?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
igb: rework handling of the vfta and vlvf registers in relation to mng_vlan
[linux-2.6.git] / drivers / net / igb / igb_main.c
blob91709272ae1f90acfb1a8dbb9d0cff0b462698f2
1 /*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <net/checksum.h>
36 #include <net/ip6_checksum.h>
37 #include <linux/net_tstamp.h>
38 #include <linux/mii.h>
39 #include <linux/ethtool.h>
40 #include <linux/if_vlan.h>
41 #include <linux/pci.h>
42 #include <linux/pci-aspm.h>
43 #include <linux/delay.h>
44 #include <linux/interrupt.h>
45 #include <linux/if_ether.h>
46 #include <linux/aer.h>
47 #ifdef CONFIG_IGB_DCA
48 #include <linux/dca.h>
49 #endif
50 #include "igb.h"
51
52 #define DRV_VERSION "1.3.16-k2"
53 char igb_driver_name[] = "igb";
54 char igb_driver_version[] = DRV_VERSION;
55 static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
57 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
58
59 static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61 };
62
63 static struct pci_device_id igb_pci_tbl[] = {
64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76 };
77
78 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80 void igb_reset(struct igb_adapter *);
81 static int igb_setup_all_tx_resources(struct igb_adapter *);
82 static int igb_setup_all_rx_resources(struct igb_adapter *);
83 static void igb_free_all_tx_resources(struct igb_adapter *);
84 static void igb_free_all_rx_resources(struct igb_adapter *);
85 static void igb_setup_mrqc(struct igb_adapter *);
86 void igb_update_stats(struct igb_adapter *);
87 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88 static void __devexit igb_remove(struct pci_dev *pdev);
89 static int igb_sw_init(struct igb_adapter *);
90 static int igb_open(struct net_device *);
91 static int igb_close(struct net_device *);
92 static void igb_configure_tx(struct igb_adapter *);
93 static void igb_configure_rx(struct igb_adapter *);
94 static void igb_clean_all_tx_rings(struct igb_adapter *);
95 static void igb_clean_all_rx_rings(struct igb_adapter *);
96 static void igb_clean_tx_ring(struct igb_ring *);
97 static void igb_clean_rx_ring(struct igb_ring *);
98 static void igb_set_rx_mode(struct net_device *);
99 static void igb_update_phy_info(unsigned long);
100 static void igb_watchdog(unsigned long);
101 static void igb_watchdog_task(struct work_struct *);
102 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
103 static struct net_device_stats *igb_get_stats(struct net_device *);
104 static int igb_change_mtu(struct net_device *, int);
105 static int igb_set_mac(struct net_device *, void *);
106 static void igb_set_uta(struct igb_adapter *adapter);
107 static irqreturn_t igb_intr(int irq, void *);
108 static irqreturn_t igb_intr_msi(int irq, void *);
109 static irqreturn_t igb_msix_other(int irq, void *);
110 static irqreturn_t igb_msix_ring(int irq, void *);
111 #ifdef CONFIG_IGB_DCA
112 static void igb_update_dca(struct igb_q_vector *);
113 static void igb_setup_dca(struct igb_adapter *);
114 #endif /* CONFIG_IGB_DCA */
115 static bool igb_clean_tx_irq(struct igb_q_vector *);
116 static int igb_poll(struct napi_struct *, int);
117 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
118 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119 static void igb_tx_timeout(struct net_device *);
120 static void igb_reset_task(struct work_struct *);
121 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122 static void igb_vlan_rx_add_vid(struct net_device *, u16);
123 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124 static void igb_restore_vlan(struct igb_adapter *);
125 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
126 static void igb_ping_all_vfs(struct igb_adapter *);
127 static void igb_msg_task(struct igb_adapter *);
128 static void igb_vmm_control(struct igb_adapter *);
129 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
130 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
131
132 #ifdef CONFIG_PM
133 static int igb_suspend(struct pci_dev *, pm_message_t);
134 static int igb_resume(struct pci_dev *);
135 #endif
136 static void igb_shutdown(struct pci_dev *);
137 #ifdef CONFIG_IGB_DCA
138 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139 static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143 };
144 #endif
145 #ifdef CONFIG_NET_POLL_CONTROLLER
146 /* for netdump / net console */
147 static void igb_netpoll(struct net_device *);
148 #endif
149 #ifdef CONFIG_PCI_IOV
150 static unsigned int max_vfs = 0;
151 module_param(max_vfs, uint, 0);
152 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154 #endif /* CONFIG_PCI_IOV */
155
156 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159 static void igb_io_resume(struct pci_dev *);
160
161 static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165 };
166
167
168 static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173 #ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177 #endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180 };
181
182 static int global_quad_port_a; /* global quad port a indication */
183
184 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
185 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
186 MODULE_LICENSE("GPL");
187 MODULE_VERSION(DRV_VERSION);
188
189 /**
190 * igb_read_clock - read raw cycle counter (to be used by time counter)
191 */
192 static cycle_t igb_read_clock(const struct cyclecounter *tc)
193 {
194 struct igb_adapter *adapter =
195 container_of(tc, struct igb_adapter, cycles);
196 struct e1000_hw *hw = &adapter->hw;
197 u64 stamp = 0;
198 int shift = 0;
199
200 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
201 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
202 return stamp;
203 }
204
205 #ifdef DEBUG
206 /**
207 * igb_get_hw_dev_name - return device name string
208 * used by hardware layer to print debugging information
209 **/
210 char *igb_get_hw_dev_name(struct e1000_hw *hw)
211 {
212 struct igb_adapter *adapter = hw->back;
213 return adapter->netdev->name;
214 }
215
216 /**
217 * igb_get_time_str - format current NIC and system time as string
218 */
219 static char *igb_get_time_str(struct igb_adapter *adapter,
220 char buffer[160])
221 {
222 cycle_t hw = adapter->cycles.read(&adapter->cycles);
223 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
224 struct timespec sys;
225 struct timespec delta;
226 getnstimeofday(&sys);
227
228 delta = timespec_sub(nic, sys);
229
230 sprintf(buffer,
231 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
232 hw,
233 (long)nic.tv_sec, nic.tv_nsec,
234 (long)sys.tv_sec, sys.tv_nsec,
235 (long)delta.tv_sec, delta.tv_nsec);
236
237 return buffer;
238 }
239 #endif
240
241 /**
242 * igb_init_module - Driver Registration Routine
243 *
244 * igb_init_module is the first routine called when the driver is
245 * loaded. All it does is register with the PCI subsystem.
246 **/
247 static int __init igb_init_module(void)
248 {
249 int ret;
250 printk(KERN_INFO "%s - version %s\n",
251 igb_driver_string, igb_driver_version);
252
253 printk(KERN_INFO "%s\n", igb_copyright);
254
255 global_quad_port_a = 0;
256
257 #ifdef CONFIG_IGB_DCA
258 dca_register_notify(&dca_notifier);
259 #endif
260
261 ret = pci_register_driver(&igb_driver);
262 return ret;
263 }
264
265 module_init(igb_init_module);
266
267 /**
268 * igb_exit_module - Driver Exit Cleanup Routine
269 *
270 * igb_exit_module is called just before the driver is removed
271 * from memory.
272 **/
273 static void __exit igb_exit_module(void)
274 {
275 #ifdef CONFIG_IGB_DCA
276 dca_unregister_notify(&dca_notifier);
277 #endif
278 pci_unregister_driver(&igb_driver);
279 }
280
281 module_exit(igb_exit_module);
282
283 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
284 /**
285 * igb_cache_ring_register - Descriptor ring to register mapping
286 * @adapter: board private structure to initialize
287 *
288 * Once we know the feature-set enabled for the device, we'll cache
289 * the register offset the descriptor ring is assigned to.
290 **/
291 static void igb_cache_ring_register(struct igb_adapter *adapter)
292 {
293 int i;
294 u32 rbase_offset = adapter->vfs_allocated_count;
295
296 switch (adapter->hw.mac.type) {
297 case e1000_82576:
298 /* The queues are allocated for virtualization such that VF 0
299 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
300 * In order to avoid collision we start at the first free queue
301 * and continue consuming queues in the same sequence
302 */
303 for (i = 0; i < adapter->num_rx_queues; i++)
304 adapter->rx_ring[i].reg_idx = rbase_offset +
305 Q_IDX_82576(i);
306 for (i = 0; i < adapter->num_tx_queues; i++)
307 adapter->tx_ring[i].reg_idx = rbase_offset +
308 Q_IDX_82576(i);
309 break;
310 case e1000_82575:
311 default:
312 for (i = 0; i < adapter->num_rx_queues; i++)
313 adapter->rx_ring[i].reg_idx = i;
314 for (i = 0; i < adapter->num_tx_queues; i++)
315 adapter->tx_ring[i].reg_idx = i;
316 break;
317 }
318 }
319
320 static void igb_free_queues(struct igb_adapter *adapter)
321 {
322 kfree(adapter->tx_ring);
323 kfree(adapter->rx_ring);
324
325 adapter->tx_ring = NULL;
326 adapter->rx_ring = NULL;
327
328 adapter->num_rx_queues = 0;
329 adapter->num_tx_queues = 0;
330 }
331
332 /**
333 * igb_alloc_queues - Allocate memory for all rings
334 * @adapter: board private structure to initialize
335 *
336 * We allocate one ring per queue at run-time since we don't know the
337 * number of queues at compile-time.
338 **/
339 static int igb_alloc_queues(struct igb_adapter *adapter)
340 {
341 int i;
342
343 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
344 sizeof(struct igb_ring), GFP_KERNEL);
345 if (!adapter->tx_ring)
346 goto err;
347
348 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
349 sizeof(struct igb_ring), GFP_KERNEL);
350 if (!adapter->rx_ring)
351 goto err;
352
353 for (i = 0; i < adapter->num_tx_queues; i++) {
354 struct igb_ring *ring = &(adapter->tx_ring[i]);
355 ring->count = adapter->tx_ring_count;
356 ring->queue_index = i;
357 ring->pdev = adapter->pdev;
358 ring->netdev = adapter->netdev;
359 /* For 82575, context index must be unique per ring. */
360 if (adapter->hw.mac.type == e1000_82575)
361 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
362 }
363
364 for (i = 0; i < adapter->num_rx_queues; i++) {
365 struct igb_ring *ring = &(adapter->rx_ring[i]);
366 ring->count = adapter->rx_ring_count;
367 ring->queue_index = i;
368 ring->pdev = adapter->pdev;
369 ring->netdev = adapter->netdev;
370 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
371 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
372 /* set flag indicating ring supports SCTP checksum offload */
373 if (adapter->hw.mac.type >= e1000_82576)
374 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
375 }
376
377 igb_cache_ring_register(adapter);
378
379 return 0;
380
381 err:
382 igb_free_queues(adapter);
383
384 return -ENOMEM;
385 }
386
387 #define IGB_N0_QUEUE -1
388 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
389 {
390 u32 msixbm = 0;
391 struct igb_adapter *adapter = q_vector->adapter;
392 struct e1000_hw *hw = &adapter->hw;
393 u32 ivar, index;
394 int rx_queue = IGB_N0_QUEUE;
395 int tx_queue = IGB_N0_QUEUE;
396
397 if (q_vector->rx_ring)
398 rx_queue = q_vector->rx_ring->reg_idx;
399 if (q_vector->tx_ring)
400 tx_queue = q_vector->tx_ring->reg_idx;
401
402 switch (hw->mac.type) {
403 case e1000_82575:
404 /* The 82575 assigns vectors using a bitmask, which matches the
405 bitmask for the EICR/EIMS/EIMC registers. To assign one
406 or more queues to a vector, we write the appropriate bits
407 into the MSIXBM register for that vector. */
408 if (rx_queue > IGB_N0_QUEUE)
409 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
410 if (tx_queue > IGB_N0_QUEUE)
411 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
412 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
413 q_vector->eims_value = msixbm;
414 break;
415 case e1000_82576:
416 /* 82576 uses a table-based method for assigning vectors.
417 Each queue has a single entry in the table to which we write
418 a vector number along with a "valid" bit. Sadly, the layout
419 of the table is somewhat counterintuitive. */
420 if (rx_queue > IGB_N0_QUEUE) {
421 index = (rx_queue & 0x7);
422 ivar = array_rd32(E1000_IVAR0, index);
423 if (rx_queue < 8) {
424 /* vector goes into low byte of register */
425 ivar = ivar & 0xFFFFFF00;
426 ivar |= msix_vector | E1000_IVAR_VALID;
427 } else {
428 /* vector goes into third byte of register */
429 ivar = ivar & 0xFF00FFFF;
430 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
431 }
432 array_wr32(E1000_IVAR0, index, ivar);
433 }
434 if (tx_queue > IGB_N0_QUEUE) {
435 index = (tx_queue & 0x7);
436 ivar = array_rd32(E1000_IVAR0, index);
437 if (tx_queue < 8) {
438 /* vector goes into second byte of register */
439 ivar = ivar & 0xFFFF00FF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
441 } else {
442 /* vector goes into high byte of register */
443 ivar = ivar & 0x00FFFFFF;
444 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
445 }
446 array_wr32(E1000_IVAR0, index, ivar);
447 }
448 q_vector->eims_value = 1 << msix_vector;
449 break;
450 default:
451 BUG();
452 break;
453 }
454 }
455
456 /**
457 * igb_configure_msix - Configure MSI-X hardware
458 *
459 * igb_configure_msix sets up the hardware to properly
460 * generate MSI-X interrupts.
461 **/
462 static void igb_configure_msix(struct igb_adapter *adapter)
463 {
464 u32 tmp;
465 int i, vector = 0;
466 struct e1000_hw *hw = &adapter->hw;
467
468 adapter->eims_enable_mask = 0;
469
470 /* set vector for other causes, i.e. link changes */
471 switch (hw->mac.type) {
472 case e1000_82575:
473 tmp = rd32(E1000_CTRL_EXT);
474 /* enable MSI-X PBA support*/
475 tmp |= E1000_CTRL_EXT_PBA_CLR;
476
477 /* Auto-Mask interrupts upon ICR read. */
478 tmp |= E1000_CTRL_EXT_EIAME;
479 tmp |= E1000_CTRL_EXT_IRCA;
480
481 wr32(E1000_CTRL_EXT, tmp);
482
483 /* enable msix_other interrupt */
484 array_wr32(E1000_MSIXBM(0), vector++,
485 E1000_EIMS_OTHER);
486 adapter->eims_other = E1000_EIMS_OTHER;
487
488 break;
489
490 case e1000_82576:
491 /* Turn on MSI-X capability first, or our settings
492 * won't stick. And it will take days to debug. */
493 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
494 E1000_GPIE_PBA | E1000_GPIE_EIAME |
495 E1000_GPIE_NSICR);
496
497 /* enable msix_other interrupt */
498 adapter->eims_other = 1 << vector;
499 tmp = (vector++ | E1000_IVAR_VALID) << 8;
500
501 wr32(E1000_IVAR_MISC, tmp);
502 break;
503 default:
504 /* do nothing, since nothing else supports MSI-X */
505 break;
506 } /* switch (hw->mac.type) */
507
508 adapter->eims_enable_mask |= adapter->eims_other;
509
510 for (i = 0; i < adapter->num_q_vectors; i++) {
511 struct igb_q_vector *q_vector = adapter->q_vector[i];
512 igb_assign_vector(q_vector, vector++);
513 adapter->eims_enable_mask |= q_vector->eims_value;
514 }
515
516 wrfl();
517 }
518
519 /**
520 * igb_request_msix - Initialize MSI-X interrupts
521 *
522 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
523 * kernel.
524 **/
525 static int igb_request_msix(struct igb_adapter *adapter)
526 {
527 struct net_device *netdev = adapter->netdev;
528 struct e1000_hw *hw = &adapter->hw;
529 int i, err = 0, vector = 0;
530
531 err = request_irq(adapter->msix_entries[vector].vector,
532 &igb_msix_other, 0, netdev->name, adapter);
533 if (err)
534 goto out;
535 vector++;
536
537 for (i = 0; i < adapter->num_q_vectors; i++) {
538 struct igb_q_vector *q_vector = adapter->q_vector[i];
539
540 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
541
542 if (q_vector->rx_ring && q_vector->tx_ring)
543 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
544 q_vector->rx_ring->queue_index);
545 else if (q_vector->tx_ring)
546 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
547 q_vector->tx_ring->queue_index);
548 else if (q_vector->rx_ring)
549 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
550 q_vector->rx_ring->queue_index);
551 else
552 sprintf(q_vector->name, "%s-unused", netdev->name);
553
554 err = request_irq(adapter->msix_entries[vector].vector,
555 &igb_msix_ring, 0, q_vector->name,
556 q_vector);
557 if (err)
558 goto out;
559 vector++;
560 }
561
562 igb_configure_msix(adapter);
563 return 0;
564 out:
565 return err;
566 }
567
568 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
569 {
570 if (adapter->msix_entries) {
571 pci_disable_msix(adapter->pdev);
572 kfree(adapter->msix_entries);
573 adapter->msix_entries = NULL;
574 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
575 pci_disable_msi(adapter->pdev);
576 }
577 }
578
579 /**
580 * igb_free_q_vectors - Free memory allocated for interrupt vectors
581 * @adapter: board private structure to initialize
582 *
583 * This function frees the memory allocated to the q_vectors. In addition if
584 * NAPI is enabled it will delete any references to the NAPI struct prior
585 * to freeing the q_vector.
586 **/
587 static void igb_free_q_vectors(struct igb_adapter *adapter)
588 {
589 int v_idx;
590
591 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
592 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
593 adapter->q_vector[v_idx] = NULL;
594 netif_napi_del(&q_vector->napi);
595 kfree(q_vector);
596 }
597 adapter->num_q_vectors = 0;
598 }
599
600 /**
601 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
602 *
603 * This function resets the device so that it has 0 rx queues, tx queues, and
604 * MSI-X interrupts allocated.
605 */
606 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
607 {
608 igb_free_queues(adapter);
609 igb_free_q_vectors(adapter);
610 igb_reset_interrupt_capability(adapter);
611 }
612
613 /**
614 * igb_set_interrupt_capability - set MSI or MSI-X if supported
615 *
616 * Attempt to configure interrupts using the best available
617 * capabilities of the hardware and kernel.
618 **/
619 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
620 {
621 int err;
622 int numvecs, i;
623
624 /* Number of supported queues. */
625 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
626 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
627
628 /* start with one vector for every rx queue */
629 numvecs = adapter->num_rx_queues;
630
631 /* if tx handler is seperate add 1 for every tx queue */
632 numvecs += adapter->num_tx_queues;
633
634 /* store the number of vectors reserved for queues */
635 adapter->num_q_vectors = numvecs;
636
637 /* add 1 vector for link status interrupts */
638 numvecs++;
639 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
640 GFP_KERNEL);
641 if (!adapter->msix_entries)
642 goto msi_only;
643
644 for (i = 0; i < numvecs; i++)
645 adapter->msix_entries[i].entry = i;
646
647 err = pci_enable_msix(adapter->pdev,
648 adapter->msix_entries,
649 numvecs);
650 if (err == 0)
651 goto out;
652
653 igb_reset_interrupt_capability(adapter);
654
655 /* If we can't do MSI-X, try MSI */
656 msi_only:
657 #ifdef CONFIG_PCI_IOV
658 /* disable SR-IOV for non MSI-X configurations */
659 if (adapter->vf_data) {
660 struct e1000_hw *hw = &adapter->hw;
661 /* disable iov and allow time for transactions to clear */
662 pci_disable_sriov(adapter->pdev);
663 msleep(500);
664
665 kfree(adapter->vf_data);
666 adapter->vf_data = NULL;
667 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
668 msleep(100);
669 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
670 }
671 #endif
672 adapter->vfs_allocated_count = 0;
673 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
674 adapter->num_rx_queues = 1;
675 adapter->num_tx_queues = 1;
676 adapter->num_q_vectors = 1;
677 if (!pci_enable_msi(adapter->pdev))
678 adapter->flags |= IGB_FLAG_HAS_MSI;
679 out:
680 /* Notify the stack of the (possibly) reduced Tx Queue count. */
681 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
682 return;
683 }
684
685 /**
686 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
687 * @adapter: board private structure to initialize
688 *
689 * We allocate one q_vector per queue interrupt. If allocation fails we
690 * return -ENOMEM.
691 **/
692 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
693 {
694 struct igb_q_vector *q_vector;
695 struct e1000_hw *hw = &adapter->hw;
696 int v_idx;
697
698 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
699 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
700 if (!q_vector)
701 goto err_out;
702 q_vector->adapter = adapter;
703 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
704 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
705 q_vector->itr_val = IGB_START_ITR;
706 q_vector->set_itr = 1;
707 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
708 adapter->q_vector[v_idx] = q_vector;
709 }
710 return 0;
711
712 err_out:
713 while (v_idx) {
714 v_idx--;
715 q_vector = adapter->q_vector[v_idx];
716 netif_napi_del(&q_vector->napi);
717 kfree(q_vector);
718 adapter->q_vector[v_idx] = NULL;
719 }
720 return -ENOMEM;
721 }
722
723 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
724 int ring_idx, int v_idx)
725 {
726 struct igb_q_vector *q_vector;
727
728 q_vector = adapter->q_vector[v_idx];
729 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
730 q_vector->rx_ring->q_vector = q_vector;
731 q_vector->itr_val = adapter->rx_itr_setting;
732 if (q_vector->itr_val && q_vector->itr_val <= 3)
733 q_vector->itr_val = IGB_START_ITR;
734 }
735
736 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
737 int ring_idx, int v_idx)
738 {
739 struct igb_q_vector *q_vector;
740
741 q_vector = adapter->q_vector[v_idx];
742 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
743 q_vector->tx_ring->q_vector = q_vector;
744 q_vector->itr_val = adapter->tx_itr_setting;
745 if (q_vector->itr_val && q_vector->itr_val <= 3)
746 q_vector->itr_val = IGB_START_ITR;
747 }
748
749 /**
750 * igb_map_ring_to_vector - maps allocated queues to vectors
751 *
752 * This function maps the recently allocated queues to vectors.
753 **/
754 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
755 {
756 int i;
757 int v_idx = 0;
758
759 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
760 (adapter->num_q_vectors < adapter->num_tx_queues))
761 return -ENOMEM;
762
763 if (adapter->num_q_vectors >=
764 (adapter->num_rx_queues + adapter->num_tx_queues)) {
765 for (i = 0; i < adapter->num_rx_queues; i++)
766 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
767 for (i = 0; i < adapter->num_tx_queues; i++)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
769 } else {
770 for (i = 0; i < adapter->num_rx_queues; i++) {
771 if (i < adapter->num_tx_queues)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx);
773 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
774 }
775 for (; i < adapter->num_tx_queues; i++)
776 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
777 }
778 return 0;
779 }
780
781 /**
782 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
783 *
784 * This function initializes the interrupts and allocates all of the queues.
785 **/
786 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
787 {
788 struct pci_dev *pdev = adapter->pdev;
789 int err;
790
791 igb_set_interrupt_capability(adapter);
792
793 err = igb_alloc_q_vectors(adapter);
794 if (err) {
795 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
796 goto err_alloc_q_vectors;
797 }
798
799 err = igb_alloc_queues(adapter);
800 if (err) {
801 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
802 goto err_alloc_queues;
803 }
804
805 err = igb_map_ring_to_vector(adapter);
806 if (err) {
807 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
808 goto err_map_queues;
809 }
810
811
812 return 0;
813 err_map_queues:
814 igb_free_queues(adapter);
815 err_alloc_queues:
816 igb_free_q_vectors(adapter);
817 err_alloc_q_vectors:
818 igb_reset_interrupt_capability(adapter);
819 return err;
820 }
821
822 /**
823 * igb_request_irq - initialize interrupts
824 *
825 * Attempts to configure interrupts using the best available
826 * capabilities of the hardware and kernel.
827 **/
828 static int igb_request_irq(struct igb_adapter *adapter)
829 {
830 struct net_device *netdev = adapter->netdev;
831 struct pci_dev *pdev = adapter->pdev;
832 struct e1000_hw *hw = &adapter->hw;
833 int err = 0;
834
835 if (adapter->msix_entries) {
836 err = igb_request_msix(adapter);
837 if (!err)
838 goto request_done;
839 /* fall back to MSI */
840 igb_clear_interrupt_scheme(adapter);
841 if (!pci_enable_msi(adapter->pdev))
842 adapter->flags |= IGB_FLAG_HAS_MSI;
843 igb_free_all_tx_resources(adapter);
844 igb_free_all_rx_resources(adapter);
845 adapter->num_tx_queues = 1;
846 adapter->num_rx_queues = 1;
847 adapter->num_q_vectors = 1;
848 err = igb_alloc_q_vectors(adapter);
849 if (err) {
850 dev_err(&pdev->dev,
851 "Unable to allocate memory for vectors\n");
852 goto request_done;
853 }
854 err = igb_alloc_queues(adapter);
855 if (err) {
856 dev_err(&pdev->dev,
857 "Unable to allocate memory for queues\n");
858 igb_free_q_vectors(adapter);
859 goto request_done;
860 }
861 igb_setup_all_tx_resources(adapter);
862 igb_setup_all_rx_resources(adapter);
863 } else {
864 switch (hw->mac.type) {
865 case e1000_82575:
866 wr32(E1000_MSIXBM(0),
867 (E1000_EICR_RX_QUEUE0 |
868 E1000_EICR_TX_QUEUE0 |
869 E1000_EIMS_OTHER));
870 break;
871 case e1000_82576:
872 wr32(E1000_IVAR0, E1000_IVAR_VALID);
873 break;
874 default:
875 break;
876 }
877 }
878
879 if (adapter->flags & IGB_FLAG_HAS_MSI) {
880 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
881 netdev->name, adapter);
882 if (!err)
883 goto request_done;
884
885 /* fall back to legacy interrupts */
886 igb_reset_interrupt_capability(adapter);
887 adapter->flags &= ~IGB_FLAG_HAS_MSI;
888 }
889
890 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
891 netdev->name, adapter);
892
893 if (err)
894 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
895 err);
896
897 request_done:
898 return err;
899 }
900
901 static void igb_free_irq(struct igb_adapter *adapter)
902 {
903 if (adapter->msix_entries) {
904 int vector = 0, i;
905
906 free_irq(adapter->msix_entries[vector++].vector, adapter);
907
908 for (i = 0; i < adapter->num_q_vectors; i++) {
909 struct igb_q_vector *q_vector = adapter->q_vector[i];
910 free_irq(adapter->msix_entries[vector++].vector,
911 q_vector);
912 }
913 } else {
914 free_irq(adapter->pdev->irq, adapter);
915 }
916 }
917
918 /**
919 * igb_irq_disable - Mask off interrupt generation on the NIC
920 * @adapter: board private structure
921 **/
922 static void igb_irq_disable(struct igb_adapter *adapter)
923 {
924 struct e1000_hw *hw = &adapter->hw;
925
926 if (adapter->msix_entries) {
927 u32 regval = rd32(E1000_EIAM);
928 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
929 wr32(E1000_EIMC, adapter->eims_enable_mask);
930 regval = rd32(E1000_EIAC);
931 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
932 }
933
934 wr32(E1000_IAM, 0);
935 wr32(E1000_IMC, ~0);
936 wrfl();
937 synchronize_irq(adapter->pdev->irq);
938 }
939
940 /**
941 * igb_irq_enable - Enable default interrupt generation settings
942 * @adapter: board private structure
943 **/
944 static void igb_irq_enable(struct igb_adapter *adapter)
945 {
946 struct e1000_hw *hw = &adapter->hw;
947
948 if (adapter->msix_entries) {
949 u32 regval = rd32(E1000_EIAC);
950 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
951 regval = rd32(E1000_EIAM);
952 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
953 wr32(E1000_EIMS, adapter->eims_enable_mask);
954 if (adapter->vfs_allocated_count)
955 wr32(E1000_MBVFIMR, 0xFF);
956 wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB |
957 E1000_IMS_DOUTSYNC));
958 } else {
959 wr32(E1000_IMS, IMS_ENABLE_MASK);
960 wr32(E1000_IAM, IMS_ENABLE_MASK);
961 }
962 }
963
964 static void igb_update_mng_vlan(struct igb_adapter *adapter)
965 {
966 struct e1000_hw *hw = &adapter->hw;
967 u16 vid = adapter->hw.mng_cookie.vlan_id;
968 u16 old_vid = adapter->mng_vlan_id;
969
970 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
971 /* add VID to filter table */
972 igb_vfta_set(hw, vid, true);
973 adapter->mng_vlan_id = vid;
974 } else {
975 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
976 }
977
978 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
979 (vid != old_vid) &&
980 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
981 /* remove VID from filter table */
982 igb_vfta_set(hw, old_vid, false);
983 }
984 }
985
986 /**
987 * igb_release_hw_control - release control of the h/w to f/w
988 * @adapter: address of board private structure
989 *
990 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
991 * For ASF and Pass Through versions of f/w this means that the
992 * driver is no longer loaded.
993 *
994 **/
995 static void igb_release_hw_control(struct igb_adapter *adapter)
996 {
997 struct e1000_hw *hw = &adapter->hw;
998 u32 ctrl_ext;
999
1000 /* Let firmware take over control of h/w */
1001 ctrl_ext = rd32(E1000_CTRL_EXT);
1002 wr32(E1000_CTRL_EXT,
1003 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1004 }
1005
1006
1007 /**
1008 * igb_get_hw_control - get control of the h/w from f/w
1009 * @adapter: address of board private structure
1010 *
1011 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1012 * For ASF and Pass Through versions of f/w this means that
1013 * the driver is loaded.
1014 *
1015 **/
1016 static void igb_get_hw_control(struct igb_adapter *adapter)
1017 {
1018 struct e1000_hw *hw = &adapter->hw;
1019 u32 ctrl_ext;
1020
1021 /* Let firmware know the driver has taken over */
1022 ctrl_ext = rd32(E1000_CTRL_EXT);
1023 wr32(E1000_CTRL_EXT,
1024 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1025 }
1026
1027 /**
1028 * igb_configure - configure the hardware for RX and TX
1029 * @adapter: private board structure
1030 **/
1031 static void igb_configure(struct igb_adapter *adapter)
1032 {
1033 struct net_device *netdev = adapter->netdev;
1034 int i;
1035
1036 igb_get_hw_control(adapter);
1037 igb_set_rx_mode(netdev);
1038
1039 igb_restore_vlan(adapter);
1040
1041 igb_setup_tctl(adapter);
1042 igb_setup_mrqc(adapter);
1043 igb_setup_rctl(adapter);
1044
1045 igb_configure_tx(adapter);
1046 igb_configure_rx(adapter);
1047
1048 igb_rx_fifo_flush_82575(&adapter->hw);
1049
1050 /* call igb_desc_unused which always leaves
1051 * at least 1 descriptor unused to make sure
1052 * next_to_use != next_to_clean */
1053 for (i = 0; i < adapter->num_rx_queues; i++) {
1054 struct igb_ring *ring = &adapter->rx_ring[i];
1055 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1056 }
1057
1058
1059 adapter->tx_queue_len = netdev->tx_queue_len;
1060 }
1061
1062
1063 /**
1064 * igb_up - Open the interface and prepare it to handle traffic
1065 * @adapter: board private structure
1066 **/
1067
1068 int igb_up(struct igb_adapter *adapter)
1069 {
1070 struct e1000_hw *hw = &adapter->hw;
1071 int i;
1072
1073 /* hardware has been reset, we need to reload some things */
1074 igb_configure(adapter);
1075
1076 clear_bit(__IGB_DOWN, &adapter->state);
1077
1078 for (i = 0; i < adapter->num_q_vectors; i++) {
1079 struct igb_q_vector *q_vector = adapter->q_vector[i];
1080 napi_enable(&q_vector->napi);
1081 }
1082 if (adapter->msix_entries)
1083 igb_configure_msix(adapter);
1084
1085 /* Clear any pending interrupts. */
1086 rd32(E1000_ICR);
1087 igb_irq_enable(adapter);
1088
1089 /* notify VFs that reset has been completed */
1090 if (adapter->vfs_allocated_count) {
1091 u32 reg_data = rd32(E1000_CTRL_EXT);
1092 reg_data |= E1000_CTRL_EXT_PFRSTD;
1093 wr32(E1000_CTRL_EXT, reg_data);
1094 }
1095
1096 netif_tx_start_all_queues(adapter->netdev);
1097
1098 /* Fire a link change interrupt to start the watchdog. */
1099 wr32(E1000_ICS, E1000_ICS_LSC);
1100 return 0;
1101 }
1102
1103 void igb_down(struct igb_adapter *adapter)
1104 {
1105 struct e1000_hw *hw = &adapter->hw;
1106 struct net_device *netdev = adapter->netdev;
1107 u32 tctl, rctl;
1108 int i;
1109
1110 /* signal that we're down so the interrupt handler does not
1111 * reschedule our watchdog timer */
1112 set_bit(__IGB_DOWN, &adapter->state);
1113
1114 /* disable receives in the hardware */
1115 rctl = rd32(E1000_RCTL);
1116 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1117 /* flush and sleep below */
1118
1119 netif_tx_stop_all_queues(netdev);
1120
1121 /* disable transmits in the hardware */
1122 tctl = rd32(E1000_TCTL);
1123 tctl &= ~E1000_TCTL_EN;
1124 wr32(E1000_TCTL, tctl);
1125 /* flush both disables and wait for them to finish */
1126 wrfl();
1127 msleep(10);
1128
1129 for (i = 0; i < adapter->num_q_vectors; i++) {
1130 struct igb_q_vector *q_vector = adapter->q_vector[i];
1131 napi_disable(&q_vector->napi);
1132 }
1133
1134 igb_irq_disable(adapter);
1135
1136 del_timer_sync(&adapter->watchdog_timer);
1137 del_timer_sync(&adapter->phy_info_timer);
1138
1139 netdev->tx_queue_len = adapter->tx_queue_len;
1140 netif_carrier_off(netdev);
1141
1142 /* record the stats before reset*/
1143 igb_update_stats(adapter);
1144
1145 adapter->link_speed = 0;
1146 adapter->link_duplex = 0;
1147
1148 if (!pci_channel_offline(adapter->pdev))
1149 igb_reset(adapter);
1150 igb_clean_all_tx_rings(adapter);
1151 igb_clean_all_rx_rings(adapter);
1152 #ifdef CONFIG_IGB_DCA
1153
1154 /* since we reset the hardware DCA settings were cleared */
1155 igb_setup_dca(adapter);
1156 #endif
1157 }
1158
1159 void igb_reinit_locked(struct igb_adapter *adapter)
1160 {
1161 WARN_ON(in_interrupt());
1162 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1163 msleep(1);
1164 igb_down(adapter);
1165 igb_up(adapter);
1166 clear_bit(__IGB_RESETTING, &adapter->state);
1167 }
1168
1169 void igb_reset(struct igb_adapter *adapter)
1170 {
1171 struct e1000_hw *hw = &adapter->hw;
1172 struct e1000_mac_info *mac = &hw->mac;
1173 struct e1000_fc_info *fc = &hw->fc;
1174 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1175 u16 hwm;
1176
1177 /* Repartition Pba for greater than 9k mtu
1178 * To take effect CTRL.RST is required.
1179 */
1180 switch (mac->type) {
1181 case e1000_82576:
1182 pba = rd32(E1000_RXPBS);
1183 pba &= E1000_RXPBS_SIZE_MASK_82576;
1184 break;
1185 case e1000_82575:
1186 default:
1187 pba = E1000_PBA_34K;
1188 break;
1189 }
1190
1191 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1192 (mac->type < e1000_82576)) {
1193 /* adjust PBA for jumbo frames */
1194 wr32(E1000_PBA, pba);
1195
1196 /* To maintain wire speed transmits, the Tx FIFO should be
1197 * large enough to accommodate two full transmit packets,
1198 * rounded up to the next 1KB and expressed in KB. Likewise,
1199 * the Rx FIFO should be large enough to accommodate at least
1200 * one full receive packet and is similarly rounded up and
1201 * expressed in KB. */
1202 pba = rd32(E1000_PBA);
1203 /* upper 16 bits has Tx packet buffer allocation size in KB */
1204 tx_space = pba >> 16;
1205 /* lower 16 bits has Rx packet buffer allocation size in KB */
1206 pba &= 0xffff;
1207 /* the tx fifo also stores 16 bytes of information about the tx
1208 * but don't include ethernet FCS because hardware appends it */
1209 min_tx_space = (adapter->max_frame_size +
1210 sizeof(union e1000_adv_tx_desc) -
1211 ETH_FCS_LEN) * 2;
1212 min_tx_space = ALIGN(min_tx_space, 1024);
1213 min_tx_space >>= 10;
1214 /* software strips receive CRC, so leave room for it */
1215 min_rx_space = adapter->max_frame_size;
1216 min_rx_space = ALIGN(min_rx_space, 1024);
1217 min_rx_space >>= 10;
1218
1219 /* If current Tx allocation is less than the min Tx FIFO size,
1220 * and the min Tx FIFO size is less than the current Rx FIFO
1221 * allocation, take space away from current Rx allocation */
1222 if (tx_space < min_tx_space &&
1223 ((min_tx_space - tx_space) < pba)) {
1224 pba = pba - (min_tx_space - tx_space);
1225
1226 /* if short on rx space, rx wins and must trump tx
1227 * adjustment */
1228 if (pba < min_rx_space)
1229 pba = min_rx_space;
1230 }
1231 wr32(E1000_PBA, pba);
1232 }
1233
1234 /* flow control settings */
1235 /* The high water mark must be low enough to fit one full frame
1236 * (or the size used for early receive) above it in the Rx FIFO.
1237 * Set it to the lower of:
1238 * - 90% of the Rx FIFO size, or
1239 * - the full Rx FIFO size minus one full frame */
1240 hwm = min(((pba << 10) * 9 / 10),
1241 ((pba << 10) - 2 * adapter->max_frame_size));
1242
1243 if (mac->type < e1000_82576) {
1244 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1245 fc->low_water = fc->high_water - 8;
1246 } else {
1247 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1248 fc->low_water = fc->high_water - 16;
1249 }
1250 fc->pause_time = 0xFFFF;
1251 fc->send_xon = 1;
1252 fc->current_mode = fc->requested_mode;
1253
1254 /* disable receive for all VFs and wait one second */
1255 if (adapter->vfs_allocated_count) {
1256 int i;
1257 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1258 adapter->vf_data[i].flags = 0;
1259
1260 /* ping all the active vfs to let them know we are going down */
1261 igb_ping_all_vfs(adapter);
1262
1263 /* disable transmits and receives */
1264 wr32(E1000_VFRE, 0);
1265 wr32(E1000_VFTE, 0);
1266 }
1267
1268 /* Allow time for pending master requests to run */
1269 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1270 wr32(E1000_WUC, 0);
1271
1272 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1273 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1274
1275 igb_update_mng_vlan(adapter);
1276
1277 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1278 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1279
1280 igb_reset_adaptive(&adapter->hw);
1281 igb_get_phy_info(&adapter->hw);
1282 }
1283
1284 static const struct net_device_ops igb_netdev_ops = {
1285 .ndo_open = igb_open,
1286 .ndo_stop = igb_close,
1287 .ndo_start_xmit = igb_xmit_frame_adv,
1288 .ndo_get_stats = igb_get_stats,
1289 .ndo_set_rx_mode = igb_set_rx_mode,
1290 .ndo_set_multicast_list = igb_set_rx_mode,
1291 .ndo_set_mac_address = igb_set_mac,
1292 .ndo_change_mtu = igb_change_mtu,
1293 .ndo_do_ioctl = igb_ioctl,
1294 .ndo_tx_timeout = igb_tx_timeout,
1295 .ndo_validate_addr = eth_validate_addr,
1296 .ndo_vlan_rx_register = igb_vlan_rx_register,
1297 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1298 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1299 #ifdef CONFIG_NET_POLL_CONTROLLER
1300 .ndo_poll_controller = igb_netpoll,
1301 #endif
1302 };
1303
1304 /**
1305 * igb_probe - Device Initialization Routine
1306 * @pdev: PCI device information struct
1307 * @ent: entry in igb_pci_tbl
1308 *
1309 * Returns 0 on success, negative on failure
1310 *
1311 * igb_probe initializes an adapter identified by a pci_dev structure.
1312 * The OS initialization, configuring of the adapter private structure,
1313 * and a hardware reset occur.
1314 **/
1315 static int __devinit igb_probe(struct pci_dev *pdev,
1316 const struct pci_device_id *ent)
1317 {
1318 struct net_device *netdev;
1319 struct igb_adapter *adapter;
1320 struct e1000_hw *hw;
1321 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1322 unsigned long mmio_start, mmio_len;
1323 int err, pci_using_dac;
1324 u16 eeprom_data = 0;
1325 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1326 u32 part_num;
1327
1328 err = pci_enable_device_mem(pdev);
1329 if (err)
1330 return err;
1331
1332 pci_using_dac = 0;
1333 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1334 if (!err) {
1335 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1336 if (!err)
1337 pci_using_dac = 1;
1338 } else {
1339 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1340 if (err) {
1341 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1342 if (err) {
1343 dev_err(&pdev->dev, "No usable DMA "
1344 "configuration, aborting\n");
1345 goto err_dma;
1346 }
1347 }
1348 }
1349
1350 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1351 IORESOURCE_MEM),
1352 igb_driver_name);
1353 if (err)
1354 goto err_pci_reg;
1355
1356 pci_enable_pcie_error_reporting(pdev);
1357
1358 pci_set_master(pdev);
1359 pci_save_state(pdev);
1360
1361 err = -ENOMEM;
1362 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1363 IGB_ABS_MAX_TX_QUEUES);
1364 if (!netdev)
1365 goto err_alloc_etherdev;
1366
1367 SET_NETDEV_DEV(netdev, &pdev->dev);
1368
1369 pci_set_drvdata(pdev, netdev);
1370 adapter = netdev_priv(netdev);
1371 adapter->netdev = netdev;
1372 adapter->pdev = pdev;
1373 hw = &adapter->hw;
1374 hw->back = adapter;
1375 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1376
1377 mmio_start = pci_resource_start(pdev, 0);
1378 mmio_len = pci_resource_len(pdev, 0);
1379
1380 err = -EIO;
1381 hw->hw_addr = ioremap(mmio_start, mmio_len);
1382 if (!hw->hw_addr)
1383 goto err_ioremap;
1384
1385 netdev->netdev_ops = &igb_netdev_ops;
1386 igb_set_ethtool_ops(netdev);
1387 netdev->watchdog_timeo = 5 * HZ;
1388
1389 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1390
1391 netdev->mem_start = mmio_start;
1392 netdev->mem_end = mmio_start + mmio_len;
1393
1394 /* PCI config space info */
1395 hw->vendor_id = pdev->vendor;
1396 hw->device_id = pdev->device;
1397 hw->revision_id = pdev->revision;
1398 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1399 hw->subsystem_device_id = pdev->subsystem_device;
1400
1401 /* setup the private structure */
1402 hw->back = adapter;
1403 /* Copy the default MAC, PHY and NVM function pointers */
1404 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1405 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1406 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1407 /* Initialize skew-specific constants */
1408 err = ei->get_invariants(hw);
1409 if (err)
1410 goto err_sw_init;
1411
1412 #ifdef CONFIG_PCI_IOV
1413 /* since iov functionality isn't critical to base device function we
1414 * can accept failure. If it fails we don't allow iov to be enabled */
1415 if (hw->mac.type == e1000_82576) {
1416 /* 82576 supports a maximum of 7 VFs in addition to the PF */
1417 unsigned int num_vfs = (max_vfs > 7) ? 7 : max_vfs;
1418 int i;
1419 unsigned char mac_addr[ETH_ALEN];
1420
1421 if (num_vfs) {
1422 adapter->vf_data = kcalloc(num_vfs,
1423 sizeof(struct vf_data_storage),
1424 GFP_KERNEL);
1425 if (!adapter->vf_data) {
1426 dev_err(&pdev->dev,
1427 "Could not allocate VF private data - "
1428 "IOV enable failed\n");
1429 } else {
1430 err = pci_enable_sriov(pdev, num_vfs);
1431 if (!err) {
1432 adapter->vfs_allocated_count = num_vfs;
1433 dev_info(&pdev->dev,
1434 "%d vfs allocated\n",
1435 num_vfs);
1436 for (i = 0;
1437 i < adapter->vfs_allocated_count;
1438 i++) {
1439 random_ether_addr(mac_addr);
1440 igb_set_vf_mac(adapter, i,
1441 mac_addr);
1442 }
1443 } else {
1444 kfree(adapter->vf_data);
1445 adapter->vf_data = NULL;
1446 }
1447 }
1448 }
1449 }
1450
1451 #endif
1452 /* setup the private structure */
1453 err = igb_sw_init(adapter);
1454 if (err)
1455 goto err_sw_init;
1456
1457 igb_get_bus_info_pcie(hw);
1458
1459 hw->phy.autoneg_wait_to_complete = false;
1460 hw->mac.adaptive_ifs = true;
1461
1462 /* Copper options */
1463 if (hw->phy.media_type == e1000_media_type_copper) {
1464 hw->phy.mdix = AUTO_ALL_MODES;
1465 hw->phy.disable_polarity_correction = false;
1466 hw->phy.ms_type = e1000_ms_hw_default;
1467 }
1468
1469 if (igb_check_reset_block(hw))
1470 dev_info(&pdev->dev,
1471 "PHY reset is blocked due to SOL/IDER session.\n");
1472
1473 netdev->features = NETIF_F_SG |
1474 NETIF_F_IP_CSUM |
1475 NETIF_F_HW_VLAN_TX |
1476 NETIF_F_HW_VLAN_RX |
1477 NETIF_F_HW_VLAN_FILTER;
1478
1479 netdev->features |= NETIF_F_IPV6_CSUM;
1480 netdev->features |= NETIF_F_TSO;
1481 netdev->features |= NETIF_F_TSO6;
1482
1483 netdev->features |= NETIF_F_GRO;
1484
1485 netdev->vlan_features |= NETIF_F_TSO;
1486 netdev->vlan_features |= NETIF_F_TSO6;
1487 netdev->vlan_features |= NETIF_F_IP_CSUM;
1488 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1489 netdev->vlan_features |= NETIF_F_SG;
1490
1491 if (pci_using_dac)
1492 netdev->features |= NETIF_F_HIGHDMA;
1493
1494 if (adapter->hw.mac.type == e1000_82576)
1495 netdev->features |= NETIF_F_SCTP_CSUM;
1496
1497 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1498
1499 /* before reading the NVM, reset the controller to put the device in a
1500 * known good starting state */
1501 hw->mac.ops.reset_hw(hw);
1502
1503 /* make sure the NVM is good */
1504 if (igb_validate_nvm_checksum(hw) < 0) {
1505 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1506 err = -EIO;
1507 goto err_eeprom;
1508 }
1509
1510 /* copy the MAC address out of the NVM */
1511 if (hw->mac.ops.read_mac_addr(hw))
1512 dev_err(&pdev->dev, "NVM Read Error\n");
1513
1514 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1515 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1516
1517 if (!is_valid_ether_addr(netdev->perm_addr)) {
1518 dev_err(&pdev->dev, "Invalid MAC Address\n");
1519 err = -EIO;
1520 goto err_eeprom;
1521 }
1522
1523 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1524 (unsigned long) adapter);
1525 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1526 (unsigned long) adapter);
1527
1528 INIT_WORK(&adapter->reset_task, igb_reset_task);
1529 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1530
1531 /* Initialize link properties that are user-changeable */
1532 adapter->fc_autoneg = true;
1533 hw->mac.autoneg = true;
1534 hw->phy.autoneg_advertised = 0x2f;
1535
1536 hw->fc.requested_mode = e1000_fc_default;
1537 hw->fc.current_mode = e1000_fc_default;
1538
1539 igb_validate_mdi_setting(hw);
1540
1541 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1542 * enable the ACPI Magic Packet filter
1543 */
1544
1545 if (hw->bus.func == 0)
1546 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1547 else if (hw->bus.func == 1)
1548 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1549
1550 if (eeprom_data & eeprom_apme_mask)
1551 adapter->eeprom_wol |= E1000_WUFC_MAG;
1552
1553 /* now that we have the eeprom settings, apply the special cases where
1554 * the eeprom may be wrong or the board simply won't support wake on
1555 * lan on a particular port */
1556 switch (pdev->device) {
1557 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1558 adapter->eeprom_wol = 0;
1559 break;
1560 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1561 case E1000_DEV_ID_82576_FIBER:
1562 case E1000_DEV_ID_82576_SERDES:
1563 /* Wake events only supported on port A for dual fiber
1564 * regardless of eeprom setting */
1565 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1566 adapter->eeprom_wol = 0;
1567 break;
1568 case E1000_DEV_ID_82576_QUAD_COPPER:
1569 /* if quad port adapter, disable WoL on all but port A */
1570 if (global_quad_port_a != 0)
1571 adapter->eeprom_wol = 0;
1572 else
1573 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1574 /* Reset for multiple quad port adapters */
1575 if (++global_quad_port_a == 4)
1576 global_quad_port_a = 0;
1577 break;
1578 }
1579
1580 /* initialize the wol settings based on the eeprom settings */
1581 adapter->wol = adapter->eeprom_wol;
1582 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1583
1584 /* reset the hardware with the new settings */
1585 igb_reset(adapter);
1586
1587 /* let the f/w know that the h/w is now under the control of the
1588 * driver. */
1589 igb_get_hw_control(adapter);
1590
1591 strcpy(netdev->name, "eth%d");
1592 err = register_netdev(netdev);
1593 if (err)
1594 goto err_register;
1595
1596 /* carrier off reporting is important to ethtool even BEFORE open */
1597 netif_carrier_off(netdev);
1598
1599 #ifdef CONFIG_IGB_DCA
1600 if (dca_add_requester(&pdev->dev) == 0) {
1601 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1602 dev_info(&pdev->dev, "DCA enabled\n");
1603 igb_setup_dca(adapter);
1604 }
1605
1606 #endif
1607
1608 switch (hw->mac.type) {
1609 case e1000_82576:
1610 /*
1611 * Initialize hardware timer: we keep it running just in case
1612 * that some program needs it later on.
1613 */
1614 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1615 adapter->cycles.read = igb_read_clock;
1616 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1617 adapter->cycles.mult = 1;
1618 /**
1619 * Scale the NIC clock cycle by a large factor so that
1620 * relatively small clock corrections can be added or
1621 * substracted at each clock tick. The drawbacks of a large
1622 * factor are a) that the clock register overflows more quickly
1623 * (not such a big deal) and b) that the increment per tick has
1624 * to fit into 24 bits. As a result we need to use a shift of
1625 * 19 so we can fit a value of 16 into the TIMINCA register.
1626 */
1627 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1628 wr32(E1000_TIMINCA,
1629 (1 << E1000_TIMINCA_16NS_SHIFT) |
1630 (16 << IGB_82576_TSYNC_SHIFT));
1631
1632 /* Set registers so that rollover occurs soon to test this. */
1633 wr32(E1000_SYSTIML, 0x00000000);
1634 wr32(E1000_SYSTIMH, 0xFF800000);
1635 wrfl();
1636
1637 timecounter_init(&adapter->clock,
1638 &adapter->cycles,
1639 ktime_to_ns(ktime_get_real()));
1640 /*
1641 * Synchronize our NIC clock against system wall clock. NIC
1642 * time stamp reading requires ~3us per sample, each sample
1643 * was pretty stable even under load => only require 10
1644 * samples for each offset comparison.
1645 */
1646 memset(&adapter->compare, 0, sizeof(adapter->compare));
1647 adapter->compare.source = &adapter->clock;
1648 adapter->compare.target = ktime_get_real;
1649 adapter->compare.num_samples = 10;
1650 timecompare_update(&adapter->compare, 0);
1651 break;
1652 case e1000_82575:
1653 /* 82575 does not support timesync */
1654 default:
1655 break;
1656 }
1657
1658 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1659 /* print bus type/speed/width info */
1660 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1661 netdev->name,
1662 ((hw->bus.speed == e1000_bus_speed_2500)
1663 ? "2.5Gb/s" : "unknown"),
1664 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1665 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1666 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1667 "unknown"),
1668 netdev->dev_addr);
1669
1670 igb_read_part_num(hw, &part_num);
1671 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1672 (part_num >> 8), (part_num & 0xff));
1673
1674 dev_info(&pdev->dev,
1675 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1676 adapter->msix_entries ? "MSI-X" :
1677 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1678 adapter->num_rx_queues, adapter->num_tx_queues);
1679
1680 return 0;
1681
1682 err_register:
1683 igb_release_hw_control(adapter);
1684 err_eeprom:
1685 if (!igb_check_reset_block(hw))
1686 igb_reset_phy(hw);
1687
1688 if (hw->flash_address)
1689 iounmap(hw->flash_address);
1690 err_sw_init:
1691 igb_clear_interrupt_scheme(adapter);
1692 iounmap(hw->hw_addr);
1693 err_ioremap:
1694 free_netdev(netdev);
1695 err_alloc_etherdev:
1696 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1697 IORESOURCE_MEM));
1698 err_pci_reg:
1699 err_dma:
1700 pci_disable_device(pdev);
1701 return err;
1702 }
1703
1704 /**
1705 * igb_remove - Device Removal Routine
1706 * @pdev: PCI device information struct
1707 *
1708 * igb_remove is called by the PCI subsystem to alert the driver
1709 * that it should release a PCI device. The could be caused by a
1710 * Hot-Plug event, or because the driver is going to be removed from
1711 * memory.
1712 **/
1713 static void __devexit igb_remove(struct pci_dev *pdev)
1714 {
1715 struct net_device *netdev = pci_get_drvdata(pdev);
1716 struct igb_adapter *adapter = netdev_priv(netdev);
1717 struct e1000_hw *hw = &adapter->hw;
1718
1719 /* flush_scheduled work may reschedule our watchdog task, so
1720 * explicitly disable watchdog tasks from being rescheduled */
1721 set_bit(__IGB_DOWN, &adapter->state);
1722 del_timer_sync(&adapter->watchdog_timer);
1723 del_timer_sync(&adapter->phy_info_timer);
1724
1725 flush_scheduled_work();
1726
1727 #ifdef CONFIG_IGB_DCA
1728 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
1729 dev_info(&pdev->dev, "DCA disabled\n");
1730 dca_remove_requester(&pdev->dev);
1731 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
1732 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
1733 }
1734 #endif
1735
1736 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1737 * would have already happened in close and is redundant. */
1738 igb_release_hw_control(adapter);
1739
1740 unregister_netdev(netdev);
1741
1742 if (!igb_check_reset_block(&adapter->hw))
1743 igb_reset_phy(&adapter->hw);
1744
1745 igb_clear_interrupt_scheme(adapter);
1746
1747 #ifdef CONFIG_PCI_IOV
1748 /* reclaim resources allocated to VFs */
1749 if (adapter->vf_data) {
1750 /* disable iov and allow time for transactions to clear */
1751 pci_disable_sriov(pdev);
1752 msleep(500);
1753
1754 kfree(adapter->vf_data);
1755 adapter->vf_data = NULL;
1756 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1757 msleep(100);
1758 dev_info(&pdev->dev, "IOV Disabled\n");
1759 }
1760 #endif
1761 iounmap(hw->hw_addr);
1762 if (hw->flash_address)
1763 iounmap(hw->flash_address);
1764 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1765 IORESOURCE_MEM));
1766
1767 free_netdev(netdev);
1768
1769 pci_disable_pcie_error_reporting(pdev);
1770
1771 pci_disable_device(pdev);
1772 }
1773
1774 /**
1775 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1776 * @adapter: board private structure to initialize
1777 *
1778 * igb_sw_init initializes the Adapter private data structure.
1779 * Fields are initialized based on PCI device information and
1780 * OS network device settings (MTU size).
1781 **/
1782 static int __devinit igb_sw_init(struct igb_adapter *adapter)
1783 {
1784 struct e1000_hw *hw = &adapter->hw;
1785 struct net_device *netdev = adapter->netdev;
1786 struct pci_dev *pdev = adapter->pdev;
1787
1788 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1789
1790 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1791 adapter->rx_ring_count = IGB_DEFAULT_RXD;
1792 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1793 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1794
1795 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1796 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1797
1798 /* This call may decrease the number of queues depending on
1799 * interrupt mode. */
1800 if (igb_init_interrupt_scheme(adapter)) {
1801 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1802 return -ENOMEM;
1803 }
1804
1805 /* Explicitly disable IRQ since the NIC can be in any state. */
1806 igb_irq_disable(adapter);
1807
1808 set_bit(__IGB_DOWN, &adapter->state);
1809 return 0;
1810 }
1811
1812 /**
1813 * igb_open - Called when a network interface is made active
1814 * @netdev: network interface device structure
1815 *
1816 * Returns 0 on success, negative value on failure
1817 *
1818 * The open entry point is called when a network interface is made
1819 * active by the system (IFF_UP). At this point all resources needed
1820 * for transmit and receive operations are allocated, the interrupt
1821 * handler is registered with the OS, the watchdog timer is started,
1822 * and the stack is notified that the interface is ready.
1823 **/
1824 static int igb_open(struct net_device *netdev)
1825 {
1826 struct igb_adapter *adapter = netdev_priv(netdev);
1827 struct e1000_hw *hw = &adapter->hw;
1828 int err;
1829 int i;
1830
1831 /* disallow open during test */
1832 if (test_bit(__IGB_TESTING, &adapter->state))
1833 return -EBUSY;
1834
1835 netif_carrier_off(netdev);
1836
1837 /* allocate transmit descriptors */
1838 err = igb_setup_all_tx_resources(adapter);
1839 if (err)
1840 goto err_setup_tx;
1841
1842 /* allocate receive descriptors */
1843 err = igb_setup_all_rx_resources(adapter);
1844 if (err)
1845 goto err_setup_rx;
1846
1847 /* e1000_power_up_phy(adapter); */
1848
1849 /* before we allocate an interrupt, we must be ready to handle it.
1850 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1851 * as soon as we call pci_request_irq, so we have to setup our
1852 * clean_rx handler before we do so. */
1853 igb_configure(adapter);
1854
1855 err = igb_request_irq(adapter);
1856 if (err)
1857 goto err_req_irq;
1858
1859 /* From here on the code is the same as igb_up() */
1860 clear_bit(__IGB_DOWN, &adapter->state);
1861
1862 for (i = 0; i < adapter->num_q_vectors; i++) {
1863 struct igb_q_vector *q_vector = adapter->q_vector[i];
1864 napi_enable(&q_vector->napi);
1865 }
1866
1867 /* Clear any pending interrupts. */
1868 rd32(E1000_ICR);
1869
1870 igb_irq_enable(adapter);
1871
1872 /* notify VFs that reset has been completed */
1873 if (adapter->vfs_allocated_count) {
1874 u32 reg_data = rd32(E1000_CTRL_EXT);
1875 reg_data |= E1000_CTRL_EXT_PFRSTD;
1876 wr32(E1000_CTRL_EXT, reg_data);
1877 }
1878
1879 netif_tx_start_all_queues(netdev);
1880
1881 /* Fire a link status change interrupt to start the watchdog. */
1882 wr32(E1000_ICS, E1000_ICS_LSC);
1883
1884 return 0;
1885
1886 err_req_irq:
1887 igb_release_hw_control(adapter);
1888 /* e1000_power_down_phy(adapter); */
1889 igb_free_all_rx_resources(adapter);
1890 err_setup_rx:
1891 igb_free_all_tx_resources(adapter);
1892 err_setup_tx:
1893 igb_reset(adapter);
1894
1895 return err;
1896 }
1897
1898 /**
1899 * igb_close - Disables a network interface
1900 * @netdev: network interface device structure
1901 *
1902 * Returns 0, this is not allowed to fail
1903 *
1904 * The close entry point is called when an interface is de-activated
1905 * by the OS. The hardware is still under the driver's control, but
1906 * needs to be disabled. A global MAC reset is issued to stop the
1907 * hardware, and all transmit and receive resources are freed.
1908 **/
1909 static int igb_close(struct net_device *netdev)
1910 {
1911 struct igb_adapter *adapter = netdev_priv(netdev);
1912
1913 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1914 igb_down(adapter);
1915
1916 igb_free_irq(adapter);
1917
1918 igb_free_all_tx_resources(adapter);
1919 igb_free_all_rx_resources(adapter);
1920
1921 return 0;
1922 }
1923
1924 /**
1925 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
1926 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1927 *
1928 * Return 0 on success, negative on failure
1929 **/
1930 int igb_setup_tx_resources(struct igb_ring *tx_ring)
1931 {
1932 struct pci_dev *pdev = tx_ring->pdev;
1933 int size;
1934
1935 size = sizeof(struct igb_buffer) * tx_ring->count;
1936 tx_ring->buffer_info = vmalloc(size);
1937 if (!tx_ring->buffer_info)
1938 goto err;
1939 memset(tx_ring->buffer_info, 0, size);
1940
1941 /* round up to nearest 4K */
1942 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
1943 tx_ring->size = ALIGN(tx_ring->size, 4096);
1944
1945 tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
1946 &tx_ring->dma);
1947
1948 if (!tx_ring->desc)
1949 goto err;
1950
1951 tx_ring->next_to_use = 0;
1952 tx_ring->next_to_clean = 0;
1953 return 0;
1954
1955 err:
1956 vfree(tx_ring->buffer_info);
1957 dev_err(&pdev->dev,
1958 "Unable to allocate memory for the transmit descriptor ring\n");
1959 return -ENOMEM;
1960 }
1961
1962 /**
1963 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1964 * (Descriptors) for all queues
1965 * @adapter: board private structure
1966 *
1967 * Return 0 on success, negative on failure
1968 **/
1969 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1970 {
1971 int i, err = 0;
1972 int r_idx;
1973
1974 for (i = 0; i < adapter->num_tx_queues; i++) {
1975 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
1976 if (err) {
1977 dev_err(&adapter->pdev->dev,
1978 "Allocation for Tx Queue %u failed\n", i);
1979 for (i--; i >= 0; i--)
1980 igb_free_tx_resources(&adapter->tx_ring[i]);
1981 break;
1982 }
1983 }
1984
1985 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
1986 r_idx = i % adapter->num_tx_queues;
1987 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
1988 }
1989 return err;
1990 }
1991
1992 /**
1993 * igb_setup_tctl - configure the transmit control registers
1994 * @adapter: Board private structure
1995 **/
1996 void igb_setup_tctl(struct igb_adapter *adapter)
1997 {
1998 struct e1000_hw *hw = &adapter->hw;
1999 u32 tctl;
2000
2001 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2002 wr32(E1000_TXDCTL(0), 0);
2003
2004 /* Program the Transmit Control Register */
2005 tctl = rd32(E1000_TCTL);
2006 tctl &= ~E1000_TCTL_CT;
2007 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2008 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2009
2010 igb_config_collision_dist(hw);
2011
2012 /* Enable transmits */
2013 tctl |= E1000_TCTL_EN;
2014
2015 wr32(E1000_TCTL, tctl);
2016 }
2017
2018 /**
2019 * igb_configure_tx_ring - Configure transmit ring after Reset
2020 * @adapter: board private structure
2021 * @ring: tx ring to configure
2022 *
2023 * Configure a transmit ring after a reset.
2024 **/
2025 void igb_configure_tx_ring(struct igb_adapter *adapter,
2026 struct igb_ring *ring)
2027 {
2028 struct e1000_hw *hw = &adapter->hw;
2029 u32 txdctl;
2030 u64 tdba = ring->dma;
2031 int reg_idx = ring->reg_idx;
2032
2033 /* disable the queue */
2034 txdctl = rd32(E1000_TXDCTL(reg_idx));
2035 wr32(E1000_TXDCTL(reg_idx),
2036 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2037 wrfl();
2038 mdelay(10);
2039
2040 wr32(E1000_TDLEN(reg_idx),
2041 ring->count * sizeof(union e1000_adv_tx_desc));
2042 wr32(E1000_TDBAL(reg_idx),
2043 tdba & 0x00000000ffffffffULL);
2044 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2045
2046 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2047 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2048 writel(0, ring->head);
2049 writel(0, ring->tail);
2050
2051 txdctl |= IGB_TX_PTHRESH;
2052 txdctl |= IGB_TX_HTHRESH << 8;
2053 txdctl |= IGB_TX_WTHRESH << 16;
2054
2055 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2056 wr32(E1000_TXDCTL(reg_idx), txdctl);
2057 }
2058
2059 /**
2060 * igb_configure_tx - Configure transmit Unit after Reset
2061 * @adapter: board private structure
2062 *
2063 * Configure the Tx unit of the MAC after a reset.
2064 **/
2065 static void igb_configure_tx(struct igb_adapter *adapter)
2066 {
2067 int i;
2068
2069 for (i = 0; i < adapter->num_tx_queues; i++)
2070 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
2071 }
2072
2073 /**
2074 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2075 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2076 *
2077 * Returns 0 on success, negative on failure
2078 **/
2079 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2080 {
2081 struct pci_dev *pdev = rx_ring->pdev;
2082 int size, desc_len;
2083
2084 size = sizeof(struct igb_buffer) * rx_ring->count;
2085 rx_ring->buffer_info = vmalloc(size);
2086 if (!rx_ring->buffer_info)
2087 goto err;
2088 memset(rx_ring->buffer_info, 0, size);
2089
2090 desc_len = sizeof(union e1000_adv_rx_desc);
2091
2092 /* Round up to nearest 4K */
2093 rx_ring->size = rx_ring->count * desc_len;
2094 rx_ring->size = ALIGN(rx_ring->size, 4096);
2095
2096 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2097 &rx_ring->dma);
2098
2099 if (!rx_ring->desc)
2100 goto err;
2101
2102 rx_ring->next_to_clean = 0;
2103 rx_ring->next_to_use = 0;
2104
2105 return 0;
2106
2107 err:
2108 vfree(rx_ring->buffer_info);
2109 dev_err(&pdev->dev, "Unable to allocate memory for "
2110 "the receive descriptor ring\n");
2111 return -ENOMEM;
2112 }
2113
2114 /**
2115 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2116 * (Descriptors) for all queues
2117 * @adapter: board private structure
2118 *
2119 * Return 0 on success, negative on failure
2120 **/
2121 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2122 {
2123 int i, err = 0;
2124
2125 for (i = 0; i < adapter->num_rx_queues; i++) {
2126 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
2127 if (err) {
2128 dev_err(&adapter->pdev->dev,
2129 "Allocation for Rx Queue %u failed\n", i);
2130 for (i--; i >= 0; i--)
2131 igb_free_rx_resources(&adapter->rx_ring[i]);
2132 break;
2133 }
2134 }
2135
2136 return err;
2137 }
2138
2139 /**
2140 * igb_setup_mrqc - configure the multiple receive queue control registers
2141 * @adapter: Board private structure
2142 **/
2143 static void igb_setup_mrqc(struct igb_adapter *adapter)
2144 {
2145 struct e1000_hw *hw = &adapter->hw;
2146 u32 mrqc, rxcsum;
2147 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2148 union e1000_reta {
2149 u32 dword;
2150 u8 bytes[4];
2151 } reta;
2152 static const u8 rsshash[40] = {
2153 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2154 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2155 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2156 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2157
2158 /* Fill out hash function seeds */
2159 for (j = 0; j < 10; j++) {
2160 u32 rsskey = rsshash[(j * 4)];
2161 rsskey |= rsshash[(j * 4) + 1] << 8;
2162 rsskey |= rsshash[(j * 4) + 2] << 16;
2163 rsskey |= rsshash[(j * 4) + 3] << 24;
2164 array_wr32(E1000_RSSRK(0), j, rsskey);
2165 }
2166
2167 num_rx_queues = adapter->num_rx_queues;
2168
2169 if (adapter->vfs_allocated_count) {
2170 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2171 switch (hw->mac.type) {
2172 case e1000_82576:
2173 shift = 3;
2174 num_rx_queues = 2;
2175 break;
2176 case e1000_82575:
2177 shift = 2;
2178 shift2 = 6;
2179 default:
2180 break;
2181 }
2182 } else {
2183 if (hw->mac.type == e1000_82575)
2184 shift = 6;
2185 }
2186
2187 for (j = 0; j < (32 * 4); j++) {
2188 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2189 if (shift2)
2190 reta.bytes[j & 3] |= num_rx_queues << shift2;
2191 if ((j & 3) == 3)
2192 wr32(E1000_RETA(j >> 2), reta.dword);
2193 }
2194
2195 /*
2196 * Disable raw packet checksumming so that RSS hash is placed in
2197 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2198 * offloads as they are enabled by default
2199 */
2200 rxcsum = rd32(E1000_RXCSUM);
2201 rxcsum |= E1000_RXCSUM_PCSD;
2202
2203 if (adapter->hw.mac.type >= e1000_82576)
2204 /* Enable Receive Checksum Offload for SCTP */
2205 rxcsum |= E1000_RXCSUM_CRCOFL;
2206
2207 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2208 wr32(E1000_RXCSUM, rxcsum);
2209
2210 /* If VMDq is enabled then we set the appropriate mode for that, else
2211 * we default to RSS so that an RSS hash is calculated per packet even
2212 * if we are only using one queue */
2213 if (adapter->vfs_allocated_count) {
2214 if (hw->mac.type > e1000_82575) {
2215 /* Set the default pool for the PF's first queue */
2216 u32 vtctl = rd32(E1000_VT_CTL);
2217 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2218 E1000_VT_CTL_DISABLE_DEF_POOL);
2219 vtctl |= adapter->vfs_allocated_count <<
2220 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2221 wr32(E1000_VT_CTL, vtctl);
2222 }
2223 if (adapter->num_rx_queues > 1)
2224 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2225 else
2226 mrqc = E1000_MRQC_ENABLE_VMDQ;
2227 } else {
2228 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2229 }
2230 igb_vmm_control(adapter);
2231
2232 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2233 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2234 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2235 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2236 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2237 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2238 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2239 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2240
2241 wr32(E1000_MRQC, mrqc);
2242 }
2243
2244 /**
2245 * igb_setup_rctl - configure the receive control registers
2246 * @adapter: Board private structure
2247 **/
2248 void igb_setup_rctl(struct igb_adapter *adapter)
2249 {
2250 struct e1000_hw *hw = &adapter->hw;
2251 u32 rctl;
2252
2253 rctl = rd32(E1000_RCTL);
2254
2255 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2256 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2257
2258 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2259 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2260
2261 /*
2262 * enable stripping of CRC. It's unlikely this will break BMC
2263 * redirection as it did with e1000. Newer features require
2264 * that the HW strips the CRC.
2265 */
2266 rctl |= E1000_RCTL_SECRC;
2267
2268 /*
2269 * disable store bad packets and clear size bits.
2270 */
2271 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2272
2273 /* enable LPE to prevent packets larger than max_frame_size */
2274 rctl |= E1000_RCTL_LPE;
2275
2276 /* disable queue 0 to prevent tail write w/o re-config */
2277 wr32(E1000_RXDCTL(0), 0);
2278
2279 /* Attention!!! For SR-IOV PF driver operations you must enable
2280 * queue drop for all VF and PF queues to prevent head of line blocking
2281 * if an un-trusted VF does not provide descriptors to hardware.
2282 */
2283 if (adapter->vfs_allocated_count) {
2284 /* set all queue drop enable bits */
2285 wr32(E1000_QDE, ALL_QUEUES);
2286 }
2287
2288 wr32(E1000_RCTL, rctl);
2289 }
2290
2291 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2292 int vfn)
2293 {
2294 struct e1000_hw *hw = &adapter->hw;
2295 u32 vmolr;
2296
2297 /* if it isn't the PF check to see if VFs are enabled and
2298 * increase the size to support vlan tags */
2299 if (vfn < adapter->vfs_allocated_count &&
2300 adapter->vf_data[vfn].vlans_enabled)
2301 size += VLAN_TAG_SIZE;
2302
2303 vmolr = rd32(E1000_VMOLR(vfn));
2304 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2305 vmolr |= size | E1000_VMOLR_LPE;
2306 wr32(E1000_VMOLR(vfn), vmolr);
2307
2308 return 0;
2309 }
2310
2311 /**
2312 * igb_rlpml_set - set maximum receive packet size
2313 * @adapter: board private structure
2314 *
2315 * Configure maximum receivable packet size.
2316 **/
2317 static void igb_rlpml_set(struct igb_adapter *adapter)
2318 {
2319 u32 max_frame_size = adapter->max_frame_size;
2320 struct e1000_hw *hw = &adapter->hw;
2321 u16 pf_id = adapter->vfs_allocated_count;
2322
2323 if (adapter->vlgrp)
2324 max_frame_size += VLAN_TAG_SIZE;
2325
2326 /* if vfs are enabled we set RLPML to the largest possible request
2327 * size and set the VMOLR RLPML to the size we need */
2328 if (pf_id) {
2329 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2330 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2331 }
2332
2333 wr32(E1000_RLPML, max_frame_size);
2334 }
2335
2336 static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2337 {
2338 struct e1000_hw *hw = &adapter->hw;
2339 u32 vmolr;
2340
2341 /*
2342 * This register exists only on 82576 and newer so if we are older then
2343 * we should exit and do nothing
2344 */
2345 if (hw->mac.type < e1000_82576)
2346 return;
2347
2348 vmolr = rd32(E1000_VMOLR(vfn));
2349 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2350 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2351
2352 /* clear all bits that might not be set */
2353 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2354
2355 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2356 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2357 /*
2358 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2359 * multicast packets
2360 */
2361 if (vfn <= adapter->vfs_allocated_count)
2362 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2363
2364 wr32(E1000_VMOLR(vfn), vmolr);
2365 }
2366
2367 /**
2368 * igb_configure_rx_ring - Configure a receive ring after Reset
2369 * @adapter: board private structure
2370 * @ring: receive ring to be configured
2371 *
2372 * Configure the Rx unit of the MAC after a reset.
2373 **/
2374 void igb_configure_rx_ring(struct igb_adapter *adapter,
2375 struct igb_ring *ring)
2376 {
2377 struct e1000_hw *hw = &adapter->hw;
2378 u64 rdba = ring->dma;
2379 int reg_idx = ring->reg_idx;
2380 u32 srrctl, rxdctl;
2381
2382 /* disable the queue */
2383 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2384 wr32(E1000_RXDCTL(reg_idx),
2385 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2386
2387 /* Set DMA base address registers */
2388 wr32(E1000_RDBAL(reg_idx),
2389 rdba & 0x00000000ffffffffULL);
2390 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2391 wr32(E1000_RDLEN(reg_idx),
2392 ring->count * sizeof(union e1000_adv_rx_desc));
2393
2394 /* initialize head and tail */
2395 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2396 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2397 writel(0, ring->head);
2398 writel(0, ring->tail);
2399
2400 /* set descriptor configuration */
2401 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2402 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2403 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2404 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2405 srrctl |= IGB_RXBUFFER_16384 >>
2406 E1000_SRRCTL_BSIZEPKT_SHIFT;
2407 #else
2408 srrctl |= (PAGE_SIZE / 2) >>
2409 E1000_SRRCTL_BSIZEPKT_SHIFT;
2410 #endif
2411 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2412 } else {
2413 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2414 E1000_SRRCTL_BSIZEPKT_SHIFT;
2415 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2416 }
2417
2418 wr32(E1000_SRRCTL(reg_idx), srrctl);
2419
2420 /* set filtering for VMDQ pools */
2421 igb_set_vmolr(adapter, reg_idx & 0x7);
2422
2423 /* enable receive descriptor fetching */
2424 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2425 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2426 rxdctl &= 0xFFF00000;
2427 rxdctl |= IGB_RX_PTHRESH;
2428 rxdctl |= IGB_RX_HTHRESH << 8;
2429 rxdctl |= IGB_RX_WTHRESH << 16;
2430 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2431 }
2432
2433 /**
2434 * igb_configure_rx - Configure receive Unit after Reset
2435 * @adapter: board private structure
2436 *
2437 * Configure the Rx unit of the MAC after a reset.
2438 **/
2439 static void igb_configure_rx(struct igb_adapter *adapter)
2440 {
2441 int i;
2442
2443 /* set UTA to appropriate mode */
2444 igb_set_uta(adapter);
2445
2446 /* set the correct pool for the PF default MAC address in entry 0 */
2447 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2448 adapter->vfs_allocated_count);
2449
2450 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2451 * the Base and Length of the Rx Descriptor Ring */
2452 for (i = 0; i < adapter->num_rx_queues; i++)
2453 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
2454 }
2455
2456 /**
2457 * igb_free_tx_resources - Free Tx Resources per Queue
2458 * @tx_ring: Tx descriptor ring for a specific queue
2459 *
2460 * Free all transmit software resources
2461 **/
2462 void igb_free_tx_resources(struct igb_ring *tx_ring)
2463 {
2464 igb_clean_tx_ring(tx_ring);
2465
2466 vfree(tx_ring->buffer_info);
2467 tx_ring->buffer_info = NULL;
2468
2469 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2470 tx_ring->desc, tx_ring->dma);
2471
2472 tx_ring->desc = NULL;
2473 }
2474
2475 /**
2476 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2477 * @adapter: board private structure
2478 *
2479 * Free all transmit software resources
2480 **/
2481 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2482 {
2483 int i;
2484
2485 for (i = 0; i < adapter->num_tx_queues; i++)
2486 igb_free_tx_resources(&adapter->tx_ring[i]);
2487 }
2488
2489 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2490 struct igb_buffer *buffer_info)
2491 {
2492 buffer_info->dma = 0;
2493 if (buffer_info->skb) {
2494 skb_dma_unmap(&tx_ring->pdev->dev,
2495 buffer_info->skb,
2496 DMA_TO_DEVICE);
2497 dev_kfree_skb_any(buffer_info->skb);
2498 buffer_info->skb = NULL;
2499 }
2500 buffer_info->time_stamp = 0;
2501 /* buffer_info must be completely set up in the transmit path */
2502 }
2503
2504 /**
2505 * igb_clean_tx_ring - Free Tx Buffers
2506 * @tx_ring: ring to be cleaned
2507 **/
2508 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
2509 {
2510 struct igb_buffer *buffer_info;
2511 unsigned long size;
2512 unsigned int i;
2513
2514 if (!tx_ring->buffer_info)
2515 return;
2516 /* Free all the Tx ring sk_buffs */
2517
2518 for (i = 0; i < tx_ring->count; i++) {
2519 buffer_info = &tx_ring->buffer_info[i];
2520 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
2521 }
2522
2523 size = sizeof(struct igb_buffer) * tx_ring->count;
2524 memset(tx_ring->buffer_info, 0, size);
2525
2526 /* Zero out the descriptor ring */
2527
2528 memset(tx_ring->desc, 0, tx_ring->size);
2529
2530 tx_ring->next_to_use = 0;
2531 tx_ring->next_to_clean = 0;
2532
2533 writel(0, tx_ring->head);
2534 writel(0, tx_ring->tail);
2535 }
2536
2537 /**
2538 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2539 * @adapter: board private structure
2540 **/
2541 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2542 {
2543 int i;
2544
2545 for (i = 0; i < adapter->num_tx_queues; i++)
2546 igb_clean_tx_ring(&adapter->tx_ring[i]);
2547 }
2548
2549 /**
2550 * igb_free_rx_resources - Free Rx Resources
2551 * @rx_ring: ring to clean the resources from
2552 *
2553 * Free all receive software resources
2554 **/
2555 void igb_free_rx_resources(struct igb_ring *rx_ring)
2556 {
2557 igb_clean_rx_ring(rx_ring);
2558
2559 vfree(rx_ring->buffer_info);
2560 rx_ring->buffer_info = NULL;
2561
2562 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2563 rx_ring->desc, rx_ring->dma);
2564
2565 rx_ring->desc = NULL;
2566 }
2567
2568 /**
2569 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2570 * @adapter: board private structure
2571 *
2572 * Free all receive software resources
2573 **/
2574 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2575 {
2576 int i;
2577
2578 for (i = 0; i < adapter->num_rx_queues; i++)
2579 igb_free_rx_resources(&adapter->rx_ring[i]);
2580 }
2581
2582 /**
2583 * igb_clean_rx_ring - Free Rx Buffers per Queue
2584 * @rx_ring: ring to free buffers from
2585 **/
2586 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
2587 {
2588 struct igb_buffer *buffer_info;
2589 unsigned long size;
2590 unsigned int i;
2591
2592 if (!rx_ring->buffer_info)
2593 return;
2594 /* Free all the Rx ring sk_buffs */
2595 for (i = 0; i < rx_ring->count; i++) {
2596 buffer_info = &rx_ring->buffer_info[i];
2597 if (buffer_info->dma) {
2598 pci_unmap_single(rx_ring->pdev,
2599 buffer_info->dma,
2600 rx_ring->rx_buffer_len,
2601 PCI_DMA_FROMDEVICE);
2602 buffer_info->dma = 0;
2603 }
2604
2605 if (buffer_info->skb) {
2606 dev_kfree_skb(buffer_info->skb);
2607 buffer_info->skb = NULL;
2608 }
2609 if (buffer_info->page_dma) {
2610 pci_unmap_page(rx_ring->pdev,
2611 buffer_info->page_dma,
2612 PAGE_SIZE / 2,
2613 PCI_DMA_FROMDEVICE);
2614 buffer_info->page_dma = 0;
2615 }
2616 if (buffer_info->page) {
2617 put_page(buffer_info->page);
2618 buffer_info->page = NULL;
2619 buffer_info->page_offset = 0;
2620 }
2621 }
2622
2623 size = sizeof(struct igb_buffer) * rx_ring->count;
2624 memset(rx_ring->buffer_info, 0, size);
2625
2626 /* Zero out the descriptor ring */
2627 memset(rx_ring->desc, 0, rx_ring->size);
2628
2629 rx_ring->next_to_clean = 0;
2630 rx_ring->next_to_use = 0;
2631
2632 writel(0, rx_ring->head);
2633 writel(0, rx_ring->tail);
2634 }
2635
2636 /**
2637 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2638 * @adapter: board private structure
2639 **/
2640 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2641 {
2642 int i;
2643
2644 for (i = 0; i < adapter->num_rx_queues; i++)
2645 igb_clean_rx_ring(&adapter->rx_ring[i]);
2646 }
2647
2648 /**
2649 * igb_set_mac - Change the Ethernet Address of the NIC
2650 * @netdev: network interface device structure
2651 * @p: pointer to an address structure
2652 *
2653 * Returns 0 on success, negative on failure
2654 **/
2655 static int igb_set_mac(struct net_device *netdev, void *p)
2656 {
2657 struct igb_adapter *adapter = netdev_priv(netdev);
2658 struct e1000_hw *hw = &adapter->hw;
2659 struct sockaddr *addr = p;
2660
2661 if (!is_valid_ether_addr(addr->sa_data))
2662 return -EADDRNOTAVAIL;
2663
2664 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2665 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2666
2667 /* set the correct pool for the new PF MAC address in entry 0 */
2668 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2669 adapter->vfs_allocated_count);
2670
2671 return 0;
2672 }
2673
2674 /**
2675 * igb_write_mc_addr_list - write multicast addresses to MTA
2676 * @netdev: network interface device structure
2677 *
2678 * Writes multicast address list to the MTA hash table.
2679 * Returns: -ENOMEM on failure
2680 * 0 on no addresses written
2681 * X on writing X addresses to MTA
2682 **/
2683 static int igb_write_mc_addr_list(struct net_device *netdev)
2684 {
2685 struct igb_adapter *adapter = netdev_priv(netdev);
2686 struct e1000_hw *hw = &adapter->hw;
2687 struct dev_mc_list *mc_ptr = netdev->mc_list;
2688 u8 *mta_list;
2689 u32 vmolr = 0;
2690 int i;
2691
2692 if (!netdev->mc_count) {
2693 /* nothing to program, so clear mc list */
2694 igb_update_mc_addr_list(hw, NULL, 0);
2695 igb_restore_vf_multicasts(adapter);
2696 return 0;
2697 }
2698
2699 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2700 if (!mta_list)
2701 return -ENOMEM;
2702
2703 /* set vmolr receive overflow multicast bit */
2704 vmolr |= E1000_VMOLR_ROMPE;
2705
2706 /* The shared function expects a packed array of only addresses. */
2707 mc_ptr = netdev->mc_list;
2708
2709 for (i = 0; i < netdev->mc_count; i++) {
2710 if (!mc_ptr)
2711 break;
2712 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2713 mc_ptr = mc_ptr->next;
2714 }
2715 igb_update_mc_addr_list(hw, mta_list, i);
2716 kfree(mta_list);
2717
2718 return netdev->mc_count;
2719 }
2720
2721 /**
2722 * igb_write_uc_addr_list - write unicast addresses to RAR table
2723 * @netdev: network interface device structure
2724 *
2725 * Writes unicast address list to the RAR table.
2726 * Returns: -ENOMEM on failure/insufficient address space
2727 * 0 on no addresses written
2728 * X on writing X addresses to the RAR table
2729 **/
2730 static int igb_write_uc_addr_list(struct net_device *netdev)
2731 {
2732 struct igb_adapter *adapter = netdev_priv(netdev);
2733 struct e1000_hw *hw = &adapter->hw;
2734 unsigned int vfn = adapter->vfs_allocated_count;
2735 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2736 int count = 0;
2737
2738 /* return ENOMEM indicating insufficient memory for addresses */
2739 if (netdev->uc.count > rar_entries)
2740 return -ENOMEM;
2741
2742 if (netdev->uc.count && rar_entries) {
2743 struct netdev_hw_addr *ha;
2744 list_for_each_entry(ha, &netdev->uc.list, list) {
2745 if (!rar_entries)
2746 break;
2747 igb_rar_set_qsel(adapter, ha->addr,
2748 rar_entries--,
2749 vfn);
2750 count++;
2751 }
2752 }
2753 /* write the addresses in reverse order to avoid write combining */
2754 for (; rar_entries > 0 ; rar_entries--) {
2755 wr32(E1000_RAH(rar_entries), 0);
2756 wr32(E1000_RAL(rar_entries), 0);
2757 }
2758 wrfl();
2759
2760 return count;
2761 }
2762
2763 /**
2764 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2765 * @netdev: network interface device structure
2766 *
2767 * The set_rx_mode entry point is called whenever the unicast or multicast
2768 * address lists or the network interface flags are updated. This routine is
2769 * responsible for configuring the hardware for proper unicast, multicast,
2770 * promiscuous mode, and all-multi behavior.
2771 **/
2772 static void igb_set_rx_mode(struct net_device *netdev)
2773 {
2774 struct igb_adapter *adapter = netdev_priv(netdev);
2775 struct e1000_hw *hw = &adapter->hw;
2776 unsigned int vfn = adapter->vfs_allocated_count;
2777 u32 rctl, vmolr = 0;
2778 int count;
2779
2780 /* Check for Promiscuous and All Multicast modes */
2781 rctl = rd32(E1000_RCTL);
2782
2783 /* clear the effected bits */
2784 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2785
2786 if (netdev->flags & IFF_PROMISC) {
2787 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2788 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2789 } else {
2790 if (netdev->flags & IFF_ALLMULTI) {
2791 rctl |= E1000_RCTL_MPE;
2792 vmolr |= E1000_VMOLR_MPME;
2793 } else {
2794 /*
2795 * Write addresses to the MTA, if the attempt fails
2796 * then we should just turn on promiscous mode so
2797 * that we can at least receive multicast traffic
2798 */
2799 count = igb_write_mc_addr_list(netdev);
2800 if (count < 0) {
2801 rctl |= E1000_RCTL_MPE;
2802 vmolr |= E1000_VMOLR_MPME;
2803 } else if (count) {
2804 vmolr |= E1000_VMOLR_ROMPE;
2805 }
2806 }
2807 /*
2808 * Write addresses to available RAR registers, if there is not
2809 * sufficient space to store all the addresses then enable
2810 * unicast promiscous mode
2811 */
2812 count = igb_write_uc_addr_list(netdev);
2813 if (count < 0) {
2814 rctl |= E1000_RCTL_UPE;
2815 vmolr |= E1000_VMOLR_ROPE;
2816 }
2817 rctl |= E1000_RCTL_VFE;
2818 }
2819 wr32(E1000_RCTL, rctl);
2820
2821 /*
2822 * In order to support SR-IOV and eventually VMDq it is necessary to set
2823 * the VMOLR to enable the appropriate modes. Without this workaround
2824 * we will have issues with VLAN tag stripping not being done for frames
2825 * that are only arriving because we are the default pool
2826 */
2827 if (hw->mac.type < e1000_82576)
2828 return;
2829
2830 vmolr |= rd32(E1000_VMOLR(vfn)) &
2831 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2832 wr32(E1000_VMOLR(vfn), vmolr);
2833 igb_restore_vf_multicasts(adapter);
2834 }
2835
2836 /* Need to wait a few seconds after link up to get diagnostic information from
2837 * the phy */
2838 static void igb_update_phy_info(unsigned long data)
2839 {
2840 struct igb_adapter *adapter = (struct igb_adapter *) data;
2841 igb_get_phy_info(&adapter->hw);
2842 }
2843
2844 /**
2845 * igb_has_link - check shared code for link and determine up/down
2846 * @adapter: pointer to driver private info
2847 **/
2848 static bool igb_has_link(struct igb_adapter *adapter)
2849 {
2850 struct e1000_hw *hw = &adapter->hw;
2851 bool link_active = false;
2852 s32 ret_val = 0;
2853
2854 /* get_link_status is set on LSC (link status) interrupt or
2855 * rx sequence error interrupt. get_link_status will stay
2856 * false until the e1000_check_for_link establishes link
2857 * for copper adapters ONLY
2858 */
2859 switch (hw->phy.media_type) {
2860 case e1000_media_type_copper:
2861 if (hw->mac.get_link_status) {
2862 ret_val = hw->mac.ops.check_for_link(hw);
2863 link_active = !hw->mac.get_link_status;
2864 } else {
2865 link_active = true;
2866 }
2867 break;
2868 case e1000_media_type_internal_serdes:
2869 ret_val = hw->mac.ops.check_for_link(hw);
2870 link_active = hw->mac.serdes_has_link;
2871 break;
2872 default:
2873 case e1000_media_type_unknown:
2874 break;
2875 }
2876
2877 return link_active;
2878 }
2879
2880 /**
2881 * igb_watchdog - Timer Call-back
2882 * @data: pointer to adapter cast into an unsigned long
2883 **/
2884 static void igb_watchdog(unsigned long data)
2885 {
2886 struct igb_adapter *adapter = (struct igb_adapter *)data;
2887 /* Do the rest outside of interrupt context */
2888 schedule_work(&adapter->watchdog_task);
2889 }
2890
2891 static void igb_watchdog_task(struct work_struct *work)
2892 {
2893 struct igb_adapter *adapter = container_of(work,
2894 struct igb_adapter, watchdog_task);
2895 struct e1000_hw *hw = &adapter->hw;
2896 struct net_device *netdev = adapter->netdev;
2897 struct igb_ring *tx_ring = adapter->tx_ring;
2898 u32 link;
2899 int i;
2900
2901 link = igb_has_link(adapter);
2902 if ((netif_carrier_ok(netdev)) && link)
2903 goto link_up;
2904
2905 if (link) {
2906 if (!netif_carrier_ok(netdev)) {
2907 u32 ctrl;
2908 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2909 &adapter->link_speed,
2910 &adapter->link_duplex);
2911
2912 ctrl = rd32(E1000_CTRL);
2913 /* Links status message must follow this format */
2914 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
2915 "Flow Control: %s\n",
2916 netdev->name,
2917 adapter->link_speed,
2918 adapter->link_duplex == FULL_DUPLEX ?
2919 "Full Duplex" : "Half Duplex",
2920 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2921 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2922 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2923 E1000_CTRL_TFCE) ? "TX" : "None")));
2924
2925 /* tweak tx_queue_len according to speed/duplex and
2926 * adjust the timeout factor */
2927 netdev->tx_queue_len = adapter->tx_queue_len;
2928 adapter->tx_timeout_factor = 1;
2929 switch (adapter->link_speed) {
2930 case SPEED_10:
2931 netdev->tx_queue_len = 10;
2932 adapter->tx_timeout_factor = 14;
2933 break;
2934 case SPEED_100:
2935 netdev->tx_queue_len = 100;
2936 /* maybe add some timeout factor ? */
2937 break;
2938 }
2939
2940 netif_carrier_on(netdev);
2941
2942 igb_ping_all_vfs(adapter);
2943
2944 /* link state has changed, schedule phy info update */
2945 if (!test_bit(__IGB_DOWN, &adapter->state))
2946 mod_timer(&adapter->phy_info_timer,
2947 round_jiffies(jiffies + 2 * HZ));
2948 }
2949 } else {
2950 if (netif_carrier_ok(netdev)) {
2951 adapter->link_speed = 0;
2952 adapter->link_duplex = 0;
2953 /* Links status message must follow this format */
2954 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2955 netdev->name);
2956 netif_carrier_off(netdev);
2957
2958 igb_ping_all_vfs(adapter);
2959
2960 /* link state has changed, schedule phy info update */
2961 if (!test_bit(__IGB_DOWN, &adapter->state))
2962 mod_timer(&adapter->phy_info_timer,
2963 round_jiffies(jiffies + 2 * HZ));
2964 }
2965 }
2966
2967 link_up:
2968 igb_update_stats(adapter);
2969
2970 hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2971 adapter->tpt_old = adapter->stats.tpt;
2972 hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
2973 adapter->colc_old = adapter->stats.colc;
2974
2975 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
2976 adapter->gorc_old = adapter->stats.gorc;
2977 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
2978 adapter->gotc_old = adapter->stats.gotc;
2979
2980 igb_update_adaptive(&adapter->hw);
2981
2982 if (!netif_carrier_ok(netdev)) {
2983 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
2984 /* We've lost link, so the controller stops DMA,
2985 * but we've got queued Tx work that's never going
2986 * to get done, so reset controller to flush Tx.
2987 * (Do the reset outside of interrupt context). */
2988 adapter->tx_timeout_count++;
2989 schedule_work(&adapter->reset_task);
2990 /* return immediately since reset is imminent */
2991 return;
2992 }
2993 }
2994
2995 /* Cause software interrupt to ensure rx ring is cleaned */
2996 if (adapter->msix_entries) {
2997 u32 eics = 0;
2998 for (i = 0; i < adapter->num_q_vectors; i++) {
2999 struct igb_q_vector *q_vector = adapter->q_vector[i];
3000 eics |= q_vector->eims_value;
3001 }
3002 wr32(E1000_EICS, eics);
3003 } else {
3004 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3005 }
3006
3007 /* Force detection of hung controller every watchdog period */
3008 tx_ring->detect_tx_hung = true;
3009
3010 /* Reset the timer */
3011 if (!test_bit(__IGB_DOWN, &adapter->state))
3012 mod_timer(&adapter->watchdog_timer,
3013 round_jiffies(jiffies + 2 * HZ));
3014 }
3015
3016 enum latency_range {
3017 lowest_latency = 0,
3018 low_latency = 1,
3019 bulk_latency = 2,
3020 latency_invalid = 255
3021 };
3022
3023 /**
3024 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3025 *
3026 * Stores a new ITR value based on strictly on packet size. This
3027 * algorithm is less sophisticated than that used in igb_update_itr,
3028 * due to the difficulty of synchronizing statistics across multiple
3029 * receive rings. The divisors and thresholds used by this fuction
3030 * were determined based on theoretical maximum wire speed and testing
3031 * data, in order to minimize response time while increasing bulk
3032 * throughput.
3033 * This functionality is controlled by the InterruptThrottleRate module
3034 * parameter (see igb_param.c)
3035 * NOTE: This function is called only when operating in a multiqueue
3036 * receive environment.
3037 * @q_vector: pointer to q_vector
3038 **/
3039 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3040 {
3041 int new_val = q_vector->itr_val;
3042 int avg_wire_size = 0;
3043 struct igb_adapter *adapter = q_vector->adapter;
3044
3045 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3046 * ints/sec - ITR timer value of 120 ticks.
3047 */
3048 if (adapter->link_speed != SPEED_1000) {
3049 new_val = 976;
3050 goto set_itr_val;
3051 }
3052
3053 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3054 struct igb_ring *ring = q_vector->rx_ring;
3055 avg_wire_size = ring->total_bytes / ring->total_packets;
3056 }
3057
3058 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3059 struct igb_ring *ring = q_vector->tx_ring;
3060 avg_wire_size = max_t(u32, avg_wire_size,
3061 (ring->total_bytes /
3062 ring->total_packets));
3063 }
3064
3065 /* if avg_wire_size isn't set no work was done */
3066 if (!avg_wire_size)
3067 goto clear_counts;
3068
3069 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3070 avg_wire_size += 24;
3071
3072 /* Don't starve jumbo frames */
3073 avg_wire_size = min(avg_wire_size, 3000);
3074
3075 /* Give a little boost to mid-size frames */
3076 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3077 new_val = avg_wire_size / 3;
3078 else
3079 new_val = avg_wire_size / 2;
3080
3081 set_itr_val:
3082 if (new_val != q_vector->itr_val) {
3083 q_vector->itr_val = new_val;
3084 q_vector->set_itr = 1;
3085 }
3086 clear_counts:
3087 if (q_vector->rx_ring) {
3088 q_vector->rx_ring->total_bytes = 0;
3089 q_vector->rx_ring->total_packets = 0;
3090 }
3091 if (q_vector->tx_ring) {
3092 q_vector->tx_ring->total_bytes = 0;
3093 q_vector->tx_ring->total_packets = 0;
3094 }
3095 }
3096
3097 /**
3098 * igb_update_itr - update the dynamic ITR value based on statistics
3099 * Stores a new ITR value based on packets and byte
3100 * counts during the last interrupt. The advantage of per interrupt
3101 * computation is faster updates and more accurate ITR for the current
3102 * traffic pattern. Constants in this function were computed
3103 * based on theoretical maximum wire speed and thresholds were set based
3104 * on testing data as well as attempting to minimize response time
3105 * while increasing bulk throughput.
3106 * this functionality is controlled by the InterruptThrottleRate module
3107 * parameter (see igb_param.c)
3108 * NOTE: These calculations are only valid when operating in a single-
3109 * queue environment.
3110 * @adapter: pointer to adapter
3111 * @itr_setting: current q_vector->itr_val
3112 * @packets: the number of packets during this measurement interval
3113 * @bytes: the number of bytes during this measurement interval
3114 **/
3115 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3116 int packets, int bytes)
3117 {
3118 unsigned int retval = itr_setting;
3119
3120 if (packets == 0)
3121 goto update_itr_done;
3122
3123 switch (itr_setting) {
3124 case lowest_latency:
3125 /* handle TSO and jumbo frames */
3126 if (bytes/packets > 8000)
3127 retval = bulk_latency;
3128 else if ((packets < 5) && (bytes > 512))
3129 retval = low_latency;
3130 break;
3131 case low_latency: /* 50 usec aka 20000 ints/s */
3132 if (bytes > 10000) {
3133 /* this if handles the TSO accounting */
3134 if (bytes/packets > 8000) {
3135 retval = bulk_latency;
3136 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3137 retval = bulk_latency;
3138 } else if ((packets > 35)) {
3139 retval = lowest_latency;
3140 }
3141 } else if (bytes/packets > 2000) {
3142 retval = bulk_latency;
3143 } else if (packets <= 2 && bytes < 512) {
3144 retval = lowest_latency;
3145 }
3146 break;
3147 case bulk_latency: /* 250 usec aka 4000 ints/s */
3148 if (bytes > 25000) {
3149 if (packets > 35)
3150 retval = low_latency;
3151 } else if (bytes < 1500) {
3152 retval = low_latency;
3153 }
3154 break;
3155 }
3156
3157 update_itr_done:
3158 return retval;
3159 }
3160
3161 static void igb_set_itr(struct igb_adapter *adapter)
3162 {
3163 struct igb_q_vector *q_vector = adapter->q_vector[0];
3164 u16 current_itr;
3165 u32 new_itr = q_vector->itr_val;
3166
3167 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3168 if (adapter->link_speed != SPEED_1000) {
3169 current_itr = 0;
3170 new_itr = 4000;
3171 goto set_itr_now;
3172 }
3173
3174 adapter->rx_itr = igb_update_itr(adapter,
3175 adapter->rx_itr,
3176 adapter->rx_ring->total_packets,
3177 adapter->rx_ring->total_bytes);
3178
3179 adapter->tx_itr = igb_update_itr(adapter,
3180 adapter->tx_itr,
3181 adapter->tx_ring->total_packets,
3182 adapter->tx_ring->total_bytes);
3183 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3184
3185 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3186 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3187 current_itr = low_latency;
3188
3189 switch (current_itr) {
3190 /* counts and packets in update_itr are dependent on these numbers */
3191 case lowest_latency:
3192 new_itr = 56; /* aka 70,000 ints/sec */
3193 break;
3194 case low_latency:
3195 new_itr = 196; /* aka 20,000 ints/sec */
3196 break;
3197 case bulk_latency:
3198 new_itr = 980; /* aka 4,000 ints/sec */
3199 break;
3200 default:
3201 break;
3202 }
3203
3204 set_itr_now:
3205 adapter->rx_ring->total_bytes = 0;
3206 adapter->rx_ring->total_packets = 0;
3207 adapter->tx_ring->total_bytes = 0;
3208 adapter->tx_ring->total_packets = 0;
3209
3210 if (new_itr != q_vector->itr_val) {
3211 /* this attempts to bias the interrupt rate towards Bulk
3212 * by adding intermediate steps when interrupt rate is
3213 * increasing */
3214 new_itr = new_itr > q_vector->itr_val ?
3215 max((new_itr * q_vector->itr_val) /
3216 (new_itr + (q_vector->itr_val >> 2)),
3217 new_itr) :
3218 new_itr;
3219 /* Don't write the value here; it resets the adapter's
3220 * internal timer, and causes us to delay far longer than
3221 * we should between interrupts. Instead, we write the ITR
3222 * value at the beginning of the next interrupt so the timing
3223 * ends up being correct.
3224 */
3225 q_vector->itr_val = new_itr;
3226 q_vector->set_itr = 1;
3227 }
3228
3229 return;
3230 }
3231
3232 #define IGB_TX_FLAGS_CSUM 0x00000001
3233 #define IGB_TX_FLAGS_VLAN 0x00000002
3234 #define IGB_TX_FLAGS_TSO 0x00000004
3235 #define IGB_TX_FLAGS_IPV4 0x00000008
3236 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3237 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3238 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3239
3240 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3241 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3242 {
3243 struct e1000_adv_tx_context_desc *context_desc;
3244 unsigned int i;
3245 int err;
3246 struct igb_buffer *buffer_info;
3247 u32 info = 0, tu_cmd = 0;
3248 u32 mss_l4len_idx, l4len;
3249 *hdr_len = 0;
3250
3251 if (skb_header_cloned(skb)) {
3252 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3253 if (err)
3254 return err;
3255 }
3256
3257 l4len = tcp_hdrlen(skb);
3258 *hdr_len += l4len;
3259
3260 if (skb->protocol == htons(ETH_P_IP)) {
3261 struct iphdr *iph = ip_hdr(skb);
3262 iph->tot_len = 0;
3263 iph->check = 0;
3264 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3265 iph->daddr, 0,
3266 IPPROTO_TCP,
3267 0);
3268 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3269 ipv6_hdr(skb)->payload_len = 0;
3270 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3271 &ipv6_hdr(skb)->daddr,
3272 0, IPPROTO_TCP, 0);
3273 }
3274
3275 i = tx_ring->next_to_use;
3276
3277 buffer_info = &tx_ring->buffer_info[i];
3278 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3279 /* VLAN MACLEN IPLEN */
3280 if (tx_flags & IGB_TX_FLAGS_VLAN)
3281 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3282 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3283 *hdr_len += skb_network_offset(skb);
3284 info |= skb_network_header_len(skb);
3285 *hdr_len += skb_network_header_len(skb);
3286 context_desc->vlan_macip_lens = cpu_to_le32(info);
3287
3288 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3289 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3290
3291 if (skb->protocol == htons(ETH_P_IP))
3292 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3293 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3294
3295 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3296
3297 /* MSS L4LEN IDX */
3298 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3299 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3300
3301 /* For 82575, context index must be unique per ring. */
3302 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3303 mss_l4len_idx |= tx_ring->reg_idx << 4;
3304
3305 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3306 context_desc->seqnum_seed = 0;
3307
3308 buffer_info->time_stamp = jiffies;
3309 buffer_info->next_to_watch = i;
3310 buffer_info->dma = 0;
3311 i++;
3312 if (i == tx_ring->count)
3313 i = 0;
3314
3315 tx_ring->next_to_use = i;
3316
3317 return true;
3318 }
3319
3320 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3321 struct sk_buff *skb, u32 tx_flags)
3322 {
3323 struct e1000_adv_tx_context_desc *context_desc;
3324 struct pci_dev *pdev = tx_ring->pdev;
3325 struct igb_buffer *buffer_info;
3326 u32 info = 0, tu_cmd = 0;
3327 unsigned int i;
3328
3329 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3330 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3331 i = tx_ring->next_to_use;
3332 buffer_info = &tx_ring->buffer_info[i];
3333 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3334
3335 if (tx_flags & IGB_TX_FLAGS_VLAN)
3336 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3337 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3338 if (skb->ip_summed == CHECKSUM_PARTIAL)
3339 info |= skb_network_header_len(skb);
3340
3341 context_desc->vlan_macip_lens = cpu_to_le32(info);
3342
3343 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3344
3345 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3346 __be16 protocol;
3347
3348 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3349 const struct vlan_ethhdr *vhdr =
3350 (const struct vlan_ethhdr*)skb->data;
3351
3352 protocol = vhdr->h_vlan_encapsulated_proto;
3353 } else {
3354 protocol = skb->protocol;
3355 }
3356
3357 switch (protocol) {
3358 case cpu_to_be16(ETH_P_IP):
3359 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3360 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3361 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3362 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3363 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3364 break;
3365 case cpu_to_be16(ETH_P_IPV6):
3366 /* XXX what about other V6 headers?? */
3367 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3368 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3369 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3370 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3371 break;
3372 default:
3373 if (unlikely(net_ratelimit()))
3374 dev_warn(&pdev->dev,
3375 "partial checksum but proto=%x!\n",
3376 skb->protocol);
3377 break;
3378 }
3379 }
3380
3381 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3382 context_desc->seqnum_seed = 0;
3383 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3384 context_desc->mss_l4len_idx =
3385 cpu_to_le32(tx_ring->reg_idx << 4);
3386
3387 buffer_info->time_stamp = jiffies;
3388 buffer_info->next_to_watch = i;
3389 buffer_info->dma = 0;
3390
3391 i++;
3392 if (i == tx_ring->count)
3393 i = 0;
3394 tx_ring->next_to_use = i;
3395
3396 return true;
3397 }
3398 return false;
3399 }
3400
3401 #define IGB_MAX_TXD_PWR 16
3402 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3403
3404 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3405 unsigned int first)
3406 {
3407 struct igb_buffer *buffer_info;
3408 struct pci_dev *pdev = tx_ring->pdev;
3409 unsigned int len = skb_headlen(skb);
3410 unsigned int count = 0, i;
3411 unsigned int f;
3412 dma_addr_t *map;
3413
3414 i = tx_ring->next_to_use;
3415
3416 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3417 dev_err(&pdev->dev, "TX DMA map failed\n");
3418 return 0;
3419 }
3420
3421 map = skb_shinfo(skb)->dma_maps;
3422
3423 buffer_info = &tx_ring->buffer_info[i];
3424 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3425 buffer_info->length = len;
3426 /* set time_stamp *before* dma to help avoid a possible race */
3427 buffer_info->time_stamp = jiffies;
3428 buffer_info->next_to_watch = i;
3429 buffer_info->dma = skb_shinfo(skb)->dma_head;
3430
3431 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3432 struct skb_frag_struct *frag;
3433
3434 i++;
3435 if (i == tx_ring->count)
3436 i = 0;
3437
3438 frag = &skb_shinfo(skb)->frags[f];
3439 len = frag->size;
3440
3441 buffer_info = &tx_ring->buffer_info[i];
3442 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3443 buffer_info->length = len;
3444 buffer_info->time_stamp = jiffies;
3445 buffer_info->next_to_watch = i;
3446 buffer_info->dma = map[count];
3447 count++;
3448 }
3449
3450 tx_ring->buffer_info[i].skb = skb;
3451 tx_ring->buffer_info[first].next_to_watch = i;
3452
3453 return count + 1;
3454 }
3455
3456 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3457 int tx_flags, int count, u32 paylen,
3458 u8 hdr_len)
3459 {
3460 union e1000_adv_tx_desc *tx_desc = NULL;
3461 struct igb_buffer *buffer_info;
3462 u32 olinfo_status = 0, cmd_type_len;
3463 unsigned int i;
3464
3465 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3466 E1000_ADVTXD_DCMD_DEXT);
3467
3468 if (tx_flags & IGB_TX_FLAGS_VLAN)
3469 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3470
3471 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3472 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3473
3474 if (tx_flags & IGB_TX_FLAGS_TSO) {
3475 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3476
3477 /* insert tcp checksum */
3478 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3479
3480 /* insert ip checksum */
3481 if (tx_flags & IGB_TX_FLAGS_IPV4)
3482 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3483
3484 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3485 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3486 }
3487
3488 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3489 (tx_flags & (IGB_TX_FLAGS_CSUM |
3490 IGB_TX_FLAGS_TSO |
3491 IGB_TX_FLAGS_VLAN)))
3492 olinfo_status |= tx_ring->reg_idx << 4;
3493
3494 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3495
3496 i = tx_ring->next_to_use;
3497 while (count--) {
3498 buffer_info = &tx_ring->buffer_info[i];
3499 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3500 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3501 tx_desc->read.cmd_type_len =
3502 cpu_to_le32(cmd_type_len | buffer_info->length);
3503 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3504 i++;
3505 if (i == tx_ring->count)
3506 i = 0;
3507 }
3508
3509 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
3510 /* Force memory writes to complete before letting h/w
3511 * know there are new descriptors to fetch. (Only
3512 * applicable for weak-ordered memory model archs,
3513 * such as IA-64). */
3514 wmb();
3515
3516 tx_ring->next_to_use = i;
3517 writel(i, tx_ring->tail);
3518 /* we need this if more than one processor can write to our tail
3519 * at a time, it syncronizes IO on IA64/Altix systems */
3520 mmiowb();
3521 }
3522
3523 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3524 {
3525 struct net_device *netdev = tx_ring->netdev;
3526
3527 netif_stop_subqueue(netdev, tx_ring->queue_index);
3528
3529 /* Herbert's original patch had:
3530 * smp_mb__after_netif_stop_queue();
3531 * but since that doesn't exist yet, just open code it. */
3532 smp_mb();
3533
3534 /* We need to check again in a case another CPU has just
3535 * made room available. */
3536 if (igb_desc_unused(tx_ring) < size)
3537 return -EBUSY;
3538
3539 /* A reprieve! */
3540 netif_wake_subqueue(netdev, tx_ring->queue_index);
3541 tx_ring->tx_stats.restart_queue++;
3542 return 0;
3543 }
3544
3545 static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3546 {
3547 if (igb_desc_unused(tx_ring) >= size)
3548 return 0;
3549 return __igb_maybe_stop_tx(tx_ring, size);
3550 }
3551
3552 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3553 struct igb_ring *tx_ring)
3554 {
3555 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
3556 unsigned int first;
3557 unsigned int tx_flags = 0;
3558 u8 hdr_len = 0;
3559 int count = 0;
3560 int tso = 0;
3561 union skb_shared_tx *shtx = skb_tx(skb);
3562
3563 /* need: 1 descriptor per page,
3564 * + 2 desc gap to keep tail from touching head,
3565 * + 1 desc for skb->data,
3566 * + 1 desc for context descriptor,
3567 * otherwise try next time */
3568 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
3569 /* this is a hard error */
3570 return NETDEV_TX_BUSY;
3571 }
3572
3573 if (unlikely(shtx->hardware)) {
3574 shtx->in_progress = 1;
3575 tx_flags |= IGB_TX_FLAGS_TSTAMP;
3576 }
3577
3578 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
3579 tx_flags |= IGB_TX_FLAGS_VLAN;
3580 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3581 }
3582
3583 if (skb->protocol == htons(ETH_P_IP))
3584 tx_flags |= IGB_TX_FLAGS_IPV4;
3585
3586 first = tx_ring->next_to_use;
3587 if (skb_is_gso(skb)) {
3588 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3589 if (tso < 0) {
3590 dev_kfree_skb_any(skb);
3591 return NETDEV_TX_OK;
3592 }
3593 }
3594
3595 if (tso)
3596 tx_flags |= IGB_TX_FLAGS_TSO;
3597 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
3598 (skb->ip_summed == CHECKSUM_PARTIAL))
3599 tx_flags |= IGB_TX_FLAGS_CSUM;
3600
3601 /*
3602 * count reflects descriptors mapped, if 0 then mapping error
3603 * has occured and we need to rewind the descriptor queue
3604 */
3605 count = igb_tx_map_adv(tx_ring, skb, first);
3606
3607 if (!count) {
3608 dev_kfree_skb_any(skb);
3609 tx_ring->buffer_info[first].time_stamp = 0;
3610 tx_ring->next_to_use = first;
3611 return NETDEV_TX_OK;
3612 }
3613
3614 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3615
3616 /* Make sure there is space in the ring for the next send. */
3617 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
3618
3619 return NETDEV_TX_OK;
3620 }
3621
3622 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3623 struct net_device *netdev)
3624 {
3625 struct igb_adapter *adapter = netdev_priv(netdev);
3626 struct igb_ring *tx_ring;
3627 int r_idx = 0;
3628
3629 if (test_bit(__IGB_DOWN, &adapter->state)) {
3630 dev_kfree_skb_any(skb);
3631 return NETDEV_TX_OK;
3632 }
3633
3634 if (skb->len <= 0) {
3635 dev_kfree_skb_any(skb);
3636 return NETDEV_TX_OK;
3637 }
3638
3639 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
3640 tx_ring = adapter->multi_tx_table[r_idx];
3641
3642 /* This goes back to the question of how to logically map a tx queue
3643 * to a flow. Right now, performance is impacted slightly negatively
3644 * if using multiple tx queues. If the stack breaks away from a
3645 * single qdisc implementation, we can look at this again. */
3646 return igb_xmit_frame_ring_adv(skb, tx_ring);
3647 }
3648
3649 /**
3650 * igb_tx_timeout - Respond to a Tx Hang
3651 * @netdev: network interface device structure
3652 **/
3653 static void igb_tx_timeout(struct net_device *netdev)
3654 {
3655 struct igb_adapter *adapter = netdev_priv(netdev);
3656 struct e1000_hw *hw = &adapter->hw;
3657
3658 /* Do the reset outside of interrupt context */
3659 adapter->tx_timeout_count++;
3660 schedule_work(&adapter->reset_task);
3661 wr32(E1000_EICS,
3662 (adapter->eims_enable_mask & ~adapter->eims_other));
3663 }
3664
3665 static void igb_reset_task(struct work_struct *work)
3666 {
3667 struct igb_adapter *adapter;
3668 adapter = container_of(work, struct igb_adapter, reset_task);
3669
3670 igb_reinit_locked(adapter);
3671 }
3672
3673 /**
3674 * igb_get_stats - Get System Network Statistics
3675 * @netdev: network interface device structure
3676 *
3677 * Returns the address of the device statistics structure.
3678 * The statistics are actually updated from the timer callback.
3679 **/
3680 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
3681 {
3682 /* only return the current stats */
3683 return &netdev->stats;
3684 }
3685
3686 /**
3687 * igb_change_mtu - Change the Maximum Transfer Unit
3688 * @netdev: network interface device structure
3689 * @new_mtu: new value for maximum frame size
3690 *
3691 * Returns 0 on success, negative on failure
3692 **/
3693 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3694 {
3695 struct igb_adapter *adapter = netdev_priv(netdev);
3696 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3697 u32 rx_buffer_len, i;
3698
3699 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3700 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3701 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3702 return -EINVAL;
3703 }
3704
3705 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3706 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3707 return -EINVAL;
3708 }
3709
3710 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3711 msleep(1);
3712
3713 /* igb_down has a dependency on max_frame_size */
3714 adapter->max_frame_size = max_frame;
3715 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3716 * means we reserve 2 more, this pushes us to allocate from the next
3717 * larger slab size.
3718 * i.e. RXBUFFER_2048 --> size-4096 slab
3719 */
3720
3721 if (max_frame <= IGB_RXBUFFER_1024)
3722 rx_buffer_len = IGB_RXBUFFER_1024;
3723 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
3724 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3725 else
3726 rx_buffer_len = IGB_RXBUFFER_128;
3727
3728 if (netif_running(netdev))
3729 igb_down(adapter);
3730
3731 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3732 netdev->mtu, new_mtu);
3733 netdev->mtu = new_mtu;
3734
3735 for (i = 0; i < adapter->num_rx_queues; i++)
3736 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3737
3738 if (netif_running(netdev))
3739 igb_up(adapter);
3740 else
3741 igb_reset(adapter);
3742
3743 clear_bit(__IGB_RESETTING, &adapter->state);
3744
3745 return 0;
3746 }
3747
3748 /**
3749 * igb_update_stats - Update the board statistics counters
3750 * @adapter: board private structure
3751 **/
3752
3753 void igb_update_stats(struct igb_adapter *adapter)
3754 {
3755 struct net_device *netdev = adapter->netdev;
3756 struct e1000_hw *hw = &adapter->hw;
3757 struct pci_dev *pdev = adapter->pdev;
3758 u16 phy_tmp;
3759
3760 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3761
3762 /*
3763 * Prevent stats update while adapter is being reset, or if the pci
3764 * connection is down.
3765 */
3766 if (adapter->link_speed == 0)
3767 return;
3768 if (pci_channel_offline(pdev))
3769 return;
3770
3771 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3772 adapter->stats.gprc += rd32(E1000_GPRC);
3773 adapter->stats.gorc += rd32(E1000_GORCL);
3774 rd32(E1000_GORCH); /* clear GORCL */
3775 adapter->stats.bprc += rd32(E1000_BPRC);
3776 adapter->stats.mprc += rd32(E1000_MPRC);
3777 adapter->stats.roc += rd32(E1000_ROC);
3778
3779 adapter->stats.prc64 += rd32(E1000_PRC64);
3780 adapter->stats.prc127 += rd32(E1000_PRC127);
3781 adapter->stats.prc255 += rd32(E1000_PRC255);
3782 adapter->stats.prc511 += rd32(E1000_PRC511);
3783 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3784 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3785 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3786 adapter->stats.sec += rd32(E1000_SEC);
3787
3788 adapter->stats.mpc += rd32(E1000_MPC);
3789 adapter->stats.scc += rd32(E1000_SCC);
3790 adapter->stats.ecol += rd32(E1000_ECOL);
3791 adapter->stats.mcc += rd32(E1000_MCC);
3792 adapter->stats.latecol += rd32(E1000_LATECOL);
3793 adapter->stats.dc += rd32(E1000_DC);
3794 adapter->stats.rlec += rd32(E1000_RLEC);
3795 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3796 adapter->stats.xontxc += rd32(E1000_XONTXC);
3797 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3798 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3799 adapter->stats.fcruc += rd32(E1000_FCRUC);
3800 adapter->stats.gptc += rd32(E1000_GPTC);
3801 adapter->stats.gotc += rd32(E1000_GOTCL);
3802 rd32(E1000_GOTCH); /* clear GOTCL */
3803 adapter->stats.rnbc += rd32(E1000_RNBC);
3804 adapter->stats.ruc += rd32(E1000_RUC);
3805 adapter->stats.rfc += rd32(E1000_RFC);
3806 adapter->stats.rjc += rd32(E1000_RJC);
3807 adapter->stats.tor += rd32(E1000_TORH);
3808 adapter->stats.tot += rd32(E1000_TOTH);
3809 adapter->stats.tpr += rd32(E1000_TPR);
3810
3811 adapter->stats.ptc64 += rd32(E1000_PTC64);
3812 adapter->stats.ptc127 += rd32(E1000_PTC127);
3813 adapter->stats.ptc255 += rd32(E1000_PTC255);
3814 adapter->stats.ptc511 += rd32(E1000_PTC511);
3815 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3816 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3817
3818 adapter->stats.mptc += rd32(E1000_MPTC);
3819 adapter->stats.bptc += rd32(E1000_BPTC);
3820
3821 /* used for adaptive IFS */
3822
3823 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3824 adapter->stats.tpt += hw->mac.tx_packet_delta;
3825 hw->mac.collision_delta = rd32(E1000_COLC);
3826 adapter->stats.colc += hw->mac.collision_delta;
3827
3828 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3829 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3830 adapter->stats.tncrs += rd32(E1000_TNCRS);
3831 adapter->stats.tsctc += rd32(E1000_TSCTC);
3832 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3833
3834 adapter->stats.iac += rd32(E1000_IAC);
3835 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3836 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3837 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3838 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3839 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3840 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3841 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3842 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3843
3844 /* Fill out the OS statistics structure */
3845 netdev->stats.multicast = adapter->stats.mprc;
3846 netdev->stats.collisions = adapter->stats.colc;
3847
3848 /* Rx Errors */
3849
3850 if (hw->mac.type != e1000_82575) {
3851 u32 rqdpc_tmp;
3852 u64 rqdpc_total = 0;
3853 int i;
3854 /* Read out drops stats per RX queue. Notice RQDPC (Receive
3855 * Queue Drop Packet Count) stats only gets incremented, if
3856 * the DROP_EN but it set (in the SRRCTL register for that
3857 * queue). If DROP_EN bit is NOT set, then the some what
3858 * equivalent count is stored in RNBC (not per queue basis).
3859 * Also note the drop count is due to lack of available
3860 * descriptors.
3861 */
3862 for (i = 0; i < adapter->num_rx_queues; i++) {
3863 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0xFFF;
3864 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3865 rqdpc_total += adapter->rx_ring[i].rx_stats.drops;
3866 }
3867 netdev->stats.rx_fifo_errors = rqdpc_total;
3868 }
3869
3870 /* Note RNBC (Receive No Buffers Count) is an not an exact
3871 * drop count as the hardware FIFO might save the day. Thats
3872 * one of the reason for saving it in rx_fifo_errors, as its
3873 * potentially not a true drop.
3874 */
3875 netdev->stats.rx_fifo_errors += adapter->stats.rnbc;
3876
3877 /* RLEC on some newer hardware can be incorrect so build
3878 * our own version based on RUC and ROC */
3879 netdev->stats.rx_errors = adapter->stats.rxerrc +
3880 adapter->stats.crcerrs + adapter->stats.algnerrc +
3881 adapter->stats.ruc + adapter->stats.roc +
3882 adapter->stats.cexterr;
3883 netdev->stats.rx_length_errors = adapter->stats.ruc +
3884 adapter->stats.roc;
3885 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3886 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3887 netdev->stats.rx_missed_errors = adapter->stats.mpc;
3888
3889 /* Tx Errors */
3890 netdev->stats.tx_errors = adapter->stats.ecol +
3891 adapter->stats.latecol;
3892 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3893 netdev->stats.tx_window_errors = adapter->stats.latecol;
3894 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3895
3896 /* Tx Dropped needs to be maintained elsewhere */
3897
3898 /* Phy Stats */
3899 if (hw->phy.media_type == e1000_media_type_copper) {
3900 if ((adapter->link_speed == SPEED_1000) &&
3901 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3902 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3903 adapter->phy_stats.idle_errors += phy_tmp;
3904 }
3905 }
3906
3907 /* Management Stats */
3908 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3909 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3910 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3911 }
3912
3913 static irqreturn_t igb_msix_other(int irq, void *data)
3914 {
3915 struct igb_adapter *adapter = data;
3916 struct e1000_hw *hw = &adapter->hw;
3917 u32 icr = rd32(E1000_ICR);
3918 /* reading ICR causes bit 31 of EICR to be cleared */
3919
3920 if (icr & E1000_ICR_DOUTSYNC) {
3921 /* HW is reporting DMA is out of sync */
3922 adapter->stats.doosync++;
3923 }
3924
3925 /* Check for a mailbox event */
3926 if (icr & E1000_ICR_VMMB)
3927 igb_msg_task(adapter);
3928
3929 if (icr & E1000_ICR_LSC) {
3930 hw->mac.get_link_status = 1;
3931 /* guard against interrupt when we're going down */
3932 if (!test_bit(__IGB_DOWN, &adapter->state))
3933 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3934 }
3935
3936 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_VMMB);
3937 wr32(E1000_EIMS, adapter->eims_other);
3938
3939 return IRQ_HANDLED;
3940 }
3941
3942 static void igb_write_itr(struct igb_q_vector *q_vector)
3943 {
3944 u32 itr_val = q_vector->itr_val & 0x7FFC;
3945
3946 if (!q_vector->set_itr)
3947 return;
3948
3949 if (!itr_val)
3950 itr_val = 0x4;
3951
3952 if (q_vector->itr_shift)
3953 itr_val |= itr_val << q_vector->itr_shift;
3954 else
3955 itr_val |= 0x8000000;
3956
3957 writel(itr_val, q_vector->itr_register);
3958 q_vector->set_itr = 0;
3959 }
3960
3961 static irqreturn_t igb_msix_ring(int irq, void *data)
3962 {
3963 struct igb_q_vector *q_vector = data;
3964
3965 /* Write the ITR value calculated from the previous interrupt. */
3966 igb_write_itr(q_vector);
3967
3968 napi_schedule(&q_vector->napi);
3969
3970 return IRQ_HANDLED;
3971 }
3972
3973 #ifdef CONFIG_IGB_DCA
3974 static void igb_update_dca(struct igb_q_vector *q_vector)
3975 {
3976 struct igb_adapter *adapter = q_vector->adapter;
3977 struct e1000_hw *hw = &adapter->hw;
3978 int cpu = get_cpu();
3979
3980 if (q_vector->cpu == cpu)
3981 goto out_no_update;
3982
3983 if (q_vector->tx_ring) {
3984 int q = q_vector->tx_ring->reg_idx;
3985 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
3986 if (hw->mac.type == e1000_82575) {
3987 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
3988 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
3989 } else {
3990 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
3991 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
3992 E1000_DCA_TXCTRL_CPUID_SHIFT;
3993 }
3994 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
3995 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
3996 }
3997 if (q_vector->rx_ring) {
3998 int q = q_vector->rx_ring->reg_idx;
3999 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4000 if (hw->mac.type == e1000_82575) {
4001 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4002 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4003 } else {
4004 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4005 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4006 E1000_DCA_RXCTRL_CPUID_SHIFT;
4007 }
4008 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4009 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4010 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4011 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4012 }
4013 q_vector->cpu = cpu;
4014 out_no_update:
4015 put_cpu();
4016 }
4017
4018 static void igb_setup_dca(struct igb_adapter *adapter)
4019 {
4020 struct e1000_hw *hw = &adapter->hw;
4021 int i;
4022
4023 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4024 return;
4025
4026 /* Always use CB2 mode, difference is masked in the CB driver. */
4027 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4028
4029 for (i = 0; i < adapter->num_q_vectors; i++) {
4030 struct igb_q_vector *q_vector = adapter->q_vector[i];
4031 q_vector->cpu = -1;
4032 igb_update_dca(q_vector);
4033 }
4034 }
4035
4036 static int __igb_notify_dca(struct device *dev, void *data)
4037 {
4038 struct net_device *netdev = dev_get_drvdata(dev);
4039 struct igb_adapter *adapter = netdev_priv(netdev);
4040 struct e1000_hw *hw = &adapter->hw;
4041 unsigned long event = *(unsigned long *)data;
4042
4043 switch (event) {
4044 case DCA_PROVIDER_ADD:
4045 /* if already enabled, don't do it again */
4046 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4047 break;
4048 /* Always use CB2 mode, difference is masked
4049 * in the CB driver. */
4050 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4051 if (dca_add_requester(dev) == 0) {
4052 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4053 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4054 igb_setup_dca(adapter);
4055 break;
4056 }
4057 /* Fall Through since DCA is disabled. */
4058 case DCA_PROVIDER_REMOVE:
4059 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4060 /* without this a class_device is left
4061 * hanging around in the sysfs model */
4062 dca_remove_requester(dev);
4063 dev_info(&adapter->pdev->dev, "DCA disabled\n");
4064 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4065 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4066 }
4067 break;
4068 }
4069
4070 return 0;
4071 }
4072
4073 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4074 void *p)
4075 {
4076 int ret_val;
4077
4078 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4079 __igb_notify_dca);
4080
4081 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4082 }
4083 #endif /* CONFIG_IGB_DCA */
4084
4085 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4086 {
4087 struct e1000_hw *hw = &adapter->hw;
4088 u32 ping;
4089 int i;
4090
4091 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4092 ping = E1000_PF_CONTROL_MSG;
4093 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4094 ping |= E1000_VT_MSGTYPE_CTS;
4095 igb_write_mbx(hw, &ping, 1, i);
4096 }
4097 }
4098
4099 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4100 {
4101 struct e1000_hw *hw = &adapter->hw;
4102 u32 vmolr = rd32(E1000_VMOLR(vf));
4103 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4104
4105 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4106 IGB_VF_FLAG_MULTI_PROMISC);
4107 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4108
4109 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4110 vmolr |= E1000_VMOLR_MPME;
4111 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4112 } else {
4113 /*
4114 * if we have hashes and we are clearing a multicast promisc
4115 * flag we need to write the hashes to the MTA as this step
4116 * was previously skipped
4117 */
4118 if (vf_data->num_vf_mc_hashes > 30) {
4119 vmolr |= E1000_VMOLR_MPME;
4120 } else if (vf_data->num_vf_mc_hashes) {
4121 int j;
4122 vmolr |= E1000_VMOLR_ROMPE;
4123 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4124 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4125 }
4126 }
4127
4128 wr32(E1000_VMOLR(vf), vmolr);
4129
4130 /* there are flags left unprocessed, likely not supported */
4131 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4132 return -EINVAL;
4133
4134 return 0;
4135
4136 }
4137
4138 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4139 u32 *msgbuf, u32 vf)
4140 {
4141 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4142 u16 *hash_list = (u16 *)&msgbuf[1];
4143 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4144 int i;
4145
4146 /* salt away the number of multicast addresses assigned
4147 * to this VF for later use to restore when the PF multi cast
4148 * list changes
4149 */
4150 vf_data->num_vf_mc_hashes = n;
4151
4152 /* only up to 30 hash values supported */
4153 if (n > 30)
4154 n = 30;
4155
4156 /* store the hashes for later use */
4157 for (i = 0; i < n; i++)
4158 vf_data->vf_mc_hashes[i] = hash_list[i];
4159
4160 /* Flush and reset the mta with the new values */
4161 igb_set_rx_mode(adapter->netdev);
4162
4163 return 0;
4164 }
4165
4166 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4167 {
4168 struct e1000_hw *hw = &adapter->hw;
4169 struct vf_data_storage *vf_data;
4170 int i, j;
4171
4172 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4173 u32 vmolr = rd32(E1000_VMOLR(i));
4174 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4175
4176 vf_data = &adapter->vf_data[i];
4177
4178 if ((vf_data->num_vf_mc_hashes > 30) ||
4179 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4180 vmolr |= E1000_VMOLR_MPME;
4181 } else if (vf_data->num_vf_mc_hashes) {
4182 vmolr |= E1000_VMOLR_ROMPE;
4183 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4184 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4185 }
4186 wr32(E1000_VMOLR(i), vmolr);
4187 }
4188 }
4189
4190 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4191 {
4192 struct e1000_hw *hw = &adapter->hw;
4193 u32 pool_mask, reg, vid;
4194 int i;
4195
4196 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4197
4198 /* Find the vlan filter for this id */
4199 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4200 reg = rd32(E1000_VLVF(i));
4201
4202 /* remove the vf from the pool */
4203 reg &= ~pool_mask;
4204
4205 /* if pool is empty then remove entry from vfta */
4206 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4207 (reg & E1000_VLVF_VLANID_ENABLE)) {
4208 reg = 0;
4209 vid = reg & E1000_VLVF_VLANID_MASK;
4210 igb_vfta_set(hw, vid, false);
4211 }
4212
4213 wr32(E1000_VLVF(i), reg);
4214 }
4215
4216 adapter->vf_data[vf].vlans_enabled = 0;
4217 }
4218
4219 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4220 {
4221 struct e1000_hw *hw = &adapter->hw;
4222 u32 reg, i;
4223
4224 /* The vlvf table only exists on 82576 hardware and newer */
4225 if (hw->mac.type < e1000_82576)
4226 return -1;
4227
4228 /* we only need to do this if VMDq is enabled */
4229 if (!adapter->vfs_allocated_count)
4230 return -1;
4231
4232 /* Find the vlan filter for this id */
4233 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4234 reg = rd32(E1000_VLVF(i));
4235 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4236 vid == (reg & E1000_VLVF_VLANID_MASK))
4237 break;
4238 }
4239
4240 if (add) {
4241 if (i == E1000_VLVF_ARRAY_SIZE) {
4242 /* Did not find a matching VLAN ID entry that was
4243 * enabled. Search for a free filter entry, i.e.
4244 * one without the enable bit set
4245 */
4246 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4247 reg = rd32(E1000_VLVF(i));
4248 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4249 break;
4250 }
4251 }
4252 if (i < E1000_VLVF_ARRAY_SIZE) {
4253 /* Found an enabled/available entry */
4254 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4255
4256 /* if !enabled we need to set this up in vfta */
4257 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4258 /* add VID to filter table */
4259 igb_vfta_set(hw, vid, true);
4260 reg |= E1000_VLVF_VLANID_ENABLE;
4261 }
4262 reg &= ~E1000_VLVF_VLANID_MASK;
4263 reg |= vid;
4264 wr32(E1000_VLVF(i), reg);
4265
4266 /* do not modify RLPML for PF devices */
4267 if (vf >= adapter->vfs_allocated_count)
4268 return 0;
4269
4270 if (!adapter->vf_data[vf].vlans_enabled) {
4271 u32 size;
4272 reg = rd32(E1000_VMOLR(vf));
4273 size = reg & E1000_VMOLR_RLPML_MASK;
4274 size += 4;
4275 reg &= ~E1000_VMOLR_RLPML_MASK;
4276 reg |= size;
4277 wr32(E1000_VMOLR(vf), reg);
4278 }
4279
4280 adapter->vf_data[vf].vlans_enabled++;
4281 return 0;
4282 }
4283 } else {
4284 if (i < E1000_VLVF_ARRAY_SIZE) {
4285 /* remove vf from the pool */
4286 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4287 /* if pool is empty then remove entry from vfta */
4288 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4289 reg = 0;
4290 igb_vfta_set(hw, vid, false);
4291 }
4292 wr32(E1000_VLVF(i), reg);
4293
4294 /* do not modify RLPML for PF devices */
4295 if (vf >= adapter->vfs_allocated_count)
4296 return 0;
4297
4298 adapter->vf_data[vf].vlans_enabled--;
4299 if (!adapter->vf_data[vf].vlans_enabled) {
4300 u32 size;
4301 reg = rd32(E1000_VMOLR(vf));
4302 size = reg & E1000_VMOLR_RLPML_MASK;
4303 size -= 4;
4304 reg &= ~E1000_VMOLR_RLPML_MASK;
4305 reg |= size;
4306 wr32(E1000_VMOLR(vf), reg);
4307 }
4308 return 0;
4309 }
4310 }
4311 return -1;
4312 }
4313
4314 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4315 {
4316 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4317 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4318
4319 return igb_vlvf_set(adapter, vid, add, vf);
4320 }
4321
4322 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4323 {
4324 /* clear all flags */
4325 adapter->vf_data[vf].flags = 0;
4326 adapter->vf_data[vf].last_nack = jiffies;
4327
4328 /* reset offloads to defaults */
4329 igb_set_vmolr(adapter, vf);
4330
4331 /* reset vlans for device */
4332 igb_clear_vf_vfta(adapter, vf);
4333
4334 /* reset multicast table array for vf */
4335 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4336
4337 /* Flush and reset the mta with the new values */
4338 igb_set_rx_mode(adapter->netdev);
4339 }
4340
4341 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4342 {
4343 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4344
4345 /* generate a new mac address as we were hotplug removed/added */
4346 random_ether_addr(vf_mac);
4347
4348 /* process remaining reset events */
4349 igb_vf_reset(adapter, vf);
4350 }
4351
4352 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4353 {
4354 struct e1000_hw *hw = &adapter->hw;
4355 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4356 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4357 u32 reg, msgbuf[3];
4358 u8 *addr = (u8 *)(&msgbuf[1]);
4359
4360 /* process all the same items cleared in a function level reset */
4361 igb_vf_reset(adapter, vf);
4362
4363 /* set vf mac address */
4364 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4365
4366 /* enable transmit and receive for vf */
4367 reg = rd32(E1000_VFTE);
4368 wr32(E1000_VFTE, reg | (1 << vf));
4369 reg = rd32(E1000_VFRE);
4370 wr32(E1000_VFRE, reg | (1 << vf));
4371
4372 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4373
4374 /* reply to reset with ack and vf mac address */
4375 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4376 memcpy(addr, vf_mac, 6);
4377 igb_write_mbx(hw, msgbuf, 3, vf);
4378 }
4379
4380 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4381 {
4382 unsigned char *addr = (char *)&msg[1];
4383 int err = -1;
4384
4385 if (is_valid_ether_addr(addr))
4386 err = igb_set_vf_mac(adapter, vf, addr);
4387
4388 return err;
4389 }
4390
4391 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4392 {
4393 struct e1000_hw *hw = &adapter->hw;
4394 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4395 u32 msg = E1000_VT_MSGTYPE_NACK;
4396
4397 /* if device isn't clear to send it shouldn't be reading either */
4398 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4399 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4400 igb_write_mbx(hw, &msg, 1, vf);
4401 vf_data->last_nack = jiffies;
4402 }
4403 }
4404
4405 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4406 {
4407 struct pci_dev *pdev = adapter->pdev;
4408 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4409 struct e1000_hw *hw = &adapter->hw;
4410 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4411 s32 retval;
4412
4413 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4414
4415 if (retval)
4416 dev_err(&pdev->dev, "Error receiving message from VF\n");
4417
4418 /* this is a message we already processed, do nothing */
4419 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4420 return;
4421
4422 /*
4423 * until the vf completes a reset it should not be
4424 * allowed to start any configuration.
4425 */
4426
4427 if (msgbuf[0] == E1000_VF_RESET) {
4428 igb_vf_reset_msg(adapter, vf);
4429 return;
4430 }
4431
4432 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4433 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4434 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4435 igb_write_mbx(hw, msgbuf, 1, vf);
4436 vf_data->last_nack = jiffies;
4437 }
4438 return;
4439 }
4440
4441 switch ((msgbuf[0] & 0xFFFF)) {
4442 case E1000_VF_SET_MAC_ADDR:
4443 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4444 break;
4445 case E1000_VF_SET_PROMISC:
4446 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4447 break;
4448 case E1000_VF_SET_MULTICAST:
4449 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4450 break;
4451 case E1000_VF_SET_LPE:
4452 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4453 break;
4454 case E1000_VF_SET_VLAN:
4455 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4456 break;
4457 default:
4458 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4459 retval = -1;
4460 break;
4461 }
4462
4463 /* notify the VF of the results of what it sent us */
4464 if (retval)
4465 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4466 else
4467 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4468
4469 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4470
4471 igb_write_mbx(hw, msgbuf, 1, vf);
4472 }
4473
4474 static void igb_msg_task(struct igb_adapter *adapter)
4475 {
4476 struct e1000_hw *hw = &adapter->hw;
4477 u32 vf;
4478
4479 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4480 /* process any reset requests */
4481 if (!igb_check_for_rst(hw, vf))
4482 igb_vf_reset_event(adapter, vf);
4483
4484 /* process any messages pending */
4485 if (!igb_check_for_msg(hw, vf))
4486 igb_rcv_msg_from_vf(adapter, vf);
4487
4488 /* process any acks */
4489 if (!igb_check_for_ack(hw, vf))
4490 igb_rcv_ack_from_vf(adapter, vf);
4491 }
4492 }
4493
4494 /**
4495 * igb_set_uta - Set unicast filter table address
4496 * @adapter: board private structure
4497 *
4498 * The unicast table address is a register array of 32-bit registers.
4499 * The table is meant to be used in a way similar to how the MTA is used
4500 * however due to certain limitations in the hardware it is necessary to
4501 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4502 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4503 **/
4504 static void igb_set_uta(struct igb_adapter *adapter)
4505 {
4506 struct e1000_hw *hw = &adapter->hw;
4507 int i;
4508
4509 /* The UTA table only exists on 82576 hardware and newer */
4510 if (hw->mac.type < e1000_82576)
4511 return;
4512
4513 /* we only need to do this if VMDq is enabled */
4514 if (!adapter->vfs_allocated_count)
4515 return;
4516
4517 for (i = 0; i < hw->mac.uta_reg_count; i++)
4518 array_wr32(E1000_UTA, i, ~0);
4519 }
4520
4521 /**
4522 * igb_intr_msi - Interrupt Handler
4523 * @irq: interrupt number
4524 * @data: pointer to a network interface device structure
4525 **/
4526 static irqreturn_t igb_intr_msi(int irq, void *data)
4527 {
4528 struct igb_adapter *adapter = data;
4529 struct igb_q_vector *q_vector = adapter->q_vector[0];
4530 struct e1000_hw *hw = &adapter->hw;
4531 /* read ICR disables interrupts using IAM */
4532 u32 icr = rd32(E1000_ICR);
4533
4534 igb_write_itr(q_vector);
4535
4536 if (icr & E1000_ICR_DOUTSYNC) {
4537 /* HW is reporting DMA is out of sync */
4538 adapter->stats.doosync++;
4539 }
4540
4541 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4542 hw->mac.get_link_status = 1;
4543 if (!test_bit(__IGB_DOWN, &adapter->state))
4544 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4545 }
4546
4547 napi_schedule(&q_vector->napi);
4548
4549 return IRQ_HANDLED;
4550 }
4551
4552 /**
4553 * igb_intr - Legacy Interrupt Handler
4554 * @irq: interrupt number
4555 * @data: pointer to a network interface device structure
4556 **/
4557 static irqreturn_t igb_intr(int irq, void *data)
4558 {
4559 struct igb_adapter *adapter = data;
4560 struct igb_q_vector *q_vector = adapter->q_vector[0];
4561 struct e1000_hw *hw = &adapter->hw;
4562 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4563 * need for the IMC write */
4564 u32 icr = rd32(E1000_ICR);
4565 if (!icr)
4566 return IRQ_NONE; /* Not our interrupt */
4567
4568 igb_write_itr(q_vector);
4569
4570 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4571 * not set, then the adapter didn't send an interrupt */
4572 if (!(icr & E1000_ICR_INT_ASSERTED))
4573 return IRQ_NONE;
4574
4575 if (icr & E1000_ICR_DOUTSYNC) {
4576 /* HW is reporting DMA is out of sync */
4577 adapter->stats.doosync++;
4578 }
4579
4580 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4581 hw->mac.get_link_status = 1;
4582 /* guard against interrupt when we're going down */
4583 if (!test_bit(__IGB_DOWN, &adapter->state))
4584 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4585 }
4586
4587 napi_schedule(&q_vector->napi);
4588
4589 return IRQ_HANDLED;
4590 }
4591
4592 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
4593 {
4594 struct igb_adapter *adapter = q_vector->adapter;
4595 struct e1000_hw *hw = &adapter->hw;
4596
4597 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4598 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
4599 if (!adapter->msix_entries)
4600 igb_set_itr(adapter);
4601 else
4602 igb_update_ring_itr(q_vector);
4603 }
4604
4605 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4606 if (adapter->msix_entries)
4607 wr32(E1000_EIMS, q_vector->eims_value);
4608 else
4609 igb_irq_enable(adapter);
4610 }
4611 }
4612
4613 /**
4614 * igb_poll - NAPI Rx polling callback
4615 * @napi: napi polling structure
4616 * @budget: count of how many packets we should handle
4617 **/
4618 static int igb_poll(struct napi_struct *napi, int budget)
4619 {
4620 struct igb_q_vector *q_vector = container_of(napi,
4621 struct igb_q_vector,
4622 napi);
4623 int tx_clean_complete = 1, work_done = 0;
4624
4625 #ifdef CONFIG_IGB_DCA
4626 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4627 igb_update_dca(q_vector);
4628 #endif
4629 if (q_vector->tx_ring)
4630 tx_clean_complete = igb_clean_tx_irq(q_vector);
4631
4632 if (q_vector->rx_ring)
4633 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4634
4635 if (!tx_clean_complete)
4636 work_done = budget;
4637
4638 /* If not enough Rx work done, exit the polling mode */
4639 if (work_done < budget) {
4640 napi_complete(napi);
4641 igb_ring_irq_enable(q_vector);
4642 }
4643
4644 return work_done;
4645 }
4646
4647 /**
4648 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
4649 * @adapter: board private structure
4650 * @shhwtstamps: timestamp structure to update
4651 * @regval: unsigned 64bit system time value.
4652 *
4653 * We need to convert the system time value stored in the RX/TXSTMP registers
4654 * into a hwtstamp which can be used by the upper level timestamping functions
4655 */
4656 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4657 struct skb_shared_hwtstamps *shhwtstamps,
4658 u64 regval)
4659 {
4660 u64 ns;
4661
4662 ns = timecounter_cyc2time(&adapter->clock, regval);
4663 timecompare_update(&adapter->compare, ns);
4664 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4665 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4666 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4667 }
4668
4669 /**
4670 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4671 * @q_vector: pointer to q_vector containing needed info
4672 * @skb: packet that was just sent
4673 *
4674 * If we were asked to do hardware stamping and such a time stamp is
4675 * available, then it must have been for this skb here because we only
4676 * allow only one such packet into the queue.
4677 */
4678 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
4679 {
4680 struct igb_adapter *adapter = q_vector->adapter;
4681 union skb_shared_tx *shtx = skb_tx(skb);
4682 struct e1000_hw *hw = &adapter->hw;
4683 struct skb_shared_hwtstamps shhwtstamps;
4684 u64 regval;
4685
4686 /* if skb does not support hw timestamp or TX stamp not valid exit */
4687 if (likely(!shtx->hardware) ||
4688 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4689 return;
4690
4691 regval = rd32(E1000_TXSTMPL);
4692 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4693
4694 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4695 skb_tstamp_tx(skb, &shhwtstamps);
4696 }
4697
4698 /**
4699 * igb_clean_tx_irq - Reclaim resources after transmit completes
4700 * @q_vector: pointer to q_vector containing needed info
4701 * returns true if ring is completely cleaned
4702 **/
4703 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
4704 {
4705 struct igb_adapter *adapter = q_vector->adapter;
4706 struct igb_ring *tx_ring = q_vector->tx_ring;
4707 struct net_device *netdev = tx_ring->netdev;
4708 struct e1000_hw *hw = &adapter->hw;
4709 struct igb_buffer *buffer_info;
4710 struct sk_buff *skb;
4711 union e1000_adv_tx_desc *tx_desc, *eop_desc;
4712 unsigned int total_bytes = 0, total_packets = 0;
4713 unsigned int i, eop, count = 0;
4714 bool cleaned = false;
4715
4716 i = tx_ring->next_to_clean;
4717 eop = tx_ring->buffer_info[i].next_to_watch;
4718 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4719
4720 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4721 (count < tx_ring->count)) {
4722 for (cleaned = false; !cleaned; count++) {
4723 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4724 buffer_info = &tx_ring->buffer_info[i];
4725 cleaned = (i == eop);
4726 skb = buffer_info->skb;
4727
4728 if (skb) {
4729 unsigned int segs, bytecount;
4730 /* gso_segs is currently only valid for tcp */
4731 segs = skb_shinfo(skb)->gso_segs ?: 1;
4732 /* multiply data chunks by size of headers */
4733 bytecount = ((segs - 1) * skb_headlen(skb)) +
4734 skb->len;
4735 total_packets += segs;
4736 total_bytes += bytecount;
4737
4738 igb_tx_hwtstamp(q_vector, skb);
4739 }
4740
4741 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4742 tx_desc->wb.status = 0;
4743
4744 i++;
4745 if (i == tx_ring->count)
4746 i = 0;
4747 }
4748 eop = tx_ring->buffer_info[i].next_to_watch;
4749 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4750 }
4751
4752 tx_ring->next_to_clean = i;
4753
4754 if (unlikely(count &&
4755 netif_carrier_ok(netdev) &&
4756 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
4757 /* Make sure that anybody stopping the queue after this
4758 * sees the new next_to_clean.
4759 */
4760 smp_mb();
4761 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4762 !(test_bit(__IGB_DOWN, &adapter->state))) {
4763 netif_wake_subqueue(netdev, tx_ring->queue_index);
4764 tx_ring->tx_stats.restart_queue++;
4765 }
4766 }
4767
4768 if (tx_ring->detect_tx_hung) {
4769 /* Detect a transmit hang in hardware, this serializes the
4770 * check with the clearing of time_stamp and movement of i */
4771 tx_ring->detect_tx_hung = false;
4772 if (tx_ring->buffer_info[i].time_stamp &&
4773 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4774 (adapter->tx_timeout_factor * HZ))
4775 && !(rd32(E1000_STATUS) &
4776 E1000_STATUS_TXOFF)) {
4777
4778 /* detected Tx unit hang */
4779 dev_err(&tx_ring->pdev->dev,
4780 "Detected Tx Unit Hang\n"
4781 " Tx Queue <%d>\n"
4782 " TDH <%x>\n"
4783 " TDT <%x>\n"
4784 " next_to_use <%x>\n"
4785 " next_to_clean <%x>\n"
4786 "buffer_info[next_to_clean]\n"
4787 " time_stamp <%lx>\n"
4788 " next_to_watch <%x>\n"
4789 " jiffies <%lx>\n"
4790 " desc.status <%x>\n",
4791 tx_ring->queue_index,
4792 readl(tx_ring->head),
4793 readl(tx_ring->tail),
4794 tx_ring->next_to_use,
4795 tx_ring->next_to_clean,
4796 tx_ring->buffer_info[i].time_stamp,
4797 eop,
4798 jiffies,
4799 eop_desc->wb.status);
4800 netif_stop_subqueue(netdev, tx_ring->queue_index);
4801 }
4802 }
4803 tx_ring->total_bytes += total_bytes;
4804 tx_ring->total_packets += total_packets;
4805 tx_ring->tx_stats.bytes += total_bytes;
4806 tx_ring->tx_stats.packets += total_packets;
4807 netdev->stats.tx_bytes += total_bytes;
4808 netdev->stats.tx_packets += total_packets;
4809 return (count < tx_ring->count);
4810 }
4811
4812 /**
4813 * igb_receive_skb - helper function to handle rx indications
4814 * @q_vector: structure containing interrupt and ring information
4815 * @skb: packet to send up
4816 * @vlan_tag: vlan tag for packet
4817 **/
4818 static void igb_receive_skb(struct igb_q_vector *q_vector,
4819 struct sk_buff *skb,
4820 u16 vlan_tag)
4821 {
4822 struct igb_adapter *adapter = q_vector->adapter;
4823
4824 if (vlan_tag)
4825 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4826 vlan_tag, skb);
4827 else
4828 napi_gro_receive(&q_vector->napi, skb);
4829 }
4830
4831 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
4832 u32 status_err, struct sk_buff *skb)
4833 {
4834 skb->ip_summed = CHECKSUM_NONE;
4835
4836 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
4837 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4838 (status_err & E1000_RXD_STAT_IXSM))
4839 return;
4840
4841 /* TCP/UDP checksum error bit is set */
4842 if (status_err &
4843 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
4844 /*
4845 * work around errata with sctp packets where the TCPE aka
4846 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4847 * packets, (aka let the stack check the crc32c)
4848 */
4849 if ((skb->len == 60) &&
4850 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
4851 ring->rx_stats.csum_err++;
4852
4853 /* let the stack verify checksum errors */
4854 return;
4855 }
4856 /* It must be a TCP or UDP packet with a valid checksum */
4857 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4858 skb->ip_summed = CHECKSUM_UNNECESSARY;
4859
4860 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
4861 }
4862
4863 static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4864 struct sk_buff *skb)
4865 {
4866 struct igb_adapter *adapter = q_vector->adapter;
4867 struct e1000_hw *hw = &adapter->hw;
4868 u64 regval;
4869
4870 /*
4871 * If this bit is set, then the RX registers contain the time stamp. No
4872 * other packet will be time stamped until we read these registers, so
4873 * read the registers to make them available again. Because only one
4874 * packet can be time stamped at a time, we know that the register
4875 * values must belong to this one here and therefore we don't need to
4876 * compare any of the additional attributes stored for it.
4877 *
4878 * If nothing went wrong, then it should have a skb_shared_tx that we
4879 * can turn into a skb_shared_hwtstamps.
4880 */
4881 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4882 return;
4883 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4884 return;
4885
4886 regval = rd32(E1000_RXSTMPL);
4887 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4888
4889 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4890 }
4891 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
4892 union e1000_adv_rx_desc *rx_desc)
4893 {
4894 /* HW will not DMA in data larger than the given buffer, even if it
4895 * parses the (NFS, of course) header to be larger. In that case, it
4896 * fills the header buffer and spills the rest into the page.
4897 */
4898 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4899 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4900 if (hlen > rx_ring->rx_buffer_len)
4901 hlen = rx_ring->rx_buffer_len;
4902 return hlen;
4903 }
4904
4905 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4906 int *work_done, int budget)
4907 {
4908 struct igb_ring *rx_ring = q_vector->rx_ring;
4909 struct net_device *netdev = rx_ring->netdev;
4910 struct pci_dev *pdev = rx_ring->pdev;
4911 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4912 struct igb_buffer *buffer_info , *next_buffer;
4913 struct sk_buff *skb;
4914 bool cleaned = false;
4915 int cleaned_count = 0;
4916 unsigned int total_bytes = 0, total_packets = 0;
4917 unsigned int i;
4918 u32 staterr;
4919 u16 length;
4920 u16 vlan_tag;
4921
4922 i = rx_ring->next_to_clean;
4923 buffer_info = &rx_ring->buffer_info[i];
4924 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4925 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4926
4927 while (staterr & E1000_RXD_STAT_DD) {
4928 if (*work_done >= budget)
4929 break;
4930 (*work_done)++;
4931
4932 skb = buffer_info->skb;
4933 prefetch(skb->data - NET_IP_ALIGN);
4934 buffer_info->skb = NULL;
4935
4936 i++;
4937 if (i == rx_ring->count)
4938 i = 0;
4939 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4940 prefetch(next_rxd);
4941 next_buffer = &rx_ring->buffer_info[i];
4942
4943 length = le16_to_cpu(rx_desc->wb.upper.length);
4944 cleaned = true;
4945 cleaned_count++;
4946
4947 if (buffer_info->dma) {
4948 pci_unmap_single(pdev, buffer_info->dma,
4949 rx_ring->rx_buffer_len,
4950 PCI_DMA_FROMDEVICE);
4951 buffer_info->dma = 0;
4952 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
4953 skb_put(skb, length);
4954 goto send_up;
4955 }
4956 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
4957 }
4958
4959 if (length) {
4960 pci_unmap_page(pdev, buffer_info->page_dma,
4961 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
4962 buffer_info->page_dma = 0;
4963
4964 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4965 buffer_info->page,
4966 buffer_info->page_offset,
4967 length);
4968
4969 if (page_count(buffer_info->page) != 1)
4970 buffer_info->page = NULL;
4971 else
4972 get_page(buffer_info->page);
4973
4974 skb->len += length;
4975 skb->data_len += length;
4976
4977 skb->truesize += length;
4978 }
4979
4980 if (!(staterr & E1000_RXD_STAT_EOP)) {
4981 buffer_info->skb = next_buffer->skb;
4982 buffer_info->dma = next_buffer->dma;
4983 next_buffer->skb = skb;
4984 next_buffer->dma = 0;
4985 goto next_desc;
4986 }
4987 send_up:
4988 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
4989 dev_kfree_skb_irq(skb);
4990 goto next_desc;
4991 }
4992
4993 igb_rx_hwtstamp(q_vector, staterr, skb);
4994 total_bytes += skb->len;
4995 total_packets++;
4996
4997 igb_rx_checksum_adv(rx_ring, staterr, skb);
4998
4999 skb->protocol = eth_type_trans(skb, netdev);
5000 skb_record_rx_queue(skb, rx_ring->queue_index);
5001
5002 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5003 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5004
5005 igb_receive_skb(q_vector, skb, vlan_tag);
5006
5007 next_desc:
5008 rx_desc->wb.upper.status_error = 0;
5009
5010 /* return some buffers to hardware, one at a time is too slow */
5011 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5012 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5013 cleaned_count = 0;
5014 }
5015
5016 /* use prefetched values */
5017 rx_desc = next_rxd;
5018 buffer_info = next_buffer;
5019 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5020 }
5021
5022 rx_ring->next_to_clean = i;
5023 cleaned_count = igb_desc_unused(rx_ring);
5024
5025 if (cleaned_count)
5026 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5027
5028 rx_ring->total_packets += total_packets;
5029 rx_ring->total_bytes += total_bytes;
5030 rx_ring->rx_stats.packets += total_packets;
5031 rx_ring->rx_stats.bytes += total_bytes;
5032 netdev->stats.rx_bytes += total_bytes;
5033 netdev->stats.rx_packets += total_packets;
5034 return cleaned;
5035 }
5036
5037 /**
5038 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5039 * @adapter: address of board private structure
5040 **/
5041 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5042 {
5043 struct net_device *netdev = rx_ring->netdev;
5044 union e1000_adv_rx_desc *rx_desc;
5045 struct igb_buffer *buffer_info;
5046 struct sk_buff *skb;
5047 unsigned int i;
5048 int bufsz;
5049
5050 i = rx_ring->next_to_use;
5051 buffer_info = &rx_ring->buffer_info[i];
5052
5053 bufsz = rx_ring->rx_buffer_len;
5054
5055 while (cleaned_count--) {
5056 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5057
5058 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5059 if (!buffer_info->page) {
5060 buffer_info->page = alloc_page(GFP_ATOMIC);
5061 if (!buffer_info->page) {
5062 rx_ring->rx_stats.alloc_failed++;
5063 goto no_buffers;
5064 }
5065 buffer_info->page_offset = 0;
5066 } else {
5067 buffer_info->page_offset ^= PAGE_SIZE / 2;
5068 }
5069 buffer_info->page_dma =
5070 pci_map_page(rx_ring->pdev, buffer_info->page,
5071 buffer_info->page_offset,
5072 PAGE_SIZE / 2,
5073 PCI_DMA_FROMDEVICE);
5074 }
5075
5076 if (!buffer_info->skb) {
5077 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5078 if (!skb) {
5079 rx_ring->rx_stats.alloc_failed++;
5080 goto no_buffers;
5081 }
5082
5083 buffer_info->skb = skb;
5084 buffer_info->dma = pci_map_single(rx_ring->pdev,
5085 skb->data,
5086 bufsz,
5087 PCI_DMA_FROMDEVICE);
5088 }
5089 /* Refresh the desc even if buffer_addrs didn't change because
5090 * each write-back erases this info. */
5091 if (bufsz < IGB_RXBUFFER_1024) {
5092 rx_desc->read.pkt_addr =
5093 cpu_to_le64(buffer_info->page_dma);
5094 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5095 } else {
5096 rx_desc->read.pkt_addr =
5097 cpu_to_le64(buffer_info->dma);
5098 rx_desc->read.hdr_addr = 0;
5099 }
5100
5101 i++;
5102 if (i == rx_ring->count)
5103 i = 0;
5104 buffer_info = &rx_ring->buffer_info[i];
5105 }
5106
5107 no_buffers:
5108 if (rx_ring->next_to_use != i) {
5109 rx_ring->next_to_use = i;
5110 if (i == 0)
5111 i = (rx_ring->count - 1);
5112 else
5113 i--;
5114
5115 /* Force memory writes to complete before letting h/w
5116 * know there are new descriptors to fetch. (Only
5117 * applicable for weak-ordered memory model archs,
5118 * such as IA-64). */
5119 wmb();
5120 writel(i, rx_ring->tail);
5121 }
5122 }
5123
5124 /**
5125 * igb_mii_ioctl -
5126 * @netdev:
5127 * @ifreq:
5128 * @cmd:
5129 **/
5130 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5131 {
5132 struct igb_adapter *adapter = netdev_priv(netdev);
5133 struct mii_ioctl_data *data = if_mii(ifr);
5134
5135 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5136 return -EOPNOTSUPP;
5137
5138 switch (cmd) {
5139 case SIOCGMIIPHY:
5140 data->phy_id = adapter->hw.phy.addr;
5141 break;
5142 case SIOCGMIIREG:
5143 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5144 &data->val_out))
5145 return -EIO;
5146 break;
5147 case SIOCSMIIREG:
5148 default:
5149 return -EOPNOTSUPP;
5150 }
5151 return 0;
5152 }
5153
5154 /**
5155 * igb_hwtstamp_ioctl - control hardware time stamping
5156 * @netdev:
5157 * @ifreq:
5158 * @cmd:
5159 *
5160 * Outgoing time stamping can be enabled and disabled. Play nice and
5161 * disable it when requested, although it shouldn't case any overhead
5162 * when no packet needs it. At most one packet in the queue may be
5163 * marked for time stamping, otherwise it would be impossible to tell
5164 * for sure to which packet the hardware time stamp belongs.
5165 *
5166 * Incoming time stamping has to be configured via the hardware
5167 * filters. Not all combinations are supported, in particular event
5168 * type has to be specified. Matching the kind of event packet is
5169 * not supported, with the exception of "all V2 events regardless of
5170 * level 2 or 4".
5171 *
5172 **/
5173 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5174 struct ifreq *ifr, int cmd)
5175 {
5176 struct igb_adapter *adapter = netdev_priv(netdev);
5177 struct e1000_hw *hw = &adapter->hw;
5178 struct hwtstamp_config config;
5179 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5180 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5181 u32 tsync_rx_cfg = 0;
5182 bool is_l4 = false;
5183 bool is_l2 = false;
5184 u32 regval;
5185
5186 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5187 return -EFAULT;
5188
5189 /* reserved for future extensions */
5190 if (config.flags)
5191 return -EINVAL;
5192
5193 switch (config.tx_type) {
5194 case HWTSTAMP_TX_OFF:
5195 tsync_tx_ctl = 0;
5196 case HWTSTAMP_TX_ON:
5197 break;
5198 default:
5199 return -ERANGE;
5200 }
5201
5202 switch (config.rx_filter) {
5203 case HWTSTAMP_FILTER_NONE:
5204 tsync_rx_ctl = 0;
5205 break;
5206 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5207 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5208 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5209 case HWTSTAMP_FILTER_ALL:
5210 /*
5211 * register TSYNCRXCFG must be set, therefore it is not
5212 * possible to time stamp both Sync and Delay_Req messages
5213 * => fall back to time stamping all packets
5214 */
5215 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5216 config.rx_filter = HWTSTAMP_FILTER_ALL;
5217 break;
5218 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5219 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5220 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5221 is_l4 = true;
5222 break;
5223 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5224 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5225 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5226 is_l4 = true;
5227 break;
5228 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5229 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5230 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5231 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5232 is_l2 = true;
5233 is_l4 = true;
5234 config.rx_filter = HWTSTAMP_FILTER_SOME;
5235 break;
5236 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5237 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5238 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5239 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5240 is_l2 = true;
5241 is_l4 = true;
5242 config.rx_filter = HWTSTAMP_FILTER_SOME;
5243 break;
5244 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5245 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5246 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5247 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5248 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5249 is_l2 = true;
5250 break;
5251 default:
5252 return -ERANGE;
5253 }
5254
5255 if (hw->mac.type == e1000_82575) {
5256 if (tsync_rx_ctl | tsync_tx_ctl)
5257 return -EINVAL;
5258 return 0;
5259 }
5260
5261 /* enable/disable TX */
5262 regval = rd32(E1000_TSYNCTXCTL);
5263 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5264 regval |= tsync_tx_ctl;
5265 wr32(E1000_TSYNCTXCTL, regval);
5266
5267 /* enable/disable RX */
5268 regval = rd32(E1000_TSYNCRXCTL);
5269 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5270 regval |= tsync_rx_ctl;
5271 wr32(E1000_TSYNCRXCTL, regval);
5272
5273 /* define which PTP packets are time stamped */
5274 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5275
5276 /* define ethertype filter for timestamped packets */
5277 if (is_l2)
5278 wr32(E1000_ETQF(3),
5279 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5280 E1000_ETQF_1588 | /* enable timestamping */
5281 ETH_P_1588)); /* 1588 eth protocol type */
5282 else
5283 wr32(E1000_ETQF(3), 0);
5284
5285 #define PTP_PORT 319
5286 /* L4 Queue Filter[3]: filter by destination port and protocol */
5287 if (is_l4) {
5288 u32 ftqf = (IPPROTO_UDP /* UDP */
5289 | E1000_FTQF_VF_BP /* VF not compared */
5290 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5291 | E1000_FTQF_MASK); /* mask all inputs */
5292 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5293
5294 wr32(E1000_IMIR(3), htons(PTP_PORT));
5295 wr32(E1000_IMIREXT(3),
5296 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5297 if (hw->mac.type == e1000_82576) {
5298 /* enable source port check */
5299 wr32(E1000_SPQF(3), htons(PTP_PORT));
5300 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5301 }
5302 wr32(E1000_FTQF(3), ftqf);
5303 } else {
5304 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5305 }
5306 wrfl();
5307
5308 adapter->hwtstamp_config = config;
5309
5310 /* clear TX/RX time stamp registers, just to be sure */
5311 regval = rd32(E1000_TXSTMPH);
5312 regval = rd32(E1000_RXSTMPH);
5313
5314 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5315 -EFAULT : 0;
5316 }
5317
5318 /**
5319 * igb_ioctl -
5320 * @netdev:
5321 * @ifreq:
5322 * @cmd:
5323 **/
5324 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5325 {
5326 switch (cmd) {
5327 case SIOCGMIIPHY:
5328 case SIOCGMIIREG:
5329 case SIOCSMIIREG:
5330 return igb_mii_ioctl(netdev, ifr, cmd);
5331 case SIOCSHWTSTAMP:
5332 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5333 default:
5334 return -EOPNOTSUPP;
5335 }
5336 }
5337
5338 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5339 {
5340 struct igb_adapter *adapter = hw->back;
5341 u16 cap_offset;
5342
5343 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5344 if (!cap_offset)
5345 return -E1000_ERR_CONFIG;
5346
5347 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5348
5349 return 0;
5350 }
5351
5352 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5353 {
5354 struct igb_adapter *adapter = hw->back;
5355 u16 cap_offset;
5356
5357 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5358 if (!cap_offset)
5359 return -E1000_ERR_CONFIG;
5360
5361 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5362
5363 return 0;
5364 }
5365
5366 static void igb_vlan_rx_register(struct net_device *netdev,
5367 struct vlan_group *grp)
5368 {
5369 struct igb_adapter *adapter = netdev_priv(netdev);
5370 struct e1000_hw *hw = &adapter->hw;
5371 u32 ctrl, rctl;
5372
5373 igb_irq_disable(adapter);
5374 adapter->vlgrp = grp;
5375
5376 if (grp) {
5377 /* enable VLAN tag insert/strip */
5378 ctrl = rd32(E1000_CTRL);
5379 ctrl |= E1000_CTRL_VME;
5380 wr32(E1000_CTRL, ctrl);
5381
5382 /* Disable CFI check */
5383 rctl = rd32(E1000_RCTL);
5384 rctl &= ~E1000_RCTL_CFIEN;
5385 wr32(E1000_RCTL, rctl);
5386 } else {
5387 /* disable VLAN tag insert/strip */
5388 ctrl = rd32(E1000_CTRL);
5389 ctrl &= ~E1000_CTRL_VME;
5390 wr32(E1000_CTRL, ctrl);
5391 }
5392
5393 igb_rlpml_set(adapter);
5394
5395 if (!test_bit(__IGB_DOWN, &adapter->state))
5396 igb_irq_enable(adapter);
5397 }
5398
5399 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5400 {
5401 struct igb_adapter *adapter = netdev_priv(netdev);
5402 struct e1000_hw *hw = &adapter->hw;
5403 int pf_id = adapter->vfs_allocated_count;
5404
5405 /* attempt to add filter to vlvf array */
5406 igb_vlvf_set(adapter, vid, true, pf_id);
5407
5408 /* add the filter since PF can receive vlans w/o entry in vlvf */
5409 igb_vfta_set(hw, vid, true);
5410 }
5411
5412 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5413 {
5414 struct igb_adapter *adapter = netdev_priv(netdev);
5415 struct e1000_hw *hw = &adapter->hw;
5416 int pf_id = adapter->vfs_allocated_count;
5417 s32 err;
5418
5419 igb_irq_disable(adapter);
5420 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5421
5422 if (!test_bit(__IGB_DOWN, &adapter->state))
5423 igb_irq_enable(adapter);
5424
5425 /* remove vlan from VLVF table array */
5426 err = igb_vlvf_set(adapter, vid, false, pf_id);
5427
5428 /* if vid was not present in VLVF just remove it from table */
5429 if (err)
5430 igb_vfta_set(hw, vid, false);
5431 }
5432
5433 static void igb_restore_vlan(struct igb_adapter *adapter)
5434 {
5435 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5436
5437 if (adapter->vlgrp) {
5438 u16 vid;
5439 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5440 if (!vlan_group_get_device(adapter->vlgrp, vid))
5441 continue;
5442 igb_vlan_rx_add_vid(adapter->netdev, vid);
5443 }
5444 }
5445 }
5446
5447 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5448 {
5449 struct e1000_mac_info *mac = &adapter->hw.mac;
5450
5451 mac->autoneg = 0;
5452
5453 switch (spddplx) {
5454 case SPEED_10 + DUPLEX_HALF:
5455 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5456 break;
5457 case SPEED_10 + DUPLEX_FULL:
5458 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5459 break;
5460 case SPEED_100 + DUPLEX_HALF:
5461 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5462 break;
5463 case SPEED_100 + DUPLEX_FULL:
5464 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5465 break;
5466 case SPEED_1000 + DUPLEX_FULL:
5467 mac->autoneg = 1;
5468 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5469 break;
5470 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5471 default:
5472 dev_err(&adapter->pdev->dev,
5473 "Unsupported Speed/Duplex configuration\n");
5474 return -EINVAL;
5475 }
5476 return 0;
5477 }
5478
5479 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
5480 {
5481 struct net_device *netdev = pci_get_drvdata(pdev);
5482 struct igb_adapter *adapter = netdev_priv(netdev);
5483 struct e1000_hw *hw = &adapter->hw;
5484 u32 ctrl, rctl, status;
5485 u32 wufc = adapter->wol;
5486 #ifdef CONFIG_PM
5487 int retval = 0;
5488 #endif
5489
5490 netif_device_detach(netdev);
5491
5492 if (netif_running(netdev))
5493 igb_close(netdev);
5494
5495 igb_clear_interrupt_scheme(adapter);
5496
5497 #ifdef CONFIG_PM
5498 retval = pci_save_state(pdev);
5499 if (retval)
5500 return retval;
5501 #endif
5502
5503 status = rd32(E1000_STATUS);
5504 if (status & E1000_STATUS_LU)
5505 wufc &= ~E1000_WUFC_LNKC;
5506
5507 if (wufc) {
5508 igb_setup_rctl(adapter);
5509 igb_set_rx_mode(netdev);
5510
5511 /* turn on all-multi mode if wake on multicast is enabled */
5512 if (wufc & E1000_WUFC_MC) {
5513 rctl = rd32(E1000_RCTL);
5514 rctl |= E1000_RCTL_MPE;
5515 wr32(E1000_RCTL, rctl);
5516 }
5517
5518 ctrl = rd32(E1000_CTRL);
5519 /* advertise wake from D3Cold */
5520 #define E1000_CTRL_ADVD3WUC 0x00100000
5521 /* phy power management enable */
5522 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5523 ctrl |= E1000_CTRL_ADVD3WUC;
5524 wr32(E1000_CTRL, ctrl);
5525
5526 /* Allow time for pending master requests to run */
5527 igb_disable_pcie_master(&adapter->hw);
5528
5529 wr32(E1000_WUC, E1000_WUC_PME_EN);
5530 wr32(E1000_WUFC, wufc);
5531 } else {
5532 wr32(E1000_WUC, 0);
5533 wr32(E1000_WUFC, 0);
5534 }
5535
5536 *enable_wake = wufc || adapter->en_mng_pt;
5537 if (!*enable_wake)
5538 igb_shutdown_serdes_link_82575(hw);
5539
5540 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5541 * would have already happened in close and is redundant. */
5542 igb_release_hw_control(adapter);
5543
5544 pci_disable_device(pdev);
5545
5546 return 0;
5547 }
5548
5549 #ifdef CONFIG_PM
5550 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5551 {
5552 int retval;
5553 bool wake;
5554
5555 retval = __igb_shutdown(pdev, &wake);
5556 if (retval)
5557 return retval;
5558
5559 if (wake) {
5560 pci_prepare_to_sleep(pdev);
5561 } else {
5562 pci_wake_from_d3(pdev, false);
5563 pci_set_power_state(pdev, PCI_D3hot);
5564 }
5565
5566 return 0;
5567 }
5568
5569 static int igb_resume(struct pci_dev *pdev)
5570 {
5571 struct net_device *netdev = pci_get_drvdata(pdev);
5572 struct igb_adapter *adapter = netdev_priv(netdev);
5573 struct e1000_hw *hw = &adapter->hw;
5574 u32 err;
5575
5576 pci_set_power_state(pdev, PCI_D0);
5577 pci_restore_state(pdev);
5578
5579 err = pci_enable_device_mem(pdev);
5580 if (err) {
5581 dev_err(&pdev->dev,
5582 "igb: Cannot enable PCI device from suspend\n");
5583 return err;
5584 }
5585 pci_set_master(pdev);
5586
5587 pci_enable_wake(pdev, PCI_D3hot, 0);
5588 pci_enable_wake(pdev, PCI_D3cold, 0);
5589
5590 if (igb_init_interrupt_scheme(adapter)) {
5591 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5592 return -ENOMEM;
5593 }
5594
5595 /* e1000_power_up_phy(adapter); */
5596
5597 igb_reset(adapter);
5598
5599 /* let the f/w know that the h/w is now under the control of the
5600 * driver. */
5601 igb_get_hw_control(adapter);
5602
5603 wr32(E1000_WUS, ~0);
5604
5605 if (netif_running(netdev)) {
5606 err = igb_open(netdev);
5607 if (err)
5608 return err;
5609 }
5610
5611 netif_device_attach(netdev);
5612
5613 return 0;
5614 }
5615 #endif
5616
5617 static void igb_shutdown(struct pci_dev *pdev)
5618 {
5619 bool wake;
5620
5621 __igb_shutdown(pdev, &wake);
5622
5623 if (system_state == SYSTEM_POWER_OFF) {
5624 pci_wake_from_d3(pdev, wake);
5625 pci_set_power_state(pdev, PCI_D3hot);
5626 }
5627 }
5628
5629 #ifdef CONFIG_NET_POLL_CONTROLLER
5630 /*
5631 * Polling 'interrupt' - used by things like netconsole to send skbs
5632 * without having to re-enable interrupts. It's not called while
5633 * the interrupt routine is executing.
5634 */
5635 static void igb_netpoll(struct net_device *netdev)
5636 {
5637 struct igb_adapter *adapter = netdev_priv(netdev);
5638 struct e1000_hw *hw = &adapter->hw;
5639 int i;
5640
5641 if (!adapter->msix_entries) {
5642 struct igb_q_vector *q_vector = adapter->q_vector[0];
5643 igb_irq_disable(adapter);
5644 napi_schedule(&q_vector->napi);
5645 return;
5646 }
5647
5648 for (i = 0; i < adapter->num_q_vectors; i++) {
5649 struct igb_q_vector *q_vector = adapter->q_vector[i];
5650 wr32(E1000_EIMC, q_vector->eims_value);
5651 napi_schedule(&q_vector->napi);
5652 }
5653 }
5654 #endif /* CONFIG_NET_POLL_CONTROLLER */
5655
5656 /**
5657 * igb_io_error_detected - called when PCI error is detected
5658 * @pdev: Pointer to PCI device
5659 * @state: The current pci connection state
5660 *
5661 * This function is called after a PCI bus error affecting
5662 * this device has been detected.
5663 */
5664 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5665 pci_channel_state_t state)
5666 {
5667 struct net_device *netdev = pci_get_drvdata(pdev);
5668 struct igb_adapter *adapter = netdev_priv(netdev);
5669
5670 netif_device_detach(netdev);
5671
5672 if (state == pci_channel_io_perm_failure)
5673 return PCI_ERS_RESULT_DISCONNECT;
5674
5675 if (netif_running(netdev))
5676 igb_down(adapter);
5677 pci_disable_device(pdev);
5678
5679 /* Request a slot slot reset. */
5680 return PCI_ERS_RESULT_NEED_RESET;
5681 }
5682
5683 /**
5684 * igb_io_slot_reset - called after the pci bus has been reset.
5685 * @pdev: Pointer to PCI device
5686 *
5687 * Restart the card from scratch, as if from a cold-boot. Implementation
5688 * resembles the first-half of the igb_resume routine.
5689 */
5690 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5691 {
5692 struct net_device *netdev = pci_get_drvdata(pdev);
5693 struct igb_adapter *adapter = netdev_priv(netdev);
5694 struct e1000_hw *hw = &adapter->hw;
5695 pci_ers_result_t result;
5696 int err;
5697
5698 if (pci_enable_device_mem(pdev)) {
5699 dev_err(&pdev->dev,
5700 "Cannot re-enable PCI device after reset.\n");
5701 result = PCI_ERS_RESULT_DISCONNECT;
5702 } else {
5703 pci_set_master(pdev);
5704 pci_restore_state(pdev);
5705
5706 pci_enable_wake(pdev, PCI_D3hot, 0);
5707 pci_enable_wake(pdev, PCI_D3cold, 0);
5708
5709 igb_reset(adapter);
5710 wr32(E1000_WUS, ~0);
5711 result = PCI_ERS_RESULT_RECOVERED;
5712 }
5713
5714 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5715 if (err) {
5716 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5717 "failed 0x%0x\n", err);
5718 /* non-fatal, continue */
5719 }
5720
5721 return result;
5722 }
5723
5724 /**
5725 * igb_io_resume - called when traffic can start flowing again.
5726 * @pdev: Pointer to PCI device
5727 *
5728 * This callback is called when the error recovery driver tells us that
5729 * its OK to resume normal operation. Implementation resembles the
5730 * second-half of the igb_resume routine.
5731 */
5732 static void igb_io_resume(struct pci_dev *pdev)
5733 {
5734 struct net_device *netdev = pci_get_drvdata(pdev);
5735 struct igb_adapter *adapter = netdev_priv(netdev);
5736
5737 if (netif_running(netdev)) {
5738 if (igb_up(adapter)) {
5739 dev_err(&pdev->dev, "igb_up failed after reset\n");
5740 return;
5741 }
5742 }
5743
5744 netif_device_attach(netdev);
5745
5746 /* let the f/w know that the h/w is now under the control of the
5747 * driver. */
5748 igb_get_hw_control(adapter);
5749 }
5750
5751 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5752 u8 qsel)
5753 {
5754 u32 rar_low, rar_high;
5755 struct e1000_hw *hw = &adapter->hw;
5756
5757 /* HW expects these in little endian so we reverse the byte order
5758 * from network order (big endian) to little endian
5759 */
5760 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5761 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5762 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5763
5764 /* Indicate to hardware the Address is Valid. */
5765 rar_high |= E1000_RAH_AV;
5766
5767 if (hw->mac.type == e1000_82575)
5768 rar_high |= E1000_RAH_POOL_1 * qsel;
5769 else
5770 rar_high |= E1000_RAH_POOL_1 << qsel;
5771
5772 wr32(E1000_RAL(index), rar_low);
5773 wrfl();
5774 wr32(E1000_RAH(index), rar_high);
5775 wrfl();
5776 }
5777
5778 static int igb_set_vf_mac(struct igb_adapter *adapter,
5779 int vf, unsigned char *mac_addr)
5780 {
5781 struct e1000_hw *hw = &adapter->hw;
5782 /* VF MAC addresses start at end of receive addresses and moves
5783 * torwards the first, as a result a collision should not be possible */
5784 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5785
5786 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
5787
5788 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
5789
5790 return 0;
5791 }
5792
5793 static void igb_vmm_control(struct igb_adapter *adapter)
5794 {
5795 struct e1000_hw *hw = &adapter->hw;
5796 u32 reg;
5797
5798 /* replication is not supported for 82575 */
5799 if (hw->mac.type == e1000_82575)
5800 return;
5801
5802 /* enable replication vlan tag stripping */
5803 reg = rd32(E1000_RPLOLR);
5804 reg |= E1000_RPLOLR_STRVLAN;
5805 wr32(E1000_RPLOLR, reg);
5806
5807 /* notify HW that the MAC is adding vlan tags */
5808 reg = rd32(E1000_DTXCTL);
5809 reg |= E1000_DTXCTL_VLAN_ADDED;
5810 wr32(E1000_DTXCTL, reg);
5811
5812 if (adapter->vfs_allocated_count) {
5813 igb_vmdq_set_loopback_pf(hw, true);
5814 igb_vmdq_set_replication_pf(hw, true);
5815 } else {
5816 igb_vmdq_set_loopback_pf(hw, false);
5817 igb_vmdq_set_replication_pf(hw, false);
5818 }
5819 }
5820
5821 /* igb_main.c */
Linus' kernel tree