?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
stmmac: update the driver documentation
[linux-2.6/x86.git] / drivers / net / igb / igb_main.c
blob892d196f17accfffc4996f830e02ea74d777c415
1 /*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <linux/slab.h>
36 #include <net/checksum.h>
37 #include <net/ip6_checksum.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
41 #include <linux/if_vlan.h>
42 #include <linux/pci.h>
43 #include <linux/pci-aspm.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/if_ether.h>
47 #include <linux/aer.h>
48 #ifdef CONFIG_IGB_DCA
49 #include <linux/dca.h>
50 #endif
51 #include "igb.h"
52
53 #define DRV_VERSION "2.1.0-k2"
54 char igb_driver_name[] = "igb";
55 char igb_driver_version[] = DRV_VERSION;
56 static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
59
60 static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62 };
63
64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
87 /* required last entry */
88 {0, }
89 };
90
91 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
92
93 void igb_reset(struct igb_adapter *);
94 static int igb_setup_all_tx_resources(struct igb_adapter *);
95 static int igb_setup_all_rx_resources(struct igb_adapter *);
96 static void igb_free_all_tx_resources(struct igb_adapter *);
97 static void igb_free_all_rx_resources(struct igb_adapter *);
98 static void igb_setup_mrqc(struct igb_adapter *);
99 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
100 static void __devexit igb_remove(struct pci_dev *pdev);
101 static int igb_sw_init(struct igb_adapter *);
102 static int igb_open(struct net_device *);
103 static int igb_close(struct net_device *);
104 static void igb_configure_tx(struct igb_adapter *);
105 static void igb_configure_rx(struct igb_adapter *);
106 static void igb_clean_all_tx_rings(struct igb_adapter *);
107 static void igb_clean_all_rx_rings(struct igb_adapter *);
108 static void igb_clean_tx_ring(struct igb_ring *);
109 static void igb_clean_rx_ring(struct igb_ring *);
110 static void igb_set_rx_mode(struct net_device *);
111 static void igb_update_phy_info(unsigned long);
112 static void igb_watchdog(unsigned long);
113 static void igb_watchdog_task(struct work_struct *);
114 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
115 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
116 struct rtnl_link_stats64 *stats);
117 static int igb_change_mtu(struct net_device *, int);
118 static int igb_set_mac(struct net_device *, void *);
119 static void igb_set_uta(struct igb_adapter *adapter);
120 static irqreturn_t igb_intr(int irq, void *);
121 static irqreturn_t igb_intr_msi(int irq, void *);
122 static irqreturn_t igb_msix_other(int irq, void *);
123 static irqreturn_t igb_msix_ring(int irq, void *);
124 #ifdef CONFIG_IGB_DCA
125 static void igb_update_dca(struct igb_q_vector *);
126 static void igb_setup_dca(struct igb_adapter *);
127 #endif /* CONFIG_IGB_DCA */
128 static bool igb_clean_tx_irq(struct igb_q_vector *);
129 static int igb_poll(struct napi_struct *, int);
130 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
131 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
132 static void igb_tx_timeout(struct net_device *);
133 static void igb_reset_task(struct work_struct *);
134 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
135 static void igb_vlan_rx_add_vid(struct net_device *, u16);
136 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
137 static void igb_restore_vlan(struct igb_adapter *);
138 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
139 static void igb_ping_all_vfs(struct igb_adapter *);
140 static void igb_msg_task(struct igb_adapter *);
141 static void igb_vmm_control(struct igb_adapter *);
142 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
143 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
144 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
145 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
146 int vf, u16 vlan, u8 qos);
147 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
148 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
149 struct ifla_vf_info *ivi);
150
151 #ifdef CONFIG_PM
152 static int igb_suspend(struct pci_dev *, pm_message_t);
153 static int igb_resume(struct pci_dev *);
154 #endif
155 static void igb_shutdown(struct pci_dev *);
156 #ifdef CONFIG_IGB_DCA
157 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
158 static struct notifier_block dca_notifier = {
159 .notifier_call = igb_notify_dca,
160 .next = NULL,
161 .priority = 0
162 };
163 #endif
164 #ifdef CONFIG_NET_POLL_CONTROLLER
165 /* for netdump / net console */
166 static void igb_netpoll(struct net_device *);
167 #endif
168 #ifdef CONFIG_PCI_IOV
169 static unsigned int max_vfs = 0;
170 module_param(max_vfs, uint, 0);
171 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
172 "per physical function");
173 #endif /* CONFIG_PCI_IOV */
174
175 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
176 pci_channel_state_t);
177 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
178 static void igb_io_resume(struct pci_dev *);
179
180 static struct pci_error_handlers igb_err_handler = {
181 .error_detected = igb_io_error_detected,
182 .slot_reset = igb_io_slot_reset,
183 .resume = igb_io_resume,
184 };
185
186
187 static struct pci_driver igb_driver = {
188 .name = igb_driver_name,
189 .id_table = igb_pci_tbl,
190 .probe = igb_probe,
191 .remove = __devexit_p(igb_remove),
192 #ifdef CONFIG_PM
193 /* Power Managment Hooks */
194 .suspend = igb_suspend,
195 .resume = igb_resume,
196 #endif
197 .shutdown = igb_shutdown,
198 .err_handler = &igb_err_handler
199 };
200
201 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
202 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
203 MODULE_LICENSE("GPL");
204 MODULE_VERSION(DRV_VERSION);
205
206 struct igb_reg_info {
207 u32 ofs;
208 char *name;
209 };
210
211 static const struct igb_reg_info igb_reg_info_tbl[] = {
212
213 /* General Registers */
214 {E1000_CTRL, "CTRL"},
215 {E1000_STATUS, "STATUS"},
216 {E1000_CTRL_EXT, "CTRL_EXT"},
217
218 /* Interrupt Registers */
219 {E1000_ICR, "ICR"},
220
221 /* RX Registers */
222 {E1000_RCTL, "RCTL"},
223 {E1000_RDLEN(0), "RDLEN"},
224 {E1000_RDH(0), "RDH"},
225 {E1000_RDT(0), "RDT"},
226 {E1000_RXDCTL(0), "RXDCTL"},
227 {E1000_RDBAL(0), "RDBAL"},
228 {E1000_RDBAH(0), "RDBAH"},
229
230 /* TX Registers */
231 {E1000_TCTL, "TCTL"},
232 {E1000_TDBAL(0), "TDBAL"},
233 {E1000_TDBAH(0), "TDBAH"},
234 {E1000_TDLEN(0), "TDLEN"},
235 {E1000_TDH(0), "TDH"},
236 {E1000_TDT(0), "TDT"},
237 {E1000_TXDCTL(0), "TXDCTL"},
238 {E1000_TDFH, "TDFH"},
239 {E1000_TDFT, "TDFT"},
240 {E1000_TDFHS, "TDFHS"},
241 {E1000_TDFPC, "TDFPC"},
242
243 /* List Terminator */
244 {}
245 };
246
247 /*
248 * igb_regdump - register printout routine
249 */
250 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
251 {
252 int n = 0;
253 char rname[16];
254 u32 regs[8];
255
256 switch (reginfo->ofs) {
257 case E1000_RDLEN(0):
258 for (n = 0; n < 4; n++)
259 regs[n] = rd32(E1000_RDLEN(n));
260 break;
261 case E1000_RDH(0):
262 for (n = 0; n < 4; n++)
263 regs[n] = rd32(E1000_RDH(n));
264 break;
265 case E1000_RDT(0):
266 for (n = 0; n < 4; n++)
267 regs[n] = rd32(E1000_RDT(n));
268 break;
269 case E1000_RXDCTL(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RXDCTL(n));
272 break;
273 case E1000_RDBAL(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDBAL(n));
276 break;
277 case E1000_RDBAH(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDBAH(n));
280 break;
281 case E1000_TDBAL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RDBAL(n));
284 break;
285 case E1000_TDBAH(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_TDBAH(n));
288 break;
289 case E1000_TDLEN(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_TDLEN(n));
292 break;
293 case E1000_TDH(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_TDH(n));
296 break;
297 case E1000_TDT(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDT(n));
300 break;
301 case E1000_TXDCTL(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TXDCTL(n));
304 break;
305 default:
306 printk(KERN_INFO "%-15s %08x\n",
307 reginfo->name, rd32(reginfo->ofs));
308 return;
309 }
310
311 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
312 printk(KERN_INFO "%-15s ", rname);
313 for (n = 0; n < 4; n++)
314 printk(KERN_CONT "%08x ", regs[n]);
315 printk(KERN_CONT "\n");
316 }
317
318 /*
319 * igb_dump - Print registers, tx-rings and rx-rings
320 */
321 static void igb_dump(struct igb_adapter *adapter)
322 {
323 struct net_device *netdev = adapter->netdev;
324 struct e1000_hw *hw = &adapter->hw;
325 struct igb_reg_info *reginfo;
326 int n = 0;
327 struct igb_ring *tx_ring;
328 union e1000_adv_tx_desc *tx_desc;
329 struct my_u0 { u64 a; u64 b; } *u0;
330 struct igb_buffer *buffer_info;
331 struct igb_ring *rx_ring;
332 union e1000_adv_rx_desc *rx_desc;
333 u32 staterr;
334 int i = 0;
335
336 if (!netif_msg_hw(adapter))
337 return;
338
339 /* Print netdevice Info */
340 if (netdev) {
341 dev_info(&adapter->pdev->dev, "Net device Info\n");
342 printk(KERN_INFO "Device Name state "
343 "trans_start last_rx\n");
344 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
345 netdev->name,
346 netdev->state,
347 netdev->trans_start,
348 netdev->last_rx);
349 }
350
351 /* Print Registers */
352 dev_info(&adapter->pdev->dev, "Register Dump\n");
353 printk(KERN_INFO " Register Name Value\n");
354 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
355 reginfo->name; reginfo++) {
356 igb_regdump(hw, reginfo);
357 }
358
359 /* Print TX Ring Summary */
360 if (!netdev || !netif_running(netdev))
361 goto exit;
362
363 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
364 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
365 " leng ntw timestamp\n");
366 for (n = 0; n < adapter->num_tx_queues; n++) {
367 tx_ring = adapter->tx_ring[n];
368 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
369 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
370 n, tx_ring->next_to_use, tx_ring->next_to_clean,
371 (u64)buffer_info->dma,
372 buffer_info->length,
373 buffer_info->next_to_watch,
374 (u64)buffer_info->time_stamp);
375 }
376
377 /* Print TX Rings */
378 if (!netif_msg_tx_done(adapter))
379 goto rx_ring_summary;
380
381 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
382
383 /* Transmit Descriptor Formats
384 *
385 * Advanced Transmit Descriptor
386 * +--------------------------------------------------------------+
387 * 0 | Buffer Address [63:0] |
388 * +--------------------------------------------------------------+
389 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
390 * +--------------------------------------------------------------+
391 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 */
393
394 for (n = 0; n < adapter->num_tx_queues; n++) {
395 tx_ring = adapter->tx_ring[n];
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "T [desc] [address 63:0 ] "
400 "[PlPOCIStDDM Ln] [bi->dma ] "
401 "leng ntw timestamp bi->skb\n");
402
403 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
404 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
405 buffer_info = &tx_ring->buffer_info[i];
406 u0 = (struct my_u0 *)tx_desc;
407 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
408 " %04X %3X %016llX %p", i,
409 le64_to_cpu(u0->a),
410 le64_to_cpu(u0->b),
411 (u64)buffer_info->dma,
412 buffer_info->length,
413 buffer_info->next_to_watch,
414 (u64)buffer_info->time_stamp,
415 buffer_info->skb);
416 if (i == tx_ring->next_to_use &&
417 i == tx_ring->next_to_clean)
418 printk(KERN_CONT " NTC/U\n");
419 else if (i == tx_ring->next_to_use)
420 printk(KERN_CONT " NTU\n");
421 else if (i == tx_ring->next_to_clean)
422 printk(KERN_CONT " NTC\n");
423 else
424 printk(KERN_CONT "\n");
425
426 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
427 print_hex_dump(KERN_INFO, "",
428 DUMP_PREFIX_ADDRESS,
429 16, 1, phys_to_virt(buffer_info->dma),
430 buffer_info->length, true);
431 }
432 }
433
434 /* Print RX Rings Summary */
435 rx_ring_summary:
436 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
437 printk(KERN_INFO "Queue [NTU] [NTC]\n");
438 for (n = 0; n < adapter->num_rx_queues; n++) {
439 rx_ring = adapter->rx_ring[n];
440 printk(KERN_INFO " %5d %5X %5X\n", n,
441 rx_ring->next_to_use, rx_ring->next_to_clean);
442 }
443
444 /* Print RX Rings */
445 if (!netif_msg_rx_status(adapter))
446 goto exit;
447
448 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
449
450 /* Advanced Receive Descriptor (Read) Format
451 * 63 1 0
452 * +-----------------------------------------------------+
453 * 0 | Packet Buffer Address [63:1] |A0/NSE|
454 * +----------------------------------------------+------+
455 * 8 | Header Buffer Address [63:1] | DD |
456 * +-----------------------------------------------------+
457 *
458 *
459 * Advanced Receive Descriptor (Write-Back) Format
460 *
461 * 63 48 47 32 31 30 21 20 17 16 4 3 0
462 * +------------------------------------------------------+
463 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
464 * | Checksum Ident | | | | Type | Type |
465 * +------------------------------------------------------+
466 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
467 * +------------------------------------------------------+
468 * 63 48 47 32 31 20 19 0
469 */
470
471 for (n = 0; n < adapter->num_rx_queues; n++) {
472 rx_ring = adapter->rx_ring[n];
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "R [desc] [ PktBuf A0] "
477 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
478 "<-- Adv Rx Read format\n");
479 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
480 "[vl er S cks ln] ---------------- [bi->skb] "
481 "<-- Adv Rx Write-Back format\n");
482
483 for (i = 0; i < rx_ring->count; i++) {
484 buffer_info = &rx_ring->buffer_info[i];
485 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
486 u0 = (struct my_u0 *)rx_desc;
487 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
488 if (staterr & E1000_RXD_STAT_DD) {
489 /* Descriptor Done */
490 printk(KERN_INFO "RWB[0x%03X] %016llX "
491 "%016llX ---------------- %p", i,
492 le64_to_cpu(u0->a),
493 le64_to_cpu(u0->b),
494 buffer_info->skb);
495 } else {
496 printk(KERN_INFO "R [0x%03X] %016llX "
497 "%016llX %016llX %p", i,
498 le64_to_cpu(u0->a),
499 le64_to_cpu(u0->b),
500 (u64)buffer_info->dma,
501 buffer_info->skb);
502
503 if (netif_msg_pktdata(adapter)) {
504 print_hex_dump(KERN_INFO, "",
505 DUMP_PREFIX_ADDRESS,
506 16, 1,
507 phys_to_virt(buffer_info->dma),
508 rx_ring->rx_buffer_len, true);
509 if (rx_ring->rx_buffer_len
510 < IGB_RXBUFFER_1024)
511 print_hex_dump(KERN_INFO, "",
512 DUMP_PREFIX_ADDRESS,
513 16, 1,
514 phys_to_virt(
515 buffer_info->page_dma +
516 buffer_info->page_offset),
517 PAGE_SIZE/2, true);
518 }
519 }
520
521 if (i == rx_ring->next_to_use)
522 printk(KERN_CONT " NTU\n");
523 else if (i == rx_ring->next_to_clean)
524 printk(KERN_CONT " NTC\n");
525 else
526 printk(KERN_CONT "\n");
527
528 }
529 }
530
531 exit:
532 return;
533 }
534
535
536 /**
537 * igb_read_clock - read raw cycle counter (to be used by time counter)
538 */
539 static cycle_t igb_read_clock(const struct cyclecounter *tc)
540 {
541 struct igb_adapter *adapter =
542 container_of(tc, struct igb_adapter, cycles);
543 struct e1000_hw *hw = &adapter->hw;
544 u64 stamp = 0;
545 int shift = 0;
546
547 /*
548 * The timestamp latches on lowest register read. For the 82580
549 * the lowest register is SYSTIMR instead of SYSTIML. However we never
550 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
551 */
552 if (hw->mac.type == e1000_82580) {
553 stamp = rd32(E1000_SYSTIMR) >> 8;
554 shift = IGB_82580_TSYNC_SHIFT;
555 }
556
557 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
558 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
559 return stamp;
560 }
561
562 /**
563 * igb_get_hw_dev - return device
564 * used by hardware layer to print debugging information
565 **/
566 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
567 {
568 struct igb_adapter *adapter = hw->back;
569 return adapter->netdev;
570 }
571
572 /**
573 * igb_init_module - Driver Registration Routine
574 *
575 * igb_init_module is the first routine called when the driver is
576 * loaded. All it does is register with the PCI subsystem.
577 **/
578 static int __init igb_init_module(void)
579 {
580 int ret;
581 printk(KERN_INFO "%s - version %s\n",
582 igb_driver_string, igb_driver_version);
583
584 printk(KERN_INFO "%s\n", igb_copyright);
585
586 #ifdef CONFIG_IGB_DCA
587 dca_register_notify(&dca_notifier);
588 #endif
589 ret = pci_register_driver(&igb_driver);
590 return ret;
591 }
592
593 module_init(igb_init_module);
594
595 /**
596 * igb_exit_module - Driver Exit Cleanup Routine
597 *
598 * igb_exit_module is called just before the driver is removed
599 * from memory.
600 **/
601 static void __exit igb_exit_module(void)
602 {
603 #ifdef CONFIG_IGB_DCA
604 dca_unregister_notify(&dca_notifier);
605 #endif
606 pci_unregister_driver(&igb_driver);
607 }
608
609 module_exit(igb_exit_module);
610
611 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
612 /**
613 * igb_cache_ring_register - Descriptor ring to register mapping
614 * @adapter: board private structure to initialize
615 *
616 * Once we know the feature-set enabled for the device, we'll cache
617 * the register offset the descriptor ring is assigned to.
618 **/
619 static void igb_cache_ring_register(struct igb_adapter *adapter)
620 {
621 int i = 0, j = 0;
622 u32 rbase_offset = adapter->vfs_allocated_count;
623
624 switch (adapter->hw.mac.type) {
625 case e1000_82576:
626 /* The queues are allocated for virtualization such that VF 0
627 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
628 * In order to avoid collision we start at the first free queue
629 * and continue consuming queues in the same sequence
630 */
631 if (adapter->vfs_allocated_count) {
632 for (; i < adapter->rss_queues; i++)
633 adapter->rx_ring[i]->reg_idx = rbase_offset +
634 Q_IDX_82576(i);
635 }
636 case e1000_82575:
637 case e1000_82580:
638 case e1000_i350:
639 default:
640 for (; i < adapter->num_rx_queues; i++)
641 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
642 for (; j < adapter->num_tx_queues; j++)
643 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
644 break;
645 }
646 }
647
648 static void igb_free_queues(struct igb_adapter *adapter)
649 {
650 int i;
651
652 for (i = 0; i < adapter->num_tx_queues; i++) {
653 kfree(adapter->tx_ring[i]);
654 adapter->tx_ring[i] = NULL;
655 }
656 for (i = 0; i < adapter->num_rx_queues; i++) {
657 kfree(adapter->rx_ring[i]);
658 adapter->rx_ring[i] = NULL;
659 }
660 adapter->num_rx_queues = 0;
661 adapter->num_tx_queues = 0;
662 }
663
664 /**
665 * igb_alloc_queues - Allocate memory for all rings
666 * @adapter: board private structure to initialize
667 *
668 * We allocate one ring per queue at run-time since we don't know the
669 * number of queues at compile-time.
670 **/
671 static int igb_alloc_queues(struct igb_adapter *adapter)
672 {
673 struct igb_ring *ring;
674 int i;
675
676 for (i = 0; i < adapter->num_tx_queues; i++) {
677 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 if (!ring)
679 goto err;
680 ring->count = adapter->tx_ring_count;
681 ring->queue_index = i;
682 ring->dev = &adapter->pdev->dev;
683 ring->netdev = adapter->netdev;
684 /* For 82575, context index must be unique per ring. */
685 if (adapter->hw.mac.type == e1000_82575)
686 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
687 adapter->tx_ring[i] = ring;
688 }
689
690 for (i = 0; i < adapter->num_rx_queues; i++) {
691 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 if (!ring)
693 goto err;
694 ring->count = adapter->rx_ring_count;
695 ring->queue_index = i;
696 ring->dev = &adapter->pdev->dev;
697 ring->netdev = adapter->netdev;
698 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
699 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
700 /* set flag indicating ring supports SCTP checksum offload */
701 if (adapter->hw.mac.type >= e1000_82576)
702 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
703 adapter->rx_ring[i] = ring;
704 }
705
706 igb_cache_ring_register(adapter);
707
708 return 0;
709
710 err:
711 igb_free_queues(adapter);
712
713 return -ENOMEM;
714 }
715
716 #define IGB_N0_QUEUE -1
717 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
718 {
719 u32 msixbm = 0;
720 struct igb_adapter *adapter = q_vector->adapter;
721 struct e1000_hw *hw = &adapter->hw;
722 u32 ivar, index;
723 int rx_queue = IGB_N0_QUEUE;
724 int tx_queue = IGB_N0_QUEUE;
725
726 if (q_vector->rx_ring)
727 rx_queue = q_vector->rx_ring->reg_idx;
728 if (q_vector->tx_ring)
729 tx_queue = q_vector->tx_ring->reg_idx;
730
731 switch (hw->mac.type) {
732 case e1000_82575:
733 /* The 82575 assigns vectors using a bitmask, which matches the
734 bitmask for the EICR/EIMS/EIMC registers. To assign one
735 or more queues to a vector, we write the appropriate bits
736 into the MSIXBM register for that vector. */
737 if (rx_queue > IGB_N0_QUEUE)
738 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
739 if (tx_queue > IGB_N0_QUEUE)
740 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
741 if (!adapter->msix_entries && msix_vector == 0)
742 msixbm |= E1000_EIMS_OTHER;
743 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
744 q_vector->eims_value = msixbm;
745 break;
746 case e1000_82576:
747 /* 82576 uses a table-based method for assigning vectors.
748 Each queue has a single entry in the table to which we write
749 a vector number along with a "valid" bit. Sadly, the layout
750 of the table is somewhat counterintuitive. */
751 if (rx_queue > IGB_N0_QUEUE) {
752 index = (rx_queue & 0x7);
753 ivar = array_rd32(E1000_IVAR0, index);
754 if (rx_queue < 8) {
755 /* vector goes into low byte of register */
756 ivar = ivar & 0xFFFFFF00;
757 ivar |= msix_vector | E1000_IVAR_VALID;
758 } else {
759 /* vector goes into third byte of register */
760 ivar = ivar & 0xFF00FFFF;
761 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
762 }
763 array_wr32(E1000_IVAR0, index, ivar);
764 }
765 if (tx_queue > IGB_N0_QUEUE) {
766 index = (tx_queue & 0x7);
767 ivar = array_rd32(E1000_IVAR0, index);
768 if (tx_queue < 8) {
769 /* vector goes into second byte of register */
770 ivar = ivar & 0xFFFF00FF;
771 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
772 } else {
773 /* vector goes into high byte of register */
774 ivar = ivar & 0x00FFFFFF;
775 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
776 }
777 array_wr32(E1000_IVAR0, index, ivar);
778 }
779 q_vector->eims_value = 1 << msix_vector;
780 break;
781 case e1000_82580:
782 case e1000_i350:
783 /* 82580 uses the same table-based approach as 82576 but has fewer
784 entries as a result we carry over for queues greater than 4. */
785 if (rx_queue > IGB_N0_QUEUE) {
786 index = (rx_queue >> 1);
787 ivar = array_rd32(E1000_IVAR0, index);
788 if (rx_queue & 0x1) {
789 /* vector goes into third byte of register */
790 ivar = ivar & 0xFF00FFFF;
791 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
792 } else {
793 /* vector goes into low byte of register */
794 ivar = ivar & 0xFFFFFF00;
795 ivar |= msix_vector | E1000_IVAR_VALID;
796 }
797 array_wr32(E1000_IVAR0, index, ivar);
798 }
799 if (tx_queue > IGB_N0_QUEUE) {
800 index = (tx_queue >> 1);
801 ivar = array_rd32(E1000_IVAR0, index);
802 if (tx_queue & 0x1) {
803 /* vector goes into high byte of register */
804 ivar = ivar & 0x00FFFFFF;
805 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
806 } else {
807 /* vector goes into second byte of register */
808 ivar = ivar & 0xFFFF00FF;
809 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
810 }
811 array_wr32(E1000_IVAR0, index, ivar);
812 }
813 q_vector->eims_value = 1 << msix_vector;
814 break;
815 default:
816 BUG();
817 break;
818 }
819
820 /* add q_vector eims value to global eims_enable_mask */
821 adapter->eims_enable_mask |= q_vector->eims_value;
822
823 /* configure q_vector to set itr on first interrupt */
824 q_vector->set_itr = 1;
825 }
826
827 /**
828 * igb_configure_msix - Configure MSI-X hardware
829 *
830 * igb_configure_msix sets up the hardware to properly
831 * generate MSI-X interrupts.
832 **/
833 static void igb_configure_msix(struct igb_adapter *adapter)
834 {
835 u32 tmp;
836 int i, vector = 0;
837 struct e1000_hw *hw = &adapter->hw;
838
839 adapter->eims_enable_mask = 0;
840
841 /* set vector for other causes, i.e. link changes */
842 switch (hw->mac.type) {
843 case e1000_82575:
844 tmp = rd32(E1000_CTRL_EXT);
845 /* enable MSI-X PBA support*/
846 tmp |= E1000_CTRL_EXT_PBA_CLR;
847
848 /* Auto-Mask interrupts upon ICR read. */
849 tmp |= E1000_CTRL_EXT_EIAME;
850 tmp |= E1000_CTRL_EXT_IRCA;
851
852 wr32(E1000_CTRL_EXT, tmp);
853
854 /* enable msix_other interrupt */
855 array_wr32(E1000_MSIXBM(0), vector++,
856 E1000_EIMS_OTHER);
857 adapter->eims_other = E1000_EIMS_OTHER;
858
859 break;
860
861 case e1000_82576:
862 case e1000_82580:
863 case e1000_i350:
864 /* Turn on MSI-X capability first, or our settings
865 * won't stick. And it will take days to debug. */
866 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
867 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 E1000_GPIE_NSICR);
869
870 /* enable msix_other interrupt */
871 adapter->eims_other = 1 << vector;
872 tmp = (vector++ | E1000_IVAR_VALID) << 8;
873
874 wr32(E1000_IVAR_MISC, tmp);
875 break;
876 default:
877 /* do nothing, since nothing else supports MSI-X */
878 break;
879 } /* switch (hw->mac.type) */
880
881 adapter->eims_enable_mask |= adapter->eims_other;
882
883 for (i = 0; i < adapter->num_q_vectors; i++)
884 igb_assign_vector(adapter->q_vector[i], vector++);
885
886 wrfl();
887 }
888
889 /**
890 * igb_request_msix - Initialize MSI-X interrupts
891 *
892 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 * kernel.
894 **/
895 static int igb_request_msix(struct igb_adapter *adapter)
896 {
897 struct net_device *netdev = adapter->netdev;
898 struct e1000_hw *hw = &adapter->hw;
899 int i, err = 0, vector = 0;
900
901 err = request_irq(adapter->msix_entries[vector].vector,
902 igb_msix_other, 0, netdev->name, adapter);
903 if (err)
904 goto out;
905 vector++;
906
907 for (i = 0; i < adapter->num_q_vectors; i++) {
908 struct igb_q_vector *q_vector = adapter->q_vector[i];
909
910 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
911
912 if (q_vector->rx_ring && q_vector->tx_ring)
913 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
914 q_vector->rx_ring->queue_index);
915 else if (q_vector->tx_ring)
916 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
917 q_vector->tx_ring->queue_index);
918 else if (q_vector->rx_ring)
919 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
920 q_vector->rx_ring->queue_index);
921 else
922 sprintf(q_vector->name, "%s-unused", netdev->name);
923
924 err = request_irq(adapter->msix_entries[vector].vector,
925 igb_msix_ring, 0, q_vector->name,
926 q_vector);
927 if (err)
928 goto out;
929 vector++;
930 }
931
932 igb_configure_msix(adapter);
933 return 0;
934 out:
935 return err;
936 }
937
938 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
939 {
940 if (adapter->msix_entries) {
941 pci_disable_msix(adapter->pdev);
942 kfree(adapter->msix_entries);
943 adapter->msix_entries = NULL;
944 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
945 pci_disable_msi(adapter->pdev);
946 }
947 }
948
949 /**
950 * igb_free_q_vectors - Free memory allocated for interrupt vectors
951 * @adapter: board private structure to initialize
952 *
953 * This function frees the memory allocated to the q_vectors. In addition if
954 * NAPI is enabled it will delete any references to the NAPI struct prior
955 * to freeing the q_vector.
956 **/
957 static void igb_free_q_vectors(struct igb_adapter *adapter)
958 {
959 int v_idx;
960
961 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
962 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
963 adapter->q_vector[v_idx] = NULL;
964 if (!q_vector)
965 continue;
966 netif_napi_del(&q_vector->napi);
967 kfree(q_vector);
968 }
969 adapter->num_q_vectors = 0;
970 }
971
972 /**
973 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
974 *
975 * This function resets the device so that it has 0 rx queues, tx queues, and
976 * MSI-X interrupts allocated.
977 */
978 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
979 {
980 igb_free_queues(adapter);
981 igb_free_q_vectors(adapter);
982 igb_reset_interrupt_capability(adapter);
983 }
984
985 /**
986 * igb_set_interrupt_capability - set MSI or MSI-X if supported
987 *
988 * Attempt to configure interrupts using the best available
989 * capabilities of the hardware and kernel.
990 **/
991 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
992 {
993 int err;
994 int numvecs, i;
995
996 /* Number of supported queues. */
997 adapter->num_rx_queues = adapter->rss_queues;
998 if (adapter->vfs_allocated_count)
999 adapter->num_tx_queues = 1;
1000 else
1001 adapter->num_tx_queues = adapter->rss_queues;
1002
1003 /* start with one vector for every rx queue */
1004 numvecs = adapter->num_rx_queues;
1005
1006 /* if tx handler is separate add 1 for every tx queue */
1007 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1008 numvecs += adapter->num_tx_queues;
1009
1010 /* store the number of vectors reserved for queues */
1011 adapter->num_q_vectors = numvecs;
1012
1013 /* add 1 vector for link status interrupts */
1014 numvecs++;
1015 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1016 GFP_KERNEL);
1017 if (!adapter->msix_entries)
1018 goto msi_only;
1019
1020 for (i = 0; i < numvecs; i++)
1021 adapter->msix_entries[i].entry = i;
1022
1023 err = pci_enable_msix(adapter->pdev,
1024 adapter->msix_entries,
1025 numvecs);
1026 if (err == 0)
1027 goto out;
1028
1029 igb_reset_interrupt_capability(adapter);
1030
1031 /* If we can't do MSI-X, try MSI */
1032 msi_only:
1033 #ifdef CONFIG_PCI_IOV
1034 /* disable SR-IOV for non MSI-X configurations */
1035 if (adapter->vf_data) {
1036 struct e1000_hw *hw = &adapter->hw;
1037 /* disable iov and allow time for transactions to clear */
1038 pci_disable_sriov(adapter->pdev);
1039 msleep(500);
1040
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1044 msleep(100);
1045 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 }
1047 #endif
1048 adapter->vfs_allocated_count = 0;
1049 adapter->rss_queues = 1;
1050 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1051 adapter->num_rx_queues = 1;
1052 adapter->num_tx_queues = 1;
1053 adapter->num_q_vectors = 1;
1054 if (!pci_enable_msi(adapter->pdev))
1055 adapter->flags |= IGB_FLAG_HAS_MSI;
1056 out:
1057 /* Notify the stack of the (possibly) reduced queue counts. */
1058 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1059 return netif_set_real_num_rx_queues(adapter->netdev,
1060 adapter->num_rx_queues);
1061 }
1062
1063 /**
1064 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1065 * @adapter: board private structure to initialize
1066 *
1067 * We allocate one q_vector per queue interrupt. If allocation fails we
1068 * return -ENOMEM.
1069 **/
1070 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1071 {
1072 struct igb_q_vector *q_vector;
1073 struct e1000_hw *hw = &adapter->hw;
1074 int v_idx;
1075
1076 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1077 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1078 if (!q_vector)
1079 goto err_out;
1080 q_vector->adapter = adapter;
1081 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1082 q_vector->itr_val = IGB_START_ITR;
1083 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1084 adapter->q_vector[v_idx] = q_vector;
1085 }
1086 return 0;
1087
1088 err_out:
1089 igb_free_q_vectors(adapter);
1090 return -ENOMEM;
1091 }
1092
1093 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1094 int ring_idx, int v_idx)
1095 {
1096 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1097
1098 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1099 q_vector->rx_ring->q_vector = q_vector;
1100 q_vector->itr_val = adapter->rx_itr_setting;
1101 if (q_vector->itr_val && q_vector->itr_val <= 3)
1102 q_vector->itr_val = IGB_START_ITR;
1103 }
1104
1105 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107 {
1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1109
1110 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1111 q_vector->tx_ring->q_vector = q_vector;
1112 q_vector->itr_val = adapter->tx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
1115 }
1116
1117 /**
1118 * igb_map_ring_to_vector - maps allocated queues to vectors
1119 *
1120 * This function maps the recently allocated queues to vectors.
1121 **/
1122 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123 {
1124 int i;
1125 int v_idx = 0;
1126
1127 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1128 (adapter->num_q_vectors < adapter->num_tx_queues))
1129 return -ENOMEM;
1130
1131 if (adapter->num_q_vectors >=
1132 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1133 for (i = 0; i < adapter->num_rx_queues; i++)
1134 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1135 for (i = 0; i < adapter->num_tx_queues; i++)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1137 } else {
1138 for (i = 0; i < adapter->num_rx_queues; i++) {
1139 if (i < adapter->num_tx_queues)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1141 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1142 }
1143 for (; i < adapter->num_tx_queues; i++)
1144 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1145 }
1146 return 0;
1147 }
1148
1149 /**
1150 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1151 *
1152 * This function initializes the interrupts and allocates all of the queues.
1153 **/
1154 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1155 {
1156 struct pci_dev *pdev = adapter->pdev;
1157 int err;
1158
1159 err = igb_set_interrupt_capability(adapter);
1160 if (err)
1161 return err;
1162
1163 err = igb_alloc_q_vectors(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1166 goto err_alloc_q_vectors;
1167 }
1168
1169 err = igb_alloc_queues(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1172 goto err_alloc_queues;
1173 }
1174
1175 err = igb_map_ring_to_vector(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1178 goto err_map_queues;
1179 }
1180
1181
1182 return 0;
1183 err_map_queues:
1184 igb_free_queues(adapter);
1185 err_alloc_queues:
1186 igb_free_q_vectors(adapter);
1187 err_alloc_q_vectors:
1188 igb_reset_interrupt_capability(adapter);
1189 return err;
1190 }
1191
1192 /**
1193 * igb_request_irq - initialize interrupts
1194 *
1195 * Attempts to configure interrupts using the best available
1196 * capabilities of the hardware and kernel.
1197 **/
1198 static int igb_request_irq(struct igb_adapter *adapter)
1199 {
1200 struct net_device *netdev = adapter->netdev;
1201 struct pci_dev *pdev = adapter->pdev;
1202 int err = 0;
1203
1204 if (adapter->msix_entries) {
1205 err = igb_request_msix(adapter);
1206 if (!err)
1207 goto request_done;
1208 /* fall back to MSI */
1209 igb_clear_interrupt_scheme(adapter);
1210 if (!pci_enable_msi(adapter->pdev))
1211 adapter->flags |= IGB_FLAG_HAS_MSI;
1212 igb_free_all_tx_resources(adapter);
1213 igb_free_all_rx_resources(adapter);
1214 adapter->num_tx_queues = 1;
1215 adapter->num_rx_queues = 1;
1216 adapter->num_q_vectors = 1;
1217 err = igb_alloc_q_vectors(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for vectors\n");
1221 goto request_done;
1222 }
1223 err = igb_alloc_queues(adapter);
1224 if (err) {
1225 dev_err(&pdev->dev,
1226 "Unable to allocate memory for queues\n");
1227 igb_free_q_vectors(adapter);
1228 goto request_done;
1229 }
1230 igb_setup_all_tx_resources(adapter);
1231 igb_setup_all_rx_resources(adapter);
1232 } else {
1233 igb_assign_vector(adapter->q_vector[0], 0);
1234 }
1235
1236 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1237 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1238 netdev->name, adapter);
1239 if (!err)
1240 goto request_done;
1241
1242 /* fall back to legacy interrupts */
1243 igb_reset_interrupt_capability(adapter);
1244 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1245 }
1246
1247 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1248 netdev->name, adapter);
1249
1250 if (err)
1251 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 err);
1253
1254 request_done:
1255 return err;
1256 }
1257
1258 static void igb_free_irq(struct igb_adapter *adapter)
1259 {
1260 if (adapter->msix_entries) {
1261 int vector = 0, i;
1262
1263 free_irq(adapter->msix_entries[vector++].vector, adapter);
1264
1265 for (i = 0; i < adapter->num_q_vectors; i++) {
1266 struct igb_q_vector *q_vector = adapter->q_vector[i];
1267 free_irq(adapter->msix_entries[vector++].vector,
1268 q_vector);
1269 }
1270 } else {
1271 free_irq(adapter->pdev->irq, adapter);
1272 }
1273 }
1274
1275 /**
1276 * igb_irq_disable - Mask off interrupt generation on the NIC
1277 * @adapter: board private structure
1278 **/
1279 static void igb_irq_disable(struct igb_adapter *adapter)
1280 {
1281 struct e1000_hw *hw = &adapter->hw;
1282
1283 /*
1284 * we need to be careful when disabling interrupts. The VFs are also
1285 * mapped into these registers and so clearing the bits can cause
1286 * issues on the VF drivers so we only need to clear what we set
1287 */
1288 if (adapter->msix_entries) {
1289 u32 regval = rd32(E1000_EIAM);
1290 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_EIMC, adapter->eims_enable_mask);
1292 regval = rd32(E1000_EIAC);
1293 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1294 }
1295
1296 wr32(E1000_IAM, 0);
1297 wr32(E1000_IMC, ~0);
1298 wrfl();
1299 if (adapter->msix_entries) {
1300 int i;
1301 for (i = 0; i < adapter->num_q_vectors; i++)
1302 synchronize_irq(adapter->msix_entries[i].vector);
1303 } else {
1304 synchronize_irq(adapter->pdev->irq);
1305 }
1306 }
1307
1308 /**
1309 * igb_irq_enable - Enable default interrupt generation settings
1310 * @adapter: board private structure
1311 **/
1312 static void igb_irq_enable(struct igb_adapter *adapter)
1313 {
1314 struct e1000_hw *hw = &adapter->hw;
1315
1316 if (adapter->msix_entries) {
1317 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1318 u32 regval = rd32(E1000_EIAC);
1319 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1320 regval = rd32(E1000_EIAM);
1321 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1322 wr32(E1000_EIMS, adapter->eims_enable_mask);
1323 if (adapter->vfs_allocated_count) {
1324 wr32(E1000_MBVFIMR, 0xFF);
1325 ims |= E1000_IMS_VMMB;
1326 }
1327 if (adapter->hw.mac.type == e1000_82580)
1328 ims |= E1000_IMS_DRSTA;
1329
1330 wr32(E1000_IMS, ims);
1331 } else {
1332 wr32(E1000_IMS, IMS_ENABLE_MASK |
1333 E1000_IMS_DRSTA);
1334 wr32(E1000_IAM, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
1336 }
1337 }
1338
1339 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1340 {
1341 struct e1000_hw *hw = &adapter->hw;
1342 u16 vid = adapter->hw.mng_cookie.vlan_id;
1343 u16 old_vid = adapter->mng_vlan_id;
1344
1345 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1346 /* add VID to filter table */
1347 igb_vfta_set(hw, vid, true);
1348 adapter->mng_vlan_id = vid;
1349 } else {
1350 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1351 }
1352
1353 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1354 (vid != old_vid) &&
1355 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1356 /* remove VID from filter table */
1357 igb_vfta_set(hw, old_vid, false);
1358 }
1359 }
1360
1361 /**
1362 * igb_release_hw_control - release control of the h/w to f/w
1363 * @adapter: address of board private structure
1364 *
1365 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1366 * For ASF and Pass Through versions of f/w this means that the
1367 * driver is no longer loaded.
1368 *
1369 **/
1370 static void igb_release_hw_control(struct igb_adapter *adapter)
1371 {
1372 struct e1000_hw *hw = &adapter->hw;
1373 u32 ctrl_ext;
1374
1375 /* Let firmware take over control of h/w */
1376 ctrl_ext = rd32(E1000_CTRL_EXT);
1377 wr32(E1000_CTRL_EXT,
1378 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1379 }
1380
1381 /**
1382 * igb_get_hw_control - get control of the h/w from f/w
1383 * @adapter: address of board private structure
1384 *
1385 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1386 * For ASF and Pass Through versions of f/w this means that
1387 * the driver is loaded.
1388 *
1389 **/
1390 static void igb_get_hw_control(struct igb_adapter *adapter)
1391 {
1392 struct e1000_hw *hw = &adapter->hw;
1393 u32 ctrl_ext;
1394
1395 /* Let firmware know the driver has taken over */
1396 ctrl_ext = rd32(E1000_CTRL_EXT);
1397 wr32(E1000_CTRL_EXT,
1398 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1399 }
1400
1401 /**
1402 * igb_configure - configure the hardware for RX and TX
1403 * @adapter: private board structure
1404 **/
1405 static void igb_configure(struct igb_adapter *adapter)
1406 {
1407 struct net_device *netdev = adapter->netdev;
1408 int i;
1409
1410 igb_get_hw_control(adapter);
1411 igb_set_rx_mode(netdev);
1412
1413 igb_restore_vlan(adapter);
1414
1415 igb_setup_tctl(adapter);
1416 igb_setup_mrqc(adapter);
1417 igb_setup_rctl(adapter);
1418
1419 igb_configure_tx(adapter);
1420 igb_configure_rx(adapter);
1421
1422 igb_rx_fifo_flush_82575(&adapter->hw);
1423
1424 /* call igb_desc_unused which always leaves
1425 * at least 1 descriptor unused to make sure
1426 * next_to_use != next_to_clean */
1427 for (i = 0; i < adapter->num_rx_queues; i++) {
1428 struct igb_ring *ring = adapter->rx_ring[i];
1429 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1430 }
1431 }
1432
1433 /**
1434 * igb_power_up_link - Power up the phy/serdes link
1435 * @adapter: address of board private structure
1436 **/
1437 void igb_power_up_link(struct igb_adapter *adapter)
1438 {
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_up_phy_copper(&adapter->hw);
1441 else
1442 igb_power_up_serdes_link_82575(&adapter->hw);
1443 }
1444
1445 /**
1446 * igb_power_down_link - Power down the phy/serdes link
1447 * @adapter: address of board private structure
1448 */
1449 static void igb_power_down_link(struct igb_adapter *adapter)
1450 {
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_down_phy_copper_82575(&adapter->hw);
1453 else
1454 igb_shutdown_serdes_link_82575(&adapter->hw);
1455 }
1456
1457 /**
1458 * igb_up - Open the interface and prepare it to handle traffic
1459 * @adapter: board private structure
1460 **/
1461 int igb_up(struct igb_adapter *adapter)
1462 {
1463 struct e1000_hw *hw = &adapter->hw;
1464 int i;
1465
1466 /* hardware has been reset, we need to reload some things */
1467 igb_configure(adapter);
1468
1469 clear_bit(__IGB_DOWN, &adapter->state);
1470
1471 for (i = 0; i < adapter->num_q_vectors; i++) {
1472 struct igb_q_vector *q_vector = adapter->q_vector[i];
1473 napi_enable(&q_vector->napi);
1474 }
1475 if (adapter->msix_entries)
1476 igb_configure_msix(adapter);
1477 else
1478 igb_assign_vector(adapter->q_vector[0], 0);
1479
1480 /* Clear any pending interrupts. */
1481 rd32(E1000_ICR);
1482 igb_irq_enable(adapter);
1483
1484 /* notify VFs that reset has been completed */
1485 if (adapter->vfs_allocated_count) {
1486 u32 reg_data = rd32(E1000_CTRL_EXT);
1487 reg_data |= E1000_CTRL_EXT_PFRSTD;
1488 wr32(E1000_CTRL_EXT, reg_data);
1489 }
1490
1491 netif_tx_start_all_queues(adapter->netdev);
1492
1493 /* start the watchdog. */
1494 hw->mac.get_link_status = 1;
1495 schedule_work(&adapter->watchdog_task);
1496
1497 return 0;
1498 }
1499
1500 void igb_down(struct igb_adapter *adapter)
1501 {
1502 struct net_device *netdev = adapter->netdev;
1503 struct e1000_hw *hw = &adapter->hw;
1504 u32 tctl, rctl;
1505 int i;
1506
1507 /* signal that we're down so the interrupt handler does not
1508 * reschedule our watchdog timer */
1509 set_bit(__IGB_DOWN, &adapter->state);
1510
1511 /* disable receives in the hardware */
1512 rctl = rd32(E1000_RCTL);
1513 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1514 /* flush and sleep below */
1515
1516 netif_tx_stop_all_queues(netdev);
1517
1518 /* disable transmits in the hardware */
1519 tctl = rd32(E1000_TCTL);
1520 tctl &= ~E1000_TCTL_EN;
1521 wr32(E1000_TCTL, tctl);
1522 /* flush both disables and wait for them to finish */
1523 wrfl();
1524 msleep(10);
1525
1526 for (i = 0; i < adapter->num_q_vectors; i++) {
1527 struct igb_q_vector *q_vector = adapter->q_vector[i];
1528 napi_disable(&q_vector->napi);
1529 }
1530
1531 igb_irq_disable(adapter);
1532
1533 del_timer_sync(&adapter->watchdog_timer);
1534 del_timer_sync(&adapter->phy_info_timer);
1535
1536 netif_carrier_off(netdev);
1537
1538 /* record the stats before reset*/
1539 spin_lock(&adapter->stats64_lock);
1540 igb_update_stats(adapter, &adapter->stats64);
1541 spin_unlock(&adapter->stats64_lock);
1542
1543 adapter->link_speed = 0;
1544 adapter->link_duplex = 0;
1545
1546 if (!pci_channel_offline(adapter->pdev))
1547 igb_reset(adapter);
1548 igb_clean_all_tx_rings(adapter);
1549 igb_clean_all_rx_rings(adapter);
1550 #ifdef CONFIG_IGB_DCA
1551
1552 /* since we reset the hardware DCA settings were cleared */
1553 igb_setup_dca(adapter);
1554 #endif
1555 }
1556
1557 void igb_reinit_locked(struct igb_adapter *adapter)
1558 {
1559 WARN_ON(in_interrupt());
1560 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1561 msleep(1);
1562 igb_down(adapter);
1563 igb_up(adapter);
1564 clear_bit(__IGB_RESETTING, &adapter->state);
1565 }
1566
1567 void igb_reset(struct igb_adapter *adapter)
1568 {
1569 struct pci_dev *pdev = adapter->pdev;
1570 struct e1000_hw *hw = &adapter->hw;
1571 struct e1000_mac_info *mac = &hw->mac;
1572 struct e1000_fc_info *fc = &hw->fc;
1573 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1574 u16 hwm;
1575
1576 /* Repartition Pba for greater than 9k mtu
1577 * To take effect CTRL.RST is required.
1578 */
1579 switch (mac->type) {
1580 case e1000_i350:
1581 case e1000_82580:
1582 pba = rd32(E1000_RXPBS);
1583 pba = igb_rxpbs_adjust_82580(pba);
1584 break;
1585 case e1000_82576:
1586 pba = rd32(E1000_RXPBS);
1587 pba &= E1000_RXPBS_SIZE_MASK_82576;
1588 break;
1589 case e1000_82575:
1590 default:
1591 pba = E1000_PBA_34K;
1592 break;
1593 }
1594
1595 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1596 (mac->type < e1000_82576)) {
1597 /* adjust PBA for jumbo frames */
1598 wr32(E1000_PBA, pba);
1599
1600 /* To maintain wire speed transmits, the Tx FIFO should be
1601 * large enough to accommodate two full transmit packets,
1602 * rounded up to the next 1KB and expressed in KB. Likewise,
1603 * the Rx FIFO should be large enough to accommodate at least
1604 * one full receive packet and is similarly rounded up and
1605 * expressed in KB. */
1606 pba = rd32(E1000_PBA);
1607 /* upper 16 bits has Tx packet buffer allocation size in KB */
1608 tx_space = pba >> 16;
1609 /* lower 16 bits has Rx packet buffer allocation size in KB */
1610 pba &= 0xffff;
1611 /* the tx fifo also stores 16 bytes of information about the tx
1612 * but don't include ethernet FCS because hardware appends it */
1613 min_tx_space = (adapter->max_frame_size +
1614 sizeof(union e1000_adv_tx_desc) -
1615 ETH_FCS_LEN) * 2;
1616 min_tx_space = ALIGN(min_tx_space, 1024);
1617 min_tx_space >>= 10;
1618 /* software strips receive CRC, so leave room for it */
1619 min_rx_space = adapter->max_frame_size;
1620 min_rx_space = ALIGN(min_rx_space, 1024);
1621 min_rx_space >>= 10;
1622
1623 /* If current Tx allocation is less than the min Tx FIFO size,
1624 * and the min Tx FIFO size is less than the current Rx FIFO
1625 * allocation, take space away from current Rx allocation */
1626 if (tx_space < min_tx_space &&
1627 ((min_tx_space - tx_space) < pba)) {
1628 pba = pba - (min_tx_space - tx_space);
1629
1630 /* if short on rx space, rx wins and must trump tx
1631 * adjustment */
1632 if (pba < min_rx_space)
1633 pba = min_rx_space;
1634 }
1635 wr32(E1000_PBA, pba);
1636 }
1637
1638 /* flow control settings */
1639 /* The high water mark must be low enough to fit one full frame
1640 * (or the size used for early receive) above it in the Rx FIFO.
1641 * Set it to the lower of:
1642 * - 90% of the Rx FIFO size, or
1643 * - the full Rx FIFO size minus one full frame */
1644 hwm = min(((pba << 10) * 9 / 10),
1645 ((pba << 10) - 2 * adapter->max_frame_size));
1646
1647 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1648 fc->low_water = fc->high_water - 16;
1649 fc->pause_time = 0xFFFF;
1650 fc->send_xon = 1;
1651 fc->current_mode = fc->requested_mode;
1652
1653 /* disable receive for all VFs and wait one second */
1654 if (adapter->vfs_allocated_count) {
1655 int i;
1656 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1657 adapter->vf_data[i].flags = 0;
1658
1659 /* ping all the active vfs to let them know we are going down */
1660 igb_ping_all_vfs(adapter);
1661
1662 /* disable transmits and receives */
1663 wr32(E1000_VFRE, 0);
1664 wr32(E1000_VFTE, 0);
1665 }
1666
1667 /* Allow time for pending master requests to run */
1668 hw->mac.ops.reset_hw(hw);
1669 wr32(E1000_WUC, 0);
1670
1671 if (hw->mac.ops.init_hw(hw))
1672 dev_err(&pdev->dev, "Hardware Error\n");
1673
1674 if (hw->mac.type == e1000_82580) {
1675 u32 reg = rd32(E1000_PCIEMISC);
1676 wr32(E1000_PCIEMISC,
1677 reg & ~E1000_PCIEMISC_LX_DECISION);
1678 }
1679 if (!netif_running(adapter->netdev))
1680 igb_power_down_link(adapter);
1681
1682 igb_update_mng_vlan(adapter);
1683
1684 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1685 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1686
1687 igb_get_phy_info(hw);
1688 }
1689
1690 static const struct net_device_ops igb_netdev_ops = {
1691 .ndo_open = igb_open,
1692 .ndo_stop = igb_close,
1693 .ndo_start_xmit = igb_xmit_frame_adv,
1694 .ndo_get_stats64 = igb_get_stats64,
1695 .ndo_set_rx_mode = igb_set_rx_mode,
1696 .ndo_set_multicast_list = igb_set_rx_mode,
1697 .ndo_set_mac_address = igb_set_mac,
1698 .ndo_change_mtu = igb_change_mtu,
1699 .ndo_do_ioctl = igb_ioctl,
1700 .ndo_tx_timeout = igb_tx_timeout,
1701 .ndo_validate_addr = eth_validate_addr,
1702 .ndo_vlan_rx_register = igb_vlan_rx_register,
1703 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1704 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1705 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1706 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1707 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1708 .ndo_get_vf_config = igb_ndo_get_vf_config,
1709 #ifdef CONFIG_NET_POLL_CONTROLLER
1710 .ndo_poll_controller = igb_netpoll,
1711 #endif
1712 };
1713
1714 /**
1715 * igb_probe - Device Initialization Routine
1716 * @pdev: PCI device information struct
1717 * @ent: entry in igb_pci_tbl
1718 *
1719 * Returns 0 on success, negative on failure
1720 *
1721 * igb_probe initializes an adapter identified by a pci_dev structure.
1722 * The OS initialization, configuring of the adapter private structure,
1723 * and a hardware reset occur.
1724 **/
1725 static int __devinit igb_probe(struct pci_dev *pdev,
1726 const struct pci_device_id *ent)
1727 {
1728 struct net_device *netdev;
1729 struct igb_adapter *adapter;
1730 struct e1000_hw *hw;
1731 u16 eeprom_data = 0;
1732 static int global_quad_port_a; /* global quad port a indication */
1733 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1734 unsigned long mmio_start, mmio_len;
1735 int err, pci_using_dac;
1736 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1737 u32 part_num;
1738
1739 /* Catch broken hardware that put the wrong VF device ID in
1740 * the PCIe SR-IOV capability.
1741 */
1742 if (pdev->is_virtfn) {
1743 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1744 pci_name(pdev), pdev->vendor, pdev->device);
1745 return -EINVAL;
1746 }
1747
1748 err = pci_enable_device_mem(pdev);
1749 if (err)
1750 return err;
1751
1752 pci_using_dac = 0;
1753 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1754 if (!err) {
1755 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1756 if (!err)
1757 pci_using_dac = 1;
1758 } else {
1759 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1760 if (err) {
1761 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1762 if (err) {
1763 dev_err(&pdev->dev, "No usable DMA "
1764 "configuration, aborting\n");
1765 goto err_dma;
1766 }
1767 }
1768 }
1769
1770 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1771 IORESOURCE_MEM),
1772 igb_driver_name);
1773 if (err)
1774 goto err_pci_reg;
1775
1776 pci_enable_pcie_error_reporting(pdev);
1777
1778 pci_set_master(pdev);
1779 pci_save_state(pdev);
1780
1781 err = -ENOMEM;
1782 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1783 IGB_ABS_MAX_TX_QUEUES);
1784 if (!netdev)
1785 goto err_alloc_etherdev;
1786
1787 SET_NETDEV_DEV(netdev, &pdev->dev);
1788
1789 pci_set_drvdata(pdev, netdev);
1790 adapter = netdev_priv(netdev);
1791 adapter->netdev = netdev;
1792 adapter->pdev = pdev;
1793 hw = &adapter->hw;
1794 hw->back = adapter;
1795 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1796
1797 mmio_start = pci_resource_start(pdev, 0);
1798 mmio_len = pci_resource_len(pdev, 0);
1799
1800 err = -EIO;
1801 hw->hw_addr = ioremap(mmio_start, mmio_len);
1802 if (!hw->hw_addr)
1803 goto err_ioremap;
1804
1805 netdev->netdev_ops = &igb_netdev_ops;
1806 igb_set_ethtool_ops(netdev);
1807 netdev->watchdog_timeo = 5 * HZ;
1808
1809 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1810
1811 netdev->mem_start = mmio_start;
1812 netdev->mem_end = mmio_start + mmio_len;
1813
1814 /* PCI config space info */
1815 hw->vendor_id = pdev->vendor;
1816 hw->device_id = pdev->device;
1817 hw->revision_id = pdev->revision;
1818 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1819 hw->subsystem_device_id = pdev->subsystem_device;
1820
1821 /* Copy the default MAC, PHY and NVM function pointers */
1822 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1823 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1824 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1825 /* Initialize skew-specific constants */
1826 err = ei->get_invariants(hw);
1827 if (err)
1828 goto err_sw_init;
1829
1830 /* setup the private structure */
1831 err = igb_sw_init(adapter);
1832 if (err)
1833 goto err_sw_init;
1834
1835 igb_get_bus_info_pcie(hw);
1836
1837 hw->phy.autoneg_wait_to_complete = false;
1838
1839 /* Copper options */
1840 if (hw->phy.media_type == e1000_media_type_copper) {
1841 hw->phy.mdix = AUTO_ALL_MODES;
1842 hw->phy.disable_polarity_correction = false;
1843 hw->phy.ms_type = e1000_ms_hw_default;
1844 }
1845
1846 if (igb_check_reset_block(hw))
1847 dev_info(&pdev->dev,
1848 "PHY reset is blocked due to SOL/IDER session.\n");
1849
1850 netdev->features = NETIF_F_SG |
1851 NETIF_F_IP_CSUM |
1852 NETIF_F_HW_VLAN_TX |
1853 NETIF_F_HW_VLAN_RX |
1854 NETIF_F_HW_VLAN_FILTER;
1855
1856 netdev->features |= NETIF_F_IPV6_CSUM;
1857 netdev->features |= NETIF_F_TSO;
1858 netdev->features |= NETIF_F_TSO6;
1859 netdev->features |= NETIF_F_GRO;
1860
1861 netdev->vlan_features |= NETIF_F_TSO;
1862 netdev->vlan_features |= NETIF_F_TSO6;
1863 netdev->vlan_features |= NETIF_F_IP_CSUM;
1864 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1865 netdev->vlan_features |= NETIF_F_SG;
1866
1867 if (pci_using_dac) {
1868 netdev->features |= NETIF_F_HIGHDMA;
1869 netdev->vlan_features |= NETIF_F_HIGHDMA;
1870 }
1871
1872 if (hw->mac.type >= e1000_82576)
1873 netdev->features |= NETIF_F_SCTP_CSUM;
1874
1875 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1876
1877 /* before reading the NVM, reset the controller to put the device in a
1878 * known good starting state */
1879 hw->mac.ops.reset_hw(hw);
1880
1881 /* make sure the NVM is good */
1882 if (igb_validate_nvm_checksum(hw) < 0) {
1883 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1884 err = -EIO;
1885 goto err_eeprom;
1886 }
1887
1888 /* copy the MAC address out of the NVM */
1889 if (hw->mac.ops.read_mac_addr(hw))
1890 dev_err(&pdev->dev, "NVM Read Error\n");
1891
1892 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1893 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1894
1895 if (!is_valid_ether_addr(netdev->perm_addr)) {
1896 dev_err(&pdev->dev, "Invalid MAC Address\n");
1897 err = -EIO;
1898 goto err_eeprom;
1899 }
1900
1901 setup_timer(&adapter->watchdog_timer, igb_watchdog,
1902 (unsigned long) adapter);
1903 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
1904 (unsigned long) adapter);
1905
1906 INIT_WORK(&adapter->reset_task, igb_reset_task);
1907 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1908
1909 /* Initialize link properties that are user-changeable */
1910 adapter->fc_autoneg = true;
1911 hw->mac.autoneg = true;
1912 hw->phy.autoneg_advertised = 0x2f;
1913
1914 hw->fc.requested_mode = e1000_fc_default;
1915 hw->fc.current_mode = e1000_fc_default;
1916
1917 igb_validate_mdi_setting(hw);
1918
1919 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1920 * enable the ACPI Magic Packet filter
1921 */
1922
1923 if (hw->bus.func == 0)
1924 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1925 else if (hw->mac.type == e1000_82580)
1926 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1927 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1928 &eeprom_data);
1929 else if (hw->bus.func == 1)
1930 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1931
1932 if (eeprom_data & eeprom_apme_mask)
1933 adapter->eeprom_wol |= E1000_WUFC_MAG;
1934
1935 /* now that we have the eeprom settings, apply the special cases where
1936 * the eeprom may be wrong or the board simply won't support wake on
1937 * lan on a particular port */
1938 switch (pdev->device) {
1939 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1940 adapter->eeprom_wol = 0;
1941 break;
1942 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1943 case E1000_DEV_ID_82576_FIBER:
1944 case E1000_DEV_ID_82576_SERDES:
1945 /* Wake events only supported on port A for dual fiber
1946 * regardless of eeprom setting */
1947 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1948 adapter->eeprom_wol = 0;
1949 break;
1950 case E1000_DEV_ID_82576_QUAD_COPPER:
1951 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1952 /* if quad port adapter, disable WoL on all but port A */
1953 if (global_quad_port_a != 0)
1954 adapter->eeprom_wol = 0;
1955 else
1956 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1957 /* Reset for multiple quad port adapters */
1958 if (++global_quad_port_a == 4)
1959 global_quad_port_a = 0;
1960 break;
1961 }
1962
1963 /* initialize the wol settings based on the eeprom settings */
1964 adapter->wol = adapter->eeprom_wol;
1965 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1966
1967 /* reset the hardware with the new settings */
1968 igb_reset(adapter);
1969
1970 /* let the f/w know that the h/w is now under the control of the
1971 * driver. */
1972 igb_get_hw_control(adapter);
1973
1974 strcpy(netdev->name, "eth%d");
1975 err = register_netdev(netdev);
1976 if (err)
1977 goto err_register;
1978
1979 /* carrier off reporting is important to ethtool even BEFORE open */
1980 netif_carrier_off(netdev);
1981
1982 #ifdef CONFIG_IGB_DCA
1983 if (dca_add_requester(&pdev->dev) == 0) {
1984 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1985 dev_info(&pdev->dev, "DCA enabled\n");
1986 igb_setup_dca(adapter);
1987 }
1988
1989 #endif
1990 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1991 /* print bus type/speed/width info */
1992 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1993 netdev->name,
1994 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1995 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1996 "unknown"),
1997 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1998 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1999 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2000 "unknown"),
2001 netdev->dev_addr);
2002
2003 igb_read_part_num(hw, &part_num);
2004 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
2005 (part_num >> 8), (part_num & 0xff));
2006
2007 dev_info(&pdev->dev,
2008 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2009 adapter->msix_entries ? "MSI-X" :
2010 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2011 adapter->num_rx_queues, adapter->num_tx_queues);
2012
2013 return 0;
2014
2015 err_register:
2016 igb_release_hw_control(adapter);
2017 err_eeprom:
2018 if (!igb_check_reset_block(hw))
2019 igb_reset_phy(hw);
2020
2021 if (hw->flash_address)
2022 iounmap(hw->flash_address);
2023 err_sw_init:
2024 igb_clear_interrupt_scheme(adapter);
2025 iounmap(hw->hw_addr);
2026 err_ioremap:
2027 free_netdev(netdev);
2028 err_alloc_etherdev:
2029 pci_release_selected_regions(pdev,
2030 pci_select_bars(pdev, IORESOURCE_MEM));
2031 err_pci_reg:
2032 err_dma:
2033 pci_disable_device(pdev);
2034 return err;
2035 }
2036
2037 /**
2038 * igb_remove - Device Removal Routine
2039 * @pdev: PCI device information struct
2040 *
2041 * igb_remove is called by the PCI subsystem to alert the driver
2042 * that it should release a PCI device. The could be caused by a
2043 * Hot-Plug event, or because the driver is going to be removed from
2044 * memory.
2045 **/
2046 static void __devexit igb_remove(struct pci_dev *pdev)
2047 {
2048 struct net_device *netdev = pci_get_drvdata(pdev);
2049 struct igb_adapter *adapter = netdev_priv(netdev);
2050 struct e1000_hw *hw = &adapter->hw;
2051
2052 /* flush_scheduled work may reschedule our watchdog task, so
2053 * explicitly disable watchdog tasks from being rescheduled */
2054 set_bit(__IGB_DOWN, &adapter->state);
2055 del_timer_sync(&adapter->watchdog_timer);
2056 del_timer_sync(&adapter->phy_info_timer);
2057
2058 flush_scheduled_work();
2059
2060 #ifdef CONFIG_IGB_DCA
2061 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2062 dev_info(&pdev->dev, "DCA disabled\n");
2063 dca_remove_requester(&pdev->dev);
2064 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2065 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2066 }
2067 #endif
2068
2069 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2070 * would have already happened in close and is redundant. */
2071 igb_release_hw_control(adapter);
2072
2073 unregister_netdev(netdev);
2074
2075 igb_clear_interrupt_scheme(adapter);
2076
2077 #ifdef CONFIG_PCI_IOV
2078 /* reclaim resources allocated to VFs */
2079 if (adapter->vf_data) {
2080 /* disable iov and allow time for transactions to clear */
2081 pci_disable_sriov(pdev);
2082 msleep(500);
2083
2084 kfree(adapter->vf_data);
2085 adapter->vf_data = NULL;
2086 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2087 msleep(100);
2088 dev_info(&pdev->dev, "IOV Disabled\n");
2089 }
2090 #endif
2091
2092 iounmap(hw->hw_addr);
2093 if (hw->flash_address)
2094 iounmap(hw->flash_address);
2095 pci_release_selected_regions(pdev,
2096 pci_select_bars(pdev, IORESOURCE_MEM));
2097
2098 free_netdev(netdev);
2099
2100 pci_disable_pcie_error_reporting(pdev);
2101
2102 pci_disable_device(pdev);
2103 }
2104
2105 /**
2106 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2107 * @adapter: board private structure to initialize
2108 *
2109 * This function initializes the vf specific data storage and then attempts to
2110 * allocate the VFs. The reason for ordering it this way is because it is much
2111 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2112 * the memory for the VFs.
2113 **/
2114 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2115 {
2116 #ifdef CONFIG_PCI_IOV
2117 struct pci_dev *pdev = adapter->pdev;
2118
2119 if (adapter->vfs_allocated_count) {
2120 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2121 sizeof(struct vf_data_storage),
2122 GFP_KERNEL);
2123 /* if allocation failed then we do not support SR-IOV */
2124 if (!adapter->vf_data) {
2125 adapter->vfs_allocated_count = 0;
2126 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2127 "Data Storage\n");
2128 }
2129 }
2130
2131 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2132 kfree(adapter->vf_data);
2133 adapter->vf_data = NULL;
2134 #endif /* CONFIG_PCI_IOV */
2135 adapter->vfs_allocated_count = 0;
2136 #ifdef CONFIG_PCI_IOV
2137 } else {
2138 unsigned char mac_addr[ETH_ALEN];
2139 int i;
2140 dev_info(&pdev->dev, "%d vfs allocated\n",
2141 adapter->vfs_allocated_count);
2142 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2143 random_ether_addr(mac_addr);
2144 igb_set_vf_mac(adapter, i, mac_addr);
2145 }
2146 }
2147 #endif /* CONFIG_PCI_IOV */
2148 }
2149
2150
2151 /**
2152 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2153 * @adapter: board private structure to initialize
2154 *
2155 * igb_init_hw_timer initializes the function pointer and values for the hw
2156 * timer found in hardware.
2157 **/
2158 static void igb_init_hw_timer(struct igb_adapter *adapter)
2159 {
2160 struct e1000_hw *hw = &adapter->hw;
2161
2162 switch (hw->mac.type) {
2163 case e1000_i350:
2164 case e1000_82580:
2165 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2166 adapter->cycles.read = igb_read_clock;
2167 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2168 adapter->cycles.mult = 1;
2169 /*
2170 * The 82580 timesync updates the system timer every 8ns by 8ns
2171 * and the value cannot be shifted. Instead we need to shift
2172 * the registers to generate a 64bit timer value. As a result
2173 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2174 * 24 in order to generate a larger value for synchronization.
2175 */
2176 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2177 /* disable system timer temporarily by setting bit 31 */
2178 wr32(E1000_TSAUXC, 0x80000000);
2179 wrfl();
2180
2181 /* Set registers so that rollover occurs soon to test this. */
2182 wr32(E1000_SYSTIMR, 0x00000000);
2183 wr32(E1000_SYSTIML, 0x80000000);
2184 wr32(E1000_SYSTIMH, 0x000000FF);
2185 wrfl();
2186
2187 /* enable system timer by clearing bit 31 */
2188 wr32(E1000_TSAUXC, 0x0);
2189 wrfl();
2190
2191 timecounter_init(&adapter->clock,
2192 &adapter->cycles,
2193 ktime_to_ns(ktime_get_real()));
2194 /*
2195 * Synchronize our NIC clock against system wall clock. NIC
2196 * time stamp reading requires ~3us per sample, each sample
2197 * was pretty stable even under load => only require 10
2198 * samples for each offset comparison.
2199 */
2200 memset(&adapter->compare, 0, sizeof(adapter->compare));
2201 adapter->compare.source = &adapter->clock;
2202 adapter->compare.target = ktime_get_real;
2203 adapter->compare.num_samples = 10;
2204 timecompare_update(&adapter->compare, 0);
2205 break;
2206 case e1000_82576:
2207 /*
2208 * Initialize hardware timer: we keep it running just in case
2209 * that some program needs it later on.
2210 */
2211 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2212 adapter->cycles.read = igb_read_clock;
2213 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2214 adapter->cycles.mult = 1;
2215 /**
2216 * Scale the NIC clock cycle by a large factor so that
2217 * relatively small clock corrections can be added or
2218 * substracted at each clock tick. The drawbacks of a large
2219 * factor are a) that the clock register overflows more quickly
2220 * (not such a big deal) and b) that the increment per tick has
2221 * to fit into 24 bits. As a result we need to use a shift of
2222 * 19 so we can fit a value of 16 into the TIMINCA register.
2223 */
2224 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2225 wr32(E1000_TIMINCA,
2226 (1 << E1000_TIMINCA_16NS_SHIFT) |
2227 (16 << IGB_82576_TSYNC_SHIFT));
2228
2229 /* Set registers so that rollover occurs soon to test this. */
2230 wr32(E1000_SYSTIML, 0x00000000);
2231 wr32(E1000_SYSTIMH, 0xFF800000);
2232 wrfl();
2233
2234 timecounter_init(&adapter->clock,
2235 &adapter->cycles,
2236 ktime_to_ns(ktime_get_real()));
2237 /*
2238 * Synchronize our NIC clock against system wall clock. NIC
2239 * time stamp reading requires ~3us per sample, each sample
2240 * was pretty stable even under load => only require 10
2241 * samples for each offset comparison.
2242 */
2243 memset(&adapter->compare, 0, sizeof(adapter->compare));
2244 adapter->compare.source = &adapter->clock;
2245 adapter->compare.target = ktime_get_real;
2246 adapter->compare.num_samples = 10;
2247 timecompare_update(&adapter->compare, 0);
2248 break;
2249 case e1000_82575:
2250 /* 82575 does not support timesync */
2251 default:
2252 break;
2253 }
2254
2255 }
2256
2257 /**
2258 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2259 * @adapter: board private structure to initialize
2260 *
2261 * igb_sw_init initializes the Adapter private data structure.
2262 * Fields are initialized based on PCI device information and
2263 * OS network device settings (MTU size).
2264 **/
2265 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2266 {
2267 struct e1000_hw *hw = &adapter->hw;
2268 struct net_device *netdev = adapter->netdev;
2269 struct pci_dev *pdev = adapter->pdev;
2270
2271 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2272
2273 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2274 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2275 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2276 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2277
2278 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2279 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2280
2281 spin_lock_init(&adapter->stats64_lock);
2282 #ifdef CONFIG_PCI_IOV
2283 if (hw->mac.type == e1000_82576)
2284 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
2285
2286 #endif /* CONFIG_PCI_IOV */
2287 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2288
2289 /*
2290 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2291 * then we should combine the queues into a queue pair in order to
2292 * conserve interrupts due to limited supply
2293 */
2294 if ((adapter->rss_queues > 4) ||
2295 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2296 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2297
2298 /* This call may decrease the number of queues */
2299 if (igb_init_interrupt_scheme(adapter)) {
2300 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2301 return -ENOMEM;
2302 }
2303
2304 igb_init_hw_timer(adapter);
2305 igb_probe_vfs(adapter);
2306
2307 /* Explicitly disable IRQ since the NIC can be in any state. */
2308 igb_irq_disable(adapter);
2309
2310 set_bit(__IGB_DOWN, &adapter->state);
2311 return 0;
2312 }
2313
2314 /**
2315 * igb_open - Called when a network interface is made active
2316 * @netdev: network interface device structure
2317 *
2318 * Returns 0 on success, negative value on failure
2319 *
2320 * The open entry point is called when a network interface is made
2321 * active by the system (IFF_UP). At this point all resources needed
2322 * for transmit and receive operations are allocated, the interrupt
2323 * handler is registered with the OS, the watchdog timer is started,
2324 * and the stack is notified that the interface is ready.
2325 **/
2326 static int igb_open(struct net_device *netdev)
2327 {
2328 struct igb_adapter *adapter = netdev_priv(netdev);
2329 struct e1000_hw *hw = &adapter->hw;
2330 int err;
2331 int i;
2332
2333 /* disallow open during test */
2334 if (test_bit(__IGB_TESTING, &adapter->state))
2335 return -EBUSY;
2336
2337 netif_carrier_off(netdev);
2338
2339 /* allocate transmit descriptors */
2340 err = igb_setup_all_tx_resources(adapter);
2341 if (err)
2342 goto err_setup_tx;
2343
2344 /* allocate receive descriptors */
2345 err = igb_setup_all_rx_resources(adapter);
2346 if (err)
2347 goto err_setup_rx;
2348
2349 igb_power_up_link(adapter);
2350
2351 /* before we allocate an interrupt, we must be ready to handle it.
2352 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2353 * as soon as we call pci_request_irq, so we have to setup our
2354 * clean_rx handler before we do so. */
2355 igb_configure(adapter);
2356
2357 err = igb_request_irq(adapter);
2358 if (err)
2359 goto err_req_irq;
2360
2361 /* From here on the code is the same as igb_up() */
2362 clear_bit(__IGB_DOWN, &adapter->state);
2363
2364 for (i = 0; i < adapter->num_q_vectors; i++) {
2365 struct igb_q_vector *q_vector = adapter->q_vector[i];
2366 napi_enable(&q_vector->napi);
2367 }
2368
2369 /* Clear any pending interrupts. */
2370 rd32(E1000_ICR);
2371
2372 igb_irq_enable(adapter);
2373
2374 /* notify VFs that reset has been completed */
2375 if (adapter->vfs_allocated_count) {
2376 u32 reg_data = rd32(E1000_CTRL_EXT);
2377 reg_data |= E1000_CTRL_EXT_PFRSTD;
2378 wr32(E1000_CTRL_EXT, reg_data);
2379 }
2380
2381 netif_tx_start_all_queues(netdev);
2382
2383 /* start the watchdog. */
2384 hw->mac.get_link_status = 1;
2385 schedule_work(&adapter->watchdog_task);
2386
2387 return 0;
2388
2389 err_req_irq:
2390 igb_release_hw_control(adapter);
2391 igb_power_down_link(adapter);
2392 igb_free_all_rx_resources(adapter);
2393 err_setup_rx:
2394 igb_free_all_tx_resources(adapter);
2395 err_setup_tx:
2396 igb_reset(adapter);
2397
2398 return err;
2399 }
2400
2401 /**
2402 * igb_close - Disables a network interface
2403 * @netdev: network interface device structure
2404 *
2405 * Returns 0, this is not allowed to fail
2406 *
2407 * The close entry point is called when an interface is de-activated
2408 * by the OS. The hardware is still under the driver's control, but
2409 * needs to be disabled. A global MAC reset is issued to stop the
2410 * hardware, and all transmit and receive resources are freed.
2411 **/
2412 static int igb_close(struct net_device *netdev)
2413 {
2414 struct igb_adapter *adapter = netdev_priv(netdev);
2415
2416 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2417 igb_down(adapter);
2418
2419 igb_free_irq(adapter);
2420
2421 igb_free_all_tx_resources(adapter);
2422 igb_free_all_rx_resources(adapter);
2423
2424 return 0;
2425 }
2426
2427 /**
2428 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2429 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2430 *
2431 * Return 0 on success, negative on failure
2432 **/
2433 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2434 {
2435 struct device *dev = tx_ring->dev;
2436 int size;
2437
2438 size = sizeof(struct igb_buffer) * tx_ring->count;
2439 tx_ring->buffer_info = vmalloc(size);
2440 if (!tx_ring->buffer_info)
2441 goto err;
2442 memset(tx_ring->buffer_info, 0, size);
2443
2444 /* round up to nearest 4K */
2445 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2446 tx_ring->size = ALIGN(tx_ring->size, 4096);
2447
2448 tx_ring->desc = dma_alloc_coherent(dev,
2449 tx_ring->size,
2450 &tx_ring->dma,
2451 GFP_KERNEL);
2452
2453 if (!tx_ring->desc)
2454 goto err;
2455
2456 tx_ring->next_to_use = 0;
2457 tx_ring->next_to_clean = 0;
2458 return 0;
2459
2460 err:
2461 vfree(tx_ring->buffer_info);
2462 dev_err(dev,
2463 "Unable to allocate memory for the transmit descriptor ring\n");
2464 return -ENOMEM;
2465 }
2466
2467 /**
2468 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2469 * (Descriptors) for all queues
2470 * @adapter: board private structure
2471 *
2472 * Return 0 on success, negative on failure
2473 **/
2474 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2475 {
2476 struct pci_dev *pdev = adapter->pdev;
2477 int i, err = 0;
2478
2479 for (i = 0; i < adapter->num_tx_queues; i++) {
2480 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2481 if (err) {
2482 dev_err(&pdev->dev,
2483 "Allocation for Tx Queue %u failed\n", i);
2484 for (i--; i >= 0; i--)
2485 igb_free_tx_resources(adapter->tx_ring[i]);
2486 break;
2487 }
2488 }
2489
2490 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2491 int r_idx = i % adapter->num_tx_queues;
2492 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2493 }
2494 return err;
2495 }
2496
2497 /**
2498 * igb_setup_tctl - configure the transmit control registers
2499 * @adapter: Board private structure
2500 **/
2501 void igb_setup_tctl(struct igb_adapter *adapter)
2502 {
2503 struct e1000_hw *hw = &adapter->hw;
2504 u32 tctl;
2505
2506 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2507 wr32(E1000_TXDCTL(0), 0);
2508
2509 /* Program the Transmit Control Register */
2510 tctl = rd32(E1000_TCTL);
2511 tctl &= ~E1000_TCTL_CT;
2512 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2513 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2514
2515 igb_config_collision_dist(hw);
2516
2517 /* Enable transmits */
2518 tctl |= E1000_TCTL_EN;
2519
2520 wr32(E1000_TCTL, tctl);
2521 }
2522
2523 /**
2524 * igb_configure_tx_ring - Configure transmit ring after Reset
2525 * @adapter: board private structure
2526 * @ring: tx ring to configure
2527 *
2528 * Configure a transmit ring after a reset.
2529 **/
2530 void igb_configure_tx_ring(struct igb_adapter *adapter,
2531 struct igb_ring *ring)
2532 {
2533 struct e1000_hw *hw = &adapter->hw;
2534 u32 txdctl;
2535 u64 tdba = ring->dma;
2536 int reg_idx = ring->reg_idx;
2537
2538 /* disable the queue */
2539 txdctl = rd32(E1000_TXDCTL(reg_idx));
2540 wr32(E1000_TXDCTL(reg_idx),
2541 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2542 wrfl();
2543 mdelay(10);
2544
2545 wr32(E1000_TDLEN(reg_idx),
2546 ring->count * sizeof(union e1000_adv_tx_desc));
2547 wr32(E1000_TDBAL(reg_idx),
2548 tdba & 0x00000000ffffffffULL);
2549 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2550
2551 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2552 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2553 writel(0, ring->head);
2554 writel(0, ring->tail);
2555
2556 txdctl |= IGB_TX_PTHRESH;
2557 txdctl |= IGB_TX_HTHRESH << 8;
2558 txdctl |= IGB_TX_WTHRESH << 16;
2559
2560 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2561 wr32(E1000_TXDCTL(reg_idx), txdctl);
2562 }
2563
2564 /**
2565 * igb_configure_tx - Configure transmit Unit after Reset
2566 * @adapter: board private structure
2567 *
2568 * Configure the Tx unit of the MAC after a reset.
2569 **/
2570 static void igb_configure_tx(struct igb_adapter *adapter)
2571 {
2572 int i;
2573
2574 for (i = 0; i < adapter->num_tx_queues; i++)
2575 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2576 }
2577
2578 /**
2579 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2580 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2581 *
2582 * Returns 0 on success, negative on failure
2583 **/
2584 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2585 {
2586 struct device *dev = rx_ring->dev;
2587 int size, desc_len;
2588
2589 size = sizeof(struct igb_buffer) * rx_ring->count;
2590 rx_ring->buffer_info = vmalloc(size);
2591 if (!rx_ring->buffer_info)
2592 goto err;
2593 memset(rx_ring->buffer_info, 0, size);
2594
2595 desc_len = sizeof(union e1000_adv_rx_desc);
2596
2597 /* Round up to nearest 4K */
2598 rx_ring->size = rx_ring->count * desc_len;
2599 rx_ring->size = ALIGN(rx_ring->size, 4096);
2600
2601 rx_ring->desc = dma_alloc_coherent(dev,
2602 rx_ring->size,
2603 &rx_ring->dma,
2604 GFP_KERNEL);
2605
2606 if (!rx_ring->desc)
2607 goto err;
2608
2609 rx_ring->next_to_clean = 0;
2610 rx_ring->next_to_use = 0;
2611
2612 return 0;
2613
2614 err:
2615 vfree(rx_ring->buffer_info);
2616 rx_ring->buffer_info = NULL;
2617 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2618 " ring\n");
2619 return -ENOMEM;
2620 }
2621
2622 /**
2623 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2624 * (Descriptors) for all queues
2625 * @adapter: board private structure
2626 *
2627 * Return 0 on success, negative on failure
2628 **/
2629 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2630 {
2631 struct pci_dev *pdev = adapter->pdev;
2632 int i, err = 0;
2633
2634 for (i = 0; i < adapter->num_rx_queues; i++) {
2635 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2636 if (err) {
2637 dev_err(&pdev->dev,
2638 "Allocation for Rx Queue %u failed\n", i);
2639 for (i--; i >= 0; i--)
2640 igb_free_rx_resources(adapter->rx_ring[i]);
2641 break;
2642 }
2643 }
2644
2645 return err;
2646 }
2647
2648 /**
2649 * igb_setup_mrqc - configure the multiple receive queue control registers
2650 * @adapter: Board private structure
2651 **/
2652 static void igb_setup_mrqc(struct igb_adapter *adapter)
2653 {
2654 struct e1000_hw *hw = &adapter->hw;
2655 u32 mrqc, rxcsum;
2656 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2657 union e1000_reta {
2658 u32 dword;
2659 u8 bytes[4];
2660 } reta;
2661 static const u8 rsshash[40] = {
2662 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2663 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2664 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2665 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2666
2667 /* Fill out hash function seeds */
2668 for (j = 0; j < 10; j++) {
2669 u32 rsskey = rsshash[(j * 4)];
2670 rsskey |= rsshash[(j * 4) + 1] << 8;
2671 rsskey |= rsshash[(j * 4) + 2] << 16;
2672 rsskey |= rsshash[(j * 4) + 3] << 24;
2673 array_wr32(E1000_RSSRK(0), j, rsskey);
2674 }
2675
2676 num_rx_queues = adapter->rss_queues;
2677
2678 if (adapter->vfs_allocated_count) {
2679 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2680 switch (hw->mac.type) {
2681 case e1000_i350:
2682 case e1000_82580:
2683 num_rx_queues = 1;
2684 shift = 0;
2685 break;
2686 case e1000_82576:
2687 shift = 3;
2688 num_rx_queues = 2;
2689 break;
2690 case e1000_82575:
2691 shift = 2;
2692 shift2 = 6;
2693 default:
2694 break;
2695 }
2696 } else {
2697 if (hw->mac.type == e1000_82575)
2698 shift = 6;
2699 }
2700
2701 for (j = 0; j < (32 * 4); j++) {
2702 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2703 if (shift2)
2704 reta.bytes[j & 3] |= num_rx_queues << shift2;
2705 if ((j & 3) == 3)
2706 wr32(E1000_RETA(j >> 2), reta.dword);
2707 }
2708
2709 /*
2710 * Disable raw packet checksumming so that RSS hash is placed in
2711 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2712 * offloads as they are enabled by default
2713 */
2714 rxcsum = rd32(E1000_RXCSUM);
2715 rxcsum |= E1000_RXCSUM_PCSD;
2716
2717 if (adapter->hw.mac.type >= e1000_82576)
2718 /* Enable Receive Checksum Offload for SCTP */
2719 rxcsum |= E1000_RXCSUM_CRCOFL;
2720
2721 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2722 wr32(E1000_RXCSUM, rxcsum);
2723
2724 /* If VMDq is enabled then we set the appropriate mode for that, else
2725 * we default to RSS so that an RSS hash is calculated per packet even
2726 * if we are only using one queue */
2727 if (adapter->vfs_allocated_count) {
2728 if (hw->mac.type > e1000_82575) {
2729 /* Set the default pool for the PF's first queue */
2730 u32 vtctl = rd32(E1000_VT_CTL);
2731 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2732 E1000_VT_CTL_DISABLE_DEF_POOL);
2733 vtctl |= adapter->vfs_allocated_count <<
2734 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2735 wr32(E1000_VT_CTL, vtctl);
2736 }
2737 if (adapter->rss_queues > 1)
2738 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2739 else
2740 mrqc = E1000_MRQC_ENABLE_VMDQ;
2741 } else {
2742 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2743 }
2744 igb_vmm_control(adapter);
2745
2746 /*
2747 * Generate RSS hash based on TCP port numbers and/or
2748 * IPv4/v6 src and dst addresses since UDP cannot be
2749 * hashed reliably due to IP fragmentation
2750 */
2751 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2752 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2753 E1000_MRQC_RSS_FIELD_IPV6 |
2754 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2755 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2756
2757 wr32(E1000_MRQC, mrqc);
2758 }
2759
2760 /**
2761 * igb_setup_rctl - configure the receive control registers
2762 * @adapter: Board private structure
2763 **/
2764 void igb_setup_rctl(struct igb_adapter *adapter)
2765 {
2766 struct e1000_hw *hw = &adapter->hw;
2767 u32 rctl;
2768
2769 rctl = rd32(E1000_RCTL);
2770
2771 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2772 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2773
2774 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2775 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2776
2777 /*
2778 * enable stripping of CRC. It's unlikely this will break BMC
2779 * redirection as it did with e1000. Newer features require
2780 * that the HW strips the CRC.
2781 */
2782 rctl |= E1000_RCTL_SECRC;
2783
2784 /* disable store bad packets and clear size bits. */
2785 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2786
2787 /* enable LPE to prevent packets larger than max_frame_size */
2788 rctl |= E1000_RCTL_LPE;
2789
2790 /* disable queue 0 to prevent tail write w/o re-config */
2791 wr32(E1000_RXDCTL(0), 0);
2792
2793 /* Attention!!! For SR-IOV PF driver operations you must enable
2794 * queue drop for all VF and PF queues to prevent head of line blocking
2795 * if an un-trusted VF does not provide descriptors to hardware.
2796 */
2797 if (adapter->vfs_allocated_count) {
2798 /* set all queue drop enable bits */
2799 wr32(E1000_QDE, ALL_QUEUES);
2800 }
2801
2802 wr32(E1000_RCTL, rctl);
2803 }
2804
2805 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2806 int vfn)
2807 {
2808 struct e1000_hw *hw = &adapter->hw;
2809 u32 vmolr;
2810
2811 /* if it isn't the PF check to see if VFs are enabled and
2812 * increase the size to support vlan tags */
2813 if (vfn < adapter->vfs_allocated_count &&
2814 adapter->vf_data[vfn].vlans_enabled)
2815 size += VLAN_TAG_SIZE;
2816
2817 vmolr = rd32(E1000_VMOLR(vfn));
2818 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2819 vmolr |= size | E1000_VMOLR_LPE;
2820 wr32(E1000_VMOLR(vfn), vmolr);
2821
2822 return 0;
2823 }
2824
2825 /**
2826 * igb_rlpml_set - set maximum receive packet size
2827 * @adapter: board private structure
2828 *
2829 * Configure maximum receivable packet size.
2830 **/
2831 static void igb_rlpml_set(struct igb_adapter *adapter)
2832 {
2833 u32 max_frame_size = adapter->max_frame_size;
2834 struct e1000_hw *hw = &adapter->hw;
2835 u16 pf_id = adapter->vfs_allocated_count;
2836
2837 if (adapter->vlgrp)
2838 max_frame_size += VLAN_TAG_SIZE;
2839
2840 /* if vfs are enabled we set RLPML to the largest possible request
2841 * size and set the VMOLR RLPML to the size we need */
2842 if (pf_id) {
2843 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2844 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2845 }
2846
2847 wr32(E1000_RLPML, max_frame_size);
2848 }
2849
2850 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2851 int vfn, bool aupe)
2852 {
2853 struct e1000_hw *hw = &adapter->hw;
2854 u32 vmolr;
2855
2856 /*
2857 * This register exists only on 82576 and newer so if we are older then
2858 * we should exit and do nothing
2859 */
2860 if (hw->mac.type < e1000_82576)
2861 return;
2862
2863 vmolr = rd32(E1000_VMOLR(vfn));
2864 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2865 if (aupe)
2866 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2867 else
2868 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2869
2870 /* clear all bits that might not be set */
2871 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2872
2873 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2874 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2875 /*
2876 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2877 * multicast packets
2878 */
2879 if (vfn <= adapter->vfs_allocated_count)
2880 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2881
2882 wr32(E1000_VMOLR(vfn), vmolr);
2883 }
2884
2885 /**
2886 * igb_configure_rx_ring - Configure a receive ring after Reset
2887 * @adapter: board private structure
2888 * @ring: receive ring to be configured
2889 *
2890 * Configure the Rx unit of the MAC after a reset.
2891 **/
2892 void igb_configure_rx_ring(struct igb_adapter *adapter,
2893 struct igb_ring *ring)
2894 {
2895 struct e1000_hw *hw = &adapter->hw;
2896 u64 rdba = ring->dma;
2897 int reg_idx = ring->reg_idx;
2898 u32 srrctl, rxdctl;
2899
2900 /* disable the queue */
2901 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2902 wr32(E1000_RXDCTL(reg_idx),
2903 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2904
2905 /* Set DMA base address registers */
2906 wr32(E1000_RDBAL(reg_idx),
2907 rdba & 0x00000000ffffffffULL);
2908 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2909 wr32(E1000_RDLEN(reg_idx),
2910 ring->count * sizeof(union e1000_adv_rx_desc));
2911
2912 /* initialize head and tail */
2913 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2914 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2915 writel(0, ring->head);
2916 writel(0, ring->tail);
2917
2918 /* set descriptor configuration */
2919 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2920 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2921 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2922 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2923 srrctl |= IGB_RXBUFFER_16384 >>
2924 E1000_SRRCTL_BSIZEPKT_SHIFT;
2925 #else
2926 srrctl |= (PAGE_SIZE / 2) >>
2927 E1000_SRRCTL_BSIZEPKT_SHIFT;
2928 #endif
2929 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2930 } else {
2931 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2932 E1000_SRRCTL_BSIZEPKT_SHIFT;
2933 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2934 }
2935 if (hw->mac.type == e1000_82580)
2936 srrctl |= E1000_SRRCTL_TIMESTAMP;
2937 /* Only set Drop Enable if we are supporting multiple queues */
2938 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2939 srrctl |= E1000_SRRCTL_DROP_EN;
2940
2941 wr32(E1000_SRRCTL(reg_idx), srrctl);
2942
2943 /* set filtering for VMDQ pools */
2944 igb_set_vmolr(adapter, reg_idx & 0x7, true);
2945
2946 /* enable receive descriptor fetching */
2947 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2948 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2949 rxdctl &= 0xFFF00000;
2950 rxdctl |= IGB_RX_PTHRESH;
2951 rxdctl |= IGB_RX_HTHRESH << 8;
2952 rxdctl |= IGB_RX_WTHRESH << 16;
2953 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2954 }
2955
2956 /**
2957 * igb_configure_rx - Configure receive Unit after Reset
2958 * @adapter: board private structure
2959 *
2960 * Configure the Rx unit of the MAC after a reset.
2961 **/
2962 static void igb_configure_rx(struct igb_adapter *adapter)
2963 {
2964 int i;
2965
2966 /* set UTA to appropriate mode */
2967 igb_set_uta(adapter);
2968
2969 /* set the correct pool for the PF default MAC address in entry 0 */
2970 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2971 adapter->vfs_allocated_count);
2972
2973 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2974 * the Base and Length of the Rx Descriptor Ring */
2975 for (i = 0; i < adapter->num_rx_queues; i++)
2976 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2977 }
2978
2979 /**
2980 * igb_free_tx_resources - Free Tx Resources per Queue
2981 * @tx_ring: Tx descriptor ring for a specific queue
2982 *
2983 * Free all transmit software resources
2984 **/
2985 void igb_free_tx_resources(struct igb_ring *tx_ring)
2986 {
2987 igb_clean_tx_ring(tx_ring);
2988
2989 vfree(tx_ring->buffer_info);
2990 tx_ring->buffer_info = NULL;
2991
2992 /* if not set, then don't free */
2993 if (!tx_ring->desc)
2994 return;
2995
2996 dma_free_coherent(tx_ring->dev, tx_ring->size,
2997 tx_ring->desc, tx_ring->dma);
2998
2999 tx_ring->desc = NULL;
3000 }
3001
3002 /**
3003 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3004 * @adapter: board private structure
3005 *
3006 * Free all transmit software resources
3007 **/
3008 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3009 {
3010 int i;
3011
3012 for (i = 0; i < adapter->num_tx_queues; i++)
3013 igb_free_tx_resources(adapter->tx_ring[i]);
3014 }
3015
3016 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3017 struct igb_buffer *buffer_info)
3018 {
3019 if (buffer_info->dma) {
3020 if (buffer_info->mapped_as_page)
3021 dma_unmap_page(tx_ring->dev,
3022 buffer_info->dma,
3023 buffer_info->length,
3024 DMA_TO_DEVICE);
3025 else
3026 dma_unmap_single(tx_ring->dev,
3027 buffer_info->dma,
3028 buffer_info->length,
3029 DMA_TO_DEVICE);
3030 buffer_info->dma = 0;
3031 }
3032 if (buffer_info->skb) {
3033 dev_kfree_skb_any(buffer_info->skb);
3034 buffer_info->skb = NULL;
3035 }
3036 buffer_info->time_stamp = 0;
3037 buffer_info->length = 0;
3038 buffer_info->next_to_watch = 0;
3039 buffer_info->mapped_as_page = false;
3040 }
3041
3042 /**
3043 * igb_clean_tx_ring - Free Tx Buffers
3044 * @tx_ring: ring to be cleaned
3045 **/
3046 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3047 {
3048 struct igb_buffer *buffer_info;
3049 unsigned long size;
3050 unsigned int i;
3051
3052 if (!tx_ring->buffer_info)
3053 return;
3054 /* Free all the Tx ring sk_buffs */
3055
3056 for (i = 0; i < tx_ring->count; i++) {
3057 buffer_info = &tx_ring->buffer_info[i];
3058 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3059 }
3060
3061 size = sizeof(struct igb_buffer) * tx_ring->count;
3062 memset(tx_ring->buffer_info, 0, size);
3063
3064 /* Zero out the descriptor ring */
3065 memset(tx_ring->desc, 0, tx_ring->size);
3066
3067 tx_ring->next_to_use = 0;
3068 tx_ring->next_to_clean = 0;
3069 }
3070
3071 /**
3072 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3073 * @adapter: board private structure
3074 **/
3075 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3076 {
3077 int i;
3078
3079 for (i = 0; i < adapter->num_tx_queues; i++)
3080 igb_clean_tx_ring(adapter->tx_ring[i]);
3081 }
3082
3083 /**
3084 * igb_free_rx_resources - Free Rx Resources
3085 * @rx_ring: ring to clean the resources from
3086 *
3087 * Free all receive software resources
3088 **/
3089 void igb_free_rx_resources(struct igb_ring *rx_ring)
3090 {
3091 igb_clean_rx_ring(rx_ring);
3092
3093 vfree(rx_ring->buffer_info);
3094 rx_ring->buffer_info = NULL;
3095
3096 /* if not set, then don't free */
3097 if (!rx_ring->desc)
3098 return;
3099
3100 dma_free_coherent(rx_ring->dev, rx_ring->size,
3101 rx_ring->desc, rx_ring->dma);
3102
3103 rx_ring->desc = NULL;
3104 }
3105
3106 /**
3107 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3108 * @adapter: board private structure
3109 *
3110 * Free all receive software resources
3111 **/
3112 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3113 {
3114 int i;
3115
3116 for (i = 0; i < adapter->num_rx_queues; i++)
3117 igb_free_rx_resources(adapter->rx_ring[i]);
3118 }
3119
3120 /**
3121 * igb_clean_rx_ring - Free Rx Buffers per Queue
3122 * @rx_ring: ring to free buffers from
3123 **/
3124 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3125 {
3126 struct igb_buffer *buffer_info;
3127 unsigned long size;
3128 unsigned int i;
3129
3130 if (!rx_ring->buffer_info)
3131 return;
3132
3133 /* Free all the Rx ring sk_buffs */
3134 for (i = 0; i < rx_ring->count; i++) {
3135 buffer_info = &rx_ring->buffer_info[i];
3136 if (buffer_info->dma) {
3137 dma_unmap_single(rx_ring->dev,
3138 buffer_info->dma,
3139 rx_ring->rx_buffer_len,
3140 DMA_FROM_DEVICE);
3141 buffer_info->dma = 0;
3142 }
3143
3144 if (buffer_info->skb) {
3145 dev_kfree_skb(buffer_info->skb);
3146 buffer_info->skb = NULL;
3147 }
3148 if (buffer_info->page_dma) {
3149 dma_unmap_page(rx_ring->dev,
3150 buffer_info->page_dma,
3151 PAGE_SIZE / 2,
3152 DMA_FROM_DEVICE);
3153 buffer_info->page_dma = 0;
3154 }
3155 if (buffer_info->page) {
3156 put_page(buffer_info->page);
3157 buffer_info->page = NULL;
3158 buffer_info->page_offset = 0;
3159 }
3160 }
3161
3162 size = sizeof(struct igb_buffer) * rx_ring->count;
3163 memset(rx_ring->buffer_info, 0, size);
3164
3165 /* Zero out the descriptor ring */
3166 memset(rx_ring->desc, 0, rx_ring->size);
3167
3168 rx_ring->next_to_clean = 0;
3169 rx_ring->next_to_use = 0;
3170 }
3171
3172 /**
3173 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3174 * @adapter: board private structure
3175 **/
3176 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3177 {
3178 int i;
3179
3180 for (i = 0; i < adapter->num_rx_queues; i++)
3181 igb_clean_rx_ring(adapter->rx_ring[i]);
3182 }
3183
3184 /**
3185 * igb_set_mac - Change the Ethernet Address of the NIC
3186 * @netdev: network interface device structure
3187 * @p: pointer to an address structure
3188 *
3189 * Returns 0 on success, negative on failure
3190 **/
3191 static int igb_set_mac(struct net_device *netdev, void *p)
3192 {
3193 struct igb_adapter *adapter = netdev_priv(netdev);
3194 struct e1000_hw *hw = &adapter->hw;
3195 struct sockaddr *addr = p;
3196
3197 if (!is_valid_ether_addr(addr->sa_data))
3198 return -EADDRNOTAVAIL;
3199
3200 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3201 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3202
3203 /* set the correct pool for the new PF MAC address in entry 0 */
3204 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3205 adapter->vfs_allocated_count);
3206
3207 return 0;
3208 }
3209
3210 /**
3211 * igb_write_mc_addr_list - write multicast addresses to MTA
3212 * @netdev: network interface device structure
3213 *
3214 * Writes multicast address list to the MTA hash table.
3215 * Returns: -ENOMEM on failure
3216 * 0 on no addresses written
3217 * X on writing X addresses to MTA
3218 **/
3219 static int igb_write_mc_addr_list(struct net_device *netdev)
3220 {
3221 struct igb_adapter *adapter = netdev_priv(netdev);
3222 struct e1000_hw *hw = &adapter->hw;
3223 struct netdev_hw_addr *ha;
3224 u8 *mta_list;
3225 int i;
3226
3227 if (netdev_mc_empty(netdev)) {
3228 /* nothing to program, so clear mc list */
3229 igb_update_mc_addr_list(hw, NULL, 0);
3230 igb_restore_vf_multicasts(adapter);
3231 return 0;
3232 }
3233
3234 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3235 if (!mta_list)
3236 return -ENOMEM;
3237
3238 /* The shared function expects a packed array of only addresses. */
3239 i = 0;
3240 netdev_for_each_mc_addr(ha, netdev)
3241 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3242
3243 igb_update_mc_addr_list(hw, mta_list, i);
3244 kfree(mta_list);
3245
3246 return netdev_mc_count(netdev);
3247 }
3248
3249 /**
3250 * igb_write_uc_addr_list - write unicast addresses to RAR table
3251 * @netdev: network interface device structure
3252 *
3253 * Writes unicast address list to the RAR table.
3254 * Returns: -ENOMEM on failure/insufficient address space
3255 * 0 on no addresses written
3256 * X on writing X addresses to the RAR table
3257 **/
3258 static int igb_write_uc_addr_list(struct net_device *netdev)
3259 {
3260 struct igb_adapter *adapter = netdev_priv(netdev);
3261 struct e1000_hw *hw = &adapter->hw;
3262 unsigned int vfn = adapter->vfs_allocated_count;
3263 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3264 int count = 0;
3265
3266 /* return ENOMEM indicating insufficient memory for addresses */
3267 if (netdev_uc_count(netdev) > rar_entries)
3268 return -ENOMEM;
3269
3270 if (!netdev_uc_empty(netdev) && rar_entries) {
3271 struct netdev_hw_addr *ha;
3272
3273 netdev_for_each_uc_addr(ha, netdev) {
3274 if (!rar_entries)
3275 break;
3276 igb_rar_set_qsel(adapter, ha->addr,
3277 rar_entries--,
3278 vfn);
3279 count++;
3280 }
3281 }
3282 /* write the addresses in reverse order to avoid write combining */
3283 for (; rar_entries > 0 ; rar_entries--) {
3284 wr32(E1000_RAH(rar_entries), 0);
3285 wr32(E1000_RAL(rar_entries), 0);
3286 }
3287 wrfl();
3288
3289 return count;
3290 }
3291
3292 /**
3293 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3294 * @netdev: network interface device structure
3295 *
3296 * The set_rx_mode entry point is called whenever the unicast or multicast
3297 * address lists or the network interface flags are updated. This routine is
3298 * responsible for configuring the hardware for proper unicast, multicast,
3299 * promiscuous mode, and all-multi behavior.
3300 **/
3301 static void igb_set_rx_mode(struct net_device *netdev)
3302 {
3303 struct igb_adapter *adapter = netdev_priv(netdev);
3304 struct e1000_hw *hw = &adapter->hw;
3305 unsigned int vfn = adapter->vfs_allocated_count;
3306 u32 rctl, vmolr = 0;
3307 int count;
3308
3309 /* Check for Promiscuous and All Multicast modes */
3310 rctl = rd32(E1000_RCTL);
3311
3312 /* clear the effected bits */
3313 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3314
3315 if (netdev->flags & IFF_PROMISC) {
3316 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3317 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3318 } else {
3319 if (netdev->flags & IFF_ALLMULTI) {
3320 rctl |= E1000_RCTL_MPE;
3321 vmolr |= E1000_VMOLR_MPME;
3322 } else {
3323 /*
3324 * Write addresses to the MTA, if the attempt fails
3325 * then we should just turn on promiscous mode so
3326 * that we can at least receive multicast traffic
3327 */
3328 count = igb_write_mc_addr_list(netdev);
3329 if (count < 0) {
3330 rctl |= E1000_RCTL_MPE;
3331 vmolr |= E1000_VMOLR_MPME;
3332 } else if (count) {
3333 vmolr |= E1000_VMOLR_ROMPE;
3334 }
3335 }
3336 /*
3337 * Write addresses to available RAR registers, if there is not
3338 * sufficient space to store all the addresses then enable
3339 * unicast promiscous mode
3340 */
3341 count = igb_write_uc_addr_list(netdev);
3342 if (count < 0) {
3343 rctl |= E1000_RCTL_UPE;
3344 vmolr |= E1000_VMOLR_ROPE;
3345 }
3346 rctl |= E1000_RCTL_VFE;
3347 }
3348 wr32(E1000_RCTL, rctl);
3349
3350 /*
3351 * In order to support SR-IOV and eventually VMDq it is necessary to set
3352 * the VMOLR to enable the appropriate modes. Without this workaround
3353 * we will have issues with VLAN tag stripping not being done for frames
3354 * that are only arriving because we are the default pool
3355 */
3356 if (hw->mac.type < e1000_82576)
3357 return;
3358
3359 vmolr |= rd32(E1000_VMOLR(vfn)) &
3360 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3361 wr32(E1000_VMOLR(vfn), vmolr);
3362 igb_restore_vf_multicasts(adapter);
3363 }
3364
3365 /* Need to wait a few seconds after link up to get diagnostic information from
3366 * the phy */
3367 static void igb_update_phy_info(unsigned long data)
3368 {
3369 struct igb_adapter *adapter = (struct igb_adapter *) data;
3370 igb_get_phy_info(&adapter->hw);
3371 }
3372
3373 /**
3374 * igb_has_link - check shared code for link and determine up/down
3375 * @adapter: pointer to driver private info
3376 **/
3377 bool igb_has_link(struct igb_adapter *adapter)
3378 {
3379 struct e1000_hw *hw = &adapter->hw;
3380 bool link_active = false;
3381 s32 ret_val = 0;
3382
3383 /* get_link_status is set on LSC (link status) interrupt or
3384 * rx sequence error interrupt. get_link_status will stay
3385 * false until the e1000_check_for_link establishes link
3386 * for copper adapters ONLY
3387 */
3388 switch (hw->phy.media_type) {
3389 case e1000_media_type_copper:
3390 if (hw->mac.get_link_status) {
3391 ret_val = hw->mac.ops.check_for_link(hw);
3392 link_active = !hw->mac.get_link_status;
3393 } else {
3394 link_active = true;
3395 }
3396 break;
3397 case e1000_media_type_internal_serdes:
3398 ret_val = hw->mac.ops.check_for_link(hw);
3399 link_active = hw->mac.serdes_has_link;
3400 break;
3401 default:
3402 case e1000_media_type_unknown:
3403 break;
3404 }
3405
3406 return link_active;
3407 }
3408
3409 /**
3410 * igb_watchdog - Timer Call-back
3411 * @data: pointer to adapter cast into an unsigned long
3412 **/
3413 static void igb_watchdog(unsigned long data)
3414 {
3415 struct igb_adapter *adapter = (struct igb_adapter *)data;
3416 /* Do the rest outside of interrupt context */
3417 schedule_work(&adapter->watchdog_task);
3418 }
3419
3420 static void igb_watchdog_task(struct work_struct *work)
3421 {
3422 struct igb_adapter *adapter = container_of(work,
3423 struct igb_adapter,
3424 watchdog_task);
3425 struct e1000_hw *hw = &adapter->hw;
3426 struct net_device *netdev = adapter->netdev;
3427 u32 link;
3428 int i;
3429
3430 link = igb_has_link(adapter);
3431 if (link) {
3432 if (!netif_carrier_ok(netdev)) {
3433 u32 ctrl;
3434 hw->mac.ops.get_speed_and_duplex(hw,
3435 &adapter->link_speed,
3436 &adapter->link_duplex);
3437
3438 ctrl = rd32(E1000_CTRL);
3439 /* Links status message must follow this format */
3440 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3441 "Flow Control: %s\n",
3442 netdev->name,
3443 adapter->link_speed,
3444 adapter->link_duplex == FULL_DUPLEX ?
3445 "Full Duplex" : "Half Duplex",
3446 ((ctrl & E1000_CTRL_TFCE) &&
3447 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3448 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3449 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3450
3451 /* adjust timeout factor according to speed/duplex */
3452 adapter->tx_timeout_factor = 1;
3453 switch (adapter->link_speed) {
3454 case SPEED_10:
3455 adapter->tx_timeout_factor = 14;
3456 break;
3457 case SPEED_100:
3458 /* maybe add some timeout factor ? */
3459 break;
3460 }
3461
3462 netif_carrier_on(netdev);
3463
3464 igb_ping_all_vfs(adapter);
3465
3466 /* link state has changed, schedule phy info update */
3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3470 }
3471 } else {
3472 if (netif_carrier_ok(netdev)) {
3473 adapter->link_speed = 0;
3474 adapter->link_duplex = 0;
3475 /* Links status message must follow this format */
3476 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3477 netdev->name);
3478 netif_carrier_off(netdev);
3479
3480 igb_ping_all_vfs(adapter);
3481
3482 /* link state has changed, schedule phy info update */
3483 if (!test_bit(__IGB_DOWN, &adapter->state))
3484 mod_timer(&adapter->phy_info_timer,
3485 round_jiffies(jiffies + 2 * HZ));
3486 }
3487 }
3488
3489 spin_lock(&adapter->stats64_lock);
3490 igb_update_stats(adapter, &adapter->stats64);
3491 spin_unlock(&adapter->stats64_lock);
3492
3493 for (i = 0; i < adapter->num_tx_queues; i++) {
3494 struct igb_ring *tx_ring = adapter->tx_ring[i];
3495 if (!netif_carrier_ok(netdev)) {
3496 /* We've lost link, so the controller stops DMA,
3497 * but we've got queued Tx work that's never going
3498 * to get done, so reset controller to flush Tx.
3499 * (Do the reset outside of interrupt context). */
3500 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3501 adapter->tx_timeout_count++;
3502 schedule_work(&adapter->reset_task);
3503 /* return immediately since reset is imminent */
3504 return;
3505 }
3506 }
3507
3508 /* Force detection of hung controller every watchdog period */
3509 tx_ring->detect_tx_hung = true;
3510 }
3511
3512 /* Cause software interrupt to ensure rx ring is cleaned */
3513 if (adapter->msix_entries) {
3514 u32 eics = 0;
3515 for (i = 0; i < adapter->num_q_vectors; i++) {
3516 struct igb_q_vector *q_vector = adapter->q_vector[i];
3517 eics |= q_vector->eims_value;
3518 }
3519 wr32(E1000_EICS, eics);
3520 } else {
3521 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3522 }
3523
3524 /* Reset the timer */
3525 if (!test_bit(__IGB_DOWN, &adapter->state))
3526 mod_timer(&adapter->watchdog_timer,
3527 round_jiffies(jiffies + 2 * HZ));
3528 }
3529
3530 enum latency_range {
3531 lowest_latency = 0,
3532 low_latency = 1,
3533 bulk_latency = 2,
3534 latency_invalid = 255
3535 };
3536
3537 /**
3538 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3539 *
3540 * Stores a new ITR value based on strictly on packet size. This
3541 * algorithm is less sophisticated than that used in igb_update_itr,
3542 * due to the difficulty of synchronizing statistics across multiple
3543 * receive rings. The divisors and thresholds used by this function
3544 * were determined based on theoretical maximum wire speed and testing
3545 * data, in order to minimize response time while increasing bulk
3546 * throughput.
3547 * This functionality is controlled by the InterruptThrottleRate module
3548 * parameter (see igb_param.c)
3549 * NOTE: This function is called only when operating in a multiqueue
3550 * receive environment.
3551 * @q_vector: pointer to q_vector
3552 **/
3553 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3554 {
3555 int new_val = q_vector->itr_val;
3556 int avg_wire_size = 0;
3557 struct igb_adapter *adapter = q_vector->adapter;
3558 struct igb_ring *ring;
3559 unsigned int packets;
3560
3561 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3562 * ints/sec - ITR timer value of 120 ticks.
3563 */
3564 if (adapter->link_speed != SPEED_1000) {
3565 new_val = 976;
3566 goto set_itr_val;
3567 }
3568
3569 ring = q_vector->rx_ring;
3570 if (ring) {
3571 packets = ACCESS_ONCE(ring->total_packets);
3572
3573 if (packets)
3574 avg_wire_size = ring->total_bytes / packets;
3575 }
3576
3577 ring = q_vector->tx_ring;
3578 if (ring) {
3579 packets = ACCESS_ONCE(ring->total_packets);
3580
3581 if (packets)
3582 avg_wire_size = max_t(u32, avg_wire_size,
3583 ring->total_bytes / packets);
3584 }
3585
3586 /* if avg_wire_size isn't set no work was done */
3587 if (!avg_wire_size)
3588 goto clear_counts;
3589
3590 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3591 avg_wire_size += 24;
3592
3593 /* Don't starve jumbo frames */
3594 avg_wire_size = min(avg_wire_size, 3000);
3595
3596 /* Give a little boost to mid-size frames */
3597 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3598 new_val = avg_wire_size / 3;
3599 else
3600 new_val = avg_wire_size / 2;
3601
3602 /* when in itr mode 3 do not exceed 20K ints/sec */
3603 if (adapter->rx_itr_setting == 3 && new_val < 196)
3604 new_val = 196;
3605
3606 set_itr_val:
3607 if (new_val != q_vector->itr_val) {
3608 q_vector->itr_val = new_val;
3609 q_vector->set_itr = 1;
3610 }
3611 clear_counts:
3612 if (q_vector->rx_ring) {
3613 q_vector->rx_ring->total_bytes = 0;
3614 q_vector->rx_ring->total_packets = 0;
3615 }
3616 if (q_vector->tx_ring) {
3617 q_vector->tx_ring->total_bytes = 0;
3618 q_vector->tx_ring->total_packets = 0;
3619 }
3620 }
3621
3622 /**
3623 * igb_update_itr - update the dynamic ITR value based on statistics
3624 * Stores a new ITR value based on packets and byte
3625 * counts during the last interrupt. The advantage of per interrupt
3626 * computation is faster updates and more accurate ITR for the current
3627 * traffic pattern. Constants in this function were computed
3628 * based on theoretical maximum wire speed and thresholds were set based
3629 * on testing data as well as attempting to minimize response time
3630 * while increasing bulk throughput.
3631 * this functionality is controlled by the InterruptThrottleRate module
3632 * parameter (see igb_param.c)
3633 * NOTE: These calculations are only valid when operating in a single-
3634 * queue environment.
3635 * @adapter: pointer to adapter
3636 * @itr_setting: current q_vector->itr_val
3637 * @packets: the number of packets during this measurement interval
3638 * @bytes: the number of bytes during this measurement interval
3639 **/
3640 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3641 int packets, int bytes)
3642 {
3643 unsigned int retval = itr_setting;
3644
3645 if (packets == 0)
3646 goto update_itr_done;
3647
3648 switch (itr_setting) {
3649 case lowest_latency:
3650 /* handle TSO and jumbo frames */
3651 if (bytes/packets > 8000)
3652 retval = bulk_latency;
3653 else if ((packets < 5) && (bytes > 512))
3654 retval = low_latency;
3655 break;
3656 case low_latency: /* 50 usec aka 20000 ints/s */
3657 if (bytes > 10000) {
3658 /* this if handles the TSO accounting */
3659 if (bytes/packets > 8000) {
3660 retval = bulk_latency;
3661 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3662 retval = bulk_latency;
3663 } else if ((packets > 35)) {
3664 retval = lowest_latency;
3665 }
3666 } else if (bytes/packets > 2000) {
3667 retval = bulk_latency;
3668 } else if (packets <= 2 && bytes < 512) {
3669 retval = lowest_latency;
3670 }
3671 break;
3672 case bulk_latency: /* 250 usec aka 4000 ints/s */
3673 if (bytes > 25000) {
3674 if (packets > 35)
3675 retval = low_latency;
3676 } else if (bytes < 1500) {
3677 retval = low_latency;
3678 }
3679 break;
3680 }
3681
3682 update_itr_done:
3683 return retval;
3684 }
3685
3686 static void igb_set_itr(struct igb_adapter *adapter)
3687 {
3688 struct igb_q_vector *q_vector = adapter->q_vector[0];
3689 u16 current_itr;
3690 u32 new_itr = q_vector->itr_val;
3691
3692 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3693 if (adapter->link_speed != SPEED_1000) {
3694 current_itr = 0;
3695 new_itr = 4000;
3696 goto set_itr_now;
3697 }
3698
3699 adapter->rx_itr = igb_update_itr(adapter,
3700 adapter->rx_itr,
3701 q_vector->rx_ring->total_packets,
3702 q_vector->rx_ring->total_bytes);
3703
3704 adapter->tx_itr = igb_update_itr(adapter,
3705 adapter->tx_itr,
3706 q_vector->tx_ring->total_packets,
3707 q_vector->tx_ring->total_bytes);
3708 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3709
3710 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3711 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3712 current_itr = low_latency;
3713
3714 switch (current_itr) {
3715 /* counts and packets in update_itr are dependent on these numbers */
3716 case lowest_latency:
3717 new_itr = 56; /* aka 70,000 ints/sec */
3718 break;
3719 case low_latency:
3720 new_itr = 196; /* aka 20,000 ints/sec */
3721 break;
3722 case bulk_latency:
3723 new_itr = 980; /* aka 4,000 ints/sec */
3724 break;
3725 default:
3726 break;
3727 }
3728
3729 set_itr_now:
3730 q_vector->rx_ring->total_bytes = 0;
3731 q_vector->rx_ring->total_packets = 0;
3732 q_vector->tx_ring->total_bytes = 0;
3733 q_vector->tx_ring->total_packets = 0;
3734
3735 if (new_itr != q_vector->itr_val) {
3736 /* this attempts to bias the interrupt rate towards Bulk
3737 * by adding intermediate steps when interrupt rate is
3738 * increasing */
3739 new_itr = new_itr > q_vector->itr_val ?
3740 max((new_itr * q_vector->itr_val) /
3741 (new_itr + (q_vector->itr_val >> 2)),
3742 new_itr) :
3743 new_itr;
3744 /* Don't write the value here; it resets the adapter's
3745 * internal timer, and causes us to delay far longer than
3746 * we should between interrupts. Instead, we write the ITR
3747 * value at the beginning of the next interrupt so the timing
3748 * ends up being correct.
3749 */
3750 q_vector->itr_val = new_itr;
3751 q_vector->set_itr = 1;
3752 }
3753 }
3754
3755 #define IGB_TX_FLAGS_CSUM 0x00000001
3756 #define IGB_TX_FLAGS_VLAN 0x00000002
3757 #define IGB_TX_FLAGS_TSO 0x00000004
3758 #define IGB_TX_FLAGS_IPV4 0x00000008
3759 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3760 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3761 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3762
3763 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3764 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3765 {
3766 struct e1000_adv_tx_context_desc *context_desc;
3767 unsigned int i;
3768 int err;
3769 struct igb_buffer *buffer_info;
3770 u32 info = 0, tu_cmd = 0;
3771 u32 mss_l4len_idx;
3772 u8 l4len;
3773
3774 if (skb_header_cloned(skb)) {
3775 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3776 if (err)
3777 return err;
3778 }
3779
3780 l4len = tcp_hdrlen(skb);
3781 *hdr_len += l4len;
3782
3783 if (skb->protocol == htons(ETH_P_IP)) {
3784 struct iphdr *iph = ip_hdr(skb);
3785 iph->tot_len = 0;
3786 iph->check = 0;
3787 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3788 iph->daddr, 0,
3789 IPPROTO_TCP,
3790 0);
3791 } else if (skb_is_gso_v6(skb)) {
3792 ipv6_hdr(skb)->payload_len = 0;
3793 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3794 &ipv6_hdr(skb)->daddr,
3795 0, IPPROTO_TCP, 0);
3796 }
3797
3798 i = tx_ring->next_to_use;
3799
3800 buffer_info = &tx_ring->buffer_info[i];
3801 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3802 /* VLAN MACLEN IPLEN */
3803 if (tx_flags & IGB_TX_FLAGS_VLAN)
3804 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3805 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3806 *hdr_len += skb_network_offset(skb);
3807 info |= skb_network_header_len(skb);
3808 *hdr_len += skb_network_header_len(skb);
3809 context_desc->vlan_macip_lens = cpu_to_le32(info);
3810
3811 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3812 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3813
3814 if (skb->protocol == htons(ETH_P_IP))
3815 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3816 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3817
3818 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3819
3820 /* MSS L4LEN IDX */
3821 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3822 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3823
3824 /* For 82575, context index must be unique per ring. */
3825 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3826 mss_l4len_idx |= tx_ring->reg_idx << 4;
3827
3828 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3829 context_desc->seqnum_seed = 0;
3830
3831 buffer_info->time_stamp = jiffies;
3832 buffer_info->next_to_watch = i;
3833 buffer_info->dma = 0;
3834 i++;
3835 if (i == tx_ring->count)
3836 i = 0;
3837
3838 tx_ring->next_to_use = i;
3839
3840 return true;
3841 }
3842
3843 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3844 struct sk_buff *skb, u32 tx_flags)
3845 {
3846 struct e1000_adv_tx_context_desc *context_desc;
3847 struct device *dev = tx_ring->dev;
3848 struct igb_buffer *buffer_info;
3849 u32 info = 0, tu_cmd = 0;
3850 unsigned int i;
3851
3852 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3853 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3854 i = tx_ring->next_to_use;
3855 buffer_info = &tx_ring->buffer_info[i];
3856 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3857
3858 if (tx_flags & IGB_TX_FLAGS_VLAN)
3859 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3860
3861 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3862 if (skb->ip_summed == CHECKSUM_PARTIAL)
3863 info |= skb_network_header_len(skb);
3864
3865 context_desc->vlan_macip_lens = cpu_to_le32(info);
3866
3867 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3868
3869 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3870 __be16 protocol;
3871
3872 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3873 const struct vlan_ethhdr *vhdr =
3874 (const struct vlan_ethhdr*)skb->data;
3875
3876 protocol = vhdr->h_vlan_encapsulated_proto;
3877 } else {
3878 protocol = skb->protocol;
3879 }
3880
3881 switch (protocol) {
3882 case cpu_to_be16(ETH_P_IP):
3883 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3884 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3885 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3886 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3887 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3888 break;
3889 case cpu_to_be16(ETH_P_IPV6):
3890 /* XXX what about other V6 headers?? */
3891 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3892 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3893 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3894 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3895 break;
3896 default:
3897 if (unlikely(net_ratelimit()))
3898 dev_warn(dev,
3899 "partial checksum but proto=%x!\n",
3900 skb->protocol);
3901 break;
3902 }
3903 }
3904
3905 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3906 context_desc->seqnum_seed = 0;
3907 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3908 context_desc->mss_l4len_idx =
3909 cpu_to_le32(tx_ring->reg_idx << 4);
3910
3911 buffer_info->time_stamp = jiffies;
3912 buffer_info->next_to_watch = i;
3913 buffer_info->dma = 0;
3914
3915 i++;
3916 if (i == tx_ring->count)
3917 i = 0;
3918 tx_ring->next_to_use = i;
3919
3920 return true;
3921 }
3922 return false;
3923 }
3924
3925 #define IGB_MAX_TXD_PWR 16
3926 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3927
3928 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3929 unsigned int first)
3930 {
3931 struct igb_buffer *buffer_info;
3932 struct device *dev = tx_ring->dev;
3933 unsigned int hlen = skb_headlen(skb);
3934 unsigned int count = 0, i;
3935 unsigned int f;
3936 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3937
3938 i = tx_ring->next_to_use;
3939
3940 buffer_info = &tx_ring->buffer_info[i];
3941 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3942 buffer_info->length = hlen;
3943 /* set time_stamp *before* dma to help avoid a possible race */
3944 buffer_info->time_stamp = jiffies;
3945 buffer_info->next_to_watch = i;
3946 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3947 DMA_TO_DEVICE);
3948 if (dma_mapping_error(dev, buffer_info->dma))
3949 goto dma_error;
3950
3951 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3952 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3953 unsigned int len = frag->size;
3954
3955 count++;
3956 i++;
3957 if (i == tx_ring->count)
3958 i = 0;
3959
3960 buffer_info = &tx_ring->buffer_info[i];
3961 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3962 buffer_info->length = len;
3963 buffer_info->time_stamp = jiffies;
3964 buffer_info->next_to_watch = i;
3965 buffer_info->mapped_as_page = true;
3966 buffer_info->dma = dma_map_page(dev,
3967 frag->page,
3968 frag->page_offset,
3969 len,
3970 DMA_TO_DEVICE);
3971 if (dma_mapping_error(dev, buffer_info->dma))
3972 goto dma_error;
3973
3974 }
3975
3976 tx_ring->buffer_info[i].skb = skb;
3977 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
3978 /* multiply data chunks by size of headers */
3979 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3980 tx_ring->buffer_info[i].gso_segs = gso_segs;
3981 tx_ring->buffer_info[first].next_to_watch = i;
3982
3983 return ++count;
3984
3985 dma_error:
3986 dev_err(dev, "TX DMA map failed\n");
3987
3988 /* clear timestamp and dma mappings for failed buffer_info mapping */
3989 buffer_info->dma = 0;
3990 buffer_info->time_stamp = 0;
3991 buffer_info->length = 0;
3992 buffer_info->next_to_watch = 0;
3993 buffer_info->mapped_as_page = false;
3994
3995 /* clear timestamp and dma mappings for remaining portion of packet */
3996 while (count--) {
3997 if (i == 0)
3998 i = tx_ring->count;
3999 i--;
4000 buffer_info = &tx_ring->buffer_info[i];
4001 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4002 }
4003
4004 return 0;
4005 }
4006
4007 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
4008 u32 tx_flags, int count, u32 paylen,
4009 u8 hdr_len)
4010 {
4011 union e1000_adv_tx_desc *tx_desc;
4012 struct igb_buffer *buffer_info;
4013 u32 olinfo_status = 0, cmd_type_len;
4014 unsigned int i = tx_ring->next_to_use;
4015
4016 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4017 E1000_ADVTXD_DCMD_DEXT);
4018
4019 if (tx_flags & IGB_TX_FLAGS_VLAN)
4020 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4021
4022 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4023 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4024
4025 if (tx_flags & IGB_TX_FLAGS_TSO) {
4026 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4027
4028 /* insert tcp checksum */
4029 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4030
4031 /* insert ip checksum */
4032 if (tx_flags & IGB_TX_FLAGS_IPV4)
4033 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4034
4035 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4036 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4037 }
4038
4039 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4040 (tx_flags & (IGB_TX_FLAGS_CSUM |
4041 IGB_TX_FLAGS_TSO |
4042 IGB_TX_FLAGS_VLAN)))
4043 olinfo_status |= tx_ring->reg_idx << 4;
4044
4045 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4046
4047 do {
4048 buffer_info = &tx_ring->buffer_info[i];
4049 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4050 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4051 tx_desc->read.cmd_type_len =
4052 cpu_to_le32(cmd_type_len | buffer_info->length);
4053 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4054 count--;
4055 i++;
4056 if (i == tx_ring->count)
4057 i = 0;
4058 } while (count > 0);
4059
4060 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4061 /* Force memory writes to complete before letting h/w
4062 * know there are new descriptors to fetch. (Only
4063 * applicable for weak-ordered memory model archs,
4064 * such as IA-64). */
4065 wmb();
4066
4067 tx_ring->next_to_use = i;
4068 writel(i, tx_ring->tail);
4069 /* we need this if more than one processor can write to our tail
4070 * at a time, it syncronizes IO on IA64/Altix systems */
4071 mmiowb();
4072 }
4073
4074 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4075 {
4076 struct net_device *netdev = tx_ring->netdev;
4077
4078 netif_stop_subqueue(netdev, tx_ring->queue_index);
4079
4080 /* Herbert's original patch had:
4081 * smp_mb__after_netif_stop_queue();
4082 * but since that doesn't exist yet, just open code it. */
4083 smp_mb();
4084
4085 /* We need to check again in a case another CPU has just
4086 * made room available. */
4087 if (igb_desc_unused(tx_ring) < size)
4088 return -EBUSY;
4089
4090 /* A reprieve! */
4091 netif_wake_subqueue(netdev, tx_ring->queue_index);
4092
4093 u64_stats_update_begin(&tx_ring->tx_syncp2);
4094 tx_ring->tx_stats.restart_queue2++;
4095 u64_stats_update_end(&tx_ring->tx_syncp2);
4096
4097 return 0;
4098 }
4099
4100 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4101 {
4102 if (igb_desc_unused(tx_ring) >= size)
4103 return 0;
4104 return __igb_maybe_stop_tx(tx_ring, size);
4105 }
4106
4107 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4108 struct igb_ring *tx_ring)
4109 {
4110 int tso = 0, count;
4111 u32 tx_flags = 0;
4112 u16 first;
4113 u8 hdr_len = 0;
4114
4115 /* need: 1 descriptor per page,
4116 * + 2 desc gap to keep tail from touching head,
4117 * + 1 desc for skb->data,
4118 * + 1 desc for context descriptor,
4119 * otherwise try next time */
4120 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4121 /* this is a hard error */
4122 return NETDEV_TX_BUSY;
4123 }
4124
4125 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4126 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4127 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4128 }
4129
4130 if (vlan_tx_tag_present(skb)) {
4131 tx_flags |= IGB_TX_FLAGS_VLAN;
4132 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4133 }
4134
4135 if (skb->protocol == htons(ETH_P_IP))
4136 tx_flags |= IGB_TX_FLAGS_IPV4;
4137
4138 first = tx_ring->next_to_use;
4139 if (skb_is_gso(skb)) {
4140 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4141
4142 if (tso < 0) {
4143 dev_kfree_skb_any(skb);
4144 return NETDEV_TX_OK;
4145 }
4146 }
4147
4148 if (tso)
4149 tx_flags |= IGB_TX_FLAGS_TSO;
4150 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4151 (skb->ip_summed == CHECKSUM_PARTIAL))
4152 tx_flags |= IGB_TX_FLAGS_CSUM;
4153
4154 /*
4155 * count reflects descriptors mapped, if 0 or less then mapping error
4156 * has occured and we need to rewind the descriptor queue
4157 */
4158 count = igb_tx_map_adv(tx_ring, skb, first);
4159 if (!count) {
4160 dev_kfree_skb_any(skb);
4161 tx_ring->buffer_info[first].time_stamp = 0;
4162 tx_ring->next_to_use = first;
4163 return NETDEV_TX_OK;
4164 }
4165
4166 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4167
4168 /* Make sure there is space in the ring for the next send. */
4169 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4170
4171 return NETDEV_TX_OK;
4172 }
4173
4174 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4175 struct net_device *netdev)
4176 {
4177 struct igb_adapter *adapter = netdev_priv(netdev);
4178 struct igb_ring *tx_ring;
4179 int r_idx = 0;
4180
4181 if (test_bit(__IGB_DOWN, &adapter->state)) {
4182 dev_kfree_skb_any(skb);
4183 return NETDEV_TX_OK;
4184 }
4185
4186 if (skb->len <= 0) {
4187 dev_kfree_skb_any(skb);
4188 return NETDEV_TX_OK;
4189 }
4190
4191 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4192 tx_ring = adapter->multi_tx_table[r_idx];
4193
4194 /* This goes back to the question of how to logically map a tx queue
4195 * to a flow. Right now, performance is impacted slightly negatively
4196 * if using multiple tx queues. If the stack breaks away from a
4197 * single qdisc implementation, we can look at this again. */
4198 return igb_xmit_frame_ring_adv(skb, tx_ring);
4199 }
4200
4201 /**
4202 * igb_tx_timeout - Respond to a Tx Hang
4203 * @netdev: network interface device structure
4204 **/
4205 static void igb_tx_timeout(struct net_device *netdev)
4206 {
4207 struct igb_adapter *adapter = netdev_priv(netdev);
4208 struct e1000_hw *hw = &adapter->hw;
4209
4210 /* Do the reset outside of interrupt context */
4211 adapter->tx_timeout_count++;
4212
4213 if (hw->mac.type == e1000_82580)
4214 hw->dev_spec._82575.global_device_reset = true;
4215
4216 schedule_work(&adapter->reset_task);
4217 wr32(E1000_EICS,
4218 (adapter->eims_enable_mask & ~adapter->eims_other));
4219 }
4220
4221 static void igb_reset_task(struct work_struct *work)
4222 {
4223 struct igb_adapter *adapter;
4224 adapter = container_of(work, struct igb_adapter, reset_task);
4225
4226 igb_dump(adapter);
4227 netdev_err(adapter->netdev, "Reset adapter\n");
4228 igb_reinit_locked(adapter);
4229 }
4230
4231 /**
4232 * igb_get_stats64 - Get System Network Statistics
4233 * @netdev: network interface device structure
4234 * @stats: rtnl_link_stats64 pointer
4235 *
4236 **/
4237 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4238 struct rtnl_link_stats64 *stats)
4239 {
4240 struct igb_adapter *adapter = netdev_priv(netdev);
4241
4242 spin_lock(&adapter->stats64_lock);
4243 igb_update_stats(adapter, &adapter->stats64);
4244 memcpy(stats, &adapter->stats64, sizeof(*stats));
4245 spin_unlock(&adapter->stats64_lock);
4246
4247 return stats;
4248 }
4249
4250 /**
4251 * igb_change_mtu - Change the Maximum Transfer Unit
4252 * @netdev: network interface device structure
4253 * @new_mtu: new value for maximum frame size
4254 *
4255 * Returns 0 on success, negative on failure
4256 **/
4257 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4258 {
4259 struct igb_adapter *adapter = netdev_priv(netdev);
4260 struct pci_dev *pdev = adapter->pdev;
4261 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4262 u32 rx_buffer_len, i;
4263
4264 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4265 dev_err(&pdev->dev, "Invalid MTU setting\n");
4266 return -EINVAL;
4267 }
4268
4269 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4270 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4271 return -EINVAL;
4272 }
4273
4274 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4275 msleep(1);
4276
4277 /* igb_down has a dependency on max_frame_size */
4278 adapter->max_frame_size = max_frame;
4279
4280 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4281 * means we reserve 2 more, this pushes us to allocate from the next
4282 * larger slab size.
4283 * i.e. RXBUFFER_2048 --> size-4096 slab
4284 */
4285
4286 if (adapter->hw.mac.type == e1000_82580)
4287 max_frame += IGB_TS_HDR_LEN;
4288
4289 if (max_frame <= IGB_RXBUFFER_1024)
4290 rx_buffer_len = IGB_RXBUFFER_1024;
4291 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4292 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4293 else
4294 rx_buffer_len = IGB_RXBUFFER_128;
4295
4296 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4297 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4298 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4299
4300 if ((adapter->hw.mac.type == e1000_82580) &&
4301 (rx_buffer_len == IGB_RXBUFFER_128))
4302 rx_buffer_len += IGB_RXBUFFER_64;
4303
4304 if (netif_running(netdev))
4305 igb_down(adapter);
4306
4307 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4308 netdev->mtu, new_mtu);
4309 netdev->mtu = new_mtu;
4310
4311 for (i = 0; i < adapter->num_rx_queues; i++)
4312 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4313
4314 if (netif_running(netdev))
4315 igb_up(adapter);
4316 else
4317 igb_reset(adapter);
4318
4319 clear_bit(__IGB_RESETTING, &adapter->state);
4320
4321 return 0;
4322 }
4323
4324 /**
4325 * igb_update_stats - Update the board statistics counters
4326 * @adapter: board private structure
4327 **/
4328
4329 void igb_update_stats(struct igb_adapter *adapter,
4330 struct rtnl_link_stats64 *net_stats)
4331 {
4332 struct e1000_hw *hw = &adapter->hw;
4333 struct pci_dev *pdev = adapter->pdev;
4334 u32 reg, mpc;
4335 u16 phy_tmp;
4336 int i;
4337 u64 bytes, packets;
4338 unsigned int start;
4339 u64 _bytes, _packets;
4340
4341 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4342
4343 /*
4344 * Prevent stats update while adapter is being reset, or if the pci
4345 * connection is down.
4346 */
4347 if (adapter->link_speed == 0)
4348 return;
4349 if (pci_channel_offline(pdev))
4350 return;
4351
4352 bytes = 0;
4353 packets = 0;
4354 for (i = 0; i < adapter->num_rx_queues; i++) {
4355 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4356 struct igb_ring *ring = adapter->rx_ring[i];
4357
4358 ring->rx_stats.drops += rqdpc_tmp;
4359 net_stats->rx_fifo_errors += rqdpc_tmp;
4360
4361 do {
4362 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4363 _bytes = ring->rx_stats.bytes;
4364 _packets = ring->rx_stats.packets;
4365 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4366 bytes += _bytes;
4367 packets += _packets;
4368 }
4369
4370 net_stats->rx_bytes = bytes;
4371 net_stats->rx_packets = packets;
4372
4373 bytes = 0;
4374 packets = 0;
4375 for (i = 0; i < adapter->num_tx_queues; i++) {
4376 struct igb_ring *ring = adapter->tx_ring[i];
4377 do {
4378 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4379 _bytes = ring->tx_stats.bytes;
4380 _packets = ring->tx_stats.packets;
4381 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4382 bytes += _bytes;
4383 packets += _packets;
4384 }
4385 net_stats->tx_bytes = bytes;
4386 net_stats->tx_packets = packets;
4387
4388 /* read stats registers */
4389 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4390 adapter->stats.gprc += rd32(E1000_GPRC);
4391 adapter->stats.gorc += rd32(E1000_GORCL);
4392 rd32(E1000_GORCH); /* clear GORCL */
4393 adapter->stats.bprc += rd32(E1000_BPRC);
4394 adapter->stats.mprc += rd32(E1000_MPRC);
4395 adapter->stats.roc += rd32(E1000_ROC);
4396
4397 adapter->stats.prc64 += rd32(E1000_PRC64);
4398 adapter->stats.prc127 += rd32(E1000_PRC127);
4399 adapter->stats.prc255 += rd32(E1000_PRC255);
4400 adapter->stats.prc511 += rd32(E1000_PRC511);
4401 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4402 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4403 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4404 adapter->stats.sec += rd32(E1000_SEC);
4405
4406 mpc = rd32(E1000_MPC);
4407 adapter->stats.mpc += mpc;
4408 net_stats->rx_fifo_errors += mpc;
4409 adapter->stats.scc += rd32(E1000_SCC);
4410 adapter->stats.ecol += rd32(E1000_ECOL);
4411 adapter->stats.mcc += rd32(E1000_MCC);
4412 adapter->stats.latecol += rd32(E1000_LATECOL);
4413 adapter->stats.dc += rd32(E1000_DC);
4414 adapter->stats.rlec += rd32(E1000_RLEC);
4415 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4416 adapter->stats.xontxc += rd32(E1000_XONTXC);
4417 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4418 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4419 adapter->stats.fcruc += rd32(E1000_FCRUC);
4420 adapter->stats.gptc += rd32(E1000_GPTC);
4421 adapter->stats.gotc += rd32(E1000_GOTCL);
4422 rd32(E1000_GOTCH); /* clear GOTCL */
4423 adapter->stats.rnbc += rd32(E1000_RNBC);
4424 adapter->stats.ruc += rd32(E1000_RUC);
4425 adapter->stats.rfc += rd32(E1000_RFC);
4426 adapter->stats.rjc += rd32(E1000_RJC);
4427 adapter->stats.tor += rd32(E1000_TORH);
4428 adapter->stats.tot += rd32(E1000_TOTH);
4429 adapter->stats.tpr += rd32(E1000_TPR);
4430
4431 adapter->stats.ptc64 += rd32(E1000_PTC64);
4432 adapter->stats.ptc127 += rd32(E1000_PTC127);
4433 adapter->stats.ptc255 += rd32(E1000_PTC255);
4434 adapter->stats.ptc511 += rd32(E1000_PTC511);
4435 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4436 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4437
4438 adapter->stats.mptc += rd32(E1000_MPTC);
4439 adapter->stats.bptc += rd32(E1000_BPTC);
4440
4441 adapter->stats.tpt += rd32(E1000_TPT);
4442 adapter->stats.colc += rd32(E1000_COLC);
4443
4444 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4445 /* read internal phy specific stats */
4446 reg = rd32(E1000_CTRL_EXT);
4447 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4448 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4449 adapter->stats.tncrs += rd32(E1000_TNCRS);
4450 }
4451
4452 adapter->stats.tsctc += rd32(E1000_TSCTC);
4453 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4454
4455 adapter->stats.iac += rd32(E1000_IAC);
4456 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4457 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4458 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4459 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4460 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4461 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4462 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4463 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4464
4465 /* Fill out the OS statistics structure */
4466 net_stats->multicast = adapter->stats.mprc;
4467 net_stats->collisions = adapter->stats.colc;
4468
4469 /* Rx Errors */
4470
4471 /* RLEC on some newer hardware can be incorrect so build
4472 * our own version based on RUC and ROC */
4473 net_stats->rx_errors = adapter->stats.rxerrc +
4474 adapter->stats.crcerrs + adapter->stats.algnerrc +
4475 adapter->stats.ruc + adapter->stats.roc +
4476 adapter->stats.cexterr;
4477 net_stats->rx_length_errors = adapter->stats.ruc +
4478 adapter->stats.roc;
4479 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4480 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4481 net_stats->rx_missed_errors = adapter->stats.mpc;
4482
4483 /* Tx Errors */
4484 net_stats->tx_errors = adapter->stats.ecol +
4485 adapter->stats.latecol;
4486 net_stats->tx_aborted_errors = adapter->stats.ecol;
4487 net_stats->tx_window_errors = adapter->stats.latecol;
4488 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4489
4490 /* Tx Dropped needs to be maintained elsewhere */
4491
4492 /* Phy Stats */
4493 if (hw->phy.media_type == e1000_media_type_copper) {
4494 if ((adapter->link_speed == SPEED_1000) &&
4495 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4496 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4497 adapter->phy_stats.idle_errors += phy_tmp;
4498 }
4499 }
4500
4501 /* Management Stats */
4502 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4503 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4504 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4505 }
4506
4507 static irqreturn_t igb_msix_other(int irq, void *data)
4508 {
4509 struct igb_adapter *adapter = data;
4510 struct e1000_hw *hw = &adapter->hw;
4511 u32 icr = rd32(E1000_ICR);
4512 /* reading ICR causes bit 31 of EICR to be cleared */
4513
4514 if (icr & E1000_ICR_DRSTA)
4515 schedule_work(&adapter->reset_task);
4516
4517 if (icr & E1000_ICR_DOUTSYNC) {
4518 /* HW is reporting DMA is out of sync */
4519 adapter->stats.doosync++;
4520 }
4521
4522 /* Check for a mailbox event */
4523 if (icr & E1000_ICR_VMMB)
4524 igb_msg_task(adapter);
4525
4526 if (icr & E1000_ICR_LSC) {
4527 hw->mac.get_link_status = 1;
4528 /* guard against interrupt when we're going down */
4529 if (!test_bit(__IGB_DOWN, &adapter->state))
4530 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4531 }
4532
4533 if (adapter->vfs_allocated_count)
4534 wr32(E1000_IMS, E1000_IMS_LSC |
4535 E1000_IMS_VMMB |
4536 E1000_IMS_DOUTSYNC);
4537 else
4538 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4539 wr32(E1000_EIMS, adapter->eims_other);
4540
4541 return IRQ_HANDLED;
4542 }
4543
4544 static void igb_write_itr(struct igb_q_vector *q_vector)
4545 {
4546 struct igb_adapter *adapter = q_vector->adapter;
4547 u32 itr_val = q_vector->itr_val & 0x7FFC;
4548
4549 if (!q_vector->set_itr)
4550 return;
4551
4552 if (!itr_val)
4553 itr_val = 0x4;
4554
4555 if (adapter->hw.mac.type == e1000_82575)
4556 itr_val |= itr_val << 16;
4557 else
4558 itr_val |= 0x8000000;
4559
4560 writel(itr_val, q_vector->itr_register);
4561 q_vector->set_itr = 0;
4562 }
4563
4564 static irqreturn_t igb_msix_ring(int irq, void *data)
4565 {
4566 struct igb_q_vector *q_vector = data;
4567
4568 /* Write the ITR value calculated from the previous interrupt. */
4569 igb_write_itr(q_vector);
4570
4571 napi_schedule(&q_vector->napi);
4572
4573 return IRQ_HANDLED;
4574 }
4575
4576 #ifdef CONFIG_IGB_DCA
4577 static void igb_update_dca(struct igb_q_vector *q_vector)
4578 {
4579 struct igb_adapter *adapter = q_vector->adapter;
4580 struct e1000_hw *hw = &adapter->hw;
4581 int cpu = get_cpu();
4582
4583 if (q_vector->cpu == cpu)
4584 goto out_no_update;
4585
4586 if (q_vector->tx_ring) {
4587 int q = q_vector->tx_ring->reg_idx;
4588 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4589 if (hw->mac.type == e1000_82575) {
4590 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4591 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4592 } else {
4593 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4594 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4595 E1000_DCA_TXCTRL_CPUID_SHIFT;
4596 }
4597 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4598 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4599 }
4600 if (q_vector->rx_ring) {
4601 int q = q_vector->rx_ring->reg_idx;
4602 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4603 if (hw->mac.type == e1000_82575) {
4604 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4605 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4606 } else {
4607 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4608 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4609 E1000_DCA_RXCTRL_CPUID_SHIFT;
4610 }
4611 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4612 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4613 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4614 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4615 }
4616 q_vector->cpu = cpu;
4617 out_no_update:
4618 put_cpu();
4619 }
4620
4621 static void igb_setup_dca(struct igb_adapter *adapter)
4622 {
4623 struct e1000_hw *hw = &adapter->hw;
4624 int i;
4625
4626 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4627 return;
4628
4629 /* Always use CB2 mode, difference is masked in the CB driver. */
4630 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4631
4632 for (i = 0; i < adapter->num_q_vectors; i++) {
4633 adapter->q_vector[i]->cpu = -1;
4634 igb_update_dca(adapter->q_vector[i]);
4635 }
4636 }
4637
4638 static int __igb_notify_dca(struct device *dev, void *data)
4639 {
4640 struct net_device *netdev = dev_get_drvdata(dev);
4641 struct igb_adapter *adapter = netdev_priv(netdev);
4642 struct pci_dev *pdev = adapter->pdev;
4643 struct e1000_hw *hw = &adapter->hw;
4644 unsigned long event = *(unsigned long *)data;
4645
4646 switch (event) {
4647 case DCA_PROVIDER_ADD:
4648 /* if already enabled, don't do it again */
4649 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4650 break;
4651 if (dca_add_requester(dev) == 0) {
4652 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4653 dev_info(&pdev->dev, "DCA enabled\n");
4654 igb_setup_dca(adapter);
4655 break;
4656 }
4657 /* Fall Through since DCA is disabled. */
4658 case DCA_PROVIDER_REMOVE:
4659 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4660 /* without this a class_device is left
4661 * hanging around in the sysfs model */
4662 dca_remove_requester(dev);
4663 dev_info(&pdev->dev, "DCA disabled\n");
4664 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4665 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4666 }
4667 break;
4668 }
4669
4670 return 0;
4671 }
4672
4673 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4674 void *p)
4675 {
4676 int ret_val;
4677
4678 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4679 __igb_notify_dca);
4680
4681 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4682 }
4683 #endif /* CONFIG_IGB_DCA */
4684
4685 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4686 {
4687 struct e1000_hw *hw = &adapter->hw;
4688 u32 ping;
4689 int i;
4690
4691 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4692 ping = E1000_PF_CONTROL_MSG;
4693 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4694 ping |= E1000_VT_MSGTYPE_CTS;
4695 igb_write_mbx(hw, &ping, 1, i);
4696 }
4697 }
4698
4699 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4700 {
4701 struct e1000_hw *hw = &adapter->hw;
4702 u32 vmolr = rd32(E1000_VMOLR(vf));
4703 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4704
4705 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
4706 IGB_VF_FLAG_MULTI_PROMISC);
4707 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4708
4709 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4710 vmolr |= E1000_VMOLR_MPME;
4711 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
4712 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4713 } else {
4714 /*
4715 * if we have hashes and we are clearing a multicast promisc
4716 * flag we need to write the hashes to the MTA as this step
4717 * was previously skipped
4718 */
4719 if (vf_data->num_vf_mc_hashes > 30) {
4720 vmolr |= E1000_VMOLR_MPME;
4721 } else if (vf_data->num_vf_mc_hashes) {
4722 int j;
4723 vmolr |= E1000_VMOLR_ROMPE;
4724 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4725 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4726 }
4727 }
4728
4729 wr32(E1000_VMOLR(vf), vmolr);
4730
4731 /* there are flags left unprocessed, likely not supported */
4732 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4733 return -EINVAL;
4734
4735 return 0;
4736
4737 }
4738
4739 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4740 u32 *msgbuf, u32 vf)
4741 {
4742 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4743 u16 *hash_list = (u16 *)&msgbuf[1];
4744 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4745 int i;
4746
4747 /* salt away the number of multicast addresses assigned
4748 * to this VF for later use to restore when the PF multi cast
4749 * list changes
4750 */
4751 vf_data->num_vf_mc_hashes = n;
4752
4753 /* only up to 30 hash values supported */
4754 if (n > 30)
4755 n = 30;
4756
4757 /* store the hashes for later use */
4758 for (i = 0; i < n; i++)
4759 vf_data->vf_mc_hashes[i] = hash_list[i];
4760
4761 /* Flush and reset the mta with the new values */
4762 igb_set_rx_mode(adapter->netdev);
4763
4764 return 0;
4765 }
4766
4767 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4768 {
4769 struct e1000_hw *hw = &adapter->hw;
4770 struct vf_data_storage *vf_data;
4771 int i, j;
4772
4773 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4774 u32 vmolr = rd32(E1000_VMOLR(i));
4775 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4776
4777 vf_data = &adapter->vf_data[i];
4778
4779 if ((vf_data->num_vf_mc_hashes > 30) ||
4780 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4781 vmolr |= E1000_VMOLR_MPME;
4782 } else if (vf_data->num_vf_mc_hashes) {
4783 vmolr |= E1000_VMOLR_ROMPE;
4784 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4785 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4786 }
4787 wr32(E1000_VMOLR(i), vmolr);
4788 }
4789 }
4790
4791 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4792 {
4793 struct e1000_hw *hw = &adapter->hw;
4794 u32 pool_mask, reg, vid;
4795 int i;
4796
4797 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4798
4799 /* Find the vlan filter for this id */
4800 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4801 reg = rd32(E1000_VLVF(i));
4802
4803 /* remove the vf from the pool */
4804 reg &= ~pool_mask;
4805
4806 /* if pool is empty then remove entry from vfta */
4807 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4808 (reg & E1000_VLVF_VLANID_ENABLE)) {
4809 reg = 0;
4810 vid = reg & E1000_VLVF_VLANID_MASK;
4811 igb_vfta_set(hw, vid, false);
4812 }
4813
4814 wr32(E1000_VLVF(i), reg);
4815 }
4816
4817 adapter->vf_data[vf].vlans_enabled = 0;
4818 }
4819
4820 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4821 {
4822 struct e1000_hw *hw = &adapter->hw;
4823 u32 reg, i;
4824
4825 /* The vlvf table only exists on 82576 hardware and newer */
4826 if (hw->mac.type < e1000_82576)
4827 return -1;
4828
4829 /* we only need to do this if VMDq is enabled */
4830 if (!adapter->vfs_allocated_count)
4831 return -1;
4832
4833 /* Find the vlan filter for this id */
4834 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4835 reg = rd32(E1000_VLVF(i));
4836 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4837 vid == (reg & E1000_VLVF_VLANID_MASK))
4838 break;
4839 }
4840
4841 if (add) {
4842 if (i == E1000_VLVF_ARRAY_SIZE) {
4843 /* Did not find a matching VLAN ID entry that was
4844 * enabled. Search for a free filter entry, i.e.
4845 * one without the enable bit set
4846 */
4847 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4848 reg = rd32(E1000_VLVF(i));
4849 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4850 break;
4851 }
4852 }
4853 if (i < E1000_VLVF_ARRAY_SIZE) {
4854 /* Found an enabled/available entry */
4855 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4856
4857 /* if !enabled we need to set this up in vfta */
4858 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4859 /* add VID to filter table */
4860 igb_vfta_set(hw, vid, true);
4861 reg |= E1000_VLVF_VLANID_ENABLE;
4862 }
4863 reg &= ~E1000_VLVF_VLANID_MASK;
4864 reg |= vid;
4865 wr32(E1000_VLVF(i), reg);
4866
4867 /* do not modify RLPML for PF devices */
4868 if (vf >= adapter->vfs_allocated_count)
4869 return 0;
4870
4871 if (!adapter->vf_data[vf].vlans_enabled) {
4872 u32 size;
4873 reg = rd32(E1000_VMOLR(vf));
4874 size = reg & E1000_VMOLR_RLPML_MASK;
4875 size += 4;
4876 reg &= ~E1000_VMOLR_RLPML_MASK;
4877 reg |= size;
4878 wr32(E1000_VMOLR(vf), reg);
4879 }
4880
4881 adapter->vf_data[vf].vlans_enabled++;
4882 return 0;
4883 }
4884 } else {
4885 if (i < E1000_VLVF_ARRAY_SIZE) {
4886 /* remove vf from the pool */
4887 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4888 /* if pool is empty then remove entry from vfta */
4889 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4890 reg = 0;
4891 igb_vfta_set(hw, vid, false);
4892 }
4893 wr32(E1000_VLVF(i), reg);
4894
4895 /* do not modify RLPML for PF devices */
4896 if (vf >= adapter->vfs_allocated_count)
4897 return 0;
4898
4899 adapter->vf_data[vf].vlans_enabled--;
4900 if (!adapter->vf_data[vf].vlans_enabled) {
4901 u32 size;
4902 reg = rd32(E1000_VMOLR(vf));
4903 size = reg & E1000_VMOLR_RLPML_MASK;
4904 size -= 4;
4905 reg &= ~E1000_VMOLR_RLPML_MASK;
4906 reg |= size;
4907 wr32(E1000_VMOLR(vf), reg);
4908 }
4909 }
4910 }
4911 return 0;
4912 }
4913
4914 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4915 {
4916 struct e1000_hw *hw = &adapter->hw;
4917
4918 if (vid)
4919 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4920 else
4921 wr32(E1000_VMVIR(vf), 0);
4922 }
4923
4924 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4925 int vf, u16 vlan, u8 qos)
4926 {
4927 int err = 0;
4928 struct igb_adapter *adapter = netdev_priv(netdev);
4929
4930 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4931 return -EINVAL;
4932 if (vlan || qos) {
4933 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4934 if (err)
4935 goto out;
4936 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4937 igb_set_vmolr(adapter, vf, !vlan);
4938 adapter->vf_data[vf].pf_vlan = vlan;
4939 adapter->vf_data[vf].pf_qos = qos;
4940 dev_info(&adapter->pdev->dev,
4941 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4942 if (test_bit(__IGB_DOWN, &adapter->state)) {
4943 dev_warn(&adapter->pdev->dev,
4944 "The VF VLAN has been set,"
4945 " but the PF device is not up.\n");
4946 dev_warn(&adapter->pdev->dev,
4947 "Bring the PF device up before"
4948 " attempting to use the VF device.\n");
4949 }
4950 } else {
4951 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4952 false, vf);
4953 igb_set_vmvir(adapter, vlan, vf);
4954 igb_set_vmolr(adapter, vf, true);
4955 adapter->vf_data[vf].pf_vlan = 0;
4956 adapter->vf_data[vf].pf_qos = 0;
4957 }
4958 out:
4959 return err;
4960 }
4961
4962 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4963 {
4964 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4965 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4966
4967 return igb_vlvf_set(adapter, vid, add, vf);
4968 }
4969
4970 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4971 {
4972 /* clear flags */
4973 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4974 adapter->vf_data[vf].last_nack = jiffies;
4975
4976 /* reset offloads to defaults */
4977 igb_set_vmolr(adapter, vf, true);
4978
4979 /* reset vlans for device */
4980 igb_clear_vf_vfta(adapter, vf);
4981 if (adapter->vf_data[vf].pf_vlan)
4982 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4983 adapter->vf_data[vf].pf_vlan,
4984 adapter->vf_data[vf].pf_qos);
4985 else
4986 igb_clear_vf_vfta(adapter, vf);
4987
4988 /* reset multicast table array for vf */
4989 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4990
4991 /* Flush and reset the mta with the new values */
4992 igb_set_rx_mode(adapter->netdev);
4993 }
4994
4995 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4996 {
4997 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4998
4999 /* generate a new mac address as we were hotplug removed/added */
5000 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5001 random_ether_addr(vf_mac);
5002
5003 /* process remaining reset events */
5004 igb_vf_reset(adapter, vf);
5005 }
5006
5007 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5008 {
5009 struct e1000_hw *hw = &adapter->hw;
5010 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5011 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5012 u32 reg, msgbuf[3];
5013 u8 *addr = (u8 *)(&msgbuf[1]);
5014
5015 /* process all the same items cleared in a function level reset */
5016 igb_vf_reset(adapter, vf);
5017
5018 /* set vf mac address */
5019 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5020
5021 /* enable transmit and receive for vf */
5022 reg = rd32(E1000_VFTE);
5023 wr32(E1000_VFTE, reg | (1 << vf));
5024 reg = rd32(E1000_VFRE);
5025 wr32(E1000_VFRE, reg | (1 << vf));
5026
5027 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
5028
5029 /* reply to reset with ack and vf mac address */
5030 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5031 memcpy(addr, vf_mac, 6);
5032 igb_write_mbx(hw, msgbuf, 3, vf);
5033 }
5034
5035 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5036 {
5037 /*
5038 * The VF MAC Address is stored in a packed array of bytes
5039 * starting at the second 32 bit word of the msg array
5040 */
5041 unsigned char *addr = (char *)&msg[1];
5042 int err = -1;
5043
5044 if (is_valid_ether_addr(addr))
5045 err = igb_set_vf_mac(adapter, vf, addr);
5046
5047 return err;
5048 }
5049
5050 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5051 {
5052 struct e1000_hw *hw = &adapter->hw;
5053 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5054 u32 msg = E1000_VT_MSGTYPE_NACK;
5055
5056 /* if device isn't clear to send it shouldn't be reading either */
5057 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5058 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5059 igb_write_mbx(hw, &msg, 1, vf);
5060 vf_data->last_nack = jiffies;
5061 }
5062 }
5063
5064 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5065 {
5066 struct pci_dev *pdev = adapter->pdev;
5067 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5068 struct e1000_hw *hw = &adapter->hw;
5069 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5070 s32 retval;
5071
5072 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5073
5074 if (retval) {
5075 /* if receive failed revoke VF CTS stats and restart init */
5076 dev_err(&pdev->dev, "Error receiving message from VF\n");
5077 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5078 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5079 return;
5080 goto out;
5081 }
5082
5083 /* this is a message we already processed, do nothing */
5084 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5085 return;
5086
5087 /*
5088 * until the vf completes a reset it should not be
5089 * allowed to start any configuration.
5090 */
5091
5092 if (msgbuf[0] == E1000_VF_RESET) {
5093 igb_vf_reset_msg(adapter, vf);
5094 return;
5095 }
5096
5097 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5098 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5099 return;
5100 retval = -1;
5101 goto out;
5102 }
5103
5104 switch ((msgbuf[0] & 0xFFFF)) {
5105 case E1000_VF_SET_MAC_ADDR:
5106 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5107 break;
5108 case E1000_VF_SET_PROMISC:
5109 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5110 break;
5111 case E1000_VF_SET_MULTICAST:
5112 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5113 break;
5114 case E1000_VF_SET_LPE:
5115 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5116 break;
5117 case E1000_VF_SET_VLAN:
5118 if (adapter->vf_data[vf].pf_vlan)
5119 retval = -1;
5120 else
5121 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5122 break;
5123 default:
5124 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5125 retval = -1;
5126 break;
5127 }
5128
5129 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5130 out:
5131 /* notify the VF of the results of what it sent us */
5132 if (retval)
5133 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5134 else
5135 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5136
5137 igb_write_mbx(hw, msgbuf, 1, vf);
5138 }
5139
5140 static void igb_msg_task(struct igb_adapter *adapter)
5141 {
5142 struct e1000_hw *hw = &adapter->hw;
5143 u32 vf;
5144
5145 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5146 /* process any reset requests */
5147 if (!igb_check_for_rst(hw, vf))
5148 igb_vf_reset_event(adapter, vf);
5149
5150 /* process any messages pending */
5151 if (!igb_check_for_msg(hw, vf))
5152 igb_rcv_msg_from_vf(adapter, vf);
5153
5154 /* process any acks */
5155 if (!igb_check_for_ack(hw, vf))
5156 igb_rcv_ack_from_vf(adapter, vf);
5157 }
5158 }
5159
5160 /**
5161 * igb_set_uta - Set unicast filter table address
5162 * @adapter: board private structure
5163 *
5164 * The unicast table address is a register array of 32-bit registers.
5165 * The table is meant to be used in a way similar to how the MTA is used
5166 * however due to certain limitations in the hardware it is necessary to
5167 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5168 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5169 **/
5170 static void igb_set_uta(struct igb_adapter *adapter)
5171 {
5172 struct e1000_hw *hw = &adapter->hw;
5173 int i;
5174
5175 /* The UTA table only exists on 82576 hardware and newer */
5176 if (hw->mac.type < e1000_82576)
5177 return;
5178
5179 /* we only need to do this if VMDq is enabled */
5180 if (!adapter->vfs_allocated_count)
5181 return;
5182
5183 for (i = 0; i < hw->mac.uta_reg_count; i++)
5184 array_wr32(E1000_UTA, i, ~0);
5185 }
5186
5187 /**
5188 * igb_intr_msi - Interrupt Handler
5189 * @irq: interrupt number
5190 * @data: pointer to a network interface device structure
5191 **/
5192 static irqreturn_t igb_intr_msi(int irq, void *data)
5193 {
5194 struct igb_adapter *adapter = data;
5195 struct igb_q_vector *q_vector = adapter->q_vector[0];
5196 struct e1000_hw *hw = &adapter->hw;
5197 /* read ICR disables interrupts using IAM */
5198 u32 icr = rd32(E1000_ICR);
5199
5200 igb_write_itr(q_vector);
5201
5202 if (icr & E1000_ICR_DRSTA)
5203 schedule_work(&adapter->reset_task);
5204
5205 if (icr & E1000_ICR_DOUTSYNC) {
5206 /* HW is reporting DMA is out of sync */
5207 adapter->stats.doosync++;
5208 }
5209
5210 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5211 hw->mac.get_link_status = 1;
5212 if (!test_bit(__IGB_DOWN, &adapter->state))
5213 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5214 }
5215
5216 napi_schedule(&q_vector->napi);
5217
5218 return IRQ_HANDLED;
5219 }
5220
5221 /**
5222 * igb_intr - Legacy Interrupt Handler
5223 * @irq: interrupt number
5224 * @data: pointer to a network interface device structure
5225 **/
5226 static irqreturn_t igb_intr(int irq, void *data)
5227 {
5228 struct igb_adapter *adapter = data;
5229 struct igb_q_vector *q_vector = adapter->q_vector[0];
5230 struct e1000_hw *hw = &adapter->hw;
5231 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5232 * need for the IMC write */
5233 u32 icr = rd32(E1000_ICR);
5234 if (!icr)
5235 return IRQ_NONE; /* Not our interrupt */
5236
5237 igb_write_itr(q_vector);
5238
5239 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5240 * not set, then the adapter didn't send an interrupt */
5241 if (!(icr & E1000_ICR_INT_ASSERTED))
5242 return IRQ_NONE;
5243
5244 if (icr & E1000_ICR_DRSTA)
5245 schedule_work(&adapter->reset_task);
5246
5247 if (icr & E1000_ICR_DOUTSYNC) {
5248 /* HW is reporting DMA is out of sync */
5249 adapter->stats.doosync++;
5250 }
5251
5252 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5253 hw->mac.get_link_status = 1;
5254 /* guard against interrupt when we're going down */
5255 if (!test_bit(__IGB_DOWN, &adapter->state))
5256 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5257 }
5258
5259 napi_schedule(&q_vector->napi);
5260
5261 return IRQ_HANDLED;
5262 }
5263
5264 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5265 {
5266 struct igb_adapter *adapter = q_vector->adapter;
5267 struct e1000_hw *hw = &adapter->hw;
5268
5269 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5270 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5271 if (!adapter->msix_entries)
5272 igb_set_itr(adapter);
5273 else
5274 igb_update_ring_itr(q_vector);
5275 }
5276
5277 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5278 if (adapter->msix_entries)
5279 wr32(E1000_EIMS, q_vector->eims_value);
5280 else
5281 igb_irq_enable(adapter);
5282 }
5283 }
5284
5285 /**
5286 * igb_poll - NAPI Rx polling callback
5287 * @napi: napi polling structure
5288 * @budget: count of how many packets we should handle
5289 **/
5290 static int igb_poll(struct napi_struct *napi, int budget)
5291 {
5292 struct igb_q_vector *q_vector = container_of(napi,
5293 struct igb_q_vector,
5294 napi);
5295 int tx_clean_complete = 1, work_done = 0;
5296
5297 #ifdef CONFIG_IGB_DCA
5298 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5299 igb_update_dca(q_vector);
5300 #endif
5301 if (q_vector->tx_ring)
5302 tx_clean_complete = igb_clean_tx_irq(q_vector);
5303
5304 if (q_vector->rx_ring)
5305 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5306
5307 if (!tx_clean_complete)
5308 work_done = budget;
5309
5310 /* If not enough Rx work done, exit the polling mode */
5311 if (work_done < budget) {
5312 napi_complete(napi);
5313 igb_ring_irq_enable(q_vector);
5314 }
5315
5316 return work_done;
5317 }
5318
5319 /**
5320 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5321 * @adapter: board private structure
5322 * @shhwtstamps: timestamp structure to update
5323 * @regval: unsigned 64bit system time value.
5324 *
5325 * We need to convert the system time value stored in the RX/TXSTMP registers
5326 * into a hwtstamp which can be used by the upper level timestamping functions
5327 */
5328 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5329 struct skb_shared_hwtstamps *shhwtstamps,
5330 u64 regval)
5331 {
5332 u64 ns;
5333
5334 /*
5335 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5336 * 24 to match clock shift we setup earlier.
5337 */
5338 if (adapter->hw.mac.type == e1000_82580)
5339 regval <<= IGB_82580_TSYNC_SHIFT;
5340
5341 ns = timecounter_cyc2time(&adapter->clock, regval);
5342 timecompare_update(&adapter->compare, ns);
5343 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5344 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5345 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5346 }
5347
5348 /**
5349 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5350 * @q_vector: pointer to q_vector containing needed info
5351 * @buffer: pointer to igb_buffer structure
5352 *
5353 * If we were asked to do hardware stamping and such a time stamp is
5354 * available, then it must have been for this skb here because we only
5355 * allow only one such packet into the queue.
5356 */
5357 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5358 {
5359 struct igb_adapter *adapter = q_vector->adapter;
5360 struct e1000_hw *hw = &adapter->hw;
5361 struct skb_shared_hwtstamps shhwtstamps;
5362 u64 regval;
5363
5364 /* if skb does not support hw timestamp or TX stamp not valid exit */
5365 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
5366 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5367 return;
5368
5369 regval = rd32(E1000_TXSTMPL);
5370 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5371
5372 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5373 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5374 }
5375
5376 /**
5377 * igb_clean_tx_irq - Reclaim resources after transmit completes
5378 * @q_vector: pointer to q_vector containing needed info
5379 * returns true if ring is completely cleaned
5380 **/
5381 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5382 {
5383 struct igb_adapter *adapter = q_vector->adapter;
5384 struct igb_ring *tx_ring = q_vector->tx_ring;
5385 struct net_device *netdev = tx_ring->netdev;
5386 struct e1000_hw *hw = &adapter->hw;
5387 struct igb_buffer *buffer_info;
5388 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5389 unsigned int total_bytes = 0, total_packets = 0;
5390 unsigned int i, eop, count = 0;
5391 bool cleaned = false;
5392
5393 i = tx_ring->next_to_clean;
5394 eop = tx_ring->buffer_info[i].next_to_watch;
5395 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5396
5397 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5398 (count < tx_ring->count)) {
5399 rmb(); /* read buffer_info after eop_desc status */
5400 for (cleaned = false; !cleaned; count++) {
5401 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5402 buffer_info = &tx_ring->buffer_info[i];
5403 cleaned = (i == eop);
5404
5405 if (buffer_info->skb) {
5406 total_bytes += buffer_info->bytecount;
5407 /* gso_segs is currently only valid for tcp */
5408 total_packets += buffer_info->gso_segs;
5409 igb_tx_hwtstamp(q_vector, buffer_info);
5410 }
5411
5412 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5413 tx_desc->wb.status = 0;
5414
5415 i++;
5416 if (i == tx_ring->count)
5417 i = 0;
5418 }
5419 eop = tx_ring->buffer_info[i].next_to_watch;
5420 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5421 }
5422
5423 tx_ring->next_to_clean = i;
5424
5425 if (unlikely(count &&
5426 netif_carrier_ok(netdev) &&
5427 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5428 /* Make sure that anybody stopping the queue after this
5429 * sees the new next_to_clean.
5430 */
5431 smp_mb();
5432 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5433 !(test_bit(__IGB_DOWN, &adapter->state))) {
5434 netif_wake_subqueue(netdev, tx_ring->queue_index);
5435
5436 u64_stats_update_begin(&tx_ring->tx_syncp);
5437 tx_ring->tx_stats.restart_queue++;
5438 u64_stats_update_end(&tx_ring->tx_syncp);
5439 }
5440 }
5441
5442 if (tx_ring->detect_tx_hung) {
5443 /* Detect a transmit hang in hardware, this serializes the
5444 * check with the clearing of time_stamp and movement of i */
5445 tx_ring->detect_tx_hung = false;
5446 if (tx_ring->buffer_info[i].time_stamp &&
5447 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5448 (adapter->tx_timeout_factor * HZ)) &&
5449 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5450
5451 /* detected Tx unit hang */
5452 dev_err(tx_ring->dev,
5453 "Detected Tx Unit Hang\n"
5454 " Tx Queue <%d>\n"
5455 " TDH <%x>\n"
5456 " TDT <%x>\n"
5457 " next_to_use <%x>\n"
5458 " next_to_clean <%x>\n"
5459 "buffer_info[next_to_clean]\n"
5460 " time_stamp <%lx>\n"
5461 " next_to_watch <%x>\n"
5462 " jiffies <%lx>\n"
5463 " desc.status <%x>\n",
5464 tx_ring->queue_index,
5465 readl(tx_ring->head),
5466 readl(tx_ring->tail),
5467 tx_ring->next_to_use,
5468 tx_ring->next_to_clean,
5469 tx_ring->buffer_info[eop].time_stamp,
5470 eop,
5471 jiffies,
5472 eop_desc->wb.status);
5473 netif_stop_subqueue(netdev, tx_ring->queue_index);
5474 }
5475 }
5476 tx_ring->total_bytes += total_bytes;
5477 tx_ring->total_packets += total_packets;
5478 u64_stats_update_begin(&tx_ring->tx_syncp);
5479 tx_ring->tx_stats.bytes += total_bytes;
5480 tx_ring->tx_stats.packets += total_packets;
5481 u64_stats_update_end(&tx_ring->tx_syncp);
5482 return count < tx_ring->count;
5483 }
5484
5485 /**
5486 * igb_receive_skb - helper function to handle rx indications
5487 * @q_vector: structure containing interrupt and ring information
5488 * @skb: packet to send up
5489 * @vlan_tag: vlan tag for packet
5490 **/
5491 static void igb_receive_skb(struct igb_q_vector *q_vector,
5492 struct sk_buff *skb,
5493 u16 vlan_tag)
5494 {
5495 struct igb_adapter *adapter = q_vector->adapter;
5496
5497 if (vlan_tag && adapter->vlgrp)
5498 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5499 vlan_tag, skb);
5500 else
5501 napi_gro_receive(&q_vector->napi, skb);
5502 }
5503
5504 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5505 u32 status_err, struct sk_buff *skb)
5506 {
5507 skb_checksum_none_assert(skb);
5508
5509 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5510 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5511 (status_err & E1000_RXD_STAT_IXSM))
5512 return;
5513
5514 /* TCP/UDP checksum error bit is set */
5515 if (status_err &
5516 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5517 /*
5518 * work around errata with sctp packets where the TCPE aka
5519 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5520 * packets, (aka let the stack check the crc32c)
5521 */
5522 if ((skb->len == 60) &&
5523 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5524 u64_stats_update_begin(&ring->rx_syncp);
5525 ring->rx_stats.csum_err++;
5526 u64_stats_update_end(&ring->rx_syncp);
5527 }
5528 /* let the stack verify checksum errors */
5529 return;
5530 }
5531 /* It must be a TCP or UDP packet with a valid checksum */
5532 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5533 skb->ip_summed = CHECKSUM_UNNECESSARY;
5534
5535 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5536 }
5537
5538 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5539 struct sk_buff *skb)
5540 {
5541 struct igb_adapter *adapter = q_vector->adapter;
5542 struct e1000_hw *hw = &adapter->hw;
5543 u64 regval;
5544
5545 /*
5546 * If this bit is set, then the RX registers contain the time stamp. No
5547 * other packet will be time stamped until we read these registers, so
5548 * read the registers to make them available again. Because only one
5549 * packet can be time stamped at a time, we know that the register
5550 * values must belong to this one here and therefore we don't need to
5551 * compare any of the additional attributes stored for it.
5552 *
5553 * If nothing went wrong, then it should have a shared tx_flags that we
5554 * can turn into a skb_shared_hwtstamps.
5555 */
5556 if (staterr & E1000_RXDADV_STAT_TSIP) {
5557 u32 *stamp = (u32 *)skb->data;
5558 regval = le32_to_cpu(*(stamp + 2));
5559 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5560 skb_pull(skb, IGB_TS_HDR_LEN);
5561 } else {
5562 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5563 return;
5564
5565 regval = rd32(E1000_RXSTMPL);
5566 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5567 }
5568
5569 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5570 }
5571 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5572 union e1000_adv_rx_desc *rx_desc)
5573 {
5574 /* HW will not DMA in data larger than the given buffer, even if it
5575 * parses the (NFS, of course) header to be larger. In that case, it
5576 * fills the header buffer and spills the rest into the page.
5577 */
5578 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5579 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5580 if (hlen > rx_ring->rx_buffer_len)
5581 hlen = rx_ring->rx_buffer_len;
5582 return hlen;
5583 }
5584
5585 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5586 int *work_done, int budget)
5587 {
5588 struct igb_ring *rx_ring = q_vector->rx_ring;
5589 struct net_device *netdev = rx_ring->netdev;
5590 struct device *dev = rx_ring->dev;
5591 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5592 struct igb_buffer *buffer_info , *next_buffer;
5593 struct sk_buff *skb;
5594 bool cleaned = false;
5595 int cleaned_count = 0;
5596 int current_node = numa_node_id();
5597 unsigned int total_bytes = 0, total_packets = 0;
5598 unsigned int i;
5599 u32 staterr;
5600 u16 length;
5601 u16 vlan_tag;
5602
5603 i = rx_ring->next_to_clean;
5604 buffer_info = &rx_ring->buffer_info[i];
5605 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5606 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5607
5608 while (staterr & E1000_RXD_STAT_DD) {
5609 if (*work_done >= budget)
5610 break;
5611 (*work_done)++;
5612 rmb(); /* read descriptor and rx_buffer_info after status DD */
5613
5614 skb = buffer_info->skb;
5615 prefetch(skb->data - NET_IP_ALIGN);
5616 buffer_info->skb = NULL;
5617
5618 i++;
5619 if (i == rx_ring->count)
5620 i = 0;
5621
5622 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5623 prefetch(next_rxd);
5624 next_buffer = &rx_ring->buffer_info[i];
5625
5626 length = le16_to_cpu(rx_desc->wb.upper.length);
5627 cleaned = true;
5628 cleaned_count++;
5629
5630 if (buffer_info->dma) {
5631 dma_unmap_single(dev, buffer_info->dma,
5632 rx_ring->rx_buffer_len,
5633 DMA_FROM_DEVICE);
5634 buffer_info->dma = 0;
5635 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5636 skb_put(skb, length);
5637 goto send_up;
5638 }
5639 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5640 }
5641
5642 if (length) {
5643 dma_unmap_page(dev, buffer_info->page_dma,
5644 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5645 buffer_info->page_dma = 0;
5646
5647 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5648 buffer_info->page,
5649 buffer_info->page_offset,
5650 length);
5651
5652 if ((page_count(buffer_info->page) != 1) ||
5653 (page_to_nid(buffer_info->page) != current_node))
5654 buffer_info->page = NULL;
5655 else
5656 get_page(buffer_info->page);
5657
5658 skb->len += length;
5659 skb->data_len += length;
5660 skb->truesize += length;
5661 }
5662
5663 if (!(staterr & E1000_RXD_STAT_EOP)) {
5664 buffer_info->skb = next_buffer->skb;
5665 buffer_info->dma = next_buffer->dma;
5666 next_buffer->skb = skb;
5667 next_buffer->dma = 0;
5668 goto next_desc;
5669 }
5670 send_up:
5671 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5672 dev_kfree_skb_irq(skb);
5673 goto next_desc;
5674 }
5675
5676 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5677 igb_rx_hwtstamp(q_vector, staterr, skb);
5678 total_bytes += skb->len;
5679 total_packets++;
5680
5681 igb_rx_checksum_adv(rx_ring, staterr, skb);
5682
5683 skb->protocol = eth_type_trans(skb, netdev);
5684 skb_record_rx_queue(skb, rx_ring->queue_index);
5685
5686 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5687 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5688
5689 igb_receive_skb(q_vector, skb, vlan_tag);
5690
5691 next_desc:
5692 rx_desc->wb.upper.status_error = 0;
5693
5694 /* return some buffers to hardware, one at a time is too slow */
5695 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5696 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5697 cleaned_count = 0;
5698 }
5699
5700 /* use prefetched values */
5701 rx_desc = next_rxd;
5702 buffer_info = next_buffer;
5703 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5704 }
5705
5706 rx_ring->next_to_clean = i;
5707 cleaned_count = igb_desc_unused(rx_ring);
5708
5709 if (cleaned_count)
5710 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5711
5712 rx_ring->total_packets += total_packets;
5713 rx_ring->total_bytes += total_bytes;
5714 u64_stats_update_begin(&rx_ring->rx_syncp);
5715 rx_ring->rx_stats.packets += total_packets;
5716 rx_ring->rx_stats.bytes += total_bytes;
5717 u64_stats_update_end(&rx_ring->rx_syncp);
5718 return cleaned;
5719 }
5720
5721 /**
5722 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5723 * @adapter: address of board private structure
5724 **/
5725 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5726 {
5727 struct net_device *netdev = rx_ring->netdev;
5728 union e1000_adv_rx_desc *rx_desc;
5729 struct igb_buffer *buffer_info;
5730 struct sk_buff *skb;
5731 unsigned int i;
5732 int bufsz;
5733
5734 i = rx_ring->next_to_use;
5735 buffer_info = &rx_ring->buffer_info[i];
5736
5737 bufsz = rx_ring->rx_buffer_len;
5738
5739 while (cleaned_count--) {
5740 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5741
5742 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5743 if (!buffer_info->page) {
5744 buffer_info->page = netdev_alloc_page(netdev);
5745 if (unlikely(!buffer_info->page)) {
5746 u64_stats_update_begin(&rx_ring->rx_syncp);
5747 rx_ring->rx_stats.alloc_failed++;
5748 u64_stats_update_end(&rx_ring->rx_syncp);
5749 goto no_buffers;
5750 }
5751 buffer_info->page_offset = 0;
5752 } else {
5753 buffer_info->page_offset ^= PAGE_SIZE / 2;
5754 }
5755 buffer_info->page_dma =
5756 dma_map_page(rx_ring->dev, buffer_info->page,
5757 buffer_info->page_offset,
5758 PAGE_SIZE / 2,
5759 DMA_FROM_DEVICE);
5760 if (dma_mapping_error(rx_ring->dev,
5761 buffer_info->page_dma)) {
5762 buffer_info->page_dma = 0;
5763 u64_stats_update_begin(&rx_ring->rx_syncp);
5764 rx_ring->rx_stats.alloc_failed++;
5765 u64_stats_update_end(&rx_ring->rx_syncp);
5766 goto no_buffers;
5767 }
5768 }
5769
5770 skb = buffer_info->skb;
5771 if (!skb) {
5772 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5773 if (unlikely(!skb)) {
5774 u64_stats_update_begin(&rx_ring->rx_syncp);
5775 rx_ring->rx_stats.alloc_failed++;
5776 u64_stats_update_end(&rx_ring->rx_syncp);
5777 goto no_buffers;
5778 }
5779
5780 buffer_info->skb = skb;
5781 }
5782 if (!buffer_info->dma) {
5783 buffer_info->dma = dma_map_single(rx_ring->dev,
5784 skb->data,
5785 bufsz,
5786 DMA_FROM_DEVICE);
5787 if (dma_mapping_error(rx_ring->dev,
5788 buffer_info->dma)) {
5789 buffer_info->dma = 0;
5790 u64_stats_update_begin(&rx_ring->rx_syncp);
5791 rx_ring->rx_stats.alloc_failed++;
5792 u64_stats_update_end(&rx_ring->rx_syncp);
5793 goto no_buffers;
5794 }
5795 }
5796 /* Refresh the desc even if buffer_addrs didn't change because
5797 * each write-back erases this info. */
5798 if (bufsz < IGB_RXBUFFER_1024) {
5799 rx_desc->read.pkt_addr =
5800 cpu_to_le64(buffer_info->page_dma);
5801 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5802 } else {
5803 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5804 rx_desc->read.hdr_addr = 0;
5805 }
5806
5807 i++;
5808 if (i == rx_ring->count)
5809 i = 0;
5810 buffer_info = &rx_ring->buffer_info[i];
5811 }
5812
5813 no_buffers:
5814 if (rx_ring->next_to_use != i) {
5815 rx_ring->next_to_use = i;
5816 if (i == 0)
5817 i = (rx_ring->count - 1);
5818 else
5819 i--;
5820
5821 /* Force memory writes to complete before letting h/w
5822 * know there are new descriptors to fetch. (Only
5823 * applicable for weak-ordered memory model archs,
5824 * such as IA-64). */
5825 wmb();
5826 writel(i, rx_ring->tail);
5827 }
5828 }
5829
5830 /**
5831 * igb_mii_ioctl -
5832 * @netdev:
5833 * @ifreq:
5834 * @cmd:
5835 **/
5836 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5837 {
5838 struct igb_adapter *adapter = netdev_priv(netdev);
5839 struct mii_ioctl_data *data = if_mii(ifr);
5840
5841 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5842 return -EOPNOTSUPP;
5843
5844 switch (cmd) {
5845 case SIOCGMIIPHY:
5846 data->phy_id = adapter->hw.phy.addr;
5847 break;
5848 case SIOCGMIIREG:
5849 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5850 &data->val_out))
5851 return -EIO;
5852 break;
5853 case SIOCSMIIREG:
5854 default:
5855 return -EOPNOTSUPP;
5856 }
5857 return 0;
5858 }
5859
5860 /**
5861 * igb_hwtstamp_ioctl - control hardware time stamping
5862 * @netdev:
5863 * @ifreq:
5864 * @cmd:
5865 *
5866 * Outgoing time stamping can be enabled and disabled. Play nice and
5867 * disable it when requested, although it shouldn't case any overhead
5868 * when no packet needs it. At most one packet in the queue may be
5869 * marked for time stamping, otherwise it would be impossible to tell
5870 * for sure to which packet the hardware time stamp belongs.
5871 *
5872 * Incoming time stamping has to be configured via the hardware
5873 * filters. Not all combinations are supported, in particular event
5874 * type has to be specified. Matching the kind of event packet is
5875 * not supported, with the exception of "all V2 events regardless of
5876 * level 2 or 4".
5877 *
5878 **/
5879 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5880 struct ifreq *ifr, int cmd)
5881 {
5882 struct igb_adapter *adapter = netdev_priv(netdev);
5883 struct e1000_hw *hw = &adapter->hw;
5884 struct hwtstamp_config config;
5885 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5886 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5887 u32 tsync_rx_cfg = 0;
5888 bool is_l4 = false;
5889 bool is_l2 = false;
5890 u32 regval;
5891
5892 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5893 return -EFAULT;
5894
5895 /* reserved for future extensions */
5896 if (config.flags)
5897 return -EINVAL;
5898
5899 switch (config.tx_type) {
5900 case HWTSTAMP_TX_OFF:
5901 tsync_tx_ctl = 0;
5902 case HWTSTAMP_TX_ON:
5903 break;
5904 default:
5905 return -ERANGE;
5906 }
5907
5908 switch (config.rx_filter) {
5909 case HWTSTAMP_FILTER_NONE:
5910 tsync_rx_ctl = 0;
5911 break;
5912 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5913 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5914 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5915 case HWTSTAMP_FILTER_ALL:
5916 /*
5917 * register TSYNCRXCFG must be set, therefore it is not
5918 * possible to time stamp both Sync and Delay_Req messages
5919 * => fall back to time stamping all packets
5920 */
5921 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5922 config.rx_filter = HWTSTAMP_FILTER_ALL;
5923 break;
5924 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5925 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5926 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5927 is_l4 = true;
5928 break;
5929 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5930 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5931 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5932 is_l4 = true;
5933 break;
5934 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5935 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5936 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5937 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5938 is_l2 = true;
5939 is_l4 = true;
5940 config.rx_filter = HWTSTAMP_FILTER_SOME;
5941 break;
5942 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5943 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5944 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5945 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5946 is_l2 = true;
5947 is_l4 = true;
5948 config.rx_filter = HWTSTAMP_FILTER_SOME;
5949 break;
5950 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5951 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5952 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5953 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5954 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5955 is_l2 = true;
5956 break;
5957 default:
5958 return -ERANGE;
5959 }
5960
5961 if (hw->mac.type == e1000_82575) {
5962 if (tsync_rx_ctl | tsync_tx_ctl)
5963 return -EINVAL;
5964 return 0;
5965 }
5966
5967 /*
5968 * Per-packet timestamping only works if all packets are
5969 * timestamped, so enable timestamping in all packets as
5970 * long as one rx filter was configured.
5971 */
5972 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5973 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5974 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5975 }
5976
5977 /* enable/disable TX */
5978 regval = rd32(E1000_TSYNCTXCTL);
5979 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5980 regval |= tsync_tx_ctl;
5981 wr32(E1000_TSYNCTXCTL, regval);
5982
5983 /* enable/disable RX */
5984 regval = rd32(E1000_TSYNCRXCTL);
5985 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5986 regval |= tsync_rx_ctl;
5987 wr32(E1000_TSYNCRXCTL, regval);
5988
5989 /* define which PTP packets are time stamped */
5990 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5991
5992 /* define ethertype filter for timestamped packets */
5993 if (is_l2)
5994 wr32(E1000_ETQF(3),
5995 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5996 E1000_ETQF_1588 | /* enable timestamping */
5997 ETH_P_1588)); /* 1588 eth protocol type */
5998 else
5999 wr32(E1000_ETQF(3), 0);
6000
6001 #define PTP_PORT 319
6002 /* L4 Queue Filter[3]: filter by destination port and protocol */
6003 if (is_l4) {
6004 u32 ftqf = (IPPROTO_UDP /* UDP */
6005 | E1000_FTQF_VF_BP /* VF not compared */
6006 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6007 | E1000_FTQF_MASK); /* mask all inputs */
6008 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6009
6010 wr32(E1000_IMIR(3), htons(PTP_PORT));
6011 wr32(E1000_IMIREXT(3),
6012 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6013 if (hw->mac.type == e1000_82576) {
6014 /* enable source port check */
6015 wr32(E1000_SPQF(3), htons(PTP_PORT));
6016 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6017 }
6018 wr32(E1000_FTQF(3), ftqf);
6019 } else {
6020 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6021 }
6022 wrfl();
6023
6024 adapter->hwtstamp_config = config;
6025
6026 /* clear TX/RX time stamp registers, just to be sure */
6027 regval = rd32(E1000_TXSTMPH);
6028 regval = rd32(E1000_RXSTMPH);
6029
6030 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6031 -EFAULT : 0;
6032 }
6033
6034 /**
6035 * igb_ioctl -
6036 * @netdev:
6037 * @ifreq:
6038 * @cmd:
6039 **/
6040 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6041 {
6042 switch (cmd) {
6043 case SIOCGMIIPHY:
6044 case SIOCGMIIREG:
6045 case SIOCSMIIREG:
6046 return igb_mii_ioctl(netdev, ifr, cmd);
6047 case SIOCSHWTSTAMP:
6048 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6049 default:
6050 return -EOPNOTSUPP;
6051 }
6052 }
6053
6054 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6055 {
6056 struct igb_adapter *adapter = hw->back;
6057 u16 cap_offset;
6058
6059 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6060 if (!cap_offset)
6061 return -E1000_ERR_CONFIG;
6062
6063 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6064
6065 return 0;
6066 }
6067
6068 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6069 {
6070 struct igb_adapter *adapter = hw->back;
6071 u16 cap_offset;
6072
6073 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6074 if (!cap_offset)
6075 return -E1000_ERR_CONFIG;
6076
6077 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6078
6079 return 0;
6080 }
6081
6082 static void igb_vlan_rx_register(struct net_device *netdev,
6083 struct vlan_group *grp)
6084 {
6085 struct igb_adapter *adapter = netdev_priv(netdev);
6086 struct e1000_hw *hw = &adapter->hw;
6087 u32 ctrl, rctl;
6088
6089 igb_irq_disable(adapter);
6090 adapter->vlgrp = grp;
6091
6092 if (grp) {
6093 /* enable VLAN tag insert/strip */
6094 ctrl = rd32(E1000_CTRL);
6095 ctrl |= E1000_CTRL_VME;
6096 wr32(E1000_CTRL, ctrl);
6097
6098 /* Disable CFI check */
6099 rctl = rd32(E1000_RCTL);
6100 rctl &= ~E1000_RCTL_CFIEN;
6101 wr32(E1000_RCTL, rctl);
6102 } else {
6103 /* disable VLAN tag insert/strip */
6104 ctrl = rd32(E1000_CTRL);
6105 ctrl &= ~E1000_CTRL_VME;
6106 wr32(E1000_CTRL, ctrl);
6107 }
6108
6109 igb_rlpml_set(adapter);
6110
6111 if (!test_bit(__IGB_DOWN, &adapter->state))
6112 igb_irq_enable(adapter);
6113 }
6114
6115 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6116 {
6117 struct igb_adapter *adapter = netdev_priv(netdev);
6118 struct e1000_hw *hw = &adapter->hw;
6119 int pf_id = adapter->vfs_allocated_count;
6120
6121 /* attempt to add filter to vlvf array */
6122 igb_vlvf_set(adapter, vid, true, pf_id);
6123
6124 /* add the filter since PF can receive vlans w/o entry in vlvf */
6125 igb_vfta_set(hw, vid, true);
6126 }
6127
6128 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6129 {
6130 struct igb_adapter *adapter = netdev_priv(netdev);
6131 struct e1000_hw *hw = &adapter->hw;
6132 int pf_id = adapter->vfs_allocated_count;
6133 s32 err;
6134
6135 igb_irq_disable(adapter);
6136 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6137
6138 if (!test_bit(__IGB_DOWN, &adapter->state))
6139 igb_irq_enable(adapter);
6140
6141 /* remove vlan from VLVF table array */
6142 err = igb_vlvf_set(adapter, vid, false, pf_id);
6143
6144 /* if vid was not present in VLVF just remove it from table */
6145 if (err)
6146 igb_vfta_set(hw, vid, false);
6147 }
6148
6149 static void igb_restore_vlan(struct igb_adapter *adapter)
6150 {
6151 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6152
6153 if (adapter->vlgrp) {
6154 u16 vid;
6155 for (vid = 0; vid < VLAN_N_VID; vid++) {
6156 if (!vlan_group_get_device(adapter->vlgrp, vid))
6157 continue;
6158 igb_vlan_rx_add_vid(adapter->netdev, vid);
6159 }
6160 }
6161 }
6162
6163 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6164 {
6165 struct pci_dev *pdev = adapter->pdev;
6166 struct e1000_mac_info *mac = &adapter->hw.mac;
6167
6168 mac->autoneg = 0;
6169
6170 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6171 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6172 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6173 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6174 return -EINVAL;
6175 }
6176
6177 switch (spddplx) {
6178 case SPEED_10 + DUPLEX_HALF:
6179 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6180 break;
6181 case SPEED_10 + DUPLEX_FULL:
6182 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6183 break;
6184 case SPEED_100 + DUPLEX_HALF:
6185 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6186 break;
6187 case SPEED_100 + DUPLEX_FULL:
6188 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6189 break;
6190 case SPEED_1000 + DUPLEX_FULL:
6191 mac->autoneg = 1;
6192 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6193 break;
6194 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6195 default:
6196 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6197 return -EINVAL;
6198 }
6199 return 0;
6200 }
6201
6202 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6203 {
6204 struct net_device *netdev = pci_get_drvdata(pdev);
6205 struct igb_adapter *adapter = netdev_priv(netdev);
6206 struct e1000_hw *hw = &adapter->hw;
6207 u32 ctrl, rctl, status;
6208 u32 wufc = adapter->wol;
6209 #ifdef CONFIG_PM
6210 int retval = 0;
6211 #endif
6212
6213 netif_device_detach(netdev);
6214
6215 if (netif_running(netdev))
6216 igb_close(netdev);
6217
6218 igb_clear_interrupt_scheme(adapter);
6219
6220 #ifdef CONFIG_PM
6221 retval = pci_save_state(pdev);
6222 if (retval)
6223 return retval;
6224 #endif
6225
6226 status = rd32(E1000_STATUS);
6227 if (status & E1000_STATUS_LU)
6228 wufc &= ~E1000_WUFC_LNKC;
6229
6230 if (wufc) {
6231 igb_setup_rctl(adapter);
6232 igb_set_rx_mode(netdev);
6233
6234 /* turn on all-multi mode if wake on multicast is enabled */
6235 if (wufc & E1000_WUFC_MC) {
6236 rctl = rd32(E1000_RCTL);
6237 rctl |= E1000_RCTL_MPE;
6238 wr32(E1000_RCTL, rctl);
6239 }
6240
6241 ctrl = rd32(E1000_CTRL);
6242 /* advertise wake from D3Cold */
6243 #define E1000_CTRL_ADVD3WUC 0x00100000
6244 /* phy power management enable */
6245 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6246 ctrl |= E1000_CTRL_ADVD3WUC;
6247 wr32(E1000_CTRL, ctrl);
6248
6249 /* Allow time for pending master requests to run */
6250 igb_disable_pcie_master(hw);
6251
6252 wr32(E1000_WUC, E1000_WUC_PME_EN);
6253 wr32(E1000_WUFC, wufc);
6254 } else {
6255 wr32(E1000_WUC, 0);
6256 wr32(E1000_WUFC, 0);
6257 }
6258
6259 *enable_wake = wufc || adapter->en_mng_pt;
6260 if (!*enable_wake)
6261 igb_power_down_link(adapter);
6262 else
6263 igb_power_up_link(adapter);
6264
6265 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6266 * would have already happened in close and is redundant. */
6267 igb_release_hw_control(adapter);
6268
6269 pci_disable_device(pdev);
6270
6271 return 0;
6272 }
6273
6274 #ifdef CONFIG_PM
6275 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6276 {
6277 int retval;
6278 bool wake;
6279
6280 retval = __igb_shutdown(pdev, &wake);
6281 if (retval)
6282 return retval;
6283
6284 if (wake) {
6285 pci_prepare_to_sleep(pdev);
6286 } else {
6287 pci_wake_from_d3(pdev, false);
6288 pci_set_power_state(pdev, PCI_D3hot);
6289 }
6290
6291 return 0;
6292 }
6293
6294 static int igb_resume(struct pci_dev *pdev)
6295 {
6296 struct net_device *netdev = pci_get_drvdata(pdev);
6297 struct igb_adapter *adapter = netdev_priv(netdev);
6298 struct e1000_hw *hw = &adapter->hw;
6299 u32 err;
6300
6301 pci_set_power_state(pdev, PCI_D0);
6302 pci_restore_state(pdev);
6303 pci_save_state(pdev);
6304
6305 err = pci_enable_device_mem(pdev);
6306 if (err) {
6307 dev_err(&pdev->dev,
6308 "igb: Cannot enable PCI device from suspend\n");
6309 return err;
6310 }
6311 pci_set_master(pdev);
6312
6313 pci_enable_wake(pdev, PCI_D3hot, 0);
6314 pci_enable_wake(pdev, PCI_D3cold, 0);
6315
6316 if (igb_init_interrupt_scheme(adapter)) {
6317 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6318 return -ENOMEM;
6319 }
6320
6321 igb_reset(adapter);
6322
6323 /* let the f/w know that the h/w is now under the control of the
6324 * driver. */
6325 igb_get_hw_control(adapter);
6326
6327 wr32(E1000_WUS, ~0);
6328
6329 if (netif_running(netdev)) {
6330 err = igb_open(netdev);
6331 if (err)
6332 return err;
6333 }
6334
6335 netif_device_attach(netdev);
6336
6337 return 0;
6338 }
6339 #endif
6340
6341 static void igb_shutdown(struct pci_dev *pdev)
6342 {
6343 bool wake;
6344
6345 __igb_shutdown(pdev, &wake);
6346
6347 if (system_state == SYSTEM_POWER_OFF) {
6348 pci_wake_from_d3(pdev, wake);
6349 pci_set_power_state(pdev, PCI_D3hot);
6350 }
6351 }
6352
6353 #ifdef CONFIG_NET_POLL_CONTROLLER
6354 /*
6355 * Polling 'interrupt' - used by things like netconsole to send skbs
6356 * without having to re-enable interrupts. It's not called while
6357 * the interrupt routine is executing.
6358 */
6359 static void igb_netpoll(struct net_device *netdev)
6360 {
6361 struct igb_adapter *adapter = netdev_priv(netdev);
6362 struct e1000_hw *hw = &adapter->hw;
6363 int i;
6364
6365 if (!adapter->msix_entries) {
6366 struct igb_q_vector *q_vector = adapter->q_vector[0];
6367 igb_irq_disable(adapter);
6368 napi_schedule(&q_vector->napi);
6369 return;
6370 }
6371
6372 for (i = 0; i < adapter->num_q_vectors; i++) {
6373 struct igb_q_vector *q_vector = adapter->q_vector[i];
6374 wr32(E1000_EIMC, q_vector->eims_value);
6375 napi_schedule(&q_vector->napi);
6376 }
6377 }
6378 #endif /* CONFIG_NET_POLL_CONTROLLER */
6379
6380 /**
6381 * igb_io_error_detected - called when PCI error is detected
6382 * @pdev: Pointer to PCI device
6383 * @state: The current pci connection state
6384 *
6385 * This function is called after a PCI bus error affecting
6386 * this device has been detected.
6387 */
6388 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6389 pci_channel_state_t state)
6390 {
6391 struct net_device *netdev = pci_get_drvdata(pdev);
6392 struct igb_adapter *adapter = netdev_priv(netdev);
6393
6394 netif_device_detach(netdev);
6395
6396 if (state == pci_channel_io_perm_failure)
6397 return PCI_ERS_RESULT_DISCONNECT;
6398
6399 if (netif_running(netdev))
6400 igb_down(adapter);
6401 pci_disable_device(pdev);
6402
6403 /* Request a slot slot reset. */
6404 return PCI_ERS_RESULT_NEED_RESET;
6405 }
6406
6407 /**
6408 * igb_io_slot_reset - called after the pci bus has been reset.
6409 * @pdev: Pointer to PCI device
6410 *
6411 * Restart the card from scratch, as if from a cold-boot. Implementation
6412 * resembles the first-half of the igb_resume routine.
6413 */
6414 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6415 {
6416 struct net_device *netdev = pci_get_drvdata(pdev);
6417 struct igb_adapter *adapter = netdev_priv(netdev);
6418 struct e1000_hw *hw = &adapter->hw;
6419 pci_ers_result_t result;
6420 int err;
6421
6422 if (pci_enable_device_mem(pdev)) {
6423 dev_err(&pdev->dev,
6424 "Cannot re-enable PCI device after reset.\n");
6425 result = PCI_ERS_RESULT_DISCONNECT;
6426 } else {
6427 pci_set_master(pdev);
6428 pci_restore_state(pdev);
6429 pci_save_state(pdev);
6430
6431 pci_enable_wake(pdev, PCI_D3hot, 0);
6432 pci_enable_wake(pdev, PCI_D3cold, 0);
6433
6434 igb_reset(adapter);
6435 wr32(E1000_WUS, ~0);
6436 result = PCI_ERS_RESULT_RECOVERED;
6437 }
6438
6439 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6440 if (err) {
6441 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6442 "failed 0x%0x\n", err);
6443 /* non-fatal, continue */
6444 }
6445
6446 return result;
6447 }
6448
6449 /**
6450 * igb_io_resume - called when traffic can start flowing again.
6451 * @pdev: Pointer to PCI device
6452 *
6453 * This callback is called when the error recovery driver tells us that
6454 * its OK to resume normal operation. Implementation resembles the
6455 * second-half of the igb_resume routine.
6456 */
6457 static void igb_io_resume(struct pci_dev *pdev)
6458 {
6459 struct net_device *netdev = pci_get_drvdata(pdev);
6460 struct igb_adapter *adapter = netdev_priv(netdev);
6461
6462 if (netif_running(netdev)) {
6463 if (igb_up(adapter)) {
6464 dev_err(&pdev->dev, "igb_up failed after reset\n");
6465 return;
6466 }
6467 }
6468
6469 netif_device_attach(netdev);
6470
6471 /* let the f/w know that the h/w is now under the control of the
6472 * driver. */
6473 igb_get_hw_control(adapter);
6474 }
6475
6476 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6477 u8 qsel)
6478 {
6479 u32 rar_low, rar_high;
6480 struct e1000_hw *hw = &adapter->hw;
6481
6482 /* HW expects these in little endian so we reverse the byte order
6483 * from network order (big endian) to little endian
6484 */
6485 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6486 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6487 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6488
6489 /* Indicate to hardware the Address is Valid. */
6490 rar_high |= E1000_RAH_AV;
6491
6492 if (hw->mac.type == e1000_82575)
6493 rar_high |= E1000_RAH_POOL_1 * qsel;
6494 else
6495 rar_high |= E1000_RAH_POOL_1 << qsel;
6496
6497 wr32(E1000_RAL(index), rar_low);
6498 wrfl();
6499 wr32(E1000_RAH(index), rar_high);
6500 wrfl();
6501 }
6502
6503 static int igb_set_vf_mac(struct igb_adapter *adapter,
6504 int vf, unsigned char *mac_addr)
6505 {
6506 struct e1000_hw *hw = &adapter->hw;
6507 /* VF MAC addresses start at end of receive addresses and moves
6508 * torwards the first, as a result a collision should not be possible */
6509 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6510
6511 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6512
6513 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6514
6515 return 0;
6516 }
6517
6518 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6519 {
6520 struct igb_adapter *adapter = netdev_priv(netdev);
6521 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6522 return -EINVAL;
6523 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6524 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6525 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6526 " change effective.");
6527 if (test_bit(__IGB_DOWN, &adapter->state)) {
6528 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6529 " but the PF device is not up.\n");
6530 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6531 " attempting to use the VF device.\n");
6532 }
6533 return igb_set_vf_mac(adapter, vf, mac);
6534 }
6535
6536 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6537 {
6538 return -EOPNOTSUPP;
6539 }
6540
6541 static int igb_ndo_get_vf_config(struct net_device *netdev,
6542 int vf, struct ifla_vf_info *ivi)
6543 {
6544 struct igb_adapter *adapter = netdev_priv(netdev);
6545 if (vf >= adapter->vfs_allocated_count)
6546 return -EINVAL;
6547 ivi->vf = vf;
6548 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6549 ivi->tx_rate = 0;
6550 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6551 ivi->qos = adapter->vf_data[vf].pf_qos;
6552 return 0;
6553 }
6554
6555 static void igb_vmm_control(struct igb_adapter *adapter)
6556 {
6557 struct e1000_hw *hw = &adapter->hw;
6558 u32 reg;
6559
6560 switch (hw->mac.type) {
6561 case e1000_82575:
6562 default:
6563 /* replication is not supported for 82575 */
6564 return;
6565 case e1000_82576:
6566 /* notify HW that the MAC is adding vlan tags */
6567 reg = rd32(E1000_DTXCTL);
6568 reg |= E1000_DTXCTL_VLAN_ADDED;
6569 wr32(E1000_DTXCTL, reg);
6570 case e1000_82580:
6571 /* enable replication vlan tag stripping */
6572 reg = rd32(E1000_RPLOLR);
6573 reg |= E1000_RPLOLR_STRVLAN;
6574 wr32(E1000_RPLOLR, reg);
6575 case e1000_i350:
6576 /* none of the above registers are supported by i350 */
6577 break;
6578 }
6579
6580 if (adapter->vfs_allocated_count) {
6581 igb_vmdq_set_loopback_pf(hw, true);
6582 igb_vmdq_set_replication_pf(hw, true);
6583 } else {
6584 igb_vmdq_set_loopback_pf(hw, false);
6585 igb_vmdq_set_replication_pf(hw, false);
6586 }
6587 }
6588
6589 /* igb_main.c */
x86 "subsystem" repository