?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
igb: Some fine tuning
[linux-2.6/cjktty.git] / drivers / net / igb / igb_main.c
bloba364ae69ab372b7c93c9d67a42a2b033de083a10
1 /*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <linux/slab.h>
36 #include <net/checksum.h>
37 #include <net/ip6_checksum.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
41 #include <linux/if_vlan.h>
42 #include <linux/pci.h>
43 #include <linux/pci-aspm.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/if_ether.h>
47 #include <linux/aer.h>
48 #ifdef CONFIG_IGB_DCA
49 #include <linux/dca.h>
50 #endif
51 #include "igb.h"
52
53 #define DRV_VERSION "2.1.0-k2"
54 char igb_driver_name[] = "igb";
55 char igb_driver_version[] = DRV_VERSION;
56 static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
59
60 static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62 };
63
64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
89 /* required last entry */
90 {0, }
91 };
92
93 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
94
95 void igb_reset(struct igb_adapter *);
96 static int igb_setup_all_tx_resources(struct igb_adapter *);
97 static int igb_setup_all_rx_resources(struct igb_adapter *);
98 static void igb_free_all_tx_resources(struct igb_adapter *);
99 static void igb_free_all_rx_resources(struct igb_adapter *);
100 static void igb_setup_mrqc(struct igb_adapter *);
101 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
102 static void __devexit igb_remove(struct pci_dev *pdev);
103 static int igb_sw_init(struct igb_adapter *);
104 static int igb_open(struct net_device *);
105 static int igb_close(struct net_device *);
106 static void igb_configure_tx(struct igb_adapter *);
107 static void igb_configure_rx(struct igb_adapter *);
108 static void igb_clean_all_tx_rings(struct igb_adapter *);
109 static void igb_clean_all_rx_rings(struct igb_adapter *);
110 static void igb_clean_tx_ring(struct igb_ring *);
111 static void igb_clean_rx_ring(struct igb_ring *);
112 static void igb_set_rx_mode(struct net_device *);
113 static void igb_update_phy_info(unsigned long);
114 static void igb_watchdog(unsigned long);
115 static void igb_watchdog_task(struct work_struct *);
116 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
117 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
118 struct rtnl_link_stats64 *stats);
119 static int igb_change_mtu(struct net_device *, int);
120 static int igb_set_mac(struct net_device *, void *);
121 static void igb_set_uta(struct igb_adapter *adapter);
122 static irqreturn_t igb_intr(int irq, void *);
123 static irqreturn_t igb_intr_msi(int irq, void *);
124 static irqreturn_t igb_msix_other(int irq, void *);
125 static irqreturn_t igb_msix_ring(int irq, void *);
126 #ifdef CONFIG_IGB_DCA
127 static void igb_update_dca(struct igb_q_vector *);
128 static void igb_setup_dca(struct igb_adapter *);
129 #endif /* CONFIG_IGB_DCA */
130 static bool igb_clean_tx_irq(struct igb_q_vector *);
131 static int igb_poll(struct napi_struct *, int);
132 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
133 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
134 static void igb_tx_timeout(struct net_device *);
135 static void igb_reset_task(struct work_struct *);
136 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
137 static void igb_vlan_rx_add_vid(struct net_device *, u16);
138 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
139 static void igb_restore_vlan(struct igb_adapter *);
140 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
141 static void igb_ping_all_vfs(struct igb_adapter *);
142 static void igb_msg_task(struct igb_adapter *);
143 static void igb_vmm_control(struct igb_adapter *);
144 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
145 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
146 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
147 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
148 int vf, u16 vlan, u8 qos);
149 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
150 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
151 struct ifla_vf_info *ivi);
152
153 #ifdef CONFIG_PM
154 static int igb_suspend(struct pci_dev *, pm_message_t);
155 static int igb_resume(struct pci_dev *);
156 #endif
157 static void igb_shutdown(struct pci_dev *);
158 #ifdef CONFIG_IGB_DCA
159 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
160 static struct notifier_block dca_notifier = {
161 .notifier_call = igb_notify_dca,
162 .next = NULL,
163 .priority = 0
164 };
165 #endif
166 #ifdef CONFIG_NET_POLL_CONTROLLER
167 /* for netdump / net console */
168 static void igb_netpoll(struct net_device *);
169 #endif
170 #ifdef CONFIG_PCI_IOV
171 static unsigned int max_vfs = 0;
172 module_param(max_vfs, uint, 0);
173 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
174 "per physical function");
175 #endif /* CONFIG_PCI_IOV */
176
177 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
178 pci_channel_state_t);
179 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
180 static void igb_io_resume(struct pci_dev *);
181
182 static struct pci_error_handlers igb_err_handler = {
183 .error_detected = igb_io_error_detected,
184 .slot_reset = igb_io_slot_reset,
185 .resume = igb_io_resume,
186 };
187
188
189 static struct pci_driver igb_driver = {
190 .name = igb_driver_name,
191 .id_table = igb_pci_tbl,
192 .probe = igb_probe,
193 .remove = __devexit_p(igb_remove),
194 #ifdef CONFIG_PM
195 /* Power Managment Hooks */
196 .suspend = igb_suspend,
197 .resume = igb_resume,
198 #endif
199 .shutdown = igb_shutdown,
200 .err_handler = &igb_err_handler
201 };
202
203 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
204 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
205 MODULE_LICENSE("GPL");
206 MODULE_VERSION(DRV_VERSION);
207
208 struct igb_reg_info {
209 u32 ofs;
210 char *name;
211 };
212
213 static const struct igb_reg_info igb_reg_info_tbl[] = {
214
215 /* General Registers */
216 {E1000_CTRL, "CTRL"},
217 {E1000_STATUS, "STATUS"},
218 {E1000_CTRL_EXT, "CTRL_EXT"},
219
220 /* Interrupt Registers */
221 {E1000_ICR, "ICR"},
222
223 /* RX Registers */
224 {E1000_RCTL, "RCTL"},
225 {E1000_RDLEN(0), "RDLEN"},
226 {E1000_RDH(0), "RDH"},
227 {E1000_RDT(0), "RDT"},
228 {E1000_RXDCTL(0), "RXDCTL"},
229 {E1000_RDBAL(0), "RDBAL"},
230 {E1000_RDBAH(0), "RDBAH"},
231
232 /* TX Registers */
233 {E1000_TCTL, "TCTL"},
234 {E1000_TDBAL(0), "TDBAL"},
235 {E1000_TDBAH(0), "TDBAH"},
236 {E1000_TDLEN(0), "TDLEN"},
237 {E1000_TDH(0), "TDH"},
238 {E1000_TDT(0), "TDT"},
239 {E1000_TXDCTL(0), "TXDCTL"},
240 {E1000_TDFH, "TDFH"},
241 {E1000_TDFT, "TDFT"},
242 {E1000_TDFHS, "TDFHS"},
243 {E1000_TDFPC, "TDFPC"},
244
245 /* List Terminator */
246 {}
247 };
248
249 /*
250 * igb_regdump - register printout routine
251 */
252 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
253 {
254 int n = 0;
255 char rname[16];
256 u32 regs[8];
257
258 switch (reginfo->ofs) {
259 case E1000_RDLEN(0):
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDLEN(n));
262 break;
263 case E1000_RDH(0):
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDH(n));
266 break;
267 case E1000_RDT(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RDT(n));
270 break;
271 case E1000_RXDCTL(0):
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RXDCTL(n));
274 break;
275 case E1000_RDBAL(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAL(n));
278 break;
279 case E1000_RDBAH(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAH(n));
282 break;
283 case E1000_TDBAL(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_RDBAL(n));
286 break;
287 case E1000_TDBAH(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDBAH(n));
290 break;
291 case E1000_TDLEN(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDLEN(n));
294 break;
295 case E1000_TDH(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDH(n));
298 break;
299 case E1000_TDT(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TDT(n));
302 break;
303 case E1000_TXDCTL(0):
304 for (n = 0; n < 4; n++)
305 regs[n] = rd32(E1000_TXDCTL(n));
306 break;
307 default:
308 printk(KERN_INFO "%-15s %08x\n",
309 reginfo->name, rd32(reginfo->ofs));
310 return;
311 }
312
313 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
314 printk(KERN_INFO "%-15s ", rname);
315 for (n = 0; n < 4; n++)
316 printk(KERN_CONT "%08x ", regs[n]);
317 printk(KERN_CONT "\n");
318 }
319
320 /*
321 * igb_dump - Print registers, tx-rings and rx-rings
322 */
323 static void igb_dump(struct igb_adapter *adapter)
324 {
325 struct net_device *netdev = adapter->netdev;
326 struct e1000_hw *hw = &adapter->hw;
327 struct igb_reg_info *reginfo;
328 int n = 0;
329 struct igb_ring *tx_ring;
330 union e1000_adv_tx_desc *tx_desc;
331 struct my_u0 { u64 a; u64 b; } *u0;
332 struct igb_buffer *buffer_info;
333 struct igb_ring *rx_ring;
334 union e1000_adv_rx_desc *rx_desc;
335 u32 staterr;
336 int i = 0;
337
338 if (!netif_msg_hw(adapter))
339 return;
340
341 /* Print netdevice Info */
342 if (netdev) {
343 dev_info(&adapter->pdev->dev, "Net device Info\n");
344 printk(KERN_INFO "Device Name state "
345 "trans_start last_rx\n");
346 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
347 netdev->name,
348 netdev->state,
349 netdev->trans_start,
350 netdev->last_rx);
351 }
352
353 /* Print Registers */
354 dev_info(&adapter->pdev->dev, "Register Dump\n");
355 printk(KERN_INFO " Register Name Value\n");
356 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
357 reginfo->name; reginfo++) {
358 igb_regdump(hw, reginfo);
359 }
360
361 /* Print TX Ring Summary */
362 if (!netdev || !netif_running(netdev))
363 goto exit;
364
365 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
366 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
367 " leng ntw timestamp\n");
368 for (n = 0; n < adapter->num_tx_queues; n++) {
369 tx_ring = adapter->tx_ring[n];
370 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
371 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
372 n, tx_ring->next_to_use, tx_ring->next_to_clean,
373 (u64)buffer_info->dma,
374 buffer_info->length,
375 buffer_info->next_to_watch,
376 (u64)buffer_info->time_stamp);
377 }
378
379 /* Print TX Rings */
380 if (!netif_msg_tx_done(adapter))
381 goto rx_ring_summary;
382
383 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
384
385 /* Transmit Descriptor Formats
386 *
387 * Advanced Transmit Descriptor
388 * +--------------------------------------------------------------+
389 * 0 | Buffer Address [63:0] |
390 * +--------------------------------------------------------------+
391 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
392 * +--------------------------------------------------------------+
393 * 63 46 45 40 39 38 36 35 32 31 24 15 0
394 */
395
396 for (n = 0; n < adapter->num_tx_queues; n++) {
397 tx_ring = adapter->tx_ring[n];
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
400 printk(KERN_INFO "------------------------------------\n");
401 printk(KERN_INFO "T [desc] [address 63:0 ] "
402 "[PlPOCIStDDM Ln] [bi->dma ] "
403 "leng ntw timestamp bi->skb\n");
404
405 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
406 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
407 buffer_info = &tx_ring->buffer_info[i];
408 u0 = (struct my_u0 *)tx_desc;
409 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
410 " %04X %3X %016llX %p", i,
411 le64_to_cpu(u0->a),
412 le64_to_cpu(u0->b),
413 (u64)buffer_info->dma,
414 buffer_info->length,
415 buffer_info->next_to_watch,
416 (u64)buffer_info->time_stamp,
417 buffer_info->skb);
418 if (i == tx_ring->next_to_use &&
419 i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC/U\n");
421 else if (i == tx_ring->next_to_use)
422 printk(KERN_CONT " NTU\n");
423 else if (i == tx_ring->next_to_clean)
424 printk(KERN_CONT " NTC\n");
425 else
426 printk(KERN_CONT "\n");
427
428 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
429 print_hex_dump(KERN_INFO, "",
430 DUMP_PREFIX_ADDRESS,
431 16, 1, phys_to_virt(buffer_info->dma),
432 buffer_info->length, true);
433 }
434 }
435
436 /* Print RX Rings Summary */
437 rx_ring_summary:
438 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
439 printk(KERN_INFO "Queue [NTU] [NTC]\n");
440 for (n = 0; n < adapter->num_rx_queues; n++) {
441 rx_ring = adapter->rx_ring[n];
442 printk(KERN_INFO " %5d %5X %5X\n", n,
443 rx_ring->next_to_use, rx_ring->next_to_clean);
444 }
445
446 /* Print RX Rings */
447 if (!netif_msg_rx_status(adapter))
448 goto exit;
449
450 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
451
452 /* Advanced Receive Descriptor (Read) Format
453 * 63 1 0
454 * +-----------------------------------------------------+
455 * 0 | Packet Buffer Address [63:1] |A0/NSE|
456 * +----------------------------------------------+------+
457 * 8 | Header Buffer Address [63:1] | DD |
458 * +-----------------------------------------------------+
459 *
460 *
461 * Advanced Receive Descriptor (Write-Back) Format
462 *
463 * 63 48 47 32 31 30 21 20 17 16 4 3 0
464 * +------------------------------------------------------+
465 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
466 * | Checksum Ident | | | | Type | Type |
467 * +------------------------------------------------------+
468 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
469 * +------------------------------------------------------+
470 * 63 48 47 32 31 20 19 0
471 */
472
473 for (n = 0; n < adapter->num_rx_queues; n++) {
474 rx_ring = adapter->rx_ring[n];
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
477 printk(KERN_INFO "------------------------------------\n");
478 printk(KERN_INFO "R [desc] [ PktBuf A0] "
479 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
480 "<-- Adv Rx Read format\n");
481 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
482 "[vl er S cks ln] ---------------- [bi->skb] "
483 "<-- Adv Rx Write-Back format\n");
484
485 for (i = 0; i < rx_ring->count; i++) {
486 buffer_info = &rx_ring->buffer_info[i];
487 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
488 u0 = (struct my_u0 *)rx_desc;
489 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
490 if (staterr & E1000_RXD_STAT_DD) {
491 /* Descriptor Done */
492 printk(KERN_INFO "RWB[0x%03X] %016llX "
493 "%016llX ---------------- %p", i,
494 le64_to_cpu(u0->a),
495 le64_to_cpu(u0->b),
496 buffer_info->skb);
497 } else {
498 printk(KERN_INFO "R [0x%03X] %016llX "
499 "%016llX %016llX %p", i,
500 le64_to_cpu(u0->a),
501 le64_to_cpu(u0->b),
502 (u64)buffer_info->dma,
503 buffer_info->skb);
504
505 if (netif_msg_pktdata(adapter)) {
506 print_hex_dump(KERN_INFO, "",
507 DUMP_PREFIX_ADDRESS,
508 16, 1,
509 phys_to_virt(buffer_info->dma),
510 rx_ring->rx_buffer_len, true);
511 if (rx_ring->rx_buffer_len
512 < IGB_RXBUFFER_1024)
513 print_hex_dump(KERN_INFO, "",
514 DUMP_PREFIX_ADDRESS,
515 16, 1,
516 phys_to_virt(
517 buffer_info->page_dma +
518 buffer_info->page_offset),
519 PAGE_SIZE/2, true);
520 }
521 }
522
523 if (i == rx_ring->next_to_use)
524 printk(KERN_CONT " NTU\n");
525 else if (i == rx_ring->next_to_clean)
526 printk(KERN_CONT " NTC\n");
527 else
528 printk(KERN_CONT "\n");
529
530 }
531 }
532
533 exit:
534 return;
535 }
536
537
538 /**
539 * igb_read_clock - read raw cycle counter (to be used by time counter)
540 */
541 static cycle_t igb_read_clock(const struct cyclecounter *tc)
542 {
543 struct igb_adapter *adapter =
544 container_of(tc, struct igb_adapter, cycles);
545 struct e1000_hw *hw = &adapter->hw;
546 u64 stamp = 0;
547 int shift = 0;
548
549 /*
550 * The timestamp latches on lowest register read. For the 82580
551 * the lowest register is SYSTIMR instead of SYSTIML. However we never
552 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
553 */
554 if (hw->mac.type == e1000_82580) {
555 stamp = rd32(E1000_SYSTIMR) >> 8;
556 shift = IGB_82580_TSYNC_SHIFT;
557 }
558
559 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
560 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
561 return stamp;
562 }
563
564 /**
565 * igb_get_hw_dev - return device
566 * used by hardware layer to print debugging information
567 **/
568 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
569 {
570 struct igb_adapter *adapter = hw->back;
571 return adapter->netdev;
572 }
573
574 /**
575 * igb_init_module - Driver Registration Routine
576 *
577 * igb_init_module is the first routine called when the driver is
578 * loaded. All it does is register with the PCI subsystem.
579 **/
580 static int __init igb_init_module(void)
581 {
582 int ret;
583 printk(KERN_INFO "%s - version %s\n",
584 igb_driver_string, igb_driver_version);
585
586 printk(KERN_INFO "%s\n", igb_copyright);
587
588 #ifdef CONFIG_IGB_DCA
589 dca_register_notify(&dca_notifier);
590 #endif
591 ret = pci_register_driver(&igb_driver);
592 return ret;
593 }
594
595 module_init(igb_init_module);
596
597 /**
598 * igb_exit_module - Driver Exit Cleanup Routine
599 *
600 * igb_exit_module is called just before the driver is removed
601 * from memory.
602 **/
603 static void __exit igb_exit_module(void)
604 {
605 #ifdef CONFIG_IGB_DCA
606 dca_unregister_notify(&dca_notifier);
607 #endif
608 pci_unregister_driver(&igb_driver);
609 }
610
611 module_exit(igb_exit_module);
612
613 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
614 /**
615 * igb_cache_ring_register - Descriptor ring to register mapping
616 * @adapter: board private structure to initialize
617 *
618 * Once we know the feature-set enabled for the device, we'll cache
619 * the register offset the descriptor ring is assigned to.
620 **/
621 static void igb_cache_ring_register(struct igb_adapter *adapter)
622 {
623 int i = 0, j = 0;
624 u32 rbase_offset = adapter->vfs_allocated_count;
625
626 switch (adapter->hw.mac.type) {
627 case e1000_82576:
628 /* The queues are allocated for virtualization such that VF 0
629 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
630 * In order to avoid collision we start at the first free queue
631 * and continue consuming queues in the same sequence
632 */
633 if (adapter->vfs_allocated_count) {
634 for (; i < adapter->rss_queues; i++)
635 adapter->rx_ring[i]->reg_idx = rbase_offset +
636 Q_IDX_82576(i);
637 }
638 case e1000_82575:
639 case e1000_82580:
640 case e1000_i350:
641 default:
642 for (; i < adapter->num_rx_queues; i++)
643 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
644 for (; j < adapter->num_tx_queues; j++)
645 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
646 break;
647 }
648 }
649
650 static void igb_free_queues(struct igb_adapter *adapter)
651 {
652 int i;
653
654 for (i = 0; i < adapter->num_tx_queues; i++) {
655 kfree(adapter->tx_ring[i]);
656 adapter->tx_ring[i] = NULL;
657 }
658 for (i = 0; i < adapter->num_rx_queues; i++) {
659 kfree(adapter->rx_ring[i]);
660 adapter->rx_ring[i] = NULL;
661 }
662 adapter->num_rx_queues = 0;
663 adapter->num_tx_queues = 0;
664 }
665
666 /**
667 * igb_alloc_queues - Allocate memory for all rings
668 * @adapter: board private structure to initialize
669 *
670 * We allocate one ring per queue at run-time since we don't know the
671 * number of queues at compile-time.
672 **/
673 static int igb_alloc_queues(struct igb_adapter *adapter)
674 {
675 struct igb_ring *ring;
676 int i;
677
678 for (i = 0; i < adapter->num_tx_queues; i++) {
679 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
680 if (!ring)
681 goto err;
682 ring->count = adapter->tx_ring_count;
683 ring->queue_index = i;
684 ring->dev = &adapter->pdev->dev;
685 ring->netdev = adapter->netdev;
686 /* For 82575, context index must be unique per ring. */
687 if (adapter->hw.mac.type == e1000_82575)
688 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
689 adapter->tx_ring[i] = ring;
690 }
691
692 for (i = 0; i < adapter->num_rx_queues; i++) {
693 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
694 if (!ring)
695 goto err;
696 ring->count = adapter->rx_ring_count;
697 ring->queue_index = i;
698 ring->dev = &adapter->pdev->dev;
699 ring->netdev = adapter->netdev;
700 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
701 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
702 /* set flag indicating ring supports SCTP checksum offload */
703 if (adapter->hw.mac.type >= e1000_82576)
704 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
705 adapter->rx_ring[i] = ring;
706 }
707
708 igb_cache_ring_register(adapter);
709
710 return 0;
711
712 err:
713 igb_free_queues(adapter);
714
715 return -ENOMEM;
716 }
717
718 #define IGB_N0_QUEUE -1
719 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
720 {
721 u32 msixbm = 0;
722 struct igb_adapter *adapter = q_vector->adapter;
723 struct e1000_hw *hw = &adapter->hw;
724 u32 ivar, index;
725 int rx_queue = IGB_N0_QUEUE;
726 int tx_queue = IGB_N0_QUEUE;
727
728 if (q_vector->rx_ring)
729 rx_queue = q_vector->rx_ring->reg_idx;
730 if (q_vector->tx_ring)
731 tx_queue = q_vector->tx_ring->reg_idx;
732
733 switch (hw->mac.type) {
734 case e1000_82575:
735 /* The 82575 assigns vectors using a bitmask, which matches the
736 bitmask for the EICR/EIMS/EIMC registers. To assign one
737 or more queues to a vector, we write the appropriate bits
738 into the MSIXBM register for that vector. */
739 if (rx_queue > IGB_N0_QUEUE)
740 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
741 if (tx_queue > IGB_N0_QUEUE)
742 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
743 if (!adapter->msix_entries && msix_vector == 0)
744 msixbm |= E1000_EIMS_OTHER;
745 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
746 q_vector->eims_value = msixbm;
747 break;
748 case e1000_82576:
749 /* 82576 uses a table-based method for assigning vectors.
750 Each queue has a single entry in the table to which we write
751 a vector number along with a "valid" bit. Sadly, the layout
752 of the table is somewhat counterintuitive. */
753 if (rx_queue > IGB_N0_QUEUE) {
754 index = (rx_queue & 0x7);
755 ivar = array_rd32(E1000_IVAR0, index);
756 if (rx_queue < 8) {
757 /* vector goes into low byte of register */
758 ivar = ivar & 0xFFFFFF00;
759 ivar |= msix_vector | E1000_IVAR_VALID;
760 } else {
761 /* vector goes into third byte of register */
762 ivar = ivar & 0xFF00FFFF;
763 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
764 }
765 array_wr32(E1000_IVAR0, index, ivar);
766 }
767 if (tx_queue > IGB_N0_QUEUE) {
768 index = (tx_queue & 0x7);
769 ivar = array_rd32(E1000_IVAR0, index);
770 if (tx_queue < 8) {
771 /* vector goes into second byte of register */
772 ivar = ivar & 0xFFFF00FF;
773 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
774 } else {
775 /* vector goes into high byte of register */
776 ivar = ivar & 0x00FFFFFF;
777 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
778 }
779 array_wr32(E1000_IVAR0, index, ivar);
780 }
781 q_vector->eims_value = 1 << msix_vector;
782 break;
783 case e1000_82580:
784 case e1000_i350:
785 /* 82580 uses the same table-based approach as 82576 but has fewer
786 entries as a result we carry over for queues greater than 4. */
787 if (rx_queue > IGB_N0_QUEUE) {
788 index = (rx_queue >> 1);
789 ivar = array_rd32(E1000_IVAR0, index);
790 if (rx_queue & 0x1) {
791 /* vector goes into third byte of register */
792 ivar = ivar & 0xFF00FFFF;
793 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
794 } else {
795 /* vector goes into low byte of register */
796 ivar = ivar & 0xFFFFFF00;
797 ivar |= msix_vector | E1000_IVAR_VALID;
798 }
799 array_wr32(E1000_IVAR0, index, ivar);
800 }
801 if (tx_queue > IGB_N0_QUEUE) {
802 index = (tx_queue >> 1);
803 ivar = array_rd32(E1000_IVAR0, index);
804 if (tx_queue & 0x1) {
805 /* vector goes into high byte of register */
806 ivar = ivar & 0x00FFFFFF;
807 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
808 } else {
809 /* vector goes into second byte of register */
810 ivar = ivar & 0xFFFF00FF;
811 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
812 }
813 array_wr32(E1000_IVAR0, index, ivar);
814 }
815 q_vector->eims_value = 1 << msix_vector;
816 break;
817 default:
818 BUG();
819 break;
820 }
821
822 /* add q_vector eims value to global eims_enable_mask */
823 adapter->eims_enable_mask |= q_vector->eims_value;
824
825 /* configure q_vector to set itr on first interrupt */
826 q_vector->set_itr = 1;
827 }
828
829 /**
830 * igb_configure_msix - Configure MSI-X hardware
831 *
832 * igb_configure_msix sets up the hardware to properly
833 * generate MSI-X interrupts.
834 **/
835 static void igb_configure_msix(struct igb_adapter *adapter)
836 {
837 u32 tmp;
838 int i, vector = 0;
839 struct e1000_hw *hw = &adapter->hw;
840
841 adapter->eims_enable_mask = 0;
842
843 /* set vector for other causes, i.e. link changes */
844 switch (hw->mac.type) {
845 case e1000_82575:
846 tmp = rd32(E1000_CTRL_EXT);
847 /* enable MSI-X PBA support*/
848 tmp |= E1000_CTRL_EXT_PBA_CLR;
849
850 /* Auto-Mask interrupts upon ICR read. */
851 tmp |= E1000_CTRL_EXT_EIAME;
852 tmp |= E1000_CTRL_EXT_IRCA;
853
854 wr32(E1000_CTRL_EXT, tmp);
855
856 /* enable msix_other interrupt */
857 array_wr32(E1000_MSIXBM(0), vector++,
858 E1000_EIMS_OTHER);
859 adapter->eims_other = E1000_EIMS_OTHER;
860
861 break;
862
863 case e1000_82576:
864 case e1000_82580:
865 case e1000_i350:
866 /* Turn on MSI-X capability first, or our settings
867 * won't stick. And it will take days to debug. */
868 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
869 E1000_GPIE_PBA | E1000_GPIE_EIAME |
870 E1000_GPIE_NSICR);
871
872 /* enable msix_other interrupt */
873 adapter->eims_other = 1 << vector;
874 tmp = (vector++ | E1000_IVAR_VALID) << 8;
875
876 wr32(E1000_IVAR_MISC, tmp);
877 break;
878 default:
879 /* do nothing, since nothing else supports MSI-X */
880 break;
881 } /* switch (hw->mac.type) */
882
883 adapter->eims_enable_mask |= adapter->eims_other;
884
885 for (i = 0; i < adapter->num_q_vectors; i++)
886 igb_assign_vector(adapter->q_vector[i], vector++);
887
888 wrfl();
889 }
890
891 /**
892 * igb_request_msix - Initialize MSI-X interrupts
893 *
894 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
895 * kernel.
896 **/
897 static int igb_request_msix(struct igb_adapter *adapter)
898 {
899 struct net_device *netdev = adapter->netdev;
900 struct e1000_hw *hw = &adapter->hw;
901 int i, err = 0, vector = 0;
902
903 err = request_irq(adapter->msix_entries[vector].vector,
904 igb_msix_other, 0, netdev->name, adapter);
905 if (err)
906 goto out;
907 vector++;
908
909 for (i = 0; i < adapter->num_q_vectors; i++) {
910 struct igb_q_vector *q_vector = adapter->q_vector[i];
911
912 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
913
914 if (q_vector->rx_ring && q_vector->tx_ring)
915 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
916 q_vector->rx_ring->queue_index);
917 else if (q_vector->tx_ring)
918 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
919 q_vector->tx_ring->queue_index);
920 else if (q_vector->rx_ring)
921 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
922 q_vector->rx_ring->queue_index);
923 else
924 sprintf(q_vector->name, "%s-unused", netdev->name);
925
926 err = request_irq(adapter->msix_entries[vector].vector,
927 igb_msix_ring, 0, q_vector->name,
928 q_vector);
929 if (err)
930 goto out;
931 vector++;
932 }
933
934 igb_configure_msix(adapter);
935 return 0;
936 out:
937 return err;
938 }
939
940 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
941 {
942 if (adapter->msix_entries) {
943 pci_disable_msix(adapter->pdev);
944 kfree(adapter->msix_entries);
945 adapter->msix_entries = NULL;
946 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
947 pci_disable_msi(adapter->pdev);
948 }
949 }
950
951 /**
952 * igb_free_q_vectors - Free memory allocated for interrupt vectors
953 * @adapter: board private structure to initialize
954 *
955 * This function frees the memory allocated to the q_vectors. In addition if
956 * NAPI is enabled it will delete any references to the NAPI struct prior
957 * to freeing the q_vector.
958 **/
959 static void igb_free_q_vectors(struct igb_adapter *adapter)
960 {
961 int v_idx;
962
963 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
964 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
965 adapter->q_vector[v_idx] = NULL;
966 if (!q_vector)
967 continue;
968 netif_napi_del(&q_vector->napi);
969 kfree(q_vector);
970 }
971 adapter->num_q_vectors = 0;
972 }
973
974 /**
975 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
976 *
977 * This function resets the device so that it has 0 rx queues, tx queues, and
978 * MSI-X interrupts allocated.
979 */
980 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
981 {
982 igb_free_queues(adapter);
983 igb_free_q_vectors(adapter);
984 igb_reset_interrupt_capability(adapter);
985 }
986
987 /**
988 * igb_set_interrupt_capability - set MSI or MSI-X if supported
989 *
990 * Attempt to configure interrupts using the best available
991 * capabilities of the hardware and kernel.
992 **/
993 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
994 {
995 int err;
996 int numvecs, i;
997
998 /* Number of supported queues. */
999 adapter->num_rx_queues = adapter->rss_queues;
1000 if (adapter->vfs_allocated_count)
1001 adapter->num_tx_queues = 1;
1002 else
1003 adapter->num_tx_queues = adapter->rss_queues;
1004
1005 /* start with one vector for every rx queue */
1006 numvecs = adapter->num_rx_queues;
1007
1008 /* if tx handler is separate add 1 for every tx queue */
1009 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1010 numvecs += adapter->num_tx_queues;
1011
1012 /* store the number of vectors reserved for queues */
1013 adapter->num_q_vectors = numvecs;
1014
1015 /* add 1 vector for link status interrupts */
1016 numvecs++;
1017 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1018 GFP_KERNEL);
1019 if (!adapter->msix_entries)
1020 goto msi_only;
1021
1022 for (i = 0; i < numvecs; i++)
1023 adapter->msix_entries[i].entry = i;
1024
1025 err = pci_enable_msix(adapter->pdev,
1026 adapter->msix_entries,
1027 numvecs);
1028 if (err == 0)
1029 goto out;
1030
1031 igb_reset_interrupt_capability(adapter);
1032
1033 /* If we can't do MSI-X, try MSI */
1034 msi_only:
1035 #ifdef CONFIG_PCI_IOV
1036 /* disable SR-IOV for non MSI-X configurations */
1037 if (adapter->vf_data) {
1038 struct e1000_hw *hw = &adapter->hw;
1039 /* disable iov and allow time for transactions to clear */
1040 pci_disable_sriov(adapter->pdev);
1041 msleep(500);
1042
1043 kfree(adapter->vf_data);
1044 adapter->vf_data = NULL;
1045 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1046 msleep(100);
1047 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1048 }
1049 #endif
1050 adapter->vfs_allocated_count = 0;
1051 adapter->rss_queues = 1;
1052 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1053 adapter->num_rx_queues = 1;
1054 adapter->num_tx_queues = 1;
1055 adapter->num_q_vectors = 1;
1056 if (!pci_enable_msi(adapter->pdev))
1057 adapter->flags |= IGB_FLAG_HAS_MSI;
1058 out:
1059 /* Notify the stack of the (possibly) reduced queue counts. */
1060 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1061 return netif_set_real_num_rx_queues(adapter->netdev,
1062 adapter->num_rx_queues);
1063 }
1064
1065 /**
1066 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1067 * @adapter: board private structure to initialize
1068 *
1069 * We allocate one q_vector per queue interrupt. If allocation fails we
1070 * return -ENOMEM.
1071 **/
1072 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1073 {
1074 struct igb_q_vector *q_vector;
1075 struct e1000_hw *hw = &adapter->hw;
1076 int v_idx;
1077
1078 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1079 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1080 if (!q_vector)
1081 goto err_out;
1082 q_vector->adapter = adapter;
1083 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1084 q_vector->itr_val = IGB_START_ITR;
1085 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1086 adapter->q_vector[v_idx] = q_vector;
1087 }
1088 return 0;
1089
1090 err_out:
1091 igb_free_q_vectors(adapter);
1092 return -ENOMEM;
1093 }
1094
1095 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1096 int ring_idx, int v_idx)
1097 {
1098 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1099
1100 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1101 q_vector->rx_ring->q_vector = q_vector;
1102 q_vector->itr_val = adapter->rx_itr_setting;
1103 if (q_vector->itr_val && q_vector->itr_val <= 3)
1104 q_vector->itr_val = IGB_START_ITR;
1105 }
1106
1107 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1108 int ring_idx, int v_idx)
1109 {
1110 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1111
1112 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1113 q_vector->tx_ring->q_vector = q_vector;
1114 q_vector->itr_val = adapter->tx_itr_setting;
1115 if (q_vector->itr_val && q_vector->itr_val <= 3)
1116 q_vector->itr_val = IGB_START_ITR;
1117 }
1118
1119 /**
1120 * igb_map_ring_to_vector - maps allocated queues to vectors
1121 *
1122 * This function maps the recently allocated queues to vectors.
1123 **/
1124 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1125 {
1126 int i;
1127 int v_idx = 0;
1128
1129 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1130 (adapter->num_q_vectors < adapter->num_tx_queues))
1131 return -ENOMEM;
1132
1133 if (adapter->num_q_vectors >=
1134 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1135 for (i = 0; i < adapter->num_rx_queues; i++)
1136 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1137 for (i = 0; i < adapter->num_tx_queues; i++)
1138 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1139 } else {
1140 for (i = 0; i < adapter->num_rx_queues; i++) {
1141 if (i < adapter->num_tx_queues)
1142 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1143 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1144 }
1145 for (; i < adapter->num_tx_queues; i++)
1146 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1147 }
1148 return 0;
1149 }
1150
1151 /**
1152 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1153 *
1154 * This function initializes the interrupts and allocates all of the queues.
1155 **/
1156 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1157 {
1158 struct pci_dev *pdev = adapter->pdev;
1159 int err;
1160
1161 err = igb_set_interrupt_capability(adapter);
1162 if (err)
1163 return err;
1164
1165 err = igb_alloc_q_vectors(adapter);
1166 if (err) {
1167 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1168 goto err_alloc_q_vectors;
1169 }
1170
1171 err = igb_alloc_queues(adapter);
1172 if (err) {
1173 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1174 goto err_alloc_queues;
1175 }
1176
1177 err = igb_map_ring_to_vector(adapter);
1178 if (err) {
1179 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1180 goto err_map_queues;
1181 }
1182
1183
1184 return 0;
1185 err_map_queues:
1186 igb_free_queues(adapter);
1187 err_alloc_queues:
1188 igb_free_q_vectors(adapter);
1189 err_alloc_q_vectors:
1190 igb_reset_interrupt_capability(adapter);
1191 return err;
1192 }
1193
1194 /**
1195 * igb_request_irq - initialize interrupts
1196 *
1197 * Attempts to configure interrupts using the best available
1198 * capabilities of the hardware and kernel.
1199 **/
1200 static int igb_request_irq(struct igb_adapter *adapter)
1201 {
1202 struct net_device *netdev = adapter->netdev;
1203 struct pci_dev *pdev = adapter->pdev;
1204 int err = 0;
1205
1206 if (adapter->msix_entries) {
1207 err = igb_request_msix(adapter);
1208 if (!err)
1209 goto request_done;
1210 /* fall back to MSI */
1211 igb_clear_interrupt_scheme(adapter);
1212 if (!pci_enable_msi(adapter->pdev))
1213 adapter->flags |= IGB_FLAG_HAS_MSI;
1214 igb_free_all_tx_resources(adapter);
1215 igb_free_all_rx_resources(adapter);
1216 adapter->num_tx_queues = 1;
1217 adapter->num_rx_queues = 1;
1218 adapter->num_q_vectors = 1;
1219 err = igb_alloc_q_vectors(adapter);
1220 if (err) {
1221 dev_err(&pdev->dev,
1222 "Unable to allocate memory for vectors\n");
1223 goto request_done;
1224 }
1225 err = igb_alloc_queues(adapter);
1226 if (err) {
1227 dev_err(&pdev->dev,
1228 "Unable to allocate memory for queues\n");
1229 igb_free_q_vectors(adapter);
1230 goto request_done;
1231 }
1232 igb_setup_all_tx_resources(adapter);
1233 igb_setup_all_rx_resources(adapter);
1234 } else {
1235 igb_assign_vector(adapter->q_vector[0], 0);
1236 }
1237
1238 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1239 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1240 netdev->name, adapter);
1241 if (!err)
1242 goto request_done;
1243
1244 /* fall back to legacy interrupts */
1245 igb_reset_interrupt_capability(adapter);
1246 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1247 }
1248
1249 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1250 netdev->name, adapter);
1251
1252 if (err)
1253 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1254 err);
1255
1256 request_done:
1257 return err;
1258 }
1259
1260 static void igb_free_irq(struct igb_adapter *adapter)
1261 {
1262 if (adapter->msix_entries) {
1263 int vector = 0, i;
1264
1265 free_irq(adapter->msix_entries[vector++].vector, adapter);
1266
1267 for (i = 0; i < adapter->num_q_vectors; i++) {
1268 struct igb_q_vector *q_vector = adapter->q_vector[i];
1269 free_irq(adapter->msix_entries[vector++].vector,
1270 q_vector);
1271 }
1272 } else {
1273 free_irq(adapter->pdev->irq, adapter);
1274 }
1275 }
1276
1277 /**
1278 * igb_irq_disable - Mask off interrupt generation on the NIC
1279 * @adapter: board private structure
1280 **/
1281 static void igb_irq_disable(struct igb_adapter *adapter)
1282 {
1283 struct e1000_hw *hw = &adapter->hw;
1284
1285 /*
1286 * we need to be careful when disabling interrupts. The VFs are also
1287 * mapped into these registers and so clearing the bits can cause
1288 * issues on the VF drivers so we only need to clear what we set
1289 */
1290 if (adapter->msix_entries) {
1291 u32 regval = rd32(E1000_EIAM);
1292 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1293 wr32(E1000_EIMC, adapter->eims_enable_mask);
1294 regval = rd32(E1000_EIAC);
1295 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1296 }
1297
1298 wr32(E1000_IAM, 0);
1299 wr32(E1000_IMC, ~0);
1300 wrfl();
1301 if (adapter->msix_entries) {
1302 int i;
1303 for (i = 0; i < adapter->num_q_vectors; i++)
1304 synchronize_irq(adapter->msix_entries[i].vector);
1305 } else {
1306 synchronize_irq(adapter->pdev->irq);
1307 }
1308 }
1309
1310 /**
1311 * igb_irq_enable - Enable default interrupt generation settings
1312 * @adapter: board private structure
1313 **/
1314 static void igb_irq_enable(struct igb_adapter *adapter)
1315 {
1316 struct e1000_hw *hw = &adapter->hw;
1317
1318 if (adapter->msix_entries) {
1319 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1320 u32 regval = rd32(E1000_EIAC);
1321 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1322 regval = rd32(E1000_EIAM);
1323 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1324 wr32(E1000_EIMS, adapter->eims_enable_mask);
1325 if (adapter->vfs_allocated_count) {
1326 wr32(E1000_MBVFIMR, 0xFF);
1327 ims |= E1000_IMS_VMMB;
1328 }
1329 if (adapter->hw.mac.type == e1000_82580)
1330 ims |= E1000_IMS_DRSTA;
1331
1332 wr32(E1000_IMS, ims);
1333 } else {
1334 wr32(E1000_IMS, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
1336 wr32(E1000_IAM, IMS_ENABLE_MASK |
1337 E1000_IMS_DRSTA);
1338 }
1339 }
1340
1341 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1342 {
1343 struct e1000_hw *hw = &adapter->hw;
1344 u16 vid = adapter->hw.mng_cookie.vlan_id;
1345 u16 old_vid = adapter->mng_vlan_id;
1346
1347 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1348 /* add VID to filter table */
1349 igb_vfta_set(hw, vid, true);
1350 adapter->mng_vlan_id = vid;
1351 } else {
1352 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1353 }
1354
1355 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1356 (vid != old_vid) &&
1357 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1358 /* remove VID from filter table */
1359 igb_vfta_set(hw, old_vid, false);
1360 }
1361 }
1362
1363 /**
1364 * igb_release_hw_control - release control of the h/w to f/w
1365 * @adapter: address of board private structure
1366 *
1367 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1368 * For ASF and Pass Through versions of f/w this means that the
1369 * driver is no longer loaded.
1370 *
1371 **/
1372 static void igb_release_hw_control(struct igb_adapter *adapter)
1373 {
1374 struct e1000_hw *hw = &adapter->hw;
1375 u32 ctrl_ext;
1376
1377 /* Let firmware take over control of h/w */
1378 ctrl_ext = rd32(E1000_CTRL_EXT);
1379 wr32(E1000_CTRL_EXT,
1380 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1381 }
1382
1383 /**
1384 * igb_get_hw_control - get control of the h/w from f/w
1385 * @adapter: address of board private structure
1386 *
1387 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1388 * For ASF and Pass Through versions of f/w this means that
1389 * the driver is loaded.
1390 *
1391 **/
1392 static void igb_get_hw_control(struct igb_adapter *adapter)
1393 {
1394 struct e1000_hw *hw = &adapter->hw;
1395 u32 ctrl_ext;
1396
1397 /* Let firmware know the driver has taken over */
1398 ctrl_ext = rd32(E1000_CTRL_EXT);
1399 wr32(E1000_CTRL_EXT,
1400 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1401 }
1402
1403 /**
1404 * igb_configure - configure the hardware for RX and TX
1405 * @adapter: private board structure
1406 **/
1407 static void igb_configure(struct igb_adapter *adapter)
1408 {
1409 struct net_device *netdev = adapter->netdev;
1410 int i;
1411
1412 igb_get_hw_control(adapter);
1413 igb_set_rx_mode(netdev);
1414
1415 igb_restore_vlan(adapter);
1416
1417 igb_setup_tctl(adapter);
1418 igb_setup_mrqc(adapter);
1419 igb_setup_rctl(adapter);
1420
1421 igb_configure_tx(adapter);
1422 igb_configure_rx(adapter);
1423
1424 igb_rx_fifo_flush_82575(&adapter->hw);
1425
1426 /* call igb_desc_unused which always leaves
1427 * at least 1 descriptor unused to make sure
1428 * next_to_use != next_to_clean */
1429 for (i = 0; i < adapter->num_rx_queues; i++) {
1430 struct igb_ring *ring = adapter->rx_ring[i];
1431 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1432 }
1433 }
1434
1435 /**
1436 * igb_power_up_link - Power up the phy/serdes link
1437 * @adapter: address of board private structure
1438 **/
1439 void igb_power_up_link(struct igb_adapter *adapter)
1440 {
1441 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1442 igb_power_up_phy_copper(&adapter->hw);
1443 else
1444 igb_power_up_serdes_link_82575(&adapter->hw);
1445 }
1446
1447 /**
1448 * igb_power_down_link - Power down the phy/serdes link
1449 * @adapter: address of board private structure
1450 */
1451 static void igb_power_down_link(struct igb_adapter *adapter)
1452 {
1453 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1454 igb_power_down_phy_copper_82575(&adapter->hw);
1455 else
1456 igb_shutdown_serdes_link_82575(&adapter->hw);
1457 }
1458
1459 /**
1460 * igb_up - Open the interface and prepare it to handle traffic
1461 * @adapter: board private structure
1462 **/
1463 int igb_up(struct igb_adapter *adapter)
1464 {
1465 struct e1000_hw *hw = &adapter->hw;
1466 int i;
1467
1468 /* hardware has been reset, we need to reload some things */
1469 igb_configure(adapter);
1470
1471 clear_bit(__IGB_DOWN, &adapter->state);
1472
1473 for (i = 0; i < adapter->num_q_vectors; i++) {
1474 struct igb_q_vector *q_vector = adapter->q_vector[i];
1475 napi_enable(&q_vector->napi);
1476 }
1477 if (adapter->msix_entries)
1478 igb_configure_msix(adapter);
1479 else
1480 igb_assign_vector(adapter->q_vector[0], 0);
1481
1482 /* Clear any pending interrupts. */
1483 rd32(E1000_ICR);
1484 igb_irq_enable(adapter);
1485
1486 /* notify VFs that reset has been completed */
1487 if (adapter->vfs_allocated_count) {
1488 u32 reg_data = rd32(E1000_CTRL_EXT);
1489 reg_data |= E1000_CTRL_EXT_PFRSTD;
1490 wr32(E1000_CTRL_EXT, reg_data);
1491 }
1492
1493 netif_tx_start_all_queues(adapter->netdev);
1494
1495 /* start the watchdog. */
1496 hw->mac.get_link_status = 1;
1497 schedule_work(&adapter->watchdog_task);
1498
1499 return 0;
1500 }
1501
1502 void igb_down(struct igb_adapter *adapter)
1503 {
1504 struct net_device *netdev = adapter->netdev;
1505 struct e1000_hw *hw = &adapter->hw;
1506 u32 tctl, rctl;
1507 int i;
1508
1509 /* signal that we're down so the interrupt handler does not
1510 * reschedule our watchdog timer */
1511 set_bit(__IGB_DOWN, &adapter->state);
1512
1513 /* disable receives in the hardware */
1514 rctl = rd32(E1000_RCTL);
1515 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1516 /* flush and sleep below */
1517
1518 netif_tx_stop_all_queues(netdev);
1519
1520 /* disable transmits in the hardware */
1521 tctl = rd32(E1000_TCTL);
1522 tctl &= ~E1000_TCTL_EN;
1523 wr32(E1000_TCTL, tctl);
1524 /* flush both disables and wait for them to finish */
1525 wrfl();
1526 msleep(10);
1527
1528 for (i = 0; i < adapter->num_q_vectors; i++) {
1529 struct igb_q_vector *q_vector = adapter->q_vector[i];
1530 napi_disable(&q_vector->napi);
1531 }
1532
1533 igb_irq_disable(adapter);
1534
1535 del_timer_sync(&adapter->watchdog_timer);
1536 del_timer_sync(&adapter->phy_info_timer);
1537
1538 netif_carrier_off(netdev);
1539
1540 /* record the stats before reset*/
1541 spin_lock(&adapter->stats64_lock);
1542 igb_update_stats(adapter, &adapter->stats64);
1543 spin_unlock(&adapter->stats64_lock);
1544
1545 adapter->link_speed = 0;
1546 adapter->link_duplex = 0;
1547
1548 if (!pci_channel_offline(adapter->pdev))
1549 igb_reset(adapter);
1550 igb_clean_all_tx_rings(adapter);
1551 igb_clean_all_rx_rings(adapter);
1552 #ifdef CONFIG_IGB_DCA
1553
1554 /* since we reset the hardware DCA settings were cleared */
1555 igb_setup_dca(adapter);
1556 #endif
1557 }
1558
1559 void igb_reinit_locked(struct igb_adapter *adapter)
1560 {
1561 WARN_ON(in_interrupt());
1562 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1563 msleep(1);
1564 igb_down(adapter);
1565 igb_up(adapter);
1566 clear_bit(__IGB_RESETTING, &adapter->state);
1567 }
1568
1569 void igb_reset(struct igb_adapter *adapter)
1570 {
1571 struct pci_dev *pdev = adapter->pdev;
1572 struct e1000_hw *hw = &adapter->hw;
1573 struct e1000_mac_info *mac = &hw->mac;
1574 struct e1000_fc_info *fc = &hw->fc;
1575 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1576 u16 hwm;
1577
1578 /* Repartition Pba for greater than 9k mtu
1579 * To take effect CTRL.RST is required.
1580 */
1581 switch (mac->type) {
1582 case e1000_i350:
1583 case e1000_82580:
1584 pba = rd32(E1000_RXPBS);
1585 pba = igb_rxpbs_adjust_82580(pba);
1586 break;
1587 case e1000_82576:
1588 pba = rd32(E1000_RXPBS);
1589 pba &= E1000_RXPBS_SIZE_MASK_82576;
1590 break;
1591 case e1000_82575:
1592 default:
1593 pba = E1000_PBA_34K;
1594 break;
1595 }
1596
1597 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1598 (mac->type < e1000_82576)) {
1599 /* adjust PBA for jumbo frames */
1600 wr32(E1000_PBA, pba);
1601
1602 /* To maintain wire speed transmits, the Tx FIFO should be
1603 * large enough to accommodate two full transmit packets,
1604 * rounded up to the next 1KB and expressed in KB. Likewise,
1605 * the Rx FIFO should be large enough to accommodate at least
1606 * one full receive packet and is similarly rounded up and
1607 * expressed in KB. */
1608 pba = rd32(E1000_PBA);
1609 /* upper 16 bits has Tx packet buffer allocation size in KB */
1610 tx_space = pba >> 16;
1611 /* lower 16 bits has Rx packet buffer allocation size in KB */
1612 pba &= 0xffff;
1613 /* the tx fifo also stores 16 bytes of information about the tx
1614 * but don't include ethernet FCS because hardware appends it */
1615 min_tx_space = (adapter->max_frame_size +
1616 sizeof(union e1000_adv_tx_desc) -
1617 ETH_FCS_LEN) * 2;
1618 min_tx_space = ALIGN(min_tx_space, 1024);
1619 min_tx_space >>= 10;
1620 /* software strips receive CRC, so leave room for it */
1621 min_rx_space = adapter->max_frame_size;
1622 min_rx_space = ALIGN(min_rx_space, 1024);
1623 min_rx_space >>= 10;
1624
1625 /* If current Tx allocation is less than the min Tx FIFO size,
1626 * and the min Tx FIFO size is less than the current Rx FIFO
1627 * allocation, take space away from current Rx allocation */
1628 if (tx_space < min_tx_space &&
1629 ((min_tx_space - tx_space) < pba)) {
1630 pba = pba - (min_tx_space - tx_space);
1631
1632 /* if short on rx space, rx wins and must trump tx
1633 * adjustment */
1634 if (pba < min_rx_space)
1635 pba = min_rx_space;
1636 }
1637 wr32(E1000_PBA, pba);
1638 }
1639
1640 /* flow control settings */
1641 /* The high water mark must be low enough to fit one full frame
1642 * (or the size used for early receive) above it in the Rx FIFO.
1643 * Set it to the lower of:
1644 * - 90% of the Rx FIFO size, or
1645 * - the full Rx FIFO size minus one full frame */
1646 hwm = min(((pba << 10) * 9 / 10),
1647 ((pba << 10) - 2 * adapter->max_frame_size));
1648
1649 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1650 fc->low_water = fc->high_water - 16;
1651 fc->pause_time = 0xFFFF;
1652 fc->send_xon = 1;
1653 fc->current_mode = fc->requested_mode;
1654
1655 /* disable receive for all VFs and wait one second */
1656 if (adapter->vfs_allocated_count) {
1657 int i;
1658 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1659 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1660
1661 /* ping all the active vfs to let them know we are going down */
1662 igb_ping_all_vfs(adapter);
1663
1664 /* disable transmits and receives */
1665 wr32(E1000_VFRE, 0);
1666 wr32(E1000_VFTE, 0);
1667 }
1668
1669 /* Allow time for pending master requests to run */
1670 hw->mac.ops.reset_hw(hw);
1671 wr32(E1000_WUC, 0);
1672
1673 if (hw->mac.ops.init_hw(hw))
1674 dev_err(&pdev->dev, "Hardware Error\n");
1675
1676 if (hw->mac.type == e1000_82580) {
1677 u32 reg = rd32(E1000_PCIEMISC);
1678 wr32(E1000_PCIEMISC,
1679 reg & ~E1000_PCIEMISC_LX_DECISION);
1680 }
1681 if (!netif_running(adapter->netdev))
1682 igb_power_down_link(adapter);
1683
1684 igb_update_mng_vlan(adapter);
1685
1686 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1687 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1688
1689 igb_get_phy_info(hw);
1690 }
1691
1692 static const struct net_device_ops igb_netdev_ops = {
1693 .ndo_open = igb_open,
1694 .ndo_stop = igb_close,
1695 .ndo_start_xmit = igb_xmit_frame_adv,
1696 .ndo_get_stats64 = igb_get_stats64,
1697 .ndo_set_rx_mode = igb_set_rx_mode,
1698 .ndo_set_multicast_list = igb_set_rx_mode,
1699 .ndo_set_mac_address = igb_set_mac,
1700 .ndo_change_mtu = igb_change_mtu,
1701 .ndo_do_ioctl = igb_ioctl,
1702 .ndo_tx_timeout = igb_tx_timeout,
1703 .ndo_validate_addr = eth_validate_addr,
1704 .ndo_vlan_rx_register = igb_vlan_rx_register,
1705 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1706 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1707 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1708 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1709 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1710 .ndo_get_vf_config = igb_ndo_get_vf_config,
1711 #ifdef CONFIG_NET_POLL_CONTROLLER
1712 .ndo_poll_controller = igb_netpoll,
1713 #endif
1714 };
1715
1716 /**
1717 * igb_probe - Device Initialization Routine
1718 * @pdev: PCI device information struct
1719 * @ent: entry in igb_pci_tbl
1720 *
1721 * Returns 0 on success, negative on failure
1722 *
1723 * igb_probe initializes an adapter identified by a pci_dev structure.
1724 * The OS initialization, configuring of the adapter private structure,
1725 * and a hardware reset occur.
1726 **/
1727 static int __devinit igb_probe(struct pci_dev *pdev,
1728 const struct pci_device_id *ent)
1729 {
1730 struct net_device *netdev;
1731 struct igb_adapter *adapter;
1732 struct e1000_hw *hw;
1733 u16 eeprom_data = 0;
1734 s32 ret_val;
1735 static int global_quad_port_a; /* global quad port a indication */
1736 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1737 unsigned long mmio_start, mmio_len;
1738 int err, pci_using_dac;
1739 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1740 u8 part_str[E1000_PBANUM_LENGTH];
1741
1742 /* Catch broken hardware that put the wrong VF device ID in
1743 * the PCIe SR-IOV capability.
1744 */
1745 if (pdev->is_virtfn) {
1746 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1747 pci_name(pdev), pdev->vendor, pdev->device);
1748 return -EINVAL;
1749 }
1750
1751 err = pci_enable_device_mem(pdev);
1752 if (err)
1753 return err;
1754
1755 pci_using_dac = 0;
1756 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1757 if (!err) {
1758 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1759 if (!err)
1760 pci_using_dac = 1;
1761 } else {
1762 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1763 if (err) {
1764 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1765 if (err) {
1766 dev_err(&pdev->dev, "No usable DMA "
1767 "configuration, aborting\n");
1768 goto err_dma;
1769 }
1770 }
1771 }
1772
1773 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1774 IORESOURCE_MEM),
1775 igb_driver_name);
1776 if (err)
1777 goto err_pci_reg;
1778
1779 pci_enable_pcie_error_reporting(pdev);
1780
1781 pci_set_master(pdev);
1782 pci_save_state(pdev);
1783
1784 err = -ENOMEM;
1785 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1786 IGB_ABS_MAX_TX_QUEUES);
1787 if (!netdev)
1788 goto err_alloc_etherdev;
1789
1790 SET_NETDEV_DEV(netdev, &pdev->dev);
1791
1792 pci_set_drvdata(pdev, netdev);
1793 adapter = netdev_priv(netdev);
1794 adapter->netdev = netdev;
1795 adapter->pdev = pdev;
1796 hw = &adapter->hw;
1797 hw->back = adapter;
1798 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1799
1800 mmio_start = pci_resource_start(pdev, 0);
1801 mmio_len = pci_resource_len(pdev, 0);
1802
1803 err = -EIO;
1804 hw->hw_addr = ioremap(mmio_start, mmio_len);
1805 if (!hw->hw_addr)
1806 goto err_ioremap;
1807
1808 netdev->netdev_ops = &igb_netdev_ops;
1809 igb_set_ethtool_ops(netdev);
1810 netdev->watchdog_timeo = 5 * HZ;
1811
1812 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1813
1814 netdev->mem_start = mmio_start;
1815 netdev->mem_end = mmio_start + mmio_len;
1816
1817 /* PCI config space info */
1818 hw->vendor_id = pdev->vendor;
1819 hw->device_id = pdev->device;
1820 hw->revision_id = pdev->revision;
1821 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1822 hw->subsystem_device_id = pdev->subsystem_device;
1823
1824 /* Copy the default MAC, PHY and NVM function pointers */
1825 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1826 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1827 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1828 /* Initialize skew-specific constants */
1829 err = ei->get_invariants(hw);
1830 if (err)
1831 goto err_sw_init;
1832
1833 /* setup the private structure */
1834 err = igb_sw_init(adapter);
1835 if (err)
1836 goto err_sw_init;
1837
1838 igb_get_bus_info_pcie(hw);
1839
1840 hw->phy.autoneg_wait_to_complete = false;
1841
1842 /* Copper options */
1843 if (hw->phy.media_type == e1000_media_type_copper) {
1844 hw->phy.mdix = AUTO_ALL_MODES;
1845 hw->phy.disable_polarity_correction = false;
1846 hw->phy.ms_type = e1000_ms_hw_default;
1847 }
1848
1849 if (igb_check_reset_block(hw))
1850 dev_info(&pdev->dev,
1851 "PHY reset is blocked due to SOL/IDER session.\n");
1852
1853 netdev->features = NETIF_F_SG |
1854 NETIF_F_IP_CSUM |
1855 NETIF_F_HW_VLAN_TX |
1856 NETIF_F_HW_VLAN_RX |
1857 NETIF_F_HW_VLAN_FILTER;
1858
1859 netdev->features |= NETIF_F_IPV6_CSUM;
1860 netdev->features |= NETIF_F_TSO;
1861 netdev->features |= NETIF_F_TSO6;
1862 netdev->features |= NETIF_F_GRO;
1863
1864 netdev->vlan_features |= NETIF_F_TSO;
1865 netdev->vlan_features |= NETIF_F_TSO6;
1866 netdev->vlan_features |= NETIF_F_IP_CSUM;
1867 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1868 netdev->vlan_features |= NETIF_F_SG;
1869
1870 if (pci_using_dac) {
1871 netdev->features |= NETIF_F_HIGHDMA;
1872 netdev->vlan_features |= NETIF_F_HIGHDMA;
1873 }
1874
1875 if (hw->mac.type >= e1000_82576)
1876 netdev->features |= NETIF_F_SCTP_CSUM;
1877
1878 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1879
1880 /* before reading the NVM, reset the controller to put the device in a
1881 * known good starting state */
1882 hw->mac.ops.reset_hw(hw);
1883
1884 /* make sure the NVM is good */
1885 if (igb_validate_nvm_checksum(hw) < 0) {
1886 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1887 err = -EIO;
1888 goto err_eeprom;
1889 }
1890
1891 /* copy the MAC address out of the NVM */
1892 if (hw->mac.ops.read_mac_addr(hw))
1893 dev_err(&pdev->dev, "NVM Read Error\n");
1894
1895 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1896 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1897
1898 if (!is_valid_ether_addr(netdev->perm_addr)) {
1899 dev_err(&pdev->dev, "Invalid MAC Address\n");
1900 err = -EIO;
1901 goto err_eeprom;
1902 }
1903
1904 setup_timer(&adapter->watchdog_timer, igb_watchdog,
1905 (unsigned long) adapter);
1906 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
1907 (unsigned long) adapter);
1908
1909 INIT_WORK(&adapter->reset_task, igb_reset_task);
1910 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1911
1912 /* Initialize link properties that are user-changeable */
1913 adapter->fc_autoneg = true;
1914 hw->mac.autoneg = true;
1915 hw->phy.autoneg_advertised = 0x2f;
1916
1917 hw->fc.requested_mode = e1000_fc_default;
1918 hw->fc.current_mode = e1000_fc_default;
1919
1920 igb_validate_mdi_setting(hw);
1921
1922 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1923 * enable the ACPI Magic Packet filter
1924 */
1925
1926 if (hw->bus.func == 0)
1927 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1928 else if (hw->mac.type == e1000_82580)
1929 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1930 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1931 &eeprom_data);
1932 else if (hw->bus.func == 1)
1933 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1934
1935 if (eeprom_data & eeprom_apme_mask)
1936 adapter->eeprom_wol |= E1000_WUFC_MAG;
1937
1938 /* now that we have the eeprom settings, apply the special cases where
1939 * the eeprom may be wrong or the board simply won't support wake on
1940 * lan on a particular port */
1941 switch (pdev->device) {
1942 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1943 adapter->eeprom_wol = 0;
1944 break;
1945 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1946 case E1000_DEV_ID_82576_FIBER:
1947 case E1000_DEV_ID_82576_SERDES:
1948 /* Wake events only supported on port A for dual fiber
1949 * regardless of eeprom setting */
1950 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1951 adapter->eeprom_wol = 0;
1952 break;
1953 case E1000_DEV_ID_82576_QUAD_COPPER:
1954 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1955 /* if quad port adapter, disable WoL on all but port A */
1956 if (global_quad_port_a != 0)
1957 adapter->eeprom_wol = 0;
1958 else
1959 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1960 /* Reset for multiple quad port adapters */
1961 if (++global_quad_port_a == 4)
1962 global_quad_port_a = 0;
1963 break;
1964 }
1965
1966 /* initialize the wol settings based on the eeprom settings */
1967 adapter->wol = adapter->eeprom_wol;
1968 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1969
1970 /* reset the hardware with the new settings */
1971 igb_reset(adapter);
1972
1973 /* let the f/w know that the h/w is now under the control of the
1974 * driver. */
1975 igb_get_hw_control(adapter);
1976
1977 strcpy(netdev->name, "eth%d");
1978 err = register_netdev(netdev);
1979 if (err)
1980 goto err_register;
1981
1982 /* carrier off reporting is important to ethtool even BEFORE open */
1983 netif_carrier_off(netdev);
1984
1985 #ifdef CONFIG_IGB_DCA
1986 if (dca_add_requester(&pdev->dev) == 0) {
1987 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1988 dev_info(&pdev->dev, "DCA enabled\n");
1989 igb_setup_dca(adapter);
1990 }
1991
1992 #endif
1993 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1994 /* print bus type/speed/width info */
1995 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1996 netdev->name,
1997 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1998 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1999 "unknown"),
2000 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2001 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2002 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2003 "unknown"),
2004 netdev->dev_addr);
2005
2006 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2007 if (ret_val)
2008 strcpy(part_str, "Unknown");
2009 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2010 dev_info(&pdev->dev,
2011 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2012 adapter->msix_entries ? "MSI-X" :
2013 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2014 adapter->num_rx_queues, adapter->num_tx_queues);
2015
2016 return 0;
2017
2018 err_register:
2019 igb_release_hw_control(adapter);
2020 err_eeprom:
2021 if (!igb_check_reset_block(hw))
2022 igb_reset_phy(hw);
2023
2024 if (hw->flash_address)
2025 iounmap(hw->flash_address);
2026 err_sw_init:
2027 igb_clear_interrupt_scheme(adapter);
2028 iounmap(hw->hw_addr);
2029 err_ioremap:
2030 free_netdev(netdev);
2031 err_alloc_etherdev:
2032 pci_release_selected_regions(pdev,
2033 pci_select_bars(pdev, IORESOURCE_MEM));
2034 err_pci_reg:
2035 err_dma:
2036 pci_disable_device(pdev);
2037 return err;
2038 }
2039
2040 /**
2041 * igb_remove - Device Removal Routine
2042 * @pdev: PCI device information struct
2043 *
2044 * igb_remove is called by the PCI subsystem to alert the driver
2045 * that it should release a PCI device. The could be caused by a
2046 * Hot-Plug event, or because the driver is going to be removed from
2047 * memory.
2048 **/
2049 static void __devexit igb_remove(struct pci_dev *pdev)
2050 {
2051 struct net_device *netdev = pci_get_drvdata(pdev);
2052 struct igb_adapter *adapter = netdev_priv(netdev);
2053 struct e1000_hw *hw = &adapter->hw;
2054
2055 /*
2056 * The watchdog timer may be rescheduled, so explicitly
2057 * disable watchdog from being rescheduled.
2058 */
2059 set_bit(__IGB_DOWN, &adapter->state);
2060 del_timer_sync(&adapter->watchdog_timer);
2061 del_timer_sync(&adapter->phy_info_timer);
2062
2063 cancel_work_sync(&adapter->reset_task);
2064 cancel_work_sync(&adapter->watchdog_task);
2065
2066 #ifdef CONFIG_IGB_DCA
2067 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2068 dev_info(&pdev->dev, "DCA disabled\n");
2069 dca_remove_requester(&pdev->dev);
2070 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2071 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2072 }
2073 #endif
2074
2075 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2076 * would have already happened in close and is redundant. */
2077 igb_release_hw_control(adapter);
2078
2079 unregister_netdev(netdev);
2080
2081 igb_clear_interrupt_scheme(adapter);
2082
2083 #ifdef CONFIG_PCI_IOV
2084 /* reclaim resources allocated to VFs */
2085 if (adapter->vf_data) {
2086 /* disable iov and allow time for transactions to clear */
2087 pci_disable_sriov(pdev);
2088 msleep(500);
2089
2090 kfree(adapter->vf_data);
2091 adapter->vf_data = NULL;
2092 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2093 msleep(100);
2094 dev_info(&pdev->dev, "IOV Disabled\n");
2095 }
2096 #endif
2097
2098 iounmap(hw->hw_addr);
2099 if (hw->flash_address)
2100 iounmap(hw->flash_address);
2101 pci_release_selected_regions(pdev,
2102 pci_select_bars(pdev, IORESOURCE_MEM));
2103
2104 free_netdev(netdev);
2105
2106 pci_disable_pcie_error_reporting(pdev);
2107
2108 pci_disable_device(pdev);
2109 }
2110
2111 /**
2112 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2113 * @adapter: board private structure to initialize
2114 *
2115 * This function initializes the vf specific data storage and then attempts to
2116 * allocate the VFs. The reason for ordering it this way is because it is much
2117 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2118 * the memory for the VFs.
2119 **/
2120 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2121 {
2122 #ifdef CONFIG_PCI_IOV
2123 struct pci_dev *pdev = adapter->pdev;
2124
2125 if (adapter->vfs_allocated_count) {
2126 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2127 sizeof(struct vf_data_storage),
2128 GFP_KERNEL);
2129 /* if allocation failed then we do not support SR-IOV */
2130 if (!adapter->vf_data) {
2131 adapter->vfs_allocated_count = 0;
2132 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2133 "Data Storage\n");
2134 }
2135 }
2136
2137 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2138 kfree(adapter->vf_data);
2139 adapter->vf_data = NULL;
2140 #endif /* CONFIG_PCI_IOV */
2141 adapter->vfs_allocated_count = 0;
2142 #ifdef CONFIG_PCI_IOV
2143 } else {
2144 unsigned char mac_addr[ETH_ALEN];
2145 int i;
2146 dev_info(&pdev->dev, "%d vfs allocated\n",
2147 adapter->vfs_allocated_count);
2148 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2149 random_ether_addr(mac_addr);
2150 igb_set_vf_mac(adapter, i, mac_addr);
2151 }
2152 }
2153 #endif /* CONFIG_PCI_IOV */
2154 }
2155
2156
2157 /**
2158 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2159 * @adapter: board private structure to initialize
2160 *
2161 * igb_init_hw_timer initializes the function pointer and values for the hw
2162 * timer found in hardware.
2163 **/
2164 static void igb_init_hw_timer(struct igb_adapter *adapter)
2165 {
2166 struct e1000_hw *hw = &adapter->hw;
2167
2168 switch (hw->mac.type) {
2169 case e1000_i350:
2170 case e1000_82580:
2171 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2172 adapter->cycles.read = igb_read_clock;
2173 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2174 adapter->cycles.mult = 1;
2175 /*
2176 * The 82580 timesync updates the system timer every 8ns by 8ns
2177 * and the value cannot be shifted. Instead we need to shift
2178 * the registers to generate a 64bit timer value. As a result
2179 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2180 * 24 in order to generate a larger value for synchronization.
2181 */
2182 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2183 /* disable system timer temporarily by setting bit 31 */
2184 wr32(E1000_TSAUXC, 0x80000000);
2185 wrfl();
2186
2187 /* Set registers so that rollover occurs soon to test this. */
2188 wr32(E1000_SYSTIMR, 0x00000000);
2189 wr32(E1000_SYSTIML, 0x80000000);
2190 wr32(E1000_SYSTIMH, 0x000000FF);
2191 wrfl();
2192
2193 /* enable system timer by clearing bit 31 */
2194 wr32(E1000_TSAUXC, 0x0);
2195 wrfl();
2196
2197 timecounter_init(&adapter->clock,
2198 &adapter->cycles,
2199 ktime_to_ns(ktime_get_real()));
2200 /*
2201 * Synchronize our NIC clock against system wall clock. NIC
2202 * time stamp reading requires ~3us per sample, each sample
2203 * was pretty stable even under load => only require 10
2204 * samples for each offset comparison.
2205 */
2206 memset(&adapter->compare, 0, sizeof(adapter->compare));
2207 adapter->compare.source = &adapter->clock;
2208 adapter->compare.target = ktime_get_real;
2209 adapter->compare.num_samples = 10;
2210 timecompare_update(&adapter->compare, 0);
2211 break;
2212 case e1000_82576:
2213 /*
2214 * Initialize hardware timer: we keep it running just in case
2215 * that some program needs it later on.
2216 */
2217 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2218 adapter->cycles.read = igb_read_clock;
2219 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2220 adapter->cycles.mult = 1;
2221 /**
2222 * Scale the NIC clock cycle by a large factor so that
2223 * relatively small clock corrections can be added or
2224 * substracted at each clock tick. The drawbacks of a large
2225 * factor are a) that the clock register overflows more quickly
2226 * (not such a big deal) and b) that the increment per tick has
2227 * to fit into 24 bits. As a result we need to use a shift of
2228 * 19 so we can fit a value of 16 into the TIMINCA register.
2229 */
2230 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2231 wr32(E1000_TIMINCA,
2232 (1 << E1000_TIMINCA_16NS_SHIFT) |
2233 (16 << IGB_82576_TSYNC_SHIFT));
2234
2235 /* Set registers so that rollover occurs soon to test this. */
2236 wr32(E1000_SYSTIML, 0x00000000);
2237 wr32(E1000_SYSTIMH, 0xFF800000);
2238 wrfl();
2239
2240 timecounter_init(&adapter->clock,
2241 &adapter->cycles,
2242 ktime_to_ns(ktime_get_real()));
2243 /*
2244 * Synchronize our NIC clock against system wall clock. NIC
2245 * time stamp reading requires ~3us per sample, each sample
2246 * was pretty stable even under load => only require 10
2247 * samples for each offset comparison.
2248 */
2249 memset(&adapter->compare, 0, sizeof(adapter->compare));
2250 adapter->compare.source = &adapter->clock;
2251 adapter->compare.target = ktime_get_real;
2252 adapter->compare.num_samples = 10;
2253 timecompare_update(&adapter->compare, 0);
2254 break;
2255 case e1000_82575:
2256 /* 82575 does not support timesync */
2257 default:
2258 break;
2259 }
2260
2261 }
2262
2263 /**
2264 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2265 * @adapter: board private structure to initialize
2266 *
2267 * igb_sw_init initializes the Adapter private data structure.
2268 * Fields are initialized based on PCI device information and
2269 * OS network device settings (MTU size).
2270 **/
2271 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2272 {
2273 struct e1000_hw *hw = &adapter->hw;
2274 struct net_device *netdev = adapter->netdev;
2275 struct pci_dev *pdev = adapter->pdev;
2276
2277 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2278
2279 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2280 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2281 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2282 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2283
2284 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2285 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2286
2287 spin_lock_init(&adapter->stats64_lock);
2288 #ifdef CONFIG_PCI_IOV
2289 if (hw->mac.type == e1000_82576)
2290 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
2291
2292 #endif /* CONFIG_PCI_IOV */
2293 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2294
2295 /*
2296 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2297 * then we should combine the queues into a queue pair in order to
2298 * conserve interrupts due to limited supply
2299 */
2300 if ((adapter->rss_queues > 4) ||
2301 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2302 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2303
2304 /* This call may decrease the number of queues */
2305 if (igb_init_interrupt_scheme(adapter)) {
2306 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2307 return -ENOMEM;
2308 }
2309
2310 igb_init_hw_timer(adapter);
2311 igb_probe_vfs(adapter);
2312
2313 /* Explicitly disable IRQ since the NIC can be in any state. */
2314 igb_irq_disable(adapter);
2315
2316 set_bit(__IGB_DOWN, &adapter->state);
2317 return 0;
2318 }
2319
2320 /**
2321 * igb_open - Called when a network interface is made active
2322 * @netdev: network interface device structure
2323 *
2324 * Returns 0 on success, negative value on failure
2325 *
2326 * The open entry point is called when a network interface is made
2327 * active by the system (IFF_UP). At this point all resources needed
2328 * for transmit and receive operations are allocated, the interrupt
2329 * handler is registered with the OS, the watchdog timer is started,
2330 * and the stack is notified that the interface is ready.
2331 **/
2332 static int igb_open(struct net_device *netdev)
2333 {
2334 struct igb_adapter *adapter = netdev_priv(netdev);
2335 struct e1000_hw *hw = &adapter->hw;
2336 int err;
2337 int i;
2338
2339 /* disallow open during test */
2340 if (test_bit(__IGB_TESTING, &adapter->state))
2341 return -EBUSY;
2342
2343 netif_carrier_off(netdev);
2344
2345 /* allocate transmit descriptors */
2346 err = igb_setup_all_tx_resources(adapter);
2347 if (err)
2348 goto err_setup_tx;
2349
2350 /* allocate receive descriptors */
2351 err = igb_setup_all_rx_resources(adapter);
2352 if (err)
2353 goto err_setup_rx;
2354
2355 igb_power_up_link(adapter);
2356
2357 /* before we allocate an interrupt, we must be ready to handle it.
2358 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2359 * as soon as we call pci_request_irq, so we have to setup our
2360 * clean_rx handler before we do so. */
2361 igb_configure(adapter);
2362
2363 err = igb_request_irq(adapter);
2364 if (err)
2365 goto err_req_irq;
2366
2367 /* From here on the code is the same as igb_up() */
2368 clear_bit(__IGB_DOWN, &adapter->state);
2369
2370 for (i = 0; i < adapter->num_q_vectors; i++) {
2371 struct igb_q_vector *q_vector = adapter->q_vector[i];
2372 napi_enable(&q_vector->napi);
2373 }
2374
2375 /* Clear any pending interrupts. */
2376 rd32(E1000_ICR);
2377
2378 igb_irq_enable(adapter);
2379
2380 /* notify VFs that reset has been completed */
2381 if (adapter->vfs_allocated_count) {
2382 u32 reg_data = rd32(E1000_CTRL_EXT);
2383 reg_data |= E1000_CTRL_EXT_PFRSTD;
2384 wr32(E1000_CTRL_EXT, reg_data);
2385 }
2386
2387 netif_tx_start_all_queues(netdev);
2388
2389 /* start the watchdog. */
2390 hw->mac.get_link_status = 1;
2391 schedule_work(&adapter->watchdog_task);
2392
2393 return 0;
2394
2395 err_req_irq:
2396 igb_release_hw_control(adapter);
2397 igb_power_down_link(adapter);
2398 igb_free_all_rx_resources(adapter);
2399 err_setup_rx:
2400 igb_free_all_tx_resources(adapter);
2401 err_setup_tx:
2402 igb_reset(adapter);
2403
2404 return err;
2405 }
2406
2407 /**
2408 * igb_close - Disables a network interface
2409 * @netdev: network interface device structure
2410 *
2411 * Returns 0, this is not allowed to fail
2412 *
2413 * The close entry point is called when an interface is de-activated
2414 * by the OS. The hardware is still under the driver's control, but
2415 * needs to be disabled. A global MAC reset is issued to stop the
2416 * hardware, and all transmit and receive resources are freed.
2417 **/
2418 static int igb_close(struct net_device *netdev)
2419 {
2420 struct igb_adapter *adapter = netdev_priv(netdev);
2421
2422 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2423 igb_down(adapter);
2424
2425 igb_free_irq(adapter);
2426
2427 igb_free_all_tx_resources(adapter);
2428 igb_free_all_rx_resources(adapter);
2429
2430 return 0;
2431 }
2432
2433 /**
2434 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2435 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2436 *
2437 * Return 0 on success, negative on failure
2438 **/
2439 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2440 {
2441 struct device *dev = tx_ring->dev;
2442 int size;
2443
2444 size = sizeof(struct igb_buffer) * tx_ring->count;
2445 tx_ring->buffer_info = vzalloc(size);
2446 if (!tx_ring->buffer_info)
2447 goto err;
2448
2449 /* round up to nearest 4K */
2450 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2451 tx_ring->size = ALIGN(tx_ring->size, 4096);
2452
2453 tx_ring->desc = dma_alloc_coherent(dev,
2454 tx_ring->size,
2455 &tx_ring->dma,
2456 GFP_KERNEL);
2457
2458 if (!tx_ring->desc)
2459 goto err;
2460
2461 tx_ring->next_to_use = 0;
2462 tx_ring->next_to_clean = 0;
2463 return 0;
2464
2465 err:
2466 vfree(tx_ring->buffer_info);
2467 dev_err(dev,
2468 "Unable to allocate memory for the transmit descriptor ring\n");
2469 return -ENOMEM;
2470 }
2471
2472 /**
2473 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2474 * (Descriptors) for all queues
2475 * @adapter: board private structure
2476 *
2477 * Return 0 on success, negative on failure
2478 **/
2479 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2480 {
2481 struct pci_dev *pdev = adapter->pdev;
2482 int i, err = 0;
2483
2484 for (i = 0; i < adapter->num_tx_queues; i++) {
2485 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2486 if (err) {
2487 dev_err(&pdev->dev,
2488 "Allocation for Tx Queue %u failed\n", i);
2489 for (i--; i >= 0; i--)
2490 igb_free_tx_resources(adapter->tx_ring[i]);
2491 break;
2492 }
2493 }
2494
2495 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2496 int r_idx = i % adapter->num_tx_queues;
2497 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2498 }
2499 return err;
2500 }
2501
2502 /**
2503 * igb_setup_tctl - configure the transmit control registers
2504 * @adapter: Board private structure
2505 **/
2506 void igb_setup_tctl(struct igb_adapter *adapter)
2507 {
2508 struct e1000_hw *hw = &adapter->hw;
2509 u32 tctl;
2510
2511 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2512 wr32(E1000_TXDCTL(0), 0);
2513
2514 /* Program the Transmit Control Register */
2515 tctl = rd32(E1000_TCTL);
2516 tctl &= ~E1000_TCTL_CT;
2517 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2518 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2519
2520 igb_config_collision_dist(hw);
2521
2522 /* Enable transmits */
2523 tctl |= E1000_TCTL_EN;
2524
2525 wr32(E1000_TCTL, tctl);
2526 }
2527
2528 /**
2529 * igb_configure_tx_ring - Configure transmit ring after Reset
2530 * @adapter: board private structure
2531 * @ring: tx ring to configure
2532 *
2533 * Configure a transmit ring after a reset.
2534 **/
2535 void igb_configure_tx_ring(struct igb_adapter *adapter,
2536 struct igb_ring *ring)
2537 {
2538 struct e1000_hw *hw = &adapter->hw;
2539 u32 txdctl;
2540 u64 tdba = ring->dma;
2541 int reg_idx = ring->reg_idx;
2542
2543 /* disable the queue */
2544 txdctl = rd32(E1000_TXDCTL(reg_idx));
2545 wr32(E1000_TXDCTL(reg_idx),
2546 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2547 wrfl();
2548 mdelay(10);
2549
2550 wr32(E1000_TDLEN(reg_idx),
2551 ring->count * sizeof(union e1000_adv_tx_desc));
2552 wr32(E1000_TDBAL(reg_idx),
2553 tdba & 0x00000000ffffffffULL);
2554 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2555
2556 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2557 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2558 writel(0, ring->head);
2559 writel(0, ring->tail);
2560
2561 txdctl |= IGB_TX_PTHRESH;
2562 txdctl |= IGB_TX_HTHRESH << 8;
2563 txdctl |= IGB_TX_WTHRESH << 16;
2564
2565 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2566 wr32(E1000_TXDCTL(reg_idx), txdctl);
2567 }
2568
2569 /**
2570 * igb_configure_tx - Configure transmit Unit after Reset
2571 * @adapter: board private structure
2572 *
2573 * Configure the Tx unit of the MAC after a reset.
2574 **/
2575 static void igb_configure_tx(struct igb_adapter *adapter)
2576 {
2577 int i;
2578
2579 for (i = 0; i < adapter->num_tx_queues; i++)
2580 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2581 }
2582
2583 /**
2584 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2585 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2586 *
2587 * Returns 0 on success, negative on failure
2588 **/
2589 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2590 {
2591 struct device *dev = rx_ring->dev;
2592 int size, desc_len;
2593
2594 size = sizeof(struct igb_buffer) * rx_ring->count;
2595 rx_ring->buffer_info = vzalloc(size);
2596 if (!rx_ring->buffer_info)
2597 goto err;
2598
2599 desc_len = sizeof(union e1000_adv_rx_desc);
2600
2601 /* Round up to nearest 4K */
2602 rx_ring->size = rx_ring->count * desc_len;
2603 rx_ring->size = ALIGN(rx_ring->size, 4096);
2604
2605 rx_ring->desc = dma_alloc_coherent(dev,
2606 rx_ring->size,
2607 &rx_ring->dma,
2608 GFP_KERNEL);
2609
2610 if (!rx_ring->desc)
2611 goto err;
2612
2613 rx_ring->next_to_clean = 0;
2614 rx_ring->next_to_use = 0;
2615
2616 return 0;
2617
2618 err:
2619 vfree(rx_ring->buffer_info);
2620 rx_ring->buffer_info = NULL;
2621 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2622 " ring\n");
2623 return -ENOMEM;
2624 }
2625
2626 /**
2627 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2628 * (Descriptors) for all queues
2629 * @adapter: board private structure
2630 *
2631 * Return 0 on success, negative on failure
2632 **/
2633 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2634 {
2635 struct pci_dev *pdev = adapter->pdev;
2636 int i, err = 0;
2637
2638 for (i = 0; i < adapter->num_rx_queues; i++) {
2639 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2640 if (err) {
2641 dev_err(&pdev->dev,
2642 "Allocation for Rx Queue %u failed\n", i);
2643 for (i--; i >= 0; i--)
2644 igb_free_rx_resources(adapter->rx_ring[i]);
2645 break;
2646 }
2647 }
2648
2649 return err;
2650 }
2651
2652 /**
2653 * igb_setup_mrqc - configure the multiple receive queue control registers
2654 * @adapter: Board private structure
2655 **/
2656 static void igb_setup_mrqc(struct igb_adapter *adapter)
2657 {
2658 struct e1000_hw *hw = &adapter->hw;
2659 u32 mrqc, rxcsum;
2660 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2661 union e1000_reta {
2662 u32 dword;
2663 u8 bytes[4];
2664 } reta;
2665 static const u8 rsshash[40] = {
2666 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2667 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2668 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2669 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2670
2671 /* Fill out hash function seeds */
2672 for (j = 0; j < 10; j++) {
2673 u32 rsskey = rsshash[(j * 4)];
2674 rsskey |= rsshash[(j * 4) + 1] << 8;
2675 rsskey |= rsshash[(j * 4) + 2] << 16;
2676 rsskey |= rsshash[(j * 4) + 3] << 24;
2677 array_wr32(E1000_RSSRK(0), j, rsskey);
2678 }
2679
2680 num_rx_queues = adapter->rss_queues;
2681
2682 if (adapter->vfs_allocated_count) {
2683 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2684 switch (hw->mac.type) {
2685 case e1000_i350:
2686 case e1000_82580:
2687 num_rx_queues = 1;
2688 shift = 0;
2689 break;
2690 case e1000_82576:
2691 shift = 3;
2692 num_rx_queues = 2;
2693 break;
2694 case e1000_82575:
2695 shift = 2;
2696 shift2 = 6;
2697 default:
2698 break;
2699 }
2700 } else {
2701 if (hw->mac.type == e1000_82575)
2702 shift = 6;
2703 }
2704
2705 for (j = 0; j < (32 * 4); j++) {
2706 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2707 if (shift2)
2708 reta.bytes[j & 3] |= num_rx_queues << shift2;
2709 if ((j & 3) == 3)
2710 wr32(E1000_RETA(j >> 2), reta.dword);
2711 }
2712
2713 /*
2714 * Disable raw packet checksumming so that RSS hash is placed in
2715 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2716 * offloads as they are enabled by default
2717 */
2718 rxcsum = rd32(E1000_RXCSUM);
2719 rxcsum |= E1000_RXCSUM_PCSD;
2720
2721 if (adapter->hw.mac.type >= e1000_82576)
2722 /* Enable Receive Checksum Offload for SCTP */
2723 rxcsum |= E1000_RXCSUM_CRCOFL;
2724
2725 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2726 wr32(E1000_RXCSUM, rxcsum);
2727
2728 /* If VMDq is enabled then we set the appropriate mode for that, else
2729 * we default to RSS so that an RSS hash is calculated per packet even
2730 * if we are only using one queue */
2731 if (adapter->vfs_allocated_count) {
2732 if (hw->mac.type > e1000_82575) {
2733 /* Set the default pool for the PF's first queue */
2734 u32 vtctl = rd32(E1000_VT_CTL);
2735 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2736 E1000_VT_CTL_DISABLE_DEF_POOL);
2737 vtctl |= adapter->vfs_allocated_count <<
2738 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2739 wr32(E1000_VT_CTL, vtctl);
2740 }
2741 if (adapter->rss_queues > 1)
2742 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2743 else
2744 mrqc = E1000_MRQC_ENABLE_VMDQ;
2745 } else {
2746 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2747 }
2748 igb_vmm_control(adapter);
2749
2750 /*
2751 * Generate RSS hash based on TCP port numbers and/or
2752 * IPv4/v6 src and dst addresses since UDP cannot be
2753 * hashed reliably due to IP fragmentation
2754 */
2755 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2756 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2757 E1000_MRQC_RSS_FIELD_IPV6 |
2758 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2759 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2760
2761 wr32(E1000_MRQC, mrqc);
2762 }
2763
2764 /**
2765 * igb_setup_rctl - configure the receive control registers
2766 * @adapter: Board private structure
2767 **/
2768 void igb_setup_rctl(struct igb_adapter *adapter)
2769 {
2770 struct e1000_hw *hw = &adapter->hw;
2771 u32 rctl;
2772
2773 rctl = rd32(E1000_RCTL);
2774
2775 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2776 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2777
2778 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2779 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2780
2781 /*
2782 * enable stripping of CRC. It's unlikely this will break BMC
2783 * redirection as it did with e1000. Newer features require
2784 * that the HW strips the CRC.
2785 */
2786 rctl |= E1000_RCTL_SECRC;
2787
2788 /* disable store bad packets and clear size bits. */
2789 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2790
2791 /* enable LPE to prevent packets larger than max_frame_size */
2792 rctl |= E1000_RCTL_LPE;
2793
2794 /* disable queue 0 to prevent tail write w/o re-config */
2795 wr32(E1000_RXDCTL(0), 0);
2796
2797 /* Attention!!! For SR-IOV PF driver operations you must enable
2798 * queue drop for all VF and PF queues to prevent head of line blocking
2799 * if an un-trusted VF does not provide descriptors to hardware.
2800 */
2801 if (adapter->vfs_allocated_count) {
2802 /* set all queue drop enable bits */
2803 wr32(E1000_QDE, ALL_QUEUES);
2804 }
2805
2806 wr32(E1000_RCTL, rctl);
2807 }
2808
2809 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2810 int vfn)
2811 {
2812 struct e1000_hw *hw = &adapter->hw;
2813 u32 vmolr;
2814
2815 /* if it isn't the PF check to see if VFs are enabled and
2816 * increase the size to support vlan tags */
2817 if (vfn < adapter->vfs_allocated_count &&
2818 adapter->vf_data[vfn].vlans_enabled)
2819 size += VLAN_TAG_SIZE;
2820
2821 vmolr = rd32(E1000_VMOLR(vfn));
2822 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2823 vmolr |= size | E1000_VMOLR_LPE;
2824 wr32(E1000_VMOLR(vfn), vmolr);
2825
2826 return 0;
2827 }
2828
2829 /**
2830 * igb_rlpml_set - set maximum receive packet size
2831 * @adapter: board private structure
2832 *
2833 * Configure maximum receivable packet size.
2834 **/
2835 static void igb_rlpml_set(struct igb_adapter *adapter)
2836 {
2837 u32 max_frame_size = adapter->max_frame_size;
2838 struct e1000_hw *hw = &adapter->hw;
2839 u16 pf_id = adapter->vfs_allocated_count;
2840
2841 if (adapter->vlgrp)
2842 max_frame_size += VLAN_TAG_SIZE;
2843
2844 /* if vfs are enabled we set RLPML to the largest possible request
2845 * size and set the VMOLR RLPML to the size we need */
2846 if (pf_id) {
2847 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2848 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2849 }
2850
2851 wr32(E1000_RLPML, max_frame_size);
2852 }
2853
2854 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2855 int vfn, bool aupe)
2856 {
2857 struct e1000_hw *hw = &adapter->hw;
2858 u32 vmolr;
2859
2860 /*
2861 * This register exists only on 82576 and newer so if we are older then
2862 * we should exit and do nothing
2863 */
2864 if (hw->mac.type < e1000_82576)
2865 return;
2866
2867 vmolr = rd32(E1000_VMOLR(vfn));
2868 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2869 if (aupe)
2870 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2871 else
2872 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2873
2874 /* clear all bits that might not be set */
2875 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2876
2877 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2878 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2879 /*
2880 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2881 * multicast packets
2882 */
2883 if (vfn <= adapter->vfs_allocated_count)
2884 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2885
2886 wr32(E1000_VMOLR(vfn), vmolr);
2887 }
2888
2889 /**
2890 * igb_configure_rx_ring - Configure a receive ring after Reset
2891 * @adapter: board private structure
2892 * @ring: receive ring to be configured
2893 *
2894 * Configure the Rx unit of the MAC after a reset.
2895 **/
2896 void igb_configure_rx_ring(struct igb_adapter *adapter,
2897 struct igb_ring *ring)
2898 {
2899 struct e1000_hw *hw = &adapter->hw;
2900 u64 rdba = ring->dma;
2901 int reg_idx = ring->reg_idx;
2902 u32 srrctl, rxdctl;
2903
2904 /* disable the queue */
2905 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2906 wr32(E1000_RXDCTL(reg_idx),
2907 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2908
2909 /* Set DMA base address registers */
2910 wr32(E1000_RDBAL(reg_idx),
2911 rdba & 0x00000000ffffffffULL);
2912 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2913 wr32(E1000_RDLEN(reg_idx),
2914 ring->count * sizeof(union e1000_adv_rx_desc));
2915
2916 /* initialize head and tail */
2917 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2918 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2919 writel(0, ring->head);
2920 writel(0, ring->tail);
2921
2922 /* set descriptor configuration */
2923 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2924 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2925 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2926 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2927 srrctl |= IGB_RXBUFFER_16384 >>
2928 E1000_SRRCTL_BSIZEPKT_SHIFT;
2929 #else
2930 srrctl |= (PAGE_SIZE / 2) >>
2931 E1000_SRRCTL_BSIZEPKT_SHIFT;
2932 #endif
2933 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2934 } else {
2935 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2936 E1000_SRRCTL_BSIZEPKT_SHIFT;
2937 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2938 }
2939 if (hw->mac.type == e1000_82580)
2940 srrctl |= E1000_SRRCTL_TIMESTAMP;
2941 /* Only set Drop Enable if we are supporting multiple queues */
2942 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2943 srrctl |= E1000_SRRCTL_DROP_EN;
2944
2945 wr32(E1000_SRRCTL(reg_idx), srrctl);
2946
2947 /* set filtering for VMDQ pools */
2948 igb_set_vmolr(adapter, reg_idx & 0x7, true);
2949
2950 /* enable receive descriptor fetching */
2951 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2952 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2953 rxdctl &= 0xFFF00000;
2954 rxdctl |= IGB_RX_PTHRESH;
2955 rxdctl |= IGB_RX_HTHRESH << 8;
2956 rxdctl |= IGB_RX_WTHRESH << 16;
2957 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2958 }
2959
2960 /**
2961 * igb_configure_rx - Configure receive Unit after Reset
2962 * @adapter: board private structure
2963 *
2964 * Configure the Rx unit of the MAC after a reset.
2965 **/
2966 static void igb_configure_rx(struct igb_adapter *adapter)
2967 {
2968 int i;
2969
2970 /* set UTA to appropriate mode */
2971 igb_set_uta(adapter);
2972
2973 /* set the correct pool for the PF default MAC address in entry 0 */
2974 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2975 adapter->vfs_allocated_count);
2976
2977 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2978 * the Base and Length of the Rx Descriptor Ring */
2979 for (i = 0; i < adapter->num_rx_queues; i++)
2980 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2981 }
2982
2983 /**
2984 * igb_free_tx_resources - Free Tx Resources per Queue
2985 * @tx_ring: Tx descriptor ring for a specific queue
2986 *
2987 * Free all transmit software resources
2988 **/
2989 void igb_free_tx_resources(struct igb_ring *tx_ring)
2990 {
2991 igb_clean_tx_ring(tx_ring);
2992
2993 vfree(tx_ring->buffer_info);
2994 tx_ring->buffer_info = NULL;
2995
2996 /* if not set, then don't free */
2997 if (!tx_ring->desc)
2998 return;
2999
3000 dma_free_coherent(tx_ring->dev, tx_ring->size,
3001 tx_ring->desc, tx_ring->dma);
3002
3003 tx_ring->desc = NULL;
3004 }
3005
3006 /**
3007 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3008 * @adapter: board private structure
3009 *
3010 * Free all transmit software resources
3011 **/
3012 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3013 {
3014 int i;
3015
3016 for (i = 0; i < adapter->num_tx_queues; i++)
3017 igb_free_tx_resources(adapter->tx_ring[i]);
3018 }
3019
3020 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3021 struct igb_buffer *buffer_info)
3022 {
3023 if (buffer_info->dma) {
3024 if (buffer_info->mapped_as_page)
3025 dma_unmap_page(tx_ring->dev,
3026 buffer_info->dma,
3027 buffer_info->length,
3028 DMA_TO_DEVICE);
3029 else
3030 dma_unmap_single(tx_ring->dev,
3031 buffer_info->dma,
3032 buffer_info->length,
3033 DMA_TO_DEVICE);
3034 buffer_info->dma = 0;
3035 }
3036 if (buffer_info->skb) {
3037 dev_kfree_skb_any(buffer_info->skb);
3038 buffer_info->skb = NULL;
3039 }
3040 buffer_info->time_stamp = 0;
3041 buffer_info->length = 0;
3042 buffer_info->next_to_watch = 0;
3043 buffer_info->mapped_as_page = false;
3044 }
3045
3046 /**
3047 * igb_clean_tx_ring - Free Tx Buffers
3048 * @tx_ring: ring to be cleaned
3049 **/
3050 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3051 {
3052 struct igb_buffer *buffer_info;
3053 unsigned long size;
3054 unsigned int i;
3055
3056 if (!tx_ring->buffer_info)
3057 return;
3058 /* Free all the Tx ring sk_buffs */
3059
3060 for (i = 0; i < tx_ring->count; i++) {
3061 buffer_info = &tx_ring->buffer_info[i];
3062 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3063 }
3064
3065 size = sizeof(struct igb_buffer) * tx_ring->count;
3066 memset(tx_ring->buffer_info, 0, size);
3067
3068 /* Zero out the descriptor ring */
3069 memset(tx_ring->desc, 0, tx_ring->size);
3070
3071 tx_ring->next_to_use = 0;
3072 tx_ring->next_to_clean = 0;
3073 }
3074
3075 /**
3076 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3077 * @adapter: board private structure
3078 **/
3079 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3080 {
3081 int i;
3082
3083 for (i = 0; i < adapter->num_tx_queues; i++)
3084 igb_clean_tx_ring(adapter->tx_ring[i]);
3085 }
3086
3087 /**
3088 * igb_free_rx_resources - Free Rx Resources
3089 * @rx_ring: ring to clean the resources from
3090 *
3091 * Free all receive software resources
3092 **/
3093 void igb_free_rx_resources(struct igb_ring *rx_ring)
3094 {
3095 igb_clean_rx_ring(rx_ring);
3096
3097 vfree(rx_ring->buffer_info);
3098 rx_ring->buffer_info = NULL;
3099
3100 /* if not set, then don't free */
3101 if (!rx_ring->desc)
3102 return;
3103
3104 dma_free_coherent(rx_ring->dev, rx_ring->size,
3105 rx_ring->desc, rx_ring->dma);
3106
3107 rx_ring->desc = NULL;
3108 }
3109
3110 /**
3111 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3112 * @adapter: board private structure
3113 *
3114 * Free all receive software resources
3115 **/
3116 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3117 {
3118 int i;
3119
3120 for (i = 0; i < adapter->num_rx_queues; i++)
3121 igb_free_rx_resources(adapter->rx_ring[i]);
3122 }
3123
3124 /**
3125 * igb_clean_rx_ring - Free Rx Buffers per Queue
3126 * @rx_ring: ring to free buffers from
3127 **/
3128 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3129 {
3130 struct igb_buffer *buffer_info;
3131 unsigned long size;
3132 unsigned int i;
3133
3134 if (!rx_ring->buffer_info)
3135 return;
3136
3137 /* Free all the Rx ring sk_buffs */
3138 for (i = 0; i < rx_ring->count; i++) {
3139 buffer_info = &rx_ring->buffer_info[i];
3140 if (buffer_info->dma) {
3141 dma_unmap_single(rx_ring->dev,
3142 buffer_info->dma,
3143 rx_ring->rx_buffer_len,
3144 DMA_FROM_DEVICE);
3145 buffer_info->dma = 0;
3146 }
3147
3148 if (buffer_info->skb) {
3149 dev_kfree_skb(buffer_info->skb);
3150 buffer_info->skb = NULL;
3151 }
3152 if (buffer_info->page_dma) {
3153 dma_unmap_page(rx_ring->dev,
3154 buffer_info->page_dma,
3155 PAGE_SIZE / 2,
3156 DMA_FROM_DEVICE);
3157 buffer_info->page_dma = 0;
3158 }
3159 if (buffer_info->page) {
3160 put_page(buffer_info->page);
3161 buffer_info->page = NULL;
3162 buffer_info->page_offset = 0;
3163 }
3164 }
3165
3166 size = sizeof(struct igb_buffer) * rx_ring->count;
3167 memset(rx_ring->buffer_info, 0, size);
3168
3169 /* Zero out the descriptor ring */
3170 memset(rx_ring->desc, 0, rx_ring->size);
3171
3172 rx_ring->next_to_clean = 0;
3173 rx_ring->next_to_use = 0;
3174 }
3175
3176 /**
3177 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3178 * @adapter: board private structure
3179 **/
3180 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3181 {
3182 int i;
3183
3184 for (i = 0; i < adapter->num_rx_queues; i++)
3185 igb_clean_rx_ring(adapter->rx_ring[i]);
3186 }
3187
3188 /**
3189 * igb_set_mac - Change the Ethernet Address of the NIC
3190 * @netdev: network interface device structure
3191 * @p: pointer to an address structure
3192 *
3193 * Returns 0 on success, negative on failure
3194 **/
3195 static int igb_set_mac(struct net_device *netdev, void *p)
3196 {
3197 struct igb_adapter *adapter = netdev_priv(netdev);
3198 struct e1000_hw *hw = &adapter->hw;
3199 struct sockaddr *addr = p;
3200
3201 if (!is_valid_ether_addr(addr->sa_data))
3202 return -EADDRNOTAVAIL;
3203
3204 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3205 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3206
3207 /* set the correct pool for the new PF MAC address in entry 0 */
3208 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3209 adapter->vfs_allocated_count);
3210
3211 return 0;
3212 }
3213
3214 /**
3215 * igb_write_mc_addr_list - write multicast addresses to MTA
3216 * @netdev: network interface device structure
3217 *
3218 * Writes multicast address list to the MTA hash table.
3219 * Returns: -ENOMEM on failure
3220 * 0 on no addresses written
3221 * X on writing X addresses to MTA
3222 **/
3223 static int igb_write_mc_addr_list(struct net_device *netdev)
3224 {
3225 struct igb_adapter *adapter = netdev_priv(netdev);
3226 struct e1000_hw *hw = &adapter->hw;
3227 struct netdev_hw_addr *ha;
3228 u8 *mta_list;
3229 int i;
3230
3231 if (netdev_mc_empty(netdev)) {
3232 /* nothing to program, so clear mc list */
3233 igb_update_mc_addr_list(hw, NULL, 0);
3234 igb_restore_vf_multicasts(adapter);
3235 return 0;
3236 }
3237
3238 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3239 if (!mta_list)
3240 return -ENOMEM;
3241
3242 /* The shared function expects a packed array of only addresses. */
3243 i = 0;
3244 netdev_for_each_mc_addr(ha, netdev)
3245 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3246
3247 igb_update_mc_addr_list(hw, mta_list, i);
3248 kfree(mta_list);
3249
3250 return netdev_mc_count(netdev);
3251 }
3252
3253 /**
3254 * igb_write_uc_addr_list - write unicast addresses to RAR table
3255 * @netdev: network interface device structure
3256 *
3257 * Writes unicast address list to the RAR table.
3258 * Returns: -ENOMEM on failure/insufficient address space
3259 * 0 on no addresses written
3260 * X on writing X addresses to the RAR table
3261 **/
3262 static int igb_write_uc_addr_list(struct net_device *netdev)
3263 {
3264 struct igb_adapter *adapter = netdev_priv(netdev);
3265 struct e1000_hw *hw = &adapter->hw;
3266 unsigned int vfn = adapter->vfs_allocated_count;
3267 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3268 int count = 0;
3269
3270 /* return ENOMEM indicating insufficient memory for addresses */
3271 if (netdev_uc_count(netdev) > rar_entries)
3272 return -ENOMEM;
3273
3274 if (!netdev_uc_empty(netdev) && rar_entries) {
3275 struct netdev_hw_addr *ha;
3276
3277 netdev_for_each_uc_addr(ha, netdev) {
3278 if (!rar_entries)
3279 break;
3280 igb_rar_set_qsel(adapter, ha->addr,
3281 rar_entries--,
3282 vfn);
3283 count++;
3284 }
3285 }
3286 /* write the addresses in reverse order to avoid write combining */
3287 for (; rar_entries > 0 ; rar_entries--) {
3288 wr32(E1000_RAH(rar_entries), 0);
3289 wr32(E1000_RAL(rar_entries), 0);
3290 }
3291 wrfl();
3292
3293 return count;
3294 }
3295
3296 /**
3297 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3298 * @netdev: network interface device structure
3299 *
3300 * The set_rx_mode entry point is called whenever the unicast or multicast
3301 * address lists or the network interface flags are updated. This routine is
3302 * responsible for configuring the hardware for proper unicast, multicast,
3303 * promiscuous mode, and all-multi behavior.
3304 **/
3305 static void igb_set_rx_mode(struct net_device *netdev)
3306 {
3307 struct igb_adapter *adapter = netdev_priv(netdev);
3308 struct e1000_hw *hw = &adapter->hw;
3309 unsigned int vfn = adapter->vfs_allocated_count;
3310 u32 rctl, vmolr = 0;
3311 int count;
3312
3313 /* Check for Promiscuous and All Multicast modes */
3314 rctl = rd32(E1000_RCTL);
3315
3316 /* clear the effected bits */
3317 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3318
3319 if (netdev->flags & IFF_PROMISC) {
3320 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3321 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3322 } else {
3323 if (netdev->flags & IFF_ALLMULTI) {
3324 rctl |= E1000_RCTL_MPE;
3325 vmolr |= E1000_VMOLR_MPME;
3326 } else {
3327 /*
3328 * Write addresses to the MTA, if the attempt fails
3329 * then we should just turn on promiscous mode so
3330 * that we can at least receive multicast traffic
3331 */
3332 count = igb_write_mc_addr_list(netdev);
3333 if (count < 0) {
3334 rctl |= E1000_RCTL_MPE;
3335 vmolr |= E1000_VMOLR_MPME;
3336 } else if (count) {
3337 vmolr |= E1000_VMOLR_ROMPE;
3338 }
3339 }
3340 /*
3341 * Write addresses to available RAR registers, if there is not
3342 * sufficient space to store all the addresses then enable
3343 * unicast promiscous mode
3344 */
3345 count = igb_write_uc_addr_list(netdev);
3346 if (count < 0) {
3347 rctl |= E1000_RCTL_UPE;
3348 vmolr |= E1000_VMOLR_ROPE;
3349 }
3350 rctl |= E1000_RCTL_VFE;
3351 }
3352 wr32(E1000_RCTL, rctl);
3353
3354 /*
3355 * In order to support SR-IOV and eventually VMDq it is necessary to set
3356 * the VMOLR to enable the appropriate modes. Without this workaround
3357 * we will have issues with VLAN tag stripping not being done for frames
3358 * that are only arriving because we are the default pool
3359 */
3360 if (hw->mac.type < e1000_82576)
3361 return;
3362
3363 vmolr |= rd32(E1000_VMOLR(vfn)) &
3364 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3365 wr32(E1000_VMOLR(vfn), vmolr);
3366 igb_restore_vf_multicasts(adapter);
3367 }
3368
3369 /* Need to wait a few seconds after link up to get diagnostic information from
3370 * the phy */
3371 static void igb_update_phy_info(unsigned long data)
3372 {
3373 struct igb_adapter *adapter = (struct igb_adapter *) data;
3374 igb_get_phy_info(&adapter->hw);
3375 }
3376
3377 /**
3378 * igb_has_link - check shared code for link and determine up/down
3379 * @adapter: pointer to driver private info
3380 **/
3381 bool igb_has_link(struct igb_adapter *adapter)
3382 {
3383 struct e1000_hw *hw = &adapter->hw;
3384 bool link_active = false;
3385 s32 ret_val = 0;
3386
3387 /* get_link_status is set on LSC (link status) interrupt or
3388 * rx sequence error interrupt. get_link_status will stay
3389 * false until the e1000_check_for_link establishes link
3390 * for copper adapters ONLY
3391 */
3392 switch (hw->phy.media_type) {
3393 case e1000_media_type_copper:
3394 if (hw->mac.get_link_status) {
3395 ret_val = hw->mac.ops.check_for_link(hw);
3396 link_active = !hw->mac.get_link_status;
3397 } else {
3398 link_active = true;
3399 }
3400 break;
3401 case e1000_media_type_internal_serdes:
3402 ret_val = hw->mac.ops.check_for_link(hw);
3403 link_active = hw->mac.serdes_has_link;
3404 break;
3405 default:
3406 case e1000_media_type_unknown:
3407 break;
3408 }
3409
3410 return link_active;
3411 }
3412
3413 /**
3414 * igb_watchdog - Timer Call-back
3415 * @data: pointer to adapter cast into an unsigned long
3416 **/
3417 static void igb_watchdog(unsigned long data)
3418 {
3419 struct igb_adapter *adapter = (struct igb_adapter *)data;
3420 /* Do the rest outside of interrupt context */
3421 schedule_work(&adapter->watchdog_task);
3422 }
3423
3424 static void igb_watchdog_task(struct work_struct *work)
3425 {
3426 struct igb_adapter *adapter = container_of(work,
3427 struct igb_adapter,
3428 watchdog_task);
3429 struct e1000_hw *hw = &adapter->hw;
3430 struct net_device *netdev = adapter->netdev;
3431 u32 link;
3432 int i;
3433
3434 link = igb_has_link(adapter);
3435 if (link) {
3436 if (!netif_carrier_ok(netdev)) {
3437 u32 ctrl;
3438 hw->mac.ops.get_speed_and_duplex(hw,
3439 &adapter->link_speed,
3440 &adapter->link_duplex);
3441
3442 ctrl = rd32(E1000_CTRL);
3443 /* Links status message must follow this format */
3444 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3445 "Flow Control: %s\n",
3446 netdev->name,
3447 adapter->link_speed,
3448 adapter->link_duplex == FULL_DUPLEX ?
3449 "Full Duplex" : "Half Duplex",
3450 ((ctrl & E1000_CTRL_TFCE) &&
3451 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3452 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3453 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3454
3455 /* adjust timeout factor according to speed/duplex */
3456 adapter->tx_timeout_factor = 1;
3457 switch (adapter->link_speed) {
3458 case SPEED_10:
3459 adapter->tx_timeout_factor = 14;
3460 break;
3461 case SPEED_100:
3462 /* maybe add some timeout factor ? */
3463 break;
3464 }
3465
3466 netif_carrier_on(netdev);
3467
3468 igb_ping_all_vfs(adapter);
3469
3470 /* link state has changed, schedule phy info update */
3471 if (!test_bit(__IGB_DOWN, &adapter->state))
3472 mod_timer(&adapter->phy_info_timer,
3473 round_jiffies(jiffies + 2 * HZ));
3474 }
3475 } else {
3476 if (netif_carrier_ok(netdev)) {
3477 adapter->link_speed = 0;
3478 adapter->link_duplex = 0;
3479 /* Links status message must follow this format */
3480 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3481 netdev->name);
3482 netif_carrier_off(netdev);
3483
3484 igb_ping_all_vfs(adapter);
3485
3486 /* link state has changed, schedule phy info update */
3487 if (!test_bit(__IGB_DOWN, &adapter->state))
3488 mod_timer(&adapter->phy_info_timer,
3489 round_jiffies(jiffies + 2 * HZ));
3490 }
3491 }
3492
3493 spin_lock(&adapter->stats64_lock);
3494 igb_update_stats(adapter, &adapter->stats64);
3495 spin_unlock(&adapter->stats64_lock);
3496
3497 for (i = 0; i < adapter->num_tx_queues; i++) {
3498 struct igb_ring *tx_ring = adapter->tx_ring[i];
3499 if (!netif_carrier_ok(netdev)) {
3500 /* We've lost link, so the controller stops DMA,
3501 * but we've got queued Tx work that's never going
3502 * to get done, so reset controller to flush Tx.
3503 * (Do the reset outside of interrupt context). */
3504 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3505 adapter->tx_timeout_count++;
3506 schedule_work(&adapter->reset_task);
3507 /* return immediately since reset is imminent */
3508 return;
3509 }
3510 }
3511
3512 /* Force detection of hung controller every watchdog period */
3513 tx_ring->detect_tx_hung = true;
3514 }
3515
3516 /* Cause software interrupt to ensure rx ring is cleaned */
3517 if (adapter->msix_entries) {
3518 u32 eics = 0;
3519 for (i = 0; i < adapter->num_q_vectors; i++) {
3520 struct igb_q_vector *q_vector = adapter->q_vector[i];
3521 eics |= q_vector->eims_value;
3522 }
3523 wr32(E1000_EICS, eics);
3524 } else {
3525 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3526 }
3527
3528 /* Reset the timer */
3529 if (!test_bit(__IGB_DOWN, &adapter->state))
3530 mod_timer(&adapter->watchdog_timer,
3531 round_jiffies(jiffies + 2 * HZ));
3532 }
3533
3534 enum latency_range {
3535 lowest_latency = 0,
3536 low_latency = 1,
3537 bulk_latency = 2,
3538 latency_invalid = 255
3539 };
3540
3541 /**
3542 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3543 *
3544 * Stores a new ITR value based on strictly on packet size. This
3545 * algorithm is less sophisticated than that used in igb_update_itr,
3546 * due to the difficulty of synchronizing statistics across multiple
3547 * receive rings. The divisors and thresholds used by this function
3548 * were determined based on theoretical maximum wire speed and testing
3549 * data, in order to minimize response time while increasing bulk
3550 * throughput.
3551 * This functionality is controlled by the InterruptThrottleRate module
3552 * parameter (see igb_param.c)
3553 * NOTE: This function is called only when operating in a multiqueue
3554 * receive environment.
3555 * @q_vector: pointer to q_vector
3556 **/
3557 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3558 {
3559 int new_val = q_vector->itr_val;
3560 int avg_wire_size = 0;
3561 struct igb_adapter *adapter = q_vector->adapter;
3562 struct igb_ring *ring;
3563 unsigned int packets;
3564
3565 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3566 * ints/sec - ITR timer value of 120 ticks.
3567 */
3568 if (adapter->link_speed != SPEED_1000) {
3569 new_val = 976;
3570 goto set_itr_val;
3571 }
3572
3573 ring = q_vector->rx_ring;
3574 if (ring) {
3575 packets = ACCESS_ONCE(ring->total_packets);
3576
3577 if (packets)
3578 avg_wire_size = ring->total_bytes / packets;
3579 }
3580
3581 ring = q_vector->tx_ring;
3582 if (ring) {
3583 packets = ACCESS_ONCE(ring->total_packets);
3584
3585 if (packets)
3586 avg_wire_size = max_t(u32, avg_wire_size,
3587 ring->total_bytes / packets);
3588 }
3589
3590 /* if avg_wire_size isn't set no work was done */
3591 if (!avg_wire_size)
3592 goto clear_counts;
3593
3594 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3595 avg_wire_size += 24;
3596
3597 /* Don't starve jumbo frames */
3598 avg_wire_size = min(avg_wire_size, 3000);
3599
3600 /* Give a little boost to mid-size frames */
3601 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3602 new_val = avg_wire_size / 3;
3603 else
3604 new_val = avg_wire_size / 2;
3605
3606 /* when in itr mode 3 do not exceed 20K ints/sec */
3607 if (adapter->rx_itr_setting == 3 && new_val < 196)
3608 new_val = 196;
3609
3610 set_itr_val:
3611 if (new_val != q_vector->itr_val) {
3612 q_vector->itr_val = new_val;
3613 q_vector->set_itr = 1;
3614 }
3615 clear_counts:
3616 if (q_vector->rx_ring) {
3617 q_vector->rx_ring->total_bytes = 0;
3618 q_vector->rx_ring->total_packets = 0;
3619 }
3620 if (q_vector->tx_ring) {
3621 q_vector->tx_ring->total_bytes = 0;
3622 q_vector->tx_ring->total_packets = 0;
3623 }
3624 }
3625
3626 /**
3627 * igb_update_itr - update the dynamic ITR value based on statistics
3628 * Stores a new ITR value based on packets and byte
3629 * counts during the last interrupt. The advantage of per interrupt
3630 * computation is faster updates and more accurate ITR for the current
3631 * traffic pattern. Constants in this function were computed
3632 * based on theoretical maximum wire speed and thresholds were set based
3633 * on testing data as well as attempting to minimize response time
3634 * while increasing bulk throughput.
3635 * this functionality is controlled by the InterruptThrottleRate module
3636 * parameter (see igb_param.c)
3637 * NOTE: These calculations are only valid when operating in a single-
3638 * queue environment.
3639 * @adapter: pointer to adapter
3640 * @itr_setting: current q_vector->itr_val
3641 * @packets: the number of packets during this measurement interval
3642 * @bytes: the number of bytes during this measurement interval
3643 **/
3644 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3645 int packets, int bytes)
3646 {
3647 unsigned int retval = itr_setting;
3648
3649 if (packets == 0)
3650 goto update_itr_done;
3651
3652 switch (itr_setting) {
3653 case lowest_latency:
3654 /* handle TSO and jumbo frames */
3655 if (bytes/packets > 8000)
3656 retval = bulk_latency;
3657 else if ((packets < 5) && (bytes > 512))
3658 retval = low_latency;
3659 break;
3660 case low_latency: /* 50 usec aka 20000 ints/s */
3661 if (bytes > 10000) {
3662 /* this if handles the TSO accounting */
3663 if (bytes/packets > 8000) {
3664 retval = bulk_latency;
3665 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3666 retval = bulk_latency;
3667 } else if ((packets > 35)) {
3668 retval = lowest_latency;
3669 }
3670 } else if (bytes/packets > 2000) {
3671 retval = bulk_latency;
3672 } else if (packets <= 2 && bytes < 512) {
3673 retval = lowest_latency;
3674 }
3675 break;
3676 case bulk_latency: /* 250 usec aka 4000 ints/s */
3677 if (bytes > 25000) {
3678 if (packets > 35)
3679 retval = low_latency;
3680 } else if (bytes < 1500) {
3681 retval = low_latency;
3682 }
3683 break;
3684 }
3685
3686 update_itr_done:
3687 return retval;
3688 }
3689
3690 static void igb_set_itr(struct igb_adapter *adapter)
3691 {
3692 struct igb_q_vector *q_vector = adapter->q_vector[0];
3693 u16 current_itr;
3694 u32 new_itr = q_vector->itr_val;
3695
3696 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3697 if (adapter->link_speed != SPEED_1000) {
3698 current_itr = 0;
3699 new_itr = 4000;
3700 goto set_itr_now;
3701 }
3702
3703 adapter->rx_itr = igb_update_itr(adapter,
3704 adapter->rx_itr,
3705 q_vector->rx_ring->total_packets,
3706 q_vector->rx_ring->total_bytes);
3707
3708 adapter->tx_itr = igb_update_itr(adapter,
3709 adapter->tx_itr,
3710 q_vector->tx_ring->total_packets,
3711 q_vector->tx_ring->total_bytes);
3712 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3713
3714 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3715 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3716 current_itr = low_latency;
3717
3718 switch (current_itr) {
3719 /* counts and packets in update_itr are dependent on these numbers */
3720 case lowest_latency:
3721 new_itr = 56; /* aka 70,000 ints/sec */
3722 break;
3723 case low_latency:
3724 new_itr = 196; /* aka 20,000 ints/sec */
3725 break;
3726 case bulk_latency:
3727 new_itr = 980; /* aka 4,000 ints/sec */
3728 break;
3729 default:
3730 break;
3731 }
3732
3733 set_itr_now:
3734 q_vector->rx_ring->total_bytes = 0;
3735 q_vector->rx_ring->total_packets = 0;
3736 q_vector->tx_ring->total_bytes = 0;
3737 q_vector->tx_ring->total_packets = 0;
3738
3739 if (new_itr != q_vector->itr_val) {
3740 /* this attempts to bias the interrupt rate towards Bulk
3741 * by adding intermediate steps when interrupt rate is
3742 * increasing */
3743 new_itr = new_itr > q_vector->itr_val ?
3744 max((new_itr * q_vector->itr_val) /
3745 (new_itr + (q_vector->itr_val >> 2)),
3746 new_itr) :
3747 new_itr;
3748 /* Don't write the value here; it resets the adapter's
3749 * internal timer, and causes us to delay far longer than
3750 * we should between interrupts. Instead, we write the ITR
3751 * value at the beginning of the next interrupt so the timing
3752 * ends up being correct.
3753 */
3754 q_vector->itr_val = new_itr;
3755 q_vector->set_itr = 1;
3756 }
3757 }
3758
3759 #define IGB_TX_FLAGS_CSUM 0x00000001
3760 #define IGB_TX_FLAGS_VLAN 0x00000002
3761 #define IGB_TX_FLAGS_TSO 0x00000004
3762 #define IGB_TX_FLAGS_IPV4 0x00000008
3763 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3764 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3765 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3766
3767 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3768 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3769 {
3770 struct e1000_adv_tx_context_desc *context_desc;
3771 unsigned int i;
3772 int err;
3773 struct igb_buffer *buffer_info;
3774 u32 info = 0, tu_cmd = 0;
3775 u32 mss_l4len_idx;
3776 u8 l4len;
3777
3778 if (skb_header_cloned(skb)) {
3779 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3780 if (err)
3781 return err;
3782 }
3783
3784 l4len = tcp_hdrlen(skb);
3785 *hdr_len += l4len;
3786
3787 if (skb->protocol == htons(ETH_P_IP)) {
3788 struct iphdr *iph = ip_hdr(skb);
3789 iph->tot_len = 0;
3790 iph->check = 0;
3791 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3792 iph->daddr, 0,
3793 IPPROTO_TCP,
3794 0);
3795 } else if (skb_is_gso_v6(skb)) {
3796 ipv6_hdr(skb)->payload_len = 0;
3797 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3798 &ipv6_hdr(skb)->daddr,
3799 0, IPPROTO_TCP, 0);
3800 }
3801
3802 i = tx_ring->next_to_use;
3803
3804 buffer_info = &tx_ring->buffer_info[i];
3805 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3806 /* VLAN MACLEN IPLEN */
3807 if (tx_flags & IGB_TX_FLAGS_VLAN)
3808 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3809 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3810 *hdr_len += skb_network_offset(skb);
3811 info |= skb_network_header_len(skb);
3812 *hdr_len += skb_network_header_len(skb);
3813 context_desc->vlan_macip_lens = cpu_to_le32(info);
3814
3815 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3816 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3817
3818 if (skb->protocol == htons(ETH_P_IP))
3819 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3820 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3821
3822 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3823
3824 /* MSS L4LEN IDX */
3825 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3826 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3827
3828 /* For 82575, context index must be unique per ring. */
3829 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3830 mss_l4len_idx |= tx_ring->reg_idx << 4;
3831
3832 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3833 context_desc->seqnum_seed = 0;
3834
3835 buffer_info->time_stamp = jiffies;
3836 buffer_info->next_to_watch = i;
3837 buffer_info->dma = 0;
3838 i++;
3839 if (i == tx_ring->count)
3840 i = 0;
3841
3842 tx_ring->next_to_use = i;
3843
3844 return true;
3845 }
3846
3847 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3848 struct sk_buff *skb, u32 tx_flags)
3849 {
3850 struct e1000_adv_tx_context_desc *context_desc;
3851 struct device *dev = tx_ring->dev;
3852 struct igb_buffer *buffer_info;
3853 u32 info = 0, tu_cmd = 0;
3854 unsigned int i;
3855
3856 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3857 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3858 i = tx_ring->next_to_use;
3859 buffer_info = &tx_ring->buffer_info[i];
3860 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3861
3862 if (tx_flags & IGB_TX_FLAGS_VLAN)
3863 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3864
3865 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3866 if (skb->ip_summed == CHECKSUM_PARTIAL)
3867 info |= skb_network_header_len(skb);
3868
3869 context_desc->vlan_macip_lens = cpu_to_le32(info);
3870
3871 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3872
3873 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3874 __be16 protocol;
3875
3876 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3877 const struct vlan_ethhdr *vhdr =
3878 (const struct vlan_ethhdr*)skb->data;
3879
3880 protocol = vhdr->h_vlan_encapsulated_proto;
3881 } else {
3882 protocol = skb->protocol;
3883 }
3884
3885 switch (protocol) {
3886 case cpu_to_be16(ETH_P_IP):
3887 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3888 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3889 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3890 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3891 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3892 break;
3893 case cpu_to_be16(ETH_P_IPV6):
3894 /* XXX what about other V6 headers?? */
3895 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3896 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3897 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3898 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3899 break;
3900 default:
3901 if (unlikely(net_ratelimit()))
3902 dev_warn(dev,
3903 "partial checksum but proto=%x!\n",
3904 skb->protocol);
3905 break;
3906 }
3907 }
3908
3909 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3910 context_desc->seqnum_seed = 0;
3911 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3912 context_desc->mss_l4len_idx =
3913 cpu_to_le32(tx_ring->reg_idx << 4);
3914
3915 buffer_info->time_stamp = jiffies;
3916 buffer_info->next_to_watch = i;
3917 buffer_info->dma = 0;
3918
3919 i++;
3920 if (i == tx_ring->count)
3921 i = 0;
3922 tx_ring->next_to_use = i;
3923
3924 return true;
3925 }
3926 return false;
3927 }
3928
3929 #define IGB_MAX_TXD_PWR 16
3930 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3931
3932 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3933 unsigned int first)
3934 {
3935 struct igb_buffer *buffer_info;
3936 struct device *dev = tx_ring->dev;
3937 unsigned int hlen = skb_headlen(skb);
3938 unsigned int count = 0, i;
3939 unsigned int f;
3940 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3941
3942 i = tx_ring->next_to_use;
3943
3944 buffer_info = &tx_ring->buffer_info[i];
3945 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3946 buffer_info->length = hlen;
3947 /* set time_stamp *before* dma to help avoid a possible race */
3948 buffer_info->time_stamp = jiffies;
3949 buffer_info->next_to_watch = i;
3950 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3951 DMA_TO_DEVICE);
3952 if (dma_mapping_error(dev, buffer_info->dma))
3953 goto dma_error;
3954
3955 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3956 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3957 unsigned int len = frag->size;
3958
3959 count++;
3960 i++;
3961 if (i == tx_ring->count)
3962 i = 0;
3963
3964 buffer_info = &tx_ring->buffer_info[i];
3965 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3966 buffer_info->length = len;
3967 buffer_info->time_stamp = jiffies;
3968 buffer_info->next_to_watch = i;
3969 buffer_info->mapped_as_page = true;
3970 buffer_info->dma = dma_map_page(dev,
3971 frag->page,
3972 frag->page_offset,
3973 len,
3974 DMA_TO_DEVICE);
3975 if (dma_mapping_error(dev, buffer_info->dma))
3976 goto dma_error;
3977
3978 }
3979
3980 tx_ring->buffer_info[i].skb = skb;
3981 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
3982 /* multiply data chunks by size of headers */
3983 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3984 tx_ring->buffer_info[i].gso_segs = gso_segs;
3985 tx_ring->buffer_info[first].next_to_watch = i;
3986
3987 return ++count;
3988
3989 dma_error:
3990 dev_err(dev, "TX DMA map failed\n");
3991
3992 /* clear timestamp and dma mappings for failed buffer_info mapping */
3993 buffer_info->dma = 0;
3994 buffer_info->time_stamp = 0;
3995 buffer_info->length = 0;
3996 buffer_info->next_to_watch = 0;
3997 buffer_info->mapped_as_page = false;
3998
3999 /* clear timestamp and dma mappings for remaining portion of packet */
4000 while (count--) {
4001 if (i == 0)
4002 i = tx_ring->count;
4003 i--;
4004 buffer_info = &tx_ring->buffer_info[i];
4005 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4006 }
4007
4008 return 0;
4009 }
4010
4011 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
4012 u32 tx_flags, int count, u32 paylen,
4013 u8 hdr_len)
4014 {
4015 union e1000_adv_tx_desc *tx_desc;
4016 struct igb_buffer *buffer_info;
4017 u32 olinfo_status = 0, cmd_type_len;
4018 unsigned int i = tx_ring->next_to_use;
4019
4020 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4021 E1000_ADVTXD_DCMD_DEXT);
4022
4023 if (tx_flags & IGB_TX_FLAGS_VLAN)
4024 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4025
4026 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4027 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4028
4029 if (tx_flags & IGB_TX_FLAGS_TSO) {
4030 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4031
4032 /* insert tcp checksum */
4033 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4034
4035 /* insert ip checksum */
4036 if (tx_flags & IGB_TX_FLAGS_IPV4)
4037 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4038
4039 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4040 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4041 }
4042
4043 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4044 (tx_flags & (IGB_TX_FLAGS_CSUM |
4045 IGB_TX_FLAGS_TSO |
4046 IGB_TX_FLAGS_VLAN)))
4047 olinfo_status |= tx_ring->reg_idx << 4;
4048
4049 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4050
4051 do {
4052 buffer_info = &tx_ring->buffer_info[i];
4053 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4054 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4055 tx_desc->read.cmd_type_len =
4056 cpu_to_le32(cmd_type_len | buffer_info->length);
4057 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4058 count--;
4059 i++;
4060 if (i == tx_ring->count)
4061 i = 0;
4062 } while (count > 0);
4063
4064 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4065 /* Force memory writes to complete before letting h/w
4066 * know there are new descriptors to fetch. (Only
4067 * applicable for weak-ordered memory model archs,
4068 * such as IA-64). */
4069 wmb();
4070
4071 tx_ring->next_to_use = i;
4072 writel(i, tx_ring->tail);
4073 /* we need this if more than one processor can write to our tail
4074 * at a time, it syncronizes IO on IA64/Altix systems */
4075 mmiowb();
4076 }
4077
4078 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4079 {
4080 struct net_device *netdev = tx_ring->netdev;
4081
4082 netif_stop_subqueue(netdev, tx_ring->queue_index);
4083
4084 /* Herbert's original patch had:
4085 * smp_mb__after_netif_stop_queue();
4086 * but since that doesn't exist yet, just open code it. */
4087 smp_mb();
4088
4089 /* We need to check again in a case another CPU has just
4090 * made room available. */
4091 if (igb_desc_unused(tx_ring) < size)
4092 return -EBUSY;
4093
4094 /* A reprieve! */
4095 netif_wake_subqueue(netdev, tx_ring->queue_index);
4096
4097 u64_stats_update_begin(&tx_ring->tx_syncp2);
4098 tx_ring->tx_stats.restart_queue2++;
4099 u64_stats_update_end(&tx_ring->tx_syncp2);
4100
4101 return 0;
4102 }
4103
4104 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4105 {
4106 if (igb_desc_unused(tx_ring) >= size)
4107 return 0;
4108 return __igb_maybe_stop_tx(tx_ring, size);
4109 }
4110
4111 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4112 struct igb_ring *tx_ring)
4113 {
4114 int tso = 0, count;
4115 u32 tx_flags = 0;
4116 u16 first;
4117 u8 hdr_len = 0;
4118
4119 /* need: 1 descriptor per page,
4120 * + 2 desc gap to keep tail from touching head,
4121 * + 1 desc for skb->data,
4122 * + 1 desc for context descriptor,
4123 * otherwise try next time */
4124 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4125 /* this is a hard error */
4126 return NETDEV_TX_BUSY;
4127 }
4128
4129 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4130 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4131 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4132 }
4133
4134 if (vlan_tx_tag_present(skb)) {
4135 tx_flags |= IGB_TX_FLAGS_VLAN;
4136 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4137 }
4138
4139 if (skb->protocol == htons(ETH_P_IP))
4140 tx_flags |= IGB_TX_FLAGS_IPV4;
4141
4142 first = tx_ring->next_to_use;
4143 if (skb_is_gso(skb)) {
4144 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4145
4146 if (tso < 0) {
4147 dev_kfree_skb_any(skb);
4148 return NETDEV_TX_OK;
4149 }
4150 }
4151
4152 if (tso)
4153 tx_flags |= IGB_TX_FLAGS_TSO;
4154 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4155 (skb->ip_summed == CHECKSUM_PARTIAL))
4156 tx_flags |= IGB_TX_FLAGS_CSUM;
4157
4158 /*
4159 * count reflects descriptors mapped, if 0 or less then mapping error
4160 * has occured and we need to rewind the descriptor queue
4161 */
4162 count = igb_tx_map_adv(tx_ring, skb, first);
4163 if (!count) {
4164 dev_kfree_skb_any(skb);
4165 tx_ring->buffer_info[first].time_stamp = 0;
4166 tx_ring->next_to_use = first;
4167 return NETDEV_TX_OK;
4168 }
4169
4170 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4171
4172 /* Make sure there is space in the ring for the next send. */
4173 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4174
4175 return NETDEV_TX_OK;
4176 }
4177
4178 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4179 struct net_device *netdev)
4180 {
4181 struct igb_adapter *adapter = netdev_priv(netdev);
4182 struct igb_ring *tx_ring;
4183 int r_idx = 0;
4184
4185 if (test_bit(__IGB_DOWN, &adapter->state)) {
4186 dev_kfree_skb_any(skb);
4187 return NETDEV_TX_OK;
4188 }
4189
4190 if (skb->len <= 0) {
4191 dev_kfree_skb_any(skb);
4192 return NETDEV_TX_OK;
4193 }
4194
4195 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4196 tx_ring = adapter->multi_tx_table[r_idx];
4197
4198 /* This goes back to the question of how to logically map a tx queue
4199 * to a flow. Right now, performance is impacted slightly negatively
4200 * if using multiple tx queues. If the stack breaks away from a
4201 * single qdisc implementation, we can look at this again. */
4202 return igb_xmit_frame_ring_adv(skb, tx_ring);
4203 }
4204
4205 /**
4206 * igb_tx_timeout - Respond to a Tx Hang
4207 * @netdev: network interface device structure
4208 **/
4209 static void igb_tx_timeout(struct net_device *netdev)
4210 {
4211 struct igb_adapter *adapter = netdev_priv(netdev);
4212 struct e1000_hw *hw = &adapter->hw;
4213
4214 /* Do the reset outside of interrupt context */
4215 adapter->tx_timeout_count++;
4216
4217 if (hw->mac.type == e1000_82580)
4218 hw->dev_spec._82575.global_device_reset = true;
4219
4220 schedule_work(&adapter->reset_task);
4221 wr32(E1000_EICS,
4222 (adapter->eims_enable_mask & ~adapter->eims_other));
4223 }
4224
4225 static void igb_reset_task(struct work_struct *work)
4226 {
4227 struct igb_adapter *adapter;
4228 adapter = container_of(work, struct igb_adapter, reset_task);
4229
4230 igb_dump(adapter);
4231 netdev_err(adapter->netdev, "Reset adapter\n");
4232 igb_reinit_locked(adapter);
4233 }
4234
4235 /**
4236 * igb_get_stats64 - Get System Network Statistics
4237 * @netdev: network interface device structure
4238 * @stats: rtnl_link_stats64 pointer
4239 *
4240 **/
4241 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4242 struct rtnl_link_stats64 *stats)
4243 {
4244 struct igb_adapter *adapter = netdev_priv(netdev);
4245
4246 spin_lock(&adapter->stats64_lock);
4247 igb_update_stats(adapter, &adapter->stats64);
4248 memcpy(stats, &adapter->stats64, sizeof(*stats));
4249 spin_unlock(&adapter->stats64_lock);
4250
4251 return stats;
4252 }
4253
4254 /**
4255 * igb_change_mtu - Change the Maximum Transfer Unit
4256 * @netdev: network interface device structure
4257 * @new_mtu: new value for maximum frame size
4258 *
4259 * Returns 0 on success, negative on failure
4260 **/
4261 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4262 {
4263 struct igb_adapter *adapter = netdev_priv(netdev);
4264 struct pci_dev *pdev = adapter->pdev;
4265 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4266 u32 rx_buffer_len, i;
4267
4268 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4269 dev_err(&pdev->dev, "Invalid MTU setting\n");
4270 return -EINVAL;
4271 }
4272
4273 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4274 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4275 return -EINVAL;
4276 }
4277
4278 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4279 msleep(1);
4280
4281 /* igb_down has a dependency on max_frame_size */
4282 adapter->max_frame_size = max_frame;
4283
4284 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4285 * means we reserve 2 more, this pushes us to allocate from the next
4286 * larger slab size.
4287 * i.e. RXBUFFER_2048 --> size-4096 slab
4288 */
4289
4290 if (adapter->hw.mac.type == e1000_82580)
4291 max_frame += IGB_TS_HDR_LEN;
4292
4293 if (max_frame <= IGB_RXBUFFER_1024)
4294 rx_buffer_len = IGB_RXBUFFER_1024;
4295 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4296 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4297 else
4298 rx_buffer_len = IGB_RXBUFFER_128;
4299
4300 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4301 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4302 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4303
4304 if ((adapter->hw.mac.type == e1000_82580) &&
4305 (rx_buffer_len == IGB_RXBUFFER_128))
4306 rx_buffer_len += IGB_RXBUFFER_64;
4307
4308 if (netif_running(netdev))
4309 igb_down(adapter);
4310
4311 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4312 netdev->mtu, new_mtu);
4313 netdev->mtu = new_mtu;
4314
4315 for (i = 0; i < adapter->num_rx_queues; i++)
4316 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4317
4318 if (netif_running(netdev))
4319 igb_up(adapter);
4320 else
4321 igb_reset(adapter);
4322
4323 clear_bit(__IGB_RESETTING, &adapter->state);
4324
4325 return 0;
4326 }
4327
4328 /**
4329 * igb_update_stats - Update the board statistics counters
4330 * @adapter: board private structure
4331 **/
4332
4333 void igb_update_stats(struct igb_adapter *adapter,
4334 struct rtnl_link_stats64 *net_stats)
4335 {
4336 struct e1000_hw *hw = &adapter->hw;
4337 struct pci_dev *pdev = adapter->pdev;
4338 u32 reg, mpc;
4339 u16 phy_tmp;
4340 int i;
4341 u64 bytes, packets;
4342 unsigned int start;
4343 u64 _bytes, _packets;
4344
4345 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4346
4347 /*
4348 * Prevent stats update while adapter is being reset, or if the pci
4349 * connection is down.
4350 */
4351 if (adapter->link_speed == 0)
4352 return;
4353 if (pci_channel_offline(pdev))
4354 return;
4355
4356 bytes = 0;
4357 packets = 0;
4358 for (i = 0; i < adapter->num_rx_queues; i++) {
4359 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4360 struct igb_ring *ring = adapter->rx_ring[i];
4361
4362 ring->rx_stats.drops += rqdpc_tmp;
4363 net_stats->rx_fifo_errors += rqdpc_tmp;
4364
4365 do {
4366 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4367 _bytes = ring->rx_stats.bytes;
4368 _packets = ring->rx_stats.packets;
4369 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4370 bytes += _bytes;
4371 packets += _packets;
4372 }
4373
4374 net_stats->rx_bytes = bytes;
4375 net_stats->rx_packets = packets;
4376
4377 bytes = 0;
4378 packets = 0;
4379 for (i = 0; i < adapter->num_tx_queues; i++) {
4380 struct igb_ring *ring = adapter->tx_ring[i];
4381 do {
4382 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4383 _bytes = ring->tx_stats.bytes;
4384 _packets = ring->tx_stats.packets;
4385 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4386 bytes += _bytes;
4387 packets += _packets;
4388 }
4389 net_stats->tx_bytes = bytes;
4390 net_stats->tx_packets = packets;
4391
4392 /* read stats registers */
4393 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4394 adapter->stats.gprc += rd32(E1000_GPRC);
4395 adapter->stats.gorc += rd32(E1000_GORCL);
4396 rd32(E1000_GORCH); /* clear GORCL */
4397 adapter->stats.bprc += rd32(E1000_BPRC);
4398 adapter->stats.mprc += rd32(E1000_MPRC);
4399 adapter->stats.roc += rd32(E1000_ROC);
4400
4401 adapter->stats.prc64 += rd32(E1000_PRC64);
4402 adapter->stats.prc127 += rd32(E1000_PRC127);
4403 adapter->stats.prc255 += rd32(E1000_PRC255);
4404 adapter->stats.prc511 += rd32(E1000_PRC511);
4405 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4406 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4407 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4408 adapter->stats.sec += rd32(E1000_SEC);
4409
4410 mpc = rd32(E1000_MPC);
4411 adapter->stats.mpc += mpc;
4412 net_stats->rx_fifo_errors += mpc;
4413 adapter->stats.scc += rd32(E1000_SCC);
4414 adapter->stats.ecol += rd32(E1000_ECOL);
4415 adapter->stats.mcc += rd32(E1000_MCC);
4416 adapter->stats.latecol += rd32(E1000_LATECOL);
4417 adapter->stats.dc += rd32(E1000_DC);
4418 adapter->stats.rlec += rd32(E1000_RLEC);
4419 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4420 adapter->stats.xontxc += rd32(E1000_XONTXC);
4421 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4422 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4423 adapter->stats.fcruc += rd32(E1000_FCRUC);
4424 adapter->stats.gptc += rd32(E1000_GPTC);
4425 adapter->stats.gotc += rd32(E1000_GOTCL);
4426 rd32(E1000_GOTCH); /* clear GOTCL */
4427 adapter->stats.rnbc += rd32(E1000_RNBC);
4428 adapter->stats.ruc += rd32(E1000_RUC);
4429 adapter->stats.rfc += rd32(E1000_RFC);
4430 adapter->stats.rjc += rd32(E1000_RJC);
4431 adapter->stats.tor += rd32(E1000_TORH);
4432 adapter->stats.tot += rd32(E1000_TOTH);
4433 adapter->stats.tpr += rd32(E1000_TPR);
4434
4435 adapter->stats.ptc64 += rd32(E1000_PTC64);
4436 adapter->stats.ptc127 += rd32(E1000_PTC127);
4437 adapter->stats.ptc255 += rd32(E1000_PTC255);
4438 adapter->stats.ptc511 += rd32(E1000_PTC511);
4439 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4440 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4441
4442 adapter->stats.mptc += rd32(E1000_MPTC);
4443 adapter->stats.bptc += rd32(E1000_BPTC);
4444
4445 adapter->stats.tpt += rd32(E1000_TPT);
4446 adapter->stats.colc += rd32(E1000_COLC);
4447
4448 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4449 /* read internal phy specific stats */
4450 reg = rd32(E1000_CTRL_EXT);
4451 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4452 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4453 adapter->stats.tncrs += rd32(E1000_TNCRS);
4454 }
4455
4456 adapter->stats.tsctc += rd32(E1000_TSCTC);
4457 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4458
4459 adapter->stats.iac += rd32(E1000_IAC);
4460 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4461 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4462 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4463 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4464 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4465 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4466 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4467 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4468
4469 /* Fill out the OS statistics structure */
4470 net_stats->multicast = adapter->stats.mprc;
4471 net_stats->collisions = adapter->stats.colc;
4472
4473 /* Rx Errors */
4474
4475 /* RLEC on some newer hardware can be incorrect so build
4476 * our own version based on RUC and ROC */
4477 net_stats->rx_errors = adapter->stats.rxerrc +
4478 adapter->stats.crcerrs + adapter->stats.algnerrc +
4479 adapter->stats.ruc + adapter->stats.roc +
4480 adapter->stats.cexterr;
4481 net_stats->rx_length_errors = adapter->stats.ruc +
4482 adapter->stats.roc;
4483 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4484 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4485 net_stats->rx_missed_errors = adapter->stats.mpc;
4486
4487 /* Tx Errors */
4488 net_stats->tx_errors = adapter->stats.ecol +
4489 adapter->stats.latecol;
4490 net_stats->tx_aborted_errors = adapter->stats.ecol;
4491 net_stats->tx_window_errors = adapter->stats.latecol;
4492 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4493
4494 /* Tx Dropped needs to be maintained elsewhere */
4495
4496 /* Phy Stats */
4497 if (hw->phy.media_type == e1000_media_type_copper) {
4498 if ((adapter->link_speed == SPEED_1000) &&
4499 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4500 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4501 adapter->phy_stats.idle_errors += phy_tmp;
4502 }
4503 }
4504
4505 /* Management Stats */
4506 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4507 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4508 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4509 }
4510
4511 static irqreturn_t igb_msix_other(int irq, void *data)
4512 {
4513 struct igb_adapter *adapter = data;
4514 struct e1000_hw *hw = &adapter->hw;
4515 u32 icr = rd32(E1000_ICR);
4516 /* reading ICR causes bit 31 of EICR to be cleared */
4517
4518 if (icr & E1000_ICR_DRSTA)
4519 schedule_work(&adapter->reset_task);
4520
4521 if (icr & E1000_ICR_DOUTSYNC) {
4522 /* HW is reporting DMA is out of sync */
4523 adapter->stats.doosync++;
4524 }
4525
4526 /* Check for a mailbox event */
4527 if (icr & E1000_ICR_VMMB)
4528 igb_msg_task(adapter);
4529
4530 if (icr & E1000_ICR_LSC) {
4531 hw->mac.get_link_status = 1;
4532 /* guard against interrupt when we're going down */
4533 if (!test_bit(__IGB_DOWN, &adapter->state))
4534 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4535 }
4536
4537 if (adapter->vfs_allocated_count)
4538 wr32(E1000_IMS, E1000_IMS_LSC |
4539 E1000_IMS_VMMB |
4540 E1000_IMS_DOUTSYNC);
4541 else
4542 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4543 wr32(E1000_EIMS, adapter->eims_other);
4544
4545 return IRQ_HANDLED;
4546 }
4547
4548 static void igb_write_itr(struct igb_q_vector *q_vector)
4549 {
4550 struct igb_adapter *adapter = q_vector->adapter;
4551 u32 itr_val = q_vector->itr_val & 0x7FFC;
4552
4553 if (!q_vector->set_itr)
4554 return;
4555
4556 if (!itr_val)
4557 itr_val = 0x4;
4558
4559 if (adapter->hw.mac.type == e1000_82575)
4560 itr_val |= itr_val << 16;
4561 else
4562 itr_val |= 0x8000000;
4563
4564 writel(itr_val, q_vector->itr_register);
4565 q_vector->set_itr = 0;
4566 }
4567
4568 static irqreturn_t igb_msix_ring(int irq, void *data)
4569 {
4570 struct igb_q_vector *q_vector = data;
4571
4572 /* Write the ITR value calculated from the previous interrupt. */
4573 igb_write_itr(q_vector);
4574
4575 napi_schedule(&q_vector->napi);
4576
4577 return IRQ_HANDLED;
4578 }
4579
4580 #ifdef CONFIG_IGB_DCA
4581 static void igb_update_dca(struct igb_q_vector *q_vector)
4582 {
4583 struct igb_adapter *adapter = q_vector->adapter;
4584 struct e1000_hw *hw = &adapter->hw;
4585 int cpu = get_cpu();
4586
4587 if (q_vector->cpu == cpu)
4588 goto out_no_update;
4589
4590 if (q_vector->tx_ring) {
4591 int q = q_vector->tx_ring->reg_idx;
4592 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4593 if (hw->mac.type == e1000_82575) {
4594 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4595 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4596 } else {
4597 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4598 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4599 E1000_DCA_TXCTRL_CPUID_SHIFT;
4600 }
4601 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4602 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4603 }
4604 if (q_vector->rx_ring) {
4605 int q = q_vector->rx_ring->reg_idx;
4606 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4607 if (hw->mac.type == e1000_82575) {
4608 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4609 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4610 } else {
4611 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4612 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4613 E1000_DCA_RXCTRL_CPUID_SHIFT;
4614 }
4615 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4616 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4617 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4618 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4619 }
4620 q_vector->cpu = cpu;
4621 out_no_update:
4622 put_cpu();
4623 }
4624
4625 static void igb_setup_dca(struct igb_adapter *adapter)
4626 {
4627 struct e1000_hw *hw = &adapter->hw;
4628 int i;
4629
4630 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4631 return;
4632
4633 /* Always use CB2 mode, difference is masked in the CB driver. */
4634 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4635
4636 for (i = 0; i < adapter->num_q_vectors; i++) {
4637 adapter->q_vector[i]->cpu = -1;
4638 igb_update_dca(adapter->q_vector[i]);
4639 }
4640 }
4641
4642 static int __igb_notify_dca(struct device *dev, void *data)
4643 {
4644 struct net_device *netdev = dev_get_drvdata(dev);
4645 struct igb_adapter *adapter = netdev_priv(netdev);
4646 struct pci_dev *pdev = adapter->pdev;
4647 struct e1000_hw *hw = &adapter->hw;
4648 unsigned long event = *(unsigned long *)data;
4649
4650 switch (event) {
4651 case DCA_PROVIDER_ADD:
4652 /* if already enabled, don't do it again */
4653 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4654 break;
4655 if (dca_add_requester(dev) == 0) {
4656 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4657 dev_info(&pdev->dev, "DCA enabled\n");
4658 igb_setup_dca(adapter);
4659 break;
4660 }
4661 /* Fall Through since DCA is disabled. */
4662 case DCA_PROVIDER_REMOVE:
4663 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4664 /* without this a class_device is left
4665 * hanging around in the sysfs model */
4666 dca_remove_requester(dev);
4667 dev_info(&pdev->dev, "DCA disabled\n");
4668 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4669 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4670 }
4671 break;
4672 }
4673
4674 return 0;
4675 }
4676
4677 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4678 void *p)
4679 {
4680 int ret_val;
4681
4682 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4683 __igb_notify_dca);
4684
4685 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4686 }
4687 #endif /* CONFIG_IGB_DCA */
4688
4689 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4690 {
4691 struct e1000_hw *hw = &adapter->hw;
4692 u32 ping;
4693 int i;
4694
4695 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4696 ping = E1000_PF_CONTROL_MSG;
4697 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4698 ping |= E1000_VT_MSGTYPE_CTS;
4699 igb_write_mbx(hw, &ping, 1, i);
4700 }
4701 }
4702
4703 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4704 {
4705 struct e1000_hw *hw = &adapter->hw;
4706 u32 vmolr = rd32(E1000_VMOLR(vf));
4707 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4708
4709 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
4710 IGB_VF_FLAG_MULTI_PROMISC);
4711 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4712
4713 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4714 vmolr |= E1000_VMOLR_MPME;
4715 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
4716 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4717 } else {
4718 /*
4719 * if we have hashes and we are clearing a multicast promisc
4720 * flag we need to write the hashes to the MTA as this step
4721 * was previously skipped
4722 */
4723 if (vf_data->num_vf_mc_hashes > 30) {
4724 vmolr |= E1000_VMOLR_MPME;
4725 } else if (vf_data->num_vf_mc_hashes) {
4726 int j;
4727 vmolr |= E1000_VMOLR_ROMPE;
4728 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4729 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4730 }
4731 }
4732
4733 wr32(E1000_VMOLR(vf), vmolr);
4734
4735 /* there are flags left unprocessed, likely not supported */
4736 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4737 return -EINVAL;
4738
4739 return 0;
4740
4741 }
4742
4743 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4744 u32 *msgbuf, u32 vf)
4745 {
4746 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4747 u16 *hash_list = (u16 *)&msgbuf[1];
4748 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4749 int i;
4750
4751 /* salt away the number of multicast addresses assigned
4752 * to this VF for later use to restore when the PF multi cast
4753 * list changes
4754 */
4755 vf_data->num_vf_mc_hashes = n;
4756
4757 /* only up to 30 hash values supported */
4758 if (n > 30)
4759 n = 30;
4760
4761 /* store the hashes for later use */
4762 for (i = 0; i < n; i++)
4763 vf_data->vf_mc_hashes[i] = hash_list[i];
4764
4765 /* Flush and reset the mta with the new values */
4766 igb_set_rx_mode(adapter->netdev);
4767
4768 return 0;
4769 }
4770
4771 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4772 {
4773 struct e1000_hw *hw = &adapter->hw;
4774 struct vf_data_storage *vf_data;
4775 int i, j;
4776
4777 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4778 u32 vmolr = rd32(E1000_VMOLR(i));
4779 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4780
4781 vf_data = &adapter->vf_data[i];
4782
4783 if ((vf_data->num_vf_mc_hashes > 30) ||
4784 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4785 vmolr |= E1000_VMOLR_MPME;
4786 } else if (vf_data->num_vf_mc_hashes) {
4787 vmolr |= E1000_VMOLR_ROMPE;
4788 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4789 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4790 }
4791 wr32(E1000_VMOLR(i), vmolr);
4792 }
4793 }
4794
4795 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4796 {
4797 struct e1000_hw *hw = &adapter->hw;
4798 u32 pool_mask, reg, vid;
4799 int i;
4800
4801 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4802
4803 /* Find the vlan filter for this id */
4804 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4805 reg = rd32(E1000_VLVF(i));
4806
4807 /* remove the vf from the pool */
4808 reg &= ~pool_mask;
4809
4810 /* if pool is empty then remove entry from vfta */
4811 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4812 (reg & E1000_VLVF_VLANID_ENABLE)) {
4813 reg = 0;
4814 vid = reg & E1000_VLVF_VLANID_MASK;
4815 igb_vfta_set(hw, vid, false);
4816 }
4817
4818 wr32(E1000_VLVF(i), reg);
4819 }
4820
4821 adapter->vf_data[vf].vlans_enabled = 0;
4822 }
4823
4824 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4825 {
4826 struct e1000_hw *hw = &adapter->hw;
4827 u32 reg, i;
4828
4829 /* The vlvf table only exists on 82576 hardware and newer */
4830 if (hw->mac.type < e1000_82576)
4831 return -1;
4832
4833 /* we only need to do this if VMDq is enabled */
4834 if (!adapter->vfs_allocated_count)
4835 return -1;
4836
4837 /* Find the vlan filter for this id */
4838 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4839 reg = rd32(E1000_VLVF(i));
4840 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4841 vid == (reg & E1000_VLVF_VLANID_MASK))
4842 break;
4843 }
4844
4845 if (add) {
4846 if (i == E1000_VLVF_ARRAY_SIZE) {
4847 /* Did not find a matching VLAN ID entry that was
4848 * enabled. Search for a free filter entry, i.e.
4849 * one without the enable bit set
4850 */
4851 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4852 reg = rd32(E1000_VLVF(i));
4853 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4854 break;
4855 }
4856 }
4857 if (i < E1000_VLVF_ARRAY_SIZE) {
4858 /* Found an enabled/available entry */
4859 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4860
4861 /* if !enabled we need to set this up in vfta */
4862 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4863 /* add VID to filter table */
4864 igb_vfta_set(hw, vid, true);
4865 reg |= E1000_VLVF_VLANID_ENABLE;
4866 }
4867 reg &= ~E1000_VLVF_VLANID_MASK;
4868 reg |= vid;
4869 wr32(E1000_VLVF(i), reg);
4870
4871 /* do not modify RLPML for PF devices */
4872 if (vf >= adapter->vfs_allocated_count)
4873 return 0;
4874
4875 if (!adapter->vf_data[vf].vlans_enabled) {
4876 u32 size;
4877 reg = rd32(E1000_VMOLR(vf));
4878 size = reg & E1000_VMOLR_RLPML_MASK;
4879 size += 4;
4880 reg &= ~E1000_VMOLR_RLPML_MASK;
4881 reg |= size;
4882 wr32(E1000_VMOLR(vf), reg);
4883 }
4884
4885 adapter->vf_data[vf].vlans_enabled++;
4886 return 0;
4887 }
4888 } else {
4889 if (i < E1000_VLVF_ARRAY_SIZE) {
4890 /* remove vf from the pool */
4891 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4892 /* if pool is empty then remove entry from vfta */
4893 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4894 reg = 0;
4895 igb_vfta_set(hw, vid, false);
4896 }
4897 wr32(E1000_VLVF(i), reg);
4898
4899 /* do not modify RLPML for PF devices */
4900 if (vf >= adapter->vfs_allocated_count)
4901 return 0;
4902
4903 adapter->vf_data[vf].vlans_enabled--;
4904 if (!adapter->vf_data[vf].vlans_enabled) {
4905 u32 size;
4906 reg = rd32(E1000_VMOLR(vf));
4907 size = reg & E1000_VMOLR_RLPML_MASK;
4908 size -= 4;
4909 reg &= ~E1000_VMOLR_RLPML_MASK;
4910 reg |= size;
4911 wr32(E1000_VMOLR(vf), reg);
4912 }
4913 }
4914 }
4915 return 0;
4916 }
4917
4918 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4919 {
4920 struct e1000_hw *hw = &adapter->hw;
4921
4922 if (vid)
4923 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4924 else
4925 wr32(E1000_VMVIR(vf), 0);
4926 }
4927
4928 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4929 int vf, u16 vlan, u8 qos)
4930 {
4931 int err = 0;
4932 struct igb_adapter *adapter = netdev_priv(netdev);
4933
4934 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4935 return -EINVAL;
4936 if (vlan || qos) {
4937 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4938 if (err)
4939 goto out;
4940 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4941 igb_set_vmolr(adapter, vf, !vlan);
4942 adapter->vf_data[vf].pf_vlan = vlan;
4943 adapter->vf_data[vf].pf_qos = qos;
4944 dev_info(&adapter->pdev->dev,
4945 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4946 if (test_bit(__IGB_DOWN, &adapter->state)) {
4947 dev_warn(&adapter->pdev->dev,
4948 "The VF VLAN has been set,"
4949 " but the PF device is not up.\n");
4950 dev_warn(&adapter->pdev->dev,
4951 "Bring the PF device up before"
4952 " attempting to use the VF device.\n");
4953 }
4954 } else {
4955 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4956 false, vf);
4957 igb_set_vmvir(adapter, vlan, vf);
4958 igb_set_vmolr(adapter, vf, true);
4959 adapter->vf_data[vf].pf_vlan = 0;
4960 adapter->vf_data[vf].pf_qos = 0;
4961 }
4962 out:
4963 return err;
4964 }
4965
4966 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4967 {
4968 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4969 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4970
4971 return igb_vlvf_set(adapter, vid, add, vf);
4972 }
4973
4974 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4975 {
4976 /* clear flags - except flag that indicates PF has set the MAC */
4977 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
4978 adapter->vf_data[vf].last_nack = jiffies;
4979
4980 /* reset offloads to defaults */
4981 igb_set_vmolr(adapter, vf, true);
4982
4983 /* reset vlans for device */
4984 igb_clear_vf_vfta(adapter, vf);
4985 if (adapter->vf_data[vf].pf_vlan)
4986 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4987 adapter->vf_data[vf].pf_vlan,
4988 adapter->vf_data[vf].pf_qos);
4989 else
4990 igb_clear_vf_vfta(adapter, vf);
4991
4992 /* reset multicast table array for vf */
4993 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4994
4995 /* Flush and reset the mta with the new values */
4996 igb_set_rx_mode(adapter->netdev);
4997 }
4998
4999 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5000 {
5001 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5002
5003 /* generate a new mac address as we were hotplug removed/added */
5004 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5005 random_ether_addr(vf_mac);
5006
5007 /* process remaining reset events */
5008 igb_vf_reset(adapter, vf);
5009 }
5010
5011 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5012 {
5013 struct e1000_hw *hw = &adapter->hw;
5014 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5015 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5016 u32 reg, msgbuf[3];
5017 u8 *addr = (u8 *)(&msgbuf[1]);
5018
5019 /* process all the same items cleared in a function level reset */
5020 igb_vf_reset(adapter, vf);
5021
5022 /* set vf mac address */
5023 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5024
5025 /* enable transmit and receive for vf */
5026 reg = rd32(E1000_VFTE);
5027 wr32(E1000_VFTE, reg | (1 << vf));
5028 reg = rd32(E1000_VFRE);
5029 wr32(E1000_VFRE, reg | (1 << vf));
5030
5031 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5032
5033 /* reply to reset with ack and vf mac address */
5034 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5035 memcpy(addr, vf_mac, 6);
5036 igb_write_mbx(hw, msgbuf, 3, vf);
5037 }
5038
5039 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5040 {
5041 /*
5042 * The VF MAC Address is stored in a packed array of bytes
5043 * starting at the second 32 bit word of the msg array
5044 */
5045 unsigned char *addr = (char *)&msg[1];
5046 int err = -1;
5047
5048 if (is_valid_ether_addr(addr))
5049 err = igb_set_vf_mac(adapter, vf, addr);
5050
5051 return err;
5052 }
5053
5054 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5055 {
5056 struct e1000_hw *hw = &adapter->hw;
5057 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5058 u32 msg = E1000_VT_MSGTYPE_NACK;
5059
5060 /* if device isn't clear to send it shouldn't be reading either */
5061 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5062 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5063 igb_write_mbx(hw, &msg, 1, vf);
5064 vf_data->last_nack = jiffies;
5065 }
5066 }
5067
5068 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5069 {
5070 struct pci_dev *pdev = adapter->pdev;
5071 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5072 struct e1000_hw *hw = &adapter->hw;
5073 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5074 s32 retval;
5075
5076 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5077
5078 if (retval) {
5079 /* if receive failed revoke VF CTS stats and restart init */
5080 dev_err(&pdev->dev, "Error receiving message from VF\n");
5081 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5082 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5083 return;
5084 goto out;
5085 }
5086
5087 /* this is a message we already processed, do nothing */
5088 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5089 return;
5090
5091 /*
5092 * until the vf completes a reset it should not be
5093 * allowed to start any configuration.
5094 */
5095
5096 if (msgbuf[0] == E1000_VF_RESET) {
5097 igb_vf_reset_msg(adapter, vf);
5098 return;
5099 }
5100
5101 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5102 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5103 return;
5104 retval = -1;
5105 goto out;
5106 }
5107
5108 switch ((msgbuf[0] & 0xFFFF)) {
5109 case E1000_VF_SET_MAC_ADDR:
5110 retval = -EINVAL;
5111 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5112 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5113 else
5114 dev_warn(&pdev->dev,
5115 "VF %d attempted to override administratively "
5116 "set MAC address\nReload the VF driver to "
5117 "resume operations\n", vf);
5118 break;
5119 case E1000_VF_SET_PROMISC:
5120 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5121 break;
5122 case E1000_VF_SET_MULTICAST:
5123 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5124 break;
5125 case E1000_VF_SET_LPE:
5126 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5127 break;
5128 case E1000_VF_SET_VLAN:
5129 retval = -1;
5130 if (vf_data->pf_vlan)
5131 dev_warn(&pdev->dev,
5132 "VF %d attempted to override administratively "
5133 "set VLAN tag\nReload the VF driver to "
5134 "resume operations\n", vf);
5135 else
5136 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5137 break;
5138 default:
5139 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5140 retval = -1;
5141 break;
5142 }
5143
5144 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5145 out:
5146 /* notify the VF of the results of what it sent us */
5147 if (retval)
5148 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5149 else
5150 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5151
5152 igb_write_mbx(hw, msgbuf, 1, vf);
5153 }
5154
5155 static void igb_msg_task(struct igb_adapter *adapter)
5156 {
5157 struct e1000_hw *hw = &adapter->hw;
5158 u32 vf;
5159
5160 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5161 /* process any reset requests */
5162 if (!igb_check_for_rst(hw, vf))
5163 igb_vf_reset_event(adapter, vf);
5164
5165 /* process any messages pending */
5166 if (!igb_check_for_msg(hw, vf))
5167 igb_rcv_msg_from_vf(adapter, vf);
5168
5169 /* process any acks */
5170 if (!igb_check_for_ack(hw, vf))
5171 igb_rcv_ack_from_vf(adapter, vf);
5172 }
5173 }
5174
5175 /**
5176 * igb_set_uta - Set unicast filter table address
5177 * @adapter: board private structure
5178 *
5179 * The unicast table address is a register array of 32-bit registers.
5180 * The table is meant to be used in a way similar to how the MTA is used
5181 * however due to certain limitations in the hardware it is necessary to
5182 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5183 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5184 **/
5185 static void igb_set_uta(struct igb_adapter *adapter)
5186 {
5187 struct e1000_hw *hw = &adapter->hw;
5188 int i;
5189
5190 /* The UTA table only exists on 82576 hardware and newer */
5191 if (hw->mac.type < e1000_82576)
5192 return;
5193
5194 /* we only need to do this if VMDq is enabled */
5195 if (!adapter->vfs_allocated_count)
5196 return;
5197
5198 for (i = 0; i < hw->mac.uta_reg_count; i++)
5199 array_wr32(E1000_UTA, i, ~0);
5200 }
5201
5202 /**
5203 * igb_intr_msi - Interrupt Handler
5204 * @irq: interrupt number
5205 * @data: pointer to a network interface device structure
5206 **/
5207 static irqreturn_t igb_intr_msi(int irq, void *data)
5208 {
5209 struct igb_adapter *adapter = data;
5210 struct igb_q_vector *q_vector = adapter->q_vector[0];
5211 struct e1000_hw *hw = &adapter->hw;
5212 /* read ICR disables interrupts using IAM */
5213 u32 icr = rd32(E1000_ICR);
5214
5215 igb_write_itr(q_vector);
5216
5217 if (icr & E1000_ICR_DRSTA)
5218 schedule_work(&adapter->reset_task);
5219
5220 if (icr & E1000_ICR_DOUTSYNC) {
5221 /* HW is reporting DMA is out of sync */
5222 adapter->stats.doosync++;
5223 }
5224
5225 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5226 hw->mac.get_link_status = 1;
5227 if (!test_bit(__IGB_DOWN, &adapter->state))
5228 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5229 }
5230
5231 napi_schedule(&q_vector->napi);
5232
5233 return IRQ_HANDLED;
5234 }
5235
5236 /**
5237 * igb_intr - Legacy Interrupt Handler
5238 * @irq: interrupt number
5239 * @data: pointer to a network interface device structure
5240 **/
5241 static irqreturn_t igb_intr(int irq, void *data)
5242 {
5243 struct igb_adapter *adapter = data;
5244 struct igb_q_vector *q_vector = adapter->q_vector[0];
5245 struct e1000_hw *hw = &adapter->hw;
5246 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5247 * need for the IMC write */
5248 u32 icr = rd32(E1000_ICR);
5249 if (!icr)
5250 return IRQ_NONE; /* Not our interrupt */
5251
5252 igb_write_itr(q_vector);
5253
5254 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5255 * not set, then the adapter didn't send an interrupt */
5256 if (!(icr & E1000_ICR_INT_ASSERTED))
5257 return IRQ_NONE;
5258
5259 if (icr & E1000_ICR_DRSTA)
5260 schedule_work(&adapter->reset_task);
5261
5262 if (icr & E1000_ICR_DOUTSYNC) {
5263 /* HW is reporting DMA is out of sync */
5264 adapter->stats.doosync++;
5265 }
5266
5267 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5268 hw->mac.get_link_status = 1;
5269 /* guard against interrupt when we're going down */
5270 if (!test_bit(__IGB_DOWN, &adapter->state))
5271 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5272 }
5273
5274 napi_schedule(&q_vector->napi);
5275
5276 return IRQ_HANDLED;
5277 }
5278
5279 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5280 {
5281 struct igb_adapter *adapter = q_vector->adapter;
5282 struct e1000_hw *hw = &adapter->hw;
5283
5284 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5285 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5286 if (!adapter->msix_entries)
5287 igb_set_itr(adapter);
5288 else
5289 igb_update_ring_itr(q_vector);
5290 }
5291
5292 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5293 if (adapter->msix_entries)
5294 wr32(E1000_EIMS, q_vector->eims_value);
5295 else
5296 igb_irq_enable(adapter);
5297 }
5298 }
5299
5300 /**
5301 * igb_poll - NAPI Rx polling callback
5302 * @napi: napi polling structure
5303 * @budget: count of how many packets we should handle
5304 **/
5305 static int igb_poll(struct napi_struct *napi, int budget)
5306 {
5307 struct igb_q_vector *q_vector = container_of(napi,
5308 struct igb_q_vector,
5309 napi);
5310 int tx_clean_complete = 1, work_done = 0;
5311
5312 #ifdef CONFIG_IGB_DCA
5313 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5314 igb_update_dca(q_vector);
5315 #endif
5316 if (q_vector->tx_ring)
5317 tx_clean_complete = igb_clean_tx_irq(q_vector);
5318
5319 if (q_vector->rx_ring)
5320 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5321
5322 if (!tx_clean_complete)
5323 work_done = budget;
5324
5325 /* If not enough Rx work done, exit the polling mode */
5326 if (work_done < budget) {
5327 napi_complete(napi);
5328 igb_ring_irq_enable(q_vector);
5329 }
5330
5331 return work_done;
5332 }
5333
5334 /**
5335 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5336 * @adapter: board private structure
5337 * @shhwtstamps: timestamp structure to update
5338 * @regval: unsigned 64bit system time value.
5339 *
5340 * We need to convert the system time value stored in the RX/TXSTMP registers
5341 * into a hwtstamp which can be used by the upper level timestamping functions
5342 */
5343 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5344 struct skb_shared_hwtstamps *shhwtstamps,
5345 u64 regval)
5346 {
5347 u64 ns;
5348
5349 /*
5350 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5351 * 24 to match clock shift we setup earlier.
5352 */
5353 if (adapter->hw.mac.type == e1000_82580)
5354 regval <<= IGB_82580_TSYNC_SHIFT;
5355
5356 ns = timecounter_cyc2time(&adapter->clock, regval);
5357 timecompare_update(&adapter->compare, ns);
5358 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5359 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5360 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5361 }
5362
5363 /**
5364 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5365 * @q_vector: pointer to q_vector containing needed info
5366 * @buffer: pointer to igb_buffer structure
5367 *
5368 * If we were asked to do hardware stamping and such a time stamp is
5369 * available, then it must have been for this skb here because we only
5370 * allow only one such packet into the queue.
5371 */
5372 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5373 {
5374 struct igb_adapter *adapter = q_vector->adapter;
5375 struct e1000_hw *hw = &adapter->hw;
5376 struct skb_shared_hwtstamps shhwtstamps;
5377 u64 regval;
5378
5379 /* if skb does not support hw timestamp or TX stamp not valid exit */
5380 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
5381 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5382 return;
5383
5384 regval = rd32(E1000_TXSTMPL);
5385 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5386
5387 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5388 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5389 }
5390
5391 /**
5392 * igb_clean_tx_irq - Reclaim resources after transmit completes
5393 * @q_vector: pointer to q_vector containing needed info
5394 * returns true if ring is completely cleaned
5395 **/
5396 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5397 {
5398 struct igb_adapter *adapter = q_vector->adapter;
5399 struct igb_ring *tx_ring = q_vector->tx_ring;
5400 struct net_device *netdev = tx_ring->netdev;
5401 struct e1000_hw *hw = &adapter->hw;
5402 struct igb_buffer *buffer_info;
5403 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5404 unsigned int total_bytes = 0, total_packets = 0;
5405 unsigned int i, eop, count = 0;
5406 bool cleaned = false;
5407
5408 i = tx_ring->next_to_clean;
5409 eop = tx_ring->buffer_info[i].next_to_watch;
5410 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5411
5412 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5413 (count < tx_ring->count)) {
5414 rmb(); /* read buffer_info after eop_desc status */
5415 for (cleaned = false; !cleaned; count++) {
5416 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5417 buffer_info = &tx_ring->buffer_info[i];
5418 cleaned = (i == eop);
5419
5420 if (buffer_info->skb) {
5421 total_bytes += buffer_info->bytecount;
5422 /* gso_segs is currently only valid for tcp */
5423 total_packets += buffer_info->gso_segs;
5424 igb_tx_hwtstamp(q_vector, buffer_info);
5425 }
5426
5427 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5428 tx_desc->wb.status = 0;
5429
5430 i++;
5431 if (i == tx_ring->count)
5432 i = 0;
5433 }
5434 eop = tx_ring->buffer_info[i].next_to_watch;
5435 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5436 }
5437
5438 tx_ring->next_to_clean = i;
5439
5440 if (unlikely(count &&
5441 netif_carrier_ok(netdev) &&
5442 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5443 /* Make sure that anybody stopping the queue after this
5444 * sees the new next_to_clean.
5445 */
5446 smp_mb();
5447 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5448 !(test_bit(__IGB_DOWN, &adapter->state))) {
5449 netif_wake_subqueue(netdev, tx_ring->queue_index);
5450
5451 u64_stats_update_begin(&tx_ring->tx_syncp);
5452 tx_ring->tx_stats.restart_queue++;
5453 u64_stats_update_end(&tx_ring->tx_syncp);
5454 }
5455 }
5456
5457 if (tx_ring->detect_tx_hung) {
5458 /* Detect a transmit hang in hardware, this serializes the
5459 * check with the clearing of time_stamp and movement of i */
5460 tx_ring->detect_tx_hung = false;
5461 if (tx_ring->buffer_info[i].time_stamp &&
5462 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5463 (adapter->tx_timeout_factor * HZ)) &&
5464 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5465
5466 /* detected Tx unit hang */
5467 dev_err(tx_ring->dev,
5468 "Detected Tx Unit Hang\n"
5469 " Tx Queue <%d>\n"
5470 " TDH <%x>\n"
5471 " TDT <%x>\n"
5472 " next_to_use <%x>\n"
5473 " next_to_clean <%x>\n"
5474 "buffer_info[next_to_clean]\n"
5475 " time_stamp <%lx>\n"
5476 " next_to_watch <%x>\n"
5477 " jiffies <%lx>\n"
5478 " desc.status <%x>\n",
5479 tx_ring->queue_index,
5480 readl(tx_ring->head),
5481 readl(tx_ring->tail),
5482 tx_ring->next_to_use,
5483 tx_ring->next_to_clean,
5484 tx_ring->buffer_info[eop].time_stamp,
5485 eop,
5486 jiffies,
5487 eop_desc->wb.status);
5488 netif_stop_subqueue(netdev, tx_ring->queue_index);
5489 }
5490 }
5491 tx_ring->total_bytes += total_bytes;
5492 tx_ring->total_packets += total_packets;
5493 u64_stats_update_begin(&tx_ring->tx_syncp);
5494 tx_ring->tx_stats.bytes += total_bytes;
5495 tx_ring->tx_stats.packets += total_packets;
5496 u64_stats_update_end(&tx_ring->tx_syncp);
5497 return count < tx_ring->count;
5498 }
5499
5500 /**
5501 * igb_receive_skb - helper function to handle rx indications
5502 * @q_vector: structure containing interrupt and ring information
5503 * @skb: packet to send up
5504 * @vlan_tag: vlan tag for packet
5505 **/
5506 static void igb_receive_skb(struct igb_q_vector *q_vector,
5507 struct sk_buff *skb,
5508 u16 vlan_tag)
5509 {
5510 struct igb_adapter *adapter = q_vector->adapter;
5511
5512 if (vlan_tag && adapter->vlgrp)
5513 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5514 vlan_tag, skb);
5515 else
5516 napi_gro_receive(&q_vector->napi, skb);
5517 }
5518
5519 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5520 u32 status_err, struct sk_buff *skb)
5521 {
5522 skb_checksum_none_assert(skb);
5523
5524 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5525 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5526 (status_err & E1000_RXD_STAT_IXSM))
5527 return;
5528
5529 /* TCP/UDP checksum error bit is set */
5530 if (status_err &
5531 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5532 /*
5533 * work around errata with sctp packets where the TCPE aka
5534 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5535 * packets, (aka let the stack check the crc32c)
5536 */
5537 if ((skb->len == 60) &&
5538 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5539 u64_stats_update_begin(&ring->rx_syncp);
5540 ring->rx_stats.csum_err++;
5541 u64_stats_update_end(&ring->rx_syncp);
5542 }
5543 /* let the stack verify checksum errors */
5544 return;
5545 }
5546 /* It must be a TCP or UDP packet with a valid checksum */
5547 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5548 skb->ip_summed = CHECKSUM_UNNECESSARY;
5549
5550 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5551 }
5552
5553 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5554 struct sk_buff *skb)
5555 {
5556 struct igb_adapter *adapter = q_vector->adapter;
5557 struct e1000_hw *hw = &adapter->hw;
5558 u64 regval;
5559
5560 /*
5561 * If this bit is set, then the RX registers contain the time stamp. No
5562 * other packet will be time stamped until we read these registers, so
5563 * read the registers to make them available again. Because only one
5564 * packet can be time stamped at a time, we know that the register
5565 * values must belong to this one here and therefore we don't need to
5566 * compare any of the additional attributes stored for it.
5567 *
5568 * If nothing went wrong, then it should have a shared tx_flags that we
5569 * can turn into a skb_shared_hwtstamps.
5570 */
5571 if (staterr & E1000_RXDADV_STAT_TSIP) {
5572 u32 *stamp = (u32 *)skb->data;
5573 regval = le32_to_cpu(*(stamp + 2));
5574 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5575 skb_pull(skb, IGB_TS_HDR_LEN);
5576 } else {
5577 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5578 return;
5579
5580 regval = rd32(E1000_RXSTMPL);
5581 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5582 }
5583
5584 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5585 }
5586 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5587 union e1000_adv_rx_desc *rx_desc)
5588 {
5589 /* HW will not DMA in data larger than the given buffer, even if it
5590 * parses the (NFS, of course) header to be larger. In that case, it
5591 * fills the header buffer and spills the rest into the page.
5592 */
5593 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5594 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5595 if (hlen > rx_ring->rx_buffer_len)
5596 hlen = rx_ring->rx_buffer_len;
5597 return hlen;
5598 }
5599
5600 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5601 int *work_done, int budget)
5602 {
5603 struct igb_ring *rx_ring = q_vector->rx_ring;
5604 struct net_device *netdev = rx_ring->netdev;
5605 struct device *dev = rx_ring->dev;
5606 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5607 struct igb_buffer *buffer_info , *next_buffer;
5608 struct sk_buff *skb;
5609 bool cleaned = false;
5610 int cleaned_count = 0;
5611 int current_node = numa_node_id();
5612 unsigned int total_bytes = 0, total_packets = 0;
5613 unsigned int i;
5614 u32 staterr;
5615 u16 length;
5616 u16 vlan_tag;
5617
5618 i = rx_ring->next_to_clean;
5619 buffer_info = &rx_ring->buffer_info[i];
5620 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5621 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5622
5623 while (staterr & E1000_RXD_STAT_DD) {
5624 if (*work_done >= budget)
5625 break;
5626 (*work_done)++;
5627 rmb(); /* read descriptor and rx_buffer_info after status DD */
5628
5629 skb = buffer_info->skb;
5630 prefetch(skb->data - NET_IP_ALIGN);
5631 buffer_info->skb = NULL;
5632
5633 i++;
5634 if (i == rx_ring->count)
5635 i = 0;
5636
5637 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5638 prefetch(next_rxd);
5639 next_buffer = &rx_ring->buffer_info[i];
5640
5641 length = le16_to_cpu(rx_desc->wb.upper.length);
5642 cleaned = true;
5643 cleaned_count++;
5644
5645 if (buffer_info->dma) {
5646 dma_unmap_single(dev, buffer_info->dma,
5647 rx_ring->rx_buffer_len,
5648 DMA_FROM_DEVICE);
5649 buffer_info->dma = 0;
5650 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5651 skb_put(skb, length);
5652 goto send_up;
5653 }
5654 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5655 }
5656
5657 if (length) {
5658 dma_unmap_page(dev, buffer_info->page_dma,
5659 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5660 buffer_info->page_dma = 0;
5661
5662 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5663 buffer_info->page,
5664 buffer_info->page_offset,
5665 length);
5666
5667 if ((page_count(buffer_info->page) != 1) ||
5668 (page_to_nid(buffer_info->page) != current_node))
5669 buffer_info->page = NULL;
5670 else
5671 get_page(buffer_info->page);
5672
5673 skb->len += length;
5674 skb->data_len += length;
5675 skb->truesize += length;
5676 }
5677
5678 if (!(staterr & E1000_RXD_STAT_EOP)) {
5679 buffer_info->skb = next_buffer->skb;
5680 buffer_info->dma = next_buffer->dma;
5681 next_buffer->skb = skb;
5682 next_buffer->dma = 0;
5683 goto next_desc;
5684 }
5685 send_up:
5686 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5687 dev_kfree_skb_irq(skb);
5688 goto next_desc;
5689 }
5690
5691 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5692 igb_rx_hwtstamp(q_vector, staterr, skb);
5693 total_bytes += skb->len;
5694 total_packets++;
5695
5696 igb_rx_checksum_adv(rx_ring, staterr, skb);
5697
5698 skb->protocol = eth_type_trans(skb, netdev);
5699 skb_record_rx_queue(skb, rx_ring->queue_index);
5700
5701 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5702 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5703
5704 igb_receive_skb(q_vector, skb, vlan_tag);
5705
5706 next_desc:
5707 rx_desc->wb.upper.status_error = 0;
5708
5709 /* return some buffers to hardware, one at a time is too slow */
5710 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5711 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5712 cleaned_count = 0;
5713 }
5714
5715 /* use prefetched values */
5716 rx_desc = next_rxd;
5717 buffer_info = next_buffer;
5718 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5719 }
5720
5721 rx_ring->next_to_clean = i;
5722 cleaned_count = igb_desc_unused(rx_ring);
5723
5724 if (cleaned_count)
5725 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5726
5727 rx_ring->total_packets += total_packets;
5728 rx_ring->total_bytes += total_bytes;
5729 u64_stats_update_begin(&rx_ring->rx_syncp);
5730 rx_ring->rx_stats.packets += total_packets;
5731 rx_ring->rx_stats.bytes += total_bytes;
5732 u64_stats_update_end(&rx_ring->rx_syncp);
5733 return cleaned;
5734 }
5735
5736 /**
5737 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5738 * @adapter: address of board private structure
5739 **/
5740 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5741 {
5742 struct net_device *netdev = rx_ring->netdev;
5743 union e1000_adv_rx_desc *rx_desc;
5744 struct igb_buffer *buffer_info;
5745 struct sk_buff *skb;
5746 unsigned int i;
5747 int bufsz;
5748
5749 i = rx_ring->next_to_use;
5750 buffer_info = &rx_ring->buffer_info[i];
5751
5752 bufsz = rx_ring->rx_buffer_len;
5753
5754 while (cleaned_count--) {
5755 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5756
5757 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5758 if (!buffer_info->page) {
5759 buffer_info->page = netdev_alloc_page(netdev);
5760 if (unlikely(!buffer_info->page)) {
5761 u64_stats_update_begin(&rx_ring->rx_syncp);
5762 rx_ring->rx_stats.alloc_failed++;
5763 u64_stats_update_end(&rx_ring->rx_syncp);
5764 goto no_buffers;
5765 }
5766 buffer_info->page_offset = 0;
5767 } else {
5768 buffer_info->page_offset ^= PAGE_SIZE / 2;
5769 }
5770 buffer_info->page_dma =
5771 dma_map_page(rx_ring->dev, buffer_info->page,
5772 buffer_info->page_offset,
5773 PAGE_SIZE / 2,
5774 DMA_FROM_DEVICE);
5775 if (dma_mapping_error(rx_ring->dev,
5776 buffer_info->page_dma)) {
5777 buffer_info->page_dma = 0;
5778 u64_stats_update_begin(&rx_ring->rx_syncp);
5779 rx_ring->rx_stats.alloc_failed++;
5780 u64_stats_update_end(&rx_ring->rx_syncp);
5781 goto no_buffers;
5782 }
5783 }
5784
5785 skb = buffer_info->skb;
5786 if (!skb) {
5787 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5788 if (unlikely(!skb)) {
5789 u64_stats_update_begin(&rx_ring->rx_syncp);
5790 rx_ring->rx_stats.alloc_failed++;
5791 u64_stats_update_end(&rx_ring->rx_syncp);
5792 goto no_buffers;
5793 }
5794
5795 buffer_info->skb = skb;
5796 }
5797 if (!buffer_info->dma) {
5798 buffer_info->dma = dma_map_single(rx_ring->dev,
5799 skb->data,
5800 bufsz,
5801 DMA_FROM_DEVICE);
5802 if (dma_mapping_error(rx_ring->dev,
5803 buffer_info->dma)) {
5804 buffer_info->dma = 0;
5805 u64_stats_update_begin(&rx_ring->rx_syncp);
5806 rx_ring->rx_stats.alloc_failed++;
5807 u64_stats_update_end(&rx_ring->rx_syncp);
5808 goto no_buffers;
5809 }
5810 }
5811 /* Refresh the desc even if buffer_addrs didn't change because
5812 * each write-back erases this info. */
5813 if (bufsz < IGB_RXBUFFER_1024) {
5814 rx_desc->read.pkt_addr =
5815 cpu_to_le64(buffer_info->page_dma);
5816 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5817 } else {
5818 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5819 rx_desc->read.hdr_addr = 0;
5820 }
5821
5822 i++;
5823 if (i == rx_ring->count)
5824 i = 0;
5825 buffer_info = &rx_ring->buffer_info[i];
5826 }
5827
5828 no_buffers:
5829 if (rx_ring->next_to_use != i) {
5830 rx_ring->next_to_use = i;
5831 if (i == 0)
5832 i = (rx_ring->count - 1);
5833 else
5834 i--;
5835
5836 /* Force memory writes to complete before letting h/w
5837 * know there are new descriptors to fetch. (Only
5838 * applicable for weak-ordered memory model archs,
5839 * such as IA-64). */
5840 wmb();
5841 writel(i, rx_ring->tail);
5842 }
5843 }
5844
5845 /**
5846 * igb_mii_ioctl -
5847 * @netdev:
5848 * @ifreq:
5849 * @cmd:
5850 **/
5851 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5852 {
5853 struct igb_adapter *adapter = netdev_priv(netdev);
5854 struct mii_ioctl_data *data = if_mii(ifr);
5855
5856 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5857 return -EOPNOTSUPP;
5858
5859 switch (cmd) {
5860 case SIOCGMIIPHY:
5861 data->phy_id = adapter->hw.phy.addr;
5862 break;
5863 case SIOCGMIIREG:
5864 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5865 &data->val_out))
5866 return -EIO;
5867 break;
5868 case SIOCSMIIREG:
5869 default:
5870 return -EOPNOTSUPP;
5871 }
5872 return 0;
5873 }
5874
5875 /**
5876 * igb_hwtstamp_ioctl - control hardware time stamping
5877 * @netdev:
5878 * @ifreq:
5879 * @cmd:
5880 *
5881 * Outgoing time stamping can be enabled and disabled. Play nice and
5882 * disable it when requested, although it shouldn't case any overhead
5883 * when no packet needs it. At most one packet in the queue may be
5884 * marked for time stamping, otherwise it would be impossible to tell
5885 * for sure to which packet the hardware time stamp belongs.
5886 *
5887 * Incoming time stamping has to be configured via the hardware
5888 * filters. Not all combinations are supported, in particular event
5889 * type has to be specified. Matching the kind of event packet is
5890 * not supported, with the exception of "all V2 events regardless of
5891 * level 2 or 4".
5892 *
5893 **/
5894 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5895 struct ifreq *ifr, int cmd)
5896 {
5897 struct igb_adapter *adapter = netdev_priv(netdev);
5898 struct e1000_hw *hw = &adapter->hw;
5899 struct hwtstamp_config config;
5900 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5901 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5902 u32 tsync_rx_cfg = 0;
5903 bool is_l4 = false;
5904 bool is_l2 = false;
5905 u32 regval;
5906
5907 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5908 return -EFAULT;
5909
5910 /* reserved for future extensions */
5911 if (config.flags)
5912 return -EINVAL;
5913
5914 switch (config.tx_type) {
5915 case HWTSTAMP_TX_OFF:
5916 tsync_tx_ctl = 0;
5917 case HWTSTAMP_TX_ON:
5918 break;
5919 default:
5920 return -ERANGE;
5921 }
5922
5923 switch (config.rx_filter) {
5924 case HWTSTAMP_FILTER_NONE:
5925 tsync_rx_ctl = 0;
5926 break;
5927 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5928 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5929 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5930 case HWTSTAMP_FILTER_ALL:
5931 /*
5932 * register TSYNCRXCFG must be set, therefore it is not
5933 * possible to time stamp both Sync and Delay_Req messages
5934 * => fall back to time stamping all packets
5935 */
5936 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5937 config.rx_filter = HWTSTAMP_FILTER_ALL;
5938 break;
5939 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5940 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5941 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5942 is_l4 = true;
5943 break;
5944 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5945 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5946 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5947 is_l4 = true;
5948 break;
5949 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5950 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5951 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5952 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5953 is_l2 = true;
5954 is_l4 = true;
5955 config.rx_filter = HWTSTAMP_FILTER_SOME;
5956 break;
5957 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5958 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5959 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5960 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5961 is_l2 = true;
5962 is_l4 = true;
5963 config.rx_filter = HWTSTAMP_FILTER_SOME;
5964 break;
5965 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5966 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5967 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5968 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5969 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5970 is_l2 = true;
5971 break;
5972 default:
5973 return -ERANGE;
5974 }
5975
5976 if (hw->mac.type == e1000_82575) {
5977 if (tsync_rx_ctl | tsync_tx_ctl)
5978 return -EINVAL;
5979 return 0;
5980 }
5981
5982 /*
5983 * Per-packet timestamping only works if all packets are
5984 * timestamped, so enable timestamping in all packets as
5985 * long as one rx filter was configured.
5986 */
5987 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5988 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5989 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5990 }
5991
5992 /* enable/disable TX */
5993 regval = rd32(E1000_TSYNCTXCTL);
5994 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5995 regval |= tsync_tx_ctl;
5996 wr32(E1000_TSYNCTXCTL, regval);
5997
5998 /* enable/disable RX */
5999 regval = rd32(E1000_TSYNCRXCTL);
6000 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6001 regval |= tsync_rx_ctl;
6002 wr32(E1000_TSYNCRXCTL, regval);
6003
6004 /* define which PTP packets are time stamped */
6005 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6006
6007 /* define ethertype filter for timestamped packets */
6008 if (is_l2)
6009 wr32(E1000_ETQF(3),
6010 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6011 E1000_ETQF_1588 | /* enable timestamping */
6012 ETH_P_1588)); /* 1588 eth protocol type */
6013 else
6014 wr32(E1000_ETQF(3), 0);
6015
6016 #define PTP_PORT 319
6017 /* L4 Queue Filter[3]: filter by destination port and protocol */
6018 if (is_l4) {
6019 u32 ftqf = (IPPROTO_UDP /* UDP */
6020 | E1000_FTQF_VF_BP /* VF not compared */
6021 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6022 | E1000_FTQF_MASK); /* mask all inputs */
6023 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6024
6025 wr32(E1000_IMIR(3), htons(PTP_PORT));
6026 wr32(E1000_IMIREXT(3),
6027 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6028 if (hw->mac.type == e1000_82576) {
6029 /* enable source port check */
6030 wr32(E1000_SPQF(3), htons(PTP_PORT));
6031 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6032 }
6033 wr32(E1000_FTQF(3), ftqf);
6034 } else {
6035 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6036 }
6037 wrfl();
6038
6039 adapter->hwtstamp_config = config;
6040
6041 /* clear TX/RX time stamp registers, just to be sure */
6042 regval = rd32(E1000_TXSTMPH);
6043 regval = rd32(E1000_RXSTMPH);
6044
6045 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6046 -EFAULT : 0;
6047 }
6048
6049 /**
6050 * igb_ioctl -
6051 * @netdev:
6052 * @ifreq:
6053 * @cmd:
6054 **/
6055 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6056 {
6057 switch (cmd) {
6058 case SIOCGMIIPHY:
6059 case SIOCGMIIREG:
6060 case SIOCSMIIREG:
6061 return igb_mii_ioctl(netdev, ifr, cmd);
6062 case SIOCSHWTSTAMP:
6063 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6064 default:
6065 return -EOPNOTSUPP;
6066 }
6067 }
6068
6069 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6070 {
6071 struct igb_adapter *adapter = hw->back;
6072 u16 cap_offset;
6073
6074 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6075 if (!cap_offset)
6076 return -E1000_ERR_CONFIG;
6077
6078 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6079
6080 return 0;
6081 }
6082
6083 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6084 {
6085 struct igb_adapter *adapter = hw->back;
6086 u16 cap_offset;
6087
6088 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6089 if (!cap_offset)
6090 return -E1000_ERR_CONFIG;
6091
6092 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6093
6094 return 0;
6095 }
6096
6097 static void igb_vlan_rx_register(struct net_device *netdev,
6098 struct vlan_group *grp)
6099 {
6100 struct igb_adapter *adapter = netdev_priv(netdev);
6101 struct e1000_hw *hw = &adapter->hw;
6102 u32 ctrl, rctl;
6103
6104 igb_irq_disable(adapter);
6105 adapter->vlgrp = grp;
6106
6107 if (grp) {
6108 /* enable VLAN tag insert/strip */
6109 ctrl = rd32(E1000_CTRL);
6110 ctrl |= E1000_CTRL_VME;
6111 wr32(E1000_CTRL, ctrl);
6112
6113 /* Disable CFI check */
6114 rctl = rd32(E1000_RCTL);
6115 rctl &= ~E1000_RCTL_CFIEN;
6116 wr32(E1000_RCTL, rctl);
6117 } else {
6118 /* disable VLAN tag insert/strip */
6119 ctrl = rd32(E1000_CTRL);
6120 ctrl &= ~E1000_CTRL_VME;
6121 wr32(E1000_CTRL, ctrl);
6122 }
6123
6124 igb_rlpml_set(adapter);
6125
6126 if (!test_bit(__IGB_DOWN, &adapter->state))
6127 igb_irq_enable(adapter);
6128 }
6129
6130 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6131 {
6132 struct igb_adapter *adapter = netdev_priv(netdev);
6133 struct e1000_hw *hw = &adapter->hw;
6134 int pf_id = adapter->vfs_allocated_count;
6135
6136 /* attempt to add filter to vlvf array */
6137 igb_vlvf_set(adapter, vid, true, pf_id);
6138
6139 /* add the filter since PF can receive vlans w/o entry in vlvf */
6140 igb_vfta_set(hw, vid, true);
6141 }
6142
6143 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6144 {
6145 struct igb_adapter *adapter = netdev_priv(netdev);
6146 struct e1000_hw *hw = &adapter->hw;
6147 int pf_id = adapter->vfs_allocated_count;
6148 s32 err;
6149
6150 igb_irq_disable(adapter);
6151 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6152
6153 if (!test_bit(__IGB_DOWN, &adapter->state))
6154 igb_irq_enable(adapter);
6155
6156 /* remove vlan from VLVF table array */
6157 err = igb_vlvf_set(adapter, vid, false, pf_id);
6158
6159 /* if vid was not present in VLVF just remove it from table */
6160 if (err)
6161 igb_vfta_set(hw, vid, false);
6162 }
6163
6164 static void igb_restore_vlan(struct igb_adapter *adapter)
6165 {
6166 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6167
6168 if (adapter->vlgrp) {
6169 u16 vid;
6170 for (vid = 0; vid < VLAN_N_VID; vid++) {
6171 if (!vlan_group_get_device(adapter->vlgrp, vid))
6172 continue;
6173 igb_vlan_rx_add_vid(adapter->netdev, vid);
6174 }
6175 }
6176 }
6177
6178 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6179 {
6180 struct pci_dev *pdev = adapter->pdev;
6181 struct e1000_mac_info *mac = &adapter->hw.mac;
6182
6183 mac->autoneg = 0;
6184
6185 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6186 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6187 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6188 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6189 return -EINVAL;
6190 }
6191
6192 switch (spddplx) {
6193 case SPEED_10 + DUPLEX_HALF:
6194 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6195 break;
6196 case SPEED_10 + DUPLEX_FULL:
6197 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6198 break;
6199 case SPEED_100 + DUPLEX_HALF:
6200 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6201 break;
6202 case SPEED_100 + DUPLEX_FULL:
6203 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6204 break;
6205 case SPEED_1000 + DUPLEX_FULL:
6206 mac->autoneg = 1;
6207 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6208 break;
6209 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6210 default:
6211 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6212 return -EINVAL;
6213 }
6214 return 0;
6215 }
6216
6217 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6218 {
6219 struct net_device *netdev = pci_get_drvdata(pdev);
6220 struct igb_adapter *adapter = netdev_priv(netdev);
6221 struct e1000_hw *hw = &adapter->hw;
6222 u32 ctrl, rctl, status;
6223 u32 wufc = adapter->wol;
6224 #ifdef CONFIG_PM
6225 int retval = 0;
6226 #endif
6227
6228 netif_device_detach(netdev);
6229
6230 if (netif_running(netdev))
6231 igb_close(netdev);
6232
6233 igb_clear_interrupt_scheme(adapter);
6234
6235 #ifdef CONFIG_PM
6236 retval = pci_save_state(pdev);
6237 if (retval)
6238 return retval;
6239 #endif
6240
6241 status = rd32(E1000_STATUS);
6242 if (status & E1000_STATUS_LU)
6243 wufc &= ~E1000_WUFC_LNKC;
6244
6245 if (wufc) {
6246 igb_setup_rctl(adapter);
6247 igb_set_rx_mode(netdev);
6248
6249 /* turn on all-multi mode if wake on multicast is enabled */
6250 if (wufc & E1000_WUFC_MC) {
6251 rctl = rd32(E1000_RCTL);
6252 rctl |= E1000_RCTL_MPE;
6253 wr32(E1000_RCTL, rctl);
6254 }
6255
6256 ctrl = rd32(E1000_CTRL);
6257 /* advertise wake from D3Cold */
6258 #define E1000_CTRL_ADVD3WUC 0x00100000
6259 /* phy power management enable */
6260 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6261 ctrl |= E1000_CTRL_ADVD3WUC;
6262 wr32(E1000_CTRL, ctrl);
6263
6264 /* Allow time for pending master requests to run */
6265 igb_disable_pcie_master(hw);
6266
6267 wr32(E1000_WUC, E1000_WUC_PME_EN);
6268 wr32(E1000_WUFC, wufc);
6269 } else {
6270 wr32(E1000_WUC, 0);
6271 wr32(E1000_WUFC, 0);
6272 }
6273
6274 *enable_wake = wufc || adapter->en_mng_pt;
6275 if (!*enable_wake)
6276 igb_power_down_link(adapter);
6277 else
6278 igb_power_up_link(adapter);
6279
6280 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6281 * would have already happened in close and is redundant. */
6282 igb_release_hw_control(adapter);
6283
6284 pci_disable_device(pdev);
6285
6286 return 0;
6287 }
6288
6289 #ifdef CONFIG_PM
6290 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6291 {
6292 int retval;
6293 bool wake;
6294
6295 retval = __igb_shutdown(pdev, &wake);
6296 if (retval)
6297 return retval;
6298
6299 if (wake) {
6300 pci_prepare_to_sleep(pdev);
6301 } else {
6302 pci_wake_from_d3(pdev, false);
6303 pci_set_power_state(pdev, PCI_D3hot);
6304 }
6305
6306 return 0;
6307 }
6308
6309 static int igb_resume(struct pci_dev *pdev)
6310 {
6311 struct net_device *netdev = pci_get_drvdata(pdev);
6312 struct igb_adapter *adapter = netdev_priv(netdev);
6313 struct e1000_hw *hw = &adapter->hw;
6314 u32 err;
6315
6316 pci_set_power_state(pdev, PCI_D0);
6317 pci_restore_state(pdev);
6318 pci_save_state(pdev);
6319
6320 err = pci_enable_device_mem(pdev);
6321 if (err) {
6322 dev_err(&pdev->dev,
6323 "igb: Cannot enable PCI device from suspend\n");
6324 return err;
6325 }
6326 pci_set_master(pdev);
6327
6328 pci_enable_wake(pdev, PCI_D3hot, 0);
6329 pci_enable_wake(pdev, PCI_D3cold, 0);
6330
6331 if (igb_init_interrupt_scheme(adapter)) {
6332 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6333 return -ENOMEM;
6334 }
6335
6336 igb_reset(adapter);
6337
6338 /* let the f/w know that the h/w is now under the control of the
6339 * driver. */
6340 igb_get_hw_control(adapter);
6341
6342 wr32(E1000_WUS, ~0);
6343
6344 if (netif_running(netdev)) {
6345 err = igb_open(netdev);
6346 if (err)
6347 return err;
6348 }
6349
6350 netif_device_attach(netdev);
6351
6352 return 0;
6353 }
6354 #endif
6355
6356 static void igb_shutdown(struct pci_dev *pdev)
6357 {
6358 bool wake;
6359
6360 __igb_shutdown(pdev, &wake);
6361
6362 if (system_state == SYSTEM_POWER_OFF) {
6363 pci_wake_from_d3(pdev, wake);
6364 pci_set_power_state(pdev, PCI_D3hot);
6365 }
6366 }
6367
6368 #ifdef CONFIG_NET_POLL_CONTROLLER
6369 /*
6370 * Polling 'interrupt' - used by things like netconsole to send skbs
6371 * without having to re-enable interrupts. It's not called while
6372 * the interrupt routine is executing.
6373 */
6374 static void igb_netpoll(struct net_device *netdev)
6375 {
6376 struct igb_adapter *adapter = netdev_priv(netdev);
6377 struct e1000_hw *hw = &adapter->hw;
6378 int i;
6379
6380 if (!adapter->msix_entries) {
6381 struct igb_q_vector *q_vector = adapter->q_vector[0];
6382 igb_irq_disable(adapter);
6383 napi_schedule(&q_vector->napi);
6384 return;
6385 }
6386
6387 for (i = 0; i < adapter->num_q_vectors; i++) {
6388 struct igb_q_vector *q_vector = adapter->q_vector[i];
6389 wr32(E1000_EIMC, q_vector->eims_value);
6390 napi_schedule(&q_vector->napi);
6391 }
6392 }
6393 #endif /* CONFIG_NET_POLL_CONTROLLER */
6394
6395 /**
6396 * igb_io_error_detected - called when PCI error is detected
6397 * @pdev: Pointer to PCI device
6398 * @state: The current pci connection state
6399 *
6400 * This function is called after a PCI bus error affecting
6401 * this device has been detected.
6402 */
6403 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6404 pci_channel_state_t state)
6405 {
6406 struct net_device *netdev = pci_get_drvdata(pdev);
6407 struct igb_adapter *adapter = netdev_priv(netdev);
6408
6409 netif_device_detach(netdev);
6410
6411 if (state == pci_channel_io_perm_failure)
6412 return PCI_ERS_RESULT_DISCONNECT;
6413
6414 if (netif_running(netdev))
6415 igb_down(adapter);
6416 pci_disable_device(pdev);
6417
6418 /* Request a slot slot reset. */
6419 return PCI_ERS_RESULT_NEED_RESET;
6420 }
6421
6422 /**
6423 * igb_io_slot_reset - called after the pci bus has been reset.
6424 * @pdev: Pointer to PCI device
6425 *
6426 * Restart the card from scratch, as if from a cold-boot. Implementation
6427 * resembles the first-half of the igb_resume routine.
6428 */
6429 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6430 {
6431 struct net_device *netdev = pci_get_drvdata(pdev);
6432 struct igb_adapter *adapter = netdev_priv(netdev);
6433 struct e1000_hw *hw = &adapter->hw;
6434 pci_ers_result_t result;
6435 int err;
6436
6437 if (pci_enable_device_mem(pdev)) {
6438 dev_err(&pdev->dev,
6439 "Cannot re-enable PCI device after reset.\n");
6440 result = PCI_ERS_RESULT_DISCONNECT;
6441 } else {
6442 pci_set_master(pdev);
6443 pci_restore_state(pdev);
6444 pci_save_state(pdev);
6445
6446 pci_enable_wake(pdev, PCI_D3hot, 0);
6447 pci_enable_wake(pdev, PCI_D3cold, 0);
6448
6449 igb_reset(adapter);
6450 wr32(E1000_WUS, ~0);
6451 result = PCI_ERS_RESULT_RECOVERED;
6452 }
6453
6454 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6455 if (err) {
6456 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6457 "failed 0x%0x\n", err);
6458 /* non-fatal, continue */
6459 }
6460
6461 return result;
6462 }
6463
6464 /**
6465 * igb_io_resume - called when traffic can start flowing again.
6466 * @pdev: Pointer to PCI device
6467 *
6468 * This callback is called when the error recovery driver tells us that
6469 * its OK to resume normal operation. Implementation resembles the
6470 * second-half of the igb_resume routine.
6471 */
6472 static void igb_io_resume(struct pci_dev *pdev)
6473 {
6474 struct net_device *netdev = pci_get_drvdata(pdev);
6475 struct igb_adapter *adapter = netdev_priv(netdev);
6476
6477 if (netif_running(netdev)) {
6478 if (igb_up(adapter)) {
6479 dev_err(&pdev->dev, "igb_up failed after reset\n");
6480 return;
6481 }
6482 }
6483
6484 netif_device_attach(netdev);
6485
6486 /* let the f/w know that the h/w is now under the control of the
6487 * driver. */
6488 igb_get_hw_control(adapter);
6489 }
6490
6491 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6492 u8 qsel)
6493 {
6494 u32 rar_low, rar_high;
6495 struct e1000_hw *hw = &adapter->hw;
6496
6497 /* HW expects these in little endian so we reverse the byte order
6498 * from network order (big endian) to little endian
6499 */
6500 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6501 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6502 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6503
6504 /* Indicate to hardware the Address is Valid. */
6505 rar_high |= E1000_RAH_AV;
6506
6507 if (hw->mac.type == e1000_82575)
6508 rar_high |= E1000_RAH_POOL_1 * qsel;
6509 else
6510 rar_high |= E1000_RAH_POOL_1 << qsel;
6511
6512 wr32(E1000_RAL(index), rar_low);
6513 wrfl();
6514 wr32(E1000_RAH(index), rar_high);
6515 wrfl();
6516 }
6517
6518 static int igb_set_vf_mac(struct igb_adapter *adapter,
6519 int vf, unsigned char *mac_addr)
6520 {
6521 struct e1000_hw *hw = &adapter->hw;
6522 /* VF MAC addresses start at end of receive addresses and moves
6523 * torwards the first, as a result a collision should not be possible */
6524 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6525
6526 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6527
6528 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6529
6530 return 0;
6531 }
6532
6533 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6534 {
6535 struct igb_adapter *adapter = netdev_priv(netdev);
6536 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6537 return -EINVAL;
6538 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6539 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6540 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6541 " change effective.");
6542 if (test_bit(__IGB_DOWN, &adapter->state)) {
6543 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6544 " but the PF device is not up.\n");
6545 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6546 " attempting to use the VF device.\n");
6547 }
6548 return igb_set_vf_mac(adapter, vf, mac);
6549 }
6550
6551 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6552 {
6553 return -EOPNOTSUPP;
6554 }
6555
6556 static int igb_ndo_get_vf_config(struct net_device *netdev,
6557 int vf, struct ifla_vf_info *ivi)
6558 {
6559 struct igb_adapter *adapter = netdev_priv(netdev);
6560 if (vf >= adapter->vfs_allocated_count)
6561 return -EINVAL;
6562 ivi->vf = vf;
6563 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6564 ivi->tx_rate = 0;
6565 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6566 ivi->qos = adapter->vf_data[vf].pf_qos;
6567 return 0;
6568 }
6569
6570 static void igb_vmm_control(struct igb_adapter *adapter)
6571 {
6572 struct e1000_hw *hw = &adapter->hw;
6573 u32 reg;
6574
6575 switch (hw->mac.type) {
6576 case e1000_82575:
6577 default:
6578 /* replication is not supported for 82575 */
6579 return;
6580 case e1000_82576:
6581 /* notify HW that the MAC is adding vlan tags */
6582 reg = rd32(E1000_DTXCTL);
6583 reg |= E1000_DTXCTL_VLAN_ADDED;
6584 wr32(E1000_DTXCTL, reg);
6585 case e1000_82580:
6586 /* enable replication vlan tag stripping */
6587 reg = rd32(E1000_RPLOLR);
6588 reg |= E1000_RPLOLR_STRVLAN;
6589 wr32(E1000_RPLOLR, reg);
6590 case e1000_i350:
6591 /* none of the above registers are supported by i350 */
6592 break;
6593 }
6594
6595 if (adapter->vfs_allocated_count) {
6596 igb_vmdq_set_loopback_pf(hw, true);
6597 igb_vmdq_set_replication_pf(hw, true);
6598 } else {
6599 igb_vmdq_set_loopback_pf(hw, false);
6600 igb_vmdq_set_replication_pf(hw, false);
6601 }
6602 }
6603
6604 /* igb_main.c */
CJK support for tty framebuffer vt