?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
i40e: genericize the partition bandwidth control
[linux-2.6/btrfs-unstable.git] / drivers / net / ethernet / intel / i40e / i40e_main.c
blob8d7bd85933bb8019de9c9fcc6d97635565b204d7
1 /*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
4 * Copyright(c) 2013 - 2017 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
27 #include <linux/etherdevice.h>
28 #include <linux/of_net.h>
29 #include <linux/pci.h>
30 #include <linux/bpf.h>
31
32 /* Local includes */
33 #include "i40e.h"
34 #include "i40e_diag.h"
35 #include <net/udp_tunnel.h>
36 /* All i40e tracepoints are defined by the include below, which
37 * must be included exactly once across the whole kernel with
38 * CREATE_TRACE_POINTS defined
39 */
40 #define CREATE_TRACE_POINTS
41 #include "i40e_trace.h"
42
43 const char i40e_driver_name[] = "i40e";
44 static const char i40e_driver_string[] =
45 "Intel(R) Ethernet Connection XL710 Network Driver";
46
47 #define DRV_KERN "-k"
48
49 #define DRV_VERSION_MAJOR 2
50 #define DRV_VERSION_MINOR 1
51 #define DRV_VERSION_BUILD 14
52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
53 __stringify(DRV_VERSION_MINOR) "." \
54 __stringify(DRV_VERSION_BUILD) DRV_KERN
55 const char i40e_driver_version_str[] = DRV_VERSION;
56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
57
58 /* a bit of forward declarations */
59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
61 static int i40e_add_vsi(struct i40e_vsi *vsi);
62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
64 static int i40e_setup_misc_vector(struct i40e_pf *pf);
65 static void i40e_determine_queue_usage(struct i40e_pf *pf);
66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
68 static int i40e_reset(struct i40e_pf *pf);
69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
70 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
71 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
72
73 /* i40e_pci_tbl - PCI Device ID Table
74 *
75 * Last entry must be all 0s
76 *
77 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
78 * Class, Class Mask, private data (not used) }
79 */
80 static const struct pci_device_id i40e_pci_tbl[] = {
81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
98 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
99 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
100 /* required last entry */
101 {0, }
102 };
103 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
104
105 #define I40E_MAX_VF_COUNT 128
106 static int debug = -1;
107 module_param(debug, uint, 0);
108 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
109
110 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
112 MODULE_LICENSE("GPL");
113 MODULE_VERSION(DRV_VERSION);
114
115 static struct workqueue_struct *i40e_wq;
116
117 /**
118 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
119 * @hw: pointer to the HW structure
120 * @mem: ptr to mem struct to fill out
121 * @size: size of memory requested
122 * @alignment: what to align the allocation to
123 **/
124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
125 u64 size, u32 alignment)
126 {
127 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
128
129 mem->size = ALIGN(size, alignment);
130 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
131 &mem->pa, GFP_KERNEL);
132 if (!mem->va)
133 return -ENOMEM;
134
135 return 0;
136 }
137
138 /**
139 * i40e_free_dma_mem_d - OS specific memory free for shared code
140 * @hw: pointer to the HW structure
141 * @mem: ptr to mem struct to free
142 **/
143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
144 {
145 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
146
147 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
148 mem->va = NULL;
149 mem->pa = 0;
150 mem->size = 0;
151
152 return 0;
153 }
154
155 /**
156 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
157 * @hw: pointer to the HW structure
158 * @mem: ptr to mem struct to fill out
159 * @size: size of memory requested
160 **/
161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
162 u32 size)
163 {
164 mem->size = size;
165 mem->va = kzalloc(size, GFP_KERNEL);
166
167 if (!mem->va)
168 return -ENOMEM;
169
170 return 0;
171 }
172
173 /**
174 * i40e_free_virt_mem_d - OS specific memory free for shared code
175 * @hw: pointer to the HW structure
176 * @mem: ptr to mem struct to free
177 **/
178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
179 {
180 /* it's ok to kfree a NULL pointer */
181 kfree(mem->va);
182 mem->va = NULL;
183 mem->size = 0;
184
185 return 0;
186 }
187
188 /**
189 * i40e_get_lump - find a lump of free generic resource
190 * @pf: board private structure
191 * @pile: the pile of resource to search
192 * @needed: the number of items needed
193 * @id: an owner id to stick on the items assigned
194 *
195 * Returns the base item index of the lump, or negative for error
196 *
197 * The search_hint trick and lack of advanced fit-finding only work
198 * because we're highly likely to have all the same size lump requests.
199 * Linear search time and any fragmentation should be minimal.
200 **/
201 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
202 u16 needed, u16 id)
203 {
204 int ret = -ENOMEM;
205 int i, j;
206
207 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
208 dev_info(&pf->pdev->dev,
209 "param err: pile=%p needed=%d id=0x%04x\n",
210 pile, needed, id);
211 return -EINVAL;
212 }
213
214 /* start the linear search with an imperfect hint */
215 i = pile->search_hint;
216 while (i < pile->num_entries) {
217 /* skip already allocated entries */
218 if (pile->list[i] & I40E_PILE_VALID_BIT) {
219 i++;
220 continue;
221 }
222
223 /* do we have enough in this lump? */
224 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
225 if (pile->list[i+j] & I40E_PILE_VALID_BIT)
226 break;
227 }
228
229 if (j == needed) {
230 /* there was enough, so assign it to the requestor */
231 for (j = 0; j < needed; j++)
232 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
233 ret = i;
234 pile->search_hint = i + j;
235 break;
236 }
237
238 /* not enough, so skip over it and continue looking */
239 i += j;
240 }
241
242 return ret;
243 }
244
245 /**
246 * i40e_put_lump - return a lump of generic resource
247 * @pile: the pile of resource to search
248 * @index: the base item index
249 * @id: the owner id of the items assigned
250 *
251 * Returns the count of items in the lump
252 **/
253 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
254 {
255 int valid_id = (id | I40E_PILE_VALID_BIT);
256 int count = 0;
257 int i;
258
259 if (!pile || index >= pile->num_entries)
260 return -EINVAL;
261
262 for (i = index;
263 i < pile->num_entries && pile->list[i] == valid_id;
264 i++) {
265 pile->list[i] = 0;
266 count++;
267 }
268
269 if (count && index < pile->search_hint)
270 pile->search_hint = index;
271
272 return count;
273 }
274
275 /**
276 * i40e_find_vsi_from_id - searches for the vsi with the given id
277 * @pf - the pf structure to search for the vsi
278 * @id - id of the vsi it is searching for
279 **/
280 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
281 {
282 int i;
283
284 for (i = 0; i < pf->num_alloc_vsi; i++)
285 if (pf->vsi[i] && (pf->vsi[i]->id == id))
286 return pf->vsi[i];
287
288 return NULL;
289 }
290
291 /**
292 * i40e_service_event_schedule - Schedule the service task to wake up
293 * @pf: board private structure
294 *
295 * If not already scheduled, this puts the task into the work queue
296 **/
297 void i40e_service_event_schedule(struct i40e_pf *pf)
298 {
299 if (!test_bit(__I40E_DOWN, pf->state) &&
300 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
301 queue_work(i40e_wq, &pf->service_task);
302 }
303
304 /**
305 * i40e_tx_timeout - Respond to a Tx Hang
306 * @netdev: network interface device structure
307 *
308 * If any port has noticed a Tx timeout, it is likely that the whole
309 * device is munged, not just the one netdev port, so go for the full
310 * reset.
311 **/
312 static void i40e_tx_timeout(struct net_device *netdev)
313 {
314 struct i40e_netdev_priv *np = netdev_priv(netdev);
315 struct i40e_vsi *vsi = np->vsi;
316 struct i40e_pf *pf = vsi->back;
317 struct i40e_ring *tx_ring = NULL;
318 unsigned int i, hung_queue = 0;
319 u32 head, val;
320
321 pf->tx_timeout_count++;
322
323 /* find the stopped queue the same way the stack does */
324 for (i = 0; i < netdev->num_tx_queues; i++) {
325 struct netdev_queue *q;
326 unsigned long trans_start;
327
328 q = netdev_get_tx_queue(netdev, i);
329 trans_start = q->trans_start;
330 if (netif_xmit_stopped(q) &&
331 time_after(jiffies,
332 (trans_start + netdev->watchdog_timeo))) {
333 hung_queue = i;
334 break;
335 }
336 }
337
338 if (i == netdev->num_tx_queues) {
339 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
340 } else {
341 /* now that we have an index, find the tx_ring struct */
342 for (i = 0; i < vsi->num_queue_pairs; i++) {
343 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
344 if (hung_queue ==
345 vsi->tx_rings[i]->queue_index) {
346 tx_ring = vsi->tx_rings[i];
347 break;
348 }
349 }
350 }
351 }
352
353 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
354 pf->tx_timeout_recovery_level = 1; /* reset after some time */
355 else if (time_before(jiffies,
356 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
357 return; /* don't do any new action before the next timeout */
358
359 if (tx_ring) {
360 head = i40e_get_head(tx_ring);
361 /* Read interrupt register */
362 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
363 val = rd32(&pf->hw,
364 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
365 tx_ring->vsi->base_vector - 1));
366 else
367 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
368
369 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
370 vsi->seid, hung_queue, tx_ring->next_to_clean,
371 head, tx_ring->next_to_use,
372 readl(tx_ring->tail), val);
373 }
374
375 pf->tx_timeout_last_recovery = jiffies;
376 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
377 pf->tx_timeout_recovery_level, hung_queue);
378
379 switch (pf->tx_timeout_recovery_level) {
380 case 1:
381 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
382 break;
383 case 2:
384 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
385 break;
386 case 3:
387 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
388 break;
389 default:
390 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
391 break;
392 }
393
394 i40e_service_event_schedule(pf);
395 pf->tx_timeout_recovery_level++;
396 }
397
398 /**
399 * i40e_get_vsi_stats_struct - Get System Network Statistics
400 * @vsi: the VSI we care about
401 *
402 * Returns the address of the device statistics structure.
403 * The statistics are actually updated from the service task.
404 **/
405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
406 {
407 return &vsi->net_stats;
408 }
409
410 /**
411 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
412 * @ring: Tx ring to get statistics from
413 * @stats: statistics entry to be updated
414 **/
415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
416 struct rtnl_link_stats64 *stats)
417 {
418 u64 bytes, packets;
419 unsigned int start;
420
421 do {
422 start = u64_stats_fetch_begin_irq(&ring->syncp);
423 packets = ring->stats.packets;
424 bytes = ring->stats.bytes;
425 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
426
427 stats->tx_packets += packets;
428 stats->tx_bytes += bytes;
429 }
430
431 /**
432 * i40e_get_netdev_stats_struct - Get statistics for netdev interface
433 * @netdev: network interface device structure
434 *
435 * Returns the address of the device statistics structure.
436 * The statistics are actually updated from the service task.
437 **/
438 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
439 struct rtnl_link_stats64 *stats)
440 {
441 struct i40e_netdev_priv *np = netdev_priv(netdev);
442 struct i40e_ring *tx_ring, *rx_ring;
443 struct i40e_vsi *vsi = np->vsi;
444 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
445 int i;
446
447 if (test_bit(__I40E_VSI_DOWN, vsi->state))
448 return;
449
450 if (!vsi->tx_rings)
451 return;
452
453 rcu_read_lock();
454 for (i = 0; i < vsi->num_queue_pairs; i++) {
455 u64 bytes, packets;
456 unsigned int start;
457
458 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
459 if (!tx_ring)
460 continue;
461 i40e_get_netdev_stats_struct_tx(tx_ring, stats);
462
463 rx_ring = &tx_ring[1];
464
465 do {
466 start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
467 packets = rx_ring->stats.packets;
468 bytes = rx_ring->stats.bytes;
469 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
470
471 stats->rx_packets += packets;
472 stats->rx_bytes += bytes;
473
474 if (i40e_enabled_xdp_vsi(vsi))
475 i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats);
476 }
477 rcu_read_unlock();
478
479 /* following stats updated by i40e_watchdog_subtask() */
480 stats->multicast = vsi_stats->multicast;
481 stats->tx_errors = vsi_stats->tx_errors;
482 stats->tx_dropped = vsi_stats->tx_dropped;
483 stats->rx_errors = vsi_stats->rx_errors;
484 stats->rx_dropped = vsi_stats->rx_dropped;
485 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
486 stats->rx_length_errors = vsi_stats->rx_length_errors;
487 }
488
489 /**
490 * i40e_vsi_reset_stats - Resets all stats of the given vsi
491 * @vsi: the VSI to have its stats reset
492 **/
493 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
494 {
495 struct rtnl_link_stats64 *ns;
496 int i;
497
498 if (!vsi)
499 return;
500
501 ns = i40e_get_vsi_stats_struct(vsi);
502 memset(ns, 0, sizeof(*ns));
503 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
504 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
505 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
506 if (vsi->rx_rings && vsi->rx_rings[0]) {
507 for (i = 0; i < vsi->num_queue_pairs; i++) {
508 memset(&vsi->rx_rings[i]->stats, 0,
509 sizeof(vsi->rx_rings[i]->stats));
510 memset(&vsi->rx_rings[i]->rx_stats, 0,
511 sizeof(vsi->rx_rings[i]->rx_stats));
512 memset(&vsi->tx_rings[i]->stats, 0,
513 sizeof(vsi->tx_rings[i]->stats));
514 memset(&vsi->tx_rings[i]->tx_stats, 0,
515 sizeof(vsi->tx_rings[i]->tx_stats));
516 }
517 }
518 vsi->stat_offsets_loaded = false;
519 }
520
521 /**
522 * i40e_pf_reset_stats - Reset all of the stats for the given PF
523 * @pf: the PF to be reset
524 **/
525 void i40e_pf_reset_stats(struct i40e_pf *pf)
526 {
527 int i;
528
529 memset(&pf->stats, 0, sizeof(pf->stats));
530 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
531 pf->stat_offsets_loaded = false;
532
533 for (i = 0; i < I40E_MAX_VEB; i++) {
534 if (pf->veb[i]) {
535 memset(&pf->veb[i]->stats, 0,
536 sizeof(pf->veb[i]->stats));
537 memset(&pf->veb[i]->stats_offsets, 0,
538 sizeof(pf->veb[i]->stats_offsets));
539 pf->veb[i]->stat_offsets_loaded = false;
540 }
541 }
542 pf->hw_csum_rx_error = 0;
543 }
544
545 /**
546 * i40e_stat_update48 - read and update a 48 bit stat from the chip
547 * @hw: ptr to the hardware info
548 * @hireg: the high 32 bit reg to read
549 * @loreg: the low 32 bit reg to read
550 * @offset_loaded: has the initial offset been loaded yet
551 * @offset: ptr to current offset value
552 * @stat: ptr to the stat
553 *
554 * Since the device stats are not reset at PFReset, they likely will not
555 * be zeroed when the driver starts. We'll save the first values read
556 * and use them as offsets to be subtracted from the raw values in order
557 * to report stats that count from zero. In the process, we also manage
558 * the potential roll-over.
559 **/
560 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
561 bool offset_loaded, u64 *offset, u64 *stat)
562 {
563 u64 new_data;
564
565 if (hw->device_id == I40E_DEV_ID_QEMU) {
566 new_data = rd32(hw, loreg);
567 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
568 } else {
569 new_data = rd64(hw, loreg);
570 }
571 if (!offset_loaded)
572 *offset = new_data;
573 if (likely(new_data >= *offset))
574 *stat = new_data - *offset;
575 else
576 *stat = (new_data + BIT_ULL(48)) - *offset;
577 *stat &= 0xFFFFFFFFFFFFULL;
578 }
579
580 /**
581 * i40e_stat_update32 - read and update a 32 bit stat from the chip
582 * @hw: ptr to the hardware info
583 * @reg: the hw reg to read
584 * @offset_loaded: has the initial offset been loaded yet
585 * @offset: ptr to current offset value
586 * @stat: ptr to the stat
587 **/
588 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
589 bool offset_loaded, u64 *offset, u64 *stat)
590 {
591 u32 new_data;
592
593 new_data = rd32(hw, reg);
594 if (!offset_loaded)
595 *offset = new_data;
596 if (likely(new_data >= *offset))
597 *stat = (u32)(new_data - *offset);
598 else
599 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
600 }
601
602 /**
603 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
604 * @vsi: the VSI to be updated
605 **/
606 void i40e_update_eth_stats(struct i40e_vsi *vsi)
607 {
608 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
609 struct i40e_pf *pf = vsi->back;
610 struct i40e_hw *hw = &pf->hw;
611 struct i40e_eth_stats *oes;
612 struct i40e_eth_stats *es; /* device's eth stats */
613
614 es = &vsi->eth_stats;
615 oes = &vsi->eth_stats_offsets;
616
617 /* Gather up the stats that the hw collects */
618 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
619 vsi->stat_offsets_loaded,
620 &oes->tx_errors, &es->tx_errors);
621 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
622 vsi->stat_offsets_loaded,
623 &oes->rx_discards, &es->rx_discards);
624 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
625 vsi->stat_offsets_loaded,
626 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
627 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
628 vsi->stat_offsets_loaded,
629 &oes->tx_errors, &es->tx_errors);
630
631 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
632 I40E_GLV_GORCL(stat_idx),
633 vsi->stat_offsets_loaded,
634 &oes->rx_bytes, &es->rx_bytes);
635 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
636 I40E_GLV_UPRCL(stat_idx),
637 vsi->stat_offsets_loaded,
638 &oes->rx_unicast, &es->rx_unicast);
639 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
640 I40E_GLV_MPRCL(stat_idx),
641 vsi->stat_offsets_loaded,
642 &oes->rx_multicast, &es->rx_multicast);
643 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
644 I40E_GLV_BPRCL(stat_idx),
645 vsi->stat_offsets_loaded,
646 &oes->rx_broadcast, &es->rx_broadcast);
647
648 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
649 I40E_GLV_GOTCL(stat_idx),
650 vsi->stat_offsets_loaded,
651 &oes->tx_bytes, &es->tx_bytes);
652 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
653 I40E_GLV_UPTCL(stat_idx),
654 vsi->stat_offsets_loaded,
655 &oes->tx_unicast, &es->tx_unicast);
656 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
657 I40E_GLV_MPTCL(stat_idx),
658 vsi->stat_offsets_loaded,
659 &oes->tx_multicast, &es->tx_multicast);
660 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
661 I40E_GLV_BPTCL(stat_idx),
662 vsi->stat_offsets_loaded,
663 &oes->tx_broadcast, &es->tx_broadcast);
664 vsi->stat_offsets_loaded = true;
665 }
666
667 /**
668 * i40e_update_veb_stats - Update Switch component statistics
669 * @veb: the VEB being updated
670 **/
671 static void i40e_update_veb_stats(struct i40e_veb *veb)
672 {
673 struct i40e_pf *pf = veb->pf;
674 struct i40e_hw *hw = &pf->hw;
675 struct i40e_eth_stats *oes;
676 struct i40e_eth_stats *es; /* device's eth stats */
677 struct i40e_veb_tc_stats *veb_oes;
678 struct i40e_veb_tc_stats *veb_es;
679 int i, idx = 0;
680
681 idx = veb->stats_idx;
682 es = &veb->stats;
683 oes = &veb->stats_offsets;
684 veb_es = &veb->tc_stats;
685 veb_oes = &veb->tc_stats_offsets;
686
687 /* Gather up the stats that the hw collects */
688 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
689 veb->stat_offsets_loaded,
690 &oes->tx_discards, &es->tx_discards);
691 if (hw->revision_id > 0)
692 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
693 veb->stat_offsets_loaded,
694 &oes->rx_unknown_protocol,
695 &es->rx_unknown_protocol);
696 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
697 veb->stat_offsets_loaded,
698 &oes->rx_bytes, &es->rx_bytes);
699 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
700 veb->stat_offsets_loaded,
701 &oes->rx_unicast, &es->rx_unicast);
702 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
703 veb->stat_offsets_loaded,
704 &oes->rx_multicast, &es->rx_multicast);
705 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
706 veb->stat_offsets_loaded,
707 &oes->rx_broadcast, &es->rx_broadcast);
708
709 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
710 veb->stat_offsets_loaded,
711 &oes->tx_bytes, &es->tx_bytes);
712 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
713 veb->stat_offsets_loaded,
714 &oes->tx_unicast, &es->tx_unicast);
715 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
716 veb->stat_offsets_loaded,
717 &oes->tx_multicast, &es->tx_multicast);
718 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
719 veb->stat_offsets_loaded,
720 &oes->tx_broadcast, &es->tx_broadcast);
721 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
722 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
723 I40E_GLVEBTC_RPCL(i, idx),
724 veb->stat_offsets_loaded,
725 &veb_oes->tc_rx_packets[i],
726 &veb_es->tc_rx_packets[i]);
727 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
728 I40E_GLVEBTC_RBCL(i, idx),
729 veb->stat_offsets_loaded,
730 &veb_oes->tc_rx_bytes[i],
731 &veb_es->tc_rx_bytes[i]);
732 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
733 I40E_GLVEBTC_TPCL(i, idx),
734 veb->stat_offsets_loaded,
735 &veb_oes->tc_tx_packets[i],
736 &veb_es->tc_tx_packets[i]);
737 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
738 I40E_GLVEBTC_TBCL(i, idx),
739 veb->stat_offsets_loaded,
740 &veb_oes->tc_tx_bytes[i],
741 &veb_es->tc_tx_bytes[i]);
742 }
743 veb->stat_offsets_loaded = true;
744 }
745
746 /**
747 * i40e_update_vsi_stats - Update the vsi statistics counters.
748 * @vsi: the VSI to be updated
749 *
750 * There are a few instances where we store the same stat in a
751 * couple of different structs. This is partly because we have
752 * the netdev stats that need to be filled out, which is slightly
753 * different from the "eth_stats" defined by the chip and used in
754 * VF communications. We sort it out here.
755 **/
756 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
757 {
758 struct i40e_pf *pf = vsi->back;
759 struct rtnl_link_stats64 *ons;
760 struct rtnl_link_stats64 *ns; /* netdev stats */
761 struct i40e_eth_stats *oes;
762 struct i40e_eth_stats *es; /* device's eth stats */
763 u32 tx_restart, tx_busy;
764 struct i40e_ring *p;
765 u32 rx_page, rx_buf;
766 u64 bytes, packets;
767 unsigned int start;
768 u64 tx_linearize;
769 u64 tx_force_wb;
770 u64 rx_p, rx_b;
771 u64 tx_p, tx_b;
772 u16 q;
773
774 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
775 test_bit(__I40E_CONFIG_BUSY, pf->state))
776 return;
777
778 ns = i40e_get_vsi_stats_struct(vsi);
779 ons = &vsi->net_stats_offsets;
780 es = &vsi->eth_stats;
781 oes = &vsi->eth_stats_offsets;
782
783 /* Gather up the netdev and vsi stats that the driver collects
784 * on the fly during packet processing
785 */
786 rx_b = rx_p = 0;
787 tx_b = tx_p = 0;
788 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
789 rx_page = 0;
790 rx_buf = 0;
791 rcu_read_lock();
792 for (q = 0; q < vsi->num_queue_pairs; q++) {
793 /* locate Tx ring */
794 p = ACCESS_ONCE(vsi->tx_rings[q]);
795
796 do {
797 start = u64_stats_fetch_begin_irq(&p->syncp);
798 packets = p->stats.packets;
799 bytes = p->stats.bytes;
800 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
801 tx_b += bytes;
802 tx_p += packets;
803 tx_restart += p->tx_stats.restart_queue;
804 tx_busy += p->tx_stats.tx_busy;
805 tx_linearize += p->tx_stats.tx_linearize;
806 tx_force_wb += p->tx_stats.tx_force_wb;
807
808 /* Rx queue is part of the same block as Tx queue */
809 p = &p[1];
810 do {
811 start = u64_stats_fetch_begin_irq(&p->syncp);
812 packets = p->stats.packets;
813 bytes = p->stats.bytes;
814 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
815 rx_b += bytes;
816 rx_p += packets;
817 rx_buf += p->rx_stats.alloc_buff_failed;
818 rx_page += p->rx_stats.alloc_page_failed;
819 }
820 rcu_read_unlock();
821 vsi->tx_restart = tx_restart;
822 vsi->tx_busy = tx_busy;
823 vsi->tx_linearize = tx_linearize;
824 vsi->tx_force_wb = tx_force_wb;
825 vsi->rx_page_failed = rx_page;
826 vsi->rx_buf_failed = rx_buf;
827
828 ns->rx_packets = rx_p;
829 ns->rx_bytes = rx_b;
830 ns->tx_packets = tx_p;
831 ns->tx_bytes = tx_b;
832
833 /* update netdev stats from eth stats */
834 i40e_update_eth_stats(vsi);
835 ons->tx_errors = oes->tx_errors;
836 ns->tx_errors = es->tx_errors;
837 ons->multicast = oes->rx_multicast;
838 ns->multicast = es->rx_multicast;
839 ons->rx_dropped = oes->rx_discards;
840 ns->rx_dropped = es->rx_discards;
841 ons->tx_dropped = oes->tx_discards;
842 ns->tx_dropped = es->tx_discards;
843
844 /* pull in a couple PF stats if this is the main vsi */
845 if (vsi == pf->vsi[pf->lan_vsi]) {
846 ns->rx_crc_errors = pf->stats.crc_errors;
847 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
848 ns->rx_length_errors = pf->stats.rx_length_errors;
849 }
850 }
851
852 /**
853 * i40e_update_pf_stats - Update the PF statistics counters.
854 * @pf: the PF to be updated
855 **/
856 static void i40e_update_pf_stats(struct i40e_pf *pf)
857 {
858 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
859 struct i40e_hw_port_stats *nsd = &pf->stats;
860 struct i40e_hw *hw = &pf->hw;
861 u32 val;
862 int i;
863
864 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
865 I40E_GLPRT_GORCL(hw->port),
866 pf->stat_offsets_loaded,
867 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
868 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
869 I40E_GLPRT_GOTCL(hw->port),
870 pf->stat_offsets_loaded,
871 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
872 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
873 pf->stat_offsets_loaded,
874 &osd->eth.rx_discards,
875 &nsd->eth.rx_discards);
876 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
877 I40E_GLPRT_UPRCL(hw->port),
878 pf->stat_offsets_loaded,
879 &osd->eth.rx_unicast,
880 &nsd->eth.rx_unicast);
881 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
882 I40E_GLPRT_MPRCL(hw->port),
883 pf->stat_offsets_loaded,
884 &osd->eth.rx_multicast,
885 &nsd->eth.rx_multicast);
886 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
887 I40E_GLPRT_BPRCL(hw->port),
888 pf->stat_offsets_loaded,
889 &osd->eth.rx_broadcast,
890 &nsd->eth.rx_broadcast);
891 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
892 I40E_GLPRT_UPTCL(hw->port),
893 pf->stat_offsets_loaded,
894 &osd->eth.tx_unicast,
895 &nsd->eth.tx_unicast);
896 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
897 I40E_GLPRT_MPTCL(hw->port),
898 pf->stat_offsets_loaded,
899 &osd->eth.tx_multicast,
900 &nsd->eth.tx_multicast);
901 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
902 I40E_GLPRT_BPTCL(hw->port),
903 pf->stat_offsets_loaded,
904 &osd->eth.tx_broadcast,
905 &nsd->eth.tx_broadcast);
906
907 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
908 pf->stat_offsets_loaded,
909 &osd->tx_dropped_link_down,
910 &nsd->tx_dropped_link_down);
911
912 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
913 pf->stat_offsets_loaded,
914 &osd->crc_errors, &nsd->crc_errors);
915
916 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
917 pf->stat_offsets_loaded,
918 &osd->illegal_bytes, &nsd->illegal_bytes);
919
920 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
921 pf->stat_offsets_loaded,
922 &osd->mac_local_faults,
923 &nsd->mac_local_faults);
924 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
925 pf->stat_offsets_loaded,
926 &osd->mac_remote_faults,
927 &nsd->mac_remote_faults);
928
929 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
930 pf->stat_offsets_loaded,
931 &osd->rx_length_errors,
932 &nsd->rx_length_errors);
933
934 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
935 pf->stat_offsets_loaded,
936 &osd->link_xon_rx, &nsd->link_xon_rx);
937 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
938 pf->stat_offsets_loaded,
939 &osd->link_xon_tx, &nsd->link_xon_tx);
940 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
941 pf->stat_offsets_loaded,
942 &osd->link_xoff_rx, &nsd->link_xoff_rx);
943 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
944 pf->stat_offsets_loaded,
945 &osd->link_xoff_tx, &nsd->link_xoff_tx);
946
947 for (i = 0; i < 8; i++) {
948 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
949 pf->stat_offsets_loaded,
950 &osd->priority_xoff_rx[i],
951 &nsd->priority_xoff_rx[i]);
952 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
953 pf->stat_offsets_loaded,
954 &osd->priority_xon_rx[i],
955 &nsd->priority_xon_rx[i]);
956 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
957 pf->stat_offsets_loaded,
958 &osd->priority_xon_tx[i],
959 &nsd->priority_xon_tx[i]);
960 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
961 pf->stat_offsets_loaded,
962 &osd->priority_xoff_tx[i],
963 &nsd->priority_xoff_tx[i]);
964 i40e_stat_update32(hw,
965 I40E_GLPRT_RXON2OFFCNT(hw->port, i),
966 pf->stat_offsets_loaded,
967 &osd->priority_xon_2_xoff[i],
968 &nsd->priority_xon_2_xoff[i]);
969 }
970
971 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
972 I40E_GLPRT_PRC64L(hw->port),
973 pf->stat_offsets_loaded,
974 &osd->rx_size_64, &nsd->rx_size_64);
975 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
976 I40E_GLPRT_PRC127L(hw->port),
977 pf->stat_offsets_loaded,
978 &osd->rx_size_127, &nsd->rx_size_127);
979 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
980 I40E_GLPRT_PRC255L(hw->port),
981 pf->stat_offsets_loaded,
982 &osd->rx_size_255, &nsd->rx_size_255);
983 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
984 I40E_GLPRT_PRC511L(hw->port),
985 pf->stat_offsets_loaded,
986 &osd->rx_size_511, &nsd->rx_size_511);
987 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
988 I40E_GLPRT_PRC1023L(hw->port),
989 pf->stat_offsets_loaded,
990 &osd->rx_size_1023, &nsd->rx_size_1023);
991 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
992 I40E_GLPRT_PRC1522L(hw->port),
993 pf->stat_offsets_loaded,
994 &osd->rx_size_1522, &nsd->rx_size_1522);
995 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
996 I40E_GLPRT_PRC9522L(hw->port),
997 pf->stat_offsets_loaded,
998 &osd->rx_size_big, &nsd->rx_size_big);
999
1000 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1001 I40E_GLPRT_PTC64L(hw->port),
1002 pf->stat_offsets_loaded,
1003 &osd->tx_size_64, &nsd->tx_size_64);
1004 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1005 I40E_GLPRT_PTC127L(hw->port),
1006 pf->stat_offsets_loaded,
1007 &osd->tx_size_127, &nsd->tx_size_127);
1008 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1009 I40E_GLPRT_PTC255L(hw->port),
1010 pf->stat_offsets_loaded,
1011 &osd->tx_size_255, &nsd->tx_size_255);
1012 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1013 I40E_GLPRT_PTC511L(hw->port),
1014 pf->stat_offsets_loaded,
1015 &osd->tx_size_511, &nsd->tx_size_511);
1016 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1017 I40E_GLPRT_PTC1023L(hw->port),
1018 pf->stat_offsets_loaded,
1019 &osd->tx_size_1023, &nsd->tx_size_1023);
1020 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1021 I40E_GLPRT_PTC1522L(hw->port),
1022 pf->stat_offsets_loaded,
1023 &osd->tx_size_1522, &nsd->tx_size_1522);
1024 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1025 I40E_GLPRT_PTC9522L(hw->port),
1026 pf->stat_offsets_loaded,
1027 &osd->tx_size_big, &nsd->tx_size_big);
1028
1029 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1030 pf->stat_offsets_loaded,
1031 &osd->rx_undersize, &nsd->rx_undersize);
1032 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1033 pf->stat_offsets_loaded,
1034 &osd->rx_fragments, &nsd->rx_fragments);
1035 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1036 pf->stat_offsets_loaded,
1037 &osd->rx_oversize, &nsd->rx_oversize);
1038 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1039 pf->stat_offsets_loaded,
1040 &osd->rx_jabber, &nsd->rx_jabber);
1041
1042 /* FDIR stats */
1043 i40e_stat_update32(hw,
1044 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)),
1045 pf->stat_offsets_loaded,
1046 &osd->fd_atr_match, &nsd->fd_atr_match);
1047 i40e_stat_update32(hw,
1048 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)),
1049 pf->stat_offsets_loaded,
1050 &osd->fd_sb_match, &nsd->fd_sb_match);
1051 i40e_stat_update32(hw,
1052 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)),
1053 pf->stat_offsets_loaded,
1054 &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match);
1055
1056 val = rd32(hw, I40E_PRTPM_EEE_STAT);
1057 nsd->tx_lpi_status =
1058 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1059 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1060 nsd->rx_lpi_status =
1061 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1062 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1063 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1064 pf->stat_offsets_loaded,
1065 &osd->tx_lpi_count, &nsd->tx_lpi_count);
1066 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1067 pf->stat_offsets_loaded,
1068 &osd->rx_lpi_count, &nsd->rx_lpi_count);
1069
1070 if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1071 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED))
1072 nsd->fd_sb_status = true;
1073 else
1074 nsd->fd_sb_status = false;
1075
1076 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1077 !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED))
1078 nsd->fd_atr_status = true;
1079 else
1080 nsd->fd_atr_status = false;
1081
1082 pf->stat_offsets_loaded = true;
1083 }
1084
1085 /**
1086 * i40e_update_stats - Update the various statistics counters.
1087 * @vsi: the VSI to be updated
1088 *
1089 * Update the various stats for this VSI and its related entities.
1090 **/
1091 void i40e_update_stats(struct i40e_vsi *vsi)
1092 {
1093 struct i40e_pf *pf = vsi->back;
1094
1095 if (vsi == pf->vsi[pf->lan_vsi])
1096 i40e_update_pf_stats(pf);
1097
1098 i40e_update_vsi_stats(vsi);
1099 }
1100
1101 /**
1102 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1103 * @vsi: the VSI to be searched
1104 * @macaddr: the MAC address
1105 * @vlan: the vlan
1106 *
1107 * Returns ptr to the filter object or NULL
1108 **/
1109 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1110 const u8 *macaddr, s16 vlan)
1111 {
1112 struct i40e_mac_filter *f;
1113 u64 key;
1114
1115 if (!vsi || !macaddr)
1116 return NULL;
1117
1118 key = i40e_addr_to_hkey(macaddr);
1119 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1120 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1121 (vlan == f->vlan))
1122 return f;
1123 }
1124 return NULL;
1125 }
1126
1127 /**
1128 * i40e_find_mac - Find a mac addr in the macvlan filters list
1129 * @vsi: the VSI to be searched
1130 * @macaddr: the MAC address we are searching for
1131 *
1132 * Returns the first filter with the provided MAC address or NULL if
1133 * MAC address was not found
1134 **/
1135 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1136 {
1137 struct i40e_mac_filter *f;
1138 u64 key;
1139
1140 if (!vsi || !macaddr)
1141 return NULL;
1142
1143 key = i40e_addr_to_hkey(macaddr);
1144 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1145 if ((ether_addr_equal(macaddr, f->macaddr)))
1146 return f;
1147 }
1148 return NULL;
1149 }
1150
1151 /**
1152 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1153 * @vsi: the VSI to be searched
1154 *
1155 * Returns true if VSI is in vlan mode or false otherwise
1156 **/
1157 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1158 {
1159 /* If we have a PVID, always operate in VLAN mode */
1160 if (vsi->info.pvid)
1161 return true;
1162
1163 /* We need to operate in VLAN mode whenever we have any filters with
1164 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1165 * time, incurring search cost repeatedly. However, we can notice two
1166 * things:
1167 *
1168 * 1) the only place where we can gain a VLAN filter is in
1169 * i40e_add_filter.
1170 *
1171 * 2) the only place where filters are actually removed is in
1172 * i40e_sync_filters_subtask.
1173 *
1174 * Thus, we can simply use a boolean value, has_vlan_filters which we
1175 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1176 * we have to perform the full search after deleting filters in
1177 * i40e_sync_filters_subtask, but we already have to search
1178 * filters here and can perform the check at the same time. This
1179 * results in avoiding embedding a loop for VLAN mode inside another
1180 * loop over all the filters, and should maintain correctness as noted
1181 * above.
1182 */
1183 return vsi->has_vlan_filter;
1184 }
1185
1186 /**
1187 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1188 * @vsi: the VSI to configure
1189 * @tmp_add_list: list of filters ready to be added
1190 * @tmp_del_list: list of filters ready to be deleted
1191 * @vlan_filters: the number of active VLAN filters
1192 *
1193 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1194 * behave as expected. If we have any active VLAN filters remaining or about
1195 * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1196 * so that they only match against untagged traffic. If we no longer have any
1197 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1198 * so that they match against both tagged and untagged traffic. In this way,
1199 * we ensure that we correctly receive the desired traffic. This ensures that
1200 * when we have an active VLAN we will receive only untagged traffic and
1201 * traffic matching active VLANs. If we have no active VLANs then we will
1202 * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1203 *
1204 * Finally, in a similar fashion, this function also corrects filters when
1205 * there is an active PVID assigned to this VSI.
1206 *
1207 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1208 *
1209 * This function is only expected to be called from within
1210 * i40e_sync_vsi_filters.
1211 *
1212 * NOTE: This function expects to be called while under the
1213 * mac_filter_hash_lock
1214 */
1215 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1216 struct hlist_head *tmp_add_list,
1217 struct hlist_head *tmp_del_list,
1218 int vlan_filters)
1219 {
1220 s16 pvid = le16_to_cpu(vsi->info.pvid);
1221 struct i40e_mac_filter *f, *add_head;
1222 struct i40e_new_mac_filter *new;
1223 struct hlist_node *h;
1224 int bkt, new_vlan;
1225
1226 /* To determine if a particular filter needs to be replaced we
1227 * have the three following conditions:
1228 *
1229 * a) if we have a PVID assigned, then all filters which are
1230 * not marked as VLAN=PVID must be replaced with filters that
1231 * are.
1232 * b) otherwise, if we have any active VLANS, all filters
1233 * which are marked as VLAN=-1 must be replaced with
1234 * filters marked as VLAN=0
1235 * c) finally, if we do not have any active VLANS, all filters
1236 * which are marked as VLAN=0 must be replaced with filters
1237 * marked as VLAN=-1
1238 */
1239
1240 /* Update the filters about to be added in place */
1241 hlist_for_each_entry(new, tmp_add_list, hlist) {
1242 if (pvid && new->f->vlan != pvid)
1243 new->f->vlan = pvid;
1244 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1245 new->f->vlan = 0;
1246 else if (!vlan_filters && new->f->vlan == 0)
1247 new->f->vlan = I40E_VLAN_ANY;
1248 }
1249
1250 /* Update the remaining active filters */
1251 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1252 /* Combine the checks for whether a filter needs to be changed
1253 * and then determine the new VLAN inside the if block, in
1254 * order to avoid duplicating code for adding the new filter
1255 * then deleting the old filter.
1256 */
1257 if ((pvid && f->vlan != pvid) ||
1258 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1259 (!vlan_filters && f->vlan == 0)) {
1260 /* Determine the new vlan we will be adding */
1261 if (pvid)
1262 new_vlan = pvid;
1263 else if (vlan_filters)
1264 new_vlan = 0;
1265 else
1266 new_vlan = I40E_VLAN_ANY;
1267
1268 /* Create the new filter */
1269 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1270 if (!add_head)
1271 return -ENOMEM;
1272
1273 /* Create a temporary i40e_new_mac_filter */
1274 new = kzalloc(sizeof(*new), GFP_ATOMIC);
1275 if (!new)
1276 return -ENOMEM;
1277
1278 new->f = add_head;
1279 new->state = add_head->state;
1280
1281 /* Add the new filter to the tmp list */
1282 hlist_add_head(&new->hlist, tmp_add_list);
1283
1284 /* Put the original filter into the delete list */
1285 f->state = I40E_FILTER_REMOVE;
1286 hash_del(&f->hlist);
1287 hlist_add_head(&f->hlist, tmp_del_list);
1288 }
1289 }
1290
1291 vsi->has_vlan_filter = !!vlan_filters;
1292
1293 return 0;
1294 }
1295
1296 /**
1297 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1298 * @vsi: the PF Main VSI - inappropriate for any other VSI
1299 * @macaddr: the MAC address
1300 *
1301 * Remove whatever filter the firmware set up so the driver can manage
1302 * its own filtering intelligently.
1303 **/
1304 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1305 {
1306 struct i40e_aqc_remove_macvlan_element_data element;
1307 struct i40e_pf *pf = vsi->back;
1308
1309 /* Only appropriate for the PF main VSI */
1310 if (vsi->type != I40E_VSI_MAIN)
1311 return;
1312
1313 memset(&element, 0, sizeof(element));
1314 ether_addr_copy(element.mac_addr, macaddr);
1315 element.vlan_tag = 0;
1316 /* Ignore error returns, some firmware does it this way... */
1317 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1318 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1319
1320 memset(&element, 0, sizeof(element));
1321 ether_addr_copy(element.mac_addr, macaddr);
1322 element.vlan_tag = 0;
1323 /* ...and some firmware does it this way. */
1324 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1325 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1326 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1327 }
1328
1329 /**
1330 * i40e_add_filter - Add a mac/vlan filter to the VSI
1331 * @vsi: the VSI to be searched
1332 * @macaddr: the MAC address
1333 * @vlan: the vlan
1334 *
1335 * Returns ptr to the filter object or NULL when no memory available.
1336 *
1337 * NOTE: This function is expected to be called with mac_filter_hash_lock
1338 * being held.
1339 **/
1340 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1341 const u8 *macaddr, s16 vlan)
1342 {
1343 struct i40e_mac_filter *f;
1344 u64 key;
1345
1346 if (!vsi || !macaddr)
1347 return NULL;
1348
1349 f = i40e_find_filter(vsi, macaddr, vlan);
1350 if (!f) {
1351 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1352 if (!f)
1353 return NULL;
1354
1355 /* Update the boolean indicating if we need to function in
1356 * VLAN mode.
1357 */
1358 if (vlan >= 0)
1359 vsi->has_vlan_filter = true;
1360
1361 ether_addr_copy(f->macaddr, macaddr);
1362 f->vlan = vlan;
1363 /* If we're in overflow promisc mode, set the state directly
1364 * to failed, so we don't bother to try sending the filter
1365 * to the hardware.
1366 */
1367 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))
1368 f->state = I40E_FILTER_FAILED;
1369 else
1370 f->state = I40E_FILTER_NEW;
1371 INIT_HLIST_NODE(&f->hlist);
1372
1373 key = i40e_addr_to_hkey(macaddr);
1374 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1375
1376 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1377 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1378 }
1379
1380 /* If we're asked to add a filter that has been marked for removal, it
1381 * is safe to simply restore it to active state. __i40e_del_filter
1382 * will have simply deleted any filters which were previously marked
1383 * NEW or FAILED, so if it is currently marked REMOVE it must have
1384 * previously been ACTIVE. Since we haven't yet run the sync filters
1385 * task, just restore this filter to the ACTIVE state so that the
1386 * sync task leaves it in place
1387 */
1388 if (f->state == I40E_FILTER_REMOVE)
1389 f->state = I40E_FILTER_ACTIVE;
1390
1391 return f;
1392 }
1393
1394 /**
1395 * __i40e_del_filter - Remove a specific filter from the VSI
1396 * @vsi: VSI to remove from
1397 * @f: the filter to remove from the list
1398 *
1399 * This function should be called instead of i40e_del_filter only if you know
1400 * the exact filter you will remove already, such as via i40e_find_filter or
1401 * i40e_find_mac.
1402 *
1403 * NOTE: This function is expected to be called with mac_filter_hash_lock
1404 * being held.
1405 * ANOTHER NOTE: This function MUST be called from within the context of
1406 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1407 * instead of list_for_each_entry().
1408 **/
1409 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1410 {
1411 if (!f)
1412 return;
1413
1414 /* If the filter was never added to firmware then we can just delete it
1415 * directly and we don't want to set the status to remove or else an
1416 * admin queue command will unnecessarily fire.
1417 */
1418 if ((f->state == I40E_FILTER_FAILED) ||
1419 (f->state == I40E_FILTER_NEW)) {
1420 hash_del(&f->hlist);
1421 kfree(f);
1422 } else {
1423 f->state = I40E_FILTER_REMOVE;
1424 }
1425
1426 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1427 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1428 }
1429
1430 /**
1431 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1432 * @vsi: the VSI to be searched
1433 * @macaddr: the MAC address
1434 * @vlan: the VLAN
1435 *
1436 * NOTE: This function is expected to be called with mac_filter_hash_lock
1437 * being held.
1438 * ANOTHER NOTE: This function MUST be called from within the context of
1439 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1440 * instead of list_for_each_entry().
1441 **/
1442 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1443 {
1444 struct i40e_mac_filter *f;
1445
1446 if (!vsi || !macaddr)
1447 return;
1448
1449 f = i40e_find_filter(vsi, macaddr, vlan);
1450 __i40e_del_filter(vsi, f);
1451 }
1452
1453 /**
1454 * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1455 * @vsi: the VSI to be searched
1456 * @macaddr: the mac address to be filtered
1457 *
1458 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1459 * go through all the macvlan filters and add a macvlan filter for each
1460 * unique vlan that already exists. If a PVID has been assigned, instead only
1461 * add the macaddr to that VLAN.
1462 *
1463 * Returns last filter added on success, else NULL
1464 **/
1465 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1466 const u8 *macaddr)
1467 {
1468 struct i40e_mac_filter *f, *add = NULL;
1469 struct hlist_node *h;
1470 int bkt;
1471
1472 if (vsi->info.pvid)
1473 return i40e_add_filter(vsi, macaddr,
1474 le16_to_cpu(vsi->info.pvid));
1475
1476 if (!i40e_is_vsi_in_vlan(vsi))
1477 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1478
1479 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1480 if (f->state == I40E_FILTER_REMOVE)
1481 continue;
1482 add = i40e_add_filter(vsi, macaddr, f->vlan);
1483 if (!add)
1484 return NULL;
1485 }
1486
1487 return add;
1488 }
1489
1490 /**
1491 * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1492 * @vsi: the VSI to be searched
1493 * @macaddr: the mac address to be removed
1494 *
1495 * Removes a given MAC address from a VSI regardless of what VLAN it has been
1496 * associated with.
1497 *
1498 * Returns 0 for success, or error
1499 **/
1500 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1501 {
1502 struct i40e_mac_filter *f;
1503 struct hlist_node *h;
1504 bool found = false;
1505 int bkt;
1506
1507 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1508 "Missing mac_filter_hash_lock\n");
1509 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1510 if (ether_addr_equal(macaddr, f->macaddr)) {
1511 __i40e_del_filter(vsi, f);
1512 found = true;
1513 }
1514 }
1515
1516 if (found)
1517 return 0;
1518 else
1519 return -ENOENT;
1520 }
1521
1522 /**
1523 * i40e_set_mac - NDO callback to set mac address
1524 * @netdev: network interface device structure
1525 * @p: pointer to an address structure
1526 *
1527 * Returns 0 on success, negative on failure
1528 **/
1529 static int i40e_set_mac(struct net_device *netdev, void *p)
1530 {
1531 struct i40e_netdev_priv *np = netdev_priv(netdev);
1532 struct i40e_vsi *vsi = np->vsi;
1533 struct i40e_pf *pf = vsi->back;
1534 struct i40e_hw *hw = &pf->hw;
1535 struct sockaddr *addr = p;
1536
1537 if (!is_valid_ether_addr(addr->sa_data))
1538 return -EADDRNOTAVAIL;
1539
1540 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1541 netdev_info(netdev, "already using mac address %pM\n",
1542 addr->sa_data);
1543 return 0;
1544 }
1545
1546 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1547 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1548 return -EADDRNOTAVAIL;
1549
1550 if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1551 netdev_info(netdev, "returning to hw mac address %pM\n",
1552 hw->mac.addr);
1553 else
1554 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1555
1556 spin_lock_bh(&vsi->mac_filter_hash_lock);
1557 i40e_del_mac_filter(vsi, netdev->dev_addr);
1558 i40e_add_mac_filter(vsi, addr->sa_data);
1559 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1560 ether_addr_copy(netdev->dev_addr, addr->sa_data);
1561 if (vsi->type == I40E_VSI_MAIN) {
1562 i40e_status ret;
1563
1564 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1565 I40E_AQC_WRITE_TYPE_LAA_WOL,
1566 addr->sa_data, NULL);
1567 if (ret)
1568 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1569 i40e_stat_str(hw, ret),
1570 i40e_aq_str(hw, hw->aq.asq_last_status));
1571 }
1572
1573 /* schedule our worker thread which will take care of
1574 * applying the new filter changes
1575 */
1576 i40e_service_event_schedule(vsi->back);
1577 return 0;
1578 }
1579
1580 /**
1581 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1582 * @vsi: the VSI being setup
1583 * @ctxt: VSI context structure
1584 * @enabled_tc: Enabled TCs bitmap
1585 * @is_add: True if called before Add VSI
1586 *
1587 * Setup VSI queue mapping for enabled traffic classes.
1588 **/
1589 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1590 struct i40e_vsi_context *ctxt,
1591 u8 enabled_tc,
1592 bool is_add)
1593 {
1594 struct i40e_pf *pf = vsi->back;
1595 u16 sections = 0;
1596 u8 netdev_tc = 0;
1597 u16 numtc = 0;
1598 u16 qcount;
1599 u8 offset;
1600 u16 qmap;
1601 int i;
1602 u16 num_tc_qps = 0;
1603
1604 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1605 offset = 0;
1606
1607 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1608 /* Find numtc from enabled TC bitmap */
1609 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1610 if (enabled_tc & BIT(i)) /* TC is enabled */
1611 numtc++;
1612 }
1613 if (!numtc) {
1614 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1615 numtc = 1;
1616 }
1617 } else {
1618 /* At least TC0 is enabled in case of non-DCB case */
1619 numtc = 1;
1620 }
1621
1622 vsi->tc_config.numtc = numtc;
1623 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1624 /* Number of queues per enabled TC */
1625 qcount = vsi->alloc_queue_pairs;
1626
1627 num_tc_qps = qcount / numtc;
1628 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1629
1630 /* Setup queue offset/count for all TCs for given VSI */
1631 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1632 /* See if the given TC is enabled for the given VSI */
1633 if (vsi->tc_config.enabled_tc & BIT(i)) {
1634 /* TC is enabled */
1635 int pow, num_qps;
1636
1637 switch (vsi->type) {
1638 case I40E_VSI_MAIN:
1639 qcount = min_t(int, pf->alloc_rss_size,
1640 num_tc_qps);
1641 break;
1642 case I40E_VSI_FDIR:
1643 case I40E_VSI_SRIOV:
1644 case I40E_VSI_VMDQ2:
1645 default:
1646 qcount = num_tc_qps;
1647 WARN_ON(i != 0);
1648 break;
1649 }
1650 vsi->tc_config.tc_info[i].qoffset = offset;
1651 vsi->tc_config.tc_info[i].qcount = qcount;
1652
1653 /* find the next higher power-of-2 of num queue pairs */
1654 num_qps = qcount;
1655 pow = 0;
1656 while (num_qps && (BIT_ULL(pow) < qcount)) {
1657 pow++;
1658 num_qps >>= 1;
1659 }
1660
1661 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1662 qmap =
1663 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1664 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1665
1666 offset += qcount;
1667 } else {
1668 /* TC is not enabled so set the offset to
1669 * default queue and allocate one queue
1670 * for the given TC.
1671 */
1672 vsi->tc_config.tc_info[i].qoffset = 0;
1673 vsi->tc_config.tc_info[i].qcount = 1;
1674 vsi->tc_config.tc_info[i].netdev_tc = 0;
1675
1676 qmap = 0;
1677 }
1678 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1679 }
1680
1681 /* Set actual Tx/Rx queue pairs */
1682 vsi->num_queue_pairs = offset;
1683 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1684 if (vsi->req_queue_pairs > 0)
1685 vsi->num_queue_pairs = vsi->req_queue_pairs;
1686 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1687 vsi->num_queue_pairs = pf->num_lan_msix;
1688 }
1689
1690 /* Scheduler section valid can only be set for ADD VSI */
1691 if (is_add) {
1692 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1693
1694 ctxt->info.up_enable_bits = enabled_tc;
1695 }
1696 if (vsi->type == I40E_VSI_SRIOV) {
1697 ctxt->info.mapping_flags |=
1698 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1699 for (i = 0; i < vsi->num_queue_pairs; i++)
1700 ctxt->info.queue_mapping[i] =
1701 cpu_to_le16(vsi->base_queue + i);
1702 } else {
1703 ctxt->info.mapping_flags |=
1704 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1705 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1706 }
1707 ctxt->info.valid_sections |= cpu_to_le16(sections);
1708 }
1709
1710 /**
1711 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1712 * @netdev: the netdevice
1713 * @addr: address to add
1714 *
1715 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1716 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1717 */
1718 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1719 {
1720 struct i40e_netdev_priv *np = netdev_priv(netdev);
1721 struct i40e_vsi *vsi = np->vsi;
1722
1723 if (i40e_add_mac_filter(vsi, addr))
1724 return 0;
1725 else
1726 return -ENOMEM;
1727 }
1728
1729 /**
1730 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1731 * @netdev: the netdevice
1732 * @addr: address to add
1733 *
1734 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1735 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1736 */
1737 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1738 {
1739 struct i40e_netdev_priv *np = netdev_priv(netdev);
1740 struct i40e_vsi *vsi = np->vsi;
1741
1742 i40e_del_mac_filter(vsi, addr);
1743
1744 return 0;
1745 }
1746
1747 /**
1748 * i40e_set_rx_mode - NDO callback to set the netdev filters
1749 * @netdev: network interface device structure
1750 **/
1751 static void i40e_set_rx_mode(struct net_device *netdev)
1752 {
1753 struct i40e_netdev_priv *np = netdev_priv(netdev);
1754 struct i40e_vsi *vsi = np->vsi;
1755
1756 spin_lock_bh(&vsi->mac_filter_hash_lock);
1757
1758 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1759 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1760
1761 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1762
1763 /* check for other flag changes */
1764 if (vsi->current_netdev_flags != vsi->netdev->flags) {
1765 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1766 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1767 }
1768
1769 /* schedule our worker thread which will take care of
1770 * applying the new filter changes
1771 */
1772 i40e_service_event_schedule(vsi->back);
1773 }
1774
1775 /**
1776 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1777 * @vsi: Pointer to VSI struct
1778 * @from: Pointer to list which contains MAC filter entries - changes to
1779 * those entries needs to be undone.
1780 *
1781 * MAC filter entries from this list were slated for deletion.
1782 **/
1783 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1784 struct hlist_head *from)
1785 {
1786 struct i40e_mac_filter *f;
1787 struct hlist_node *h;
1788
1789 hlist_for_each_entry_safe(f, h, from, hlist) {
1790 u64 key = i40e_addr_to_hkey(f->macaddr);
1791
1792 /* Move the element back into MAC filter list*/
1793 hlist_del(&f->hlist);
1794 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1795 }
1796 }
1797
1798 /**
1799 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1800 * @vsi: Pointer to vsi struct
1801 * @from: Pointer to list which contains MAC filter entries - changes to
1802 * those entries needs to be undone.
1803 *
1804 * MAC filter entries from this list were slated for addition.
1805 **/
1806 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1807 struct hlist_head *from)
1808 {
1809 struct i40e_new_mac_filter *new;
1810 struct hlist_node *h;
1811
1812 hlist_for_each_entry_safe(new, h, from, hlist) {
1813 /* We can simply free the wrapper structure */
1814 hlist_del(&new->hlist);
1815 kfree(new);
1816 }
1817 }
1818
1819 /**
1820 * i40e_next_entry - Get the next non-broadcast filter from a list
1821 * @next: pointer to filter in list
1822 *
1823 * Returns the next non-broadcast filter in the list. Required so that we
1824 * ignore broadcast filters within the list, since these are not handled via
1825 * the normal firmware update path.
1826 */
1827 static
1828 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
1829 {
1830 hlist_for_each_entry_continue(next, hlist) {
1831 if (!is_broadcast_ether_addr(next->f->macaddr))
1832 return next;
1833 }
1834
1835 return NULL;
1836 }
1837
1838 /**
1839 * i40e_update_filter_state - Update filter state based on return data
1840 * from firmware
1841 * @count: Number of filters added
1842 * @add_list: return data from fw
1843 * @head: pointer to first filter in current batch
1844 *
1845 * MAC filter entries from list were slated to be added to device. Returns
1846 * number of successful filters. Note that 0 does NOT mean success!
1847 **/
1848 static int
1849 i40e_update_filter_state(int count,
1850 struct i40e_aqc_add_macvlan_element_data *add_list,
1851 struct i40e_new_mac_filter *add_head)
1852 {
1853 int retval = 0;
1854 int i;
1855
1856 for (i = 0; i < count; i++) {
1857 /* Always check status of each filter. We don't need to check
1858 * the firmware return status because we pre-set the filter
1859 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1860 * request to the adminq. Thus, if it no longer matches then
1861 * we know the filter is active.
1862 */
1863 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1864 add_head->state = I40E_FILTER_FAILED;
1865 } else {
1866 add_head->state = I40E_FILTER_ACTIVE;
1867 retval++;
1868 }
1869
1870 add_head = i40e_next_filter(add_head);
1871 if (!add_head)
1872 break;
1873 }
1874
1875 return retval;
1876 }
1877
1878 /**
1879 * i40e_aqc_del_filters - Request firmware to delete a set of filters
1880 * @vsi: ptr to the VSI
1881 * @vsi_name: name to display in messages
1882 * @list: the list of filters to send to firmware
1883 * @num_del: the number of filters to delete
1884 * @retval: Set to -EIO on failure to delete
1885 *
1886 * Send a request to firmware via AdminQ to delete a set of filters. Uses
1887 * *retval instead of a return value so that success does not force ret_val to
1888 * be set to 0. This ensures that a sequence of calls to this function
1889 * preserve the previous value of *retval on successful delete.
1890 */
1891 static
1892 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1893 struct i40e_aqc_remove_macvlan_element_data *list,
1894 int num_del, int *retval)
1895 {
1896 struct i40e_hw *hw = &vsi->back->hw;
1897 i40e_status aq_ret;
1898 int aq_err;
1899
1900 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1901 aq_err = hw->aq.asq_last_status;
1902
1903 /* Explicitly ignore and do not report when firmware returns ENOENT */
1904 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1905 *retval = -EIO;
1906 dev_info(&vsi->back->pdev->dev,
1907 "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1908 vsi_name, i40e_stat_str(hw, aq_ret),
1909 i40e_aq_str(hw, aq_err));
1910 }
1911 }
1912
1913 /**
1914 * i40e_aqc_add_filters - Request firmware to add a set of filters
1915 * @vsi: ptr to the VSI
1916 * @vsi_name: name to display in messages
1917 * @list: the list of filters to send to firmware
1918 * @add_head: Position in the add hlist
1919 * @num_add: the number of filters to add
1920 * @promisc_change: set to true on exit if promiscuous mode was forced on
1921 *
1922 * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1923 * promisc_changed to true if the firmware has run out of space for more
1924 * filters.
1925 */
1926 static
1927 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1928 struct i40e_aqc_add_macvlan_element_data *list,
1929 struct i40e_new_mac_filter *add_head,
1930 int num_add, bool *promisc_changed)
1931 {
1932 struct i40e_hw *hw = &vsi->back->hw;
1933 int aq_err, fcnt;
1934
1935 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1936 aq_err = hw->aq.asq_last_status;
1937 fcnt = i40e_update_filter_state(num_add, list, add_head);
1938
1939 if (fcnt != num_add) {
1940 *promisc_changed = true;
1941 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
1942 dev_warn(&vsi->back->pdev->dev,
1943 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1944 i40e_aq_str(hw, aq_err),
1945 vsi_name);
1946 }
1947 }
1948
1949 /**
1950 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1951 * @vsi: pointer to the VSI
1952 * @f: filter data
1953 *
1954 * This function sets or clears the promiscuous broadcast flags for VLAN
1955 * filters in order to properly receive broadcast frames. Assumes that only
1956 * broadcast filters are passed.
1957 *
1958 * Returns status indicating success or failure;
1959 **/
1960 static i40e_status
1961 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1962 struct i40e_mac_filter *f)
1963 {
1964 bool enable = f->state == I40E_FILTER_NEW;
1965 struct i40e_hw *hw = &vsi->back->hw;
1966 i40e_status aq_ret;
1967
1968 if (f->vlan == I40E_VLAN_ANY) {
1969 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1970 vsi->seid,
1971 enable,
1972 NULL);
1973 } else {
1974 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1975 vsi->seid,
1976 enable,
1977 f->vlan,
1978 NULL);
1979 }
1980
1981 if (aq_ret)
1982 dev_warn(&vsi->back->pdev->dev,
1983 "Error %s setting broadcast promiscuous mode on %s\n",
1984 i40e_aq_str(hw, hw->aq.asq_last_status),
1985 vsi_name);
1986
1987 return aq_ret;
1988 }
1989
1990 /**
1991 * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1992 * @vsi: ptr to the VSI
1993 *
1994 * Push any outstanding VSI filter changes through the AdminQ.
1995 *
1996 * Returns 0 or error value
1997 **/
1998 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1999 {
2000 struct hlist_head tmp_add_list, tmp_del_list;
2001 struct i40e_mac_filter *f;
2002 struct i40e_new_mac_filter *new, *add_head = NULL;
2003 struct i40e_hw *hw = &vsi->back->hw;
2004 unsigned int failed_filters = 0;
2005 unsigned int vlan_filters = 0;
2006 bool promisc_changed = false;
2007 char vsi_name[16] = "PF";
2008 int filter_list_len = 0;
2009 i40e_status aq_ret = 0;
2010 u32 changed_flags = 0;
2011 struct hlist_node *h;
2012 struct i40e_pf *pf;
2013 int num_add = 0;
2014 int num_del = 0;
2015 int retval = 0;
2016 u16 cmd_flags;
2017 int list_size;
2018 int bkt;
2019
2020 /* empty array typed pointers, kcalloc later */
2021 struct i40e_aqc_add_macvlan_element_data *add_list;
2022 struct i40e_aqc_remove_macvlan_element_data *del_list;
2023
2024 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2025 usleep_range(1000, 2000);
2026 pf = vsi->back;
2027
2028 if (vsi->netdev) {
2029 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2030 vsi->current_netdev_flags = vsi->netdev->flags;
2031 }
2032
2033 INIT_HLIST_HEAD(&tmp_add_list);
2034 INIT_HLIST_HEAD(&tmp_del_list);
2035
2036 if (vsi->type == I40E_VSI_SRIOV)
2037 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2038 else if (vsi->type != I40E_VSI_MAIN)
2039 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2040
2041 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2042 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2043
2044 spin_lock_bh(&vsi->mac_filter_hash_lock);
2045 /* Create a list of filters to delete. */
2046 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2047 if (f->state == I40E_FILTER_REMOVE) {
2048 /* Move the element into temporary del_list */
2049 hash_del(&f->hlist);
2050 hlist_add_head(&f->hlist, &tmp_del_list);
2051
2052 /* Avoid counting removed filters */
2053 continue;
2054 }
2055 if (f->state == I40E_FILTER_NEW) {
2056 /* Create a temporary i40e_new_mac_filter */
2057 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2058 if (!new)
2059 goto err_no_memory_locked;
2060
2061 /* Store pointer to the real filter */
2062 new->f = f;
2063 new->state = f->state;
2064
2065 /* Add it to the hash list */
2066 hlist_add_head(&new->hlist, &tmp_add_list);
2067 }
2068
2069 /* Count the number of active (current and new) VLAN
2070 * filters we have now. Does not count filters which
2071 * are marked for deletion.
2072 */
2073 if (f->vlan > 0)
2074 vlan_filters++;
2075 }
2076
2077 retval = i40e_correct_mac_vlan_filters(vsi,
2078 &tmp_add_list,
2079 &tmp_del_list,
2080 vlan_filters);
2081 if (retval)
2082 goto err_no_memory_locked;
2083
2084 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2085 }
2086
2087 /* Now process 'del_list' outside the lock */
2088 if (!hlist_empty(&tmp_del_list)) {
2089 filter_list_len = hw->aq.asq_buf_size /
2090 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2091 list_size = filter_list_len *
2092 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2093 del_list = kzalloc(list_size, GFP_ATOMIC);
2094 if (!del_list)
2095 goto err_no_memory;
2096
2097 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2098 cmd_flags = 0;
2099
2100 /* handle broadcast filters by updating the broadcast
2101 * promiscuous flag and release filter list.
2102 */
2103 if (is_broadcast_ether_addr(f->macaddr)) {
2104 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2105
2106 hlist_del(&f->hlist);
2107 kfree(f);
2108 continue;
2109 }
2110
2111 /* add to delete list */
2112 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2113 if (f->vlan == I40E_VLAN_ANY) {
2114 del_list[num_del].vlan_tag = 0;
2115 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2116 } else {
2117 del_list[num_del].vlan_tag =
2118 cpu_to_le16((u16)(f->vlan));
2119 }
2120
2121 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2122 del_list[num_del].flags = cmd_flags;
2123 num_del++;
2124
2125 /* flush a full buffer */
2126 if (num_del == filter_list_len) {
2127 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2128 num_del, &retval);
2129 memset(del_list, 0, list_size);
2130 num_del = 0;
2131 }
2132 /* Release memory for MAC filter entries which were
2133 * synced up with HW.
2134 */
2135 hlist_del(&f->hlist);
2136 kfree(f);
2137 }
2138
2139 if (num_del) {
2140 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2141 num_del, &retval);
2142 }
2143
2144 kfree(del_list);
2145 del_list = NULL;
2146 }
2147
2148 if (!hlist_empty(&tmp_add_list)) {
2149 /* Do all the adds now. */
2150 filter_list_len = hw->aq.asq_buf_size /
2151 sizeof(struct i40e_aqc_add_macvlan_element_data);
2152 list_size = filter_list_len *
2153 sizeof(struct i40e_aqc_add_macvlan_element_data);
2154 add_list = kzalloc(list_size, GFP_ATOMIC);
2155 if (!add_list)
2156 goto err_no_memory;
2157
2158 num_add = 0;
2159 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2160 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2161 vsi->state)) {
2162 new->state = I40E_FILTER_FAILED;
2163 continue;
2164 }
2165
2166 /* handle broadcast filters by updating the broadcast
2167 * promiscuous flag instead of adding a MAC filter.
2168 */
2169 if (is_broadcast_ether_addr(new->f->macaddr)) {
2170 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2171 new->f))
2172 new->state = I40E_FILTER_FAILED;
2173 else
2174 new->state = I40E_FILTER_ACTIVE;
2175 continue;
2176 }
2177
2178 /* add to add array */
2179 if (num_add == 0)
2180 add_head = new;
2181 cmd_flags = 0;
2182 ether_addr_copy(add_list[num_add].mac_addr,
2183 new->f->macaddr);
2184 if (new->f->vlan == I40E_VLAN_ANY) {
2185 add_list[num_add].vlan_tag = 0;
2186 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2187 } else {
2188 add_list[num_add].vlan_tag =
2189 cpu_to_le16((u16)(new->f->vlan));
2190 }
2191 add_list[num_add].queue_number = 0;
2192 /* set invalid match method for later detection */
2193 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2194 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2195 add_list[num_add].flags = cpu_to_le16(cmd_flags);
2196 num_add++;
2197
2198 /* flush a full buffer */
2199 if (num_add == filter_list_len) {
2200 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2201 add_head, num_add,
2202 &promisc_changed);
2203 memset(add_list, 0, list_size);
2204 num_add = 0;
2205 }
2206 }
2207 if (num_add) {
2208 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2209 num_add, &promisc_changed);
2210 }
2211 /* Now move all of the filters from the temp add list back to
2212 * the VSI's list.
2213 */
2214 spin_lock_bh(&vsi->mac_filter_hash_lock);
2215 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2216 /* Only update the state if we're still NEW */
2217 if (new->f->state == I40E_FILTER_NEW)
2218 new->f->state = new->state;
2219 hlist_del(&new->hlist);
2220 kfree(new);
2221 }
2222 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2223 kfree(add_list);
2224 add_list = NULL;
2225 }
2226
2227 /* Determine the number of active and failed filters. */
2228 spin_lock_bh(&vsi->mac_filter_hash_lock);
2229 vsi->active_filters = 0;
2230 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2231 if (f->state == I40E_FILTER_ACTIVE)
2232 vsi->active_filters++;
2233 else if (f->state == I40E_FILTER_FAILED)
2234 failed_filters++;
2235 }
2236 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2237
2238 /* If promiscuous mode has changed, we need to calculate a new
2239 * threshold for when we are safe to exit
2240 */
2241 if (promisc_changed)
2242 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2243
2244 /* Check if we are able to exit overflow promiscuous mode. We can
2245 * safely exit if we didn't just enter, we no longer have any failed
2246 * filters, and we have reduced filters below the threshold value.
2247 */
2248 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) &&
2249 !promisc_changed && !failed_filters &&
2250 (vsi->active_filters < vsi->promisc_threshold)) {
2251 dev_info(&pf->pdev->dev,
2252 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2253 vsi_name);
2254 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2255 promisc_changed = true;
2256 vsi->promisc_threshold = 0;
2257 }
2258
2259 /* if the VF is not trusted do not do promisc */
2260 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2261 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2262 goto out;
2263 }
2264
2265 /* check for changes in promiscuous modes */
2266 if (changed_flags & IFF_ALLMULTI) {
2267 bool cur_multipromisc;
2268
2269 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2270 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2271 vsi->seid,
2272 cur_multipromisc,
2273 NULL);
2274 if (aq_ret) {
2275 retval = i40e_aq_rc_to_posix(aq_ret,
2276 hw->aq.asq_last_status);
2277 dev_info(&pf->pdev->dev,
2278 "set multi promisc failed on %s, err %s aq_err %s\n",
2279 vsi_name,
2280 i40e_stat_str(hw, aq_ret),
2281 i40e_aq_str(hw, hw->aq.asq_last_status));
2282 }
2283 }
2284 if ((changed_flags & IFF_PROMISC) ||
2285 (promisc_changed &&
2286 test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))) {
2287 bool cur_promisc;
2288
2289 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2290 test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2291 vsi->state));
2292 if ((vsi->type == I40E_VSI_MAIN) &&
2293 (pf->lan_veb != I40E_NO_VEB) &&
2294 !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2295 /* set defport ON for Main VSI instead of true promisc
2296 * this way we will get all unicast/multicast and VLAN
2297 * promisc behavior but will not get VF or VMDq traffic
2298 * replicated on the Main VSI.
2299 */
2300 if (pf->cur_promisc != cur_promisc) {
2301 pf->cur_promisc = cur_promisc;
2302 if (cur_promisc)
2303 aq_ret =
2304 i40e_aq_set_default_vsi(hw,
2305 vsi->seid,
2306 NULL);
2307 else
2308 aq_ret =
2309 i40e_aq_clear_default_vsi(hw,
2310 vsi->seid,
2311 NULL);
2312 if (aq_ret) {
2313 retval = i40e_aq_rc_to_posix(aq_ret,
2314 hw->aq.asq_last_status);
2315 dev_info(&pf->pdev->dev,
2316 "Set default VSI failed on %s, err %s, aq_err %s\n",
2317 vsi_name,
2318 i40e_stat_str(hw, aq_ret),
2319 i40e_aq_str(hw,
2320 hw->aq.asq_last_status));
2321 }
2322 }
2323 } else {
2324 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2325 hw,
2326 vsi->seid,
2327 cur_promisc, NULL,
2328 true);
2329 if (aq_ret) {
2330 retval =
2331 i40e_aq_rc_to_posix(aq_ret,
2332 hw->aq.asq_last_status);
2333 dev_info(&pf->pdev->dev,
2334 "set unicast promisc failed on %s, err %s, aq_err %s\n",
2335 vsi_name,
2336 i40e_stat_str(hw, aq_ret),
2337 i40e_aq_str(hw,
2338 hw->aq.asq_last_status));
2339 }
2340 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2341 hw,
2342 vsi->seid,
2343 cur_promisc, NULL);
2344 if (aq_ret) {
2345 retval =
2346 i40e_aq_rc_to_posix(aq_ret,
2347 hw->aq.asq_last_status);
2348 dev_info(&pf->pdev->dev,
2349 "set multicast promisc failed on %s, err %s, aq_err %s\n",
2350 vsi_name,
2351 i40e_stat_str(hw, aq_ret),
2352 i40e_aq_str(hw,
2353 hw->aq.asq_last_status));
2354 }
2355 }
2356 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2357 vsi->seid,
2358 cur_promisc, NULL);
2359 if (aq_ret) {
2360 retval = i40e_aq_rc_to_posix(aq_ret,
2361 pf->hw.aq.asq_last_status);
2362 dev_info(&pf->pdev->dev,
2363 "set brdcast promisc failed, err %s, aq_err %s\n",
2364 i40e_stat_str(hw, aq_ret),
2365 i40e_aq_str(hw,
2366 hw->aq.asq_last_status));
2367 }
2368 }
2369 out:
2370 /* if something went wrong then set the changed flag so we try again */
2371 if (retval)
2372 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2373
2374 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2375 return retval;
2376
2377 err_no_memory:
2378 /* Restore elements on the temporary add and delete lists */
2379 spin_lock_bh(&vsi->mac_filter_hash_lock);
2380 err_no_memory_locked:
2381 i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2382 i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2383 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2384
2385 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2386 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2387 return -ENOMEM;
2388 }
2389
2390 /**
2391 * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2392 * @pf: board private structure
2393 **/
2394 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2395 {
2396 int v;
2397
2398 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2399 return;
2400 pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2401
2402 for (v = 0; v < pf->num_alloc_vsi; v++) {
2403 if (pf->vsi[v] &&
2404 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2405 int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2406
2407 if (ret) {
2408 /* come back and try again later */
2409 pf->flags |= I40E_FLAG_FILTER_SYNC;
2410 break;
2411 }
2412 }
2413 }
2414 }
2415
2416 /**
2417 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2418 * @vsi: the vsi
2419 **/
2420 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2421 {
2422 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2423 return I40E_RXBUFFER_2048;
2424 else
2425 return I40E_RXBUFFER_3072;
2426 }
2427
2428 /**
2429 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2430 * @netdev: network interface device structure
2431 * @new_mtu: new value for maximum frame size
2432 *
2433 * Returns 0 on success, negative on failure
2434 **/
2435 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2436 {
2437 struct i40e_netdev_priv *np = netdev_priv(netdev);
2438 struct i40e_vsi *vsi = np->vsi;
2439 struct i40e_pf *pf = vsi->back;
2440
2441 if (i40e_enabled_xdp_vsi(vsi)) {
2442 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2443
2444 if (frame_size > i40e_max_xdp_frame_size(vsi))
2445 return -EINVAL;
2446 }
2447
2448 netdev_info(netdev, "changing MTU from %d to %d\n",
2449 netdev->mtu, new_mtu);
2450 netdev->mtu = new_mtu;
2451 if (netif_running(netdev))
2452 i40e_vsi_reinit_locked(vsi);
2453 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2454 I40E_FLAG_CLIENT_L2_CHANGE);
2455 return 0;
2456 }
2457
2458 /**
2459 * i40e_ioctl - Access the hwtstamp interface
2460 * @netdev: network interface device structure
2461 * @ifr: interface request data
2462 * @cmd: ioctl command
2463 **/
2464 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2465 {
2466 struct i40e_netdev_priv *np = netdev_priv(netdev);
2467 struct i40e_pf *pf = np->vsi->back;
2468
2469 switch (cmd) {
2470 case SIOCGHWTSTAMP:
2471 return i40e_ptp_get_ts_config(pf, ifr);
2472 case SIOCSHWTSTAMP:
2473 return i40e_ptp_set_ts_config(pf, ifr);
2474 default:
2475 return -EOPNOTSUPP;
2476 }
2477 }
2478
2479 /**
2480 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2481 * @vsi: the vsi being adjusted
2482 **/
2483 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2484 {
2485 struct i40e_vsi_context ctxt;
2486 i40e_status ret;
2487
2488 if ((vsi->info.valid_sections &
2489 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2490 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2491 return; /* already enabled */
2492
2493 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2494 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2495 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2496
2497 ctxt.seid = vsi->seid;
2498 ctxt.info = vsi->info;
2499 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2500 if (ret) {
2501 dev_info(&vsi->back->pdev->dev,
2502 "update vlan stripping failed, err %s aq_err %s\n",
2503 i40e_stat_str(&vsi->back->hw, ret),
2504 i40e_aq_str(&vsi->back->hw,
2505 vsi->back->hw.aq.asq_last_status));
2506 }
2507 }
2508
2509 /**
2510 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2511 * @vsi: the vsi being adjusted
2512 **/
2513 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2514 {
2515 struct i40e_vsi_context ctxt;
2516 i40e_status ret;
2517
2518 if ((vsi->info.valid_sections &
2519 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2520 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2521 I40E_AQ_VSI_PVLAN_EMOD_MASK))
2522 return; /* already disabled */
2523
2524 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2525 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2526 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2527
2528 ctxt.seid = vsi->seid;
2529 ctxt.info = vsi->info;
2530 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2531 if (ret) {
2532 dev_info(&vsi->back->pdev->dev,
2533 "update vlan stripping failed, err %s aq_err %s\n",
2534 i40e_stat_str(&vsi->back->hw, ret),
2535 i40e_aq_str(&vsi->back->hw,
2536 vsi->back->hw.aq.asq_last_status));
2537 }
2538 }
2539
2540 /**
2541 * i40e_vlan_rx_register - Setup or shutdown vlan offload
2542 * @netdev: network interface to be adjusted
2543 * @features: netdev features to test if VLAN offload is enabled or not
2544 **/
2545 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2546 {
2547 struct i40e_netdev_priv *np = netdev_priv(netdev);
2548 struct i40e_vsi *vsi = np->vsi;
2549
2550 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2551 i40e_vlan_stripping_enable(vsi);
2552 else
2553 i40e_vlan_stripping_disable(vsi);
2554 }
2555
2556 /**
2557 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2558 * @vsi: the vsi being configured
2559 * @vid: vlan id to be added (0 = untagged only , -1 = any)
2560 *
2561 * This is a helper function for adding a new MAC/VLAN filter with the
2562 * specified VLAN for each existing MAC address already in the hash table.
2563 * This function does *not* perform any accounting to update filters based on
2564 * VLAN mode.
2565 *
2566 * NOTE: this function expects to be called while under the
2567 * mac_filter_hash_lock
2568 **/
2569 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2570 {
2571 struct i40e_mac_filter *f, *add_f;
2572 struct hlist_node *h;
2573 int bkt;
2574
2575 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2576 if (f->state == I40E_FILTER_REMOVE)
2577 continue;
2578 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2579 if (!add_f) {
2580 dev_info(&vsi->back->pdev->dev,
2581 "Could not add vlan filter %d for %pM\n",
2582 vid, f->macaddr);
2583 return -ENOMEM;
2584 }
2585 }
2586
2587 return 0;
2588 }
2589
2590 /**
2591 * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2592 * @vsi: the VSI being configured
2593 * @vid: VLAN id to be added
2594 **/
2595 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2596 {
2597 int err;
2598
2599 if (!vid || vsi->info.pvid)
2600 return -EINVAL;
2601
2602 /* Locked once because all functions invoked below iterates list*/
2603 spin_lock_bh(&vsi->mac_filter_hash_lock);
2604 err = i40e_add_vlan_all_mac(vsi, vid);
2605 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2606 if (err)
2607 return err;
2608
2609 /* schedule our worker thread which will take care of
2610 * applying the new filter changes
2611 */
2612 i40e_service_event_schedule(vsi->back);
2613 return 0;
2614 }
2615
2616 /**
2617 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2618 * @vsi: the vsi being configured
2619 * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2620 *
2621 * This function should be used to remove all VLAN filters which match the
2622 * given VID. It does not schedule the service event and does not take the
2623 * mac_filter_hash_lock so it may be combined with other operations under
2624 * a single invocation of the mac_filter_hash_lock.
2625 *
2626 * NOTE: this function expects to be called while under the
2627 * mac_filter_hash_lock
2628 */
2629 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2630 {
2631 struct i40e_mac_filter *f;
2632 struct hlist_node *h;
2633 int bkt;
2634
2635 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2636 if (f->vlan == vid)
2637 __i40e_del_filter(vsi, f);
2638 }
2639 }
2640
2641 /**
2642 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2643 * @vsi: the VSI being configured
2644 * @vid: VLAN id to be removed
2645 **/
2646 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2647 {
2648 if (!vid || vsi->info.pvid)
2649 return;
2650
2651 spin_lock_bh(&vsi->mac_filter_hash_lock);
2652 i40e_rm_vlan_all_mac(vsi, vid);
2653 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2654
2655 /* schedule our worker thread which will take care of
2656 * applying the new filter changes
2657 */
2658 i40e_service_event_schedule(vsi->back);
2659 }
2660
2661 /**
2662 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2663 * @netdev: network interface to be adjusted
2664 * @vid: vlan id to be added
2665 *
2666 * net_device_ops implementation for adding vlan ids
2667 **/
2668 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2669 __always_unused __be16 proto, u16 vid)
2670 {
2671 struct i40e_netdev_priv *np = netdev_priv(netdev);
2672 struct i40e_vsi *vsi = np->vsi;
2673 int ret = 0;
2674
2675 if (vid >= VLAN_N_VID)
2676 return -EINVAL;
2677
2678 /* If the network stack called us with vid = 0 then
2679 * it is asking to receive priority tagged packets with
2680 * vlan id 0. Our HW receives them by default when configured
2681 * to receive untagged packets so there is no need to add an
2682 * extra filter for vlan 0 tagged packets.
2683 */
2684 if (vid)
2685 ret = i40e_vsi_add_vlan(vsi, vid);
2686
2687 if (!ret)
2688 set_bit(vid, vsi->active_vlans);
2689
2690 return ret;
2691 }
2692
2693 /**
2694 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2695 * @netdev: network interface to be adjusted
2696 * @vid: vlan id to be removed
2697 *
2698 * net_device_ops implementation for removing vlan ids
2699 **/
2700 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2701 __always_unused __be16 proto, u16 vid)
2702 {
2703 struct i40e_netdev_priv *np = netdev_priv(netdev);
2704 struct i40e_vsi *vsi = np->vsi;
2705
2706 /* return code is ignored as there is nothing a user
2707 * can do about failure to remove and a log message was
2708 * already printed from the other function
2709 */
2710 i40e_vsi_kill_vlan(vsi, vid);
2711
2712 clear_bit(vid, vsi->active_vlans);
2713
2714 return 0;
2715 }
2716
2717 /**
2718 * i40e_macaddr_init - explicitly write the mac address filters
2719 *
2720 * @vsi: pointer to the vsi
2721 * @macaddr: the MAC address
2722 *
2723 * This is needed when the macaddr has been obtained by other
2724 * means than the default, e.g., from Open Firmware or IDPROM.
2725 * Returns 0 on success, negative on failure
2726 **/
2727 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr)
2728 {
2729 int ret;
2730 struct i40e_aqc_add_macvlan_element_data element;
2731
2732 ret = i40e_aq_mac_address_write(&vsi->back->hw,
2733 I40E_AQC_WRITE_TYPE_LAA_WOL,
2734 macaddr, NULL);
2735 if (ret) {
2736 dev_info(&vsi->back->pdev->dev,
2737 "Addr change for VSI failed: %d\n", ret);
2738 return -EADDRNOTAVAIL;
2739 }
2740
2741 memset(&element, 0, sizeof(element));
2742 ether_addr_copy(element.mac_addr, macaddr);
2743 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
2744 ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL);
2745 if (ret) {
2746 dev_info(&vsi->back->pdev->dev,
2747 "add filter failed err %s aq_err %s\n",
2748 i40e_stat_str(&vsi->back->hw, ret),
2749 i40e_aq_str(&vsi->back->hw,
2750 vsi->back->hw.aq.asq_last_status));
2751 }
2752 return ret;
2753 }
2754
2755 /**
2756 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2757 * @vsi: the vsi being brought back up
2758 **/
2759 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2760 {
2761 u16 vid;
2762
2763 if (!vsi->netdev)
2764 return;
2765
2766 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2767
2768 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2769 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2770 vid);
2771 }
2772
2773 /**
2774 * i40e_vsi_add_pvid - Add pvid for the VSI
2775 * @vsi: the vsi being adjusted
2776 * @vid: the vlan id to set as a PVID
2777 **/
2778 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2779 {
2780 struct i40e_vsi_context ctxt;
2781 i40e_status ret;
2782
2783 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2784 vsi->info.pvid = cpu_to_le16(vid);
2785 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2786 I40E_AQ_VSI_PVLAN_INSERT_PVID |
2787 I40E_AQ_VSI_PVLAN_EMOD_STR;
2788
2789 ctxt.seid = vsi->seid;
2790 ctxt.info = vsi->info;
2791 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2792 if (ret) {
2793 dev_info(&vsi->back->pdev->dev,
2794 "add pvid failed, err %s aq_err %s\n",
2795 i40e_stat_str(&vsi->back->hw, ret),
2796 i40e_aq_str(&vsi->back->hw,
2797 vsi->back->hw.aq.asq_last_status));
2798 return -ENOENT;
2799 }
2800
2801 return 0;
2802 }
2803
2804 /**
2805 * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2806 * @vsi: the vsi being adjusted
2807 *
2808 * Just use the vlan_rx_register() service to put it back to normal
2809 **/
2810 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2811 {
2812 i40e_vlan_stripping_disable(vsi);
2813
2814 vsi->info.pvid = 0;
2815 }
2816
2817 /**
2818 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2819 * @vsi: ptr to the VSI
2820 *
2821 * If this function returns with an error, then it's possible one or
2822 * more of the rings is populated (while the rest are not). It is the
2823 * callers duty to clean those orphaned rings.
2824 *
2825 * Return 0 on success, negative on failure
2826 **/
2827 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2828 {
2829 int i, err = 0;
2830
2831 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2832 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2833
2834 if (!i40e_enabled_xdp_vsi(vsi))
2835 return err;
2836
2837 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2838 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
2839
2840 return err;
2841 }
2842
2843 /**
2844 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2845 * @vsi: ptr to the VSI
2846 *
2847 * Free VSI's transmit software resources
2848 **/
2849 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2850 {
2851 int i;
2852
2853 if (vsi->tx_rings) {
2854 for (i = 0; i < vsi->num_queue_pairs; i++)
2855 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2856 i40e_free_tx_resources(vsi->tx_rings[i]);
2857 }
2858
2859 if (vsi->xdp_rings) {
2860 for (i = 0; i < vsi->num_queue_pairs; i++)
2861 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
2862 i40e_free_tx_resources(vsi->xdp_rings[i]);
2863 }
2864 }
2865
2866 /**
2867 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2868 * @vsi: ptr to the VSI
2869 *
2870 * If this function returns with an error, then it's possible one or
2871 * more of the rings is populated (while the rest are not). It is the
2872 * callers duty to clean those orphaned rings.
2873 *
2874 * Return 0 on success, negative on failure
2875 **/
2876 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2877 {
2878 int i, err = 0;
2879
2880 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2881 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2882 return err;
2883 }
2884
2885 /**
2886 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2887 * @vsi: ptr to the VSI
2888 *
2889 * Free all receive software resources
2890 **/
2891 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2892 {
2893 int i;
2894
2895 if (!vsi->rx_rings)
2896 return;
2897
2898 for (i = 0; i < vsi->num_queue_pairs; i++)
2899 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2900 i40e_free_rx_resources(vsi->rx_rings[i]);
2901 }
2902
2903 /**
2904 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2905 * @ring: The Tx ring to configure
2906 *
2907 * This enables/disables XPS for a given Tx descriptor ring
2908 * based on the TCs enabled for the VSI that ring belongs to.
2909 **/
2910 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2911 {
2912 struct i40e_vsi *vsi = ring->vsi;
2913 cpumask_var_t mask;
2914
2915 if (!ring->q_vector || !ring->netdev)
2916 return;
2917
2918 /* Single TC mode enable XPS */
2919 if (vsi->tc_config.numtc <= 1) {
2920 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2921 netif_set_xps_queue(ring->netdev,
2922 &ring->q_vector->affinity_mask,
2923 ring->queue_index);
2924 } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
2925 /* Disable XPS to allow selection based on TC */
2926 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits);
2927 netif_set_xps_queue(ring->netdev, mask, ring->queue_index);
2928 free_cpumask_var(mask);
2929 }
2930
2931 /* schedule our worker thread which will take care of
2932 * applying the new filter changes
2933 */
2934 i40e_service_event_schedule(vsi->back);
2935 }
2936
2937 /**
2938 * i40e_configure_tx_ring - Configure a transmit ring context and rest
2939 * @ring: The Tx ring to configure
2940 *
2941 * Configure the Tx descriptor ring in the HMC context.
2942 **/
2943 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2944 {
2945 struct i40e_vsi *vsi = ring->vsi;
2946 u16 pf_q = vsi->base_queue + ring->queue_index;
2947 struct i40e_hw *hw = &vsi->back->hw;
2948 struct i40e_hmc_obj_txq tx_ctx;
2949 i40e_status err = 0;
2950 u32 qtx_ctl = 0;
2951
2952 /* some ATR related tx ring init */
2953 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2954 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2955 ring->atr_count = 0;
2956 } else {
2957 ring->atr_sample_rate = 0;
2958 }
2959
2960 /* configure XPS */
2961 i40e_config_xps_tx_ring(ring);
2962
2963 /* clear the context structure first */
2964 memset(&tx_ctx, 0, sizeof(tx_ctx));
2965
2966 tx_ctx.new_context = 1;
2967 tx_ctx.base = (ring->dma / 128);
2968 tx_ctx.qlen = ring->count;
2969 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2970 I40E_FLAG_FD_ATR_ENABLED));
2971 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2972 /* FDIR VSI tx ring can still use RS bit and writebacks */
2973 if (vsi->type != I40E_VSI_FDIR)
2974 tx_ctx.head_wb_ena = 1;
2975 tx_ctx.head_wb_addr = ring->dma +
2976 (ring->count * sizeof(struct i40e_tx_desc));
2977
2978 /* As part of VSI creation/update, FW allocates certain
2979 * Tx arbitration queue sets for each TC enabled for
2980 * the VSI. The FW returns the handles to these queue
2981 * sets as part of the response buffer to Add VSI,
2982 * Update VSI, etc. AQ commands. It is expected that
2983 * these queue set handles be associated with the Tx
2984 * queues by the driver as part of the TX queue context
2985 * initialization. This has to be done regardless of
2986 * DCB as by default everything is mapped to TC0.
2987 */
2988 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2989 tx_ctx.rdylist_act = 0;
2990
2991 /* clear the context in the HMC */
2992 err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2993 if (err) {
2994 dev_info(&vsi->back->pdev->dev,
2995 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2996 ring->queue_index, pf_q, err);
2997 return -ENOMEM;
2998 }
2999
3000 /* set the context in the HMC */
3001 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3002 if (err) {
3003 dev_info(&vsi->back->pdev->dev,
3004 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3005 ring->queue_index, pf_q, err);
3006 return -ENOMEM;
3007 }
3008
3009 /* Now associate this queue with this PCI function */
3010 if (vsi->type == I40E_VSI_VMDQ2) {
3011 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3012 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3013 I40E_QTX_CTL_VFVM_INDX_MASK;
3014 } else {
3015 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3016 }
3017
3018 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3019 I40E_QTX_CTL_PF_INDX_MASK);
3020 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3021 i40e_flush(hw);
3022
3023 /* cache tail off for easier writes later */
3024 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3025
3026 return 0;
3027 }
3028
3029 /**
3030 * i40e_configure_rx_ring - Configure a receive ring context
3031 * @ring: The Rx ring to configure
3032 *
3033 * Configure the Rx descriptor ring in the HMC context.
3034 **/
3035 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3036 {
3037 struct i40e_vsi *vsi = ring->vsi;
3038 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3039 u16 pf_q = vsi->base_queue + ring->queue_index;
3040 struct i40e_hw *hw = &vsi->back->hw;
3041 struct i40e_hmc_obj_rxq rx_ctx;
3042 i40e_status err = 0;
3043
3044 ring->state = 0;
3045
3046 /* clear the context structure first */
3047 memset(&rx_ctx, 0, sizeof(rx_ctx));
3048
3049 ring->rx_buf_len = vsi->rx_buf_len;
3050
3051 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3052 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3053
3054 rx_ctx.base = (ring->dma / 128);
3055 rx_ctx.qlen = ring->count;
3056
3057 /* use 32 byte descriptors */
3058 rx_ctx.dsize = 1;
3059
3060 /* descriptor type is always zero
3061 * rx_ctx.dtype = 0;
3062 */
3063 rx_ctx.hsplit_0 = 0;
3064
3065 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3066 if (hw->revision_id == 0)
3067 rx_ctx.lrxqthresh = 0;
3068 else
3069 rx_ctx.lrxqthresh = 2;
3070 rx_ctx.crcstrip = 1;
3071 rx_ctx.l2tsel = 1;
3072 /* this controls whether VLAN is stripped from inner headers */
3073 rx_ctx.showiv = 0;
3074 /* set the prefena field to 1 because the manual says to */
3075 rx_ctx.prefena = 1;
3076
3077 /* clear the context in the HMC */
3078 err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3079 if (err) {
3080 dev_info(&vsi->back->pdev->dev,
3081 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3082 ring->queue_index, pf_q, err);
3083 return -ENOMEM;
3084 }
3085
3086 /* set the context in the HMC */
3087 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3088 if (err) {
3089 dev_info(&vsi->back->pdev->dev,
3090 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3091 ring->queue_index, pf_q, err);
3092 return -ENOMEM;
3093 }
3094
3095 /* configure Rx buffer alignment */
3096 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3097 clear_ring_build_skb_enabled(ring);
3098 else
3099 set_ring_build_skb_enabled(ring);
3100
3101 /* cache tail for quicker writes, and clear the reg before use */
3102 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3103 writel(0, ring->tail);
3104
3105 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3106
3107 return 0;
3108 }
3109
3110 /**
3111 * i40e_vsi_configure_tx - Configure the VSI for Tx
3112 * @vsi: VSI structure describing this set of rings and resources
3113 *
3114 * Configure the Tx VSI for operation.
3115 **/
3116 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3117 {
3118 int err = 0;
3119 u16 i;
3120
3121 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3122 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3123
3124 if (!i40e_enabled_xdp_vsi(vsi))
3125 return err;
3126
3127 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3128 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3129
3130 return err;
3131 }
3132
3133 /**
3134 * i40e_vsi_configure_rx - Configure the VSI for Rx
3135 * @vsi: the VSI being configured
3136 *
3137 * Configure the Rx VSI for operation.
3138 **/
3139 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3140 {
3141 int err = 0;
3142 u16 i;
3143
3144 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3145 vsi->max_frame = I40E_MAX_RXBUFFER;
3146 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3147 #if (PAGE_SIZE < 8192)
3148 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3149 (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3150 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3151 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3152 #endif
3153 } else {
3154 vsi->max_frame = I40E_MAX_RXBUFFER;
3155 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3156 I40E_RXBUFFER_2048;
3157 }
3158
3159 /* set up individual rings */
3160 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3161 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3162
3163 return err;
3164 }
3165
3166 /**
3167 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3168 * @vsi: ptr to the VSI
3169 **/
3170 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3171 {
3172 struct i40e_ring *tx_ring, *rx_ring;
3173 u16 qoffset, qcount;
3174 int i, n;
3175
3176 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3177 /* Reset the TC information */
3178 for (i = 0; i < vsi->num_queue_pairs; i++) {
3179 rx_ring = vsi->rx_rings[i];
3180 tx_ring = vsi->tx_rings[i];
3181 rx_ring->dcb_tc = 0;
3182 tx_ring->dcb_tc = 0;
3183 }
3184 }
3185
3186 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3187 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3188 continue;
3189
3190 qoffset = vsi->tc_config.tc_info[n].qoffset;
3191 qcount = vsi->tc_config.tc_info[n].qcount;
3192 for (i = qoffset; i < (qoffset + qcount); i++) {
3193 rx_ring = vsi->rx_rings[i];
3194 tx_ring = vsi->tx_rings[i];
3195 rx_ring->dcb_tc = n;
3196 tx_ring->dcb_tc = n;
3197 }
3198 }
3199 }
3200
3201 /**
3202 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3203 * @vsi: ptr to the VSI
3204 **/
3205 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3206 {
3207 struct i40e_pf *pf = vsi->back;
3208 int err;
3209
3210 if (vsi->netdev)
3211 i40e_set_rx_mode(vsi->netdev);
3212
3213 if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
3214 err = i40e_macaddr_init(vsi, pf->hw.mac.addr);
3215 if (err) {
3216 dev_warn(&pf->pdev->dev,
3217 "could not set up macaddr; err %d\n", err);
3218 }
3219 }
3220 }
3221
3222 /**
3223 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3224 * @vsi: Pointer to the targeted VSI
3225 *
3226 * This function replays the hlist on the hw where all the SB Flow Director
3227 * filters were saved.
3228 **/
3229 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3230 {
3231 struct i40e_fdir_filter *filter;
3232 struct i40e_pf *pf = vsi->back;
3233 struct hlist_node *node;
3234
3235 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3236 return;
3237
3238 /* Reset FDir counters as we're replaying all existing filters */
3239 pf->fd_tcp4_filter_cnt = 0;
3240 pf->fd_udp4_filter_cnt = 0;
3241 pf->fd_sctp4_filter_cnt = 0;
3242 pf->fd_ip4_filter_cnt = 0;
3243
3244 hlist_for_each_entry_safe(filter, node,
3245 &pf->fdir_filter_list, fdir_node) {
3246 i40e_add_del_fdir(vsi, filter, true);
3247 }
3248 }
3249
3250 /**
3251 * i40e_vsi_configure - Set up the VSI for action
3252 * @vsi: the VSI being configured
3253 **/
3254 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3255 {
3256 int err;
3257
3258 i40e_set_vsi_rx_mode(vsi);
3259 i40e_restore_vlan(vsi);
3260 i40e_vsi_config_dcb_rings(vsi);
3261 err = i40e_vsi_configure_tx(vsi);
3262 if (!err)
3263 err = i40e_vsi_configure_rx(vsi);
3264
3265 return err;
3266 }
3267
3268 /**
3269 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3270 * @vsi: the VSI being configured
3271 **/
3272 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3273 {
3274 bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3275 struct i40e_pf *pf = vsi->back;
3276 struct i40e_hw *hw = &pf->hw;
3277 u16 vector;
3278 int i, q;
3279 u32 qp;
3280
3281 /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3282 * and PFINT_LNKLSTn registers, e.g.:
3283 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
3284 */
3285 qp = vsi->base_queue;
3286 vector = vsi->base_vector;
3287 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3288 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3289
3290 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3291 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3292 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3293 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3294 q_vector->rx.itr);
3295 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3296 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3297 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3298 q_vector->tx.itr);
3299 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3300 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3301
3302 /* Linked list for the queuepairs assigned to this vector */
3303 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3304 for (q = 0; q < q_vector->num_ringpairs; q++) {
3305 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3306 u32 val;
3307
3308 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3309 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3310 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3311 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3312 (I40E_QUEUE_TYPE_TX <<
3313 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3314
3315 wr32(hw, I40E_QINT_RQCTL(qp), val);
3316
3317 if (has_xdp) {
3318 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3319 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3320 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3321 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3322 (I40E_QUEUE_TYPE_TX <<
3323 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3324
3325 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3326 }
3327
3328 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3329 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3330 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3331 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3332 (I40E_QUEUE_TYPE_RX <<
3333 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3334
3335 /* Terminate the linked list */
3336 if (q == (q_vector->num_ringpairs - 1))
3337 val |= (I40E_QUEUE_END_OF_LIST <<
3338 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3339
3340 wr32(hw, I40E_QINT_TQCTL(qp), val);
3341 qp++;
3342 }
3343 }
3344
3345 i40e_flush(hw);
3346 }
3347
3348 /**
3349 * i40e_enable_misc_int_causes - enable the non-queue interrupts
3350 * @hw: ptr to the hardware info
3351 **/
3352 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3353 {
3354 struct i40e_hw *hw = &pf->hw;
3355 u32 val;
3356
3357 /* clear things first */
3358 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
3359 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
3360
3361 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3362 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3363 I40E_PFINT_ICR0_ENA_GRST_MASK |
3364 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3365 I40E_PFINT_ICR0_ENA_GPIO_MASK |
3366 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3367 I40E_PFINT_ICR0_ENA_VFLR_MASK |
3368 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3369
3370 if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3371 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3372
3373 if (pf->flags & I40E_FLAG_PTP)
3374 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3375
3376 wr32(hw, I40E_PFINT_ICR0_ENA, val);
3377
3378 /* SW_ITR_IDX = 0, but don't change INTENA */
3379 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3380 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3381
3382 /* OTHER_ITR_IDX = 0 */
3383 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3384 }
3385
3386 /**
3387 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3388 * @vsi: the VSI being configured
3389 **/
3390 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3391 {
3392 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3393 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3394 struct i40e_pf *pf = vsi->back;
3395 struct i40e_hw *hw = &pf->hw;
3396 u32 val;
3397
3398 /* set the ITR configuration */
3399 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3400 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3401 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3402 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3403 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3404 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3405 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3406
3407 i40e_enable_misc_int_causes(pf);
3408
3409 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3410 wr32(hw, I40E_PFINT_LNKLST0, 0);
3411
3412 /* Associate the queue pair to the vector and enable the queue int */
3413 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3414 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3415 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3416 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3417
3418 wr32(hw, I40E_QINT_RQCTL(0), val);
3419
3420 if (i40e_enabled_xdp_vsi(vsi)) {
3421 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3422 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3423 (I40E_QUEUE_TYPE_TX
3424 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3425
3426 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3427 }
3428
3429 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3430 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3431 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3432
3433 wr32(hw, I40E_QINT_TQCTL(0), val);
3434 i40e_flush(hw);
3435 }
3436
3437 /**
3438 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3439 * @pf: board private structure
3440 **/
3441 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3442 {
3443 struct i40e_hw *hw = &pf->hw;
3444
3445 wr32(hw, I40E_PFINT_DYN_CTL0,
3446 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3447 i40e_flush(hw);
3448 }
3449
3450 /**
3451 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3452 * @pf: board private structure
3453 * @clearpba: true when all pending interrupt events should be cleared
3454 **/
3455 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3456 {
3457 struct i40e_hw *hw = &pf->hw;
3458 u32 val;
3459
3460 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3461 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3462 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3463
3464 wr32(hw, I40E_PFINT_DYN_CTL0, val);
3465 i40e_flush(hw);
3466 }
3467
3468 /**
3469 * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3470 * @irq: interrupt number
3471 * @data: pointer to a q_vector
3472 **/
3473 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3474 {
3475 struct i40e_q_vector *q_vector = data;
3476
3477 if (!q_vector->tx.ring && !q_vector->rx.ring)
3478 return IRQ_HANDLED;
3479
3480 napi_schedule_irqoff(&q_vector->napi);
3481
3482 return IRQ_HANDLED;
3483 }
3484
3485 /**
3486 * i40e_irq_affinity_notify - Callback for affinity changes
3487 * @notify: context as to what irq was changed
3488 * @mask: the new affinity mask
3489 *
3490 * This is a callback function used by the irq_set_affinity_notifier function
3491 * so that we may register to receive changes to the irq affinity masks.
3492 **/
3493 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3494 const cpumask_t *mask)
3495 {
3496 struct i40e_q_vector *q_vector =
3497 container_of(notify, struct i40e_q_vector, affinity_notify);
3498
3499 q_vector->affinity_mask = *mask;
3500 }
3501
3502 /**
3503 * i40e_irq_affinity_release - Callback for affinity notifier release
3504 * @ref: internal core kernel usage
3505 *
3506 * This is a callback function used by the irq_set_affinity_notifier function
3507 * to inform the current notification subscriber that they will no longer
3508 * receive notifications.
3509 **/
3510 static void i40e_irq_affinity_release(struct kref *ref) {}
3511
3512 /**
3513 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3514 * @vsi: the VSI being configured
3515 * @basename: name for the vector
3516 *
3517 * Allocates MSI-X vectors and requests interrupts from the kernel.
3518 **/
3519 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3520 {
3521 int q_vectors = vsi->num_q_vectors;
3522 struct i40e_pf *pf = vsi->back;
3523 int base = vsi->base_vector;
3524 int rx_int_idx = 0;
3525 int tx_int_idx = 0;
3526 int vector, err;
3527 int irq_num;
3528
3529 for (vector = 0; vector < q_vectors; vector++) {
3530 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3531
3532 irq_num = pf->msix_entries[base + vector].vector;
3533
3534 if (q_vector->tx.ring && q_vector->rx.ring) {
3535 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3536 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3537 tx_int_idx++;
3538 } else if (q_vector->rx.ring) {
3539 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3540 "%s-%s-%d", basename, "rx", rx_int_idx++);
3541 } else if (q_vector->tx.ring) {
3542 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3543 "%s-%s-%d", basename, "tx", tx_int_idx++);
3544 } else {
3545 /* skip this unused q_vector */
3546 continue;
3547 }
3548 err = request_irq(irq_num,
3549 vsi->irq_handler,
3550 0,
3551 q_vector->name,
3552 q_vector);
3553 if (err) {
3554 dev_info(&pf->pdev->dev,
3555 "MSIX request_irq failed, error: %d\n", err);
3556 goto free_queue_irqs;
3557 }
3558
3559 /* register for affinity change notifications */
3560 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3561 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3562 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3563 /* assign the mask for this irq */
3564 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
3565 }
3566
3567 vsi->irqs_ready = true;
3568 return 0;
3569
3570 free_queue_irqs:
3571 while (vector) {
3572 vector--;
3573 irq_num = pf->msix_entries[base + vector].vector;
3574 irq_set_affinity_notifier(irq_num, NULL);
3575 irq_set_affinity_hint(irq_num, NULL);
3576 free_irq(irq_num, &vsi->q_vectors[vector]);
3577 }
3578 return err;
3579 }
3580
3581 /**
3582 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3583 * @vsi: the VSI being un-configured
3584 **/
3585 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3586 {
3587 struct i40e_pf *pf = vsi->back;
3588 struct i40e_hw *hw = &pf->hw;
3589 int base = vsi->base_vector;
3590 int i;
3591
3592 for (i = 0; i < vsi->num_queue_pairs; i++) {
3593 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
3594 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
3595 if (!i40e_enabled_xdp_vsi(vsi))
3596 continue;
3597 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3598 }
3599
3600 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3601 for (i = vsi->base_vector;
3602 i < (vsi->num_q_vectors + vsi->base_vector); i++)
3603 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3604
3605 i40e_flush(hw);
3606 for (i = 0; i < vsi->num_q_vectors; i++)
3607 synchronize_irq(pf->msix_entries[i + base].vector);
3608 } else {
3609 /* Legacy and MSI mode - this stops all interrupt handling */
3610 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3611 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3612 i40e_flush(hw);
3613 synchronize_irq(pf->pdev->irq);
3614 }
3615 }
3616
3617 /**
3618 * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3619 * @vsi: the VSI being configured
3620 **/
3621 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3622 {
3623 struct i40e_pf *pf = vsi->back;
3624 int i;
3625
3626 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3627 for (i = 0; i < vsi->num_q_vectors; i++)
3628 i40e_irq_dynamic_enable(vsi, i);
3629 } else {
3630 i40e_irq_dynamic_enable_icr0(pf, true);
3631 }
3632
3633 i40e_flush(&pf->hw);
3634 return 0;
3635 }
3636
3637 /**
3638 * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3639 * @pf: board private structure
3640 **/
3641 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3642 {
3643 /* Disable ICR 0 */
3644 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3645 i40e_flush(&pf->hw);
3646 }
3647
3648 /**
3649 * i40e_intr - MSI/Legacy and non-queue interrupt handler
3650 * @irq: interrupt number
3651 * @data: pointer to a q_vector
3652 *
3653 * This is the handler used for all MSI/Legacy interrupts, and deals
3654 * with both queue and non-queue interrupts. This is also used in
3655 * MSIX mode to handle the non-queue interrupts.
3656 **/
3657 static irqreturn_t i40e_intr(int irq, void *data)
3658 {
3659 struct i40e_pf *pf = (struct i40e_pf *)data;
3660 struct i40e_hw *hw = &pf->hw;
3661 irqreturn_t ret = IRQ_NONE;
3662 u32 icr0, icr0_remaining;
3663 u32 val, ena_mask;
3664
3665 icr0 = rd32(hw, I40E_PFINT_ICR0);
3666 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3667
3668 /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3669 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3670 goto enable_intr;
3671
3672 /* if interrupt but no bits showing, must be SWINT */
3673 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3674 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3675 pf->sw_int_count++;
3676
3677 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3678 (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3679 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3680 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3681 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3682 }
3683
3684 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3685 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3686 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3687 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3688
3689 /* We do not have a way to disarm Queue causes while leaving
3690 * interrupt enabled for all other causes, ideally
3691 * interrupt should be disabled while we are in NAPI but
3692 * this is not a performance path and napi_schedule()
3693 * can deal with rescheduling.
3694 */
3695 if (!test_bit(__I40E_DOWN, pf->state))
3696 napi_schedule_irqoff(&q_vector->napi);
3697 }
3698
3699 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3700 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3701 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3702 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3703 }
3704
3705 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3706 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3707 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3708 }
3709
3710 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3711 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3712 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3713 }
3714
3715 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3716 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3717 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3718 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3719 val = rd32(hw, I40E_GLGEN_RSTAT);
3720 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3721 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3722 if (val == I40E_RESET_CORER) {
3723 pf->corer_count++;
3724 } else if (val == I40E_RESET_GLOBR) {
3725 pf->globr_count++;
3726 } else if (val == I40E_RESET_EMPR) {
3727 pf->empr_count++;
3728 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3729 }
3730 }
3731
3732 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3733 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3734 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3735 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3736 rd32(hw, I40E_PFHMC_ERRORINFO),
3737 rd32(hw, I40E_PFHMC_ERRORDATA));
3738 }
3739
3740 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3741 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3742
3743 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3744 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3745 i40e_ptp_tx_hwtstamp(pf);
3746 }
3747 }
3748
3749 /* If a critical error is pending we have no choice but to reset the
3750 * device.
3751 * Report and mask out any remaining unexpected interrupts.
3752 */
3753 icr0_remaining = icr0 & ena_mask;
3754 if (icr0_remaining) {
3755 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3756 icr0_remaining);
3757 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3758 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3759 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3760 dev_info(&pf->pdev->dev, "device will be reset\n");
3761 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
3762 i40e_service_event_schedule(pf);
3763 }
3764 ena_mask &= ~icr0_remaining;
3765 }
3766 ret = IRQ_HANDLED;
3767
3768 enable_intr:
3769 /* re-enable interrupt causes */
3770 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3771 if (!test_bit(__I40E_DOWN, pf->state)) {
3772 i40e_service_event_schedule(pf);
3773 i40e_irq_dynamic_enable_icr0(pf, false);
3774 }
3775
3776 return ret;
3777 }
3778
3779 /**
3780 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3781 * @tx_ring: tx ring to clean
3782 * @budget: how many cleans we're allowed
3783 *
3784 * Returns true if there's any budget left (e.g. the clean is finished)
3785 **/
3786 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3787 {
3788 struct i40e_vsi *vsi = tx_ring->vsi;
3789 u16 i = tx_ring->next_to_clean;
3790 struct i40e_tx_buffer *tx_buf;
3791 struct i40e_tx_desc *tx_desc;
3792
3793 tx_buf = &tx_ring->tx_bi[i];
3794 tx_desc = I40E_TX_DESC(tx_ring, i);
3795 i -= tx_ring->count;
3796
3797 do {
3798 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3799
3800 /* if next_to_watch is not set then there is no work pending */
3801 if (!eop_desc)
3802 break;
3803
3804 /* prevent any other reads prior to eop_desc */
3805 read_barrier_depends();
3806
3807 /* if the descriptor isn't done, no work yet to do */
3808 if (!(eop_desc->cmd_type_offset_bsz &
3809 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3810 break;
3811
3812 /* clear next_to_watch to prevent false hangs */
3813 tx_buf->next_to_watch = NULL;
3814
3815 tx_desc->buffer_addr = 0;
3816 tx_desc->cmd_type_offset_bsz = 0;
3817 /* move past filter desc */
3818 tx_buf++;
3819 tx_desc++;
3820 i++;
3821 if (unlikely(!i)) {
3822 i -= tx_ring->count;
3823 tx_buf = tx_ring->tx_bi;
3824 tx_desc = I40E_TX_DESC(tx_ring, 0);
3825 }
3826 /* unmap skb header data */
3827 dma_unmap_single(tx_ring->dev,
3828 dma_unmap_addr(tx_buf, dma),
3829 dma_unmap_len(tx_buf, len),
3830 DMA_TO_DEVICE);
3831 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3832 kfree(tx_buf->raw_buf);
3833
3834 tx_buf->raw_buf = NULL;
3835 tx_buf->tx_flags = 0;
3836 tx_buf->next_to_watch = NULL;
3837 dma_unmap_len_set(tx_buf, len, 0);
3838 tx_desc->buffer_addr = 0;
3839 tx_desc->cmd_type_offset_bsz = 0;
3840
3841 /* move us past the eop_desc for start of next FD desc */
3842 tx_buf++;
3843 tx_desc++;
3844 i++;
3845 if (unlikely(!i)) {
3846 i -= tx_ring->count;
3847 tx_buf = tx_ring->tx_bi;
3848 tx_desc = I40E_TX_DESC(tx_ring, 0);
3849 }
3850
3851 /* update budget accounting */
3852 budget--;
3853 } while (likely(budget));
3854
3855 i += tx_ring->count;
3856 tx_ring->next_to_clean = i;
3857
3858 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3859 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3860
3861 return budget > 0;
3862 }
3863
3864 /**
3865 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3866 * @irq: interrupt number
3867 * @data: pointer to a q_vector
3868 **/
3869 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3870 {
3871 struct i40e_q_vector *q_vector = data;
3872 struct i40e_vsi *vsi;
3873
3874 if (!q_vector->tx.ring)
3875 return IRQ_HANDLED;
3876
3877 vsi = q_vector->tx.ring->vsi;
3878 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3879
3880 return IRQ_HANDLED;
3881 }
3882
3883 /**
3884 * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3885 * @vsi: the VSI being configured
3886 * @v_idx: vector index
3887 * @qp_idx: queue pair index
3888 **/
3889 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3890 {
3891 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3892 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3893 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3894
3895 tx_ring->q_vector = q_vector;
3896 tx_ring->next = q_vector->tx.ring;
3897 q_vector->tx.ring = tx_ring;
3898 q_vector->tx.count++;
3899
3900 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
3901 if (i40e_enabled_xdp_vsi(vsi)) {
3902 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
3903
3904 xdp_ring->q_vector = q_vector;
3905 xdp_ring->next = q_vector->tx.ring;
3906 q_vector->tx.ring = xdp_ring;
3907 q_vector->tx.count++;
3908 }
3909
3910 rx_ring->q_vector = q_vector;
3911 rx_ring->next = q_vector->rx.ring;
3912 q_vector->rx.ring = rx_ring;
3913 q_vector->rx.count++;
3914 }
3915
3916 /**
3917 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3918 * @vsi: the VSI being configured
3919 *
3920 * This function maps descriptor rings to the queue-specific vectors
3921 * we were allotted through the MSI-X enabling code. Ideally, we'd have
3922 * one vector per queue pair, but on a constrained vector budget, we
3923 * group the queue pairs as "efficiently" as possible.
3924 **/
3925 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3926 {
3927 int qp_remaining = vsi->num_queue_pairs;
3928 int q_vectors = vsi->num_q_vectors;
3929 int num_ringpairs;
3930 int v_start = 0;
3931 int qp_idx = 0;
3932
3933 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3934 * group them so there are multiple queues per vector.
3935 * It is also important to go through all the vectors available to be
3936 * sure that if we don't use all the vectors, that the remaining vectors
3937 * are cleared. This is especially important when decreasing the
3938 * number of queues in use.
3939 */
3940 for (; v_start < q_vectors; v_start++) {
3941 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3942
3943 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3944
3945 q_vector->num_ringpairs = num_ringpairs;
3946
3947 q_vector->rx.count = 0;
3948 q_vector->tx.count = 0;
3949 q_vector->rx.ring = NULL;
3950 q_vector->tx.ring = NULL;
3951
3952 while (num_ringpairs--) {
3953 i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3954 qp_idx++;
3955 qp_remaining--;
3956 }
3957 }
3958 }
3959
3960 /**
3961 * i40e_vsi_request_irq - Request IRQ from the OS
3962 * @vsi: the VSI being configured
3963 * @basename: name for the vector
3964 **/
3965 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3966 {
3967 struct i40e_pf *pf = vsi->back;
3968 int err;
3969
3970 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3971 err = i40e_vsi_request_irq_msix(vsi, basename);
3972 else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3973 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3974 pf->int_name, pf);
3975 else
3976 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3977 pf->int_name, pf);
3978
3979 if (err)
3980 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3981
3982 return err;
3983 }
3984
3985 #ifdef CONFIG_NET_POLL_CONTROLLER
3986 /**
3987 * i40e_netpoll - A Polling 'interrupt' handler
3988 * @netdev: network interface device structure
3989 *
3990 * This is used by netconsole to send skbs without having to re-enable
3991 * interrupts. It's not called while the normal interrupt routine is executing.
3992 **/
3993 static void i40e_netpoll(struct net_device *netdev)
3994 {
3995 struct i40e_netdev_priv *np = netdev_priv(netdev);
3996 struct i40e_vsi *vsi = np->vsi;
3997 struct i40e_pf *pf = vsi->back;
3998 int i;
3999
4000 /* if interface is down do nothing */
4001 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4002 return;
4003
4004 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4005 for (i = 0; i < vsi->num_q_vectors; i++)
4006 i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4007 } else {
4008 i40e_intr(pf->pdev->irq, netdev);
4009 }
4010 }
4011 #endif
4012
4013 #define I40E_QTX_ENA_WAIT_COUNT 50
4014
4015 /**
4016 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4017 * @pf: the PF being configured
4018 * @pf_q: the PF queue
4019 * @enable: enable or disable state of the queue
4020 *
4021 * This routine will wait for the given Tx queue of the PF to reach the
4022 * enabled or disabled state.
4023 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4024 * multiple retries; else will return 0 in case of success.
4025 **/
4026 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4027 {
4028 int i;
4029 u32 tx_reg;
4030
4031 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4032 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4033 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4034 break;
4035
4036 usleep_range(10, 20);
4037 }
4038 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4039 return -ETIMEDOUT;
4040
4041 return 0;
4042 }
4043
4044 /**
4045 * i40e_control_tx_q - Start or stop a particular Tx queue
4046 * @pf: the PF structure
4047 * @pf_q: the PF queue to configure
4048 * @enable: start or stop the queue
4049 *
4050 * This function enables or disables a single queue. Note that any delay
4051 * required after the operation is expected to be handled by the caller of
4052 * this function.
4053 **/
4054 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4055 {
4056 struct i40e_hw *hw = &pf->hw;
4057 u32 tx_reg;
4058 int i;
4059
4060 /* warn the TX unit of coming changes */
4061 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4062 if (!enable)
4063 usleep_range(10, 20);
4064
4065 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4066 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4067 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4068 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4069 break;
4070 usleep_range(1000, 2000);
4071 }
4072
4073 /* Skip if the queue is already in the requested state */
4074 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4075 return;
4076
4077 /* turn on/off the queue */
4078 if (enable) {
4079 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4080 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4081 } else {
4082 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4083 }
4084
4085 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4086 }
4087
4088 /**
4089 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4090 * @seid: VSI SEID
4091 * @pf: the PF structure
4092 * @pf_q: the PF queue to configure
4093 * @is_xdp: true if the queue is used for XDP
4094 * @enable: start or stop the queue
4095 **/
4096 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4097 bool is_xdp, bool enable)
4098 {
4099 int ret;
4100
4101 i40e_control_tx_q(pf, pf_q, enable);
4102
4103 /* wait for the change to finish */
4104 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4105 if (ret) {
4106 dev_info(&pf->pdev->dev,
4107 "VSI seid %d %sTx ring %d %sable timeout\n",
4108 seid, (is_xdp ? "XDP " : ""), pf_q,
4109 (enable ? "en" : "dis"));
4110 }
4111
4112 return ret;
4113 }
4114
4115 /**
4116 * i40e_vsi_control_tx - Start or stop a VSI's rings
4117 * @vsi: the VSI being configured
4118 * @enable: start or stop the rings
4119 **/
4120 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4121 {
4122 struct i40e_pf *pf = vsi->back;
4123 int i, pf_q, ret = 0;
4124
4125 pf_q = vsi->base_queue;
4126 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4127 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4128 pf_q,
4129 false /*is xdp*/, enable);
4130 if (ret)
4131 break;
4132
4133 if (!i40e_enabled_xdp_vsi(vsi))
4134 continue;
4135
4136 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4137 pf_q + vsi->alloc_queue_pairs,
4138 true /*is xdp*/, enable);
4139 if (ret)
4140 break;
4141 }
4142
4143 return ret;
4144 }
4145
4146 /**
4147 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4148 * @pf: the PF being configured
4149 * @pf_q: the PF queue
4150 * @enable: enable or disable state of the queue
4151 *
4152 * This routine will wait for the given Rx queue of the PF to reach the
4153 * enabled or disabled state.
4154 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4155 * multiple retries; else will return 0 in case of success.
4156 **/
4157 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4158 {
4159 int i;
4160 u32 rx_reg;
4161
4162 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4163 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4164 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4165 break;
4166
4167 usleep_range(10, 20);
4168 }
4169 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4170 return -ETIMEDOUT;
4171
4172 return 0;
4173 }
4174
4175 /**
4176 * i40e_control_rx_q - Start or stop a particular Rx queue
4177 * @pf: the PF structure
4178 * @pf_q: the PF queue to configure
4179 * @enable: start or stop the queue
4180 *
4181 * This function enables or disables a single queue. Note that any delay
4182 * required after the operation is expected to be handled by the caller of
4183 * this function.
4184 **/
4185 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4186 {
4187 struct i40e_hw *hw = &pf->hw;
4188 u32 rx_reg;
4189 int i;
4190
4191 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4192 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4193 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4194 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4195 break;
4196 usleep_range(1000, 2000);
4197 }
4198
4199 /* Skip if the queue is already in the requested state */
4200 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4201 return;
4202
4203 /* turn on/off the queue */
4204 if (enable)
4205 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4206 else
4207 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4208
4209 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4210 }
4211
4212 /**
4213 * i40e_vsi_control_rx - Start or stop a VSI's rings
4214 * @vsi: the VSI being configured
4215 * @enable: start or stop the rings
4216 **/
4217 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4218 {
4219 struct i40e_pf *pf = vsi->back;
4220 int i, pf_q, ret = 0;
4221
4222 pf_q = vsi->base_queue;
4223 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4224 i40e_control_rx_q(pf, pf_q, enable);
4225
4226 /* wait for the change to finish */
4227 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4228 if (ret) {
4229 dev_info(&pf->pdev->dev,
4230 "VSI seid %d Rx ring %d %sable timeout\n",
4231 vsi->seid, pf_q, (enable ? "en" : "dis"));
4232 break;
4233 }
4234 }
4235
4236 /* Due to HW errata, on Rx disable only, the register can indicate done
4237 * before it really is. Needs 50ms to be sure
4238 */
4239 if (!enable)
4240 mdelay(50);
4241
4242 return ret;
4243 }
4244
4245 /**
4246 * i40e_vsi_start_rings - Start a VSI's rings
4247 * @vsi: the VSI being configured
4248 **/
4249 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4250 {
4251 int ret = 0;
4252
4253 /* do rx first for enable and last for disable */
4254 ret = i40e_vsi_control_rx(vsi, true);
4255 if (ret)
4256 return ret;
4257 ret = i40e_vsi_control_tx(vsi, true);
4258
4259 return ret;
4260 }
4261
4262 /**
4263 * i40e_vsi_stop_rings - Stop a VSI's rings
4264 * @vsi: the VSI being configured
4265 **/
4266 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4267 {
4268 /* When port TX is suspended, don't wait */
4269 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4270 return i40e_vsi_stop_rings_no_wait(vsi);
4271
4272 /* do rx first for enable and last for disable
4273 * Ignore return value, we need to shutdown whatever we can
4274 */
4275 i40e_vsi_control_tx(vsi, false);
4276 i40e_vsi_control_rx(vsi, false);
4277 }
4278
4279 /**
4280 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4281 * @vsi: the VSI being shutdown
4282 *
4283 * This function stops all the rings for a VSI but does not delay to verify
4284 * that rings have been disabled. It is expected that the caller is shutting
4285 * down multiple VSIs at once and will delay together for all the VSIs after
4286 * initiating the shutdown. This is particularly useful for shutting down lots
4287 * of VFs together. Otherwise, a large delay can be incurred while configuring
4288 * each VSI in serial.
4289 **/
4290 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4291 {
4292 struct i40e_pf *pf = vsi->back;
4293 int i, pf_q;
4294
4295 pf_q = vsi->base_queue;
4296 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4297 i40e_control_tx_q(pf, pf_q, false);
4298 i40e_control_rx_q(pf, pf_q, false);
4299 }
4300 }
4301
4302 /**
4303 * i40e_vsi_free_irq - Free the irq association with the OS
4304 * @vsi: the VSI being configured
4305 **/
4306 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4307 {
4308 struct i40e_pf *pf = vsi->back;
4309 struct i40e_hw *hw = &pf->hw;
4310 int base = vsi->base_vector;
4311 u32 val, qp;
4312 int i;
4313
4314 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4315 if (!vsi->q_vectors)
4316 return;
4317
4318 if (!vsi->irqs_ready)
4319 return;
4320
4321 vsi->irqs_ready = false;
4322 for (i = 0; i < vsi->num_q_vectors; i++) {
4323 int irq_num;
4324 u16 vector;
4325
4326 vector = i + base;
4327 irq_num = pf->msix_entries[vector].vector;
4328
4329 /* free only the irqs that were actually requested */
4330 if (!vsi->q_vectors[i] ||
4331 !vsi->q_vectors[i]->num_ringpairs)
4332 continue;
4333
4334 /* clear the affinity notifier in the IRQ descriptor */
4335 irq_set_affinity_notifier(irq_num, NULL);
4336 /* clear the affinity_mask in the IRQ descriptor */
4337 irq_set_affinity_hint(irq_num, NULL);
4338 synchronize_irq(irq_num);
4339 free_irq(irq_num, vsi->q_vectors[i]);
4340
4341 /* Tear down the interrupt queue link list
4342 *
4343 * We know that they come in pairs and always
4344 * the Rx first, then the Tx. To clear the
4345 * link list, stick the EOL value into the
4346 * next_q field of the registers.
4347 */
4348 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4349 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4350 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4351 val |= I40E_QUEUE_END_OF_LIST
4352 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4353 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4354
4355 while (qp != I40E_QUEUE_END_OF_LIST) {
4356 u32 next;
4357
4358 val = rd32(hw, I40E_QINT_RQCTL(qp));
4359
4360 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4361 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4362 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4363 I40E_QINT_RQCTL_INTEVENT_MASK);
4364
4365 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4366 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4367
4368 wr32(hw, I40E_QINT_RQCTL(qp), val);
4369
4370 val = rd32(hw, I40E_QINT_TQCTL(qp));
4371
4372 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4373 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4374
4375 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4376 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4377 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4378 I40E_QINT_TQCTL_INTEVENT_MASK);
4379
4380 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4381 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4382
4383 wr32(hw, I40E_QINT_TQCTL(qp), val);
4384 qp = next;
4385 }
4386 }
4387 } else {
4388 free_irq(pf->pdev->irq, pf);
4389
4390 val = rd32(hw, I40E_PFINT_LNKLST0);
4391 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4392 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4393 val |= I40E_QUEUE_END_OF_LIST
4394 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4395 wr32(hw, I40E_PFINT_LNKLST0, val);
4396
4397 val = rd32(hw, I40E_QINT_RQCTL(qp));
4398 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4399 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4400 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4401 I40E_QINT_RQCTL_INTEVENT_MASK);
4402
4403 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4404 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4405
4406 wr32(hw, I40E_QINT_RQCTL(qp), val);
4407
4408 val = rd32(hw, I40E_QINT_TQCTL(qp));
4409
4410 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4411 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4412 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4413 I40E_QINT_TQCTL_INTEVENT_MASK);
4414
4415 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4416 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4417
4418 wr32(hw, I40E_QINT_TQCTL(qp), val);
4419 }
4420 }
4421
4422 /**
4423 * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4424 * @vsi: the VSI being configured
4425 * @v_idx: Index of vector to be freed
4426 *
4427 * This function frees the memory allocated to the q_vector. In addition if
4428 * NAPI is enabled it will delete any references to the NAPI struct prior
4429 * to freeing the q_vector.
4430 **/
4431 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4432 {
4433 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4434 struct i40e_ring *ring;
4435
4436 if (!q_vector)
4437 return;
4438
4439 /* disassociate q_vector from rings */
4440 i40e_for_each_ring(ring, q_vector->tx)
4441 ring->q_vector = NULL;
4442
4443 i40e_for_each_ring(ring, q_vector->rx)
4444 ring->q_vector = NULL;
4445
4446 /* only VSI w/ an associated netdev is set up w/ NAPI */
4447 if (vsi->netdev)
4448 netif_napi_del(&q_vector->napi);
4449
4450 vsi->q_vectors[v_idx] = NULL;
4451
4452 kfree_rcu(q_vector, rcu);
4453 }
4454
4455 /**
4456 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4457 * @vsi: the VSI being un-configured
4458 *
4459 * This frees the memory allocated to the q_vectors and
4460 * deletes references to the NAPI struct.
4461 **/
4462 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4463 {
4464 int v_idx;
4465
4466 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4467 i40e_free_q_vector(vsi, v_idx);
4468 }
4469
4470 /**
4471 * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4472 * @pf: board private structure
4473 **/
4474 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4475 {
4476 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4477 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4478 pci_disable_msix(pf->pdev);
4479 kfree(pf->msix_entries);
4480 pf->msix_entries = NULL;
4481 kfree(pf->irq_pile);
4482 pf->irq_pile = NULL;
4483 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4484 pci_disable_msi(pf->pdev);
4485 }
4486 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4487 }
4488
4489 /**
4490 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4491 * @pf: board private structure
4492 *
4493 * We go through and clear interrupt specific resources and reset the structure
4494 * to pre-load conditions
4495 **/
4496 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4497 {
4498 int i;
4499
4500 i40e_stop_misc_vector(pf);
4501 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4502 synchronize_irq(pf->msix_entries[0].vector);
4503 free_irq(pf->msix_entries[0].vector, pf);
4504 }
4505
4506 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4507 I40E_IWARP_IRQ_PILE_ID);
4508
4509 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4510 for (i = 0; i < pf->num_alloc_vsi; i++)
4511 if (pf->vsi[i])
4512 i40e_vsi_free_q_vectors(pf->vsi[i]);
4513 i40e_reset_interrupt_capability(pf);
4514 }
4515
4516 /**
4517 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4518 * @vsi: the VSI being configured
4519 **/
4520 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4521 {
4522 int q_idx;
4523
4524 if (!vsi->netdev)
4525 return;
4526
4527 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4528 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4529
4530 if (q_vector->rx.ring || q_vector->tx.ring)
4531 napi_enable(&q_vector->napi);
4532 }
4533 }
4534
4535 /**
4536 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4537 * @vsi: the VSI being configured
4538 **/
4539 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4540 {
4541 int q_idx;
4542
4543 if (!vsi->netdev)
4544 return;
4545
4546 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4547 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4548
4549 if (q_vector->rx.ring || q_vector->tx.ring)
4550 napi_disable(&q_vector->napi);
4551 }
4552 }
4553
4554 /**
4555 * i40e_vsi_close - Shut down a VSI
4556 * @vsi: the vsi to be quelled
4557 **/
4558 static void i40e_vsi_close(struct i40e_vsi *vsi)
4559 {
4560 struct i40e_pf *pf = vsi->back;
4561 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4562 i40e_down(vsi);
4563 i40e_vsi_free_irq(vsi);
4564 i40e_vsi_free_tx_resources(vsi);
4565 i40e_vsi_free_rx_resources(vsi);
4566 vsi->current_netdev_flags = 0;
4567 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4568 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4569 pf->flags |= I40E_FLAG_CLIENT_RESET;
4570 }
4571
4572 /**
4573 * i40e_quiesce_vsi - Pause a given VSI
4574 * @vsi: the VSI being paused
4575 **/
4576 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4577 {
4578 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4579 return;
4580
4581 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4582 if (vsi->netdev && netif_running(vsi->netdev))
4583 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4584 else
4585 i40e_vsi_close(vsi);
4586 }
4587
4588 /**
4589 * i40e_unquiesce_vsi - Resume a given VSI
4590 * @vsi: the VSI being resumed
4591 **/
4592 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4593 {
4594 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4595 return;
4596
4597 if (vsi->netdev && netif_running(vsi->netdev))
4598 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4599 else
4600 i40e_vsi_open(vsi); /* this clears the DOWN bit */
4601 }
4602
4603 /**
4604 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4605 * @pf: the PF
4606 **/
4607 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4608 {
4609 int v;
4610
4611 for (v = 0; v < pf->num_alloc_vsi; v++) {
4612 if (pf->vsi[v])
4613 i40e_quiesce_vsi(pf->vsi[v]);
4614 }
4615 }
4616
4617 /**
4618 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4619 * @pf: the PF
4620 **/
4621 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4622 {
4623 int v;
4624
4625 for (v = 0; v < pf->num_alloc_vsi; v++) {
4626 if (pf->vsi[v])
4627 i40e_unquiesce_vsi(pf->vsi[v]);
4628 }
4629 }
4630
4631 /**
4632 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4633 * @vsi: the VSI being configured
4634 *
4635 * Wait until all queues on a given VSI have been disabled.
4636 **/
4637 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4638 {
4639 struct i40e_pf *pf = vsi->back;
4640 int i, pf_q, ret;
4641
4642 pf_q = vsi->base_queue;
4643 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4644 /* Check and wait for the Tx queue */
4645 ret = i40e_pf_txq_wait(pf, pf_q, false);
4646 if (ret) {
4647 dev_info(&pf->pdev->dev,
4648 "VSI seid %d Tx ring %d disable timeout\n",
4649 vsi->seid, pf_q);
4650 return ret;
4651 }
4652
4653 if (!i40e_enabled_xdp_vsi(vsi))
4654 goto wait_rx;
4655
4656 /* Check and wait for the XDP Tx queue */
4657 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4658 false);
4659 if (ret) {
4660 dev_info(&pf->pdev->dev,
4661 "VSI seid %d XDP Tx ring %d disable timeout\n",
4662 vsi->seid, pf_q);
4663 return ret;
4664 }
4665 wait_rx:
4666 /* Check and wait for the Rx queue */
4667 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4668 if (ret) {
4669 dev_info(&pf->pdev->dev,
4670 "VSI seid %d Rx ring %d disable timeout\n",
4671 vsi->seid, pf_q);
4672 return ret;
4673 }
4674 }
4675
4676 return 0;
4677 }
4678
4679 #ifdef CONFIG_I40E_DCB
4680 /**
4681 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4682 * @pf: the PF
4683 *
4684 * This function waits for the queues to be in disabled state for all the
4685 * VSIs that are managed by this PF.
4686 **/
4687 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4688 {
4689 int v, ret = 0;
4690
4691 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4692 if (pf->vsi[v]) {
4693 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4694 if (ret)
4695 break;
4696 }
4697 }
4698
4699 return ret;
4700 }
4701
4702 #endif
4703
4704 /**
4705 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4706 * @q_idx: TX queue number
4707 * @vsi: Pointer to VSI struct
4708 *
4709 * This function checks specified queue for given VSI. Detects hung condition.
4710 * We proactively detect hung TX queues by checking if interrupts are disabled
4711 * but there are pending descriptors. If it appears hung, attempt to recover
4712 * by triggering a SW interrupt.
4713 **/
4714 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4715 {
4716 struct i40e_ring *tx_ring = NULL;
4717 struct i40e_pf *pf;
4718 u32 val, tx_pending;
4719 int i;
4720
4721 pf = vsi->back;
4722
4723 /* now that we have an index, find the tx_ring struct */
4724 for (i = 0; i < vsi->num_queue_pairs; i++) {
4725 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4726 if (q_idx == vsi->tx_rings[i]->queue_index) {
4727 tx_ring = vsi->tx_rings[i];
4728 break;
4729 }
4730 }
4731 }
4732
4733 if (!tx_ring)
4734 return;
4735
4736 /* Read interrupt register */
4737 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4738 val = rd32(&pf->hw,
4739 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4740 tx_ring->vsi->base_vector - 1));
4741 else
4742 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4743
4744 tx_pending = i40e_get_tx_pending(tx_ring);
4745
4746 /* Interrupts are disabled and TX pending is non-zero,
4747 * trigger the SW interrupt (don't wait). Worst case
4748 * there will be one extra interrupt which may result
4749 * into not cleaning any queues because queues are cleaned.
4750 */
4751 if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4752 i40e_force_wb(vsi, tx_ring->q_vector);
4753 }
4754
4755 /**
4756 * i40e_detect_recover_hung - Function to detect and recover hung_queues
4757 * @pf: pointer to PF struct
4758 *
4759 * LAN VSI has netdev and netdev has TX queues. This function is to check
4760 * each of those TX queues if they are hung, trigger recovery by issuing
4761 * SW interrupt.
4762 **/
4763 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4764 {
4765 struct net_device *netdev;
4766 struct i40e_vsi *vsi;
4767 int i;
4768
4769 /* Only for LAN VSI */
4770 vsi = pf->vsi[pf->lan_vsi];
4771
4772 if (!vsi)
4773 return;
4774
4775 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4776 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
4777 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
4778 return;
4779
4780 /* Make sure type is MAIN VSI */
4781 if (vsi->type != I40E_VSI_MAIN)
4782 return;
4783
4784 netdev = vsi->netdev;
4785 if (!netdev)
4786 return;
4787
4788 /* Bail out if netif_carrier is not OK */
4789 if (!netif_carrier_ok(netdev))
4790 return;
4791
4792 /* Go thru' TX queues for netdev */
4793 for (i = 0; i < netdev->num_tx_queues; i++) {
4794 struct netdev_queue *q;
4795
4796 q = netdev_get_tx_queue(netdev, i);
4797 if (q)
4798 i40e_detect_recover_hung_queue(i, vsi);
4799 }
4800 }
4801
4802 /**
4803 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4804 * @pf: pointer to PF
4805 *
4806 * Get TC map for ISCSI PF type that will include iSCSI TC
4807 * and LAN TC.
4808 **/
4809 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4810 {
4811 struct i40e_dcb_app_priority_table app;
4812 struct i40e_hw *hw = &pf->hw;
4813 u8 enabled_tc = 1; /* TC0 is always enabled */
4814 u8 tc, i;
4815 /* Get the iSCSI APP TLV */
4816 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4817
4818 for (i = 0; i < dcbcfg->numapps; i++) {
4819 app = dcbcfg->app[i];
4820 if (app.selector == I40E_APP_SEL_TCPIP &&
4821 app.protocolid == I40E_APP_PROTOID_ISCSI) {
4822 tc = dcbcfg->etscfg.prioritytable[app.priority];
4823 enabled_tc |= BIT(tc);
4824 break;
4825 }
4826 }
4827
4828 return enabled_tc;
4829 }
4830
4831 /**
4832 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
4833 * @dcbcfg: the corresponding DCBx configuration structure
4834 *
4835 * Return the number of TCs from given DCBx configuration
4836 **/
4837 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4838 {
4839 int i, tc_unused = 0;
4840 u8 num_tc = 0;
4841 u8 ret = 0;
4842
4843 /* Scan the ETS Config Priority Table to find
4844 * traffic class enabled for a given priority
4845 * and create a bitmask of enabled TCs
4846 */
4847 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4848 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4849
4850 /* Now scan the bitmask to check for
4851 * contiguous TCs starting with TC0
4852 */
4853 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4854 if (num_tc & BIT(i)) {
4855 if (!tc_unused) {
4856 ret++;
4857 } else {
4858 pr_err("Non-contiguous TC - Disabling DCB\n");
4859 return 1;
4860 }
4861 } else {
4862 tc_unused = 1;
4863 }
4864 }
4865
4866 /* There is always at least TC0 */
4867 if (!ret)
4868 ret = 1;
4869
4870 return ret;
4871 }
4872
4873 /**
4874 * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4875 * @dcbcfg: the corresponding DCBx configuration structure
4876 *
4877 * Query the current DCB configuration and return the number of
4878 * traffic classes enabled from the given DCBX config
4879 **/
4880 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4881 {
4882 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4883 u8 enabled_tc = 1;
4884 u8 i;
4885
4886 for (i = 0; i < num_tc; i++)
4887 enabled_tc |= BIT(i);
4888
4889 return enabled_tc;
4890 }
4891
4892 /**
4893 * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4894 * @pf: PF being queried
4895 *
4896 * Return number of traffic classes enabled for the given PF
4897 **/
4898 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4899 {
4900 struct i40e_hw *hw = &pf->hw;
4901 u8 i, enabled_tc = 1;
4902 u8 num_tc = 0;
4903 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4904
4905 /* If DCB is not enabled then always in single TC */
4906 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4907 return 1;
4908
4909 /* SFP mode will be enabled for all TCs on port */
4910 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4911 return i40e_dcb_get_num_tc(dcbcfg);
4912
4913 /* MFP mode return count of enabled TCs for this PF */
4914 if (pf->hw.func_caps.iscsi)
4915 enabled_tc = i40e_get_iscsi_tc_map(pf);
4916 else
4917 return 1; /* Only TC0 */
4918
4919 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4920 if (enabled_tc & BIT(i))
4921 num_tc++;
4922 }
4923 return num_tc;
4924 }
4925
4926 /**
4927 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4928 * @pf: PF being queried
4929 *
4930 * Return a bitmap for enabled traffic classes for this PF.
4931 **/
4932 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4933 {
4934 /* If DCB is not enabled for this PF then just return default TC */
4935 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4936 return I40E_DEFAULT_TRAFFIC_CLASS;
4937
4938 /* SFP mode we want PF to be enabled for all TCs */
4939 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4940 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4941
4942 /* MFP enabled and iSCSI PF type */
4943 if (pf->hw.func_caps.iscsi)
4944 return i40e_get_iscsi_tc_map(pf);
4945 else
4946 return I40E_DEFAULT_TRAFFIC_CLASS;
4947 }
4948
4949 /**
4950 * i40e_vsi_get_bw_info - Query VSI BW Information
4951 * @vsi: the VSI being queried
4952 *
4953 * Returns 0 on success, negative value on failure
4954 **/
4955 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4956 {
4957 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4958 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4959 struct i40e_pf *pf = vsi->back;
4960 struct i40e_hw *hw = &pf->hw;
4961 i40e_status ret;
4962 u32 tc_bw_max;
4963 int i;
4964
4965 /* Get the VSI level BW configuration */
4966 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4967 if (ret) {
4968 dev_info(&pf->pdev->dev,
4969 "couldn't get PF vsi bw config, err %s aq_err %s\n",
4970 i40e_stat_str(&pf->hw, ret),
4971 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4972 return -EINVAL;
4973 }
4974
4975 /* Get the VSI level BW configuration per TC */
4976 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4977 NULL);
4978 if (ret) {
4979 dev_info(&pf->pdev->dev,
4980 "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4981 i40e_stat_str(&pf->hw, ret),
4982 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4983 return -EINVAL;
4984 }
4985
4986 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4987 dev_info(&pf->pdev->dev,
4988 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4989 bw_config.tc_valid_bits,
4990 bw_ets_config.tc_valid_bits);
4991 /* Still continuing */
4992 }
4993
4994 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4995 vsi->bw_max_quanta = bw_config.max_bw;
4996 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4997 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4998 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4999 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5000 vsi->bw_ets_limit_credits[i] =
5001 le16_to_cpu(bw_ets_config.credits[i]);
5002 /* 3 bits out of 4 for each TC */
5003 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5004 }
5005
5006 return 0;
5007 }
5008
5009 /**
5010 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5011 * @vsi: the VSI being configured
5012 * @enabled_tc: TC bitmap
5013 * @bw_credits: BW shared credits per TC
5014 *
5015 * Returns 0 on success, negative value on failure
5016 **/
5017 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5018 u8 *bw_share)
5019 {
5020 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5021 i40e_status ret;
5022 int i;
5023
5024 bw_data.tc_valid_bits = enabled_tc;
5025 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5026 bw_data.tc_bw_credits[i] = bw_share[i];
5027
5028 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
5029 NULL);
5030 if (ret) {
5031 dev_info(&vsi->back->pdev->dev,
5032 "AQ command Config VSI BW allocation per TC failed = %d\n",
5033 vsi->back->hw.aq.asq_last_status);
5034 return -EINVAL;
5035 }
5036
5037 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5038 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5039
5040 return 0;
5041 }
5042
5043 /**
5044 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5045 * @vsi: the VSI being configured
5046 * @enabled_tc: TC map to be enabled
5047 *
5048 **/
5049 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5050 {
5051 struct net_device *netdev = vsi->netdev;
5052 struct i40e_pf *pf = vsi->back;
5053 struct i40e_hw *hw = &pf->hw;
5054 u8 netdev_tc = 0;
5055 int i;
5056 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5057
5058 if (!netdev)
5059 return;
5060
5061 if (!enabled_tc) {
5062 netdev_reset_tc(netdev);
5063 return;
5064 }
5065
5066 /* Set up actual enabled TCs on the VSI */
5067 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5068 return;
5069
5070 /* set per TC queues for the VSI */
5071 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5072 /* Only set TC queues for enabled tcs
5073 *
5074 * e.g. For a VSI that has TC0 and TC3 enabled the
5075 * enabled_tc bitmap would be 0x00001001; the driver
5076 * will set the numtc for netdev as 2 that will be
5077 * referenced by the netdev layer as TC 0 and 1.
5078 */
5079 if (vsi->tc_config.enabled_tc & BIT(i))
5080 netdev_set_tc_queue(netdev,
5081 vsi->tc_config.tc_info[i].netdev_tc,
5082 vsi->tc_config.tc_info[i].qcount,
5083 vsi->tc_config.tc_info[i].qoffset);
5084 }
5085
5086 /* Assign UP2TC map for the VSI */
5087 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5088 /* Get the actual TC# for the UP */
5089 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5090 /* Get the mapped netdev TC# for the UP */
5091 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5092 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5093 }
5094 }
5095
5096 /**
5097 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5098 * @vsi: the VSI being configured
5099 * @ctxt: the ctxt buffer returned from AQ VSI update param command
5100 **/
5101 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5102 struct i40e_vsi_context *ctxt)
5103 {
5104 /* copy just the sections touched not the entire info
5105 * since not all sections are valid as returned by
5106 * update vsi params
5107 */
5108 vsi->info.mapping_flags = ctxt->info.mapping_flags;
5109 memcpy(&vsi->info.queue_mapping,
5110 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5111 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5112 sizeof(vsi->info.tc_mapping));
5113 }
5114
5115 /**
5116 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5117 * @vsi: VSI to be configured
5118 * @enabled_tc: TC bitmap
5119 *
5120 * This configures a particular VSI for TCs that are mapped to the
5121 * given TC bitmap. It uses default bandwidth share for TCs across
5122 * VSIs to configure TC for a particular VSI.
5123 *
5124 * NOTE:
5125 * It is expected that the VSI queues have been quisced before calling
5126 * this function.
5127 **/
5128 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5129 {
5130 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5131 struct i40e_vsi_context ctxt;
5132 int ret = 0;
5133 int i;
5134
5135 /* Check if enabled_tc is same as existing or new TCs */
5136 if (vsi->tc_config.enabled_tc == enabled_tc)
5137 return ret;
5138
5139 /* Enable ETS TCs with equal BW Share for now across all VSIs */
5140 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5141 if (enabled_tc & BIT(i))
5142 bw_share[i] = 1;
5143 }
5144
5145 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5146 if (ret) {
5147 dev_info(&vsi->back->pdev->dev,
5148 "Failed configuring TC map %d for VSI %d\n",
5149 enabled_tc, vsi->seid);
5150 goto out;
5151 }
5152
5153 /* Update Queue Pairs Mapping for currently enabled UPs */
5154 ctxt.seid = vsi->seid;
5155 ctxt.pf_num = vsi->back->hw.pf_id;
5156 ctxt.vf_num = 0;
5157 ctxt.uplink_seid = vsi->uplink_seid;
5158 ctxt.info = vsi->info;
5159 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5160
5161 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5162 ctxt.info.valid_sections |=
5163 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5164 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5165 }
5166
5167 /* Update the VSI after updating the VSI queue-mapping information */
5168 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5169 if (ret) {
5170 dev_info(&vsi->back->pdev->dev,
5171 "Update vsi tc config failed, err %s aq_err %s\n",
5172 i40e_stat_str(&vsi->back->hw, ret),
5173 i40e_aq_str(&vsi->back->hw,
5174 vsi->back->hw.aq.asq_last_status));
5175 goto out;
5176 }
5177 /* update the local VSI info with updated queue map */
5178 i40e_vsi_update_queue_map(vsi, &ctxt);
5179 vsi->info.valid_sections = 0;
5180
5181 /* Update current VSI BW information */
5182 ret = i40e_vsi_get_bw_info(vsi);
5183 if (ret) {
5184 dev_info(&vsi->back->pdev->dev,
5185 "Failed updating vsi bw info, err %s aq_err %s\n",
5186 i40e_stat_str(&vsi->back->hw, ret),
5187 i40e_aq_str(&vsi->back->hw,
5188 vsi->back->hw.aq.asq_last_status));
5189 goto out;
5190 }
5191
5192 /* Update the netdev TC setup */
5193 i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5194 out:
5195 return ret;
5196 }
5197
5198 /**
5199 * i40e_veb_config_tc - Configure TCs for given VEB
5200 * @veb: given VEB
5201 * @enabled_tc: TC bitmap
5202 *
5203 * Configures given TC bitmap for VEB (switching) element
5204 **/
5205 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5206 {
5207 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5208 struct i40e_pf *pf = veb->pf;
5209 int ret = 0;
5210 int i;
5211
5212 /* No TCs or already enabled TCs just return */
5213 if (!enabled_tc || veb->enabled_tc == enabled_tc)
5214 return ret;
5215
5216 bw_data.tc_valid_bits = enabled_tc;
5217 /* bw_data.absolute_credits is not set (relative) */
5218
5219 /* Enable ETS TCs with equal BW Share for now */
5220 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5221 if (enabled_tc & BIT(i))
5222 bw_data.tc_bw_share_credits[i] = 1;
5223 }
5224
5225 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5226 &bw_data, NULL);
5227 if (ret) {
5228 dev_info(&pf->pdev->dev,
5229 "VEB bw config failed, err %s aq_err %s\n",
5230 i40e_stat_str(&pf->hw, ret),
5231 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5232 goto out;
5233 }
5234
5235 /* Update the BW information */
5236 ret = i40e_veb_get_bw_info(veb);
5237 if (ret) {
5238 dev_info(&pf->pdev->dev,
5239 "Failed getting veb bw config, err %s aq_err %s\n",
5240 i40e_stat_str(&pf->hw, ret),
5241 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5242 }
5243
5244 out:
5245 return ret;
5246 }
5247
5248 #ifdef CONFIG_I40E_DCB
5249 /**
5250 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5251 * @pf: PF struct
5252 *
5253 * Reconfigure VEB/VSIs on a given PF; it is assumed that
5254 * the caller would've quiesce all the VSIs before calling
5255 * this function
5256 **/
5257 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5258 {
5259 u8 tc_map = 0;
5260 int ret;
5261 u8 v;
5262
5263 /* Enable the TCs available on PF to all VEBs */
5264 tc_map = i40e_pf_get_tc_map(pf);
5265 for (v = 0; v < I40E_MAX_VEB; v++) {
5266 if (!pf->veb[v])
5267 continue;
5268 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5269 if (ret) {
5270 dev_info(&pf->pdev->dev,
5271 "Failed configuring TC for VEB seid=%d\n",
5272 pf->veb[v]->seid);
5273 /* Will try to configure as many components */
5274 }
5275 }
5276
5277 /* Update each VSI */
5278 for (v = 0; v < pf->num_alloc_vsi; v++) {
5279 if (!pf->vsi[v])
5280 continue;
5281
5282 /* - Enable all TCs for the LAN VSI
5283 * - For all others keep them at TC0 for now
5284 */
5285 if (v == pf->lan_vsi)
5286 tc_map = i40e_pf_get_tc_map(pf);
5287 else
5288 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5289
5290 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5291 if (ret) {
5292 dev_info(&pf->pdev->dev,
5293 "Failed configuring TC for VSI seid=%d\n",
5294 pf->vsi[v]->seid);
5295 /* Will try to configure as many components */
5296 } else {
5297 /* Re-configure VSI vectors based on updated TC map */
5298 i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5299 if (pf->vsi[v]->netdev)
5300 i40e_dcbnl_set_all(pf->vsi[v]);
5301 }
5302 }
5303 }
5304
5305 /**
5306 * i40e_resume_port_tx - Resume port Tx
5307 * @pf: PF struct
5308 *
5309 * Resume a port's Tx and issue a PF reset in case of failure to
5310 * resume.
5311 **/
5312 static int i40e_resume_port_tx(struct i40e_pf *pf)
5313 {
5314 struct i40e_hw *hw = &pf->hw;
5315 int ret;
5316
5317 ret = i40e_aq_resume_port_tx(hw, NULL);
5318 if (ret) {
5319 dev_info(&pf->pdev->dev,
5320 "Resume Port Tx failed, err %s aq_err %s\n",
5321 i40e_stat_str(&pf->hw, ret),
5322 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5323 /* Schedule PF reset to recover */
5324 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
5325 i40e_service_event_schedule(pf);
5326 }
5327
5328 return ret;
5329 }
5330
5331 /**
5332 * i40e_init_pf_dcb - Initialize DCB configuration
5333 * @pf: PF being configured
5334 *
5335 * Query the current DCB configuration and cache it
5336 * in the hardware structure
5337 **/
5338 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5339 {
5340 struct i40e_hw *hw = &pf->hw;
5341 int err = 0;
5342
5343 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5344 if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT)
5345 goto out;
5346
5347 /* Get the initial DCB configuration */
5348 err = i40e_init_dcb(hw);
5349 if (!err) {
5350 /* Device/Function is not DCBX capable */
5351 if ((!hw->func_caps.dcb) ||
5352 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5353 dev_info(&pf->pdev->dev,
5354 "DCBX offload is not supported or is disabled for this PF.\n");
5355 } else {
5356 /* When status is not DISABLED then DCBX in FW */
5357 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5358 DCB_CAP_DCBX_VER_IEEE;
5359
5360 pf->flags |= I40E_FLAG_DCB_CAPABLE;
5361 /* Enable DCB tagging only when more than one TC
5362 * or explicitly disable if only one TC
5363 */
5364 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5365 pf->flags |= I40E_FLAG_DCB_ENABLED;
5366 else
5367 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5368 dev_dbg(&pf->pdev->dev,
5369 "DCBX offload is supported for this PF.\n");
5370 }
5371 } else {
5372 dev_info(&pf->pdev->dev,
5373 "Query for DCB configuration failed, err %s aq_err %s\n",
5374 i40e_stat_str(&pf->hw, err),
5375 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5376 }
5377
5378 out:
5379 return err;
5380 }
5381 #endif /* CONFIG_I40E_DCB */
5382 #define SPEED_SIZE 14
5383 #define FC_SIZE 8
5384 /**
5385 * i40e_print_link_message - print link up or down
5386 * @vsi: the VSI for which link needs a message
5387 */
5388 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5389 {
5390 enum i40e_aq_link_speed new_speed;
5391 char *speed = "Unknown";
5392 char *fc = "Unknown";
5393 char *fec = "";
5394 char *an = "";
5395
5396 new_speed = vsi->back->hw.phy.link_info.link_speed;
5397
5398 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5399 return;
5400 vsi->current_isup = isup;
5401 vsi->current_speed = new_speed;
5402 if (!isup) {
5403 netdev_info(vsi->netdev, "NIC Link is Down\n");
5404 return;
5405 }
5406
5407 /* Warn user if link speed on NPAR enabled partition is not at
5408 * least 10GB
5409 */
5410 if (vsi->back->hw.func_caps.npar_enable &&
5411 (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5412 vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5413 netdev_warn(vsi->netdev,
5414 "The partition detected link speed that is less than 10Gbps\n");
5415
5416 switch (vsi->back->hw.phy.link_info.link_speed) {
5417 case I40E_LINK_SPEED_40GB:
5418 speed = "40 G";
5419 break;
5420 case I40E_LINK_SPEED_20GB:
5421 speed = "20 G";
5422 break;
5423 case I40E_LINK_SPEED_25GB:
5424 speed = "25 G";
5425 break;
5426 case I40E_LINK_SPEED_10GB:
5427 speed = "10 G";
5428 break;
5429 case I40E_LINK_SPEED_1GB:
5430 speed = "1000 M";
5431 break;
5432 case I40E_LINK_SPEED_100MB:
5433 speed = "100 M";
5434 break;
5435 default:
5436 break;
5437 }
5438
5439 switch (vsi->back->hw.fc.current_mode) {
5440 case I40E_FC_FULL:
5441 fc = "RX/TX";
5442 break;
5443 case I40E_FC_TX_PAUSE:
5444 fc = "TX";
5445 break;
5446 case I40E_FC_RX_PAUSE:
5447 fc = "RX";
5448 break;
5449 default:
5450 fc = "None";
5451 break;
5452 }
5453
5454 if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
5455 fec = ", FEC: None";
5456 an = ", Autoneg: False";
5457
5458 if (vsi->back->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
5459 an = ", Autoneg: True";
5460
5461 if (vsi->back->hw.phy.link_info.fec_info &
5462 I40E_AQ_CONFIG_FEC_KR_ENA)
5463 fec = ", FEC: CL74 FC-FEC/BASE-R";
5464 else if (vsi->back->hw.phy.link_info.fec_info &
5465 I40E_AQ_CONFIG_FEC_RS_ENA)
5466 fec = ", FEC: CL108 RS-FEC";
5467 }
5468
5469 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s, Flow Control: %s\n",
5470 speed, fec, an, fc);
5471 }
5472
5473 /**
5474 * i40e_up_complete - Finish the last steps of bringing up a connection
5475 * @vsi: the VSI being configured
5476 **/
5477 static int i40e_up_complete(struct i40e_vsi *vsi)
5478 {
5479 struct i40e_pf *pf = vsi->back;
5480 int err;
5481
5482 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5483 i40e_vsi_configure_msix(vsi);
5484 else
5485 i40e_configure_msi_and_legacy(vsi);
5486
5487 /* start rings */
5488 err = i40e_vsi_start_rings(vsi);
5489 if (err)
5490 return err;
5491
5492 clear_bit(__I40E_VSI_DOWN, vsi->state);
5493 i40e_napi_enable_all(vsi);
5494 i40e_vsi_enable_irq(vsi);
5495
5496 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5497 (vsi->netdev)) {
5498 i40e_print_link_message(vsi, true);
5499 netif_tx_start_all_queues(vsi->netdev);
5500 netif_carrier_on(vsi->netdev);
5501 } else if (vsi->netdev) {
5502 i40e_print_link_message(vsi, false);
5503 /* need to check for qualified module here*/
5504 if ((pf->hw.phy.link_info.link_info &
5505 I40E_AQ_MEDIA_AVAILABLE) &&
5506 (!(pf->hw.phy.link_info.an_info &
5507 I40E_AQ_QUALIFIED_MODULE)))
5508 netdev_err(vsi->netdev,
5509 "the driver failed to link because an unqualified module was detected.");
5510 }
5511
5512 /* replay FDIR SB filters */
5513 if (vsi->type == I40E_VSI_FDIR) {
5514 /* reset fd counters */
5515 pf->fd_add_err = 0;
5516 pf->fd_atr_cnt = 0;
5517 i40e_fdir_filter_restore(vsi);
5518 }
5519
5520 /* On the next run of the service_task, notify any clients of the new
5521 * opened netdev
5522 */
5523 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5524 i40e_service_event_schedule(pf);
5525
5526 return 0;
5527 }
5528
5529 /**
5530 * i40e_vsi_reinit_locked - Reset the VSI
5531 * @vsi: the VSI being configured
5532 *
5533 * Rebuild the ring structs after some configuration
5534 * has changed, e.g. MTU size.
5535 **/
5536 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5537 {
5538 struct i40e_pf *pf = vsi->back;
5539
5540 WARN_ON(in_interrupt());
5541 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
5542 usleep_range(1000, 2000);
5543 i40e_down(vsi);
5544
5545 i40e_up(vsi);
5546 clear_bit(__I40E_CONFIG_BUSY, pf->state);
5547 }
5548
5549 /**
5550 * i40e_up - Bring the connection back up after being down
5551 * @vsi: the VSI being configured
5552 **/
5553 int i40e_up(struct i40e_vsi *vsi)
5554 {
5555 int err;
5556
5557 err = i40e_vsi_configure(vsi);
5558 if (!err)
5559 err = i40e_up_complete(vsi);
5560
5561 return err;
5562 }
5563
5564 /**
5565 * i40e_down - Shutdown the connection processing
5566 * @vsi: the VSI being stopped
5567 **/
5568 void i40e_down(struct i40e_vsi *vsi)
5569 {
5570 int i;
5571
5572 /* It is assumed that the caller of this function
5573 * sets the vsi->state __I40E_VSI_DOWN bit.
5574 */
5575 if (vsi->netdev) {
5576 netif_carrier_off(vsi->netdev);
5577 netif_tx_disable(vsi->netdev);
5578 }
5579 i40e_vsi_disable_irq(vsi);
5580 i40e_vsi_stop_rings(vsi);
5581 i40e_napi_disable_all(vsi);
5582
5583 for (i = 0; i < vsi->num_queue_pairs; i++) {
5584 i40e_clean_tx_ring(vsi->tx_rings[i]);
5585 if (i40e_enabled_xdp_vsi(vsi))
5586 i40e_clean_tx_ring(vsi->xdp_rings[i]);
5587 i40e_clean_rx_ring(vsi->rx_rings[i]);
5588 }
5589
5590 }
5591
5592 /**
5593 * i40e_setup_tc - configure multiple traffic classes
5594 * @netdev: net device to configure
5595 * @tc: number of traffic classes to enable
5596 **/
5597 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5598 {
5599 struct i40e_netdev_priv *np = netdev_priv(netdev);
5600 struct i40e_vsi *vsi = np->vsi;
5601 struct i40e_pf *pf = vsi->back;
5602 u8 enabled_tc = 0;
5603 int ret = -EINVAL;
5604 int i;
5605
5606 /* Check if DCB enabled to continue */
5607 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5608 netdev_info(netdev, "DCB is not enabled for adapter\n");
5609 goto exit;
5610 }
5611
5612 /* Check if MFP enabled */
5613 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5614 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5615 goto exit;
5616 }
5617
5618 /* Check whether tc count is within enabled limit */
5619 if (tc > i40e_pf_get_num_tc(pf)) {
5620 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5621 goto exit;
5622 }
5623
5624 /* Generate TC map for number of tc requested */
5625 for (i = 0; i < tc; i++)
5626 enabled_tc |= BIT(i);
5627
5628 /* Requesting same TC configuration as already enabled */
5629 if (enabled_tc == vsi->tc_config.enabled_tc)
5630 return 0;
5631
5632 /* Quiesce VSI queues */
5633 i40e_quiesce_vsi(vsi);
5634
5635 /* Configure VSI for enabled TCs */
5636 ret = i40e_vsi_config_tc(vsi, enabled_tc);
5637 if (ret) {
5638 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5639 vsi->seid);
5640 goto exit;
5641 }
5642
5643 /* Unquiesce VSI */
5644 i40e_unquiesce_vsi(vsi);
5645
5646 exit:
5647 return ret;
5648 }
5649
5650 static int __i40e_setup_tc(struct net_device *netdev, u32 handle,
5651 u32 chain_index, __be16 proto,
5652 struct tc_to_netdev *tc)
5653 {
5654 if (tc->type != TC_SETUP_MQPRIO)
5655 return -EINVAL;
5656
5657 tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
5658
5659 return i40e_setup_tc(netdev, tc->mqprio->num_tc);
5660 }
5661
5662 /**
5663 * i40e_open - Called when a network interface is made active
5664 * @netdev: network interface device structure
5665 *
5666 * The open entry point is called when a network interface is made
5667 * active by the system (IFF_UP). At this point all resources needed
5668 * for transmit and receive operations are allocated, the interrupt
5669 * handler is registered with the OS, the netdev watchdog subtask is
5670 * enabled, and the stack is notified that the interface is ready.
5671 *
5672 * Returns 0 on success, negative value on failure
5673 **/
5674 int i40e_open(struct net_device *netdev)
5675 {
5676 struct i40e_netdev_priv *np = netdev_priv(netdev);
5677 struct i40e_vsi *vsi = np->vsi;
5678 struct i40e_pf *pf = vsi->back;
5679 int err;
5680
5681 /* disallow open during test or if eeprom is broken */
5682 if (test_bit(__I40E_TESTING, pf->state) ||
5683 test_bit(__I40E_BAD_EEPROM, pf->state))
5684 return -EBUSY;
5685
5686 netif_carrier_off(netdev);
5687
5688 err = i40e_vsi_open(vsi);
5689 if (err)
5690 return err;
5691
5692 /* configure global TSO hardware offload settings */
5693 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5694 TCP_FLAG_FIN) >> 16);
5695 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5696 TCP_FLAG_FIN |
5697 TCP_FLAG_CWR) >> 16);
5698 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5699
5700 udp_tunnel_get_rx_info(netdev);
5701
5702 return 0;
5703 }
5704
5705 /**
5706 * i40e_vsi_open -
5707 * @vsi: the VSI to open
5708 *
5709 * Finish initialization of the VSI.
5710 *
5711 * Returns 0 on success, negative value on failure
5712 *
5713 * Note: expects to be called while under rtnl_lock()
5714 **/
5715 int i40e_vsi_open(struct i40e_vsi *vsi)
5716 {
5717 struct i40e_pf *pf = vsi->back;
5718 char int_name[I40E_INT_NAME_STR_LEN];
5719 int err;
5720
5721 /* allocate descriptors */
5722 err = i40e_vsi_setup_tx_resources(vsi);
5723 if (err)
5724 goto err_setup_tx;
5725 err = i40e_vsi_setup_rx_resources(vsi);
5726 if (err)
5727 goto err_setup_rx;
5728
5729 err = i40e_vsi_configure(vsi);
5730 if (err)
5731 goto err_setup_rx;
5732
5733 if (vsi->netdev) {
5734 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5735 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5736 err = i40e_vsi_request_irq(vsi, int_name);
5737 if (err)
5738 goto err_setup_rx;
5739
5740 /* Notify the stack of the actual queue counts. */
5741 err = netif_set_real_num_tx_queues(vsi->netdev,
5742 vsi->num_queue_pairs);
5743 if (err)
5744 goto err_set_queues;
5745
5746 err = netif_set_real_num_rx_queues(vsi->netdev,
5747 vsi->num_queue_pairs);
5748 if (err)
5749 goto err_set_queues;
5750
5751 } else if (vsi->type == I40E_VSI_FDIR) {
5752 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5753 dev_driver_string(&pf->pdev->dev),
5754 dev_name(&pf->pdev->dev));
5755 err = i40e_vsi_request_irq(vsi, int_name);
5756
5757 } else {
5758 err = -EINVAL;
5759 goto err_setup_rx;
5760 }
5761
5762 err = i40e_up_complete(vsi);
5763 if (err)
5764 goto err_up_complete;
5765
5766 return 0;
5767
5768 err_up_complete:
5769 i40e_down(vsi);
5770 err_set_queues:
5771 i40e_vsi_free_irq(vsi);
5772 err_setup_rx:
5773 i40e_vsi_free_rx_resources(vsi);
5774 err_setup_tx:
5775 i40e_vsi_free_tx_resources(vsi);
5776 if (vsi == pf->vsi[pf->lan_vsi])
5777 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
5778
5779 return err;
5780 }
5781
5782 /**
5783 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5784 * @pf: Pointer to PF
5785 *
5786 * This function destroys the hlist where all the Flow Director
5787 * filters were saved.
5788 **/
5789 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5790 {
5791 struct i40e_fdir_filter *filter;
5792 struct i40e_flex_pit *pit_entry, *tmp;
5793 struct hlist_node *node2;
5794
5795 hlist_for_each_entry_safe(filter, node2,
5796 &pf->fdir_filter_list, fdir_node) {
5797 hlist_del(&filter->fdir_node);
5798 kfree(filter);
5799 }
5800
5801 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
5802 list_del(&pit_entry->list);
5803 kfree(pit_entry);
5804 }
5805 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
5806
5807 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
5808 list_del(&pit_entry->list);
5809 kfree(pit_entry);
5810 }
5811 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
5812
5813 pf->fdir_pf_active_filters = 0;
5814 pf->fd_tcp4_filter_cnt = 0;
5815 pf->fd_udp4_filter_cnt = 0;
5816 pf->fd_sctp4_filter_cnt = 0;
5817 pf->fd_ip4_filter_cnt = 0;
5818
5819 /* Reprogram the default input set for TCP/IPv4 */
5820 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5821 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5822 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5823
5824 /* Reprogram the default input set for UDP/IPv4 */
5825 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5826 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5827 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5828
5829 /* Reprogram the default input set for SCTP/IPv4 */
5830 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5831 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5832 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5833
5834 /* Reprogram the default input set for Other/IPv4 */
5835 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5836 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
5837 }
5838
5839 /**
5840 * i40e_close - Disables a network interface
5841 * @netdev: network interface device structure
5842 *
5843 * The close entry point is called when an interface is de-activated
5844 * by the OS. The hardware is still under the driver's control, but
5845 * this netdev interface is disabled.
5846 *
5847 * Returns 0, this is not allowed to fail
5848 **/
5849 int i40e_close(struct net_device *netdev)
5850 {
5851 struct i40e_netdev_priv *np = netdev_priv(netdev);
5852 struct i40e_vsi *vsi = np->vsi;
5853
5854 i40e_vsi_close(vsi);
5855
5856 return 0;
5857 }
5858
5859 /**
5860 * i40e_do_reset - Start a PF or Core Reset sequence
5861 * @pf: board private structure
5862 * @reset_flags: which reset is requested
5863 * @lock_acquired: indicates whether or not the lock has been acquired
5864 * before this function was called.
5865 *
5866 * The essential difference in resets is that the PF Reset
5867 * doesn't clear the packet buffers, doesn't reset the PE
5868 * firmware, and doesn't bother the other PFs on the chip.
5869 **/
5870 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
5871 {
5872 u32 val;
5873
5874 WARN_ON(in_interrupt());
5875
5876
5877 /* do the biggest reset indicated */
5878 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5879
5880 /* Request a Global Reset
5881 *
5882 * This will start the chip's countdown to the actual full
5883 * chip reset event, and a warning interrupt to be sent
5884 * to all PFs, including the requestor. Our handler
5885 * for the warning interrupt will deal with the shutdown
5886 * and recovery of the switch setup.
5887 */
5888 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5889 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5890 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5891 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5892
5893 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5894
5895 /* Request a Core Reset
5896 *
5897 * Same as Global Reset, except does *not* include the MAC/PHY
5898 */
5899 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5900 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5901 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5902 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5903 i40e_flush(&pf->hw);
5904
5905 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5906
5907 /* Request a PF Reset
5908 *
5909 * Resets only the PF-specific registers
5910 *
5911 * This goes directly to the tear-down and rebuild of
5912 * the switch, since we need to do all the recovery as
5913 * for the Core Reset.
5914 */
5915 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5916 i40e_handle_reset_warning(pf, lock_acquired);
5917
5918 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5919 int v;
5920
5921 /* Find the VSI(s) that requested a re-init */
5922 dev_info(&pf->pdev->dev,
5923 "VSI reinit requested\n");
5924 for (v = 0; v < pf->num_alloc_vsi; v++) {
5925 struct i40e_vsi *vsi = pf->vsi[v];
5926
5927 if (vsi != NULL &&
5928 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
5929 vsi->state))
5930 i40e_vsi_reinit_locked(pf->vsi[v]);
5931 }
5932 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5933 int v;
5934
5935 /* Find the VSI(s) that needs to be brought down */
5936 dev_info(&pf->pdev->dev, "VSI down requested\n");
5937 for (v = 0; v < pf->num_alloc_vsi; v++) {
5938 struct i40e_vsi *vsi = pf->vsi[v];
5939
5940 if (vsi != NULL &&
5941 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
5942 vsi->state)) {
5943 set_bit(__I40E_VSI_DOWN, vsi->state);
5944 i40e_down(vsi);
5945 }
5946 }
5947 } else {
5948 dev_info(&pf->pdev->dev,
5949 "bad reset request 0x%08x\n", reset_flags);
5950 }
5951 }
5952
5953 #ifdef CONFIG_I40E_DCB
5954 /**
5955 * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5956 * @pf: board private structure
5957 * @old_cfg: current DCB config
5958 * @new_cfg: new DCB config
5959 **/
5960 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5961 struct i40e_dcbx_config *old_cfg,
5962 struct i40e_dcbx_config *new_cfg)
5963 {
5964 bool need_reconfig = false;
5965
5966 /* Check if ETS configuration has changed */
5967 if (memcmp(&new_cfg->etscfg,
5968 &old_cfg->etscfg,
5969 sizeof(new_cfg->etscfg))) {
5970 /* If Priority Table has changed reconfig is needed */
5971 if (memcmp(&new_cfg->etscfg.prioritytable,
5972 &old_cfg->etscfg.prioritytable,
5973 sizeof(new_cfg->etscfg.prioritytable))) {
5974 need_reconfig = true;
5975 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5976 }
5977
5978 if (memcmp(&new_cfg->etscfg.tcbwtable,
5979 &old_cfg->etscfg.tcbwtable,
5980 sizeof(new_cfg->etscfg.tcbwtable)))
5981 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5982
5983 if (memcmp(&new_cfg->etscfg.tsatable,
5984 &old_cfg->etscfg.tsatable,
5985 sizeof(new_cfg->etscfg.tsatable)))
5986 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5987 }
5988
5989 /* Check if PFC configuration has changed */
5990 if (memcmp(&new_cfg->pfc,
5991 &old_cfg->pfc,
5992 sizeof(new_cfg->pfc))) {
5993 need_reconfig = true;
5994 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5995 }
5996
5997 /* Check if APP Table has changed */
5998 if (memcmp(&new_cfg->app,
5999 &old_cfg->app,
6000 sizeof(new_cfg->app))) {
6001 need_reconfig = true;
6002 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
6003 }
6004
6005 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
6006 return need_reconfig;
6007 }
6008
6009 /**
6010 * i40e_handle_lldp_event - Handle LLDP Change MIB event
6011 * @pf: board private structure
6012 * @e: event info posted on ARQ
6013 **/
6014 static int i40e_handle_lldp_event(struct i40e_pf *pf,
6015 struct i40e_arq_event_info *e)
6016 {
6017 struct i40e_aqc_lldp_get_mib *mib =
6018 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
6019 struct i40e_hw *hw = &pf->hw;
6020 struct i40e_dcbx_config tmp_dcbx_cfg;
6021 bool need_reconfig = false;
6022 int ret = 0;
6023 u8 type;
6024
6025 /* Not DCB capable or capability disabled */
6026 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
6027 return ret;
6028
6029 /* Ignore if event is not for Nearest Bridge */
6030 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
6031 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
6032 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
6033 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
6034 return ret;
6035
6036 /* Check MIB Type and return if event for Remote MIB update */
6037 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
6038 dev_dbg(&pf->pdev->dev,
6039 "LLDP event mib type %s\n", type ? "remote" : "local");
6040 if (type == I40E_AQ_LLDP_MIB_REMOTE) {
6041 /* Update the remote cached instance and return */
6042 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
6043 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
6044 &hw->remote_dcbx_config);
6045 goto exit;
6046 }
6047
6048 /* Store the old configuration */
6049 tmp_dcbx_cfg = hw->local_dcbx_config;
6050
6051 /* Reset the old DCBx configuration data */
6052 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
6053 /* Get updated DCBX data from firmware */
6054 ret = i40e_get_dcb_config(&pf->hw);
6055 if (ret) {
6056 dev_info(&pf->pdev->dev,
6057 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
6058 i40e_stat_str(&pf->hw, ret),
6059 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6060 goto exit;
6061 }
6062
6063 /* No change detected in DCBX configs */
6064 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
6065 sizeof(tmp_dcbx_cfg))) {
6066 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
6067 goto exit;
6068 }
6069
6070 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
6071 &hw->local_dcbx_config);
6072
6073 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
6074
6075 if (!need_reconfig)
6076 goto exit;
6077
6078 /* Enable DCB tagging only when more than one TC */
6079 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6080 pf->flags |= I40E_FLAG_DCB_ENABLED;
6081 else
6082 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6083
6084 set_bit(__I40E_PORT_SUSPENDED, pf->state);
6085 /* Reconfiguration needed quiesce all VSIs */
6086 i40e_pf_quiesce_all_vsi(pf);
6087
6088 /* Changes in configuration update VEB/VSI */
6089 i40e_dcb_reconfigure(pf);
6090
6091 ret = i40e_resume_port_tx(pf);
6092
6093 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6094 /* In case of error no point in resuming VSIs */
6095 if (ret)
6096 goto exit;
6097
6098 /* Wait for the PF's queues to be disabled */
6099 ret = i40e_pf_wait_queues_disabled(pf);
6100 if (ret) {
6101 /* Schedule PF reset to recover */
6102 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6103 i40e_service_event_schedule(pf);
6104 } else {
6105 i40e_pf_unquiesce_all_vsi(pf);
6106 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
6107 I40E_FLAG_CLIENT_L2_CHANGE);
6108 }
6109
6110 exit:
6111 return ret;
6112 }
6113 #endif /* CONFIG_I40E_DCB */
6114
6115 /**
6116 * i40e_do_reset_safe - Protected reset path for userland calls.
6117 * @pf: board private structure
6118 * @reset_flags: which reset is requested
6119 *
6120 **/
6121 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
6122 {
6123 rtnl_lock();
6124 i40e_do_reset(pf, reset_flags, true);
6125 rtnl_unlock();
6126 }
6127
6128 /**
6129 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
6130 * @pf: board private structure
6131 * @e: event info posted on ARQ
6132 *
6133 * Handler for LAN Queue Overflow Event generated by the firmware for PF
6134 * and VF queues
6135 **/
6136 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
6137 struct i40e_arq_event_info *e)
6138 {
6139 struct i40e_aqc_lan_overflow *data =
6140 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
6141 u32 queue = le32_to_cpu(data->prtdcb_rupto);
6142 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
6143 struct i40e_hw *hw = &pf->hw;
6144 struct i40e_vf *vf;
6145 u16 vf_id;
6146
6147 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
6148 queue, qtx_ctl);
6149
6150 /* Queue belongs to VF, find the VF and issue VF reset */
6151 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
6152 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
6153 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
6154 >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
6155 vf_id -= hw->func_caps.vf_base_id;
6156 vf = &pf->vf[vf_id];
6157 i40e_vc_notify_vf_reset(vf);
6158 /* Allow VF to process pending reset notification */
6159 msleep(20);
6160 i40e_reset_vf(vf, false);
6161 }
6162 }
6163
6164 /**
6165 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
6166 * @pf: board private structure
6167 **/
6168 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
6169 {
6170 u32 val, fcnt_prog;
6171
6172 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6173 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
6174 return fcnt_prog;
6175 }
6176
6177 /**
6178 * i40e_get_current_fd_count - Get total FD filters programmed for this PF
6179 * @pf: board private structure
6180 **/
6181 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
6182 {
6183 u32 val, fcnt_prog;
6184
6185 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6186 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
6187 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
6188 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
6189 return fcnt_prog;
6190 }
6191
6192 /**
6193 * i40e_get_global_fd_count - Get total FD filters programmed on device
6194 * @pf: board private structure
6195 **/
6196 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
6197 {
6198 u32 val, fcnt_prog;
6199
6200 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6201 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6202 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6203 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6204 return fcnt_prog;
6205 }
6206
6207 /**
6208 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6209 * @pf: board private structure
6210 **/
6211 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6212 {
6213 struct i40e_fdir_filter *filter;
6214 u32 fcnt_prog, fcnt_avail;
6215 struct hlist_node *node;
6216
6217 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6218 return;
6219
6220 /* Check if we have enough room to re-enable FDir SB capability. */
6221 fcnt_prog = i40e_get_global_fd_count(pf);
6222 fcnt_avail = pf->fdir_pf_filter_count;
6223 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6224 (pf->fd_add_err == 0) ||
6225 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6226 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) {
6227 pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED;
6228 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6229 (I40E_DEBUG_FD & pf->hw.debug_mask))
6230 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6231 }
6232 }
6233
6234 /* We should wait for even more space before re-enabling ATR.
6235 * Additionally, we cannot enable ATR as long as we still have TCP SB
6236 * rules active.
6237 */
6238 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
6239 (pf->fd_tcp4_filter_cnt == 0)) {
6240 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
6241 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6242 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6243 (I40E_DEBUG_FD & pf->hw.debug_mask))
6244 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6245 }
6246 }
6247
6248 /* if hw had a problem adding a filter, delete it */
6249 if (pf->fd_inv > 0) {
6250 hlist_for_each_entry_safe(filter, node,
6251 &pf->fdir_filter_list, fdir_node) {
6252 if (filter->fd_id == pf->fd_inv) {
6253 hlist_del(&filter->fdir_node);
6254 kfree(filter);
6255 pf->fdir_pf_active_filters--;
6256 }
6257 }
6258 }
6259 }
6260
6261 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6262 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6263 /**
6264 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6265 * @pf: board private structure
6266 **/
6267 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6268 {
6269 unsigned long min_flush_time;
6270 int flush_wait_retry = 50;
6271 bool disable_atr = false;
6272 int fd_room;
6273 int reg;
6274
6275 if (!time_after(jiffies, pf->fd_flush_timestamp +
6276 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6277 return;
6278
6279 /* If the flush is happening too quick and we have mostly SB rules we
6280 * should not re-enable ATR for some time.
6281 */
6282 min_flush_time = pf->fd_flush_timestamp +
6283 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6284 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6285
6286 if (!(time_after(jiffies, min_flush_time)) &&
6287 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6288 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6289 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6290 disable_atr = true;
6291 }
6292
6293 pf->fd_flush_timestamp = jiffies;
6294 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED;
6295 /* flush all filters */
6296 wr32(&pf->hw, I40E_PFQF_CTL_1,
6297 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6298 i40e_flush(&pf->hw);
6299 pf->fd_flush_cnt++;
6300 pf->fd_add_err = 0;
6301 do {
6302 /* Check FD flush status every 5-6msec */
6303 usleep_range(5000, 6000);
6304 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6305 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6306 break;
6307 } while (flush_wait_retry--);
6308 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6309 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6310 } else {
6311 /* replay sideband filters */
6312 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6313 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
6314 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6315 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
6316 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6317 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6318 }
6319 }
6320
6321 /**
6322 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6323 * @pf: board private structure
6324 **/
6325 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6326 {
6327 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6328 }
6329
6330 /* We can see up to 256 filter programming desc in transit if the filters are
6331 * being applied really fast; before we see the first
6332 * filter miss error on Rx queue 0. Accumulating enough error messages before
6333 * reacting will make sure we don't cause flush too often.
6334 */
6335 #define I40E_MAX_FD_PROGRAM_ERROR 256
6336
6337 /**
6338 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6339 * @pf: board private structure
6340 **/
6341 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6342 {
6343
6344 /* if interface is down do nothing */
6345 if (test_bit(__I40E_DOWN, pf->state))
6346 return;
6347
6348 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6349 i40e_fdir_flush_and_replay(pf);
6350
6351 i40e_fdir_check_and_reenable(pf);
6352
6353 }
6354
6355 /**
6356 * i40e_vsi_link_event - notify VSI of a link event
6357 * @vsi: vsi to be notified
6358 * @link_up: link up or down
6359 **/
6360 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6361 {
6362 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
6363 return;
6364
6365 switch (vsi->type) {
6366 case I40E_VSI_MAIN:
6367 if (!vsi->netdev || !vsi->netdev_registered)
6368 break;
6369
6370 if (link_up) {
6371 netif_carrier_on(vsi->netdev);
6372 netif_tx_wake_all_queues(vsi->netdev);
6373 } else {
6374 netif_carrier_off(vsi->netdev);
6375 netif_tx_stop_all_queues(vsi->netdev);
6376 }
6377 break;
6378
6379 case I40E_VSI_SRIOV:
6380 case I40E_VSI_VMDQ2:
6381 case I40E_VSI_CTRL:
6382 case I40E_VSI_IWARP:
6383 case I40E_VSI_MIRROR:
6384 default:
6385 /* there is no notification for other VSIs */
6386 break;
6387 }
6388 }
6389
6390 /**
6391 * i40e_veb_link_event - notify elements on the veb of a link event
6392 * @veb: veb to be notified
6393 * @link_up: link up or down
6394 **/
6395 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6396 {
6397 struct i40e_pf *pf;
6398 int i;
6399
6400 if (!veb || !veb->pf)
6401 return;
6402 pf = veb->pf;
6403
6404 /* depth first... */
6405 for (i = 0; i < I40E_MAX_VEB; i++)
6406 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6407 i40e_veb_link_event(pf->veb[i], link_up);
6408
6409 /* ... now the local VSIs */
6410 for (i = 0; i < pf->num_alloc_vsi; i++)
6411 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6412 i40e_vsi_link_event(pf->vsi[i], link_up);
6413 }
6414
6415 /**
6416 * i40e_link_event - Update netif_carrier status
6417 * @pf: board private structure
6418 **/
6419 static void i40e_link_event(struct i40e_pf *pf)
6420 {
6421 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6422 u8 new_link_speed, old_link_speed;
6423 i40e_status status;
6424 bool new_link, old_link;
6425
6426 /* save off old link status information */
6427 pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6428
6429 /* set this to force the get_link_status call to refresh state */
6430 pf->hw.phy.get_link_info = true;
6431
6432 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6433
6434 status = i40e_get_link_status(&pf->hw, &new_link);
6435
6436 /* On success, disable temp link polling */
6437 if (status == I40E_SUCCESS) {
6438 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
6439 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
6440 } else {
6441 /* Enable link polling temporarily until i40e_get_link_status
6442 * returns I40E_SUCCESS
6443 */
6444 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
6445 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6446 status);
6447 return;
6448 }
6449
6450 old_link_speed = pf->hw.phy.link_info_old.link_speed;
6451 new_link_speed = pf->hw.phy.link_info.link_speed;
6452
6453 if (new_link == old_link &&
6454 new_link_speed == old_link_speed &&
6455 (test_bit(__I40E_VSI_DOWN, vsi->state) ||
6456 new_link == netif_carrier_ok(vsi->netdev)))
6457 return;
6458
6459 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
6460 i40e_print_link_message(vsi, new_link);
6461
6462 /* Notify the base of the switch tree connected to
6463 * the link. Floating VEBs are not notified.
6464 */
6465 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6466 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6467 else
6468 i40e_vsi_link_event(vsi, new_link);
6469
6470 if (pf->vf)
6471 i40e_vc_notify_link_state(pf);
6472
6473 if (pf->flags & I40E_FLAG_PTP)
6474 i40e_ptp_set_increment(pf);
6475 }
6476
6477 /**
6478 * i40e_watchdog_subtask - periodic checks not using event driven response
6479 * @pf: board private structure
6480 **/
6481 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6482 {
6483 int i;
6484
6485 /* if interface is down do nothing */
6486 if (test_bit(__I40E_DOWN, pf->state) ||
6487 test_bit(__I40E_CONFIG_BUSY, pf->state))
6488 return;
6489
6490 /* make sure we don't do these things too often */
6491 if (time_before(jiffies, (pf->service_timer_previous +
6492 pf->service_timer_period)))
6493 return;
6494 pf->service_timer_previous = jiffies;
6495
6496 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
6497 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
6498 i40e_link_event(pf);
6499
6500 /* Update the stats for active netdevs so the network stack
6501 * can look at updated numbers whenever it cares to
6502 */
6503 for (i = 0; i < pf->num_alloc_vsi; i++)
6504 if (pf->vsi[i] && pf->vsi[i]->netdev)
6505 i40e_update_stats(pf->vsi[i]);
6506
6507 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6508 /* Update the stats for the active switching components */
6509 for (i = 0; i < I40E_MAX_VEB; i++)
6510 if (pf->veb[i])
6511 i40e_update_veb_stats(pf->veb[i]);
6512 }
6513
6514 i40e_ptp_rx_hang(pf);
6515 i40e_ptp_tx_hang(pf);
6516 }
6517
6518 /**
6519 * i40e_reset_subtask - Set up for resetting the device and driver
6520 * @pf: board private structure
6521 **/
6522 static void i40e_reset_subtask(struct i40e_pf *pf)
6523 {
6524 u32 reset_flags = 0;
6525
6526 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
6527 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6528 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
6529 }
6530 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
6531 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6532 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6533 }
6534 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
6535 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6536 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
6537 }
6538 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
6539 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6540 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
6541 }
6542 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
6543 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6544 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
6545 }
6546
6547 /* If there's a recovery already waiting, it takes
6548 * precedence before starting a new reset sequence.
6549 */
6550 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
6551 i40e_prep_for_reset(pf, false);
6552 i40e_reset(pf);
6553 i40e_rebuild(pf, false, false);
6554 }
6555
6556 /* If we're already down or resetting, just bail */
6557 if (reset_flags &&
6558 !test_bit(__I40E_DOWN, pf->state) &&
6559 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
6560 rtnl_lock();
6561 i40e_do_reset(pf, reset_flags, true);
6562 rtnl_unlock();
6563 }
6564 }
6565
6566 /**
6567 * i40e_handle_link_event - Handle link event
6568 * @pf: board private structure
6569 * @e: event info posted on ARQ
6570 **/
6571 static void i40e_handle_link_event(struct i40e_pf *pf,
6572 struct i40e_arq_event_info *e)
6573 {
6574 struct i40e_aqc_get_link_status *status =
6575 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6576
6577 /* Do a new status request to re-enable LSE reporting
6578 * and load new status information into the hw struct
6579 * This completely ignores any state information
6580 * in the ARQ event info, instead choosing to always
6581 * issue the AQ update link status command.
6582 */
6583 i40e_link_event(pf);
6584
6585 /* check for unqualified module, if link is down */
6586 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6587 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6588 (!(status->link_info & I40E_AQ_LINK_UP)))
6589 dev_err(&pf->pdev->dev,
6590 "The driver failed to link because an unqualified module was detected.\n");
6591 }
6592
6593 /**
6594 * i40e_clean_adminq_subtask - Clean the AdminQ rings
6595 * @pf: board private structure
6596 **/
6597 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6598 {
6599 struct i40e_arq_event_info event;
6600 struct i40e_hw *hw = &pf->hw;
6601 u16 pending, i = 0;
6602 i40e_status ret;
6603 u16 opcode;
6604 u32 oldval;
6605 u32 val;
6606
6607 /* Do not run clean AQ when PF reset fails */
6608 if (test_bit(__I40E_RESET_FAILED, pf->state))
6609 return;
6610
6611 /* check for error indications */
6612 val = rd32(&pf->hw, pf->hw.aq.arq.len);
6613 oldval = val;
6614 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6615 if (hw->debug_mask & I40E_DEBUG_AQ)
6616 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6617 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6618 }
6619 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6620 if (hw->debug_mask & I40E_DEBUG_AQ)
6621 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6622 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6623 pf->arq_overflows++;
6624 }
6625 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6626 if (hw->debug_mask & I40E_DEBUG_AQ)
6627 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6628 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6629 }
6630 if (oldval != val)
6631 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6632
6633 val = rd32(&pf->hw, pf->hw.aq.asq.len);
6634 oldval = val;
6635 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6636 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6637 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6638 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6639 }
6640 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6641 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6642 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6643 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6644 }
6645 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6646 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6647 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6648 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6649 }
6650 if (oldval != val)
6651 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6652
6653 event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6654 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6655 if (!event.msg_buf)
6656 return;
6657
6658 do {
6659 ret = i40e_clean_arq_element(hw, &event, &pending);
6660 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6661 break;
6662 else if (ret) {
6663 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6664 break;
6665 }
6666
6667 opcode = le16_to_cpu(event.desc.opcode);
6668 switch (opcode) {
6669
6670 case i40e_aqc_opc_get_link_status:
6671 i40e_handle_link_event(pf, &event);
6672 break;
6673 case i40e_aqc_opc_send_msg_to_pf:
6674 ret = i40e_vc_process_vf_msg(pf,
6675 le16_to_cpu(event.desc.retval),
6676 le32_to_cpu(event.desc.cookie_high),
6677 le32_to_cpu(event.desc.cookie_low),
6678 event.msg_buf,
6679 event.msg_len);
6680 break;
6681 case i40e_aqc_opc_lldp_update_mib:
6682 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6683 #ifdef CONFIG_I40E_DCB
6684 rtnl_lock();
6685 ret = i40e_handle_lldp_event(pf, &event);
6686 rtnl_unlock();
6687 #endif /* CONFIG_I40E_DCB */
6688 break;
6689 case i40e_aqc_opc_event_lan_overflow:
6690 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6691 i40e_handle_lan_overflow_event(pf, &event);
6692 break;
6693 case i40e_aqc_opc_send_msg_to_peer:
6694 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6695 break;
6696 case i40e_aqc_opc_nvm_erase:
6697 case i40e_aqc_opc_nvm_update:
6698 case i40e_aqc_opc_oem_post_update:
6699 i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6700 "ARQ NVM operation 0x%04x completed\n",
6701 opcode);
6702 break;
6703 default:
6704 dev_info(&pf->pdev->dev,
6705 "ARQ: Unknown event 0x%04x ignored\n",
6706 opcode);
6707 break;
6708 }
6709 } while (i++ < pf->adminq_work_limit);
6710
6711 if (i < pf->adminq_work_limit)
6712 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
6713
6714 /* re-enable Admin queue interrupt cause */
6715 val = rd32(hw, I40E_PFINT_ICR0_ENA);
6716 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6717 wr32(hw, I40E_PFINT_ICR0_ENA, val);
6718 i40e_flush(hw);
6719
6720 kfree(event.msg_buf);
6721 }
6722
6723 /**
6724 * i40e_verify_eeprom - make sure eeprom is good to use
6725 * @pf: board private structure
6726 **/
6727 static void i40e_verify_eeprom(struct i40e_pf *pf)
6728 {
6729 int err;
6730
6731 err = i40e_diag_eeprom_test(&pf->hw);
6732 if (err) {
6733 /* retry in case of garbage read */
6734 err = i40e_diag_eeprom_test(&pf->hw);
6735 if (err) {
6736 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6737 err);
6738 set_bit(__I40E_BAD_EEPROM, pf->state);
6739 }
6740 }
6741
6742 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
6743 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6744 clear_bit(__I40E_BAD_EEPROM, pf->state);
6745 }
6746 }
6747
6748 /**
6749 * i40e_enable_pf_switch_lb
6750 * @pf: pointer to the PF structure
6751 *
6752 * enable switch loop back or die - no point in a return value
6753 **/
6754 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6755 {
6756 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6757 struct i40e_vsi_context ctxt;
6758 int ret;
6759
6760 ctxt.seid = pf->main_vsi_seid;
6761 ctxt.pf_num = pf->hw.pf_id;
6762 ctxt.vf_num = 0;
6763 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6764 if (ret) {
6765 dev_info(&pf->pdev->dev,
6766 "couldn't get PF vsi config, err %s aq_err %s\n",
6767 i40e_stat_str(&pf->hw, ret),
6768 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6769 return;
6770 }
6771 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6772 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6773 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6774
6775 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6776 if (ret) {
6777 dev_info(&pf->pdev->dev,
6778 "update vsi switch failed, err %s aq_err %s\n",
6779 i40e_stat_str(&pf->hw, ret),
6780 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6781 }
6782 }
6783
6784 /**
6785 * i40e_disable_pf_switch_lb
6786 * @pf: pointer to the PF structure
6787 *
6788 * disable switch loop back or die - no point in a return value
6789 **/
6790 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6791 {
6792 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6793 struct i40e_vsi_context ctxt;
6794 int ret;
6795
6796 ctxt.seid = pf->main_vsi_seid;
6797 ctxt.pf_num = pf->hw.pf_id;
6798 ctxt.vf_num = 0;
6799 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6800 if (ret) {
6801 dev_info(&pf->pdev->dev,
6802 "couldn't get PF vsi config, err %s aq_err %s\n",
6803 i40e_stat_str(&pf->hw, ret),
6804 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6805 return;
6806 }
6807 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6808 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6809 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6810
6811 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6812 if (ret) {
6813 dev_info(&pf->pdev->dev,
6814 "update vsi switch failed, err %s aq_err %s\n",
6815 i40e_stat_str(&pf->hw, ret),
6816 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6817 }
6818 }
6819
6820 /**
6821 * i40e_config_bridge_mode - Configure the HW bridge mode
6822 * @veb: pointer to the bridge instance
6823 *
6824 * Configure the loop back mode for the LAN VSI that is downlink to the
6825 * specified HW bridge instance. It is expected this function is called
6826 * when a new HW bridge is instantiated.
6827 **/
6828 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6829 {
6830 struct i40e_pf *pf = veb->pf;
6831
6832 if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6833 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6834 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6835 if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6836 i40e_disable_pf_switch_lb(pf);
6837 else
6838 i40e_enable_pf_switch_lb(pf);
6839 }
6840
6841 /**
6842 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6843 * @veb: pointer to the VEB instance
6844 *
6845 * This is a recursive function that first builds the attached VSIs then
6846 * recurses in to build the next layer of VEB. We track the connections
6847 * through our own index numbers because the seid's from the HW could
6848 * change across the reset.
6849 **/
6850 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6851 {
6852 struct i40e_vsi *ctl_vsi = NULL;
6853 struct i40e_pf *pf = veb->pf;
6854 int v, veb_idx;
6855 int ret;
6856
6857 /* build VSI that owns this VEB, temporarily attached to base VEB */
6858 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6859 if (pf->vsi[v] &&
6860 pf->vsi[v]->veb_idx == veb->idx &&
6861 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6862 ctl_vsi = pf->vsi[v];
6863 break;
6864 }
6865 }
6866 if (!ctl_vsi) {
6867 dev_info(&pf->pdev->dev,
6868 "missing owner VSI for veb_idx %d\n", veb->idx);
6869 ret = -ENOENT;
6870 goto end_reconstitute;
6871 }
6872 if (ctl_vsi != pf->vsi[pf->lan_vsi])
6873 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6874 ret = i40e_add_vsi(ctl_vsi);
6875 if (ret) {
6876 dev_info(&pf->pdev->dev,
6877 "rebuild of veb_idx %d owner VSI failed: %d\n",
6878 veb->idx, ret);
6879 goto end_reconstitute;
6880 }
6881 i40e_vsi_reset_stats(ctl_vsi);
6882
6883 /* create the VEB in the switch and move the VSI onto the VEB */
6884 ret = i40e_add_veb(veb, ctl_vsi);
6885 if (ret)
6886 goto end_reconstitute;
6887
6888 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6889 veb->bridge_mode = BRIDGE_MODE_VEB;
6890 else
6891 veb->bridge_mode = BRIDGE_MODE_VEPA;
6892 i40e_config_bridge_mode(veb);
6893
6894 /* create the remaining VSIs attached to this VEB */
6895 for (v = 0; v < pf->num_alloc_vsi; v++) {
6896 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6897 continue;
6898
6899 if (pf->vsi[v]->veb_idx == veb->idx) {
6900 struct i40e_vsi *vsi = pf->vsi[v];
6901
6902 vsi->uplink_seid = veb->seid;
6903 ret = i40e_add_vsi(vsi);
6904 if (ret) {
6905 dev_info(&pf->pdev->dev,
6906 "rebuild of vsi_idx %d failed: %d\n",
6907 v, ret);
6908 goto end_reconstitute;
6909 }
6910 i40e_vsi_reset_stats(vsi);
6911 }
6912 }
6913
6914 /* create any VEBs attached to this VEB - RECURSION */
6915 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6916 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6917 pf->veb[veb_idx]->uplink_seid = veb->seid;
6918 ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6919 if (ret)
6920 break;
6921 }
6922 }
6923
6924 end_reconstitute:
6925 return ret;
6926 }
6927
6928 /**
6929 * i40e_get_capabilities - get info about the HW
6930 * @pf: the PF struct
6931 **/
6932 static int i40e_get_capabilities(struct i40e_pf *pf)
6933 {
6934 struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6935 u16 data_size;
6936 int buf_len;
6937 int err;
6938
6939 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6940 do {
6941 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6942 if (!cap_buf)
6943 return -ENOMEM;
6944
6945 /* this loads the data into the hw struct for us */
6946 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6947 &data_size,
6948 i40e_aqc_opc_list_func_capabilities,
6949 NULL);
6950 /* data loaded, buffer no longer needed */
6951 kfree(cap_buf);
6952
6953 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6954 /* retry with a larger buffer */
6955 buf_len = data_size;
6956 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6957 dev_info(&pf->pdev->dev,
6958 "capability discovery failed, err %s aq_err %s\n",
6959 i40e_stat_str(&pf->hw, err),
6960 i40e_aq_str(&pf->hw,
6961 pf->hw.aq.asq_last_status));
6962 return -ENODEV;
6963 }
6964 } while (err);
6965
6966 if (pf->hw.debug_mask & I40E_DEBUG_USER)
6967 dev_info(&pf->pdev->dev,
6968 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6969 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6970 pf->hw.func_caps.num_msix_vectors,
6971 pf->hw.func_caps.num_msix_vectors_vf,
6972 pf->hw.func_caps.fd_filters_guaranteed,
6973 pf->hw.func_caps.fd_filters_best_effort,
6974 pf->hw.func_caps.num_tx_qp,
6975 pf->hw.func_caps.num_vsis);
6976
6977 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6978 + pf->hw.func_caps.num_vfs)
6979 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6980 dev_info(&pf->pdev->dev,
6981 "got num_vsis %d, setting num_vsis to %d\n",
6982 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6983 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6984 }
6985
6986 return 0;
6987 }
6988
6989 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6990
6991 /**
6992 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6993 * @pf: board private structure
6994 **/
6995 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6996 {
6997 struct i40e_vsi *vsi;
6998
6999 /* quick workaround for an NVM issue that leaves a critical register
7000 * uninitialized
7001 */
7002 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
7003 static const u32 hkey[] = {
7004 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
7005 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
7006 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
7007 0x95b3a76d};
7008 int i;
7009
7010 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
7011 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
7012 }
7013
7014 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
7015 return;
7016
7017 /* find existing VSI and see if it needs configuring */
7018 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7019
7020 /* create a new VSI if none exists */
7021 if (!vsi) {
7022 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
7023 pf->vsi[pf->lan_vsi]->seid, 0);
7024 if (!vsi) {
7025 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
7026 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7027 return;
7028 }
7029 }
7030
7031 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
7032 }
7033
7034 /**
7035 * i40e_fdir_teardown - release the Flow Director resources
7036 * @pf: board private structure
7037 **/
7038 static void i40e_fdir_teardown(struct i40e_pf *pf)
7039 {
7040 struct i40e_vsi *vsi;
7041
7042 i40e_fdir_filter_exit(pf);
7043 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7044 if (vsi)
7045 i40e_vsi_release(vsi);
7046 }
7047
7048 /**
7049 * i40e_prep_for_reset - prep for the core to reset
7050 * @pf: board private structure
7051 * @lock_acquired: indicates whether or not the lock has been acquired
7052 * before this function was called.
7053 *
7054 * Close up the VFs and other things in prep for PF Reset.
7055 **/
7056 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
7057 {
7058 struct i40e_hw *hw = &pf->hw;
7059 i40e_status ret = 0;
7060 u32 v;
7061
7062 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
7063 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7064 return;
7065 if (i40e_check_asq_alive(&pf->hw))
7066 i40e_vc_notify_reset(pf);
7067
7068 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
7069
7070 /* quiesce the VSIs and their queues that are not already DOWN */
7071 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
7072 if (!lock_acquired)
7073 rtnl_lock();
7074 i40e_pf_quiesce_all_vsi(pf);
7075 if (!lock_acquired)
7076 rtnl_unlock();
7077
7078 for (v = 0; v < pf->num_alloc_vsi; v++) {
7079 if (pf->vsi[v])
7080 pf->vsi[v]->seid = 0;
7081 }
7082
7083 i40e_shutdown_adminq(&pf->hw);
7084
7085 /* call shutdown HMC */
7086 if (hw->hmc.hmc_obj) {
7087 ret = i40e_shutdown_lan_hmc(hw);
7088 if (ret)
7089 dev_warn(&pf->pdev->dev,
7090 "shutdown_lan_hmc failed: %d\n", ret);
7091 }
7092 }
7093
7094 /**
7095 * i40e_send_version - update firmware with driver version
7096 * @pf: PF struct
7097 */
7098 static void i40e_send_version(struct i40e_pf *pf)
7099 {
7100 struct i40e_driver_version dv;
7101
7102 dv.major_version = DRV_VERSION_MAJOR;
7103 dv.minor_version = DRV_VERSION_MINOR;
7104 dv.build_version = DRV_VERSION_BUILD;
7105 dv.subbuild_version = 0;
7106 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
7107 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7108 }
7109
7110 /**
7111 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
7112 * @pf: board private structure
7113 **/
7114 static int i40e_reset(struct i40e_pf *pf)
7115 {
7116 struct i40e_hw *hw = &pf->hw;
7117 i40e_status ret;
7118
7119 ret = i40e_pf_reset(hw);
7120 if (ret) {
7121 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
7122 set_bit(__I40E_RESET_FAILED, pf->state);
7123 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7124 } else {
7125 pf->pfr_count++;
7126 }
7127 return ret;
7128 }
7129
7130 /**
7131 * i40e_rebuild - rebuild using a saved config
7132 * @pf: board private structure
7133 * @reinit: if the Main VSI needs to re-initialized.
7134 * @lock_acquired: indicates whether or not the lock has been acquired
7135 * before this function was called.
7136 **/
7137 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
7138 {
7139 struct i40e_hw *hw = &pf->hw;
7140 u8 set_fc_aq_fail = 0;
7141 i40e_status ret;
7142 u32 val;
7143 int v;
7144
7145 if (test_bit(__I40E_DOWN, pf->state))
7146 goto clear_recovery;
7147 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
7148
7149 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
7150 ret = i40e_init_adminq(&pf->hw);
7151 if (ret) {
7152 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
7153 i40e_stat_str(&pf->hw, ret),
7154 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7155 goto clear_recovery;
7156 }
7157
7158 /* re-verify the eeprom if we just had an EMP reset */
7159 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
7160 i40e_verify_eeprom(pf);
7161
7162 i40e_clear_pxe_mode(hw);
7163 ret = i40e_get_capabilities(pf);
7164 if (ret)
7165 goto end_core_reset;
7166
7167 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7168 hw->func_caps.num_rx_qp, 0, 0);
7169 if (ret) {
7170 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
7171 goto end_core_reset;
7172 }
7173 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7174 if (ret) {
7175 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
7176 goto end_core_reset;
7177 }
7178
7179 #ifdef CONFIG_I40E_DCB
7180 ret = i40e_init_pf_dcb(pf);
7181 if (ret) {
7182 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
7183 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
7184 /* Continue without DCB enabled */
7185 }
7186 #endif /* CONFIG_I40E_DCB */
7187 /* do basic switch setup */
7188 if (!lock_acquired)
7189 rtnl_lock();
7190 ret = i40e_setup_pf_switch(pf, reinit);
7191 if (ret)
7192 goto end_unlock;
7193
7194 /* The driver only wants link up/down and module qualification
7195 * reports from firmware. Note the negative logic.
7196 */
7197 ret = i40e_aq_set_phy_int_mask(&pf->hw,
7198 ~(I40E_AQ_EVENT_LINK_UPDOWN |
7199 I40E_AQ_EVENT_MEDIA_NA |
7200 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7201 if (ret)
7202 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7203 i40e_stat_str(&pf->hw, ret),
7204 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7205
7206 /* make sure our flow control settings are restored */
7207 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7208 if (ret)
7209 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7210 i40e_stat_str(&pf->hw, ret),
7211 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7212
7213 /* Rebuild the VSIs and VEBs that existed before reset.
7214 * They are still in our local switch element arrays, so only
7215 * need to rebuild the switch model in the HW.
7216 *
7217 * If there were VEBs but the reconstitution failed, we'll try
7218 * try to recover minimal use by getting the basic PF VSI working.
7219 */
7220 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7221 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7222 /* find the one VEB connected to the MAC, and find orphans */
7223 for (v = 0; v < I40E_MAX_VEB; v++) {
7224 if (!pf->veb[v])
7225 continue;
7226
7227 if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7228 pf->veb[v]->uplink_seid == 0) {
7229 ret = i40e_reconstitute_veb(pf->veb[v]);
7230
7231 if (!ret)
7232 continue;
7233
7234 /* If Main VEB failed, we're in deep doodoo,
7235 * so give up rebuilding the switch and set up
7236 * for minimal rebuild of PF VSI.
7237 * If orphan failed, we'll report the error
7238 * but try to keep going.
7239 */
7240 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7241 dev_info(&pf->pdev->dev,
7242 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7243 ret);
7244 pf->vsi[pf->lan_vsi]->uplink_seid
7245 = pf->mac_seid;
7246 break;
7247 } else if (pf->veb[v]->uplink_seid == 0) {
7248 dev_info(&pf->pdev->dev,
7249 "rebuild of orphan VEB failed: %d\n",
7250 ret);
7251 }
7252 }
7253 }
7254 }
7255
7256 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7257 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7258 /* no VEB, so rebuild only the Main VSI */
7259 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7260 if (ret) {
7261 dev_info(&pf->pdev->dev,
7262 "rebuild of Main VSI failed: %d\n", ret);
7263 goto end_unlock;
7264 }
7265 }
7266
7267 /* Reconfigure hardware for allowing smaller MSS in the case
7268 * of TSO, so that we avoid the MDD being fired and causing
7269 * a reset in the case of small MSS+TSO.
7270 */
7271 #define I40E_REG_MSS 0x000E64DC
7272 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7273 #define I40E_64BYTE_MSS 0x400000
7274 val = rd32(hw, I40E_REG_MSS);
7275 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7276 val &= ~I40E_REG_MSS_MIN_MASK;
7277 val |= I40E_64BYTE_MSS;
7278 wr32(hw, I40E_REG_MSS, val);
7279 }
7280
7281 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
7282 msleep(75);
7283 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7284 if (ret)
7285 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7286 i40e_stat_str(&pf->hw, ret),
7287 i40e_aq_str(&pf->hw,
7288 pf->hw.aq.asq_last_status));
7289 }
7290 /* reinit the misc interrupt */
7291 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7292 ret = i40e_setup_misc_vector(pf);
7293
7294 /* Add a filter to drop all Flow control frames from any VSI from being
7295 * transmitted. By doing so we stop a malicious VF from sending out
7296 * PAUSE or PFC frames and potentially controlling traffic for other
7297 * PF/VF VSIs.
7298 * The FW can still send Flow control frames if enabled.
7299 */
7300 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7301 pf->main_vsi_seid);
7302
7303 /* restart the VSIs that were rebuilt and running before the reset */
7304 i40e_pf_unquiesce_all_vsi(pf);
7305
7306 /* Release the RTNL lock before we start resetting VFs */
7307 if (!lock_acquired)
7308 rtnl_unlock();
7309
7310 i40e_reset_all_vfs(pf, true);
7311
7312 /* tell the firmware that we're starting */
7313 i40e_send_version(pf);
7314
7315 /* We've already released the lock, so don't do it again */
7316 goto end_core_reset;
7317
7318 end_unlock:
7319 if (!lock_acquired)
7320 rtnl_unlock();
7321 end_core_reset:
7322 clear_bit(__I40E_RESET_FAILED, pf->state);
7323 clear_recovery:
7324 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7325 }
7326
7327 /**
7328 * i40e_reset_and_rebuild - reset and rebuild using a saved config
7329 * @pf: board private structure
7330 * @reinit: if the Main VSI needs to re-initialized.
7331 * @lock_acquired: indicates whether or not the lock has been acquired
7332 * before this function was called.
7333 **/
7334 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
7335 bool lock_acquired)
7336 {
7337 int ret;
7338 /* Now we wait for GRST to settle out.
7339 * We don't have to delete the VEBs or VSIs from the hw switch
7340 * because the reset will make them disappear.
7341 */
7342 ret = i40e_reset(pf);
7343 if (!ret)
7344 i40e_rebuild(pf, reinit, lock_acquired);
7345 }
7346
7347 /**
7348 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7349 * @pf: board private structure
7350 *
7351 * Close up the VFs and other things in prep for a Core Reset,
7352 * then get ready to rebuild the world.
7353 * @lock_acquired: indicates whether or not the lock has been acquired
7354 * before this function was called.
7355 **/
7356 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
7357 {
7358 i40e_prep_for_reset(pf, lock_acquired);
7359 i40e_reset_and_rebuild(pf, false, lock_acquired);
7360 }
7361
7362 /**
7363 * i40e_handle_mdd_event
7364 * @pf: pointer to the PF structure
7365 *
7366 * Called from the MDD irq handler to identify possibly malicious vfs
7367 **/
7368 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7369 {
7370 struct i40e_hw *hw = &pf->hw;
7371 bool mdd_detected = false;
7372 bool pf_mdd_detected = false;
7373 struct i40e_vf *vf;
7374 u32 reg;
7375 int i;
7376
7377 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
7378 return;
7379
7380 /* find what triggered the MDD event */
7381 reg = rd32(hw, I40E_GL_MDET_TX);
7382 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7383 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7384 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7385 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7386 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7387 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7388 I40E_GL_MDET_TX_EVENT_SHIFT;
7389 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7390 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7391 pf->hw.func_caps.base_queue;
7392 if (netif_msg_tx_err(pf))
7393 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7394 event, queue, pf_num, vf_num);
7395 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7396 mdd_detected = true;
7397 }
7398 reg = rd32(hw, I40E_GL_MDET_RX);
7399 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7400 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7401 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7402 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7403 I40E_GL_MDET_RX_EVENT_SHIFT;
7404 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7405 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7406 pf->hw.func_caps.base_queue;
7407 if (netif_msg_rx_err(pf))
7408 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7409 event, queue, func);
7410 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7411 mdd_detected = true;
7412 }
7413
7414 if (mdd_detected) {
7415 reg = rd32(hw, I40E_PF_MDET_TX);
7416 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7417 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7418 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7419 pf_mdd_detected = true;
7420 }
7421 reg = rd32(hw, I40E_PF_MDET_RX);
7422 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7423 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7424 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7425 pf_mdd_detected = true;
7426 }
7427 /* Queue belongs to the PF, initiate a reset */
7428 if (pf_mdd_detected) {
7429 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7430 i40e_service_event_schedule(pf);
7431 }
7432 }
7433
7434 /* see if one of the VFs needs its hand slapped */
7435 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7436 vf = &(pf->vf[i]);
7437 reg = rd32(hw, I40E_VP_MDET_TX(i));
7438 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7439 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7440 vf->num_mdd_events++;
7441 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7442 i);
7443 }
7444
7445 reg = rd32(hw, I40E_VP_MDET_RX(i));
7446 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7447 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7448 vf->num_mdd_events++;
7449 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7450 i);
7451 }
7452
7453 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7454 dev_info(&pf->pdev->dev,
7455 "Too many MDD events on VF %d, disabled\n", i);
7456 dev_info(&pf->pdev->dev,
7457 "Use PF Control I/F to re-enable the VF\n");
7458 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
7459 }
7460 }
7461
7462 /* re-enable mdd interrupt cause */
7463 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
7464 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7465 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7466 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7467 i40e_flush(hw);
7468 }
7469
7470 /**
7471 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
7472 * @pf: board private structure
7473 **/
7474 static void i40e_sync_udp_filters(struct i40e_pf *pf)
7475 {
7476 int i;
7477
7478 /* loop through and set pending bit for all active UDP filters */
7479 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7480 if (pf->udp_ports[i].port)
7481 pf->pending_udp_bitmap |= BIT_ULL(i);
7482 }
7483
7484 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
7485 }
7486
7487 /**
7488 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7489 * @pf: board private structure
7490 **/
7491 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7492 {
7493 struct i40e_hw *hw = &pf->hw;
7494 i40e_status ret;
7495 u16 port;
7496 int i;
7497
7498 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7499 return;
7500
7501 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7502
7503 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7504 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7505 pf->pending_udp_bitmap &= ~BIT_ULL(i);
7506 port = pf->udp_ports[i].port;
7507 if (port)
7508 ret = i40e_aq_add_udp_tunnel(hw, port,
7509 pf->udp_ports[i].type,
7510 NULL, NULL);
7511 else
7512 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7513
7514 if (ret) {
7515 dev_dbg(&pf->pdev->dev,
7516 "%s %s port %d, index %d failed, err %s aq_err %s\n",
7517 pf->udp_ports[i].type ? "vxlan" : "geneve",
7518 port ? "add" : "delete",
7519 port, i,
7520 i40e_stat_str(&pf->hw, ret),
7521 i40e_aq_str(&pf->hw,
7522 pf->hw.aq.asq_last_status));
7523 pf->udp_ports[i].port = 0;
7524 }
7525 }
7526 }
7527 }
7528
7529 /**
7530 * i40e_service_task - Run the driver's async subtasks
7531 * @work: pointer to work_struct containing our data
7532 **/
7533 static void i40e_service_task(struct work_struct *work)
7534 {
7535 struct i40e_pf *pf = container_of(work,
7536 struct i40e_pf,
7537 service_task);
7538 unsigned long start_time = jiffies;
7539
7540 /* don't bother with service tasks if a reset is in progress */
7541 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7542 return;
7543
7544 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
7545 return;
7546
7547 i40e_detect_recover_hung(pf);
7548 i40e_sync_filters_subtask(pf);
7549 i40e_reset_subtask(pf);
7550 i40e_handle_mdd_event(pf);
7551 i40e_vc_process_vflr_event(pf);
7552 i40e_watchdog_subtask(pf);
7553 i40e_fdir_reinit_subtask(pf);
7554 if (pf->flags & I40E_FLAG_CLIENT_RESET) {
7555 /* Client subtask will reopen next time through. */
7556 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
7557 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
7558 } else {
7559 i40e_client_subtask(pf);
7560 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
7561 i40e_notify_client_of_l2_param_changes(
7562 pf->vsi[pf->lan_vsi]);
7563 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
7564 }
7565 }
7566 i40e_sync_filters_subtask(pf);
7567 i40e_sync_udp_filters_subtask(pf);
7568 i40e_clean_adminq_subtask(pf);
7569
7570 /* flush memory to make sure state is correct before next watchdog */
7571 smp_mb__before_atomic();
7572 clear_bit(__I40E_SERVICE_SCHED, pf->state);
7573
7574 /* If the tasks have taken longer than one timer cycle or there
7575 * is more work to be done, reschedule the service task now
7576 * rather than wait for the timer to tick again.
7577 */
7578 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7579 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
7580 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
7581 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
7582 i40e_service_event_schedule(pf);
7583 }
7584
7585 /**
7586 * i40e_service_timer - timer callback
7587 * @data: pointer to PF struct
7588 **/
7589 static void i40e_service_timer(unsigned long data)
7590 {
7591 struct i40e_pf *pf = (struct i40e_pf *)data;
7592
7593 mod_timer(&pf->service_timer,
7594 round_jiffies(jiffies + pf->service_timer_period));
7595 i40e_service_event_schedule(pf);
7596 }
7597
7598 /**
7599 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7600 * @vsi: the VSI being configured
7601 **/
7602 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7603 {
7604 struct i40e_pf *pf = vsi->back;
7605
7606 switch (vsi->type) {
7607 case I40E_VSI_MAIN:
7608 vsi->alloc_queue_pairs = pf->num_lan_qps;
7609 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7610 I40E_REQ_DESCRIPTOR_MULTIPLE);
7611 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7612 vsi->num_q_vectors = pf->num_lan_msix;
7613 else
7614 vsi->num_q_vectors = 1;
7615
7616 break;
7617
7618 case I40E_VSI_FDIR:
7619 vsi->alloc_queue_pairs = 1;
7620 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7621 I40E_REQ_DESCRIPTOR_MULTIPLE);
7622 vsi->num_q_vectors = pf->num_fdsb_msix;
7623 break;
7624
7625 case I40E_VSI_VMDQ2:
7626 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7627 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7628 I40E_REQ_DESCRIPTOR_MULTIPLE);
7629 vsi->num_q_vectors = pf->num_vmdq_msix;
7630 break;
7631
7632 case I40E_VSI_SRIOV:
7633 vsi->alloc_queue_pairs = pf->num_vf_qps;
7634 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7635 I40E_REQ_DESCRIPTOR_MULTIPLE);
7636 break;
7637
7638 default:
7639 WARN_ON(1);
7640 return -ENODATA;
7641 }
7642
7643 return 0;
7644 }
7645
7646 /**
7647 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7648 * @type: VSI pointer
7649 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7650 *
7651 * On error: returns error code (negative)
7652 * On success: returns 0
7653 **/
7654 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7655 {
7656 struct i40e_ring **next_rings;
7657 int size;
7658 int ret = 0;
7659
7660 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
7661 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
7662 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
7663 vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7664 if (!vsi->tx_rings)
7665 return -ENOMEM;
7666 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
7667 if (i40e_enabled_xdp_vsi(vsi)) {
7668 vsi->xdp_rings = next_rings;
7669 next_rings += vsi->alloc_queue_pairs;
7670 }
7671 vsi->rx_rings = next_rings;
7672
7673 if (alloc_qvectors) {
7674 /* allocate memory for q_vector pointers */
7675 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7676 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7677 if (!vsi->q_vectors) {
7678 ret = -ENOMEM;
7679 goto err_vectors;
7680 }
7681 }
7682 return ret;
7683
7684 err_vectors:
7685 kfree(vsi->tx_rings);
7686 return ret;
7687 }
7688
7689 /**
7690 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7691 * @pf: board private structure
7692 * @type: type of VSI
7693 *
7694 * On error: returns error code (negative)
7695 * On success: returns vsi index in PF (positive)
7696 **/
7697 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7698 {
7699 int ret = -ENODEV;
7700 struct i40e_vsi *vsi;
7701 int vsi_idx;
7702 int i;
7703
7704 /* Need to protect the allocation of the VSIs at the PF level */
7705 mutex_lock(&pf->switch_mutex);
7706
7707 /* VSI list may be fragmented if VSI creation/destruction has
7708 * been happening. We can afford to do a quick scan to look
7709 * for any free VSIs in the list.
7710 *
7711 * find next empty vsi slot, looping back around if necessary
7712 */
7713 i = pf->next_vsi;
7714 while (i < pf->num_alloc_vsi && pf->vsi[i])
7715 i++;
7716 if (i >= pf->num_alloc_vsi) {
7717 i = 0;
7718 while (i < pf->next_vsi && pf->vsi[i])
7719 i++;
7720 }
7721
7722 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7723 vsi_idx = i; /* Found one! */
7724 } else {
7725 ret = -ENODEV;
7726 goto unlock_pf; /* out of VSI slots! */
7727 }
7728 pf->next_vsi = ++i;
7729
7730 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7731 if (!vsi) {
7732 ret = -ENOMEM;
7733 goto unlock_pf;
7734 }
7735 vsi->type = type;
7736 vsi->back = pf;
7737 set_bit(__I40E_VSI_DOWN, vsi->state);
7738 vsi->flags = 0;
7739 vsi->idx = vsi_idx;
7740 vsi->int_rate_limit = 0;
7741 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7742 pf->rss_table_size : 64;
7743 vsi->netdev_registered = false;
7744 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7745 hash_init(vsi->mac_filter_hash);
7746 vsi->irqs_ready = false;
7747
7748 ret = i40e_set_num_rings_in_vsi(vsi);
7749 if (ret)
7750 goto err_rings;
7751
7752 ret = i40e_vsi_alloc_arrays(vsi, true);
7753 if (ret)
7754 goto err_rings;
7755
7756 /* Setup default MSIX irq handler for VSI */
7757 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7758
7759 /* Initialize VSI lock */
7760 spin_lock_init(&vsi->mac_filter_hash_lock);
7761 pf->vsi[vsi_idx] = vsi;
7762 ret = vsi_idx;
7763 goto unlock_pf;
7764
7765 err_rings:
7766 pf->next_vsi = i - 1;
7767 kfree(vsi);
7768 unlock_pf:
7769 mutex_unlock(&pf->switch_mutex);
7770 return ret;
7771 }
7772
7773 /**
7774 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7775 * @type: VSI pointer
7776 * @free_qvectors: a bool to specify if q_vectors need to be freed.
7777 *
7778 * On error: returns error code (negative)
7779 * On success: returns 0
7780 **/
7781 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7782 {
7783 /* free the ring and vector containers */
7784 if (free_qvectors) {
7785 kfree(vsi->q_vectors);
7786 vsi->q_vectors = NULL;
7787 }
7788 kfree(vsi->tx_rings);
7789 vsi->tx_rings = NULL;
7790 vsi->rx_rings = NULL;
7791 vsi->xdp_rings = NULL;
7792 }
7793
7794 /**
7795 * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7796 * and lookup table
7797 * @vsi: Pointer to VSI structure
7798 */
7799 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7800 {
7801 if (!vsi)
7802 return;
7803
7804 kfree(vsi->rss_hkey_user);
7805 vsi->rss_hkey_user = NULL;
7806
7807 kfree(vsi->rss_lut_user);
7808 vsi->rss_lut_user = NULL;
7809 }
7810
7811 /**
7812 * i40e_vsi_clear - Deallocate the VSI provided
7813 * @vsi: the VSI being un-configured
7814 **/
7815 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7816 {
7817 struct i40e_pf *pf;
7818
7819 if (!vsi)
7820 return 0;
7821
7822 if (!vsi->back)
7823 goto free_vsi;
7824 pf = vsi->back;
7825
7826 mutex_lock(&pf->switch_mutex);
7827 if (!pf->vsi[vsi->idx]) {
7828 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7829 vsi->idx, vsi->idx, vsi, vsi->type);
7830 goto unlock_vsi;
7831 }
7832
7833 if (pf->vsi[vsi->idx] != vsi) {
7834 dev_err(&pf->pdev->dev,
7835 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7836 pf->vsi[vsi->idx]->idx,
7837 pf->vsi[vsi->idx],
7838 pf->vsi[vsi->idx]->type,
7839 vsi->idx, vsi, vsi->type);
7840 goto unlock_vsi;
7841 }
7842
7843 /* updates the PF for this cleared vsi */
7844 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7845 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7846
7847 i40e_vsi_free_arrays(vsi, true);
7848 i40e_clear_rss_config_user(vsi);
7849
7850 pf->vsi[vsi->idx] = NULL;
7851 if (vsi->idx < pf->next_vsi)
7852 pf->next_vsi = vsi->idx;
7853
7854 unlock_vsi:
7855 mutex_unlock(&pf->switch_mutex);
7856 free_vsi:
7857 kfree(vsi);
7858
7859 return 0;
7860 }
7861
7862 /**
7863 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7864 * @vsi: the VSI being cleaned
7865 **/
7866 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7867 {
7868 int i;
7869
7870 if (vsi->tx_rings && vsi->tx_rings[0]) {
7871 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7872 kfree_rcu(vsi->tx_rings[i], rcu);
7873 vsi->tx_rings[i] = NULL;
7874 vsi->rx_rings[i] = NULL;
7875 if (vsi->xdp_rings)
7876 vsi->xdp_rings[i] = NULL;
7877 }
7878 }
7879 }
7880
7881 /**
7882 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7883 * @vsi: the VSI being configured
7884 **/
7885 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7886 {
7887 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
7888 struct i40e_pf *pf = vsi->back;
7889 struct i40e_ring *ring;
7890
7891 /* Set basic values in the rings to be used later during open() */
7892 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7893 /* allocate space for both Tx and Rx in one shot */
7894 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
7895 if (!ring)
7896 goto err_out;
7897
7898 ring->queue_index = i;
7899 ring->reg_idx = vsi->base_queue + i;
7900 ring->ring_active = false;
7901 ring->vsi = vsi;
7902 ring->netdev = vsi->netdev;
7903 ring->dev = &pf->pdev->dev;
7904 ring->count = vsi->num_desc;
7905 ring->size = 0;
7906 ring->dcb_tc = 0;
7907 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
7908 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7909 ring->tx_itr_setting = pf->tx_itr_default;
7910 vsi->tx_rings[i] = ring++;
7911
7912 if (!i40e_enabled_xdp_vsi(vsi))
7913 goto setup_rx;
7914
7915 ring->queue_index = vsi->alloc_queue_pairs + i;
7916 ring->reg_idx = vsi->base_queue + ring->queue_index;
7917 ring->ring_active = false;
7918 ring->vsi = vsi;
7919 ring->netdev = NULL;
7920 ring->dev = &pf->pdev->dev;
7921 ring->count = vsi->num_desc;
7922 ring->size = 0;
7923 ring->dcb_tc = 0;
7924 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
7925 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7926 set_ring_xdp(ring);
7927 ring->tx_itr_setting = pf->tx_itr_default;
7928 vsi->xdp_rings[i] = ring++;
7929
7930 setup_rx:
7931 ring->queue_index = i;
7932 ring->reg_idx = vsi->base_queue + i;
7933 ring->ring_active = false;
7934 ring->vsi = vsi;
7935 ring->netdev = vsi->netdev;
7936 ring->dev = &pf->pdev->dev;
7937 ring->count = vsi->num_desc;
7938 ring->size = 0;
7939 ring->dcb_tc = 0;
7940 ring->rx_itr_setting = pf->rx_itr_default;
7941 vsi->rx_rings[i] = ring;
7942 }
7943
7944 return 0;
7945
7946 err_out:
7947 i40e_vsi_clear_rings(vsi);
7948 return -ENOMEM;
7949 }
7950
7951 /**
7952 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7953 * @pf: board private structure
7954 * @vectors: the number of MSI-X vectors to request
7955 *
7956 * Returns the number of vectors reserved, or error
7957 **/
7958 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7959 {
7960 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7961 I40E_MIN_MSIX, vectors);
7962 if (vectors < 0) {
7963 dev_info(&pf->pdev->dev,
7964 "MSI-X vector reservation failed: %d\n", vectors);
7965 vectors = 0;
7966 }
7967
7968 return vectors;
7969 }
7970
7971 /**
7972 * i40e_init_msix - Setup the MSIX capability
7973 * @pf: board private structure
7974 *
7975 * Work with the OS to set up the MSIX vectors needed.
7976 *
7977 * Returns the number of vectors reserved or negative on failure
7978 **/
7979 static int i40e_init_msix(struct i40e_pf *pf)
7980 {
7981 struct i40e_hw *hw = &pf->hw;
7982 int cpus, extra_vectors;
7983 int vectors_left;
7984 int v_budget, i;
7985 int v_actual;
7986 int iwarp_requested = 0;
7987
7988 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7989 return -ENODEV;
7990
7991 /* The number of vectors we'll request will be comprised of:
7992 * - Add 1 for "other" cause for Admin Queue events, etc.
7993 * - The number of LAN queue pairs
7994 * - Queues being used for RSS.
7995 * We don't need as many as max_rss_size vectors.
7996 * use rss_size instead in the calculation since that
7997 * is governed by number of cpus in the system.
7998 * - assumes symmetric Tx/Rx pairing
7999 * - The number of VMDq pairs
8000 * - The CPU count within the NUMA node if iWARP is enabled
8001 * Once we count this up, try the request.
8002 *
8003 * If we can't get what we want, we'll simplify to nearly nothing
8004 * and try again. If that still fails, we punt.
8005 */
8006 vectors_left = hw->func_caps.num_msix_vectors;
8007 v_budget = 0;
8008
8009 /* reserve one vector for miscellaneous handler */
8010 if (vectors_left) {
8011 v_budget++;
8012 vectors_left--;
8013 }
8014
8015 /* reserve some vectors for the main PF traffic queues. Initially we
8016 * only reserve at most 50% of the available vectors, in the case that
8017 * the number of online CPUs is large. This ensures that we can enable
8018 * extra features as well. Once we've enabled the other features, we
8019 * will use any remaining vectors to reach as close as we can to the
8020 * number of online CPUs.
8021 */
8022 cpus = num_online_cpus();
8023 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
8024 vectors_left -= pf->num_lan_msix;
8025
8026 /* reserve one vector for sideband flow director */
8027 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8028 if (vectors_left) {
8029 pf->num_fdsb_msix = 1;
8030 v_budget++;
8031 vectors_left--;
8032 } else {
8033 pf->num_fdsb_msix = 0;
8034 }
8035 }
8036
8037 /* can we reserve enough for iWARP? */
8038 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8039 iwarp_requested = pf->num_iwarp_msix;
8040
8041 if (!vectors_left)
8042 pf->num_iwarp_msix = 0;
8043 else if (vectors_left < pf->num_iwarp_msix)
8044 pf->num_iwarp_msix = 1;
8045 v_budget += pf->num_iwarp_msix;
8046 vectors_left -= pf->num_iwarp_msix;
8047 }
8048
8049 /* any vectors left over go for VMDq support */
8050 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
8051 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
8052 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
8053
8054 if (!vectors_left) {
8055 pf->num_vmdq_msix = 0;
8056 pf->num_vmdq_qps = 0;
8057 } else {
8058 /* if we're short on vectors for what's desired, we limit
8059 * the queues per vmdq. If this is still more than are
8060 * available, the user will need to change the number of
8061 * queues/vectors used by the PF later with the ethtool
8062 * channels command
8063 */
8064 if (vmdq_vecs < vmdq_vecs_wanted)
8065 pf->num_vmdq_qps = 1;
8066 pf->num_vmdq_msix = pf->num_vmdq_qps;
8067
8068 v_budget += vmdq_vecs;
8069 vectors_left -= vmdq_vecs;
8070 }
8071 }
8072
8073 /* On systems with a large number of SMP cores, we previously limited
8074 * the number of vectors for num_lan_msix to be at most 50% of the
8075 * available vectors, to allow for other features. Now, we add back
8076 * the remaining vectors. However, we ensure that the total
8077 * num_lan_msix will not exceed num_online_cpus(). To do this, we
8078 * calculate the number of vectors we can add without going over the
8079 * cap of CPUs. For systems with a small number of CPUs this will be
8080 * zero.
8081 */
8082 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
8083 pf->num_lan_msix += extra_vectors;
8084 vectors_left -= extra_vectors;
8085
8086 WARN(vectors_left < 0,
8087 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
8088
8089 v_budget += pf->num_lan_msix;
8090 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
8091 GFP_KERNEL);
8092 if (!pf->msix_entries)
8093 return -ENOMEM;
8094
8095 for (i = 0; i < v_budget; i++)
8096 pf->msix_entries[i].entry = i;
8097 v_actual = i40e_reserve_msix_vectors(pf, v_budget);
8098
8099 if (v_actual < I40E_MIN_MSIX) {
8100 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
8101 kfree(pf->msix_entries);
8102 pf->msix_entries = NULL;
8103 pci_disable_msix(pf->pdev);
8104 return -ENODEV;
8105
8106 } else if (v_actual == I40E_MIN_MSIX) {
8107 /* Adjust for minimal MSIX use */
8108 pf->num_vmdq_vsis = 0;
8109 pf->num_vmdq_qps = 0;
8110 pf->num_lan_qps = 1;
8111 pf->num_lan_msix = 1;
8112
8113 } else if (!vectors_left) {
8114 /* If we have limited resources, we will start with no vectors
8115 * for the special features and then allocate vectors to some
8116 * of these features based on the policy and at the end disable
8117 * the features that did not get any vectors.
8118 */
8119 int vec;
8120
8121 dev_info(&pf->pdev->dev,
8122 "MSI-X vector limit reached, attempting to redistribute vectors\n");
8123 /* reserve the misc vector */
8124 vec = v_actual - 1;
8125
8126 /* Scale vector usage down */
8127 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
8128 pf->num_vmdq_vsis = 1;
8129 pf->num_vmdq_qps = 1;
8130
8131 /* partition out the remaining vectors */
8132 switch (vec) {
8133 case 2:
8134 pf->num_lan_msix = 1;
8135 break;
8136 case 3:
8137 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8138 pf->num_lan_msix = 1;
8139 pf->num_iwarp_msix = 1;
8140 } else {
8141 pf->num_lan_msix = 2;
8142 }
8143 break;
8144 default:
8145 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8146 pf->num_iwarp_msix = min_t(int, (vec / 3),
8147 iwarp_requested);
8148 pf->num_vmdq_vsis = min_t(int, (vec / 3),
8149 I40E_DEFAULT_NUM_VMDQ_VSI);
8150 } else {
8151 pf->num_vmdq_vsis = min_t(int, (vec / 2),
8152 I40E_DEFAULT_NUM_VMDQ_VSI);
8153 }
8154 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8155 pf->num_fdsb_msix = 1;
8156 vec--;
8157 }
8158 pf->num_lan_msix = min_t(int,
8159 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
8160 pf->num_lan_msix);
8161 pf->num_lan_qps = pf->num_lan_msix;
8162 break;
8163 }
8164 }
8165
8166 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8167 (pf->num_fdsb_msix == 0)) {
8168 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
8169 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8170 }
8171 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8172 (pf->num_vmdq_msix == 0)) {
8173 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
8174 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
8175 }
8176
8177 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
8178 (pf->num_iwarp_msix == 0)) {
8179 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
8180 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
8181 }
8182 i40e_debug(&pf->hw, I40E_DEBUG_INIT,
8183 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
8184 pf->num_lan_msix,
8185 pf->num_vmdq_msix * pf->num_vmdq_vsis,
8186 pf->num_fdsb_msix,
8187 pf->num_iwarp_msix);
8188
8189 return v_actual;
8190 }
8191
8192 /**
8193 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
8194 * @vsi: the VSI being configured
8195 * @v_idx: index of the vector in the vsi struct
8196 * @cpu: cpu to be used on affinity_mask
8197 *
8198 * We allocate one q_vector. If allocation fails we return -ENOMEM.
8199 **/
8200 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
8201 {
8202 struct i40e_q_vector *q_vector;
8203
8204 /* allocate q_vector */
8205 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
8206 if (!q_vector)
8207 return -ENOMEM;
8208
8209 q_vector->vsi = vsi;
8210 q_vector->v_idx = v_idx;
8211 cpumask_set_cpu(cpu, &q_vector->affinity_mask);
8212
8213 if (vsi->netdev)
8214 netif_napi_add(vsi->netdev, &q_vector->napi,
8215 i40e_napi_poll, NAPI_POLL_WEIGHT);
8216
8217 q_vector->rx.latency_range = I40E_LOW_LATENCY;
8218 q_vector->tx.latency_range = I40E_LOW_LATENCY;
8219
8220 /* tie q_vector and vsi together */
8221 vsi->q_vectors[v_idx] = q_vector;
8222
8223 return 0;
8224 }
8225
8226 /**
8227 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
8228 * @vsi: the VSI being configured
8229 *
8230 * We allocate one q_vector per queue interrupt. If allocation fails we
8231 * return -ENOMEM.
8232 **/
8233 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
8234 {
8235 struct i40e_pf *pf = vsi->back;
8236 int err, v_idx, num_q_vectors, current_cpu;
8237
8238 /* if not MSIX, give the one vector only to the LAN VSI */
8239 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
8240 num_q_vectors = vsi->num_q_vectors;
8241 else if (vsi == pf->vsi[pf->lan_vsi])
8242 num_q_vectors = 1;
8243 else
8244 return -EINVAL;
8245
8246 current_cpu = cpumask_first(cpu_online_mask);
8247
8248 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
8249 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
8250 if (err)
8251 goto err_out;
8252 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8253 if (unlikely(current_cpu >= nr_cpu_ids))
8254 current_cpu = cpumask_first(cpu_online_mask);
8255 }
8256
8257 return 0;
8258
8259 err_out:
8260 while (v_idx--)
8261 i40e_free_q_vector(vsi, v_idx);
8262
8263 return err;
8264 }
8265
8266 /**
8267 * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8268 * @pf: board private structure to initialize
8269 **/
8270 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8271 {
8272 int vectors = 0;
8273 ssize_t size;
8274
8275 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8276 vectors = i40e_init_msix(pf);
8277 if (vectors < 0) {
8278 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED |
8279 I40E_FLAG_IWARP_ENABLED |
8280 I40E_FLAG_RSS_ENABLED |
8281 I40E_FLAG_DCB_CAPABLE |
8282 I40E_FLAG_DCB_ENABLED |
8283 I40E_FLAG_SRIOV_ENABLED |
8284 I40E_FLAG_FD_SB_ENABLED |
8285 I40E_FLAG_FD_ATR_ENABLED |
8286 I40E_FLAG_VMDQ_ENABLED);
8287
8288 /* rework the queue expectations without MSIX */
8289 i40e_determine_queue_usage(pf);
8290 }
8291 }
8292
8293 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8294 (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8295 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8296 vectors = pci_enable_msi(pf->pdev);
8297 if (vectors < 0) {
8298 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8299 vectors);
8300 pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8301 }
8302 vectors = 1; /* one MSI or Legacy vector */
8303 }
8304
8305 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8306 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8307
8308 /* set up vector assignment tracking */
8309 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8310 pf->irq_pile = kzalloc(size, GFP_KERNEL);
8311 if (!pf->irq_pile) {
8312 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8313 return -ENOMEM;
8314 }
8315 pf->irq_pile->num_entries = vectors;
8316 pf->irq_pile->search_hint = 0;
8317
8318 /* track first vector for misc interrupts, ignore return */
8319 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8320
8321 return 0;
8322 }
8323
8324 /**
8325 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8326 * @pf: board private structure
8327 *
8328 * This sets up the handler for MSIX 0, which is used to manage the
8329 * non-queue interrupts, e.g. AdminQ and errors. This is not used
8330 * when in MSI or Legacy interrupt mode.
8331 **/
8332 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8333 {
8334 struct i40e_hw *hw = &pf->hw;
8335 int err = 0;
8336
8337 /* Only request the irq if this is the first time through, and
8338 * not when we're rebuilding after a Reset
8339 */
8340 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
8341 err = request_irq(pf->msix_entries[0].vector,
8342 i40e_intr, 0, pf->int_name, pf);
8343 if (err) {
8344 dev_info(&pf->pdev->dev,
8345 "request_irq for %s failed: %d\n",
8346 pf->int_name, err);
8347 return -EFAULT;
8348 }
8349 }
8350
8351 i40e_enable_misc_int_causes(pf);
8352
8353 /* associate no queues to the misc vector */
8354 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8355 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8356
8357 i40e_flush(hw);
8358
8359 i40e_irq_dynamic_enable_icr0(pf, true);
8360
8361 return err;
8362 }
8363
8364 /**
8365 * i40e_config_rss_aq - Prepare for RSS using AQ commands
8366 * @vsi: vsi structure
8367 * @seed: RSS hash seed
8368 **/
8369 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8370 u8 *lut, u16 lut_size)
8371 {
8372 struct i40e_pf *pf = vsi->back;
8373 struct i40e_hw *hw = &pf->hw;
8374 int ret = 0;
8375
8376 if (seed) {
8377 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8378 (struct i40e_aqc_get_set_rss_key_data *)seed;
8379 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8380 if (ret) {
8381 dev_info(&pf->pdev->dev,
8382 "Cannot set RSS key, err %s aq_err %s\n",
8383 i40e_stat_str(hw, ret),
8384 i40e_aq_str(hw, hw->aq.asq_last_status));
8385 return ret;
8386 }
8387 }
8388 if (lut) {
8389 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8390
8391 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8392 if (ret) {
8393 dev_info(&pf->pdev->dev,
8394 "Cannot set RSS lut, err %s aq_err %s\n",
8395 i40e_stat_str(hw, ret),
8396 i40e_aq_str(hw, hw->aq.asq_last_status));
8397 return ret;
8398 }
8399 }
8400 return ret;
8401 }
8402
8403 /**
8404 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8405 * @vsi: Pointer to vsi structure
8406 * @seed: Buffter to store the hash keys
8407 * @lut: Buffer to store the lookup table entries
8408 * @lut_size: Size of buffer to store the lookup table entries
8409 *
8410 * Return 0 on success, negative on failure
8411 */
8412 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8413 u8 *lut, u16 lut_size)
8414 {
8415 struct i40e_pf *pf = vsi->back;
8416 struct i40e_hw *hw = &pf->hw;
8417 int ret = 0;
8418
8419 if (seed) {
8420 ret = i40e_aq_get_rss_key(hw, vsi->id,
8421 (struct i40e_aqc_get_set_rss_key_data *)seed);
8422 if (ret) {
8423 dev_info(&pf->pdev->dev,
8424 "Cannot get RSS key, err %s aq_err %s\n",
8425 i40e_stat_str(&pf->hw, ret),
8426 i40e_aq_str(&pf->hw,
8427 pf->hw.aq.asq_last_status));
8428 return ret;
8429 }
8430 }
8431
8432 if (lut) {
8433 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8434
8435 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8436 if (ret) {
8437 dev_info(&pf->pdev->dev,
8438 "Cannot get RSS lut, err %s aq_err %s\n",
8439 i40e_stat_str(&pf->hw, ret),
8440 i40e_aq_str(&pf->hw,
8441 pf->hw.aq.asq_last_status));
8442 return ret;
8443 }
8444 }
8445
8446 return ret;
8447 }
8448
8449 /**
8450 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8451 * @vsi: VSI structure
8452 **/
8453 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8454 {
8455 u8 seed[I40E_HKEY_ARRAY_SIZE];
8456 struct i40e_pf *pf = vsi->back;
8457 u8 *lut;
8458 int ret;
8459
8460 if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
8461 return 0;
8462
8463 if (!vsi->rss_size)
8464 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8465 vsi->num_queue_pairs);
8466 if (!vsi->rss_size)
8467 return -EINVAL;
8468
8469 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8470 if (!lut)
8471 return -ENOMEM;
8472 /* Use the user configured hash keys and lookup table if there is one,
8473 * otherwise use default
8474 */
8475 if (vsi->rss_lut_user)
8476 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8477 else
8478 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8479 if (vsi->rss_hkey_user)
8480 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8481 else
8482 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8483 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8484 kfree(lut);
8485
8486 return ret;
8487 }
8488
8489 /**
8490 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8491 * @vsi: Pointer to vsi structure
8492 * @seed: RSS hash seed
8493 * @lut: Lookup table
8494 * @lut_size: Lookup table size
8495 *
8496 * Returns 0 on success, negative on failure
8497 **/
8498 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8499 const u8 *lut, u16 lut_size)
8500 {
8501 struct i40e_pf *pf = vsi->back;
8502 struct i40e_hw *hw = &pf->hw;
8503 u16 vf_id = vsi->vf_id;
8504 u8 i;
8505
8506 /* Fill out hash function seed */
8507 if (seed) {
8508 u32 *seed_dw = (u32 *)seed;
8509
8510 if (vsi->type == I40E_VSI_MAIN) {
8511 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8512 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8513 } else if (vsi->type == I40E_VSI_SRIOV) {
8514 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8515 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
8516 } else {
8517 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8518 }
8519 }
8520
8521 if (lut) {
8522 u32 *lut_dw = (u32 *)lut;
8523
8524 if (vsi->type == I40E_VSI_MAIN) {
8525 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8526 return -EINVAL;
8527 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8528 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8529 } else if (vsi->type == I40E_VSI_SRIOV) {
8530 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8531 return -EINVAL;
8532 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8533 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
8534 } else {
8535 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8536 }
8537 }
8538 i40e_flush(hw);
8539
8540 return 0;
8541 }
8542
8543 /**
8544 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8545 * @vsi: Pointer to VSI structure
8546 * @seed: Buffer to store the keys
8547 * @lut: Buffer to store the lookup table entries
8548 * @lut_size: Size of buffer to store the lookup table entries
8549 *
8550 * Returns 0 on success, negative on failure
8551 */
8552 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8553 u8 *lut, u16 lut_size)
8554 {
8555 struct i40e_pf *pf = vsi->back;
8556 struct i40e_hw *hw = &pf->hw;
8557 u16 i;
8558
8559 if (seed) {
8560 u32 *seed_dw = (u32 *)seed;
8561
8562 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8563 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8564 }
8565 if (lut) {
8566 u32 *lut_dw = (u32 *)lut;
8567
8568 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8569 return -EINVAL;
8570 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8571 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8572 }
8573
8574 return 0;
8575 }
8576
8577 /**
8578 * i40e_config_rss - Configure RSS keys and lut
8579 * @vsi: Pointer to VSI structure
8580 * @seed: RSS hash seed
8581 * @lut: Lookup table
8582 * @lut_size: Lookup table size
8583 *
8584 * Returns 0 on success, negative on failure
8585 */
8586 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8587 {
8588 struct i40e_pf *pf = vsi->back;
8589
8590 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8591 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8592 else
8593 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8594 }
8595
8596 /**
8597 * i40e_get_rss - Get RSS keys and lut
8598 * @vsi: Pointer to VSI structure
8599 * @seed: Buffer to store the keys
8600 * @lut: Buffer to store the lookup table entries
8601 * lut_size: Size of buffer to store the lookup table entries
8602 *
8603 * Returns 0 on success, negative on failure
8604 */
8605 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8606 {
8607 struct i40e_pf *pf = vsi->back;
8608
8609 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8610 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8611 else
8612 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8613 }
8614
8615 /**
8616 * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8617 * @pf: Pointer to board private structure
8618 * @lut: Lookup table
8619 * @rss_table_size: Lookup table size
8620 * @rss_size: Range of queue number for hashing
8621 */
8622 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8623 u16 rss_table_size, u16 rss_size)
8624 {
8625 u16 i;
8626
8627 for (i = 0; i < rss_table_size; i++)
8628 lut[i] = i % rss_size;
8629 }
8630
8631 /**
8632 * i40e_pf_config_rss - Prepare for RSS if used
8633 * @pf: board private structure
8634 **/
8635 static int i40e_pf_config_rss(struct i40e_pf *pf)
8636 {
8637 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8638 u8 seed[I40E_HKEY_ARRAY_SIZE];
8639 u8 *lut;
8640 struct i40e_hw *hw = &pf->hw;
8641 u32 reg_val;
8642 u64 hena;
8643 int ret;
8644
8645 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8646 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8647 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8648 hena |= i40e_pf_get_default_rss_hena(pf);
8649
8650 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8651 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8652
8653 /* Determine the RSS table size based on the hardware capabilities */
8654 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8655 reg_val = (pf->rss_table_size == 512) ?
8656 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8657 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8658 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8659
8660 /* Determine the RSS size of the VSI */
8661 if (!vsi->rss_size) {
8662 u16 qcount;
8663
8664 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8665 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8666 }
8667 if (!vsi->rss_size)
8668 return -EINVAL;
8669
8670 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8671 if (!lut)
8672 return -ENOMEM;
8673
8674 /* Use user configured lut if there is one, otherwise use default */
8675 if (vsi->rss_lut_user)
8676 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8677 else
8678 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8679
8680 /* Use user configured hash key if there is one, otherwise
8681 * use default.
8682 */
8683 if (vsi->rss_hkey_user)
8684 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8685 else
8686 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8687 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8688 kfree(lut);
8689
8690 return ret;
8691 }
8692
8693 /**
8694 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8695 * @pf: board private structure
8696 * @queue_count: the requested queue count for rss.
8697 *
8698 * returns 0 if rss is not enabled, if enabled returns the final rss queue
8699 * count which may be different from the requested queue count.
8700 * Note: expects to be called while under rtnl_lock()
8701 **/
8702 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8703 {
8704 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8705 int new_rss_size;
8706
8707 if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8708 return 0;
8709
8710 new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8711
8712 if (queue_count != vsi->num_queue_pairs) {
8713 u16 qcount;
8714
8715 vsi->req_queue_pairs = queue_count;
8716 i40e_prep_for_reset(pf, true);
8717
8718 pf->alloc_rss_size = new_rss_size;
8719
8720 i40e_reset_and_rebuild(pf, true, true);
8721
8722 /* Discard the user configured hash keys and lut, if less
8723 * queues are enabled.
8724 */
8725 if (queue_count < vsi->rss_size) {
8726 i40e_clear_rss_config_user(vsi);
8727 dev_dbg(&pf->pdev->dev,
8728 "discard user configured hash keys and lut\n");
8729 }
8730
8731 /* Reset vsi->rss_size, as number of enabled queues changed */
8732 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8733 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8734
8735 i40e_pf_config_rss(pf);
8736 }
8737 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
8738 vsi->req_queue_pairs, pf->rss_size_max);
8739 return pf->alloc_rss_size;
8740 }
8741
8742 /**
8743 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
8744 * @pf: board private structure
8745 **/
8746 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
8747 {
8748 i40e_status status;
8749 bool min_valid, max_valid;
8750 u32 max_bw, min_bw;
8751
8752 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8753 &min_valid, &max_valid);
8754
8755 if (!status) {
8756 if (min_valid)
8757 pf->min_bw = min_bw;
8758 if (max_valid)
8759 pf->max_bw = max_bw;
8760 }
8761
8762 return status;
8763 }
8764
8765 /**
8766 * i40e_set_partition_bw_setting - Set BW settings for this PF partition
8767 * @pf: board private structure
8768 **/
8769 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
8770 {
8771 struct i40e_aqc_configure_partition_bw_data bw_data;
8772 i40e_status status;
8773
8774 /* Set the valid bit for this PF */
8775 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8776 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
8777 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
8778
8779 /* Set the new bandwidths */
8780 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8781
8782 return status;
8783 }
8784
8785 /**
8786 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
8787 * @pf: board private structure
8788 **/
8789 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
8790 {
8791 /* Commit temporary BW setting to permanent NVM image */
8792 enum i40e_admin_queue_err last_aq_status;
8793 i40e_status ret;
8794 u16 nvm_word;
8795
8796 if (pf->hw.partition_id != 1) {
8797 dev_info(&pf->pdev->dev,
8798 "Commit BW only works on partition 1! This is partition %d",
8799 pf->hw.partition_id);
8800 ret = I40E_NOT_SUPPORTED;
8801 goto bw_commit_out;
8802 }
8803
8804 /* Acquire NVM for read access */
8805 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8806 last_aq_status = pf->hw.aq.asq_last_status;
8807 if (ret) {
8808 dev_info(&pf->pdev->dev,
8809 "Cannot acquire NVM for read access, err %s aq_err %s\n",
8810 i40e_stat_str(&pf->hw, ret),
8811 i40e_aq_str(&pf->hw, last_aq_status));
8812 goto bw_commit_out;
8813 }
8814
8815 /* Read word 0x10 of NVM - SW compatibility word 1 */
8816 ret = i40e_aq_read_nvm(&pf->hw,
8817 I40E_SR_NVM_CONTROL_WORD,
8818 0x10, sizeof(nvm_word), &nvm_word,
8819 false, NULL);
8820 /* Save off last admin queue command status before releasing
8821 * the NVM
8822 */
8823 last_aq_status = pf->hw.aq.asq_last_status;
8824 i40e_release_nvm(&pf->hw);
8825 if (ret) {
8826 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8827 i40e_stat_str(&pf->hw, ret),
8828 i40e_aq_str(&pf->hw, last_aq_status));
8829 goto bw_commit_out;
8830 }
8831
8832 /* Wait a bit for NVM release to complete */
8833 msleep(50);
8834
8835 /* Acquire NVM for write access */
8836 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8837 last_aq_status = pf->hw.aq.asq_last_status;
8838 if (ret) {
8839 dev_info(&pf->pdev->dev,
8840 "Cannot acquire NVM for write access, err %s aq_err %s\n",
8841 i40e_stat_str(&pf->hw, ret),
8842 i40e_aq_str(&pf->hw, last_aq_status));
8843 goto bw_commit_out;
8844 }
8845 /* Write it back out unchanged to initiate update NVM,
8846 * which will force a write of the shadow (alt) RAM to
8847 * the NVM - thus storing the bandwidth values permanently.
8848 */
8849 ret = i40e_aq_update_nvm(&pf->hw,
8850 I40E_SR_NVM_CONTROL_WORD,
8851 0x10, sizeof(nvm_word),
8852 &nvm_word, true, NULL);
8853 /* Save off last admin queue command status before releasing
8854 * the NVM
8855 */
8856 last_aq_status = pf->hw.aq.asq_last_status;
8857 i40e_release_nvm(&pf->hw);
8858 if (ret)
8859 dev_info(&pf->pdev->dev,
8860 "BW settings NOT SAVED, err %s aq_err %s\n",
8861 i40e_stat_str(&pf->hw, ret),
8862 i40e_aq_str(&pf->hw, last_aq_status));
8863 bw_commit_out:
8864
8865 return ret;
8866 }
8867
8868 /**
8869 * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8870 * @pf: board private structure to initialize
8871 *
8872 * i40e_sw_init initializes the Adapter private data structure.
8873 * Fields are initialized based on PCI device information and
8874 * OS network device settings (MTU size).
8875 **/
8876 static int i40e_sw_init(struct i40e_pf *pf)
8877 {
8878 int err = 0;
8879 int size;
8880
8881 /* Set default capability flags */
8882 pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8883 I40E_FLAG_MSI_ENABLED |
8884 I40E_FLAG_MSIX_ENABLED;
8885
8886 /* Set default ITR */
8887 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8888 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8889
8890 /* Depending on PF configurations, it is possible that the RSS
8891 * maximum might end up larger than the available queues
8892 */
8893 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8894 pf->alloc_rss_size = 1;
8895 pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8896 pf->rss_size_max = min_t(int, pf->rss_size_max,
8897 pf->hw.func_caps.num_tx_qp);
8898 if (pf->hw.func_caps.rss) {
8899 pf->flags |= I40E_FLAG_RSS_ENABLED;
8900 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8901 num_online_cpus());
8902 }
8903
8904 /* MFP mode enabled */
8905 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8906 pf->flags |= I40E_FLAG_MFP_ENABLED;
8907 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8908 if (i40e_get_partition_bw_setting(pf)) {
8909 dev_warn(&pf->pdev->dev,
8910 "Could not get partition bw settings\n");
8911 } else {
8912 dev_info(&pf->pdev->dev,
8913 "Partition BW Min = %8.8x, Max = %8.8x\n",
8914 pf->min_bw, pf->max_bw);
8915
8916 /* nudge the Tx scheduler */
8917 i40e_set_partition_bw_setting(pf);
8918 }
8919 }
8920
8921 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8922 (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8923 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8924 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8925 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8926 pf->hw.num_partitions > 1)
8927 dev_info(&pf->pdev->dev,
8928 "Flow Director Sideband mode Disabled in MFP mode\n");
8929 else
8930 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8931 pf->fdir_pf_filter_count =
8932 pf->hw.func_caps.fd_filters_guaranteed;
8933 pf->hw.fdir_shared_filter_count =
8934 pf->hw.func_caps.fd_filters_best_effort;
8935 }
8936
8937 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8938 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
8939 (pf->hw.aq.fw_maj_ver < 4))) {
8940 pf->flags |= I40E_FLAG_RESTART_AUTONEG;
8941 /* No DCB support for FW < v4.33 */
8942 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT;
8943 }
8944
8945 /* Disable FW LLDP if FW < v4.3 */
8946 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8947 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
8948 (pf->hw.aq.fw_maj_ver < 4)))
8949 pf->flags |= I40E_FLAG_STOP_FW_LLDP;
8950
8951 /* Use the FW Set LLDP MIB API if FW > v4.40 */
8952 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8953 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
8954 (pf->hw.aq.fw_maj_ver >= 5)))
8955 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB;
8956
8957 if (pf->hw.func_caps.vmdq) {
8958 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
8959 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
8960 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
8961 }
8962
8963 if (pf->hw.func_caps.iwarp) {
8964 pf->flags |= I40E_FLAG_IWARP_ENABLED;
8965 /* IWARP needs one extra vector for CQP just like MISC.*/
8966 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
8967 }
8968
8969 #ifdef CONFIG_PCI_IOV
8970 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
8971 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
8972 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
8973 pf->num_req_vfs = min_t(int,
8974 pf->hw.func_caps.num_vfs,
8975 I40E_MAX_VF_COUNT);
8976 }
8977 #endif /* CONFIG_PCI_IOV */
8978 if (pf->hw.mac.type == I40E_MAC_X722) {
8979 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE
8980 | I40E_FLAG_128_QP_RSS_CAPABLE
8981 | I40E_FLAG_HW_ATR_EVICT_CAPABLE
8982 | I40E_FLAG_OUTER_UDP_CSUM_CAPABLE
8983 | I40E_FLAG_WB_ON_ITR_CAPABLE
8984 | I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE
8985 | I40E_FLAG_NO_PCI_LINK_CHECK
8986 | I40E_FLAG_USE_SET_LLDP_MIB
8987 | I40E_FLAG_GENEVE_OFFLOAD_CAPABLE
8988 | I40E_FLAG_PTP_L4_CAPABLE
8989 | I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE;
8990 } else if ((pf->hw.aq.api_maj_ver > 1) ||
8991 ((pf->hw.aq.api_maj_ver == 1) &&
8992 (pf->hw.aq.api_min_ver > 4))) {
8993 /* Supported in FW API version higher than 1.4 */
8994 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
8995 }
8996
8997 /* Enable HW ATR eviction if possible */
8998 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE)
8999 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
9000
9001 pf->eeprom_version = 0xDEAD;
9002 pf->lan_veb = I40E_NO_VEB;
9003 pf->lan_vsi = I40E_NO_VSI;
9004
9005 /* By default FW has this off for performance reasons */
9006 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
9007
9008 /* set up queue assignment tracking */
9009 size = sizeof(struct i40e_lump_tracking)
9010 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
9011 pf->qp_pile = kzalloc(size, GFP_KERNEL);
9012 if (!pf->qp_pile) {
9013 err = -ENOMEM;
9014 goto sw_init_done;
9015 }
9016 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
9017 pf->qp_pile->search_hint = 0;
9018
9019 pf->tx_timeout_recovery_level = 1;
9020
9021 mutex_init(&pf->switch_mutex);
9022
9023 sw_init_done:
9024 return err;
9025 }
9026
9027 /**
9028 * i40e_set_ntuple - set the ntuple feature flag and take action
9029 * @pf: board private structure to initialize
9030 * @features: the feature set that the stack is suggesting
9031 *
9032 * returns a bool to indicate if reset needs to happen
9033 **/
9034 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
9035 {
9036 bool need_reset = false;
9037
9038 /* Check if Flow Director n-tuple support was enabled or disabled. If
9039 * the state changed, we need to reset.
9040 */
9041 if (features & NETIF_F_NTUPLE) {
9042 /* Enable filters and mark for reset */
9043 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9044 need_reset = true;
9045 /* enable FD_SB only if there is MSI-X vector */
9046 if (pf->num_fdsb_msix > 0)
9047 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9048 } else {
9049 /* turn off filters, mark for reset and clear SW filter list */
9050 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
9051 need_reset = true;
9052 i40e_fdir_filter_exit(pf);
9053 }
9054 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED |
9055 I40E_FLAG_FD_SB_AUTO_DISABLED);
9056 /* reset fd counters */
9057 pf->fd_add_err = 0;
9058 pf->fd_atr_cnt = 0;
9059 /* if ATR was auto disabled it can be re-enabled. */
9060 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
9061 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
9062 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9063 (I40E_DEBUG_FD & pf->hw.debug_mask))
9064 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
9065 }
9066 }
9067 return need_reset;
9068 }
9069
9070 /**
9071 * i40e_clear_rss_lut - clear the rx hash lookup table
9072 * @vsi: the VSI being configured
9073 **/
9074 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
9075 {
9076 struct i40e_pf *pf = vsi->back;
9077 struct i40e_hw *hw = &pf->hw;
9078 u16 vf_id = vsi->vf_id;
9079 u8 i;
9080
9081 if (vsi->type == I40E_VSI_MAIN) {
9082 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
9083 wr32(hw, I40E_PFQF_HLUT(i), 0);
9084 } else if (vsi->type == I40E_VSI_SRIOV) {
9085 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
9086 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
9087 } else {
9088 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
9089 }
9090 }
9091
9092 /**
9093 * i40e_set_features - set the netdev feature flags
9094 * @netdev: ptr to the netdev being adjusted
9095 * @features: the feature set that the stack is suggesting
9096 * Note: expects to be called while under rtnl_lock()
9097 **/
9098 static int i40e_set_features(struct net_device *netdev,
9099 netdev_features_t features)
9100 {
9101 struct i40e_netdev_priv *np = netdev_priv(netdev);
9102 struct i40e_vsi *vsi = np->vsi;
9103 struct i40e_pf *pf = vsi->back;
9104 bool need_reset;
9105
9106 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
9107 i40e_pf_config_rss(pf);
9108 else if (!(features & NETIF_F_RXHASH) &&
9109 netdev->features & NETIF_F_RXHASH)
9110 i40e_clear_rss_lut(vsi);
9111
9112 if (features & NETIF_F_HW_VLAN_CTAG_RX)
9113 i40e_vlan_stripping_enable(vsi);
9114 else
9115 i40e_vlan_stripping_disable(vsi);
9116
9117 need_reset = i40e_set_ntuple(pf, features);
9118
9119 if (need_reset)
9120 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
9121
9122 return 0;
9123 }
9124
9125 /**
9126 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
9127 * @pf: board private structure
9128 * @port: The UDP port to look up
9129 *
9130 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
9131 **/
9132 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
9133 {
9134 u8 i;
9135
9136 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9137 if (pf->udp_ports[i].port == port)
9138 return i;
9139 }
9140
9141 return i;
9142 }
9143
9144 /**
9145 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
9146 * @netdev: This physical port's netdev
9147 * @ti: Tunnel endpoint information
9148 **/
9149 static void i40e_udp_tunnel_add(struct net_device *netdev,
9150 struct udp_tunnel_info *ti)
9151 {
9152 struct i40e_netdev_priv *np = netdev_priv(netdev);
9153 struct i40e_vsi *vsi = np->vsi;
9154 struct i40e_pf *pf = vsi->back;
9155 u16 port = ntohs(ti->port);
9156 u8 next_idx;
9157 u8 idx;
9158
9159 idx = i40e_get_udp_port_idx(pf, port);
9160
9161 /* Check if port already exists */
9162 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9163 netdev_info(netdev, "port %d already offloaded\n", port);
9164 return;
9165 }
9166
9167 /* Now check if there is space to add the new port */
9168 next_idx = i40e_get_udp_port_idx(pf, 0);
9169
9170 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9171 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
9172 port);
9173 return;
9174 }
9175
9176 switch (ti->type) {
9177 case UDP_TUNNEL_TYPE_VXLAN:
9178 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
9179 break;
9180 case UDP_TUNNEL_TYPE_GENEVE:
9181 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
9182 return;
9183 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
9184 break;
9185 default:
9186 return;
9187 }
9188
9189 /* New port: add it and mark its index in the bitmap */
9190 pf->udp_ports[next_idx].port = port;
9191 pf->pending_udp_bitmap |= BIT_ULL(next_idx);
9192 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9193 }
9194
9195 /**
9196 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
9197 * @netdev: This physical port's netdev
9198 * @ti: Tunnel endpoint information
9199 **/
9200 static void i40e_udp_tunnel_del(struct net_device *netdev,
9201 struct udp_tunnel_info *ti)
9202 {
9203 struct i40e_netdev_priv *np = netdev_priv(netdev);
9204 struct i40e_vsi *vsi = np->vsi;
9205 struct i40e_pf *pf = vsi->back;
9206 u16 port = ntohs(ti->port);
9207 u8 idx;
9208
9209 idx = i40e_get_udp_port_idx(pf, port);
9210
9211 /* Check if port already exists */
9212 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
9213 goto not_found;
9214
9215 switch (ti->type) {
9216 case UDP_TUNNEL_TYPE_VXLAN:
9217 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
9218 goto not_found;
9219 break;
9220 case UDP_TUNNEL_TYPE_GENEVE:
9221 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
9222 goto not_found;
9223 break;
9224 default:
9225 goto not_found;
9226 }
9227
9228 /* if port exists, set it to 0 (mark for deletion)
9229 * and make it pending
9230 */
9231 pf->udp_ports[idx].port = 0;
9232 pf->pending_udp_bitmap |= BIT_ULL(idx);
9233 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9234
9235 return;
9236 not_found:
9237 netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
9238 port);
9239 }
9240
9241 static int i40e_get_phys_port_id(struct net_device *netdev,
9242 struct netdev_phys_item_id *ppid)
9243 {
9244 struct i40e_netdev_priv *np = netdev_priv(netdev);
9245 struct i40e_pf *pf = np->vsi->back;
9246 struct i40e_hw *hw = &pf->hw;
9247
9248 if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
9249 return -EOPNOTSUPP;
9250
9251 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9252 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9253
9254 return 0;
9255 }
9256
9257 /**
9258 * i40e_ndo_fdb_add - add an entry to the hardware database
9259 * @ndm: the input from the stack
9260 * @tb: pointer to array of nladdr (unused)
9261 * @dev: the net device pointer
9262 * @addr: the MAC address entry being added
9263 * @flags: instructions from stack about fdb operation
9264 */
9265 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9266 struct net_device *dev,
9267 const unsigned char *addr, u16 vid,
9268 u16 flags)
9269 {
9270 struct i40e_netdev_priv *np = netdev_priv(dev);
9271 struct i40e_pf *pf = np->vsi->back;
9272 int err = 0;
9273
9274 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9275 return -EOPNOTSUPP;
9276
9277 if (vid) {
9278 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9279 return -EINVAL;
9280 }
9281
9282 /* Hardware does not support aging addresses so if a
9283 * ndm_state is given only allow permanent addresses
9284 */
9285 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9286 netdev_info(dev, "FDB only supports static addresses\n");
9287 return -EINVAL;
9288 }
9289
9290 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9291 err = dev_uc_add_excl(dev, addr);
9292 else if (is_multicast_ether_addr(addr))
9293 err = dev_mc_add_excl(dev, addr);
9294 else
9295 err = -EINVAL;
9296
9297 /* Only return duplicate errors if NLM_F_EXCL is set */
9298 if (err == -EEXIST && !(flags & NLM_F_EXCL))
9299 err = 0;
9300
9301 return err;
9302 }
9303
9304 /**
9305 * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9306 * @dev: the netdev being configured
9307 * @nlh: RTNL message
9308 *
9309 * Inserts a new hardware bridge if not already created and
9310 * enables the bridging mode requested (VEB or VEPA). If the
9311 * hardware bridge has already been inserted and the request
9312 * is to change the mode then that requires a PF reset to
9313 * allow rebuild of the components with required hardware
9314 * bridge mode enabled.
9315 *
9316 * Note: expects to be called while under rtnl_lock()
9317 **/
9318 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9319 struct nlmsghdr *nlh,
9320 u16 flags)
9321 {
9322 struct i40e_netdev_priv *np = netdev_priv(dev);
9323 struct i40e_vsi *vsi = np->vsi;
9324 struct i40e_pf *pf = vsi->back;
9325 struct i40e_veb *veb = NULL;
9326 struct nlattr *attr, *br_spec;
9327 int i, rem;
9328
9329 /* Only for PF VSI for now */
9330 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9331 return -EOPNOTSUPP;
9332
9333 /* Find the HW bridge for PF VSI */
9334 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9335 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9336 veb = pf->veb[i];
9337 }
9338
9339 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9340
9341 nla_for_each_nested(attr, br_spec, rem) {
9342 __u16 mode;
9343
9344 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9345 continue;
9346
9347 mode = nla_get_u16(attr);
9348 if ((mode != BRIDGE_MODE_VEPA) &&
9349 (mode != BRIDGE_MODE_VEB))
9350 return -EINVAL;
9351
9352 /* Insert a new HW bridge */
9353 if (!veb) {
9354 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9355 vsi->tc_config.enabled_tc);
9356 if (veb) {
9357 veb->bridge_mode = mode;
9358 i40e_config_bridge_mode(veb);
9359 } else {
9360 /* No Bridge HW offload available */
9361 return -ENOENT;
9362 }
9363 break;
9364 } else if (mode != veb->bridge_mode) {
9365 /* Existing HW bridge but different mode needs reset */
9366 veb->bridge_mode = mode;
9367 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9368 if (mode == BRIDGE_MODE_VEB)
9369 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9370 else
9371 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9372 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED),
9373 true);
9374 break;
9375 }
9376 }
9377
9378 return 0;
9379 }
9380
9381 /**
9382 * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9383 * @skb: skb buff
9384 * @pid: process id
9385 * @seq: RTNL message seq #
9386 * @dev: the netdev being configured
9387 * @filter_mask: unused
9388 * @nlflags: netlink flags passed in
9389 *
9390 * Return the mode in which the hardware bridge is operating in
9391 * i.e VEB or VEPA.
9392 **/
9393 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9394 struct net_device *dev,
9395 u32 __always_unused filter_mask,
9396 int nlflags)
9397 {
9398 struct i40e_netdev_priv *np = netdev_priv(dev);
9399 struct i40e_vsi *vsi = np->vsi;
9400 struct i40e_pf *pf = vsi->back;
9401 struct i40e_veb *veb = NULL;
9402 int i;
9403
9404 /* Only for PF VSI for now */
9405 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9406 return -EOPNOTSUPP;
9407
9408 /* Find the HW bridge for the PF VSI */
9409 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9410 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9411 veb = pf->veb[i];
9412 }
9413
9414 if (!veb)
9415 return 0;
9416
9417 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9418 0, 0, nlflags, filter_mask, NULL);
9419 }
9420
9421 /**
9422 * i40e_features_check - Validate encapsulated packet conforms to limits
9423 * @skb: skb buff
9424 * @dev: This physical port's netdev
9425 * @features: Offload features that the stack believes apply
9426 **/
9427 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9428 struct net_device *dev,
9429 netdev_features_t features)
9430 {
9431 size_t len;
9432
9433 /* No point in doing any of this if neither checksum nor GSO are
9434 * being requested for this frame. We can rule out both by just
9435 * checking for CHECKSUM_PARTIAL
9436 */
9437 if (skb->ip_summed != CHECKSUM_PARTIAL)
9438 return features;
9439
9440 /* We cannot support GSO if the MSS is going to be less than
9441 * 64 bytes. If it is then we need to drop support for GSO.
9442 */
9443 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9444 features &= ~NETIF_F_GSO_MASK;
9445
9446 /* MACLEN can support at most 63 words */
9447 len = skb_network_header(skb) - skb->data;
9448 if (len & ~(63 * 2))
9449 goto out_err;
9450
9451 /* IPLEN and EIPLEN can support at most 127 dwords */
9452 len = skb_transport_header(skb) - skb_network_header(skb);
9453 if (len & ~(127 * 4))
9454 goto out_err;
9455
9456 if (skb->encapsulation) {
9457 /* L4TUNLEN can support 127 words */
9458 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9459 if (len & ~(127 * 2))
9460 goto out_err;
9461
9462 /* IPLEN can support at most 127 dwords */
9463 len = skb_inner_transport_header(skb) -
9464 skb_inner_network_header(skb);
9465 if (len & ~(127 * 4))
9466 goto out_err;
9467 }
9468
9469 /* No need to validate L4LEN as TCP is the only protocol with a
9470 * a flexible value and we support all possible values supported
9471 * by TCP, which is at most 15 dwords
9472 */
9473
9474 return features;
9475 out_err:
9476 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9477 }
9478
9479 /**
9480 * i40e_xdp_setup - add/remove an XDP program
9481 * @vsi: VSI to changed
9482 * @prog: XDP program
9483 **/
9484 static int i40e_xdp_setup(struct i40e_vsi *vsi,
9485 struct bpf_prog *prog)
9486 {
9487 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9488 struct i40e_pf *pf = vsi->back;
9489 struct bpf_prog *old_prog;
9490 bool need_reset;
9491 int i;
9492
9493 /* Don't allow frames that span over multiple buffers */
9494 if (frame_size > vsi->rx_buf_len)
9495 return -EINVAL;
9496
9497 if (!i40e_enabled_xdp_vsi(vsi) && !prog)
9498 return 0;
9499
9500 /* When turning XDP on->off/off->on we reset and rebuild the rings. */
9501 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
9502
9503 if (need_reset)
9504 i40e_prep_for_reset(pf, true);
9505
9506 old_prog = xchg(&vsi->xdp_prog, prog);
9507
9508 if (need_reset)
9509 i40e_reset_and_rebuild(pf, true, true);
9510
9511 for (i = 0; i < vsi->num_queue_pairs; i++)
9512 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
9513
9514 if (old_prog)
9515 bpf_prog_put(old_prog);
9516
9517 return 0;
9518 }
9519
9520 /**
9521 * i40e_xdp - implements ndo_xdp for i40e
9522 * @dev: netdevice
9523 * @xdp: XDP command
9524 **/
9525 static int i40e_xdp(struct net_device *dev,
9526 struct netdev_xdp *xdp)
9527 {
9528 struct i40e_netdev_priv *np = netdev_priv(dev);
9529 struct i40e_vsi *vsi = np->vsi;
9530
9531 if (vsi->type != I40E_VSI_MAIN)
9532 return -EINVAL;
9533
9534 switch (xdp->command) {
9535 case XDP_SETUP_PROG:
9536 return i40e_xdp_setup(vsi, xdp->prog);
9537 case XDP_QUERY_PROG:
9538 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
9539 return 0;
9540 default:
9541 return -EINVAL;
9542 }
9543 }
9544
9545 static const struct net_device_ops i40e_netdev_ops = {
9546 .ndo_open = i40e_open,
9547 .ndo_stop = i40e_close,
9548 .ndo_start_xmit = i40e_lan_xmit_frame,
9549 .ndo_get_stats64 = i40e_get_netdev_stats_struct,
9550 .ndo_set_rx_mode = i40e_set_rx_mode,
9551 .ndo_validate_addr = eth_validate_addr,
9552 .ndo_set_mac_address = i40e_set_mac,
9553 .ndo_change_mtu = i40e_change_mtu,
9554 .ndo_do_ioctl = i40e_ioctl,
9555 .ndo_tx_timeout = i40e_tx_timeout,
9556 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
9557 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
9558 #ifdef CONFIG_NET_POLL_CONTROLLER
9559 .ndo_poll_controller = i40e_netpoll,
9560 #endif
9561 .ndo_setup_tc = __i40e_setup_tc,
9562 .ndo_set_features = i40e_set_features,
9563 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
9564 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
9565 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
9566 .ndo_get_vf_config = i40e_ndo_get_vf_config,
9567 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
9568 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
9569 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,
9570 .ndo_udp_tunnel_add = i40e_udp_tunnel_add,
9571 .ndo_udp_tunnel_del = i40e_udp_tunnel_del,
9572 .ndo_get_phys_port_id = i40e_get_phys_port_id,
9573 .ndo_fdb_add = i40e_ndo_fdb_add,
9574 .ndo_features_check = i40e_features_check,
9575 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,
9576 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,
9577 .ndo_xdp = i40e_xdp,
9578 };
9579
9580 /**
9581 * i40e_config_netdev - Setup the netdev flags
9582 * @vsi: the VSI being configured
9583 *
9584 * Returns 0 on success, negative value on failure
9585 **/
9586 static int i40e_config_netdev(struct i40e_vsi *vsi)
9587 {
9588 struct i40e_pf *pf = vsi->back;
9589 struct i40e_hw *hw = &pf->hw;
9590 struct i40e_netdev_priv *np;
9591 struct net_device *netdev;
9592 u8 broadcast[ETH_ALEN];
9593 u8 mac_addr[ETH_ALEN];
9594 int etherdev_size;
9595 netdev_features_t hw_enc_features;
9596 netdev_features_t hw_features;
9597
9598 etherdev_size = sizeof(struct i40e_netdev_priv);
9599 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9600 if (!netdev)
9601 return -ENOMEM;
9602
9603 vsi->netdev = netdev;
9604 np = netdev_priv(netdev);
9605 np->vsi = vsi;
9606
9607 hw_enc_features = NETIF_F_SG |
9608 NETIF_F_IP_CSUM |
9609 NETIF_F_IPV6_CSUM |
9610 NETIF_F_HIGHDMA |
9611 NETIF_F_SOFT_FEATURES |
9612 NETIF_F_TSO |
9613 NETIF_F_TSO_ECN |
9614 NETIF_F_TSO6 |
9615 NETIF_F_GSO_GRE |
9616 NETIF_F_GSO_GRE_CSUM |
9617 NETIF_F_GSO_PARTIAL |
9618 NETIF_F_GSO_UDP_TUNNEL |
9619 NETIF_F_GSO_UDP_TUNNEL_CSUM |
9620 NETIF_F_SCTP_CRC |
9621 NETIF_F_RXHASH |
9622 NETIF_F_RXCSUM |
9623 0;
9624
9625 if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE))
9626 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9627
9628 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9629
9630 netdev->hw_enc_features |= hw_enc_features;
9631
9632 /* record features VLANs can make use of */
9633 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
9634
9635 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9636 netdev->hw_features |= NETIF_F_NTUPLE;
9637 hw_features = hw_enc_features |
9638 NETIF_F_HW_VLAN_CTAG_TX |
9639 NETIF_F_HW_VLAN_CTAG_RX;
9640
9641 netdev->hw_features |= hw_features;
9642
9643 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9644 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9645
9646 if (vsi->type == I40E_VSI_MAIN) {
9647 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9648 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9649 /* The following steps are necessary for two reasons. First,
9650 * some older NVM configurations load a default MAC-VLAN
9651 * filter that will accept any tagged packet, and we want to
9652 * replace this with a normal filter. Additionally, it is
9653 * possible our MAC address was provided by the platform using
9654 * Open Firmware or similar.
9655 *
9656 * Thus, we need to remove the default filter and install one
9657 * specific to the MAC address.
9658 */
9659 i40e_rm_default_mac_filter(vsi, mac_addr);
9660 spin_lock_bh(&vsi->mac_filter_hash_lock);
9661 i40e_add_mac_filter(vsi, mac_addr);
9662 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9663 } else {
9664 /* relate the VSI_VMDQ name to the VSI_MAIN name */
9665 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
9666 pf->vsi[pf->lan_vsi]->netdev->name);
9667 random_ether_addr(mac_addr);
9668
9669 spin_lock_bh(&vsi->mac_filter_hash_lock);
9670 i40e_add_mac_filter(vsi, mac_addr);
9671 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9672 }
9673
9674 /* Add the broadcast filter so that we initially will receive
9675 * broadcast packets. Note that when a new VLAN is first added the
9676 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9677 * specific filters as part of transitioning into "vlan" operation.
9678 * When more VLANs are added, the driver will copy each existing MAC
9679 * filter and add it for the new VLAN.
9680 *
9681 * Broadcast filters are handled specially by
9682 * i40e_sync_filters_subtask, as the driver must to set the broadcast
9683 * promiscuous bit instead of adding this directly as a MAC/VLAN
9684 * filter. The subtask will update the correct broadcast promiscuous
9685 * bits as VLANs become active or inactive.
9686 */
9687 eth_broadcast_addr(broadcast);
9688 spin_lock_bh(&vsi->mac_filter_hash_lock);
9689 i40e_add_mac_filter(vsi, broadcast);
9690 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9691
9692 ether_addr_copy(netdev->dev_addr, mac_addr);
9693 ether_addr_copy(netdev->perm_addr, mac_addr);
9694
9695 netdev->priv_flags |= IFF_UNICAST_FLT;
9696 netdev->priv_flags |= IFF_SUPP_NOFCS;
9697 /* Setup netdev TC information */
9698 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9699
9700 netdev->netdev_ops = &i40e_netdev_ops;
9701 netdev->watchdog_timeo = 5 * HZ;
9702 i40e_set_ethtool_ops(netdev);
9703
9704 /* MTU range: 68 - 9706 */
9705 netdev->min_mtu = ETH_MIN_MTU;
9706 netdev->max_mtu = I40E_MAX_RXBUFFER -
9707 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
9708
9709 return 0;
9710 }
9711
9712 /**
9713 * i40e_vsi_delete - Delete a VSI from the switch
9714 * @vsi: the VSI being removed
9715 *
9716 * Returns 0 on success, negative value on failure
9717 **/
9718 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9719 {
9720 /* remove default VSI is not allowed */
9721 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9722 return;
9723
9724 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9725 }
9726
9727 /**
9728 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9729 * @vsi: the VSI being queried
9730 *
9731 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9732 **/
9733 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9734 {
9735 struct i40e_veb *veb;
9736 struct i40e_pf *pf = vsi->back;
9737
9738 /* Uplink is not a bridge so default to VEB */
9739 if (vsi->veb_idx == I40E_NO_VEB)
9740 return 1;
9741
9742 veb = pf->veb[vsi->veb_idx];
9743 if (!veb) {
9744 dev_info(&pf->pdev->dev,
9745 "There is no veb associated with the bridge\n");
9746 return -ENOENT;
9747 }
9748
9749 /* Uplink is a bridge in VEPA mode */
9750 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9751 return 0;
9752 } else {
9753 /* Uplink is a bridge in VEB mode */
9754 return 1;
9755 }
9756
9757 /* VEPA is now default bridge, so return 0 */
9758 return 0;
9759 }
9760
9761 /**
9762 * i40e_add_vsi - Add a VSI to the switch
9763 * @vsi: the VSI being configured
9764 *
9765 * This initializes a VSI context depending on the VSI type to be added and
9766 * passes it down to the add_vsi aq command.
9767 **/
9768 static int i40e_add_vsi(struct i40e_vsi *vsi)
9769 {
9770 int ret = -ENODEV;
9771 struct i40e_pf *pf = vsi->back;
9772 struct i40e_hw *hw = &pf->hw;
9773 struct i40e_vsi_context ctxt;
9774 struct i40e_mac_filter *f;
9775 struct hlist_node *h;
9776 int bkt;
9777
9778 u8 enabled_tc = 0x1; /* TC0 enabled */
9779 int f_count = 0;
9780
9781 memset(&ctxt, 0, sizeof(ctxt));
9782 switch (vsi->type) {
9783 case I40E_VSI_MAIN:
9784 /* The PF's main VSI is already setup as part of the
9785 * device initialization, so we'll not bother with
9786 * the add_vsi call, but we will retrieve the current
9787 * VSI context.
9788 */
9789 ctxt.seid = pf->main_vsi_seid;
9790 ctxt.pf_num = pf->hw.pf_id;
9791 ctxt.vf_num = 0;
9792 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9793 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9794 if (ret) {
9795 dev_info(&pf->pdev->dev,
9796 "couldn't get PF vsi config, err %s aq_err %s\n",
9797 i40e_stat_str(&pf->hw, ret),
9798 i40e_aq_str(&pf->hw,
9799 pf->hw.aq.asq_last_status));
9800 return -ENOENT;
9801 }
9802 vsi->info = ctxt.info;
9803 vsi->info.valid_sections = 0;
9804
9805 vsi->seid = ctxt.seid;
9806 vsi->id = ctxt.vsi_number;
9807
9808 enabled_tc = i40e_pf_get_tc_map(pf);
9809
9810 /* MFP mode setup queue map and update VSI */
9811 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9812 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9813 memset(&ctxt, 0, sizeof(ctxt));
9814 ctxt.seid = pf->main_vsi_seid;
9815 ctxt.pf_num = pf->hw.pf_id;
9816 ctxt.vf_num = 0;
9817 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9818 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9819 if (ret) {
9820 dev_info(&pf->pdev->dev,
9821 "update vsi failed, err %s aq_err %s\n",
9822 i40e_stat_str(&pf->hw, ret),
9823 i40e_aq_str(&pf->hw,
9824 pf->hw.aq.asq_last_status));
9825 ret = -ENOENT;
9826 goto err;
9827 }
9828 /* update the local VSI info queue map */
9829 i40e_vsi_update_queue_map(vsi, &ctxt);
9830 vsi->info.valid_sections = 0;
9831 } else {
9832 /* Default/Main VSI is only enabled for TC0
9833 * reconfigure it to enable all TCs that are
9834 * available on the port in SFP mode.
9835 * For MFP case the iSCSI PF would use this
9836 * flow to enable LAN+iSCSI TC.
9837 */
9838 ret = i40e_vsi_config_tc(vsi, enabled_tc);
9839 if (ret) {
9840 dev_info(&pf->pdev->dev,
9841 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9842 enabled_tc,
9843 i40e_stat_str(&pf->hw, ret),
9844 i40e_aq_str(&pf->hw,
9845 pf->hw.aq.asq_last_status));
9846 ret = -ENOENT;
9847 }
9848 }
9849 break;
9850
9851 case I40E_VSI_FDIR:
9852 ctxt.pf_num = hw->pf_id;
9853 ctxt.vf_num = 0;
9854 ctxt.uplink_seid = vsi->uplink_seid;
9855 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9856 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9857 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9858 (i40e_is_vsi_uplink_mode_veb(vsi))) {
9859 ctxt.info.valid_sections |=
9860 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9861 ctxt.info.switch_id =
9862 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9863 }
9864 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9865 break;
9866
9867 case I40E_VSI_VMDQ2:
9868 ctxt.pf_num = hw->pf_id;
9869 ctxt.vf_num = 0;
9870 ctxt.uplink_seid = vsi->uplink_seid;
9871 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9872 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9873
9874 /* This VSI is connected to VEB so the switch_id
9875 * should be set to zero by default.
9876 */
9877 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9878 ctxt.info.valid_sections |=
9879 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9880 ctxt.info.switch_id =
9881 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9882 }
9883
9884 /* Setup the VSI tx/rx queue map for TC0 only for now */
9885 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9886 break;
9887
9888 case I40E_VSI_SRIOV:
9889 ctxt.pf_num = hw->pf_id;
9890 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9891 ctxt.uplink_seid = vsi->uplink_seid;
9892 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9893 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9894
9895 /* This VSI is connected to VEB so the switch_id
9896 * should be set to zero by default.
9897 */
9898 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9899 ctxt.info.valid_sections |=
9900 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9901 ctxt.info.switch_id =
9902 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9903 }
9904
9905 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9906 ctxt.info.valid_sections |=
9907 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9908 ctxt.info.queueing_opt_flags |=
9909 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
9910 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
9911 }
9912
9913 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9914 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9915 if (pf->vf[vsi->vf_id].spoofchk) {
9916 ctxt.info.valid_sections |=
9917 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9918 ctxt.info.sec_flags |=
9919 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9920 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9921 }
9922 /* Setup the VSI tx/rx queue map for TC0 only for now */
9923 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9924 break;
9925
9926 case I40E_VSI_IWARP:
9927 /* send down message to iWARP */
9928 break;
9929
9930 default:
9931 return -ENODEV;
9932 }
9933
9934 if (vsi->type != I40E_VSI_MAIN) {
9935 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9936 if (ret) {
9937 dev_info(&vsi->back->pdev->dev,
9938 "add vsi failed, err %s aq_err %s\n",
9939 i40e_stat_str(&pf->hw, ret),
9940 i40e_aq_str(&pf->hw,
9941 pf->hw.aq.asq_last_status));
9942 ret = -ENOENT;
9943 goto err;
9944 }
9945 vsi->info = ctxt.info;
9946 vsi->info.valid_sections = 0;
9947 vsi->seid = ctxt.seid;
9948 vsi->id = ctxt.vsi_number;
9949 }
9950
9951 vsi->active_filters = 0;
9952 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
9953 spin_lock_bh(&vsi->mac_filter_hash_lock);
9954 /* If macvlan filters already exist, force them to get loaded */
9955 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
9956 f->state = I40E_FILTER_NEW;
9957 f_count++;
9958 }
9959 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9960
9961 if (f_count) {
9962 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
9963 pf->flags |= I40E_FLAG_FILTER_SYNC;
9964 }
9965
9966 /* Update VSI BW information */
9967 ret = i40e_vsi_get_bw_info(vsi);
9968 if (ret) {
9969 dev_info(&pf->pdev->dev,
9970 "couldn't get vsi bw info, err %s aq_err %s\n",
9971 i40e_stat_str(&pf->hw, ret),
9972 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9973 /* VSI is already added so not tearing that up */
9974 ret = 0;
9975 }
9976
9977 err:
9978 return ret;
9979 }
9980
9981 /**
9982 * i40e_vsi_release - Delete a VSI and free its resources
9983 * @vsi: the VSI being removed
9984 *
9985 * Returns 0 on success or < 0 on error
9986 **/
9987 int i40e_vsi_release(struct i40e_vsi *vsi)
9988 {
9989 struct i40e_mac_filter *f;
9990 struct hlist_node *h;
9991 struct i40e_veb *veb = NULL;
9992 struct i40e_pf *pf;
9993 u16 uplink_seid;
9994 int i, n, bkt;
9995
9996 pf = vsi->back;
9997
9998 /* release of a VEB-owner or last VSI is not allowed */
9999 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
10000 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
10001 vsi->seid, vsi->uplink_seid);
10002 return -ENODEV;
10003 }
10004 if (vsi == pf->vsi[pf->lan_vsi] &&
10005 !test_bit(__I40E_DOWN, pf->state)) {
10006 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
10007 return -ENODEV;
10008 }
10009
10010 uplink_seid = vsi->uplink_seid;
10011 if (vsi->type != I40E_VSI_SRIOV) {
10012 if (vsi->netdev_registered) {
10013 vsi->netdev_registered = false;
10014 if (vsi->netdev) {
10015 /* results in a call to i40e_close() */
10016 unregister_netdev(vsi->netdev);
10017 }
10018 } else {
10019 i40e_vsi_close(vsi);
10020 }
10021 i40e_vsi_disable_irq(vsi);
10022 }
10023
10024 spin_lock_bh(&vsi->mac_filter_hash_lock);
10025
10026 /* clear the sync flag on all filters */
10027 if (vsi->netdev) {
10028 __dev_uc_unsync(vsi->netdev, NULL);
10029 __dev_mc_unsync(vsi->netdev, NULL);
10030 }
10031
10032 /* make sure any remaining filters are marked for deletion */
10033 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
10034 __i40e_del_filter(vsi, f);
10035
10036 spin_unlock_bh(&vsi->mac_filter_hash_lock);
10037
10038 i40e_sync_vsi_filters(vsi);
10039
10040 i40e_vsi_delete(vsi);
10041 i40e_vsi_free_q_vectors(vsi);
10042 if (vsi->netdev) {
10043 free_netdev(vsi->netdev);
10044 vsi->netdev = NULL;
10045 }
10046 i40e_vsi_clear_rings(vsi);
10047 i40e_vsi_clear(vsi);
10048
10049 /* If this was the last thing on the VEB, except for the
10050 * controlling VSI, remove the VEB, which puts the controlling
10051 * VSI onto the next level down in the switch.
10052 *
10053 * Well, okay, there's one more exception here: don't remove
10054 * the orphan VEBs yet. We'll wait for an explicit remove request
10055 * from up the network stack.
10056 */
10057 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
10058 if (pf->vsi[i] &&
10059 pf->vsi[i]->uplink_seid == uplink_seid &&
10060 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10061 n++; /* count the VSIs */
10062 }
10063 }
10064 for (i = 0; i < I40E_MAX_VEB; i++) {
10065 if (!pf->veb[i])
10066 continue;
10067 if (pf->veb[i]->uplink_seid == uplink_seid)
10068 n++; /* count the VEBs */
10069 if (pf->veb[i]->seid == uplink_seid)
10070 veb = pf->veb[i];
10071 }
10072 if (n == 0 && veb && veb->uplink_seid != 0)
10073 i40e_veb_release(veb);
10074
10075 return 0;
10076 }
10077
10078 /**
10079 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
10080 * @vsi: ptr to the VSI
10081 *
10082 * This should only be called after i40e_vsi_mem_alloc() which allocates the
10083 * corresponding SW VSI structure and initializes num_queue_pairs for the
10084 * newly allocated VSI.
10085 *
10086 * Returns 0 on success or negative on failure
10087 **/
10088 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
10089 {
10090 int ret = -ENOENT;
10091 struct i40e_pf *pf = vsi->back;
10092
10093 if (vsi->q_vectors[0]) {
10094 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
10095 vsi->seid);
10096 return -EEXIST;
10097 }
10098
10099 if (vsi->base_vector) {
10100 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
10101 vsi->seid, vsi->base_vector);
10102 return -EEXIST;
10103 }
10104
10105 ret = i40e_vsi_alloc_q_vectors(vsi);
10106 if (ret) {
10107 dev_info(&pf->pdev->dev,
10108 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
10109 vsi->num_q_vectors, vsi->seid, ret);
10110 vsi->num_q_vectors = 0;
10111 goto vector_setup_out;
10112 }
10113
10114 /* In Legacy mode, we do not have to get any other vector since we
10115 * piggyback on the misc/ICR0 for queue interrupts.
10116 */
10117 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10118 return ret;
10119 if (vsi->num_q_vectors)
10120 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
10121 vsi->num_q_vectors, vsi->idx);
10122 if (vsi->base_vector < 0) {
10123 dev_info(&pf->pdev->dev,
10124 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
10125 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
10126 i40e_vsi_free_q_vectors(vsi);
10127 ret = -ENOENT;
10128 goto vector_setup_out;
10129 }
10130
10131 vector_setup_out:
10132 return ret;
10133 }
10134
10135 /**
10136 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
10137 * @vsi: pointer to the vsi.
10138 *
10139 * This re-allocates a vsi's queue resources.
10140 *
10141 * Returns pointer to the successfully allocated and configured VSI sw struct
10142 * on success, otherwise returns NULL on failure.
10143 **/
10144 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
10145 {
10146 u16 alloc_queue_pairs;
10147 struct i40e_pf *pf;
10148 u8 enabled_tc;
10149 int ret;
10150
10151 if (!vsi)
10152 return NULL;
10153
10154 pf = vsi->back;
10155
10156 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10157 i40e_vsi_clear_rings(vsi);
10158
10159 i40e_vsi_free_arrays(vsi, false);
10160 i40e_set_num_rings_in_vsi(vsi);
10161 ret = i40e_vsi_alloc_arrays(vsi, false);
10162 if (ret)
10163 goto err_vsi;
10164
10165 alloc_queue_pairs = vsi->alloc_queue_pairs *
10166 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10167
10168 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10169 if (ret < 0) {
10170 dev_info(&pf->pdev->dev,
10171 "failed to get tracking for %d queues for VSI %d err %d\n",
10172 alloc_queue_pairs, vsi->seid, ret);
10173 goto err_vsi;
10174 }
10175 vsi->base_queue = ret;
10176
10177 /* Update the FW view of the VSI. Force a reset of TC and queue
10178 * layout configurations.
10179 */
10180 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10181 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10182 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10183 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10184 if (vsi->type == I40E_VSI_MAIN)
10185 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
10186
10187 /* assign it some queues */
10188 ret = i40e_alloc_rings(vsi);
10189 if (ret)
10190 goto err_rings;
10191
10192 /* map all of the rings to the q_vectors */
10193 i40e_vsi_map_rings_to_vectors(vsi);
10194 return vsi;
10195
10196 err_rings:
10197 i40e_vsi_free_q_vectors(vsi);
10198 if (vsi->netdev_registered) {
10199 vsi->netdev_registered = false;
10200 unregister_netdev(vsi->netdev);
10201 free_netdev(vsi->netdev);
10202 vsi->netdev = NULL;
10203 }
10204 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10205 err_vsi:
10206 i40e_vsi_clear(vsi);
10207 return NULL;
10208 }
10209
10210 /**
10211 * i40e_vsi_setup - Set up a VSI by a given type
10212 * @pf: board private structure
10213 * @type: VSI type
10214 * @uplink_seid: the switch element to link to
10215 * @param1: usage depends upon VSI type. For VF types, indicates VF id
10216 *
10217 * This allocates the sw VSI structure and its queue resources, then add a VSI
10218 * to the identified VEB.
10219 *
10220 * Returns pointer to the successfully allocated and configure VSI sw struct on
10221 * success, otherwise returns NULL on failure.
10222 **/
10223 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
10224 u16 uplink_seid, u32 param1)
10225 {
10226 struct i40e_vsi *vsi = NULL;
10227 struct i40e_veb *veb = NULL;
10228 u16 alloc_queue_pairs;
10229 int ret, i;
10230 int v_idx;
10231
10232 /* The requested uplink_seid must be either
10233 * - the PF's port seid
10234 * no VEB is needed because this is the PF
10235 * or this is a Flow Director special case VSI
10236 * - seid of an existing VEB
10237 * - seid of a VSI that owns an existing VEB
10238 * - seid of a VSI that doesn't own a VEB
10239 * a new VEB is created and the VSI becomes the owner
10240 * - seid of the PF VSI, which is what creates the first VEB
10241 * this is a special case of the previous
10242 *
10243 * Find which uplink_seid we were given and create a new VEB if needed
10244 */
10245 for (i = 0; i < I40E_MAX_VEB; i++) {
10246 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
10247 veb = pf->veb[i];
10248 break;
10249 }
10250 }
10251
10252 if (!veb && uplink_seid != pf->mac_seid) {
10253
10254 for (i = 0; i < pf->num_alloc_vsi; i++) {
10255 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
10256 vsi = pf->vsi[i];
10257 break;
10258 }
10259 }
10260 if (!vsi) {
10261 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
10262 uplink_seid);
10263 return NULL;
10264 }
10265
10266 if (vsi->uplink_seid == pf->mac_seid)
10267 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
10268 vsi->tc_config.enabled_tc);
10269 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
10270 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
10271 vsi->tc_config.enabled_tc);
10272 if (veb) {
10273 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
10274 dev_info(&vsi->back->pdev->dev,
10275 "New VSI creation error, uplink seid of LAN VSI expected.\n");
10276 return NULL;
10277 }
10278 /* We come up by default in VEPA mode if SRIOV is not
10279 * already enabled, in which case we can't force VEPA
10280 * mode.
10281 */
10282 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
10283 veb->bridge_mode = BRIDGE_MODE_VEPA;
10284 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
10285 }
10286 i40e_config_bridge_mode(veb);
10287 }
10288 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
10289 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
10290 veb = pf->veb[i];
10291 }
10292 if (!veb) {
10293 dev_info(&pf->pdev->dev, "couldn't add VEB\n");
10294 return NULL;
10295 }
10296
10297 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10298 uplink_seid = veb->seid;
10299 }
10300
10301 /* get vsi sw struct */
10302 v_idx = i40e_vsi_mem_alloc(pf, type);
10303 if (v_idx < 0)
10304 goto err_alloc;
10305 vsi = pf->vsi[v_idx];
10306 if (!vsi)
10307 goto err_alloc;
10308 vsi->type = type;
10309 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
10310
10311 if (type == I40E_VSI_MAIN)
10312 pf->lan_vsi = v_idx;
10313 else if (type == I40E_VSI_SRIOV)
10314 vsi->vf_id = param1;
10315 /* assign it some queues */
10316 alloc_queue_pairs = vsi->alloc_queue_pairs *
10317 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10318
10319 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10320 if (ret < 0) {
10321 dev_info(&pf->pdev->dev,
10322 "failed to get tracking for %d queues for VSI %d err=%d\n",
10323 alloc_queue_pairs, vsi->seid, ret);
10324 goto err_vsi;
10325 }
10326 vsi->base_queue = ret;
10327
10328 /* get a VSI from the hardware */
10329 vsi->uplink_seid = uplink_seid;
10330 ret = i40e_add_vsi(vsi);
10331 if (ret)
10332 goto err_vsi;
10333
10334 switch (vsi->type) {
10335 /* setup the netdev if needed */
10336 case I40E_VSI_MAIN:
10337 /* Apply relevant filters if a platform-specific mac
10338 * address was selected.
10339 */
10340 if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
10341 ret = i40e_macaddr_init(vsi, pf->hw.mac.addr);
10342 if (ret) {
10343 dev_warn(&pf->pdev->dev,
10344 "could not set up macaddr; err %d\n",
10345 ret);
10346 }
10347 }
10348 case I40E_VSI_VMDQ2:
10349 ret = i40e_config_netdev(vsi);
10350 if (ret)
10351 goto err_netdev;
10352 ret = register_netdev(vsi->netdev);
10353 if (ret)
10354 goto err_netdev;
10355 vsi->netdev_registered = true;
10356 netif_carrier_off(vsi->netdev);
10357 #ifdef CONFIG_I40E_DCB
10358 /* Setup DCB netlink interface */
10359 i40e_dcbnl_setup(vsi);
10360 #endif /* CONFIG_I40E_DCB */
10361 /* fall through */
10362
10363 case I40E_VSI_FDIR:
10364 /* set up vectors and rings if needed */
10365 ret = i40e_vsi_setup_vectors(vsi);
10366 if (ret)
10367 goto err_msix;
10368
10369 ret = i40e_alloc_rings(vsi);
10370 if (ret)
10371 goto err_rings;
10372
10373 /* map all of the rings to the q_vectors */
10374 i40e_vsi_map_rings_to_vectors(vsi);
10375
10376 i40e_vsi_reset_stats(vsi);
10377 break;
10378
10379 default:
10380 /* no netdev or rings for the other VSI types */
10381 break;
10382 }
10383
10384 if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
10385 (vsi->type == I40E_VSI_VMDQ2)) {
10386 ret = i40e_vsi_config_rss(vsi);
10387 }
10388 return vsi;
10389
10390 err_rings:
10391 i40e_vsi_free_q_vectors(vsi);
10392 err_msix:
10393 if (vsi->netdev_registered) {
10394 vsi->netdev_registered = false;
10395 unregister_netdev(vsi->netdev);
10396 free_netdev(vsi->netdev);
10397 vsi->netdev = NULL;
10398 }
10399 err_netdev:
10400 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10401 err_vsi:
10402 i40e_vsi_clear(vsi);
10403 err_alloc:
10404 return NULL;
10405 }
10406
10407 /**
10408 * i40e_veb_get_bw_info - Query VEB BW information
10409 * @veb: the veb to query
10410 *
10411 * Query the Tx scheduler BW configuration data for given VEB
10412 **/
10413 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10414 {
10415 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10416 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10417 struct i40e_pf *pf = veb->pf;
10418 struct i40e_hw *hw = &pf->hw;
10419 u32 tc_bw_max;
10420 int ret = 0;
10421 int i;
10422
10423 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10424 &bw_data, NULL);
10425 if (ret) {
10426 dev_info(&pf->pdev->dev,
10427 "query veb bw config failed, err %s aq_err %s\n",
10428 i40e_stat_str(&pf->hw, ret),
10429 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10430 goto out;
10431 }
10432
10433 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10434 &ets_data, NULL);
10435 if (ret) {
10436 dev_info(&pf->pdev->dev,
10437 "query veb bw ets config failed, err %s aq_err %s\n",
10438 i40e_stat_str(&pf->hw, ret),
10439 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10440 goto out;
10441 }
10442
10443 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10444 veb->bw_max_quanta = ets_data.tc_bw_max;
10445 veb->is_abs_credits = bw_data.absolute_credits_enable;
10446 veb->enabled_tc = ets_data.tc_valid_bits;
10447 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10448 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10449 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10450 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10451 veb->bw_tc_limit_credits[i] =
10452 le16_to_cpu(bw_data.tc_bw_limits[i]);
10453 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10454 }
10455
10456 out:
10457 return ret;
10458 }
10459
10460 /**
10461 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10462 * @pf: board private structure
10463 *
10464 * On error: returns error code (negative)
10465 * On success: returns vsi index in PF (positive)
10466 **/
10467 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10468 {
10469 int ret = -ENOENT;
10470 struct i40e_veb *veb;
10471 int i;
10472
10473 /* Need to protect the allocation of switch elements at the PF level */
10474 mutex_lock(&pf->switch_mutex);
10475
10476 /* VEB list may be fragmented if VEB creation/destruction has
10477 * been happening. We can afford to do a quick scan to look
10478 * for any free slots in the list.
10479 *
10480 * find next empty veb slot, looping back around if necessary
10481 */
10482 i = 0;
10483 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10484 i++;
10485 if (i >= I40E_MAX_VEB) {
10486 ret = -ENOMEM;
10487 goto err_alloc_veb; /* out of VEB slots! */
10488 }
10489
10490 veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10491 if (!veb) {
10492 ret = -ENOMEM;
10493 goto err_alloc_veb;
10494 }
10495 veb->pf = pf;
10496 veb->idx = i;
10497 veb->enabled_tc = 1;
10498
10499 pf->veb[i] = veb;
10500 ret = i;
10501 err_alloc_veb:
10502 mutex_unlock(&pf->switch_mutex);
10503 return ret;
10504 }
10505
10506 /**
10507 * i40e_switch_branch_release - Delete a branch of the switch tree
10508 * @branch: where to start deleting
10509 *
10510 * This uses recursion to find the tips of the branch to be
10511 * removed, deleting until we get back to and can delete this VEB.
10512 **/
10513 static void i40e_switch_branch_release(struct i40e_veb *branch)
10514 {
10515 struct i40e_pf *pf = branch->pf;
10516 u16 branch_seid = branch->seid;
10517 u16 veb_idx = branch->idx;
10518 int i;
10519
10520 /* release any VEBs on this VEB - RECURSION */
10521 for (i = 0; i < I40E_MAX_VEB; i++) {
10522 if (!pf->veb[i])
10523 continue;
10524 if (pf->veb[i]->uplink_seid == branch->seid)
10525 i40e_switch_branch_release(pf->veb[i]);
10526 }
10527
10528 /* Release the VSIs on this VEB, but not the owner VSI.
10529 *
10530 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10531 * the VEB itself, so don't use (*branch) after this loop.
10532 */
10533 for (i = 0; i < pf->num_alloc_vsi; i++) {
10534 if (!pf->vsi[i])
10535 continue;
10536 if (pf->vsi[i]->uplink_seid == branch_seid &&
10537 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10538 i40e_vsi_release(pf->vsi[i]);
10539 }
10540 }
10541
10542 /* There's one corner case where the VEB might not have been
10543 * removed, so double check it here and remove it if needed.
10544 * This case happens if the veb was created from the debugfs
10545 * commands and no VSIs were added to it.
10546 */
10547 if (pf->veb[veb_idx])
10548 i40e_veb_release(pf->veb[veb_idx]);
10549 }
10550
10551 /**
10552 * i40e_veb_clear - remove veb struct
10553 * @veb: the veb to remove
10554 **/
10555 static void i40e_veb_clear(struct i40e_veb *veb)
10556 {
10557 if (!veb)
10558 return;
10559
10560 if (veb->pf) {
10561 struct i40e_pf *pf = veb->pf;
10562
10563 mutex_lock(&pf->switch_mutex);
10564 if (pf->veb[veb->idx] == veb)
10565 pf->veb[veb->idx] = NULL;
10566 mutex_unlock(&pf->switch_mutex);
10567 }
10568
10569 kfree(veb);
10570 }
10571
10572 /**
10573 * i40e_veb_release - Delete a VEB and free its resources
10574 * @veb: the VEB being removed
10575 **/
10576 void i40e_veb_release(struct i40e_veb *veb)
10577 {
10578 struct i40e_vsi *vsi = NULL;
10579 struct i40e_pf *pf;
10580 int i, n = 0;
10581
10582 pf = veb->pf;
10583
10584 /* find the remaining VSI and check for extras */
10585 for (i = 0; i < pf->num_alloc_vsi; i++) {
10586 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10587 n++;
10588 vsi = pf->vsi[i];
10589 }
10590 }
10591 if (n != 1) {
10592 dev_info(&pf->pdev->dev,
10593 "can't remove VEB %d with %d VSIs left\n",
10594 veb->seid, n);
10595 return;
10596 }
10597
10598 /* move the remaining VSI to uplink veb */
10599 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10600 if (veb->uplink_seid) {
10601 vsi->uplink_seid = veb->uplink_seid;
10602 if (veb->uplink_seid == pf->mac_seid)
10603 vsi->veb_idx = I40E_NO_VEB;
10604 else
10605 vsi->veb_idx = veb->veb_idx;
10606 } else {
10607 /* floating VEB */
10608 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10609 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10610 }
10611
10612 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10613 i40e_veb_clear(veb);
10614 }
10615
10616 /**
10617 * i40e_add_veb - create the VEB in the switch
10618 * @veb: the VEB to be instantiated
10619 * @vsi: the controlling VSI
10620 **/
10621 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10622 {
10623 struct i40e_pf *pf = veb->pf;
10624 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10625 int ret;
10626
10627 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10628 veb->enabled_tc, false,
10629 &veb->seid, enable_stats, NULL);
10630
10631 /* get a VEB from the hardware */
10632 if (ret) {
10633 dev_info(&pf->pdev->dev,
10634 "couldn't add VEB, err %s aq_err %s\n",
10635 i40e_stat_str(&pf->hw, ret),
10636 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10637 return -EPERM;
10638 }
10639
10640 /* get statistics counter */
10641 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10642 &veb->stats_idx, NULL, NULL, NULL);
10643 if (ret) {
10644 dev_info(&pf->pdev->dev,
10645 "couldn't get VEB statistics idx, err %s aq_err %s\n",
10646 i40e_stat_str(&pf->hw, ret),
10647 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10648 return -EPERM;
10649 }
10650 ret = i40e_veb_get_bw_info(veb);
10651 if (ret) {
10652 dev_info(&pf->pdev->dev,
10653 "couldn't get VEB bw info, err %s aq_err %s\n",
10654 i40e_stat_str(&pf->hw, ret),
10655 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10656 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10657 return -ENOENT;
10658 }
10659
10660 vsi->uplink_seid = veb->seid;
10661 vsi->veb_idx = veb->idx;
10662 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10663
10664 return 0;
10665 }
10666
10667 /**
10668 * i40e_veb_setup - Set up a VEB
10669 * @pf: board private structure
10670 * @flags: VEB setup flags
10671 * @uplink_seid: the switch element to link to
10672 * @vsi_seid: the initial VSI seid
10673 * @enabled_tc: Enabled TC bit-map
10674 *
10675 * This allocates the sw VEB structure and links it into the switch
10676 * It is possible and legal for this to be a duplicate of an already
10677 * existing VEB. It is also possible for both uplink and vsi seids
10678 * to be zero, in order to create a floating VEB.
10679 *
10680 * Returns pointer to the successfully allocated VEB sw struct on
10681 * success, otherwise returns NULL on failure.
10682 **/
10683 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10684 u16 uplink_seid, u16 vsi_seid,
10685 u8 enabled_tc)
10686 {
10687 struct i40e_veb *veb, *uplink_veb = NULL;
10688 int vsi_idx, veb_idx;
10689 int ret;
10690
10691 /* if one seid is 0, the other must be 0 to create a floating relay */
10692 if ((uplink_seid == 0 || vsi_seid == 0) &&
10693 (uplink_seid + vsi_seid != 0)) {
10694 dev_info(&pf->pdev->dev,
10695 "one, not both seid's are 0: uplink=%d vsi=%d\n",
10696 uplink_seid, vsi_seid);
10697 return NULL;
10698 }
10699
10700 /* make sure there is such a vsi and uplink */
10701 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10702 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10703 break;
10704 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10705 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10706 vsi_seid);
10707 return NULL;
10708 }
10709
10710 if (uplink_seid && uplink_seid != pf->mac_seid) {
10711 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10712 if (pf->veb[veb_idx] &&
10713 pf->veb[veb_idx]->seid == uplink_seid) {
10714 uplink_veb = pf->veb[veb_idx];
10715 break;
10716 }
10717 }
10718 if (!uplink_veb) {
10719 dev_info(&pf->pdev->dev,
10720 "uplink seid %d not found\n", uplink_seid);
10721 return NULL;
10722 }
10723 }
10724
10725 /* get veb sw struct */
10726 veb_idx = i40e_veb_mem_alloc(pf);
10727 if (veb_idx < 0)
10728 goto err_alloc;
10729 veb = pf->veb[veb_idx];
10730 veb->flags = flags;
10731 veb->uplink_seid = uplink_seid;
10732 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10733 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10734
10735 /* create the VEB in the switch */
10736 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10737 if (ret)
10738 goto err_veb;
10739 if (vsi_idx == pf->lan_vsi)
10740 pf->lan_veb = veb->idx;
10741
10742 return veb;
10743
10744 err_veb:
10745 i40e_veb_clear(veb);
10746 err_alloc:
10747 return NULL;
10748 }
10749
10750 /**
10751 * i40e_setup_pf_switch_element - set PF vars based on switch type
10752 * @pf: board private structure
10753 * @ele: element we are building info from
10754 * @num_reported: total number of elements
10755 * @printconfig: should we print the contents
10756 *
10757 * helper function to assist in extracting a few useful SEID values.
10758 **/
10759 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10760 struct i40e_aqc_switch_config_element_resp *ele,
10761 u16 num_reported, bool printconfig)
10762 {
10763 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10764 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10765 u8 element_type = ele->element_type;
10766 u16 seid = le16_to_cpu(ele->seid);
10767
10768 if (printconfig)
10769 dev_info(&pf->pdev->dev,
10770 "type=%d seid=%d uplink=%d downlink=%d\n",
10771 element_type, seid, uplink_seid, downlink_seid);
10772
10773 switch (element_type) {
10774 case I40E_SWITCH_ELEMENT_TYPE_MAC:
10775 pf->mac_seid = seid;
10776 break;
10777 case I40E_SWITCH_ELEMENT_TYPE_VEB:
10778 /* Main VEB? */
10779 if (uplink_seid != pf->mac_seid)
10780 break;
10781 if (pf->lan_veb == I40E_NO_VEB) {
10782 int v;
10783
10784 /* find existing or else empty VEB */
10785 for (v = 0; v < I40E_MAX_VEB; v++) {
10786 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10787 pf->lan_veb = v;
10788 break;
10789 }
10790 }
10791 if (pf->lan_veb == I40E_NO_VEB) {
10792 v = i40e_veb_mem_alloc(pf);
10793 if (v < 0)
10794 break;
10795 pf->lan_veb = v;
10796 }
10797 }
10798
10799 pf->veb[pf->lan_veb]->seid = seid;
10800 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10801 pf->veb[pf->lan_veb]->pf = pf;
10802 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10803 break;
10804 case I40E_SWITCH_ELEMENT_TYPE_VSI:
10805 if (num_reported != 1)
10806 break;
10807 /* This is immediately after a reset so we can assume this is
10808 * the PF's VSI
10809 */
10810 pf->mac_seid = uplink_seid;
10811 pf->pf_seid = downlink_seid;
10812 pf->main_vsi_seid = seid;
10813 if (printconfig)
10814 dev_info(&pf->pdev->dev,
10815 "pf_seid=%d main_vsi_seid=%d\n",
10816 pf->pf_seid, pf->main_vsi_seid);
10817 break;
10818 case I40E_SWITCH_ELEMENT_TYPE_PF:
10819 case I40E_SWITCH_ELEMENT_TYPE_VF:
10820 case I40E_SWITCH_ELEMENT_TYPE_EMP:
10821 case I40E_SWITCH_ELEMENT_TYPE_BMC:
10822 case I40E_SWITCH_ELEMENT_TYPE_PE:
10823 case I40E_SWITCH_ELEMENT_TYPE_PA:
10824 /* ignore these for now */
10825 break;
10826 default:
10827 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10828 element_type, seid);
10829 break;
10830 }
10831 }
10832
10833 /**
10834 * i40e_fetch_switch_configuration - Get switch config from firmware
10835 * @pf: board private structure
10836 * @printconfig: should we print the contents
10837 *
10838 * Get the current switch configuration from the device and
10839 * extract a few useful SEID values.
10840 **/
10841 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10842 {
10843 struct i40e_aqc_get_switch_config_resp *sw_config;
10844 u16 next_seid = 0;
10845 int ret = 0;
10846 u8 *aq_buf;
10847 int i;
10848
10849 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10850 if (!aq_buf)
10851 return -ENOMEM;
10852
10853 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10854 do {
10855 u16 num_reported, num_total;
10856
10857 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10858 I40E_AQ_LARGE_BUF,
10859 &next_seid, NULL);
10860 if (ret) {
10861 dev_info(&pf->pdev->dev,
10862 "get switch config failed err %s aq_err %s\n",
10863 i40e_stat_str(&pf->hw, ret),
10864 i40e_aq_str(&pf->hw,
10865 pf->hw.aq.asq_last_status));
10866 kfree(aq_buf);
10867 return -ENOENT;
10868 }
10869
10870 num_reported = le16_to_cpu(sw_config->header.num_reported);
10871 num_total = le16_to_cpu(sw_config->header.num_total);
10872
10873 if (printconfig)
10874 dev_info(&pf->pdev->dev,
10875 "header: %d reported %d total\n",
10876 num_reported, num_total);
10877
10878 for (i = 0; i < num_reported; i++) {
10879 struct i40e_aqc_switch_config_element_resp *ele =
10880 &sw_config->element[i];
10881
10882 i40e_setup_pf_switch_element(pf, ele, num_reported,
10883 printconfig);
10884 }
10885 } while (next_seid != 0);
10886
10887 kfree(aq_buf);
10888 return ret;
10889 }
10890
10891 /**
10892 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10893 * @pf: board private structure
10894 * @reinit: if the Main VSI needs to re-initialized.
10895 *
10896 * Returns 0 on success, negative value on failure
10897 **/
10898 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10899 {
10900 u16 flags = 0;
10901 int ret;
10902
10903 /* find out what's out there already */
10904 ret = i40e_fetch_switch_configuration(pf, false);
10905 if (ret) {
10906 dev_info(&pf->pdev->dev,
10907 "couldn't fetch switch config, err %s aq_err %s\n",
10908 i40e_stat_str(&pf->hw, ret),
10909 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10910 return ret;
10911 }
10912 i40e_pf_reset_stats(pf);
10913
10914 /* set the switch config bit for the whole device to
10915 * support limited promisc or true promisc
10916 * when user requests promisc. The default is limited
10917 * promisc.
10918 */
10919
10920 if ((pf->hw.pf_id == 0) &&
10921 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
10922 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10923
10924 if (pf->hw.pf_id == 0) {
10925 u16 valid_flags;
10926
10927 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10928 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
10929 NULL);
10930 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
10931 dev_info(&pf->pdev->dev,
10932 "couldn't set switch config bits, err %s aq_err %s\n",
10933 i40e_stat_str(&pf->hw, ret),
10934 i40e_aq_str(&pf->hw,
10935 pf->hw.aq.asq_last_status));
10936 /* not a fatal problem, just keep going */
10937 }
10938 }
10939
10940 /* first time setup */
10941 if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10942 struct i40e_vsi *vsi = NULL;
10943 u16 uplink_seid;
10944
10945 /* Set up the PF VSI associated with the PF's main VSI
10946 * that is already in the HW switch
10947 */
10948 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10949 uplink_seid = pf->veb[pf->lan_veb]->seid;
10950 else
10951 uplink_seid = pf->mac_seid;
10952 if (pf->lan_vsi == I40E_NO_VSI)
10953 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10954 else if (reinit)
10955 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10956 if (!vsi) {
10957 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10958 i40e_fdir_teardown(pf);
10959 return -EAGAIN;
10960 }
10961 } else {
10962 /* force a reset of TC and queue layout configurations */
10963 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10964
10965 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10966 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10967 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10968 }
10969 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
10970
10971 i40e_fdir_sb_setup(pf);
10972
10973 /* Setup static PF queue filter control settings */
10974 ret = i40e_setup_pf_filter_control(pf);
10975 if (ret) {
10976 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
10977 ret);
10978 /* Failure here should not stop continuing other steps */
10979 }
10980
10981 /* enable RSS in the HW, even for only one queue, as the stack can use
10982 * the hash
10983 */
10984 if ((pf->flags & I40E_FLAG_RSS_ENABLED))
10985 i40e_pf_config_rss(pf);
10986
10987 /* fill in link information and enable LSE reporting */
10988 i40e_link_event(pf);
10989
10990 /* Initialize user-specific link properties */
10991 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
10992 I40E_AQ_AN_COMPLETED) ? true : false);
10993
10994 i40e_ptp_init(pf);
10995
10996 /* repopulate tunnel port filters */
10997 i40e_sync_udp_filters(pf);
10998
10999 return ret;
11000 }
11001
11002 /**
11003 * i40e_determine_queue_usage - Work out queue distribution
11004 * @pf: board private structure
11005 **/
11006 static void i40e_determine_queue_usage(struct i40e_pf *pf)
11007 {
11008 int queues_left;
11009
11010 pf->num_lan_qps = 0;
11011
11012 /* Find the max queues to be put into basic use. We'll always be
11013 * using TC0, whether or not DCB is running, and TC0 will get the
11014 * big RSS set.
11015 */
11016 queues_left = pf->hw.func_caps.num_tx_qp;
11017
11018 if ((queues_left == 1) ||
11019 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
11020 /* one qp for PF, no queues for anything else */
11021 queues_left = 0;
11022 pf->alloc_rss_size = pf->num_lan_qps = 1;
11023
11024 /* make sure all the fancies are disabled */
11025 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
11026 I40E_FLAG_IWARP_ENABLED |
11027 I40E_FLAG_FD_SB_ENABLED |
11028 I40E_FLAG_FD_ATR_ENABLED |
11029 I40E_FLAG_DCB_CAPABLE |
11030 I40E_FLAG_DCB_ENABLED |
11031 I40E_FLAG_SRIOV_ENABLED |
11032 I40E_FLAG_VMDQ_ENABLED);
11033 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
11034 I40E_FLAG_FD_SB_ENABLED |
11035 I40E_FLAG_FD_ATR_ENABLED |
11036 I40E_FLAG_DCB_CAPABLE))) {
11037 /* one qp for PF */
11038 pf->alloc_rss_size = pf->num_lan_qps = 1;
11039 queues_left -= pf->num_lan_qps;
11040
11041 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
11042 I40E_FLAG_IWARP_ENABLED |
11043 I40E_FLAG_FD_SB_ENABLED |
11044 I40E_FLAG_FD_ATR_ENABLED |
11045 I40E_FLAG_DCB_ENABLED |
11046 I40E_FLAG_VMDQ_ENABLED);
11047 } else {
11048 /* Not enough queues for all TCs */
11049 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
11050 (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
11051 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
11052 I40E_FLAG_DCB_ENABLED);
11053 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
11054 }
11055 pf->num_lan_qps = max_t(int, pf->rss_size_max,
11056 num_online_cpus());
11057 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
11058 pf->hw.func_caps.num_tx_qp);
11059
11060 queues_left -= pf->num_lan_qps;
11061 }
11062
11063 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11064 if (queues_left > 1) {
11065 queues_left -= 1; /* save 1 queue for FD */
11066 } else {
11067 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11068 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
11069 }
11070 }
11071
11072 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11073 pf->num_vf_qps && pf->num_req_vfs && queues_left) {
11074 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
11075 (queues_left / pf->num_vf_qps));
11076 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
11077 }
11078
11079 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11080 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
11081 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
11082 (queues_left / pf->num_vmdq_qps));
11083 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
11084 }
11085
11086 pf->queues_left = queues_left;
11087 dev_dbg(&pf->pdev->dev,
11088 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
11089 pf->hw.func_caps.num_tx_qp,
11090 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
11091 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
11092 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
11093 queues_left);
11094 }
11095
11096 /**
11097 * i40e_setup_pf_filter_control - Setup PF static filter control
11098 * @pf: PF to be setup
11099 *
11100 * i40e_setup_pf_filter_control sets up a PF's initial filter control
11101 * settings. If PE/FCoE are enabled then it will also set the per PF
11102 * based filter sizes required for them. It also enables Flow director,
11103 * ethertype and macvlan type filter settings for the pf.
11104 *
11105 * Returns 0 on success, negative on failure
11106 **/
11107 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
11108 {
11109 struct i40e_filter_control_settings *settings = &pf->filter_settings;
11110
11111 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
11112
11113 /* Flow Director is enabled */
11114 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
11115 settings->enable_fdir = true;
11116
11117 /* Ethtype and MACVLAN filters enabled for PF */
11118 settings->enable_ethtype = true;
11119 settings->enable_macvlan = true;
11120
11121 if (i40e_set_filter_control(&pf->hw, settings))
11122 return -ENOENT;
11123
11124 return 0;
11125 }
11126
11127 #define INFO_STRING_LEN 255
11128 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
11129 static void i40e_print_features(struct i40e_pf *pf)
11130 {
11131 struct i40e_hw *hw = &pf->hw;
11132 char *buf;
11133 int i;
11134
11135 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
11136 if (!buf)
11137 return;
11138
11139 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
11140 #ifdef CONFIG_PCI_IOV
11141 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
11142 #endif
11143 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
11144 pf->hw.func_caps.num_vsis,
11145 pf->vsi[pf->lan_vsi]->num_queue_pairs);
11146 if (pf->flags & I40E_FLAG_RSS_ENABLED)
11147 i += snprintf(&buf[i], REMAIN(i), " RSS");
11148 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
11149 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
11150 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11151 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
11152 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
11153 }
11154 if (pf->flags & I40E_FLAG_DCB_CAPABLE)
11155 i += snprintf(&buf[i], REMAIN(i), " DCB");
11156 i += snprintf(&buf[i], REMAIN(i), " VxLAN");
11157 i += snprintf(&buf[i], REMAIN(i), " Geneve");
11158 if (pf->flags & I40E_FLAG_PTP)
11159 i += snprintf(&buf[i], REMAIN(i), " PTP");
11160 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
11161 i += snprintf(&buf[i], REMAIN(i), " VEB");
11162 else
11163 i += snprintf(&buf[i], REMAIN(i), " VEPA");
11164
11165 dev_info(&pf->pdev->dev, "%s\n", buf);
11166 kfree(buf);
11167 WARN_ON(i > INFO_STRING_LEN);
11168 }
11169
11170 /**
11171 * i40e_get_platform_mac_addr - get platform-specific MAC address
11172 * @pdev: PCI device information struct
11173 * @pf: board private structure
11174 *
11175 * Look up the MAC address for the device. First we'll try
11176 * eth_platform_get_mac_address, which will check Open Firmware, or arch
11177 * specific fallback. Otherwise, we'll default to the stored value in
11178 * firmware.
11179 **/
11180 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
11181 {
11182 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
11183 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
11184 }
11185
11186 /**
11187 * i40e_probe - Device initialization routine
11188 * @pdev: PCI device information struct
11189 * @ent: entry in i40e_pci_tbl
11190 *
11191 * i40e_probe initializes a PF identified by a pci_dev structure.
11192 * The OS initialization, configuring of the PF private structure,
11193 * and a hardware reset occur.
11194 *
11195 * Returns 0 on success, negative on failure
11196 **/
11197 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11198 {
11199 struct i40e_aq_get_phy_abilities_resp abilities;
11200 struct i40e_pf *pf;
11201 struct i40e_hw *hw;
11202 static u16 pfs_found;
11203 u16 wol_nvm_bits;
11204 u16 link_status;
11205 int err;
11206 u32 val;
11207 u32 i;
11208 u8 set_fc_aq_fail;
11209
11210 err = pci_enable_device_mem(pdev);
11211 if (err)
11212 return err;
11213
11214 /* set up for high or low dma */
11215 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
11216 if (err) {
11217 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
11218 if (err) {
11219 dev_err(&pdev->dev,
11220 "DMA configuration failed: 0x%x\n", err);
11221 goto err_dma;
11222 }
11223 }
11224
11225 /* set up pci connections */
11226 err = pci_request_mem_regions(pdev, i40e_driver_name);
11227 if (err) {
11228 dev_info(&pdev->dev,
11229 "pci_request_selected_regions failed %d\n", err);
11230 goto err_pci_reg;
11231 }
11232
11233 pci_enable_pcie_error_reporting(pdev);
11234 pci_set_master(pdev);
11235
11236 /* Now that we have a PCI connection, we need to do the
11237 * low level device setup. This is primarily setting up
11238 * the Admin Queue structures and then querying for the
11239 * device's current profile information.
11240 */
11241 pf = kzalloc(sizeof(*pf), GFP_KERNEL);
11242 if (!pf) {
11243 err = -ENOMEM;
11244 goto err_pf_alloc;
11245 }
11246 pf->next_vsi = 0;
11247 pf->pdev = pdev;
11248 set_bit(__I40E_DOWN, pf->state);
11249
11250 hw = &pf->hw;
11251 hw->back = pf;
11252
11253 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
11254 I40E_MAX_CSR_SPACE);
11255
11256 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
11257 if (!hw->hw_addr) {
11258 err = -EIO;
11259 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
11260 (unsigned int)pci_resource_start(pdev, 0),
11261 pf->ioremap_len, err);
11262 goto err_ioremap;
11263 }
11264 hw->vendor_id = pdev->vendor;
11265 hw->device_id = pdev->device;
11266 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
11267 hw->subsystem_vendor_id = pdev->subsystem_vendor;
11268 hw->subsystem_device_id = pdev->subsystem_device;
11269 hw->bus.device = PCI_SLOT(pdev->devfn);
11270 hw->bus.func = PCI_FUNC(pdev->devfn);
11271 hw->bus.bus_id = pdev->bus->number;
11272 pf->instance = pfs_found;
11273
11274 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
11275 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
11276
11277 /* set up the locks for the AQ, do this only once in probe
11278 * and destroy them only once in remove
11279 */
11280 mutex_init(&hw->aq.asq_mutex);
11281 mutex_init(&hw->aq.arq_mutex);
11282
11283 pf->msg_enable = netif_msg_init(debug,
11284 NETIF_MSG_DRV |
11285 NETIF_MSG_PROBE |
11286 NETIF_MSG_LINK);
11287 if (debug < -1)
11288 pf->hw.debug_mask = debug;
11289
11290 /* do a special CORER for clearing PXE mode once at init */
11291 if (hw->revision_id == 0 &&
11292 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
11293 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
11294 i40e_flush(hw);
11295 msleep(200);
11296 pf->corer_count++;
11297
11298 i40e_clear_pxe_mode(hw);
11299 }
11300
11301 /* Reset here to make sure all is clean and to define PF 'n' */
11302 i40e_clear_hw(hw);
11303 err = i40e_pf_reset(hw);
11304 if (err) {
11305 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
11306 goto err_pf_reset;
11307 }
11308 pf->pfr_count++;
11309
11310 hw->aq.num_arq_entries = I40E_AQ_LEN;
11311 hw->aq.num_asq_entries = I40E_AQ_LEN;
11312 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11313 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11314 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
11315
11316 snprintf(pf->int_name, sizeof(pf->int_name) - 1,
11317 "%s-%s:misc",
11318 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
11319
11320 err = i40e_init_shared_code(hw);
11321 if (err) {
11322 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
11323 err);
11324 goto err_pf_reset;
11325 }
11326
11327 /* set up a default setting for link flow control */
11328 pf->hw.fc.requested_mode = I40E_FC_NONE;
11329
11330 err = i40e_init_adminq(hw);
11331 if (err) {
11332 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11333 dev_info(&pdev->dev,
11334 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11335 else
11336 dev_info(&pdev->dev,
11337 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11338
11339 goto err_pf_reset;
11340 }
11341
11342 /* provide nvm, fw, api versions */
11343 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11344 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11345 hw->aq.api_maj_ver, hw->aq.api_min_ver,
11346 i40e_nvm_version_str(hw));
11347
11348 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11349 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
11350 dev_info(&pdev->dev,
11351 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11352 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
11353 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
11354 dev_info(&pdev->dev,
11355 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11356
11357 i40e_verify_eeprom(pf);
11358
11359 /* Rev 0 hardware was never productized */
11360 if (hw->revision_id < 1)
11361 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11362
11363 i40e_clear_pxe_mode(hw);
11364 err = i40e_get_capabilities(pf);
11365 if (err)
11366 goto err_adminq_setup;
11367
11368 err = i40e_sw_init(pf);
11369 if (err) {
11370 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11371 goto err_sw_init;
11372 }
11373
11374 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11375 hw->func_caps.num_rx_qp, 0, 0);
11376 if (err) {
11377 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11378 goto err_init_lan_hmc;
11379 }
11380
11381 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11382 if (err) {
11383 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11384 err = -ENOENT;
11385 goto err_configure_lan_hmc;
11386 }
11387
11388 /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11389 * Ignore error return codes because if it was already disabled via
11390 * hardware settings this will fail
11391 */
11392 if (pf->flags & I40E_FLAG_STOP_FW_LLDP) {
11393 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11394 i40e_aq_stop_lldp(hw, true, NULL);
11395 }
11396
11397 /* allow a platform config to override the HW addr */
11398 i40e_get_platform_mac_addr(pdev, pf);
11399
11400 if (!is_valid_ether_addr(hw->mac.addr)) {
11401 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11402 err = -EIO;
11403 goto err_mac_addr;
11404 }
11405 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11406 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11407 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11408 if (is_valid_ether_addr(hw->mac.port_addr))
11409 pf->flags |= I40E_FLAG_PORT_ID_VALID;
11410
11411 pci_set_drvdata(pdev, pf);
11412 pci_save_state(pdev);
11413 #ifdef CONFIG_I40E_DCB
11414 err = i40e_init_pf_dcb(pf);
11415 if (err) {
11416 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11417 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11418 /* Continue without DCB enabled */
11419 }
11420 #endif /* CONFIG_I40E_DCB */
11421
11422 /* set up periodic task facility */
11423 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11424 pf->service_timer_period = HZ;
11425
11426 INIT_WORK(&pf->service_task, i40e_service_task);
11427 clear_bit(__I40E_SERVICE_SCHED, pf->state);
11428
11429 /* NVM bit on means WoL disabled for the port */
11430 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11431 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11432 pf->wol_en = false;
11433 else
11434 pf->wol_en = true;
11435 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11436
11437 /* set up the main switch operations */
11438 i40e_determine_queue_usage(pf);
11439 err = i40e_init_interrupt_scheme(pf);
11440 if (err)
11441 goto err_switch_setup;
11442
11443 /* The number of VSIs reported by the FW is the minimum guaranteed
11444 * to us; HW supports far more and we share the remaining pool with
11445 * the other PFs. We allocate space for more than the guarantee with
11446 * the understanding that we might not get them all later.
11447 */
11448 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11449 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11450 else
11451 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11452
11453 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11454 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11455 GFP_KERNEL);
11456 if (!pf->vsi) {
11457 err = -ENOMEM;
11458 goto err_switch_setup;
11459 }
11460
11461 #ifdef CONFIG_PCI_IOV
11462 /* prep for VF support */
11463 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11464 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11465 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11466 if (pci_num_vf(pdev))
11467 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11468 }
11469 #endif
11470 err = i40e_setup_pf_switch(pf, false);
11471 if (err) {
11472 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11473 goto err_vsis;
11474 }
11475
11476 /* Make sure flow control is set according to current settings */
11477 err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11478 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11479 dev_dbg(&pf->pdev->dev,
11480 "Set fc with err %s aq_err %s on get_phy_cap\n",
11481 i40e_stat_str(hw, err),
11482 i40e_aq_str(hw, hw->aq.asq_last_status));
11483 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11484 dev_dbg(&pf->pdev->dev,
11485 "Set fc with err %s aq_err %s on set_phy_config\n",
11486 i40e_stat_str(hw, err),
11487 i40e_aq_str(hw, hw->aq.asq_last_status));
11488 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11489 dev_dbg(&pf->pdev->dev,
11490 "Set fc with err %s aq_err %s on get_link_info\n",
11491 i40e_stat_str(hw, err),
11492 i40e_aq_str(hw, hw->aq.asq_last_status));
11493
11494 /* if FDIR VSI was set up, start it now */
11495 for (i = 0; i < pf->num_alloc_vsi; i++) {
11496 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11497 i40e_vsi_open(pf->vsi[i]);
11498 break;
11499 }
11500 }
11501
11502 /* The driver only wants link up/down and module qualification
11503 * reports from firmware. Note the negative logic.
11504 */
11505 err = i40e_aq_set_phy_int_mask(&pf->hw,
11506 ~(I40E_AQ_EVENT_LINK_UPDOWN |
11507 I40E_AQ_EVENT_MEDIA_NA |
11508 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11509 if (err)
11510 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11511 i40e_stat_str(&pf->hw, err),
11512 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11513
11514 /* Reconfigure hardware for allowing smaller MSS in the case
11515 * of TSO, so that we avoid the MDD being fired and causing
11516 * a reset in the case of small MSS+TSO.
11517 */
11518 val = rd32(hw, I40E_REG_MSS);
11519 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11520 val &= ~I40E_REG_MSS_MIN_MASK;
11521 val |= I40E_64BYTE_MSS;
11522 wr32(hw, I40E_REG_MSS, val);
11523 }
11524
11525 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
11526 msleep(75);
11527 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11528 if (err)
11529 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11530 i40e_stat_str(&pf->hw, err),
11531 i40e_aq_str(&pf->hw,
11532 pf->hw.aq.asq_last_status));
11533 }
11534 /* The main driver is (mostly) up and happy. We need to set this state
11535 * before setting up the misc vector or we get a race and the vector
11536 * ends up disabled forever.
11537 */
11538 clear_bit(__I40E_DOWN, pf->state);
11539
11540 /* In case of MSIX we are going to setup the misc vector right here
11541 * to handle admin queue events etc. In case of legacy and MSI
11542 * the misc functionality and queue processing is combined in
11543 * the same vector and that gets setup at open.
11544 */
11545 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11546 err = i40e_setup_misc_vector(pf);
11547 if (err) {
11548 dev_info(&pdev->dev,
11549 "setup of misc vector failed: %d\n", err);
11550 goto err_vsis;
11551 }
11552 }
11553
11554 #ifdef CONFIG_PCI_IOV
11555 /* prep for VF support */
11556 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11557 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11558 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11559 /* disable link interrupts for VFs */
11560 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11561 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11562 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11563 i40e_flush(hw);
11564
11565 if (pci_num_vf(pdev)) {
11566 dev_info(&pdev->dev,
11567 "Active VFs found, allocating resources.\n");
11568 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11569 if (err)
11570 dev_info(&pdev->dev,
11571 "Error %d allocating resources for existing VFs\n",
11572 err);
11573 }
11574 }
11575 #endif /* CONFIG_PCI_IOV */
11576
11577 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11578 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11579 pf->num_iwarp_msix,
11580 I40E_IWARP_IRQ_PILE_ID);
11581 if (pf->iwarp_base_vector < 0) {
11582 dev_info(&pdev->dev,
11583 "failed to get tracking for %d vectors for IWARP err=%d\n",
11584 pf->num_iwarp_msix, pf->iwarp_base_vector);
11585 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11586 }
11587 }
11588
11589 i40e_dbg_pf_init(pf);
11590
11591 /* tell the firmware that we're starting */
11592 i40e_send_version(pf);
11593
11594 /* since everything's happy, start the service_task timer */
11595 mod_timer(&pf->service_timer,
11596 round_jiffies(jiffies + pf->service_timer_period));
11597
11598 /* add this PF to client device list and launch a client service task */
11599 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11600 err = i40e_lan_add_device(pf);
11601 if (err)
11602 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11603 err);
11604 }
11605
11606 #define PCI_SPEED_SIZE 8
11607 #define PCI_WIDTH_SIZE 8
11608 /* Devices on the IOSF bus do not have this information
11609 * and will report PCI Gen 1 x 1 by default so don't bother
11610 * checking them.
11611 */
11612 if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) {
11613 char speed[PCI_SPEED_SIZE] = "Unknown";
11614 char width[PCI_WIDTH_SIZE] = "Unknown";
11615
11616 /* Get the negotiated link width and speed from PCI config
11617 * space
11618 */
11619 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11620 &link_status);
11621
11622 i40e_set_pci_config_data(hw, link_status);
11623
11624 switch (hw->bus.speed) {
11625 case i40e_bus_speed_8000:
11626 strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11627 case i40e_bus_speed_5000:
11628 strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11629 case i40e_bus_speed_2500:
11630 strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11631 default:
11632 break;
11633 }
11634 switch (hw->bus.width) {
11635 case i40e_bus_width_pcie_x8:
11636 strncpy(width, "8", PCI_WIDTH_SIZE); break;
11637 case i40e_bus_width_pcie_x4:
11638 strncpy(width, "4", PCI_WIDTH_SIZE); break;
11639 case i40e_bus_width_pcie_x2:
11640 strncpy(width, "2", PCI_WIDTH_SIZE); break;
11641 case i40e_bus_width_pcie_x1:
11642 strncpy(width, "1", PCI_WIDTH_SIZE); break;
11643 default:
11644 break;
11645 }
11646
11647 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11648 speed, width);
11649
11650 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11651 hw->bus.speed < i40e_bus_speed_8000) {
11652 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11653 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11654 }
11655 }
11656
11657 /* get the requested speeds from the fw */
11658 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11659 if (err)
11660 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n",
11661 i40e_stat_str(&pf->hw, err),
11662 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11663 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11664
11665 /* get the supported phy types from the fw */
11666 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11667 if (err)
11668 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n",
11669 i40e_stat_str(&pf->hw, err),
11670 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11671
11672 /* Add a filter to drop all Flow control frames from any VSI from being
11673 * transmitted. By doing so we stop a malicious VF from sending out
11674 * PAUSE or PFC frames and potentially controlling traffic for other
11675 * PF/VF VSIs.
11676 * The FW can still send Flow control frames if enabled.
11677 */
11678 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11679 pf->main_vsi_seid);
11680
11681 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11682 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11683 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS;
11684 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11685 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER;
11686 /* print a string summarizing features */
11687 i40e_print_features(pf);
11688
11689 return 0;
11690
11691 /* Unwind what we've done if something failed in the setup */
11692 err_vsis:
11693 set_bit(__I40E_DOWN, pf->state);
11694 i40e_clear_interrupt_scheme(pf);
11695 kfree(pf->vsi);
11696 err_switch_setup:
11697 i40e_reset_interrupt_capability(pf);
11698 del_timer_sync(&pf->service_timer);
11699 err_mac_addr:
11700 err_configure_lan_hmc:
11701 (void)i40e_shutdown_lan_hmc(hw);
11702 err_init_lan_hmc:
11703 kfree(pf->qp_pile);
11704 err_sw_init:
11705 err_adminq_setup:
11706 err_pf_reset:
11707 iounmap(hw->hw_addr);
11708 err_ioremap:
11709 kfree(pf);
11710 err_pf_alloc:
11711 pci_disable_pcie_error_reporting(pdev);
11712 pci_release_mem_regions(pdev);
11713 err_pci_reg:
11714 err_dma:
11715 pci_disable_device(pdev);
11716 return err;
11717 }
11718
11719 /**
11720 * i40e_remove - Device removal routine
11721 * @pdev: PCI device information struct
11722 *
11723 * i40e_remove is called by the PCI subsystem to alert the driver
11724 * that is should release a PCI device. This could be caused by a
11725 * Hot-Plug event, or because the driver is going to be removed from
11726 * memory.
11727 **/
11728 static void i40e_remove(struct pci_dev *pdev)
11729 {
11730 struct i40e_pf *pf = pci_get_drvdata(pdev);
11731 struct i40e_hw *hw = &pf->hw;
11732 i40e_status ret_code;
11733 int i;
11734
11735 i40e_dbg_pf_exit(pf);
11736
11737 i40e_ptp_stop(pf);
11738
11739 /* Disable RSS in hw */
11740 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11741 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11742
11743 /* no more scheduling of any task */
11744 set_bit(__I40E_SUSPENDED, pf->state);
11745 set_bit(__I40E_DOWN, pf->state);
11746 if (pf->service_timer.data)
11747 del_timer_sync(&pf->service_timer);
11748 if (pf->service_task.func)
11749 cancel_work_sync(&pf->service_task);
11750
11751 /* Client close must be called explicitly here because the timer
11752 * has been stopped.
11753 */
11754 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11755
11756 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11757 i40e_free_vfs(pf);
11758 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11759 }
11760
11761 i40e_fdir_teardown(pf);
11762
11763 /* If there is a switch structure or any orphans, remove them.
11764 * This will leave only the PF's VSI remaining.
11765 */
11766 for (i = 0; i < I40E_MAX_VEB; i++) {
11767 if (!pf->veb[i])
11768 continue;
11769
11770 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11771 pf->veb[i]->uplink_seid == 0)
11772 i40e_switch_branch_release(pf->veb[i]);
11773 }
11774
11775 /* Now we can shutdown the PF's VSI, just before we kill
11776 * adminq and hmc.
11777 */
11778 if (pf->vsi[pf->lan_vsi])
11779 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11780
11781 /* remove attached clients */
11782 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11783 ret_code = i40e_lan_del_device(pf);
11784 if (ret_code)
11785 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11786 ret_code);
11787 }
11788
11789 /* shutdown and destroy the HMC */
11790 if (hw->hmc.hmc_obj) {
11791 ret_code = i40e_shutdown_lan_hmc(hw);
11792 if (ret_code)
11793 dev_warn(&pdev->dev,
11794 "Failed to destroy the HMC resources: %d\n",
11795 ret_code);
11796 }
11797
11798 /* shutdown the adminq */
11799 i40e_shutdown_adminq(hw);
11800
11801 /* destroy the locks only once, here */
11802 mutex_destroy(&hw->aq.arq_mutex);
11803 mutex_destroy(&hw->aq.asq_mutex);
11804
11805 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11806 i40e_clear_interrupt_scheme(pf);
11807 for (i = 0; i < pf->num_alloc_vsi; i++) {
11808 if (pf->vsi[i]) {
11809 i40e_vsi_clear_rings(pf->vsi[i]);
11810 i40e_vsi_clear(pf->vsi[i]);
11811 pf->vsi[i] = NULL;
11812 }
11813 }
11814
11815 for (i = 0; i < I40E_MAX_VEB; i++) {
11816 kfree(pf->veb[i]);
11817 pf->veb[i] = NULL;
11818 }
11819
11820 kfree(pf->qp_pile);
11821 kfree(pf->vsi);
11822
11823 iounmap(hw->hw_addr);
11824 kfree(pf);
11825 pci_release_mem_regions(pdev);
11826
11827 pci_disable_pcie_error_reporting(pdev);
11828 pci_disable_device(pdev);
11829 }
11830
11831 /**
11832 * i40e_pci_error_detected - warning that something funky happened in PCI land
11833 * @pdev: PCI device information struct
11834 *
11835 * Called to warn that something happened and the error handling steps
11836 * are in progress. Allows the driver to quiesce things, be ready for
11837 * remediation.
11838 **/
11839 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11840 enum pci_channel_state error)
11841 {
11842 struct i40e_pf *pf = pci_get_drvdata(pdev);
11843
11844 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11845
11846 if (!pf) {
11847 dev_info(&pdev->dev,
11848 "Cannot recover - error happened during device probe\n");
11849 return PCI_ERS_RESULT_DISCONNECT;
11850 }
11851
11852 /* shutdown all operations */
11853 if (!test_bit(__I40E_SUSPENDED, pf->state)) {
11854 rtnl_lock();
11855 i40e_prep_for_reset(pf, true);
11856 rtnl_unlock();
11857 }
11858
11859 /* Request a slot reset */
11860 return PCI_ERS_RESULT_NEED_RESET;
11861 }
11862
11863 /**
11864 * i40e_pci_error_slot_reset - a PCI slot reset just happened
11865 * @pdev: PCI device information struct
11866 *
11867 * Called to find if the driver can work with the device now that
11868 * the pci slot has been reset. If a basic connection seems good
11869 * (registers are readable and have sane content) then return a
11870 * happy little PCI_ERS_RESULT_xxx.
11871 **/
11872 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11873 {
11874 struct i40e_pf *pf = pci_get_drvdata(pdev);
11875 pci_ers_result_t result;
11876 int err;
11877 u32 reg;
11878
11879 dev_dbg(&pdev->dev, "%s\n", __func__);
11880 if (pci_enable_device_mem(pdev)) {
11881 dev_info(&pdev->dev,
11882 "Cannot re-enable PCI device after reset.\n");
11883 result = PCI_ERS_RESULT_DISCONNECT;
11884 } else {
11885 pci_set_master(pdev);
11886 pci_restore_state(pdev);
11887 pci_save_state(pdev);
11888 pci_wake_from_d3(pdev, false);
11889
11890 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11891 if (reg == 0)
11892 result = PCI_ERS_RESULT_RECOVERED;
11893 else
11894 result = PCI_ERS_RESULT_DISCONNECT;
11895 }
11896
11897 err = pci_cleanup_aer_uncorrect_error_status(pdev);
11898 if (err) {
11899 dev_info(&pdev->dev,
11900 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11901 err);
11902 /* non-fatal, continue */
11903 }
11904
11905 return result;
11906 }
11907
11908 /**
11909 * i40e_pci_error_resume - restart operations after PCI error recovery
11910 * @pdev: PCI device information struct
11911 *
11912 * Called to allow the driver to bring things back up after PCI error
11913 * and/or reset recovery has finished.
11914 **/
11915 static void i40e_pci_error_resume(struct pci_dev *pdev)
11916 {
11917 struct i40e_pf *pf = pci_get_drvdata(pdev);
11918
11919 dev_dbg(&pdev->dev, "%s\n", __func__);
11920 if (test_bit(__I40E_SUSPENDED, pf->state))
11921 return;
11922
11923 rtnl_lock();
11924 i40e_handle_reset_warning(pf, true);
11925 rtnl_unlock();
11926 }
11927
11928 /**
11929 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
11930 * using the mac_address_write admin q function
11931 * @pf: pointer to i40e_pf struct
11932 **/
11933 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
11934 {
11935 struct i40e_hw *hw = &pf->hw;
11936 i40e_status ret;
11937 u8 mac_addr[6];
11938 u16 flags = 0;
11939
11940 /* Get current MAC address in case it's an LAA */
11941 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
11942 ether_addr_copy(mac_addr,
11943 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
11944 } else {
11945 dev_err(&pf->pdev->dev,
11946 "Failed to retrieve MAC address; using default\n");
11947 ether_addr_copy(mac_addr, hw->mac.addr);
11948 }
11949
11950 /* The FW expects the mac address write cmd to first be called with
11951 * one of these flags before calling it again with the multicast
11952 * enable flags.
11953 */
11954 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
11955
11956 if (hw->func_caps.flex10_enable && hw->partition_id != 1)
11957 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
11958
11959 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11960 if (ret) {
11961 dev_err(&pf->pdev->dev,
11962 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
11963 return;
11964 }
11965
11966 flags = I40E_AQC_MC_MAG_EN
11967 | I40E_AQC_WOL_PRESERVE_ON_PFR
11968 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
11969 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11970 if (ret)
11971 dev_err(&pf->pdev->dev,
11972 "Failed to enable Multicast Magic Packet wake up\n");
11973 }
11974
11975 /**
11976 * i40e_shutdown - PCI callback for shutting down
11977 * @pdev: PCI device information struct
11978 **/
11979 static void i40e_shutdown(struct pci_dev *pdev)
11980 {
11981 struct i40e_pf *pf = pci_get_drvdata(pdev);
11982 struct i40e_hw *hw = &pf->hw;
11983
11984 set_bit(__I40E_SUSPENDED, pf->state);
11985 set_bit(__I40E_DOWN, pf->state);
11986 rtnl_lock();
11987 i40e_prep_for_reset(pf, true);
11988 rtnl_unlock();
11989
11990 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11991 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11992
11993 del_timer_sync(&pf->service_timer);
11994 cancel_work_sync(&pf->service_task);
11995 i40e_fdir_teardown(pf);
11996
11997 /* Client close must be called explicitly here because the timer
11998 * has been stopped.
11999 */
12000 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
12001
12002 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
12003 i40e_enable_mc_magic_wake(pf);
12004
12005 rtnl_lock();
12006 i40e_prep_for_reset(pf, true);
12007 rtnl_unlock();
12008
12009 wr32(hw, I40E_PFPM_APM,
12010 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12011 wr32(hw, I40E_PFPM_WUFC,
12012 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12013
12014 i40e_clear_interrupt_scheme(pf);
12015
12016 if (system_state == SYSTEM_POWER_OFF) {
12017 pci_wake_from_d3(pdev, pf->wol_en);
12018 pci_set_power_state(pdev, PCI_D3hot);
12019 }
12020 }
12021
12022 #ifdef CONFIG_PM
12023 /**
12024 * i40e_suspend - PCI callback for moving to D3
12025 * @pdev: PCI device information struct
12026 **/
12027 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
12028 {
12029 struct i40e_pf *pf = pci_get_drvdata(pdev);
12030 struct i40e_hw *hw = &pf->hw;
12031 int retval = 0;
12032
12033 set_bit(__I40E_SUSPENDED, pf->state);
12034 set_bit(__I40E_DOWN, pf->state);
12035
12036 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
12037 i40e_enable_mc_magic_wake(pf);
12038
12039 rtnl_lock();
12040 i40e_prep_for_reset(pf, true);
12041 rtnl_unlock();
12042
12043 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12044 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12045
12046 i40e_stop_misc_vector(pf);
12047
12048 retval = pci_save_state(pdev);
12049 if (retval)
12050 return retval;
12051
12052 pci_wake_from_d3(pdev, pf->wol_en);
12053 pci_set_power_state(pdev, PCI_D3hot);
12054
12055 return retval;
12056 }
12057
12058 /**
12059 * i40e_resume - PCI callback for waking up from D3
12060 * @pdev: PCI device information struct
12061 **/
12062 static int i40e_resume(struct pci_dev *pdev)
12063 {
12064 struct i40e_pf *pf = pci_get_drvdata(pdev);
12065 u32 err;
12066
12067 pci_set_power_state(pdev, PCI_D0);
12068 pci_restore_state(pdev);
12069 /* pci_restore_state() clears dev->state_saves, so
12070 * call pci_save_state() again to restore it.
12071 */
12072 pci_save_state(pdev);
12073
12074 err = pci_enable_device_mem(pdev);
12075 if (err) {
12076 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
12077 return err;
12078 }
12079 pci_set_master(pdev);
12080
12081 /* no wakeup events while running */
12082 pci_wake_from_d3(pdev, false);
12083
12084 /* handling the reset will rebuild the device state */
12085 if (test_and_clear_bit(__I40E_SUSPENDED, pf->state)) {
12086 clear_bit(__I40E_DOWN, pf->state);
12087 rtnl_lock();
12088 i40e_reset_and_rebuild(pf, false, true);
12089 rtnl_unlock();
12090 }
12091
12092 return 0;
12093 }
12094
12095 #endif
12096 static const struct pci_error_handlers i40e_err_handler = {
12097 .error_detected = i40e_pci_error_detected,
12098 .slot_reset = i40e_pci_error_slot_reset,
12099 .resume = i40e_pci_error_resume,
12100 };
12101
12102 static struct pci_driver i40e_driver = {
12103 .name = i40e_driver_name,
12104 .id_table = i40e_pci_tbl,
12105 .probe = i40e_probe,
12106 .remove = i40e_remove,
12107 #ifdef CONFIG_PM
12108 .suspend = i40e_suspend,
12109 .resume = i40e_resume,
12110 #endif
12111 .shutdown = i40e_shutdown,
12112 .err_handler = &i40e_err_handler,
12113 .sriov_configure = i40e_pci_sriov_configure,
12114 };
12115
12116 /**
12117 * i40e_init_module - Driver registration routine
12118 *
12119 * i40e_init_module is the first routine called when the driver is
12120 * loaded. All it does is register with the PCI subsystem.
12121 **/
12122 static int __init i40e_init_module(void)
12123 {
12124 pr_info("%s: %s - version %s\n", i40e_driver_name,
12125 i40e_driver_string, i40e_driver_version_str);
12126 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
12127
12128 /* we will see if single thread per module is enough for now,
12129 * it can't be any worse than using the system workqueue which
12130 * was already single threaded
12131 */
12132 i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
12133 i40e_driver_name);
12134 if (!i40e_wq) {
12135 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
12136 return -ENOMEM;
12137 }
12138
12139 i40e_dbg_init();
12140 return pci_register_driver(&i40e_driver);
12141 }
12142 module_init(i40e_init_module);
12143
12144 /**
12145 * i40e_exit_module - Driver exit cleanup routine
12146 *
12147 * i40e_exit_module is called just before the driver is removed
12148 * from memory.
12149 **/
12150 static void __exit i40e_exit_module(void)
12151 {
12152 pci_unregister_driver(&i40e_driver);
12153 destroy_workqueue(i40e_wq);
12154 i40e_dbg_exit();
12155 }
12156 module_exit(i40e_exit_module);
(deprecated) Btrfs devel tree