1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2010-2011 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
10 #include <linux/module.h>
11 #include "net_driver.h"
17 #include "mcdi_pcol.h"
18 #include "farch_regs.h"
21 /* Number of longs required to track all the VIs in a VF */
22 #define VI_MASK_LENGTH BITS_TO_LONGS(1 << EFX_VI_SCALE_MAX)
24 /* Maximum number of RX queues supported */
25 #define VF_MAX_RX_QUEUES 63
28 * enum efx_vf_tx_filter_mode - TX MAC filtering behaviour
29 * @VF_TX_FILTER_OFF: Disabled
30 * @VF_TX_FILTER_AUTO: Enabled if MAC address assigned to VF and only
31 * 2 TX queues allowed per VF.
32 * @VF_TX_FILTER_ON: Enabled
34 enum efx_vf_tx_filter_mode
{
41 * struct efx_vf - Back-end resource and protocol state for a PCI VF
42 * @efx: The Efx NIC owning this VF
43 * @pci_rid: The PCI requester ID for this VF
44 * @pci_name: The PCI name (formatted address) of this VF
45 * @index: Index of VF within its port and PF.
46 * @req: VFDI incoming request work item. Incoming USR_EV events are received
47 * by the NAPI handler, but must be handled by executing MCDI requests
49 * @req_addr: VFDI incoming request DMA address (in VF's PCI address space).
50 * @req_type: Expected next incoming (from VF) %VFDI_EV_TYPE member.
51 * @req_seqno: Expected next incoming (from VF) %VFDI_EV_SEQ member.
52 * @msg_seqno: Next %VFDI_EV_SEQ member to reply to VF. Protected by
54 * @busy: VFDI request queued to be processed or being processed. Receiving
55 * a VFDI request when @busy is set is an error condition.
56 * @buf: Incoming VFDI requests are DMA from the VF into this buffer.
57 * @buftbl_base: Buffer table entries for this VF start at this index.
58 * @rx_filtering: Receive filtering has been requested by the VF driver.
59 * @rx_filter_flags: The flags sent in the %VFDI_OP_INSERT_FILTER request.
60 * @rx_filter_qid: VF relative qid for RX filter requested by VF.
61 * @rx_filter_id: Receive MAC filter ID. Only one filter per VF is supported.
62 * @tx_filter_mode: Transmit MAC filtering mode.
63 * @tx_filter_id: Transmit MAC filter ID.
64 * @addr: The MAC address and outer vlan tag of the VF.
65 * @status_addr: VF DMA address of page for &struct vfdi_status updates.
66 * @status_lock: Mutex protecting @msg_seqno, @status_addr, @addr,
67 * @peer_page_addrs and @peer_page_count from simultaneous
68 * updates by the VM and consumption by
69 * efx_sriov_update_vf_addr()
70 * @peer_page_addrs: Pointer to an array of guest pages for local addresses.
71 * @peer_page_count: Number of entries in @peer_page_count.
72 * @evq0_addrs: Array of guest pages backing evq0.
73 * @evq0_count: Number of entries in @evq0_addrs.
74 * @flush_waitq: wait queue used by %VFDI_OP_FINI_ALL_QUEUES handler
75 * to wait for flush completions.
76 * @txq_lock: Mutex for TX queue allocation.
77 * @txq_mask: Mask of initialized transmit queues.
78 * @txq_count: Number of initialized transmit queues.
79 * @rxq_mask: Mask of initialized receive queues.
80 * @rxq_count: Number of initialized receive queues.
81 * @rxq_retry_mask: Mask or receive queues that need to be flushed again
82 * due to flush failure.
83 * @rxq_retry_count: Number of receive queues in @rxq_retry_mask.
84 * @reset_work: Work item to schedule a VF reset.
89 char pci_name
[13]; /* dddd:bb:dd.f */
91 struct work_struct req
;
97 struct efx_buffer buf
;
100 enum efx_filter_flags rx_filter_flags
;
101 unsigned rx_filter_qid
;
103 enum efx_vf_tx_filter_mode tx_filter_mode
;
105 struct vfdi_endpoint addr
;
107 struct mutex status_lock
;
108 u64
*peer_page_addrs
;
109 unsigned peer_page_count
;
110 u64 evq0_addrs
[EFX_MAX_VF_EVQ_SIZE
* sizeof(efx_qword_t
) /
113 wait_queue_head_t flush_waitq
;
114 struct mutex txq_lock
;
115 unsigned long txq_mask
[VI_MASK_LENGTH
];
117 unsigned long rxq_mask
[VI_MASK_LENGTH
];
119 unsigned long rxq_retry_mask
[VI_MASK_LENGTH
];
120 atomic_t rxq_retry_count
;
121 struct work_struct reset_work
;
124 struct efx_memcpy_req
{
125 unsigned int from_rid
;
134 * struct efx_local_addr - A MAC address on the vswitch without a VF.
136 * Siena does not have a switch, so VFs can't transmit data to each
137 * other. Instead the VFs must be made aware of the local addresses
138 * on the vswitch, so that they can arrange for an alternative
139 * software datapath to be used.
141 * @link: List head for insertion into efx->local_addr_list.
142 * @addr: Ethernet address
144 struct efx_local_addr
{
145 struct list_head link
;
150 * struct efx_endpoint_page - Page of vfdi_endpoint structures
152 * @link: List head for insertion into efx->local_page_list.
153 * @ptr: Pointer to page.
154 * @addr: DMA address of page.
156 struct efx_endpoint_page
{
157 struct list_head link
;
162 /* Buffer table entries are reserved txq0,rxq0,evq0,txq1,rxq1,evq1 */
163 #define EFX_BUFTBL_TXQ_BASE(_vf, _qid) \
164 ((_vf)->buftbl_base + EFX_VF_BUFTBL_PER_VI * (_qid))
165 #define EFX_BUFTBL_RXQ_BASE(_vf, _qid) \
166 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \
167 (EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))
168 #define EFX_BUFTBL_EVQ_BASE(_vf, _qid) \
169 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \
170 (2 * EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))
172 #define EFX_FIELD_MASK(_field) \
173 ((1 << _field ## _WIDTH) - 1)
175 /* VFs can only use this many transmit channels */
176 static unsigned int vf_max_tx_channels
= 2;
177 module_param(vf_max_tx_channels
, uint
, 0444);
178 MODULE_PARM_DESC(vf_max_tx_channels
,
179 "Limit the number of TX channels VFs can use");
181 static int max_vfs
= -1;
182 module_param(max_vfs
, int, 0444);
183 MODULE_PARM_DESC(max_vfs
,
184 "Reduce the number of VFs initialized by the driver");
186 /* Workqueue used by VFDI communication. We can't use the global
187 * workqueue because it may be running the VF driver's probe()
188 * routine, which will be blocked there waiting for a VFDI response.
190 static struct workqueue_struct
*vfdi_workqueue
;
192 static unsigned abs_index(struct efx_vf
*vf
, unsigned index
)
194 return EFX_VI_BASE
+ vf
->index
* efx_vf_size(vf
->efx
) + index
;
197 static int efx_sriov_cmd(struct efx_nic
*efx
, bool enable
,
198 unsigned *vi_scale_out
, unsigned *vf_total_out
)
200 MCDI_DECLARE_BUF(inbuf
, MC_CMD_SRIOV_IN_LEN
);
201 MCDI_DECLARE_BUF(outbuf
, MC_CMD_SRIOV_OUT_LEN
);
202 unsigned vi_scale
, vf_total
;
206 MCDI_SET_DWORD(inbuf
, SRIOV_IN_ENABLE
, enable
? 1 : 0);
207 MCDI_SET_DWORD(inbuf
, SRIOV_IN_VI_BASE
, EFX_VI_BASE
);
208 MCDI_SET_DWORD(inbuf
, SRIOV_IN_VF_COUNT
, efx
->vf_count
);
210 rc
= efx_mcdi_rpc(efx
, MC_CMD_SRIOV
, inbuf
, MC_CMD_SRIOV_IN_LEN
,
211 outbuf
, MC_CMD_SRIOV_OUT_LEN
, &outlen
);
214 if (outlen
< MC_CMD_SRIOV_OUT_LEN
)
217 vf_total
= MCDI_DWORD(outbuf
, SRIOV_OUT_VF_TOTAL
);
218 vi_scale
= MCDI_DWORD(outbuf
, SRIOV_OUT_VI_SCALE
);
219 if (vi_scale
> EFX_VI_SCALE_MAX
)
223 *vi_scale_out
= vi_scale
;
225 *vf_total_out
= vf_total
;
230 static void efx_sriov_usrev(struct efx_nic
*efx
, bool enabled
)
234 EFX_POPULATE_OWORD_2(reg
,
235 FRF_CZ_USREV_DIS
, enabled
? 0 : 1,
236 FRF_CZ_DFLT_EVQ
, efx
->vfdi_channel
->channel
);
237 efx_writeo(efx
, ®
, FR_CZ_USR_EV_CFG
);
240 static int efx_sriov_memcpy(struct efx_nic
*efx
, struct efx_memcpy_req
*req
,
243 MCDI_DECLARE_BUF(inbuf
, MCDI_CTL_SDU_LEN_MAX_V1
);
244 MCDI_DECLARE_STRUCT_PTR(record
);
245 unsigned int index
, used
;
250 mb(); /* Finish writing source/reading dest before DMA starts */
252 if (WARN_ON(count
> MC_CMD_MEMCPY_IN_RECORD_MAXNUM
))
254 used
= MC_CMD_MEMCPY_IN_LEN(count
);
256 for (index
= 0; index
< count
; index
++) {
257 record
= MCDI_ARRAY_STRUCT_PTR(inbuf
, MEMCPY_IN_RECORD
, index
);
258 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_NUM_RECORDS
,
260 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_TO_RID
,
262 MCDI_SET_QWORD(record
, MEMCPY_RECORD_TYPEDEF_TO_ADDR
,
264 if (req
->from_buf
== NULL
) {
265 from_rid
= req
->from_rid
;
266 from_addr
= req
->from_addr
;
268 if (WARN_ON(used
+ req
->length
>
269 MCDI_CTL_SDU_LEN_MAX_V1
)) {
274 from_rid
= MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE
;
276 memcpy(_MCDI_PTR(inbuf
, used
), req
->from_buf
,
281 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_FROM_RID
, from_rid
);
282 MCDI_SET_QWORD(record
, MEMCPY_RECORD_TYPEDEF_FROM_ADDR
,
284 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_LENGTH
,
290 rc
= efx_mcdi_rpc(efx
, MC_CMD_MEMCPY
, inbuf
, used
, NULL
, 0, NULL
);
292 mb(); /* Don't write source/read dest before DMA is complete */
297 /* The TX filter is entirely controlled by this driver, and is modified
298 * underneath the feet of the VF
300 static void efx_sriov_reset_tx_filter(struct efx_vf
*vf
)
302 struct efx_nic
*efx
= vf
->efx
;
303 struct efx_filter_spec filter
;
307 if (vf
->tx_filter_id
!= -1) {
308 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
310 netif_dbg(efx
, hw
, efx
->net_dev
, "Removed vf %s tx filter %d\n",
311 vf
->pci_name
, vf
->tx_filter_id
);
312 vf
->tx_filter_id
= -1;
315 if (is_zero_ether_addr(vf
->addr
.mac_addr
))
318 /* Turn on TX filtering automatically if not explicitly
319 * enabled or disabled.
321 if (vf
->tx_filter_mode
== VF_TX_FILTER_AUTO
&& vf_max_tx_channels
<= 2)
322 vf
->tx_filter_mode
= VF_TX_FILTER_ON
;
324 vlan
= ntohs(vf
->addr
.tci
) & VLAN_VID_MASK
;
325 efx_filter_init_tx(&filter
, abs_index(vf
, 0));
326 rc
= efx_filter_set_eth_local(&filter
,
327 vlan
? vlan
: EFX_FILTER_VID_UNSPEC
,
331 rc
= efx_filter_insert_filter(efx
, &filter
, true);
333 netif_warn(efx
, hw
, efx
->net_dev
,
334 "Unable to migrate tx filter for vf %s\n",
337 netif_dbg(efx
, hw
, efx
->net_dev
, "Inserted vf %s tx filter %d\n",
339 vf
->tx_filter_id
= rc
;
343 /* The RX filter is managed here on behalf of the VF driver */
344 static void efx_sriov_reset_rx_filter(struct efx_vf
*vf
)
346 struct efx_nic
*efx
= vf
->efx
;
347 struct efx_filter_spec filter
;
351 if (vf
->rx_filter_id
!= -1) {
352 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
354 netif_dbg(efx
, hw
, efx
->net_dev
, "Removed vf %s rx filter %d\n",
355 vf
->pci_name
, vf
->rx_filter_id
);
356 vf
->rx_filter_id
= -1;
359 if (!vf
->rx_filtering
|| is_zero_ether_addr(vf
->addr
.mac_addr
))
362 vlan
= ntohs(vf
->addr
.tci
) & VLAN_VID_MASK
;
363 efx_filter_init_rx(&filter
, EFX_FILTER_PRI_REQUIRED
,
365 abs_index(vf
, vf
->rx_filter_qid
));
366 rc
= efx_filter_set_eth_local(&filter
,
367 vlan
? vlan
: EFX_FILTER_VID_UNSPEC
,
371 rc
= efx_filter_insert_filter(efx
, &filter
, true);
373 netif_warn(efx
, hw
, efx
->net_dev
,
374 "Unable to insert rx filter for vf %s\n",
377 netif_dbg(efx
, hw
, efx
->net_dev
, "Inserted vf %s rx filter %d\n",
379 vf
->rx_filter_id
= rc
;
383 static void __efx_sriov_update_vf_addr(struct efx_vf
*vf
)
385 efx_sriov_reset_tx_filter(vf
);
386 efx_sriov_reset_rx_filter(vf
);
387 queue_work(vfdi_workqueue
, &vf
->efx
->peer_work
);
390 /* Push the peer list to this VF. The caller must hold status_lock to interlock
391 * with VFDI requests, and they must be serialised against manipulation of
392 * local_page_list, either by acquiring local_lock or by running from
393 * efx_sriov_peer_work()
395 static void __efx_sriov_push_vf_status(struct efx_vf
*vf
)
397 struct efx_nic
*efx
= vf
->efx
;
398 struct vfdi_status
*status
= efx
->vfdi_status
.addr
;
399 struct efx_memcpy_req copy
[4];
400 struct efx_endpoint_page
*epp
;
401 unsigned int pos
, count
;
402 unsigned data_offset
;
405 WARN_ON(!mutex_is_locked(&vf
->status_lock
));
406 WARN_ON(!vf
->status_addr
);
408 status
->local
= vf
->addr
;
409 status
->generation_end
= ++status
->generation_start
;
411 memset(copy
, '\0', sizeof(copy
));
412 /* Write generation_start */
413 copy
[0].from_buf
= &status
->generation_start
;
414 copy
[0].to_rid
= vf
->pci_rid
;
415 copy
[0].to_addr
= vf
->status_addr
+ offsetof(struct vfdi_status
,
417 copy
[0].length
= sizeof(status
->generation_start
);
418 /* DMA the rest of the structure (excluding the generations). This
419 * assumes that the non-generation portion of vfdi_status is in
420 * one chunk starting at the version member.
422 data_offset
= offsetof(struct vfdi_status
, version
);
423 copy
[1].from_rid
= efx
->pci_dev
->devfn
;
424 copy
[1].from_addr
= efx
->vfdi_status
.dma_addr
+ data_offset
;
425 copy
[1].to_rid
= vf
->pci_rid
;
426 copy
[1].to_addr
= vf
->status_addr
+ data_offset
;
427 copy
[1].length
= status
->length
- data_offset
;
429 /* Copy the peer pages */
432 list_for_each_entry(epp
, &efx
->local_page_list
, link
) {
433 if (count
== vf
->peer_page_count
) {
434 /* The VF driver will know they need to provide more
435 * pages because peer_addr_count is too large.
439 copy
[pos
].from_buf
= NULL
;
440 copy
[pos
].from_rid
= efx
->pci_dev
->devfn
;
441 copy
[pos
].from_addr
= epp
->addr
;
442 copy
[pos
].to_rid
= vf
->pci_rid
;
443 copy
[pos
].to_addr
= vf
->peer_page_addrs
[count
];
444 copy
[pos
].length
= EFX_PAGE_SIZE
;
446 if (++pos
== ARRAY_SIZE(copy
)) {
447 efx_sriov_memcpy(efx
, copy
, ARRAY_SIZE(copy
));
453 /* Write generation_end */
454 copy
[pos
].from_buf
= &status
->generation_end
;
455 copy
[pos
].to_rid
= vf
->pci_rid
;
456 copy
[pos
].to_addr
= vf
->status_addr
+ offsetof(struct vfdi_status
,
458 copy
[pos
].length
= sizeof(status
->generation_end
);
459 efx_sriov_memcpy(efx
, copy
, pos
+ 1);
461 /* Notify the guest */
462 EFX_POPULATE_QWORD_3(event
,
463 FSF_AZ_EV_CODE
, FSE_CZ_EV_CODE_USER_EV
,
464 VFDI_EV_SEQ
, (vf
->msg_seqno
& 0xff),
465 VFDI_EV_TYPE
, VFDI_EV_TYPE_STATUS
);
467 efx_generate_event(efx
, EFX_VI_BASE
+ vf
->index
* efx_vf_size(efx
),
471 static void efx_sriov_bufs(struct efx_nic
*efx
, unsigned offset
,
472 u64
*addr
, unsigned count
)
477 for (pos
= 0; pos
< count
; ++pos
) {
478 EFX_POPULATE_QWORD_3(buf
,
479 FRF_AZ_BUF_ADR_REGION
, 0,
481 addr
? addr
[pos
] >> 12 : 0,
482 FRF_AZ_BUF_OWNER_ID_FBUF
, 0);
483 efx_sram_writeq(efx
, efx
->membase
+ FR_BZ_BUF_FULL_TBL
,
488 static bool bad_vf_index(struct efx_nic
*efx
, unsigned index
)
490 return index
>= efx_vf_size(efx
);
493 static bool bad_buf_count(unsigned buf_count
, unsigned max_entry_count
)
495 unsigned max_buf_count
= max_entry_count
*
496 sizeof(efx_qword_t
) / EFX_BUF_SIZE
;
498 return ((buf_count
& (buf_count
- 1)) || buf_count
> max_buf_count
);
501 /* Check that VI specified by per-port index belongs to a VF.
502 * Optionally set VF index and VI index within the VF.
504 static bool map_vi_index(struct efx_nic
*efx
, unsigned abs_index
,
505 struct efx_vf
**vf_out
, unsigned *rel_index_out
)
509 if (abs_index
< EFX_VI_BASE
)
511 vf_i
= (abs_index
- EFX_VI_BASE
) / efx_vf_size(efx
);
512 if (vf_i
>= efx
->vf_init_count
)
516 *vf_out
= efx
->vf
+ vf_i
;
518 *rel_index_out
= abs_index
% efx_vf_size(efx
);
522 static int efx_vfdi_init_evq(struct efx_vf
*vf
)
524 struct efx_nic
*efx
= vf
->efx
;
525 struct vfdi_req
*req
= vf
->buf
.addr
;
526 unsigned vf_evq
= req
->u
.init_evq
.index
;
527 unsigned buf_count
= req
->u
.init_evq
.buf_count
;
528 unsigned abs_evq
= abs_index(vf
, vf_evq
);
529 unsigned buftbl
= EFX_BUFTBL_EVQ_BASE(vf
, vf_evq
);
532 if (bad_vf_index(efx
, vf_evq
) ||
533 bad_buf_count(buf_count
, EFX_MAX_VF_EVQ_SIZE
)) {
535 netif_err(efx
, hw
, efx
->net_dev
,
536 "ERROR: Invalid INIT_EVQ from %s: evq %d bufs %d\n",
537 vf
->pci_name
, vf_evq
, buf_count
);
538 return VFDI_RC_EINVAL
;
541 efx_sriov_bufs(efx
, buftbl
, req
->u
.init_evq
.addr
, buf_count
);
543 EFX_POPULATE_OWORD_3(reg
,
544 FRF_CZ_TIMER_Q_EN
, 1,
545 FRF_CZ_HOST_NOTIFY_MODE
, 0,
546 FRF_CZ_TIMER_MODE
, FFE_CZ_TIMER_MODE_DIS
);
547 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
, abs_evq
);
548 EFX_POPULATE_OWORD_3(reg
,
550 FRF_AZ_EVQ_SIZE
, __ffs(buf_count
),
551 FRF_AZ_EVQ_BUF_BASE_ID
, buftbl
);
552 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
, abs_evq
);
555 memcpy(vf
->evq0_addrs
, req
->u
.init_evq
.addr
,
556 buf_count
* sizeof(u64
));
557 vf
->evq0_count
= buf_count
;
560 return VFDI_RC_SUCCESS
;
563 static int efx_vfdi_init_rxq(struct efx_vf
*vf
)
565 struct efx_nic
*efx
= vf
->efx
;
566 struct vfdi_req
*req
= vf
->buf
.addr
;
567 unsigned vf_rxq
= req
->u
.init_rxq
.index
;
568 unsigned vf_evq
= req
->u
.init_rxq
.evq
;
569 unsigned buf_count
= req
->u
.init_rxq
.buf_count
;
570 unsigned buftbl
= EFX_BUFTBL_RXQ_BASE(vf
, vf_rxq
);
574 if (bad_vf_index(efx
, vf_evq
) || bad_vf_index(efx
, vf_rxq
) ||
575 vf_rxq
>= VF_MAX_RX_QUEUES
||
576 bad_buf_count(buf_count
, EFX_MAX_DMAQ_SIZE
)) {
578 netif_err(efx
, hw
, efx
->net_dev
,
579 "ERROR: Invalid INIT_RXQ from %s: rxq %d evq %d "
580 "buf_count %d\n", vf
->pci_name
, vf_rxq
,
582 return VFDI_RC_EINVAL
;
584 if (__test_and_set_bit(req
->u
.init_rxq
.index
, vf
->rxq_mask
))
586 efx_sriov_bufs(efx
, buftbl
, req
->u
.init_rxq
.addr
, buf_count
);
588 label
= req
->u
.init_rxq
.label
& EFX_FIELD_MASK(FRF_AZ_RX_DESCQ_LABEL
);
589 EFX_POPULATE_OWORD_6(reg
,
590 FRF_AZ_RX_DESCQ_BUF_BASE_ID
, buftbl
,
591 FRF_AZ_RX_DESCQ_EVQ_ID
, abs_index(vf
, vf_evq
),
592 FRF_AZ_RX_DESCQ_LABEL
, label
,
593 FRF_AZ_RX_DESCQ_SIZE
, __ffs(buf_count
),
594 FRF_AZ_RX_DESCQ_JUMBO
,
595 !!(req
->u
.init_rxq
.flags
&
596 VFDI_RXQ_FLAG_SCATTER_EN
),
597 FRF_AZ_RX_DESCQ_EN
, 1);
598 efx_writeo_table(efx
, ®
, FR_BZ_RX_DESC_PTR_TBL
,
599 abs_index(vf
, vf_rxq
));
601 return VFDI_RC_SUCCESS
;
604 static int efx_vfdi_init_txq(struct efx_vf
*vf
)
606 struct efx_nic
*efx
= vf
->efx
;
607 struct vfdi_req
*req
= vf
->buf
.addr
;
608 unsigned vf_txq
= req
->u
.init_txq
.index
;
609 unsigned vf_evq
= req
->u
.init_txq
.evq
;
610 unsigned buf_count
= req
->u
.init_txq
.buf_count
;
611 unsigned buftbl
= EFX_BUFTBL_TXQ_BASE(vf
, vf_txq
);
612 unsigned label
, eth_filt_en
;
615 if (bad_vf_index(efx
, vf_evq
) || bad_vf_index(efx
, vf_txq
) ||
616 vf_txq
>= vf_max_tx_channels
||
617 bad_buf_count(buf_count
, EFX_MAX_DMAQ_SIZE
)) {
619 netif_err(efx
, hw
, efx
->net_dev
,
620 "ERROR: Invalid INIT_TXQ from %s: txq %d evq %d "
621 "buf_count %d\n", vf
->pci_name
, vf_txq
,
623 return VFDI_RC_EINVAL
;
626 mutex_lock(&vf
->txq_lock
);
627 if (__test_and_set_bit(req
->u
.init_txq
.index
, vf
->txq_mask
))
629 mutex_unlock(&vf
->txq_lock
);
630 efx_sriov_bufs(efx
, buftbl
, req
->u
.init_txq
.addr
, buf_count
);
632 eth_filt_en
= vf
->tx_filter_mode
== VF_TX_FILTER_ON
;
634 label
= req
->u
.init_txq
.label
& EFX_FIELD_MASK(FRF_AZ_TX_DESCQ_LABEL
);
635 EFX_POPULATE_OWORD_8(reg
,
636 FRF_CZ_TX_DPT_Q_MASK_WIDTH
, min(efx
->vi_scale
, 1U),
637 FRF_CZ_TX_DPT_ETH_FILT_EN
, eth_filt_en
,
638 FRF_AZ_TX_DESCQ_EN
, 1,
639 FRF_AZ_TX_DESCQ_BUF_BASE_ID
, buftbl
,
640 FRF_AZ_TX_DESCQ_EVQ_ID
, abs_index(vf
, vf_evq
),
641 FRF_AZ_TX_DESCQ_LABEL
, label
,
642 FRF_AZ_TX_DESCQ_SIZE
, __ffs(buf_count
),
643 FRF_BZ_TX_NON_IP_DROP_DIS
, 1);
644 efx_writeo_table(efx
, ®
, FR_BZ_TX_DESC_PTR_TBL
,
645 abs_index(vf
, vf_txq
));
647 return VFDI_RC_SUCCESS
;
650 /* Returns true when efx_vfdi_fini_all_queues should wake */
651 static bool efx_vfdi_flush_wake(struct efx_vf
*vf
)
653 /* Ensure that all updates are visible to efx_vfdi_fini_all_queues() */
656 return (!vf
->txq_count
&& !vf
->rxq_count
) ||
657 atomic_read(&vf
->rxq_retry_count
);
660 static void efx_vfdi_flush_clear(struct efx_vf
*vf
)
662 memset(vf
->txq_mask
, 0, sizeof(vf
->txq_mask
));
664 memset(vf
->rxq_mask
, 0, sizeof(vf
->rxq_mask
));
666 memset(vf
->rxq_retry_mask
, 0, sizeof(vf
->rxq_retry_mask
));
667 atomic_set(&vf
->rxq_retry_count
, 0);
670 static int efx_vfdi_fini_all_queues(struct efx_vf
*vf
)
672 struct efx_nic
*efx
= vf
->efx
;
674 unsigned count
= efx_vf_size(efx
);
675 unsigned vf_offset
= EFX_VI_BASE
+ vf
->index
* efx_vf_size(efx
);
676 unsigned timeout
= HZ
;
677 unsigned index
, rxqs_count
;
678 MCDI_DECLARE_BUF(inbuf
, MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX
);
681 BUILD_BUG_ON(VF_MAX_RX_QUEUES
>
682 MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM
);
685 siena_prepare_flush(efx
);
688 /* Flush all the initialized queues */
690 for (index
= 0; index
< count
; ++index
) {
691 if (test_bit(index
, vf
->txq_mask
)) {
692 EFX_POPULATE_OWORD_2(reg
,
693 FRF_AZ_TX_FLUSH_DESCQ_CMD
, 1,
694 FRF_AZ_TX_FLUSH_DESCQ
,
696 efx_writeo(efx
, ®
, FR_AZ_TX_FLUSH_DESCQ
);
698 if (test_bit(index
, vf
->rxq_mask
)) {
699 MCDI_SET_ARRAY_DWORD(
700 inbuf
, FLUSH_RX_QUEUES_IN_QID_OFST
,
701 rxqs_count
, vf_offset
+ index
);
706 atomic_set(&vf
->rxq_retry_count
, 0);
707 while (timeout
&& (vf
->rxq_count
|| vf
->txq_count
)) {
708 rc
= efx_mcdi_rpc(efx
, MC_CMD_FLUSH_RX_QUEUES
, inbuf
,
709 MC_CMD_FLUSH_RX_QUEUES_IN_LEN(rxqs_count
),
713 timeout
= wait_event_timeout(vf
->flush_waitq
,
714 efx_vfdi_flush_wake(vf
),
717 for (index
= 0; index
< count
; ++index
) {
718 if (test_and_clear_bit(index
, vf
->rxq_retry_mask
)) {
719 atomic_dec(&vf
->rxq_retry_count
);
720 MCDI_SET_ARRAY_DWORD(
721 inbuf
, FLUSH_RX_QUEUES_IN_QID_OFST
,
722 rxqs_count
, vf_offset
+ index
);
729 siena_finish_flush(efx
);
732 /* Irrespective of success/failure, fini the queues */
734 for (index
= 0; index
< count
; ++index
) {
735 efx_writeo_table(efx
, ®
, FR_BZ_RX_DESC_PTR_TBL
,
737 efx_writeo_table(efx
, ®
, FR_BZ_TX_DESC_PTR_TBL
,
739 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
,
741 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
,
744 efx_sriov_bufs(efx
, vf
->buftbl_base
, NULL
,
745 EFX_VF_BUFTBL_PER_VI
* efx_vf_size(efx
));
746 efx_vfdi_flush_clear(vf
);
750 return timeout
? 0 : VFDI_RC_ETIMEDOUT
;
753 static int efx_vfdi_insert_filter(struct efx_vf
*vf
)
755 struct efx_nic
*efx
= vf
->efx
;
756 struct vfdi_req
*req
= vf
->buf
.addr
;
757 unsigned vf_rxq
= req
->u
.mac_filter
.rxq
;
760 if (bad_vf_index(efx
, vf_rxq
) || vf
->rx_filtering
) {
762 netif_err(efx
, hw
, efx
->net_dev
,
763 "ERROR: Invalid INSERT_FILTER from %s: rxq %d "
764 "flags 0x%x\n", vf
->pci_name
, vf_rxq
,
765 req
->u
.mac_filter
.flags
);
766 return VFDI_RC_EINVAL
;
770 if (req
->u
.mac_filter
.flags
& VFDI_MAC_FILTER_FLAG_RSS
)
771 flags
|= EFX_FILTER_FLAG_RX_RSS
;
772 if (req
->u
.mac_filter
.flags
& VFDI_MAC_FILTER_FLAG_SCATTER
)
773 flags
|= EFX_FILTER_FLAG_RX_SCATTER
;
774 vf
->rx_filter_flags
= flags
;
775 vf
->rx_filter_qid
= vf_rxq
;
776 vf
->rx_filtering
= true;
778 efx_sriov_reset_rx_filter(vf
);
779 queue_work(vfdi_workqueue
, &efx
->peer_work
);
781 return VFDI_RC_SUCCESS
;
784 static int efx_vfdi_remove_all_filters(struct efx_vf
*vf
)
786 vf
->rx_filtering
= false;
787 efx_sriov_reset_rx_filter(vf
);
788 queue_work(vfdi_workqueue
, &vf
->efx
->peer_work
);
790 return VFDI_RC_SUCCESS
;
793 static int efx_vfdi_set_status_page(struct efx_vf
*vf
)
795 struct efx_nic
*efx
= vf
->efx
;
796 struct vfdi_req
*req
= vf
->buf
.addr
;
797 u64 page_count
= req
->u
.set_status_page
.peer_page_count
;
800 offsetof(struct vfdi_req
, u
.set_status_page
.peer_page_addr
[0]))
801 / sizeof(req
->u
.set_status_page
.peer_page_addr
[0]);
803 if (!req
->u
.set_status_page
.dma_addr
|| page_count
> max_page_count
) {
805 netif_err(efx
, hw
, efx
->net_dev
,
806 "ERROR: Invalid SET_STATUS_PAGE from %s\n",
808 return VFDI_RC_EINVAL
;
811 mutex_lock(&efx
->local_lock
);
812 mutex_lock(&vf
->status_lock
);
813 vf
->status_addr
= req
->u
.set_status_page
.dma_addr
;
815 kfree(vf
->peer_page_addrs
);
816 vf
->peer_page_addrs
= NULL
;
817 vf
->peer_page_count
= 0;
820 vf
->peer_page_addrs
= kcalloc(page_count
, sizeof(u64
),
822 if (vf
->peer_page_addrs
) {
823 memcpy(vf
->peer_page_addrs
,
824 req
->u
.set_status_page
.peer_page_addr
,
825 page_count
* sizeof(u64
));
826 vf
->peer_page_count
= page_count
;
830 __efx_sriov_push_vf_status(vf
);
831 mutex_unlock(&vf
->status_lock
);
832 mutex_unlock(&efx
->local_lock
);
834 return VFDI_RC_SUCCESS
;
837 static int efx_vfdi_clear_status_page(struct efx_vf
*vf
)
839 mutex_lock(&vf
->status_lock
);
841 mutex_unlock(&vf
->status_lock
);
843 return VFDI_RC_SUCCESS
;
846 typedef int (*efx_vfdi_op_t
)(struct efx_vf
*vf
);
848 static const efx_vfdi_op_t vfdi_ops
[VFDI_OP_LIMIT
] = {
849 [VFDI_OP_INIT_EVQ
] = efx_vfdi_init_evq
,
850 [VFDI_OP_INIT_TXQ
] = efx_vfdi_init_txq
,
851 [VFDI_OP_INIT_RXQ
] = efx_vfdi_init_rxq
,
852 [VFDI_OP_FINI_ALL_QUEUES
] = efx_vfdi_fini_all_queues
,
853 [VFDI_OP_INSERT_FILTER
] = efx_vfdi_insert_filter
,
854 [VFDI_OP_REMOVE_ALL_FILTERS
] = efx_vfdi_remove_all_filters
,
855 [VFDI_OP_SET_STATUS_PAGE
] = efx_vfdi_set_status_page
,
856 [VFDI_OP_CLEAR_STATUS_PAGE
] = efx_vfdi_clear_status_page
,
859 static void efx_sriov_vfdi(struct work_struct
*work
)
861 struct efx_vf
*vf
= container_of(work
, struct efx_vf
, req
);
862 struct efx_nic
*efx
= vf
->efx
;
863 struct vfdi_req
*req
= vf
->buf
.addr
;
864 struct efx_memcpy_req copy
[2];
867 /* Copy this page into the local address space */
868 memset(copy
, '\0', sizeof(copy
));
869 copy
[0].from_rid
= vf
->pci_rid
;
870 copy
[0].from_addr
= vf
->req_addr
;
871 copy
[0].to_rid
= efx
->pci_dev
->devfn
;
872 copy
[0].to_addr
= vf
->buf
.dma_addr
;
873 copy
[0].length
= EFX_PAGE_SIZE
;
874 rc
= efx_sriov_memcpy(efx
, copy
, 1);
876 /* If we can't get the request, we can't reply to the caller */
878 netif_err(efx
, hw
, efx
->net_dev
,
879 "ERROR: Unable to fetch VFDI request from %s rc %d\n",
885 if (req
->op
< VFDI_OP_LIMIT
&& vfdi_ops
[req
->op
] != NULL
) {
886 rc
= vfdi_ops
[req
->op
](vf
);
888 netif_dbg(efx
, hw
, efx
->net_dev
,
889 "vfdi request %d from %s ok\n",
890 req
->op
, vf
->pci_name
);
893 netif_dbg(efx
, hw
, efx
->net_dev
,
894 "ERROR: Unrecognised request %d from VF %s addr "
895 "%llx\n", req
->op
, vf
->pci_name
,
896 (unsigned long long)vf
->req_addr
);
897 rc
= VFDI_RC_EOPNOTSUPP
;
900 /* Allow subsequent VF requests */
904 /* Respond to the request */
906 req
->op
= VFDI_OP_RESPONSE
;
908 memset(copy
, '\0', sizeof(copy
));
909 copy
[0].from_buf
= &req
->rc
;
910 copy
[0].to_rid
= vf
->pci_rid
;
911 copy
[0].to_addr
= vf
->req_addr
+ offsetof(struct vfdi_req
, rc
);
912 copy
[0].length
= sizeof(req
->rc
);
913 copy
[1].from_buf
= &req
->op
;
914 copy
[1].to_rid
= vf
->pci_rid
;
915 copy
[1].to_addr
= vf
->req_addr
+ offsetof(struct vfdi_req
, op
);
916 copy
[1].length
= sizeof(req
->op
);
918 (void) efx_sriov_memcpy(efx
, copy
, ARRAY_SIZE(copy
));
923 /* After a reset the event queues inside the guests no longer exist. Fill the
924 * event ring in guest memory with VFDI reset events, then (re-initialise) the
925 * event queue to raise an interrupt. The guest driver will then recover.
927 static void efx_sriov_reset_vf(struct efx_vf
*vf
, struct efx_buffer
*buffer
)
929 struct efx_nic
*efx
= vf
->efx
;
930 struct efx_memcpy_req copy_req
[4];
932 unsigned int pos
, count
, k
, buftbl
, abs_evq
;
937 BUG_ON(buffer
->len
!= EFX_PAGE_SIZE
);
941 BUG_ON(vf
->evq0_count
& (vf
->evq0_count
- 1));
943 mutex_lock(&vf
->status_lock
);
944 EFX_POPULATE_QWORD_3(event
,
945 FSF_AZ_EV_CODE
, FSE_CZ_EV_CODE_USER_EV
,
946 VFDI_EV_SEQ
, vf
->msg_seqno
,
947 VFDI_EV_TYPE
, VFDI_EV_TYPE_RESET
);
949 for (pos
= 0; pos
< EFX_PAGE_SIZE
; pos
+= sizeof(event
))
950 memcpy(buffer
->addr
+ pos
, &event
, sizeof(event
));
952 for (pos
= 0; pos
< vf
->evq0_count
; pos
+= count
) {
953 count
= min_t(unsigned, vf
->evq0_count
- pos
,
954 ARRAY_SIZE(copy_req
));
955 for (k
= 0; k
< count
; k
++) {
956 copy_req
[k
].from_buf
= NULL
;
957 copy_req
[k
].from_rid
= efx
->pci_dev
->devfn
;
958 copy_req
[k
].from_addr
= buffer
->dma_addr
;
959 copy_req
[k
].to_rid
= vf
->pci_rid
;
960 copy_req
[k
].to_addr
= vf
->evq0_addrs
[pos
+ k
];
961 copy_req
[k
].length
= EFX_PAGE_SIZE
;
963 rc
= efx_sriov_memcpy(efx
, copy_req
, count
);
966 netif_err(efx
, hw
, efx
->net_dev
,
967 "ERROR: Unable to notify %s of reset"
968 ": %d\n", vf
->pci_name
, -rc
);
973 /* Reinitialise, arm and trigger evq0 */
974 abs_evq
= abs_index(vf
, 0);
975 buftbl
= EFX_BUFTBL_EVQ_BASE(vf
, 0);
976 efx_sriov_bufs(efx
, buftbl
, vf
->evq0_addrs
, vf
->evq0_count
);
978 EFX_POPULATE_OWORD_3(reg
,
979 FRF_CZ_TIMER_Q_EN
, 1,
980 FRF_CZ_HOST_NOTIFY_MODE
, 0,
981 FRF_CZ_TIMER_MODE
, FFE_CZ_TIMER_MODE_DIS
);
982 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
, abs_evq
);
983 EFX_POPULATE_OWORD_3(reg
,
985 FRF_AZ_EVQ_SIZE
, __ffs(vf
->evq0_count
),
986 FRF_AZ_EVQ_BUF_BASE_ID
, buftbl
);
987 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
, abs_evq
);
988 EFX_POPULATE_DWORD_1(ptr
, FRF_AZ_EVQ_RPTR
, 0);
989 efx_writed(efx
, &ptr
, FR_BZ_EVQ_RPTR
+ FR_BZ_EVQ_RPTR_STEP
* abs_evq
);
991 mutex_unlock(&vf
->status_lock
);
994 static void efx_sriov_reset_vf_work(struct work_struct
*work
)
996 struct efx_vf
*vf
= container_of(work
, struct efx_vf
, req
);
997 struct efx_nic
*efx
= vf
->efx
;
998 struct efx_buffer buf
;
1000 if (!efx_nic_alloc_buffer(efx
, &buf
, EFX_PAGE_SIZE
, GFP_NOIO
)) {
1001 efx_sriov_reset_vf(vf
, &buf
);
1002 efx_nic_free_buffer(efx
, &buf
);
1006 static void efx_sriov_handle_no_channel(struct efx_nic
*efx
)
1008 netif_err(efx
, drv
, efx
->net_dev
,
1009 "ERROR: IOV requires MSI-X and 1 additional interrupt"
1010 "vector. IOV disabled\n");
1014 static int efx_sriov_probe_channel(struct efx_channel
*channel
)
1016 channel
->efx
->vfdi_channel
= channel
;
1021 efx_sriov_get_channel_name(struct efx_channel
*channel
, char *buf
, size_t len
)
1023 snprintf(buf
, len
, "%s-iov", channel
->efx
->name
);
1026 static const struct efx_channel_type efx_sriov_channel_type
= {
1027 .handle_no_channel
= efx_sriov_handle_no_channel
,
1028 .pre_probe
= efx_sriov_probe_channel
,
1029 .post_remove
= efx_channel_dummy_op_void
,
1030 .get_name
= efx_sriov_get_channel_name
,
1031 /* no copy operation; channel must not be reallocated */
1032 .keep_eventq
= true,
1035 void efx_sriov_probe(struct efx_nic
*efx
)
1042 if (efx_sriov_cmd(efx
, false, &efx
->vi_scale
, &count
))
1044 if (count
> 0 && count
> max_vfs
)
1047 /* efx_nic_dimension_resources() will reduce vf_count as appopriate */
1048 efx
->vf_count
= count
;
1050 efx
->extra_channel_type
[EFX_EXTRA_CHANNEL_IOV
] = &efx_sriov_channel_type
;
1053 /* Copy the list of individual addresses into the vfdi_status.peers
1054 * array and auxillary pages, protected by %local_lock. Drop that lock
1055 * and then broadcast the address list to every VF.
1057 static void efx_sriov_peer_work(struct work_struct
*data
)
1059 struct efx_nic
*efx
= container_of(data
, struct efx_nic
, peer_work
);
1060 struct vfdi_status
*vfdi_status
= efx
->vfdi_status
.addr
;
1062 struct efx_local_addr
*local_addr
;
1063 struct vfdi_endpoint
*peer
;
1064 struct efx_endpoint_page
*epp
;
1065 struct list_head pages
;
1066 unsigned int peer_space
;
1067 unsigned int peer_count
;
1070 mutex_lock(&efx
->local_lock
);
1072 /* Move the existing peer pages off %local_page_list */
1073 INIT_LIST_HEAD(&pages
);
1074 list_splice_tail_init(&efx
->local_page_list
, &pages
);
1076 /* Populate the VF addresses starting from entry 1 (entry 0 is
1079 peer
= vfdi_status
->peers
+ 1;
1080 peer_space
= ARRAY_SIZE(vfdi_status
->peers
) - 1;
1082 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1085 mutex_lock(&vf
->status_lock
);
1086 if (vf
->rx_filtering
&& !is_zero_ether_addr(vf
->addr
.mac_addr
)) {
1090 BUG_ON(peer_space
== 0);
1092 mutex_unlock(&vf
->status_lock
);
1095 /* Fill the remaining addresses */
1096 list_for_each_entry(local_addr
, &efx
->local_addr_list
, link
) {
1097 memcpy(peer
->mac_addr
, local_addr
->addr
, ETH_ALEN
);
1101 if (--peer_space
== 0) {
1102 if (list_empty(&pages
)) {
1103 epp
= kmalloc(sizeof(*epp
), GFP_KERNEL
);
1106 epp
->ptr
= dma_alloc_coherent(
1107 &efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1108 &epp
->addr
, GFP_KERNEL
);
1114 epp
= list_first_entry(
1115 &pages
, struct efx_endpoint_page
, link
);
1116 list_del(&epp
->link
);
1119 list_add_tail(&epp
->link
, &efx
->local_page_list
);
1120 peer
= (struct vfdi_endpoint
*)epp
->ptr
;
1121 peer_space
= EFX_PAGE_SIZE
/ sizeof(struct vfdi_endpoint
);
1124 vfdi_status
->peer_count
= peer_count
;
1125 mutex_unlock(&efx
->local_lock
);
1127 /* Free any now unused endpoint pages */
1128 while (!list_empty(&pages
)) {
1129 epp
= list_first_entry(
1130 &pages
, struct efx_endpoint_page
, link
);
1131 list_del(&epp
->link
);
1132 dma_free_coherent(&efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1133 epp
->ptr
, epp
->addr
);
1137 /* Finally, push the pages */
1138 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1141 mutex_lock(&vf
->status_lock
);
1142 if (vf
->status_addr
)
1143 __efx_sriov_push_vf_status(vf
);
1144 mutex_unlock(&vf
->status_lock
);
1148 static void efx_sriov_free_local(struct efx_nic
*efx
)
1150 struct efx_local_addr
*local_addr
;
1151 struct efx_endpoint_page
*epp
;
1153 while (!list_empty(&efx
->local_addr_list
)) {
1154 local_addr
= list_first_entry(&efx
->local_addr_list
,
1155 struct efx_local_addr
, link
);
1156 list_del(&local_addr
->link
);
1160 while (!list_empty(&efx
->local_page_list
)) {
1161 epp
= list_first_entry(&efx
->local_page_list
,
1162 struct efx_endpoint_page
, link
);
1163 list_del(&epp
->link
);
1164 dma_free_coherent(&efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1165 epp
->ptr
, epp
->addr
);
1170 static int efx_sriov_vf_alloc(struct efx_nic
*efx
)
1175 efx
->vf
= kzalloc(sizeof(struct efx_vf
) * efx
->vf_count
, GFP_KERNEL
);
1179 for (index
= 0; index
< efx
->vf_count
; ++index
) {
1180 vf
= efx
->vf
+ index
;
1184 vf
->rx_filter_id
= -1;
1185 vf
->tx_filter_mode
= VF_TX_FILTER_AUTO
;
1186 vf
->tx_filter_id
= -1;
1187 INIT_WORK(&vf
->req
, efx_sriov_vfdi
);
1188 INIT_WORK(&vf
->reset_work
, efx_sriov_reset_vf_work
);
1189 init_waitqueue_head(&vf
->flush_waitq
);
1190 mutex_init(&vf
->status_lock
);
1191 mutex_init(&vf
->txq_lock
);
1197 static void efx_sriov_vfs_fini(struct efx_nic
*efx
)
1202 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1205 efx_nic_free_buffer(efx
, &vf
->buf
);
1206 kfree(vf
->peer_page_addrs
);
1207 vf
->peer_page_addrs
= NULL
;
1208 vf
->peer_page_count
= 0;
1214 static int efx_sriov_vfs_init(struct efx_nic
*efx
)
1216 struct pci_dev
*pci_dev
= efx
->pci_dev
;
1217 unsigned index
, devfn
, sriov
, buftbl_base
;
1222 sriov
= pci_find_ext_capability(pci_dev
, PCI_EXT_CAP_ID_SRIOV
);
1226 pci_read_config_word(pci_dev
, sriov
+ PCI_SRIOV_VF_OFFSET
, &offset
);
1227 pci_read_config_word(pci_dev
, sriov
+ PCI_SRIOV_VF_STRIDE
, &stride
);
1229 buftbl_base
= efx
->vf_buftbl_base
;
1230 devfn
= pci_dev
->devfn
+ offset
;
1231 for (index
= 0; index
< efx
->vf_count
; ++index
) {
1232 vf
= efx
->vf
+ index
;
1234 /* Reserve buffer entries */
1235 vf
->buftbl_base
= buftbl_base
;
1236 buftbl_base
+= EFX_VF_BUFTBL_PER_VI
* efx_vf_size(efx
);
1238 vf
->pci_rid
= devfn
;
1239 snprintf(vf
->pci_name
, sizeof(vf
->pci_name
),
1240 "%04x:%02x:%02x.%d",
1241 pci_domain_nr(pci_dev
->bus
), pci_dev
->bus
->number
,
1242 PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1244 rc
= efx_nic_alloc_buffer(efx
, &vf
->buf
, EFX_PAGE_SIZE
,
1255 efx_sriov_vfs_fini(efx
);
1259 int efx_sriov_init(struct efx_nic
*efx
)
1261 struct net_device
*net_dev
= efx
->net_dev
;
1262 struct vfdi_status
*vfdi_status
;
1265 /* Ensure there's room for vf_channel */
1266 BUILD_BUG_ON(EFX_MAX_CHANNELS
+ 1 >= EFX_VI_BASE
);
1267 /* Ensure that VI_BASE is aligned on VI_SCALE */
1268 BUILD_BUG_ON(EFX_VI_BASE
& ((1 << EFX_VI_SCALE_MAX
) - 1));
1270 if (efx
->vf_count
== 0)
1273 rc
= efx_sriov_cmd(efx
, true, NULL
, NULL
);
1277 rc
= efx_nic_alloc_buffer(efx
, &efx
->vfdi_status
, sizeof(*vfdi_status
),
1281 vfdi_status
= efx
->vfdi_status
.addr
;
1282 memset(vfdi_status
, 0, sizeof(*vfdi_status
));
1283 vfdi_status
->version
= 1;
1284 vfdi_status
->length
= sizeof(*vfdi_status
);
1285 vfdi_status
->max_tx_channels
= vf_max_tx_channels
;
1286 vfdi_status
->vi_scale
= efx
->vi_scale
;
1287 vfdi_status
->rss_rxq_count
= efx
->rss_spread
;
1288 vfdi_status
->peer_count
= 1 + efx
->vf_count
;
1289 vfdi_status
->timer_quantum_ns
= efx
->timer_quantum_ns
;
1291 rc
= efx_sriov_vf_alloc(efx
);
1295 mutex_init(&efx
->local_lock
);
1296 INIT_WORK(&efx
->peer_work
, efx_sriov_peer_work
);
1297 INIT_LIST_HEAD(&efx
->local_addr_list
);
1298 INIT_LIST_HEAD(&efx
->local_page_list
);
1300 rc
= efx_sriov_vfs_init(efx
);
1305 memcpy(vfdi_status
->peers
[0].mac_addr
,
1306 net_dev
->dev_addr
, ETH_ALEN
);
1307 efx
->vf_init_count
= efx
->vf_count
;
1310 efx_sriov_usrev(efx
, true);
1312 /* At this point we must be ready to accept VFDI requests */
1314 rc
= pci_enable_sriov(efx
->pci_dev
, efx
->vf_count
);
1318 netif_info(efx
, probe
, net_dev
,
1319 "enabled SR-IOV for %d VFs, %d VI per VF\n",
1320 efx
->vf_count
, efx_vf_size(efx
));
1324 efx_sriov_usrev(efx
, false);
1326 efx
->vf_init_count
= 0;
1328 efx_sriov_vfs_fini(efx
);
1330 cancel_work_sync(&efx
->peer_work
);
1331 efx_sriov_free_local(efx
);
1334 efx_nic_free_buffer(efx
, &efx
->vfdi_status
);
1336 efx_sriov_cmd(efx
, false, NULL
, NULL
);
1341 void efx_sriov_fini(struct efx_nic
*efx
)
1346 if (efx
->vf_init_count
== 0)
1349 /* Disable all interfaces to reconfiguration */
1350 BUG_ON(efx
->vfdi_channel
->enabled
);
1351 efx_sriov_usrev(efx
, false);
1353 efx
->vf_init_count
= 0;
1356 /* Flush all reconfiguration work */
1357 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1359 cancel_work_sync(&vf
->req
);
1360 cancel_work_sync(&vf
->reset_work
);
1362 cancel_work_sync(&efx
->peer_work
);
1364 pci_disable_sriov(efx
->pci_dev
);
1366 /* Tear down back-end state */
1367 efx_sriov_vfs_fini(efx
);
1368 efx_sriov_free_local(efx
);
1370 efx_nic_free_buffer(efx
, &efx
->vfdi_status
);
1371 efx_sriov_cmd(efx
, false, NULL
, NULL
);
1374 void efx_sriov_event(struct efx_channel
*channel
, efx_qword_t
*event
)
1376 struct efx_nic
*efx
= channel
->efx
;
1378 unsigned qid
, seq
, type
, data
;
1380 qid
= EFX_QWORD_FIELD(*event
, FSF_CZ_USER_QID
);
1382 /* USR_EV_REG_VALUE is dword0, so access the VFDI_EV fields directly */
1383 BUILD_BUG_ON(FSF_CZ_USER_EV_REG_VALUE_LBN
!= 0);
1384 seq
= EFX_QWORD_FIELD(*event
, VFDI_EV_SEQ
);
1385 type
= EFX_QWORD_FIELD(*event
, VFDI_EV_TYPE
);
1386 data
= EFX_QWORD_FIELD(*event
, VFDI_EV_DATA
);
1388 netif_vdbg(efx
, hw
, efx
->net_dev
,
1389 "USR_EV event from qid %d seq 0x%x type %d data 0x%x\n",
1390 qid
, seq
, type
, data
);
1392 if (map_vi_index(efx
, qid
, &vf
, NULL
))
1397 if (type
== VFDI_EV_TYPE_REQ_WORD0
) {
1399 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1400 vf
->req_seqno
= seq
+ 1;
1402 } else if (seq
!= (vf
->req_seqno
++ & 0xff) || type
!= vf
->req_type
)
1405 switch (vf
->req_type
) {
1406 case VFDI_EV_TYPE_REQ_WORD0
:
1407 case VFDI_EV_TYPE_REQ_WORD1
:
1408 case VFDI_EV_TYPE_REQ_WORD2
:
1409 vf
->req_addr
|= (u64
)data
<< (vf
->req_type
<< 4);
1413 case VFDI_EV_TYPE_REQ_WORD3
:
1414 vf
->req_addr
|= (u64
)data
<< 48;
1415 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1417 queue_work(vfdi_workqueue
, &vf
->req
);
1422 if (net_ratelimit())
1423 netif_err(efx
, hw
, efx
->net_dev
,
1424 "ERROR: Screaming VFDI request from %s\n",
1426 /* Reset the request and sequence number */
1427 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1428 vf
->req_seqno
= seq
+ 1;
1431 void efx_sriov_flr(struct efx_nic
*efx
, unsigned vf_i
)
1435 if (vf_i
> efx
->vf_init_count
)
1437 vf
= efx
->vf
+ vf_i
;
1438 netif_info(efx
, hw
, efx
->net_dev
,
1439 "FLR on VF %s\n", vf
->pci_name
);
1441 vf
->status_addr
= 0;
1442 efx_vfdi_remove_all_filters(vf
);
1443 efx_vfdi_flush_clear(vf
);
1448 void efx_sriov_mac_address_changed(struct efx_nic
*efx
)
1450 struct vfdi_status
*vfdi_status
= efx
->vfdi_status
.addr
;
1452 if (!efx
->vf_init_count
)
1454 memcpy(vfdi_status
->peers
[0].mac_addr
,
1455 efx
->net_dev
->dev_addr
, ETH_ALEN
);
1456 queue_work(vfdi_workqueue
, &efx
->peer_work
);
1459 void efx_sriov_tx_flush_done(struct efx_nic
*efx
, efx_qword_t
*event
)
1462 unsigned queue
, qid
;
1464 queue
= EFX_QWORD_FIELD(*event
, FSF_AZ_DRIVER_EV_SUBDATA
);
1465 if (map_vi_index(efx
, queue
, &vf
, &qid
))
1467 /* Ignore flush completions triggered by an FLR */
1468 if (!test_bit(qid
, vf
->txq_mask
))
1471 __clear_bit(qid
, vf
->txq_mask
);
1474 if (efx_vfdi_flush_wake(vf
))
1475 wake_up(&vf
->flush_waitq
);
1478 void efx_sriov_rx_flush_done(struct efx_nic
*efx
, efx_qword_t
*event
)
1481 unsigned ev_failed
, queue
, qid
;
1483 queue
= EFX_QWORD_FIELD(*event
, FSF_AZ_DRIVER_EV_RX_DESCQ_ID
);
1484 ev_failed
= EFX_QWORD_FIELD(*event
,
1485 FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL
);
1486 if (map_vi_index(efx
, queue
, &vf
, &qid
))
1488 if (!test_bit(qid
, vf
->rxq_mask
))
1492 set_bit(qid
, vf
->rxq_retry_mask
);
1493 atomic_inc(&vf
->rxq_retry_count
);
1495 __clear_bit(qid
, vf
->rxq_mask
);
1498 if (efx_vfdi_flush_wake(vf
))
1499 wake_up(&vf
->flush_waitq
);
1502 /* Called from napi. Schedule the reset work item */
1503 void efx_sriov_desc_fetch_err(struct efx_nic
*efx
, unsigned dmaq
)
1508 if (map_vi_index(efx
, dmaq
, &vf
, &rel
))
1511 if (net_ratelimit())
1512 netif_err(efx
, hw
, efx
->net_dev
,
1513 "VF %d DMA Q %d reports descriptor fetch error.\n",
1515 queue_work(vfdi_workqueue
, &vf
->reset_work
);
1519 void efx_sriov_reset(struct efx_nic
*efx
)
1522 struct efx_buffer buf
;
1527 if (efx
->vf_init_count
== 0)
1530 efx_sriov_usrev(efx
, true);
1531 (void)efx_sriov_cmd(efx
, true, NULL
, NULL
);
1533 if (efx_nic_alloc_buffer(efx
, &buf
, EFX_PAGE_SIZE
, GFP_NOIO
))
1536 for (vf_i
= 0; vf_i
< efx
->vf_init_count
; ++vf_i
) {
1537 vf
= efx
->vf
+ vf_i
;
1538 efx_sriov_reset_vf(vf
, &buf
);
1541 efx_nic_free_buffer(efx
, &buf
);
1544 int efx_init_sriov(void)
1546 /* A single threaded workqueue is sufficient. efx_sriov_vfdi() and
1547 * efx_sriov_peer_work() spend almost all their time sleeping for
1548 * MCDI to complete anyway
1550 vfdi_workqueue
= create_singlethread_workqueue("sfc_vfdi");
1551 if (!vfdi_workqueue
)
1557 void efx_fini_sriov(void)
1559 destroy_workqueue(vfdi_workqueue
);
1562 int efx_sriov_set_vf_mac(struct net_device
*net_dev
, int vf_i
, u8
*mac
)
1564 struct efx_nic
*efx
= netdev_priv(net_dev
);
1567 if (vf_i
>= efx
->vf_init_count
)
1569 vf
= efx
->vf
+ vf_i
;
1571 mutex_lock(&vf
->status_lock
);
1572 memcpy(vf
->addr
.mac_addr
, mac
, ETH_ALEN
);
1573 __efx_sriov_update_vf_addr(vf
);
1574 mutex_unlock(&vf
->status_lock
);
1579 int efx_sriov_set_vf_vlan(struct net_device
*net_dev
, int vf_i
,
1582 struct efx_nic
*efx
= netdev_priv(net_dev
);
1586 if (vf_i
>= efx
->vf_init_count
)
1588 vf
= efx
->vf
+ vf_i
;
1590 mutex_lock(&vf
->status_lock
);
1591 tci
= (vlan
& VLAN_VID_MASK
) | ((qos
& 0x7) << VLAN_PRIO_SHIFT
);
1592 vf
->addr
.tci
= htons(tci
);
1593 __efx_sriov_update_vf_addr(vf
);
1594 mutex_unlock(&vf
->status_lock
);
1599 int efx_sriov_set_vf_spoofchk(struct net_device
*net_dev
, int vf_i
,
1602 struct efx_nic
*efx
= netdev_priv(net_dev
);
1606 if (vf_i
>= efx
->vf_init_count
)
1608 vf
= efx
->vf
+ vf_i
;
1610 mutex_lock(&vf
->txq_lock
);
1611 if (vf
->txq_count
== 0) {
1612 vf
->tx_filter_mode
=
1613 spoofchk
? VF_TX_FILTER_ON
: VF_TX_FILTER_OFF
;
1616 /* This cannot be changed while TX queues are running */
1619 mutex_unlock(&vf
->txq_lock
);
1623 int efx_sriov_get_vf_config(struct net_device
*net_dev
, int vf_i
,
1624 struct ifla_vf_info
*ivi
)
1626 struct efx_nic
*efx
= netdev_priv(net_dev
);
1630 if (vf_i
>= efx
->vf_init_count
)
1632 vf
= efx
->vf
+ vf_i
;
1635 memcpy(ivi
->mac
, vf
->addr
.mac_addr
, ETH_ALEN
);
1637 tci
= ntohs(vf
->addr
.tci
);
1638 ivi
->vlan
= tci
& VLAN_VID_MASK
;
1639 ivi
->qos
= (tci
>> VLAN_PRIO_SHIFT
) & 0x7;
1640 ivi
->spoofchk
= vf
->tx_filter_mode
== VF_TX_FILTER_ON
;