2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/etherdevice.h>
35 #include <linux/tcp.h>
36 #include <linux/if_vlan.h>
37 #include <linux/delay.h>
38 #include <linux/slab.h>
40 #include <linux/mlx4/driver.h>
41 #include <linux/mlx4/device.h>
42 #include <linux/mlx4/cmd.h>
43 #include <linux/mlx4/cq.h>
48 static void mlx4_en_vlan_rx_add_vid(struct net_device
*dev
, unsigned short vid
)
50 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
51 struct mlx4_en_dev
*mdev
= priv
->mdev
;
55 en_dbg(HW
, priv
, "adding VLAN:%d\n", vid
);
57 set_bit(vid
, priv
->active_vlans
);
59 /* Add VID to port VLAN filter */
60 mutex_lock(&mdev
->state_lock
);
61 if (mdev
->device_up
&& priv
->port_up
) {
62 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
64 en_err(priv
, "Failed configuring VLAN filter\n");
66 if (mlx4_register_vlan(mdev
->dev
, priv
->port
, vid
, &idx
))
67 en_err(priv
, "failed adding vlan %d\n", vid
);
68 mutex_unlock(&mdev
->state_lock
);
72 static void mlx4_en_vlan_rx_kill_vid(struct net_device
*dev
, unsigned short vid
)
74 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
75 struct mlx4_en_dev
*mdev
= priv
->mdev
;
79 en_dbg(HW
, priv
, "Killing VID:%d\n", vid
);
81 clear_bit(vid
, priv
->active_vlans
);
83 /* Remove VID from port VLAN filter */
84 mutex_lock(&mdev
->state_lock
);
85 if (!mlx4_find_cached_vlan(mdev
->dev
, priv
->port
, vid
, &idx
))
86 mlx4_unregister_vlan(mdev
->dev
, priv
->port
, idx
);
88 en_err(priv
, "could not find vid %d in cache\n", vid
);
90 if (mdev
->device_up
&& priv
->port_up
) {
91 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
93 en_err(priv
, "Failed configuring VLAN filter\n");
95 mutex_unlock(&mdev
->state_lock
);
98 u64
mlx4_en_mac_to_u64(u8
*addr
)
103 for (i
= 0; i
< ETH_ALEN
; i
++) {
110 static int mlx4_en_set_mac(struct net_device
*dev
, void *addr
)
112 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
113 struct mlx4_en_dev
*mdev
= priv
->mdev
;
114 struct sockaddr
*saddr
= addr
;
116 if (!is_valid_ether_addr(saddr
->sa_data
))
117 return -EADDRNOTAVAIL
;
119 memcpy(dev
->dev_addr
, saddr
->sa_data
, ETH_ALEN
);
120 priv
->mac
= mlx4_en_mac_to_u64(dev
->dev_addr
);
121 queue_work(mdev
->workqueue
, &priv
->mac_task
);
125 static void mlx4_en_do_set_mac(struct work_struct
*work
)
127 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
129 struct mlx4_en_dev
*mdev
= priv
->mdev
;
132 mutex_lock(&mdev
->state_lock
);
134 /* Remove old MAC and insert the new one */
135 err
= mlx4_replace_mac(mdev
->dev
, priv
->port
,
136 priv
->base_qpn
, priv
->mac
, 0);
138 en_err(priv
, "Failed changing HW MAC address\n");
140 en_dbg(HW
, priv
, "Port is down while "
141 "registering mac, exiting...\n");
143 mutex_unlock(&mdev
->state_lock
);
146 static void mlx4_en_clear_list(struct net_device
*dev
)
148 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
150 kfree(priv
->mc_addrs
);
151 priv
->mc_addrs_cnt
= 0;
154 static void mlx4_en_cache_mclist(struct net_device
*dev
)
156 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
157 struct netdev_hw_addr
*ha
;
159 int mc_addrs_cnt
= netdev_mc_count(dev
);
162 mc_addrs
= kmalloc(mc_addrs_cnt
* ETH_ALEN
, GFP_ATOMIC
);
164 en_err(priv
, "failed to allocate multicast list\n");
168 netdev_for_each_mc_addr(ha
, dev
)
169 memcpy(mc_addrs
+ i
++ * ETH_ALEN
, ha
->addr
, ETH_ALEN
);
170 priv
->mc_addrs
= mc_addrs
;
171 priv
->mc_addrs_cnt
= mc_addrs_cnt
;
175 static void mlx4_en_set_multicast(struct net_device
*dev
)
177 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
182 queue_work(priv
->mdev
->workqueue
, &priv
->mcast_task
);
185 static void mlx4_en_do_set_multicast(struct work_struct
*work
)
187 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
189 struct mlx4_en_dev
*mdev
= priv
->mdev
;
190 struct net_device
*dev
= priv
->dev
;
192 u8 mc_list
[16] = {0};
195 mutex_lock(&mdev
->state_lock
);
196 if (!mdev
->device_up
) {
197 en_dbg(HW
, priv
, "Card is not up, "
198 "ignoring multicast change.\n");
201 if (!priv
->port_up
) {
202 en_dbg(HW
, priv
, "Port is down, "
203 "ignoring multicast change.\n");
208 * Promsicuous mode: disable all filters
211 if (dev
->flags
& IFF_PROMISC
) {
212 if (!(priv
->flags
& MLX4_EN_FLAG_PROMISC
)) {
213 if (netif_msg_rx_status(priv
))
214 en_warn(priv
, "Entering promiscuous mode\n");
215 priv
->flags
|= MLX4_EN_FLAG_PROMISC
;
217 /* Enable promiscouos mode */
218 if (!(mdev
->dev
->caps
.flags
&
219 MLX4_DEV_CAP_FLAG_VEP_UC_STEER
))
220 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
, priv
->port
,
223 err
= mlx4_unicast_promisc_add(mdev
->dev
, priv
->base_qpn
,
226 en_err(priv
, "Failed enabling "
227 "promiscuous mode\n");
229 /* Disable port multicast filter (unconditionally) */
230 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
231 0, MLX4_MCAST_DISABLE
);
233 en_err(priv
, "Failed disabling "
234 "multicast filter\n");
236 /* Add the default qp number as multicast promisc */
237 if (!(priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
)) {
238 err
= mlx4_multicast_promisc_add(mdev
->dev
, priv
->base_qpn
,
241 en_err(priv
, "Failed entering multicast promisc mode\n");
242 priv
->flags
|= MLX4_EN_FLAG_MC_PROMISC
;
245 /* Disable port VLAN filter */
246 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
248 en_err(priv
, "Failed disabling VLAN filter\n");
254 * Not in promiscuous mode
257 if (priv
->flags
& MLX4_EN_FLAG_PROMISC
) {
258 if (netif_msg_rx_status(priv
))
259 en_warn(priv
, "Leaving promiscuous mode\n");
260 priv
->flags
&= ~MLX4_EN_FLAG_PROMISC
;
262 /* Disable promiscouos mode */
263 if (!(mdev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_VEP_UC_STEER
))
264 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
, priv
->port
,
267 err
= mlx4_unicast_promisc_remove(mdev
->dev
, priv
->base_qpn
,
270 en_err(priv
, "Failed disabling promiscuous mode\n");
272 /* Disable Multicast promisc */
273 if (priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
) {
274 err
= mlx4_multicast_promisc_remove(mdev
->dev
, priv
->base_qpn
,
277 en_err(priv
, "Failed disabling multicast promiscuous mode\n");
278 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
281 /* Enable port VLAN filter */
282 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
284 en_err(priv
, "Failed enabling VLAN filter\n");
287 /* Enable/disable the multicast filter according to IFF_ALLMULTI */
288 if (dev
->flags
& IFF_ALLMULTI
) {
289 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
290 0, MLX4_MCAST_DISABLE
);
292 en_err(priv
, "Failed disabling multicast filter\n");
294 /* Add the default qp number as multicast promisc */
295 if (!(priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
)) {
296 err
= mlx4_multicast_promisc_add(mdev
->dev
, priv
->base_qpn
,
299 en_err(priv
, "Failed entering multicast promisc mode\n");
300 priv
->flags
|= MLX4_EN_FLAG_MC_PROMISC
;
304 /* Disable Multicast promisc */
305 if (priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
) {
306 err
= mlx4_multicast_promisc_remove(mdev
->dev
, priv
->base_qpn
,
309 en_err(priv
, "Failed disabling multicast promiscuous mode\n");
310 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
313 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
314 0, MLX4_MCAST_DISABLE
);
316 en_err(priv
, "Failed disabling multicast filter\n");
318 /* Detach our qp from all the multicast addresses */
319 for (i
= 0; i
< priv
->mc_addrs_cnt
; i
++) {
320 memcpy(&mc_list
[10], priv
->mc_addrs
+ i
* ETH_ALEN
, ETH_ALEN
);
321 mc_list
[5] = priv
->port
;
322 mlx4_multicast_detach(mdev
->dev
, &priv
->rss_map
.indir_qp
,
323 mc_list
, MLX4_PROT_ETH
);
325 /* Flush mcast filter and init it with broadcast address */
326 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, ETH_BCAST
,
327 1, MLX4_MCAST_CONFIG
);
329 /* Update multicast list - we cache all addresses so they won't
330 * change while HW is updated holding the command semaphor */
331 netif_tx_lock_bh(dev
);
332 mlx4_en_cache_mclist(dev
);
333 netif_tx_unlock_bh(dev
);
334 for (i
= 0; i
< priv
->mc_addrs_cnt
; i
++) {
336 mlx4_en_mac_to_u64(priv
->mc_addrs
+ i
* ETH_ALEN
);
337 memcpy(&mc_list
[10], priv
->mc_addrs
+ i
* ETH_ALEN
, ETH_ALEN
);
338 mc_list
[5] = priv
->port
;
339 mlx4_multicast_attach(mdev
->dev
, &priv
->rss_map
.indir_qp
,
340 mc_list
, 0, MLX4_PROT_ETH
);
341 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
,
342 mcast_addr
, 0, MLX4_MCAST_CONFIG
);
344 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
345 0, MLX4_MCAST_ENABLE
);
347 en_err(priv
, "Failed enabling multicast filter\n");
350 mutex_unlock(&mdev
->state_lock
);
353 #ifdef CONFIG_NET_POLL_CONTROLLER
354 static void mlx4_en_netpoll(struct net_device
*dev
)
356 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
357 struct mlx4_en_cq
*cq
;
361 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
362 cq
= &priv
->rx_cq
[i
];
363 spin_lock_irqsave(&cq
->lock
, flags
);
364 napi_synchronize(&cq
->napi
);
365 mlx4_en_process_rx_cq(dev
, cq
, 0);
366 spin_unlock_irqrestore(&cq
->lock
, flags
);
371 static void mlx4_en_tx_timeout(struct net_device
*dev
)
373 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
374 struct mlx4_en_dev
*mdev
= priv
->mdev
;
376 if (netif_msg_timer(priv
))
377 en_warn(priv
, "Tx timeout called on port:%d\n", priv
->port
);
379 priv
->port_stats
.tx_timeout
++;
380 en_dbg(DRV
, priv
, "Scheduling watchdog\n");
381 queue_work(mdev
->workqueue
, &priv
->watchdog_task
);
385 static struct net_device_stats
*mlx4_en_get_stats(struct net_device
*dev
)
387 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
389 spin_lock_bh(&priv
->stats_lock
);
390 memcpy(&priv
->ret_stats
, &priv
->stats
, sizeof(priv
->stats
));
391 spin_unlock_bh(&priv
->stats_lock
);
393 return &priv
->ret_stats
;
396 static void mlx4_en_set_default_moderation(struct mlx4_en_priv
*priv
)
398 struct mlx4_en_cq
*cq
;
401 /* If we haven't received a specific coalescing setting
402 * (module param), we set the moderation parameters as follows:
403 * - moder_cnt is set to the number of mtu sized packets to
404 * satisfy our coelsing target.
405 * - moder_time is set to a fixed value.
407 priv
->rx_frames
= MLX4_EN_RX_COAL_TARGET
;
408 priv
->rx_usecs
= MLX4_EN_RX_COAL_TIME
;
409 en_dbg(INTR
, priv
, "Default coalesing params for mtu:%d - "
410 "rx_frames:%d rx_usecs:%d\n",
411 priv
->dev
->mtu
, priv
->rx_frames
, priv
->rx_usecs
);
413 /* Setup cq moderation params */
414 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
415 cq
= &priv
->rx_cq
[i
];
416 cq
->moder_cnt
= priv
->rx_frames
;
417 cq
->moder_time
= priv
->rx_usecs
;
420 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
421 cq
= &priv
->tx_cq
[i
];
422 cq
->moder_cnt
= MLX4_EN_TX_COAL_PKTS
;
423 cq
->moder_time
= MLX4_EN_TX_COAL_TIME
;
426 /* Reset auto-moderation params */
427 priv
->pkt_rate_low
= MLX4_EN_RX_RATE_LOW
;
428 priv
->rx_usecs_low
= MLX4_EN_RX_COAL_TIME_LOW
;
429 priv
->pkt_rate_high
= MLX4_EN_RX_RATE_HIGH
;
430 priv
->rx_usecs_high
= MLX4_EN_RX_COAL_TIME_HIGH
;
431 priv
->sample_interval
= MLX4_EN_SAMPLE_INTERVAL
;
432 priv
->adaptive_rx_coal
= 1;
433 priv
->last_moder_time
= MLX4_EN_AUTO_CONF
;
434 priv
->last_moder_jiffies
= 0;
435 priv
->last_moder_packets
= 0;
436 priv
->last_moder_tx_packets
= 0;
437 priv
->last_moder_bytes
= 0;
440 static void mlx4_en_auto_moderation(struct mlx4_en_priv
*priv
)
442 unsigned long period
= (unsigned long) (jiffies
- priv
->last_moder_jiffies
);
443 struct mlx4_en_cq
*cq
;
444 unsigned long packets
;
446 unsigned long avg_pkt_size
;
447 unsigned long rx_packets
;
448 unsigned long rx_bytes
;
449 unsigned long tx_packets
;
450 unsigned long tx_pkt_diff
;
451 unsigned long rx_pkt_diff
;
455 if (!priv
->adaptive_rx_coal
|| period
< priv
->sample_interval
* HZ
)
458 spin_lock_bh(&priv
->stats_lock
);
459 rx_packets
= priv
->stats
.rx_packets
;
460 rx_bytes
= priv
->stats
.rx_bytes
;
461 tx_packets
= priv
->stats
.tx_packets
;
462 spin_unlock_bh(&priv
->stats_lock
);
464 if (!priv
->last_moder_jiffies
|| !period
)
467 tx_pkt_diff
= ((unsigned long) (tx_packets
-
468 priv
->last_moder_tx_packets
));
469 rx_pkt_diff
= ((unsigned long) (rx_packets
-
470 priv
->last_moder_packets
));
471 packets
= max(tx_pkt_diff
, rx_pkt_diff
);
472 rate
= packets
* HZ
/ period
;
473 avg_pkt_size
= packets
? ((unsigned long) (rx_bytes
-
474 priv
->last_moder_bytes
)) / packets
: 0;
476 /* Apply auto-moderation only when packet rate exceeds a rate that
478 if (rate
> MLX4_EN_RX_RATE_THRESH
&& avg_pkt_size
> MLX4_EN_AVG_PKT_SMALL
) {
479 /* If tx and rx packet rates are not balanced, assume that
480 * traffic is mainly BW bound and apply maximum moderation.
481 * Otherwise, moderate according to packet rate */
482 if (2 * tx_pkt_diff
> 3 * rx_pkt_diff
||
483 2 * rx_pkt_diff
> 3 * tx_pkt_diff
) {
484 moder_time
= priv
->rx_usecs_high
;
486 if (rate
< priv
->pkt_rate_low
)
487 moder_time
= priv
->rx_usecs_low
;
488 else if (rate
> priv
->pkt_rate_high
)
489 moder_time
= priv
->rx_usecs_high
;
491 moder_time
= (rate
- priv
->pkt_rate_low
) *
492 (priv
->rx_usecs_high
- priv
->rx_usecs_low
) /
493 (priv
->pkt_rate_high
- priv
->pkt_rate_low
) +
497 moder_time
= priv
->rx_usecs_low
;
500 en_dbg(INTR
, priv
, "tx rate:%lu rx_rate:%lu\n",
501 tx_pkt_diff
* HZ
/ period
, rx_pkt_diff
* HZ
/ period
);
503 en_dbg(INTR
, priv
, "Rx moder_time changed from:%d to %d period:%lu "
504 "[jiff] packets:%lu avg_pkt_size:%lu rate:%lu [p/s])\n",
505 priv
->last_moder_time
, moder_time
, period
, packets
,
508 if (moder_time
!= priv
->last_moder_time
) {
509 priv
->last_moder_time
= moder_time
;
510 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
511 cq
= &priv
->rx_cq
[i
];
512 cq
->moder_time
= moder_time
;
513 err
= mlx4_en_set_cq_moder(priv
, cq
);
515 en_err(priv
, "Failed modifying moderation for cq:%d\n", i
);
522 priv
->last_moder_packets
= rx_packets
;
523 priv
->last_moder_tx_packets
= tx_packets
;
524 priv
->last_moder_bytes
= rx_bytes
;
525 priv
->last_moder_jiffies
= jiffies
;
528 static void mlx4_en_do_get_stats(struct work_struct
*work
)
530 struct delayed_work
*delay
= to_delayed_work(work
);
531 struct mlx4_en_priv
*priv
= container_of(delay
, struct mlx4_en_priv
,
533 struct mlx4_en_dev
*mdev
= priv
->mdev
;
536 err
= mlx4_en_DUMP_ETH_STATS(mdev
, priv
->port
, 0);
538 en_dbg(HW
, priv
, "Could not update stats\n");
540 mutex_lock(&mdev
->state_lock
);
541 if (mdev
->device_up
) {
543 mlx4_en_auto_moderation(priv
);
545 queue_delayed_work(mdev
->workqueue
, &priv
->stats_task
, STATS_DELAY
);
547 if (mdev
->mac_removed
[MLX4_MAX_PORTS
+ 1 - priv
->port
]) {
548 queue_work(mdev
->workqueue
, &priv
->mac_task
);
549 mdev
->mac_removed
[MLX4_MAX_PORTS
+ 1 - priv
->port
] = 0;
551 mutex_unlock(&mdev
->state_lock
);
554 static void mlx4_en_linkstate(struct work_struct
*work
)
556 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
558 struct mlx4_en_dev
*mdev
= priv
->mdev
;
559 int linkstate
= priv
->link_state
;
561 mutex_lock(&mdev
->state_lock
);
562 /* If observable port state changed set carrier state and
563 * report to system log */
564 if (priv
->last_link_state
!= linkstate
) {
565 if (linkstate
== MLX4_DEV_EVENT_PORT_DOWN
) {
566 en_info(priv
, "Link Down\n");
567 netif_carrier_off(priv
->dev
);
569 en_info(priv
, "Link Up\n");
570 netif_carrier_on(priv
->dev
);
573 priv
->last_link_state
= linkstate
;
574 mutex_unlock(&mdev
->state_lock
);
578 int mlx4_en_start_port(struct net_device
*dev
)
580 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
581 struct mlx4_en_dev
*mdev
= priv
->mdev
;
582 struct mlx4_en_cq
*cq
;
583 struct mlx4_en_tx_ring
*tx_ring
;
589 u8 mc_list
[16] = {0};
592 en_dbg(DRV
, priv
, "start port called while port already up\n");
596 /* Calculate Rx buf size */
597 dev
->mtu
= min(dev
->mtu
, priv
->max_mtu
);
598 mlx4_en_calc_rx_buf(dev
);
599 en_dbg(DRV
, priv
, "Rx buf size:%d\n", priv
->rx_skb_size
);
601 /* Configure rx cq's and rings */
602 err
= mlx4_en_activate_rx_rings(priv
);
604 en_err(priv
, "Failed to activate RX rings\n");
607 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
608 cq
= &priv
->rx_cq
[i
];
610 err
= mlx4_en_activate_cq(priv
, cq
, i
);
612 en_err(priv
, "Failed activating Rx CQ\n");
615 for (j
= 0; j
< cq
->size
; j
++)
616 cq
->buf
[j
].owner_sr_opcode
= MLX4_CQE_OWNER_MASK
;
617 err
= mlx4_en_set_cq_moder(priv
, cq
);
619 en_err(priv
, "Failed setting cq moderation parameters");
620 mlx4_en_deactivate_cq(priv
, cq
);
623 mlx4_en_arm_cq(priv
, cq
);
624 priv
->rx_ring
[i
].cqn
= cq
->mcq
.cqn
;
628 /* Set port mac number */
629 en_dbg(DRV
, priv
, "Setting mac for port %d\n", priv
->port
);
630 err
= mlx4_register_mac(mdev
->dev
, priv
->port
,
631 priv
->mac
, &priv
->base_qpn
, 0);
633 en_err(priv
, "Failed setting port mac\n");
636 mdev
->mac_removed
[priv
->port
] = 0;
638 err
= mlx4_en_config_rss_steer(priv
);
640 en_err(priv
, "Failed configuring rss steering\n");
644 /* Configure tx cq's and rings */
645 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
647 cq
= &priv
->tx_cq
[i
];
648 err
= mlx4_en_activate_cq(priv
, cq
, i
);
650 en_err(priv
, "Failed allocating Tx CQ\n");
653 err
= mlx4_en_set_cq_moder(priv
, cq
);
655 en_err(priv
, "Failed setting cq moderation parameters");
656 mlx4_en_deactivate_cq(priv
, cq
);
659 en_dbg(DRV
, priv
, "Resetting index of collapsed CQ:%d to -1\n", i
);
660 cq
->buf
->wqe_index
= cpu_to_be16(0xffff);
663 tx_ring
= &priv
->tx_ring
[i
];
664 err
= mlx4_en_activate_tx_ring(priv
, tx_ring
, cq
->mcq
.cqn
);
666 en_err(priv
, "Failed allocating Tx ring\n");
667 mlx4_en_deactivate_cq(priv
, cq
);
670 /* Set initial ownership of all Tx TXBBs to SW (1) */
671 for (j
= 0; j
< tx_ring
->buf_size
; j
+= STAMP_STRIDE
)
672 *((u32
*) (tx_ring
->buf
+ j
)) = 0xffffffff;
677 err
= mlx4_SET_PORT_general(mdev
->dev
, priv
->port
,
678 priv
->rx_skb_size
+ ETH_FCS_LEN
,
679 priv
->prof
->tx_pause
,
681 priv
->prof
->rx_pause
,
684 en_err(priv
, "Failed setting port general configurations "
685 "for port %d, with error %d\n", priv
->port
, err
);
688 /* Set default qp number */
689 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
, priv
->port
, priv
->base_qpn
, 0);
691 en_err(priv
, "Failed setting default qp numbers\n");
696 en_dbg(HW
, priv
, "Initializing port\n");
697 err
= mlx4_INIT_PORT(mdev
->dev
, priv
->port
);
699 en_err(priv
, "Failed Initializing port\n");
703 /* Attach rx QP to bradcast address */
704 memset(&mc_list
[10], 0xff, ETH_ALEN
);
705 mc_list
[5] = priv
->port
;
706 if (mlx4_multicast_attach(mdev
->dev
, &priv
->rss_map
.indir_qp
, mc_list
,
708 mlx4_warn(mdev
, "Failed Attaching Broadcast\n");
710 /* Must redo promiscuous mode setup. */
711 priv
->flags
&= ~(MLX4_EN_FLAG_PROMISC
| MLX4_EN_FLAG_MC_PROMISC
);
713 /* Schedule multicast task to populate multicast list */
714 queue_work(mdev
->workqueue
, &priv
->mcast_task
);
716 priv
->port_up
= true;
717 netif_tx_start_all_queues(dev
);
722 mlx4_en_deactivate_tx_ring(priv
, &priv
->tx_ring
[tx_index
]);
723 mlx4_en_deactivate_cq(priv
, &priv
->tx_cq
[tx_index
]);
726 mlx4_en_release_rss_steer(priv
);
728 mlx4_unregister_mac(mdev
->dev
, priv
->port
, priv
->base_qpn
);
731 mlx4_en_deactivate_cq(priv
, &priv
->rx_cq
[rx_index
]);
732 for (i
= 0; i
< priv
->rx_ring_num
; i
++)
733 mlx4_en_deactivate_rx_ring(priv
, &priv
->rx_ring
[i
]);
735 return err
; /* need to close devices */
739 void mlx4_en_stop_port(struct net_device
*dev
)
741 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
742 struct mlx4_en_dev
*mdev
= priv
->mdev
;
744 u8 mc_list
[16] = {0};
746 if (!priv
->port_up
) {
747 en_dbg(DRV
, priv
, "stop port called while port already down\n");
751 /* Synchronize with tx routine */
752 netif_tx_lock_bh(dev
);
753 netif_tx_stop_all_queues(dev
);
754 netif_tx_unlock_bh(dev
);
756 /* Set port as not active */
757 priv
->port_up
= false;
759 /* Detach All multicasts */
760 memset(&mc_list
[10], 0xff, ETH_ALEN
);
761 mc_list
[5] = priv
->port
;
762 mlx4_multicast_detach(mdev
->dev
, &priv
->rss_map
.indir_qp
, mc_list
,
764 for (i
= 0; i
< priv
->mc_addrs_cnt
; i
++) {
765 memcpy(&mc_list
[10], priv
->mc_addrs
+ i
* ETH_ALEN
, ETH_ALEN
);
766 mc_list
[5] = priv
->port
;
767 mlx4_multicast_detach(mdev
->dev
, &priv
->rss_map
.indir_qp
,
768 mc_list
, MLX4_PROT_ETH
);
770 mlx4_en_clear_list(dev
);
771 /* Flush multicast filter */
772 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0, 1, MLX4_MCAST_CONFIG
);
774 /* Unregister Mac address for the port */
775 mlx4_unregister_mac(mdev
->dev
, priv
->port
, priv
->base_qpn
);
776 mdev
->mac_removed
[priv
->port
] = 1;
779 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
780 mlx4_en_deactivate_tx_ring(priv
, &priv
->tx_ring
[i
]);
781 mlx4_en_deactivate_cq(priv
, &priv
->tx_cq
[i
]);
785 for (i
= 0; i
< priv
->tx_ring_num
; i
++)
786 mlx4_en_free_tx_buf(dev
, &priv
->tx_ring
[i
]);
789 mlx4_en_release_rss_steer(priv
);
792 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
793 mlx4_en_deactivate_rx_ring(priv
, &priv
->rx_ring
[i
]);
794 while (test_bit(NAPI_STATE_SCHED
, &priv
->rx_cq
[i
].napi
.state
))
796 mlx4_en_deactivate_cq(priv
, &priv
->rx_cq
[i
]);
800 mlx4_CLOSE_PORT(mdev
->dev
, priv
->port
);
803 static void mlx4_en_restart(struct work_struct
*work
)
805 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
807 struct mlx4_en_dev
*mdev
= priv
->mdev
;
808 struct net_device
*dev
= priv
->dev
;
810 en_dbg(DRV
, priv
, "Watchdog task called for port %d\n", priv
->port
);
812 mutex_lock(&mdev
->state_lock
);
814 mlx4_en_stop_port(dev
);
815 if (mlx4_en_start_port(dev
))
816 en_err(priv
, "Failed restarting port %d\n", priv
->port
);
818 mutex_unlock(&mdev
->state_lock
);
822 static int mlx4_en_open(struct net_device
*dev
)
824 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
825 struct mlx4_en_dev
*mdev
= priv
->mdev
;
829 mutex_lock(&mdev
->state_lock
);
831 if (!mdev
->device_up
) {
832 en_err(priv
, "Cannot open - device down/disabled\n");
837 /* Reset HW statistics and performance counters */
838 if (mlx4_en_DUMP_ETH_STATS(mdev
, priv
->port
, 1))
839 en_dbg(HW
, priv
, "Failed dumping statistics\n");
841 memset(&priv
->stats
, 0, sizeof(priv
->stats
));
842 memset(&priv
->pstats
, 0, sizeof(priv
->pstats
));
844 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
845 priv
->tx_ring
[i
].bytes
= 0;
846 priv
->tx_ring
[i
].packets
= 0;
848 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
849 priv
->rx_ring
[i
].bytes
= 0;
850 priv
->rx_ring
[i
].packets
= 0;
853 err
= mlx4_en_start_port(dev
);
855 en_err(priv
, "Failed starting port:%d\n", priv
->port
);
858 mutex_unlock(&mdev
->state_lock
);
863 static int mlx4_en_close(struct net_device
*dev
)
865 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
866 struct mlx4_en_dev
*mdev
= priv
->mdev
;
868 en_dbg(IFDOWN
, priv
, "Close port called\n");
870 mutex_lock(&mdev
->state_lock
);
872 mlx4_en_stop_port(dev
);
873 netif_carrier_off(dev
);
875 mutex_unlock(&mdev
->state_lock
);
879 void mlx4_en_free_resources(struct mlx4_en_priv
*priv
)
883 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
884 if (priv
->tx_ring
[i
].tx_info
)
885 mlx4_en_destroy_tx_ring(priv
, &priv
->tx_ring
[i
]);
886 if (priv
->tx_cq
[i
].buf
)
887 mlx4_en_destroy_cq(priv
, &priv
->tx_cq
[i
]);
890 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
891 if (priv
->rx_ring
[i
].rx_info
)
892 mlx4_en_destroy_rx_ring(priv
, &priv
->rx_ring
[i
]);
893 if (priv
->rx_cq
[i
].buf
)
894 mlx4_en_destroy_cq(priv
, &priv
->rx_cq
[i
]);
898 int mlx4_en_alloc_resources(struct mlx4_en_priv
*priv
)
900 struct mlx4_en_port_profile
*prof
= priv
->prof
;
902 int base_tx_qpn
, err
;
904 err
= mlx4_qp_reserve_range(priv
->mdev
->dev
, priv
->tx_ring_num
, 256, &base_tx_qpn
);
906 en_err(priv
, "failed reserving range for TX rings\n");
910 /* Create tx Rings */
911 for (i
= 0; i
< priv
->tx_ring_num
; i
++) {
912 if (mlx4_en_create_cq(priv
, &priv
->tx_cq
[i
],
913 prof
->tx_ring_size
, i
, TX
))
916 if (mlx4_en_create_tx_ring(priv
, &priv
->tx_ring
[i
], base_tx_qpn
+ i
,
917 prof
->tx_ring_size
, TXBB_SIZE
))
921 /* Create rx Rings */
922 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
923 if (mlx4_en_create_cq(priv
, &priv
->rx_cq
[i
],
924 prof
->rx_ring_size
, i
, RX
))
927 if (mlx4_en_create_rx_ring(priv
, &priv
->rx_ring
[i
],
928 prof
->rx_ring_size
, priv
->stride
))
935 en_err(priv
, "Failed to allocate NIC resources\n");
936 mlx4_qp_release_range(priv
->mdev
->dev
, base_tx_qpn
, priv
->tx_ring_num
);
941 void mlx4_en_destroy_netdev(struct net_device
*dev
)
943 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
944 struct mlx4_en_dev
*mdev
= priv
->mdev
;
946 en_dbg(DRV
, priv
, "Destroying netdev on port:%d\n", priv
->port
);
948 /* Unregister device - this will close the port if it was up */
949 if (priv
->registered
)
950 unregister_netdev(dev
);
953 mlx4_free_hwq_res(mdev
->dev
, &priv
->res
, MLX4_EN_PAGE_SIZE
);
955 cancel_delayed_work(&priv
->stats_task
);
956 /* flush any pending task for this netdev */
957 flush_workqueue(mdev
->workqueue
);
959 /* Detach the netdev so tasks would not attempt to access it */
960 mutex_lock(&mdev
->state_lock
);
961 mdev
->pndev
[priv
->port
] = NULL
;
962 mutex_unlock(&mdev
->state_lock
);
964 mlx4_en_free_resources(priv
);
968 static int mlx4_en_change_mtu(struct net_device
*dev
, int new_mtu
)
970 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
971 struct mlx4_en_dev
*mdev
= priv
->mdev
;
974 en_dbg(DRV
, priv
, "Change MTU called - current:%d new:%d\n",
977 if ((new_mtu
< MLX4_EN_MIN_MTU
) || (new_mtu
> priv
->max_mtu
)) {
978 en_err(priv
, "Bad MTU size:%d.\n", new_mtu
);
983 if (netif_running(dev
)) {
984 mutex_lock(&mdev
->state_lock
);
985 if (!mdev
->device_up
) {
986 /* NIC is probably restarting - let watchdog task reset
988 en_dbg(DRV
, priv
, "Change MTU called with card down!?\n");
990 mlx4_en_stop_port(dev
);
991 err
= mlx4_en_start_port(dev
);
993 en_err(priv
, "Failed restarting port:%d\n",
995 queue_work(mdev
->workqueue
, &priv
->watchdog_task
);
998 mutex_unlock(&mdev
->state_lock
);
1003 static const struct net_device_ops mlx4_netdev_ops
= {
1004 .ndo_open
= mlx4_en_open
,
1005 .ndo_stop
= mlx4_en_close
,
1006 .ndo_start_xmit
= mlx4_en_xmit
,
1007 .ndo_select_queue
= mlx4_en_select_queue
,
1008 .ndo_get_stats
= mlx4_en_get_stats
,
1009 .ndo_set_rx_mode
= mlx4_en_set_multicast
,
1010 .ndo_set_mac_address
= mlx4_en_set_mac
,
1011 .ndo_validate_addr
= eth_validate_addr
,
1012 .ndo_change_mtu
= mlx4_en_change_mtu
,
1013 .ndo_tx_timeout
= mlx4_en_tx_timeout
,
1014 .ndo_vlan_rx_add_vid
= mlx4_en_vlan_rx_add_vid
,
1015 .ndo_vlan_rx_kill_vid
= mlx4_en_vlan_rx_kill_vid
,
1016 #ifdef CONFIG_NET_POLL_CONTROLLER
1017 .ndo_poll_controller
= mlx4_en_netpoll
,
1021 int mlx4_en_init_netdev(struct mlx4_en_dev
*mdev
, int port
,
1022 struct mlx4_en_port_profile
*prof
)
1024 struct net_device
*dev
;
1025 struct mlx4_en_priv
*priv
;
1029 dev
= alloc_etherdev_mqs(sizeof(struct mlx4_en_priv
),
1030 prof
->tx_ring_num
, prof
->rx_ring_num
);
1032 mlx4_err(mdev
, "Net device allocation failed\n");
1036 SET_NETDEV_DEV(dev
, &mdev
->dev
->pdev
->dev
);
1037 dev
->dev_id
= port
- 1;
1040 * Initialize driver private data
1043 priv
= netdev_priv(dev
);
1044 memset(priv
, 0, sizeof(struct mlx4_en_priv
));
1049 priv
->port_up
= false;
1050 priv
->flags
= prof
->flags
;
1051 priv
->tx_ring_num
= prof
->tx_ring_num
;
1052 priv
->rx_ring_num
= prof
->rx_ring_num
;
1053 priv
->mac_index
= -1;
1054 priv
->msg_enable
= MLX4_EN_MSG_LEVEL
;
1055 spin_lock_init(&priv
->stats_lock
);
1056 INIT_WORK(&priv
->mcast_task
, mlx4_en_do_set_multicast
);
1057 INIT_WORK(&priv
->mac_task
, mlx4_en_do_set_mac
);
1058 INIT_WORK(&priv
->watchdog_task
, mlx4_en_restart
);
1059 INIT_WORK(&priv
->linkstate_task
, mlx4_en_linkstate
);
1060 INIT_DELAYED_WORK(&priv
->stats_task
, mlx4_en_do_get_stats
);
1062 /* Query for default mac and max mtu */
1063 priv
->max_mtu
= mdev
->dev
->caps
.eth_mtu_cap
[priv
->port
];
1064 priv
->mac
= mdev
->dev
->caps
.def_mac
[priv
->port
];
1065 if (ILLEGAL_MAC(priv
->mac
)) {
1066 en_err(priv
, "Port: %d, invalid mac burned: 0x%llx, quiting\n",
1067 priv
->port
, priv
->mac
);
1072 priv
->stride
= roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc
) +
1073 DS_SIZE
* MLX4_EN_MAX_RX_FRAGS
);
1074 err
= mlx4_en_alloc_resources(priv
);
1078 /* Allocate page for receive rings */
1079 err
= mlx4_alloc_hwq_res(mdev
->dev
, &priv
->res
,
1080 MLX4_EN_PAGE_SIZE
, MLX4_EN_PAGE_SIZE
);
1082 en_err(priv
, "Failed to allocate page for rx qps\n");
1085 priv
->allocated
= 1;
1088 * Initialize netdev entry points
1090 dev
->netdev_ops
= &mlx4_netdev_ops
;
1091 dev
->watchdog_timeo
= MLX4_EN_WATCHDOG_TIMEOUT
;
1092 netif_set_real_num_tx_queues(dev
, priv
->tx_ring_num
);
1093 netif_set_real_num_rx_queues(dev
, priv
->rx_ring_num
);
1095 SET_ETHTOOL_OPS(dev
, &mlx4_en_ethtool_ops
);
1097 /* Set defualt MAC */
1098 dev
->addr_len
= ETH_ALEN
;
1099 for (i
= 0; i
< ETH_ALEN
; i
++) {
1100 dev
->dev_addr
[ETH_ALEN
- 1 - i
] = (u8
) (priv
->mac
>> (8 * i
));
1101 dev
->perm_addr
[ETH_ALEN
- 1 - i
] = (u8
) (priv
->mac
>> (8 * i
));
1105 * Set driver features
1107 dev
->hw_features
= NETIF_F_SG
| NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
;
1108 if (mdev
->LSO_support
)
1109 dev
->hw_features
|= NETIF_F_TSO
| NETIF_F_TSO6
;
1111 dev
->vlan_features
= dev
->hw_features
;
1113 dev
->hw_features
|= NETIF_F_RXCSUM
;
1114 dev
->features
= dev
->hw_features
| NETIF_F_HIGHDMA
|
1115 NETIF_F_HW_VLAN_TX
| NETIF_F_HW_VLAN_RX
|
1116 NETIF_F_HW_VLAN_FILTER
;
1118 mdev
->pndev
[port
] = dev
;
1120 netif_carrier_off(dev
);
1121 err
= register_netdev(dev
);
1123 en_err(priv
, "Netdev registration failed for port %d\n", port
);
1127 en_warn(priv
, "Using %d TX rings\n", prof
->tx_ring_num
);
1128 en_warn(priv
, "Using %d RX rings\n", prof
->rx_ring_num
);
1130 /* Configure port */
1131 err
= mlx4_SET_PORT_general(mdev
->dev
, priv
->port
,
1135 en_err(priv
, "Failed setting port general configurations "
1136 "for port %d, with error %d\n", priv
->port
, err
);
1141 en_warn(priv
, "Initializing port\n");
1142 err
= mlx4_INIT_PORT(mdev
->dev
, priv
->port
);
1144 en_err(priv
, "Failed Initializing port\n");
1147 priv
->registered
= 1;
1148 mlx4_en_set_default_moderation(priv
);
1149 queue_delayed_work(mdev
->workqueue
, &priv
->stats_task
, STATS_DELAY
);
1153 mlx4_en_destroy_netdev(dev
);