4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved.
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
24 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
25 * Use is subject to license terms.
30 /* function prototypes */
31 static mblk_t
*igb_rx_bind(igb_rx_data_t
*, uint32_t, uint32_t);
32 static mblk_t
*igb_rx_copy(igb_rx_data_t
*, uint32_t, uint32_t);
33 static void igb_rx_assoc_hcksum(mblk_t
*, uint32_t);
36 #pragma inline(igb_rx_assoc_hcksum)
41 * igb_rx_recycle - the call-back function to reclaim rx buffer
43 * This function is called when an mp is freed by the user thru
44 * freeb call (Only for mp constructed through desballoc call).
45 * It returns back the freed buffer to the free list.
48 igb_rx_recycle(caddr_t arg
)
51 igb_rx_ring_t
*rx_ring
;
52 igb_rx_data_t
*rx_data
;
53 rx_control_block_t
*recycle_rcb
;
57 recycle_rcb
= (rx_control_block_t
*)(uintptr_t)arg
;
58 rx_data
= recycle_rcb
->rx_data
;
59 rx_ring
= rx_data
->rx_ring
;
62 if (recycle_rcb
->ref_cnt
== 0) {
64 * This case only happens when rx buffers are being freed
65 * in igb_stop() and freemsg() is called.
70 ASSERT(recycle_rcb
->mp
== NULL
);
73 * Using the recycled data buffer to generate a new mblk
75 recycle_rcb
->mp
= desballoc((unsigned char *)
76 recycle_rcb
->rx_buf
.address
,
77 recycle_rcb
->rx_buf
.size
,
78 0, &recycle_rcb
->free_rtn
);
81 * Put the recycled rx control block into free list
83 mutex_enter(&rx_data
->recycle_lock
);
85 free_index
= rx_data
->rcb_tail
;
86 ASSERT(rx_data
->free_list
[free_index
] == NULL
);
88 rx_data
->free_list
[free_index
] = recycle_rcb
;
89 rx_data
->rcb_tail
= NEXT_INDEX(free_index
, 1, rx_data
->free_list_size
);
91 mutex_exit(&rx_data
->recycle_lock
);
94 * The atomic operation on the number of the available rx control
95 * blocks in the free list is used to make the recycling mutual
96 * exclusive with the receiving.
98 atomic_inc_32(&rx_data
->rcb_free
);
99 ASSERT(rx_data
->rcb_free
<= rx_data
->free_list_size
);
102 * Considering the case that the interface is unplumbed
103 * and there are still some buffers held by the upper layer.
104 * When the buffer is returned back, we need to free it.
106 ref_cnt
= atomic_dec_32_nv(&recycle_rcb
->ref_cnt
);
108 if (recycle_rcb
->mp
!= NULL
) {
109 freemsg(recycle_rcb
->mp
);
110 recycle_rcb
->mp
= NULL
;
113 igb_free_dma_buffer(&recycle_rcb
->rx_buf
);
115 mutex_enter(&igb
->rx_pending_lock
);
116 atomic_dec_32(&rx_data
->rcb_pending
);
117 atomic_dec_32(&igb
->rcb_pending
);
120 * When there is not any buffer belonging to this rx_data
121 * held by the upper layer, the rx_data can be freed.
123 if ((rx_data
->flag
& IGB_RX_STOPPED
) &&
124 (rx_data
->rcb_pending
== 0))
125 igb_free_rx_ring_data(rx_data
);
127 mutex_exit(&igb
->rx_pending_lock
);
132 * igb_rx_copy - Use copy to process the received packet
134 * This function will use bcopy to process the packet
135 * and send the copied packet upstream
138 igb_rx_copy(igb_rx_data_t
*rx_data
, uint32_t index
, uint32_t pkt_len
)
140 rx_control_block_t
*current_rcb
;
142 igb_t
*igb
= rx_data
->rx_ring
->igb
;
144 current_rcb
= rx_data
->work_list
[index
];
146 DMA_SYNC(¤t_rcb
->rx_buf
, DDI_DMA_SYNC_FORKERNEL
);
148 if (igb_check_dma_handle(
149 current_rcb
->rx_buf
.dma_handle
) != DDI_FM_OK
) {
150 ddi_fm_service_impact(igb
->dip
, DDI_SERVICE_DEGRADED
);
151 atomic_or_32(&igb
->igb_state
, IGB_ERROR
);
156 * Allocate buffer to receive this packet
158 mp
= allocb(pkt_len
+ IPHDR_ALIGN_ROOM
, 0);
160 igb_log(igb
, IGB_LOG_INFO
,
161 "igb_rx_copy: allocate buffer failed");
166 * Copy the data received into the new cluster
168 mp
->b_rptr
+= IPHDR_ALIGN_ROOM
;
169 bcopy(current_rcb
->rx_buf
.address
, mp
->b_rptr
, pkt_len
);
170 mp
->b_wptr
= mp
->b_rptr
+ pkt_len
;
176 * igb_rx_bind - Use existing DMA buffer to build mblk for receiving
178 * This function will use pre-bound DMA buffer to receive the packet
179 * and build mblk that will be sent upstream.
182 igb_rx_bind(igb_rx_data_t
*rx_data
, uint32_t index
, uint32_t pkt_len
)
184 rx_control_block_t
*current_rcb
;
185 rx_control_block_t
*free_rcb
;
188 igb_t
*igb
= rx_data
->rx_ring
->igb
;
191 * If the free list is empty, we cannot proceed to send
192 * the current DMA buffer upstream. We'll have to return
193 * and use bcopy to process the packet.
195 if (igb_atomic_reserve(&rx_data
->rcb_free
, 1) < 0)
198 current_rcb
= rx_data
->work_list
[index
];
200 * If the mp of the rx control block is NULL, try to do
203 if (current_rcb
->mp
== NULL
) {
204 current_rcb
->mp
= desballoc((unsigned char *)
205 current_rcb
->rx_buf
.address
,
206 current_rcb
->rx_buf
.size
,
207 0, ¤t_rcb
->free_rtn
);
209 * If it is failed to built a mblk using the current
210 * DMA buffer, we have to return and use bcopy to
211 * process the packet.
213 if (current_rcb
->mp
== NULL
) {
214 atomic_inc_32(&rx_data
->rcb_free
);
219 * Sync up the data received
221 DMA_SYNC(¤t_rcb
->rx_buf
, DDI_DMA_SYNC_FORKERNEL
);
223 if (igb_check_dma_handle(
224 current_rcb
->rx_buf
.dma_handle
) != DDI_FM_OK
) {
225 ddi_fm_service_impact(igb
->dip
, DDI_SERVICE_DEGRADED
);
226 atomic_or_32(&igb
->igb_state
, IGB_ERROR
);
227 atomic_inc_32(&rx_data
->rcb_free
);
231 mp
= current_rcb
->mp
;
232 current_rcb
->mp
= NULL
;
233 atomic_inc_32(¤t_rcb
->ref_cnt
);
235 mp
->b_wptr
= mp
->b_rptr
+ pkt_len
;
236 mp
->b_next
= mp
->b_cont
= NULL
;
239 * Strip off one free rx control block from the free list
241 free_index
= rx_data
->rcb_head
;
242 free_rcb
= rx_data
->free_list
[free_index
];
243 ASSERT(free_rcb
!= NULL
);
244 rx_data
->free_list
[free_index
] = NULL
;
245 rx_data
->rcb_head
= NEXT_INDEX(free_index
, 1, rx_data
->free_list_size
);
248 * Put the rx control block to the work list
250 rx_data
->work_list
[index
] = free_rcb
;
256 * igb_rx_assoc_hcksum
258 * Check the rx hardware checksum status and associate the hcksum flags
261 igb_rx_assoc_hcksum(mblk_t
*mp
, uint32_t status_error
)
263 uint32_t hcksum_flags
= 0;
265 /* Ignore Checksum Indication */
266 if (status_error
& E1000_RXD_STAT_IXSM
)
270 * Check TCP/UDP checksum
272 if (((status_error
& E1000_RXD_STAT_TCPCS
) ||
273 (status_error
& E1000_RXD_STAT_UDPCS
)) &&
274 !(status_error
& E1000_RXDEXT_STATERR_TCPE
))
275 hcksum_flags
|= HCK_FULLCKSUM_OK
;
280 if ((status_error
& E1000_RXD_STAT_IPCS
) &&
281 !(status_error
& E1000_RXDEXT_STATERR_IPE
))
282 hcksum_flags
|= HCK_IPV4_HDRCKSUM_OK
;
284 if (hcksum_flags
!= 0) {
285 mac_hcksum_set(mp
, 0, 0, 0, 0, hcksum_flags
);
290 igb_rx_ring_poll(void *arg
, int bytes
)
292 igb_rx_ring_t
*rx_ring
= (igb_rx_ring_t
*)arg
;
297 if ((bytes
== 0) || (rx_ring
->igb
->igb_state
& IGB_SUSPENDED
) ||
298 !(rx_ring
->igb
->igb_state
& IGB_STARTED
))
301 mutex_enter(&rx_ring
->rx_lock
);
302 mp
= igb_rx(rx_ring
, bytes
);
303 mutex_exit(&rx_ring
->rx_lock
);
309 * igb_rx - Receive the data of one ring
311 * This function goes throught h/w descriptor in one specified rx ring,
312 * receives the data if the descriptor status shows the data is ready.
313 * It returns a chain of mblks containing the received data, to be
314 * passed up to mac_rx().
317 igb_rx(igb_rx_ring_t
*rx_ring
, int poll_bytes
)
319 union e1000_adv_rx_desc
*current_rbd
;
320 rx_control_block_t
*current_rcb
;
327 uint32_t status_error
;
329 uint32_t total_bytes
;
330 igb_t
*igb
= rx_ring
->igb
;
331 igb_rx_data_t
*rx_data
= rx_ring
->rx_data
;
334 mblk_tail
= &mblk_head
;
336 if (igb
->igb_state
& IGB_ERROR
)
340 * Sync the receive descriptors before
341 * accepting the packets
343 DMA_SYNC(&rx_data
->rbd_area
, DDI_DMA_SYNC_FORKERNEL
);
345 if (igb_check_dma_handle(
346 rx_data
->rbd_area
.dma_handle
) != DDI_FM_OK
) {
347 ddi_fm_service_impact(igb
->dip
, DDI_SERVICE_DEGRADED
);
348 atomic_or_32(&igb
->igb_state
, IGB_ERROR
);
353 * Get the start point of rx bd ring which should be examined
356 rx_next
= rx_data
->rbd_next
;
358 current_rbd
= &rx_data
->rbd_ring
[rx_next
];
361 status_error
= current_rbd
->wb
.upper
.status_error
;
362 while (status_error
& E1000_RXD_STAT_DD
) {
364 * If hardware has found the errors, but the error
365 * is hardware checksum error, here does not discard the
366 * packet, and let upper layer compute the checksum;
367 * Otherwise discard the packet.
369 if ((status_error
& E1000_RXDEXT_ERR_FRAME_ERR_MASK
) ||
370 !(status_error
& E1000_RXD_STAT_EOP
)) {
371 IGB_DEBUG_STAT(rx_ring
->stat_frame_error
);
375 IGB_DEBUG_STAT_COND(rx_ring
->stat_cksum_error
,
376 (status_error
& E1000_RXDEXT_STATERR_TCPE
) ||
377 (status_error
& E1000_RXDEXT_STATERR_IPE
));
379 pkt_len
= current_rbd
->wb
.upper
.length
;
381 if ((poll_bytes
!= IGB_NO_POLL
) &&
382 ((pkt_len
+ total_bytes
) > poll_bytes
))
385 IGB_DEBUG_STAT(rx_ring
->stat_pkt_cnt
);
386 total_bytes
+= pkt_len
;
390 * For packets with length more than the copy threshold,
391 * we'll firstly try to use the existed DMA buffer to built
392 * a mblk and send the mblk upstream.
394 * If the first method fails, or the packet length is less
395 * than the copy threshold, we'll allocate a new mblk and
396 * copy the packet data to the mblk.
398 if (pkt_len
> igb
->rx_copy_thresh
)
399 mp
= igb_rx_bind(rx_data
, rx_next
, pkt_len
);
402 mp
= igb_rx_copy(rx_data
, rx_next
, pkt_len
);
406 * Check h/w checksum offload status
408 if (igb
->rx_hcksum_enable
)
409 igb_rx_assoc_hcksum(mp
, status_error
);
412 mblk_tail
= &mp
->b_next
;
415 /* Update per-ring rx statistics */
417 rx_ring
->rx_bytes
+= pkt_len
;
421 * Reset rx descriptor read bits
423 current_rcb
= rx_data
->work_list
[rx_next
];
424 current_rbd
->read
.pkt_addr
= current_rcb
->rx_buf
.dma_address
;
425 current_rbd
->read
.hdr_addr
= 0;
427 rx_next
= NEXT_INDEX(rx_next
, 1, rx_data
->ring_size
);
430 * The receive function is in interrupt context, so here
431 * rx_limit_per_intr is used to avoid doing receiving too long
434 if (++pkt_num
> igb
->rx_limit_per_intr
) {
435 IGB_DEBUG_STAT(rx_ring
->stat_exceed_pkt
);
439 current_rbd
= &rx_data
->rbd_ring
[rx_next
];
440 status_error
= current_rbd
->wb
.upper
.status_error
;
443 DMA_SYNC(&rx_data
->rbd_area
, DDI_DMA_SYNC_FORDEV
);
445 rx_data
->rbd_next
= rx_next
;
448 * Update the h/w tail accordingly
450 rx_tail
= PREV_INDEX(rx_next
, 1, rx_data
->ring_size
);
452 E1000_WRITE_REG(&igb
->hw
, E1000_RDT(rx_ring
->index
), rx_tail
);
454 if (igb_check_acc_handle(igb
->osdep
.reg_handle
) != DDI_FM_OK
) {
455 ddi_fm_service_impact(igb
->dip
, DDI_SERVICE_DEGRADED
);
456 atomic_or_32(&igb
->igb_state
, IGB_ERROR
);