1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2016 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18 #include <linux/pci.h>
19 #include <linux/netdevice.h>
20 #include <linux/vmalloc.h>
21 #include "liquidio_common.h"
22 #include "octeon_droq.h"
23 #include "octeon_iq.h"
24 #include "response_manager.h"
25 #include "octeon_device.h"
26 #include "octeon_main.h"
27 #include "octeon_network.h"
28 #include "cn66xx_regs.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
34 struct list_head list
;
39 struct list_head list
;
40 struct octeon_recv_info
*rinfo
;
41 octeon_dispatch_fn_t disp_fn
;
44 /** Get the argument that the user set when registering dispatch
45 * function for a given opcode/subcode.
46 * @param octeon_dev - the octeon device pointer.
47 * @param opcode - the opcode for which the dispatch argument
49 * @param subcode - the subcode for which the dispatch argument
51 * @return Success: void * (argument to the dispatch function)
52 * @return Failure: NULL
55 static inline void *octeon_get_dispatch_arg(struct octeon_device
*octeon_dev
,
56 u16 opcode
, u16 subcode
)
59 struct list_head
*dispatch
;
61 u16 combined_opcode
= OPCODE_SUBCODE(opcode
, subcode
);
63 idx
= combined_opcode
& OCTEON_OPCODE_MASK
;
65 spin_lock_bh(&octeon_dev
->dispatch
.lock
);
67 if (octeon_dev
->dispatch
.count
== 0) {
68 spin_unlock_bh(&octeon_dev
->dispatch
.lock
);
72 if (octeon_dev
->dispatch
.dlist
[idx
].opcode
== combined_opcode
) {
73 fn_arg
= octeon_dev
->dispatch
.dlist
[idx
].arg
;
75 list_for_each(dispatch
,
76 &octeon_dev
->dispatch
.dlist
[idx
].list
) {
77 if (((struct octeon_dispatch
*)dispatch
)->opcode
==
79 fn_arg
= ((struct octeon_dispatch
*)
86 spin_unlock_bh(&octeon_dev
->dispatch
.lock
);
90 /** Check for packets on Droq. This function should be called with lock held.
91 * @param droq - Droq on which count is checked.
92 * @return Returns packet count.
94 u32
octeon_droq_check_hw_for_pkts(struct octeon_droq
*droq
)
99 pkt_count
= readl(droq
->pkts_sent_reg
);
101 last_count
= pkt_count
- droq
->pkt_count
;
102 droq
->pkt_count
= pkt_count
;
104 /* we shall write to cnts at napi irq enable or end of droq tasklet */
106 atomic_add(last_count
, &droq
->pkts_pending
);
111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq
*droq
)
115 /* max_empty_descs is the max. no. of descs that can have no buffers.
116 * If the empty desc count goes beyond this value, we cannot safely
117 * read in a 64K packet sent by Octeon
118 * (64K is max pkt size from Octeon)
120 droq
->max_empty_descs
= 0;
123 droq
->max_empty_descs
++;
124 count
+= droq
->buffer_size
;
125 } while (count
< (64 * 1024));
127 droq
->max_empty_descs
= droq
->max_count
- droq
->max_empty_descs
;
130 static void octeon_droq_reset_indices(struct octeon_droq
*droq
)
134 droq
->refill_idx
= 0;
135 droq
->refill_count
= 0;
136 atomic_set(&droq
->pkts_pending
, 0);
140 octeon_droq_destroy_ring_buffers(struct octeon_device
*oct
,
141 struct octeon_droq
*droq
)
144 struct octeon_skb_page_info
*pg_info
;
146 for (i
= 0; i
< droq
->max_count
; i
++) {
147 pg_info
= &droq
->recv_buf_list
[i
].pg_info
;
150 lio_unmap_ring(oct
->pci_dev
,
155 recv_buffer_destroy(droq
->recv_buf_list
[i
].buffer
,
158 droq
->recv_buf_list
[i
].buffer
= NULL
;
161 octeon_droq_reset_indices(droq
);
165 octeon_droq_setup_ring_buffers(struct octeon_device
*oct
,
166 struct octeon_droq
*droq
)
170 struct octeon_droq_desc
*desc_ring
= droq
->desc_ring
;
172 for (i
= 0; i
< droq
->max_count
; i
++) {
173 buf
= recv_buffer_alloc(oct
, &droq
->recv_buf_list
[i
].pg_info
);
176 dev_err(&oct
->pci_dev
->dev
, "%s buffer alloc failed\n",
178 droq
->stats
.rx_alloc_failure
++;
182 droq
->recv_buf_list
[i
].buffer
= buf
;
183 droq
->recv_buf_list
[i
].data
= get_rbd(buf
);
184 desc_ring
[i
].info_ptr
= 0;
185 desc_ring
[i
].buffer_ptr
=
186 lio_map_ring(droq
->recv_buf_list
[i
].buffer
);
189 octeon_droq_reset_indices(droq
);
191 octeon_droq_compute_max_packet_bufs(droq
);
196 int octeon_delete_droq(struct octeon_device
*oct
, u32 q_no
)
198 struct octeon_droq
*droq
= oct
->droq
[q_no
];
200 dev_dbg(&oct
->pci_dev
->dev
, "%s[%d]\n", __func__
, q_no
);
202 octeon_droq_destroy_ring_buffers(oct
, droq
);
203 vfree(droq
->recv_buf_list
);
206 lio_dma_free(oct
, (droq
->max_count
* OCT_DROQ_DESC_SIZE
),
207 droq
->desc_ring
, droq
->desc_ring_dma
);
209 memset(droq
, 0, OCT_DROQ_SIZE
);
214 int octeon_init_droq(struct octeon_device
*oct
,
220 struct octeon_droq
*droq
;
221 u32 desc_ring_size
= 0, c_num_descs
= 0, c_buf_size
= 0;
222 u32 c_pkts_per_intr
= 0, c_refill_threshold
= 0;
223 int numa_node
= dev_to_node(&oct
->pci_dev
->dev
);
225 dev_dbg(&oct
->pci_dev
->dev
, "%s[%d]\n", __func__
, q_no
);
227 droq
= oct
->droq
[q_no
];
228 memset(droq
, 0, OCT_DROQ_SIZE
);
233 droq
->app_ctx
= app_ctx
;
235 droq
->app_ctx
= (void *)(size_t)q_no
;
237 c_num_descs
= num_descs
;
238 c_buf_size
= desc_size
;
239 if (OCTEON_CN6XXX(oct
)) {
240 struct octeon_config
*conf6x
= CHIP_CONF(oct
, cn6xxx
);
242 c_pkts_per_intr
= (u32
)CFG_GET_OQ_PKTS_PER_INTR(conf6x
);
244 (u32
)CFG_GET_OQ_REFILL_THRESHOLD(conf6x
);
245 } else if (OCTEON_CN23XX_PF(oct
)) {
246 struct octeon_config
*conf23
= CHIP_CONF(oct
, cn23xx_pf
);
248 c_pkts_per_intr
= (u32
)CFG_GET_OQ_PKTS_PER_INTR(conf23
);
249 c_refill_threshold
= (u32
)CFG_GET_OQ_REFILL_THRESHOLD(conf23
);
250 } else if (OCTEON_CN23XX_VF(oct
)) {
251 struct octeon_config
*conf23
= CHIP_CONF(oct
, cn23xx_vf
);
253 c_pkts_per_intr
= (u32
)CFG_GET_OQ_PKTS_PER_INTR(conf23
);
254 c_refill_threshold
= (u32
)CFG_GET_OQ_REFILL_THRESHOLD(conf23
);
259 droq
->max_count
= c_num_descs
;
260 droq
->buffer_size
= c_buf_size
;
262 desc_ring_size
= droq
->max_count
* OCT_DROQ_DESC_SIZE
;
263 droq
->desc_ring
= lio_dma_alloc(oct
, desc_ring_size
,
264 (dma_addr_t
*)&droq
->desc_ring_dma
);
266 if (!droq
->desc_ring
) {
267 dev_err(&oct
->pci_dev
->dev
,
268 "Output queue %d ring alloc failed\n", q_no
);
272 dev_dbg(&oct
->pci_dev
->dev
, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
273 q_no
, droq
->desc_ring
, droq
->desc_ring_dma
);
274 dev_dbg(&oct
->pci_dev
->dev
, "droq[%d]: num_desc: %d\n", q_no
,
277 droq
->recv_buf_list
= (struct octeon_recv_buffer
*)
278 vmalloc_node(droq
->max_count
*
279 OCT_DROQ_RECVBUF_SIZE
,
281 if (!droq
->recv_buf_list
)
282 droq
->recv_buf_list
= (struct octeon_recv_buffer
*)
283 vmalloc(droq
->max_count
*
284 OCT_DROQ_RECVBUF_SIZE
);
285 if (!droq
->recv_buf_list
) {
286 dev_err(&oct
->pci_dev
->dev
, "Output queue recv buf list alloc failed\n");
290 if (octeon_droq_setup_ring_buffers(oct
, droq
))
293 droq
->pkts_per_intr
= c_pkts_per_intr
;
294 droq
->refill_threshold
= c_refill_threshold
;
296 dev_dbg(&oct
->pci_dev
->dev
, "DROQ INIT: max_empty_descs: %d\n",
297 droq
->max_empty_descs
);
299 spin_lock_init(&droq
->lock
);
301 INIT_LIST_HEAD(&droq
->dispatch_list
);
303 /* For 56xx Pass1, this function won't be called, so no checks. */
304 oct
->fn_list
.setup_oq_regs(oct
, q_no
);
306 oct
->io_qmask
.oq
|= BIT_ULL(q_no
);
311 octeon_delete_droq(oct
, q_no
);
315 /* octeon_create_recv_info
317 * octeon_dev - pointer to the octeon device structure
318 * droq - droq in which the packet arrived.
319 * buf_cnt - no. of buffers used by the packet.
320 * idx - index in the descriptor for the first buffer in the packet.
322 * Allocates a recv_info_t and copies the buffer addresses for packet data
323 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
324 * Flags the descriptors for refill later. If available descriptors go
325 * below the threshold to receive a 64K pkt, new buffers are first allocated
326 * before the recv_pkt_t is created.
327 * This routine will be called in interrupt context.
329 * Success: Pointer to recv_info_t
332 * The droq->lock is held when this routine is called.
334 static inline struct octeon_recv_info
*octeon_create_recv_info(
335 struct octeon_device
*octeon_dev
,
336 struct octeon_droq
*droq
,
340 struct octeon_droq_info
*info
;
341 struct octeon_recv_pkt
*recv_pkt
;
342 struct octeon_recv_info
*recv_info
;
344 struct octeon_skb_page_info
*pg_info
;
346 info
= (struct octeon_droq_info
*)droq
->recv_buf_list
[idx
].data
;
348 recv_info
= octeon_alloc_recv_info(sizeof(struct __dispatch
));
352 recv_pkt
= recv_info
->recv_pkt
;
353 recv_pkt
->rh
= info
->rh
;
354 recv_pkt
->length
= (u32
)info
->length
;
355 recv_pkt
->buffer_count
= (u16
)buf_cnt
;
356 recv_pkt
->octeon_id
= (u16
)octeon_dev
->octeon_id
;
359 bytes_left
= (u32
)info
->length
;
363 pg_info
= &droq
->recv_buf_list
[idx
].pg_info
;
365 lio_unmap_ring(octeon_dev
->pci_dev
,
367 pg_info
->page
= NULL
;
371 recv_pkt
->buffer_size
[i
] =
373 droq
->buffer_size
) ? droq
->buffer_size
: bytes_left
;
375 recv_pkt
->buffer_ptr
[i
] = droq
->recv_buf_list
[idx
].buffer
;
376 droq
->recv_buf_list
[idx
].buffer
= NULL
;
378 idx
= incr_index(idx
, 1, droq
->max_count
);
379 bytes_left
-= droq
->buffer_size
;
387 /* If we were not able to refill all buffers, try to move around
388 * the buffers that were not dispatched.
391 octeon_droq_refill_pullup_descs(struct octeon_droq
*droq
,
392 struct octeon_droq_desc
*desc_ring
)
394 u32 desc_refilled
= 0;
396 u32 refill_index
= droq
->refill_idx
;
398 while (refill_index
!= droq
->read_idx
) {
399 if (droq
->recv_buf_list
[refill_index
].buffer
) {
400 droq
->recv_buf_list
[droq
->refill_idx
].buffer
=
401 droq
->recv_buf_list
[refill_index
].buffer
;
402 droq
->recv_buf_list
[droq
->refill_idx
].data
=
403 droq
->recv_buf_list
[refill_index
].data
;
404 desc_ring
[droq
->refill_idx
].buffer_ptr
=
405 desc_ring
[refill_index
].buffer_ptr
;
406 droq
->recv_buf_list
[refill_index
].buffer
= NULL
;
407 desc_ring
[refill_index
].buffer_ptr
= 0;
409 droq
->refill_idx
= incr_index(droq
->refill_idx
,
413 droq
->refill_count
--;
414 } while (droq
->recv_buf_list
[droq
->refill_idx
].buffer
);
416 refill_index
= incr_index(refill_index
, 1, droq
->max_count
);
418 return desc_refilled
;
421 /* octeon_droq_refill
423 * droq - droq in which descriptors require new buffers.
425 * Called during normal DROQ processing in interrupt mode or by the poll
426 * thread to refill the descriptors from which buffers were dispatched
427 * to upper layers. Attempts to allocate new buffers. If that fails, moves
428 * up buffers (that were not dispatched) to form a contiguous ring.
430 * No of descriptors refilled.
432 * This routine is called with droq->lock held.
435 octeon_droq_refill(struct octeon_device
*octeon_dev
, struct octeon_droq
*droq
)
437 struct octeon_droq_desc
*desc_ring
;
440 u32 desc_refilled
= 0;
441 struct octeon_skb_page_info
*pg_info
;
443 desc_ring
= droq
->desc_ring
;
445 while (droq
->refill_count
&& (desc_refilled
< droq
->max_count
)) {
446 /* If a valid buffer exists (happens if there is no dispatch),
448 * the buffer, else allocate.
450 if (!droq
->recv_buf_list
[droq
->refill_idx
].buffer
) {
452 &droq
->recv_buf_list
[droq
->refill_idx
].pg_info
;
453 /* Either recycle the existing pages or go for
457 buf
= recv_buffer_reuse(octeon_dev
, pg_info
);
459 buf
= recv_buffer_alloc(octeon_dev
, pg_info
);
460 /* If a buffer could not be allocated, no point in
464 droq
->stats
.rx_alloc_failure
++;
467 droq
->recv_buf_list
[droq
->refill_idx
].buffer
=
471 data
= get_rbd(droq
->recv_buf_list
472 [droq
->refill_idx
].buffer
);
475 droq
->recv_buf_list
[droq
->refill_idx
].data
= data
;
477 desc_ring
[droq
->refill_idx
].buffer_ptr
=
478 lio_map_ring(droq
->recv_buf_list
[
479 droq
->refill_idx
].buffer
);
481 droq
->refill_idx
= incr_index(droq
->refill_idx
, 1,
484 droq
->refill_count
--;
487 if (droq
->refill_count
)
489 octeon_droq_refill_pullup_descs(droq
, desc_ring
);
491 /* if droq->refill_count
492 * The refill count would not change in pass two. We only moved buffers
493 * to close the gap in the ring, but we would still have the same no. of
496 return desc_refilled
;
499 /** check if we can allocate packets to get out of oom.
500 * @param droq - Droq being checked.
501 * @return does not return anything
503 void octeon_droq_check_oom(struct octeon_droq
*droq
)
506 struct octeon_device
*oct
= droq
->oct_dev
;
508 if (readl(droq
->pkts_credit_reg
) <= CN23XX_SLI_DEF_BP
) {
509 spin_lock_bh(&droq
->lock
);
510 desc_refilled
= octeon_droq_refill(oct
, droq
);
512 /* Flush the droq descriptor data to memory to be sure
513 * that when we update the credits the data in memory
517 writel(desc_refilled
, droq
->pkts_credit_reg
);
518 /* make sure mmio write completes */
521 spin_unlock_bh(&droq
->lock
);
526 octeon_droq_get_bufcount(u32 buf_size
, u32 total_len
)
528 return ((total_len
+ buf_size
- 1) / buf_size
);
532 octeon_droq_dispatch_pkt(struct octeon_device
*oct
,
533 struct octeon_droq
*droq
,
535 struct octeon_droq_info
*info
)
538 octeon_dispatch_fn_t disp_fn
;
539 struct octeon_recv_info
*rinfo
;
541 cnt
= octeon_droq_get_bufcount(droq
->buffer_size
, (u32
)info
->length
);
543 disp_fn
= octeon_get_dispatch(oct
, (u16
)rh
->r
.opcode
,
546 rinfo
= octeon_create_recv_info(oct
, droq
, cnt
, droq
->read_idx
);
548 struct __dispatch
*rdisp
= rinfo
->rsvd
;
550 rdisp
->rinfo
= rinfo
;
551 rdisp
->disp_fn
= disp_fn
;
552 rinfo
->recv_pkt
->rh
= *rh
;
553 list_add_tail(&rdisp
->list
,
554 &droq
->dispatch_list
);
556 droq
->stats
.dropped_nomem
++;
559 dev_err(&oct
->pci_dev
->dev
, "DROQ: No dispatch function (opcode %u/%u)\n",
560 (unsigned int)rh
->r
.opcode
,
561 (unsigned int)rh
->r
.subcode
);
562 droq
->stats
.dropped_nodispatch
++;
568 static inline void octeon_droq_drop_packets(struct octeon_device
*oct
,
569 struct octeon_droq
*droq
,
573 struct octeon_droq_info
*info
;
575 for (i
= 0; i
< cnt
; i
++) {
576 info
= (struct octeon_droq_info
*)
577 droq
->recv_buf_list
[droq
->read_idx
].data
;
578 octeon_swap_8B_data((u64
*)info
, 2);
581 info
->length
+= OCTNET_FRM_LENGTH_SIZE
;
582 droq
->stats
.bytes_received
+= info
->length
;
583 buf_cnt
= octeon_droq_get_bufcount(droq
->buffer_size
,
586 dev_err(&oct
->pci_dev
->dev
, "DROQ: In drop: pkt with len 0\n");
590 droq
->read_idx
= incr_index(droq
->read_idx
, buf_cnt
,
592 droq
->refill_count
+= buf_cnt
;
597 octeon_droq_fast_process_packets(struct octeon_device
*oct
,
598 struct octeon_droq
*droq
,
601 struct octeon_droq_info
*info
;
603 u32 pkt
, total_len
= 0, pkt_count
;
605 pkt_count
= pkts_to_process
;
607 for (pkt
= 0; pkt
< pkt_count
; pkt
++) {
609 struct sk_buff
*nicbuf
= NULL
;
610 struct octeon_skb_page_info
*pg_info
;
613 info
= (struct octeon_droq_info
*)
614 droq
->recv_buf_list
[droq
->read_idx
].data
;
615 octeon_swap_8B_data((u64
*)info
, 2);
618 dev_err(&oct
->pci_dev
->dev
,
619 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
620 droq
->q_no
, droq
->read_idx
, pkt_count
);
621 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS
,
627 /* Len of resp hdr in included in the received data len. */
630 info
->length
+= OCTNET_FRM_LENGTH_SIZE
;
631 rh
->r_dh
.len
+= (ROUNDUP8(OCT_DROQ_INFO_SIZE
) / sizeof(u64
));
632 total_len
+= (u32
)info
->length
;
633 if (opcode_slow_path(rh
)) {
636 buf_cnt
= octeon_droq_dispatch_pkt(oct
, droq
, rh
, info
);
637 droq
->read_idx
= incr_index(droq
->read_idx
,
638 buf_cnt
, droq
->max_count
);
639 droq
->refill_count
+= buf_cnt
;
641 if (info
->length
<= droq
->buffer_size
) {
642 pkt_len
= (u32
)info
->length
;
643 nicbuf
= droq
->recv_buf_list
[
644 droq
->read_idx
].buffer
;
645 pg_info
= &droq
->recv_buf_list
[
646 droq
->read_idx
].pg_info
;
647 if (recv_buffer_recycle(oct
, pg_info
))
648 pg_info
->page
= NULL
;
649 droq
->recv_buf_list
[droq
->read_idx
].buffer
=
652 droq
->read_idx
= incr_index(droq
->read_idx
, 1,
654 droq
->refill_count
++;
656 nicbuf
= octeon_fast_packet_alloc((u32
)
659 /* nicbuf allocation can fail. We'll handle it
662 while (pkt_len
< info
->length
) {
663 int cpy_len
, idx
= droq
->read_idx
;
665 cpy_len
= ((pkt_len
+ droq
->buffer_size
)
667 ((u32
)info
->length
- pkt_len
) :
671 octeon_fast_packet_next(droq
,
675 buf
= droq
->recv_buf_list
[
677 recv_buffer_fast_free(buf
);
678 droq
->recv_buf_list
[idx
].buffer
681 droq
->stats
.rx_alloc_failure
++;
686 incr_index(droq
->read_idx
, 1,
688 droq
->refill_count
++;
693 if (droq
->ops
.fptr
) {
694 droq
->ops
.fptr(oct
->octeon_id
,
699 recv_buffer_free(nicbuf
);
704 if (droq
->refill_count
>= droq
->refill_threshold
) {
705 int desc_refilled
= octeon_droq_refill(oct
, droq
);
707 /* Flush the droq descriptor data to memory to be sure
708 * that when we update the credits the data in memory
712 writel((desc_refilled
), droq
->pkts_credit_reg
);
713 /* make sure mmio write completes */
717 } /* for (each packet)... */
719 /* Increment refill_count by the number of buffers processed. */
720 droq
->stats
.pkts_received
+= pkt
;
721 droq
->stats
.bytes_received
+= total_len
;
723 if ((droq
->ops
.drop_on_max
) && (pkts_to_process
- pkt
)) {
724 octeon_droq_drop_packets(oct
, droq
, (pkts_to_process
- pkt
));
726 droq
->stats
.dropped_toomany
+= (pkts_to_process
- pkt
);
727 return pkts_to_process
;
734 octeon_droq_process_packets(struct octeon_device
*oct
,
735 struct octeon_droq
*droq
,
738 u32 pkt_count
= 0, pkts_processed
= 0;
739 struct list_head
*tmp
, *tmp2
;
741 /* Grab the droq lock */
742 spin_lock(&droq
->lock
);
744 octeon_droq_check_hw_for_pkts(droq
);
745 pkt_count
= atomic_read(&droq
->pkts_pending
);
748 spin_unlock(&droq
->lock
);
752 if (pkt_count
> budget
)
755 pkts_processed
= octeon_droq_fast_process_packets(oct
, droq
, pkt_count
);
757 atomic_sub(pkts_processed
, &droq
->pkts_pending
);
759 /* Release the spin lock */
760 spin_unlock(&droq
->lock
);
762 list_for_each_safe(tmp
, tmp2
, &droq
->dispatch_list
) {
763 struct __dispatch
*rdisp
= (struct __dispatch
*)tmp
;
766 rdisp
->disp_fn(rdisp
->rinfo
,
767 octeon_get_dispatch_arg
769 (u16
)rdisp
->rinfo
->recv_pkt
->rh
.r
.opcode
,
770 (u16
)rdisp
->rinfo
->recv_pkt
->rh
.r
.subcode
));
773 /* If there are packets pending. schedule tasklet again */
774 if (atomic_read(&droq
->pkts_pending
))
781 * Utility function to poll for packets. check_hw_for_packets must be
782 * called before calling this routine.
786 octeon_droq_process_poll_pkts(struct octeon_device
*oct
,
787 struct octeon_droq
*droq
, u32 budget
)
789 struct list_head
*tmp
, *tmp2
;
790 u32 pkts_available
= 0, pkts_processed
= 0;
791 u32 total_pkts_processed
= 0;
793 if (budget
> droq
->max_count
)
794 budget
= droq
->max_count
;
796 spin_lock(&droq
->lock
);
798 while (total_pkts_processed
< budget
) {
799 octeon_droq_check_hw_for_pkts(droq
);
801 pkts_available
= min((budget
- total_pkts_processed
),
802 (u32
)(atomic_read(&droq
->pkts_pending
)));
804 if (pkts_available
== 0)
808 octeon_droq_fast_process_packets(oct
, droq
,
811 atomic_sub(pkts_processed
, &droq
->pkts_pending
);
813 total_pkts_processed
+= pkts_processed
;
816 spin_unlock(&droq
->lock
);
818 list_for_each_safe(tmp
, tmp2
, &droq
->dispatch_list
) {
819 struct __dispatch
*rdisp
= (struct __dispatch
*)tmp
;
822 rdisp
->disp_fn(rdisp
->rinfo
,
823 octeon_get_dispatch_arg
825 (u16
)rdisp
->rinfo
->recv_pkt
->rh
.r
.opcode
,
826 (u16
)rdisp
->rinfo
->recv_pkt
->rh
.r
.subcode
));
829 return total_pkts_processed
;
833 octeon_process_droq_poll_cmd(struct octeon_device
*oct
, u32 q_no
, int cmd
,
836 struct octeon_droq
*droq
;
838 droq
= oct
->droq
[q_no
];
840 if (cmd
== POLL_EVENT_PROCESS_PKTS
)
841 return octeon_droq_process_poll_pkts(oct
, droq
, arg
);
843 if (cmd
== POLL_EVENT_PENDING_PKTS
) {
844 u32 pkt_cnt
= atomic_read(&droq
->pkts_pending
);
846 return octeon_droq_process_packets(oct
, droq
, pkt_cnt
);
849 if (cmd
== POLL_EVENT_ENABLE_INTR
) {
853 /* Enable Pkt Interrupt */
854 switch (oct
->chip_id
) {
856 case OCTEON_CN68XX
: {
857 struct octeon_cn6xxx
*cn6xxx
=
858 (struct octeon_cn6xxx
*)oct
->chip
;
860 (&cn6xxx
->lock_for_droq_int_enb_reg
, flags
);
863 CN6XXX_SLI_PKT_TIME_INT_ENB
);
864 value
|= (1 << q_no
);
865 octeon_write_csr(oct
,
866 CN6XXX_SLI_PKT_TIME_INT_ENB
,
870 CN6XXX_SLI_PKT_CNT_INT_ENB
);
871 value
|= (1 << q_no
);
872 octeon_write_csr(oct
,
873 CN6XXX_SLI_PKT_CNT_INT_ENB
,
876 /* don't bother flushing the enables */
878 spin_unlock_irqrestore
879 (&cn6xxx
->lock_for_droq_int_enb_reg
, flags
);
883 case OCTEON_CN23XX_PF_VID
: {
884 lio_enable_irq(oct
->droq
[q_no
], oct
->instr_queue
[q_no
]);
888 case OCTEON_CN23XX_VF_VID
:
889 lio_enable_irq(oct
->droq
[q_no
], oct
->instr_queue
[q_no
]);
895 dev_err(&oct
->pci_dev
->dev
, "%s Unknown command: %d\n", __func__
, cmd
);
899 int octeon_register_droq_ops(struct octeon_device
*oct
, u32 q_no
,
900 struct octeon_droq_ops
*ops
)
902 struct octeon_droq
*droq
;
904 struct octeon_config
*oct_cfg
= NULL
;
906 oct_cfg
= octeon_get_conf(oct
);
912 dev_err(&oct
->pci_dev
->dev
, "%s: droq_ops pointer is NULL\n",
917 if (q_no
>= CFG_GET_OQ_MAX_Q(oct_cfg
)) {
918 dev_err(&oct
->pci_dev
->dev
, "%s: droq id (%d) exceeds MAX (%d)\n",
919 __func__
, q_no
, (oct
->num_oqs
- 1));
923 droq
= oct
->droq
[q_no
];
925 spin_lock_irqsave(&droq
->lock
, flags
);
927 memcpy(&droq
->ops
, ops
, sizeof(struct octeon_droq_ops
));
929 spin_unlock_irqrestore(&droq
->lock
, flags
);
934 int octeon_unregister_droq_ops(struct octeon_device
*oct
, u32 q_no
)
937 struct octeon_droq
*droq
;
938 struct octeon_config
*oct_cfg
= NULL
;
940 oct_cfg
= octeon_get_conf(oct
);
945 if (q_no
>= CFG_GET_OQ_MAX_Q(oct_cfg
)) {
946 dev_err(&oct
->pci_dev
->dev
, "%s: droq id (%d) exceeds MAX (%d)\n",
947 __func__
, q_no
, oct
->num_oqs
- 1);
951 droq
= oct
->droq
[q_no
];
954 dev_info(&oct
->pci_dev
->dev
,
955 "Droq id (%d) not available.\n", q_no
);
959 spin_lock_irqsave(&droq
->lock
, flags
);
961 droq
->ops
.fptr
= NULL
;
962 droq
->ops
.farg
= NULL
;
963 droq
->ops
.drop_on_max
= 0;
965 spin_unlock_irqrestore(&droq
->lock
, flags
);
970 int octeon_create_droq(struct octeon_device
*oct
,
971 u32 q_no
, u32 num_descs
,
972 u32 desc_size
, void *app_ctx
)
974 struct octeon_droq
*droq
;
975 int numa_node
= dev_to_node(&oct
->pci_dev
->dev
);
977 if (oct
->droq
[q_no
]) {
978 dev_dbg(&oct
->pci_dev
->dev
, "Droq already in use. Cannot create droq %d again\n",
983 /* Allocate the DS for the new droq. */
984 droq
= vmalloc_node(sizeof(*droq
), numa_node
);
986 droq
= vmalloc(sizeof(*droq
));
990 memset(droq
, 0, sizeof(struct octeon_droq
));
992 /*Disable the pkt o/p for this Q */
993 octeon_set_droq_pkt_op(oct
, q_no
, 0);
994 oct
->droq
[q_no
] = droq
;
996 /* Initialize the Droq */
997 if (octeon_init_droq(oct
, q_no
, num_descs
, desc_size
, app_ctx
)) {
998 vfree(oct
->droq
[q_no
]);
999 oct
->droq
[q_no
] = NULL
;
1005 dev_dbg(&oct
->pci_dev
->dev
, "%s: Total number of OQ: %d\n", __func__
,
1008 /* Global Droq register settings */
1010 /* As of now not required, as setting are done for all 32 Droqs at