1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2016 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
18 **********************************************************************/
19 #include <linux/pci.h>
20 #include <linux/netdevice.h>
21 #include <linux/vmalloc.h>
22 #include "liquidio_common.h"
23 #include "octeon_droq.h"
24 #include "octeon_iq.h"
25 #include "response_manager.h"
26 #include "octeon_device.h"
27 #include "octeon_main.h"
28 #include "octeon_network.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
33 struct iq_post_status
{
38 static void check_db_timeout(struct work_struct
*work
);
39 static void __check_db_timeout(struct octeon_device
*oct
, u64 iq_no
);
41 static void (*reqtype_free_fn
[MAX_OCTEON_DEVICES
][REQTYPE_LAST
+ 1]) (void *);
43 static inline int IQ_INSTR_MODE_64B(struct octeon_device
*oct
, int iq_no
)
45 struct octeon_instr_queue
*iq
=
46 (struct octeon_instr_queue
*)oct
->instr_queue
[iq_no
];
50 #define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no))
52 /* Define this to return the request status comaptible to old code */
53 /*#define OCTEON_USE_OLD_REQ_STATUS*/
55 /* Return 0 on success, 1 on failure */
56 int octeon_init_instr_queue(struct octeon_device
*oct
,
57 union oct_txpciq txpciq
,
60 struct octeon_instr_queue
*iq
;
61 struct octeon_iq_config
*conf
= NULL
;
62 u32 iq_no
= (u32
)txpciq
.s
.q_no
;
64 struct cavium_wq
*db_wq
;
65 int numa_node
= dev_to_node(&oct
->pci_dev
->dev
);
67 if (OCTEON_CN6XXX(oct
))
68 conf
= &(CFG_GET_IQ_CFG(CHIP_CONF(oct
, cn6xxx
)));
69 else if (OCTEON_CN23XX_PF(oct
))
70 conf
= &(CFG_GET_IQ_CFG(CHIP_CONF(oct
, cn23xx_pf
)));
71 else if (OCTEON_CN23XX_VF(oct
))
72 conf
= &(CFG_GET_IQ_CFG(CHIP_CONF(oct
, cn23xx_vf
)));
75 dev_err(&oct
->pci_dev
->dev
, "Unsupported Chip %x\n",
80 q_size
= (u32
)conf
->instr_type
* num_descs
;
82 iq
= oct
->instr_queue
[iq_no
];
86 iq
->base_addr
= lio_dma_alloc(oct
, q_size
, &iq
->base_addr_dma
);
88 dev_err(&oct
->pci_dev
->dev
, "Cannot allocate memory for instr queue %d\n",
93 iq
->max_count
= num_descs
;
95 /* Initialize a list to holds requests that have been posted to Octeon
96 * but has yet to be fetched by octeon
98 iq
->request_list
= vmalloc_node((sizeof(*iq
->request_list
) * num_descs
),
100 if (!iq
->request_list
)
102 vmalloc(array_size(num_descs
,
103 sizeof(*iq
->request_list
)));
104 if (!iq
->request_list
) {
105 lio_dma_free(oct
, q_size
, iq
->base_addr
, iq
->base_addr_dma
);
106 dev_err(&oct
->pci_dev
->dev
, "Alloc failed for IQ[%d] nr free list\n",
111 memset(iq
->request_list
, 0, sizeof(*iq
->request_list
) * num_descs
);
113 dev_dbg(&oct
->pci_dev
->dev
, "IQ[%d]: base: %p basedma: %llx count: %d\n",
114 iq_no
, iq
->base_addr
, iq
->base_addr_dma
, iq
->max_count
);
116 iq
->txpciq
.u64
= txpciq
.u64
;
117 iq
->fill_threshold
= (u32
)conf
->db_min
;
119 iq
->host_write_index
= 0;
120 iq
->octeon_read_index
= 0;
122 iq
->last_db_time
= 0;
123 iq
->do_auto_flush
= 1;
124 iq
->db_timeout
= (u32
)conf
->db_timeout
;
125 atomic_set(&iq
->instr_pending
, 0);
127 /* Initialize the spinlock for this instruction queue */
128 spin_lock_init(&iq
->lock
);
129 spin_lock_init(&iq
->post_lock
);
131 spin_lock_init(&iq
->iq_flush_running_lock
);
133 oct
->io_qmask
.iq
|= BIT_ULL(iq_no
);
135 /* Set the 32B/64B mode for each input queue */
136 oct
->io_qmask
.iq64B
|= ((conf
->instr_type
== 64) << iq_no
);
137 iq
->iqcmd_64B
= (conf
->instr_type
== 64);
139 oct
->fn_list
.setup_iq_regs(oct
, iq_no
);
141 oct
->check_db_wq
[iq_no
].wq
= alloc_workqueue("check_iq_db",
144 if (!oct
->check_db_wq
[iq_no
].wq
) {
145 vfree(iq
->request_list
);
146 iq
->request_list
= NULL
;
147 lio_dma_free(oct
, q_size
, iq
->base_addr
, iq
->base_addr_dma
);
148 dev_err(&oct
->pci_dev
->dev
, "check db wq create failed for iq %d\n",
153 db_wq
= &oct
->check_db_wq
[iq_no
];
155 INIT_DELAYED_WORK(&db_wq
->wk
.work
, check_db_timeout
);
156 db_wq
->wk
.ctxptr
= oct
;
157 db_wq
->wk
.ctxul
= iq_no
;
158 queue_delayed_work(db_wq
->wq
, &db_wq
->wk
.work
, msecs_to_jiffies(1));
163 int octeon_delete_instr_queue(struct octeon_device
*oct
, u32 iq_no
)
165 u64 desc_size
= 0, q_size
;
166 struct octeon_instr_queue
*iq
= oct
->instr_queue
[iq_no
];
168 cancel_delayed_work_sync(&oct
->check_db_wq
[iq_no
].wk
.work
);
169 destroy_workqueue(oct
->check_db_wq
[iq_no
].wq
);
171 if (OCTEON_CN6XXX(oct
))
173 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct
, cn6xxx
));
174 else if (OCTEON_CN23XX_PF(oct
))
176 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct
, cn23xx_pf
));
177 else if (OCTEON_CN23XX_VF(oct
))
179 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct
, cn23xx_vf
));
181 vfree(iq
->request_list
);
184 q_size
= iq
->max_count
* desc_size
;
185 lio_dma_free(oct
, (u32
)q_size
, iq
->base_addr
,
187 oct
->io_qmask
.iq
&= ~(1ULL << iq_no
);
188 vfree(oct
->instr_queue
[iq_no
]);
189 oct
->instr_queue
[iq_no
] = NULL
;
196 /* Return 0 on success, 1 on failure */
197 int octeon_setup_iq(struct octeon_device
*oct
,
200 union oct_txpciq txpciq
,
204 u32 iq_no
= (u32
)txpciq
.s
.q_no
;
205 int numa_node
= dev_to_node(&oct
->pci_dev
->dev
);
207 if (oct
->instr_queue
[iq_no
]) {
208 dev_dbg(&oct
->pci_dev
->dev
, "IQ is in use. Cannot create the IQ: %d again\n",
210 oct
->instr_queue
[iq_no
]->txpciq
.u64
= txpciq
.u64
;
211 oct
->instr_queue
[iq_no
]->app_ctx
= app_ctx
;
214 oct
->instr_queue
[iq_no
] =
215 vmalloc_node(sizeof(struct octeon_instr_queue
), numa_node
);
216 if (!oct
->instr_queue
[iq_no
])
217 oct
->instr_queue
[iq_no
] =
218 vmalloc(sizeof(struct octeon_instr_queue
));
219 if (!oct
->instr_queue
[iq_no
])
222 memset(oct
->instr_queue
[iq_no
], 0,
223 sizeof(struct octeon_instr_queue
));
225 oct
->instr_queue
[iq_no
]->q_index
= q_index
;
226 oct
->instr_queue
[iq_no
]->app_ctx
= app_ctx
;
227 oct
->instr_queue
[iq_no
]->ifidx
= ifidx
;
229 if (octeon_init_instr_queue(oct
, txpciq
, num_descs
)) {
230 vfree(oct
->instr_queue
[iq_no
]);
231 oct
->instr_queue
[iq_no
] = NULL
;
236 if (oct
->fn_list
.enable_io_queues(oct
))
242 int lio_wait_for_instr_fetch(struct octeon_device
*oct
)
244 int i
, retry
= 1000, pending
, instr_cnt
= 0;
249 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES(oct
); i
++) {
250 if (!(oct
->io_qmask
.iq
& BIT_ULL(i
)))
253 atomic_read(&oct
->instr_queue
[i
]->instr_pending
);
255 __check_db_timeout(oct
, i
);
256 instr_cnt
+= pending
;
262 schedule_timeout_uninterruptible(1);
264 } while (retry
-- && instr_cnt
);
270 ring_doorbell(struct octeon_device
*oct
, struct octeon_instr_queue
*iq
)
272 if (atomic_read(&oct
->status
) == OCT_DEV_RUNNING
) {
273 writel(iq
->fill_cnt
, iq
->doorbell_reg
);
274 /* make sure doorbell write goes through */
277 iq
->last_db_time
= jiffies
;
283 octeon_ring_doorbell_locked(struct octeon_device
*oct
, u32 iq_no
)
285 struct octeon_instr_queue
*iq
;
287 iq
= oct
->instr_queue
[iq_no
];
288 spin_lock(&iq
->post_lock
);
290 ring_doorbell(oct
, iq
);
291 spin_unlock(&iq
->post_lock
);
294 static inline void __copy_cmd_into_iq(struct octeon_instr_queue
*iq
,
299 cmdsize
= ((iq
->iqcmd_64B
) ? 64 : 32);
300 iqptr
= iq
->base_addr
+ (cmdsize
* iq
->host_write_index
);
302 memcpy(iqptr
, cmd
, cmdsize
);
305 static inline struct iq_post_status
306 __post_command2(struct octeon_instr_queue
*iq
, u8
*cmd
)
308 struct iq_post_status st
;
310 st
.status
= IQ_SEND_OK
;
312 /* This ensures that the read index does not wrap around to the same
313 * position if queue gets full before Octeon could fetch any instr.
315 if (atomic_read(&iq
->instr_pending
) >= (s32
)(iq
->max_count
- 1)) {
316 st
.status
= IQ_SEND_FAILED
;
321 if (atomic_read(&iq
->instr_pending
) >= (s32
)(iq
->max_count
- 2))
322 st
.status
= IQ_SEND_STOP
;
324 __copy_cmd_into_iq(iq
, cmd
);
326 /* "index" is returned, host_write_index is modified. */
327 st
.index
= iq
->host_write_index
;
328 iq
->host_write_index
= incr_index(iq
->host_write_index
, 1,
332 /* Flush the command into memory. We need to be sure the data is in
333 * memory before indicating that the instruction is pending.
337 atomic_inc(&iq
->instr_pending
);
343 octeon_register_reqtype_free_fn(struct octeon_device
*oct
, int reqtype
,
346 if (reqtype
> REQTYPE_LAST
) {
347 dev_err(&oct
->pci_dev
->dev
, "%s: Invalid reqtype: %d\n",
352 reqtype_free_fn
[oct
->octeon_id
][reqtype
] = fn
;
358 __add_to_request_list(struct octeon_instr_queue
*iq
,
359 int idx
, void *buf
, int reqtype
)
361 iq
->request_list
[idx
].buf
= buf
;
362 iq
->request_list
[idx
].reqtype
= reqtype
;
365 /* Can only run in process context */
367 lio_process_iq_request_list(struct octeon_device
*oct
,
368 struct octeon_instr_queue
*iq
, u32 napi_budget
)
370 struct cavium_wq
*cwq
= &oct
->dma_comp_wq
;
373 u32 old
= iq
->flush_index
;
375 unsigned int pkts_compl
= 0, bytes_compl
= 0;
376 struct octeon_soft_command
*sc
;
377 struct octeon_instr_irh
*irh
;
380 while (old
!= iq
->octeon_read_index
) {
381 reqtype
= iq
->request_list
[old
].reqtype
;
382 buf
= iq
->request_list
[old
].buf
;
384 if (reqtype
== REQTYPE_NONE
)
387 octeon_update_tx_completion_counters(buf
, reqtype
, &pkts_compl
,
391 case REQTYPE_NORESP_NET
:
392 case REQTYPE_NORESP_NET_SG
:
393 case REQTYPE_RESP_NET_SG
:
394 reqtype_free_fn
[oct
->octeon_id
][reqtype
](buf
);
396 case REQTYPE_RESP_NET
:
397 case REQTYPE_SOFT_COMMAND
:
400 if (OCTEON_CN23XX_PF(oct
) || OCTEON_CN23XX_VF(oct
))
401 irh
= (struct octeon_instr_irh
*)
404 irh
= (struct octeon_instr_irh
*)
407 /* We're expecting a response from Octeon.
408 * It's up to lio_process_ordered_list() to
409 * process sc. Add sc to the ordered soft
410 * command response list because we expect
411 * a response from Octeon.
415 [OCTEON_ORDERED_SC_LIST
].lock
,
417 atomic_inc(&oct
->response_list
418 [OCTEON_ORDERED_SC_LIST
].
420 list_add_tail(&sc
->node
, &oct
->response_list
421 [OCTEON_ORDERED_SC_LIST
].head
);
422 spin_unlock_irqrestore
424 [OCTEON_ORDERED_SC_LIST
].lock
,
428 /* This callback must not sleep */
429 sc
->callback(oct
, OCTEON_REQUEST_DONE
,
435 dev_err(&oct
->pci_dev
->dev
,
436 "%s Unknown reqtype: %d buf: %p at idx %d\n",
437 __func__
, reqtype
, buf
, old
);
440 iq
->request_list
[old
].buf
= NULL
;
441 iq
->request_list
[old
].reqtype
= 0;
445 old
= incr_index(old
, 1, iq
->max_count
);
447 if ((napi_budget
) && (inst_count
>= napi_budget
))
451 octeon_report_tx_completion_to_bql(iq
->app_ctx
, pkts_compl
,
453 iq
->flush_index
= old
;
455 if (atomic_read(&oct
->response_list
456 [OCTEON_ORDERED_SC_LIST
].pending_req_count
))
457 queue_delayed_work(cwq
->wq
, &cwq
->wk
.work
, msecs_to_jiffies(1));
462 /* Can only be called from process context */
464 octeon_flush_iq(struct octeon_device
*oct
, struct octeon_instr_queue
*iq
,
467 u32 inst_processed
= 0;
468 u32 tot_inst_processed
= 0;
471 if (!spin_trylock(&iq
->iq_flush_running_lock
))
474 spin_lock_bh(&iq
->lock
);
476 iq
->octeon_read_index
= oct
->fn_list
.update_iq_read_idx(iq
);
479 /* Process any outstanding IQ packets. */
480 if (iq
->flush_index
== iq
->octeon_read_index
)
485 lio_process_iq_request_list(oct
, iq
,
490 lio_process_iq_request_list(oct
, iq
, 0);
492 if (inst_processed
) {
493 atomic_sub(inst_processed
, &iq
->instr_pending
);
494 iq
->stats
.instr_processed
+= inst_processed
;
497 tot_inst_processed
+= inst_processed
;
498 } while (tot_inst_processed
< napi_budget
);
500 if (napi_budget
&& (tot_inst_processed
>= napi_budget
))
503 iq
->last_db_time
= jiffies
;
505 spin_unlock_bh(&iq
->lock
);
507 spin_unlock(&iq
->iq_flush_running_lock
);
512 /* Process instruction queue after timeout.
513 * This routine gets called from a workqueue or when removing the module.
515 static void __check_db_timeout(struct octeon_device
*oct
, u64 iq_no
)
517 struct octeon_instr_queue
*iq
;
523 iq
= oct
->instr_queue
[iq_no
];
527 /* return immediately, if no work pending */
528 if (!atomic_read(&iq
->instr_pending
))
530 /* If jiffies - last_db_time < db_timeout do nothing */
531 next_time
= iq
->last_db_time
+ iq
->db_timeout
;
532 if (!time_after(jiffies
, (unsigned long)next_time
))
534 iq
->last_db_time
= jiffies
;
536 /* Flush the instruction queue */
537 octeon_flush_iq(oct
, iq
, 0);
539 lio_enable_irq(NULL
, iq
);
542 /* Called by the Poll thread at regular intervals to check the instruction
543 * queue for commands to be posted and for commands that were fetched by Octeon.
545 static void check_db_timeout(struct work_struct
*work
)
547 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
548 struct octeon_device
*oct
= (struct octeon_device
*)wk
->ctxptr
;
549 u64 iq_no
= wk
->ctxul
;
550 struct cavium_wq
*db_wq
= &oct
->check_db_wq
[iq_no
];
553 __check_db_timeout(oct
, iq_no
);
554 queue_delayed_work(db_wq
->wq
, &db_wq
->wk
.work
, msecs_to_jiffies(delay
));
558 octeon_send_command(struct octeon_device
*oct
, u32 iq_no
,
559 u32 force_db
, void *cmd
, void *buf
,
560 u32 datasize
, u32 reqtype
)
563 struct iq_post_status st
;
564 struct octeon_instr_queue
*iq
= oct
->instr_queue
[iq_no
];
566 /* Get the lock and prevent other tasks and tx interrupt handler from
569 spin_lock_bh(&iq
->post_lock
);
571 st
= __post_command2(iq
, cmd
);
573 if (st
.status
!= IQ_SEND_FAILED
) {
574 xmit_stopped
= octeon_report_sent_bytes_to_bql(buf
, reqtype
);
575 __add_to_request_list(iq
, st
.index
, buf
, reqtype
);
576 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, bytes_sent
, datasize
);
577 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, instr_posted
, 1);
579 if (iq
->fill_cnt
>= MAX_OCTEON_FILL_COUNT
|| force_db
||
580 xmit_stopped
|| st
.status
== IQ_SEND_STOP
)
581 ring_doorbell(oct
, iq
);
583 INCR_INSTRQUEUE_PKT_COUNT(oct
, iq_no
, instr_dropped
, 1);
586 spin_unlock_bh(&iq
->post_lock
);
588 /* This is only done here to expedite packets being flushed
589 * for cases where there are no IQ completion interrupts.
596 octeon_prepare_soft_command(struct octeon_device
*oct
,
597 struct octeon_soft_command
*sc
,
604 struct octeon_config
*oct_cfg
;
605 struct octeon_instr_ih2
*ih2
;
606 struct octeon_instr_ih3
*ih3
;
607 struct octeon_instr_pki_ih3
*pki_ih3
;
608 struct octeon_instr_irh
*irh
;
609 struct octeon_instr_rdp
*rdp
;
611 WARN_ON(opcode
> 15);
612 WARN_ON(subcode
> 127);
614 oct_cfg
= octeon_get_conf(oct
);
616 if (OCTEON_CN23XX_PF(oct
) || OCTEON_CN23XX_VF(oct
)) {
617 ih3
= (struct octeon_instr_ih3
*)&sc
->cmd
.cmd3
.ih3
;
619 ih3
->pkind
= oct
->instr_queue
[sc
->iq_no
]->txpciq
.s
.pkind
;
621 pki_ih3
= (struct octeon_instr_pki_ih3
*)&sc
->cmd
.cmd3
.pki_ih3
;
627 oct
->instr_queue
[sc
->iq_no
]->txpciq
.s
.use_qpg
;
629 pki_ih3
->tag
= LIO_CONTROL
;
630 pki_ih3
->tagtype
= ATOMIC_TAG
;
632 oct
->instr_queue
[sc
->iq_no
]->txpciq
.s
.ctrl_qpg
;
638 ih3
->dlengsz
= sc
->datasize
;
640 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd3
.irh
;
641 irh
->opcode
= opcode
;
642 irh
->subcode
= subcode
;
644 /* opcode/subcode specific parameters (ossp) */
645 irh
->ossp
= irh_ossp
;
646 sc
->cmd
.cmd3
.ossp
[0] = ossp0
;
647 sc
->cmd
.cmd3
.ossp
[1] = ossp1
;
650 rdp
= (struct octeon_instr_rdp
*)&sc
->cmd
.cmd3
.rdp
;
651 rdp
->pcie_port
= oct
->pcie_port
;
652 rdp
->rlen
= sc
->rdatasize
;
656 /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
657 ih3
->fsz
= LIO_SOFTCMDRESP_IH3
;
661 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
662 ih3
->fsz
= LIO_PCICMD_O3
;
666 ih2
= (struct octeon_instr_ih2
*)&sc
->cmd
.cmd2
.ih2
;
667 ih2
->tagtype
= ATOMIC_TAG
;
668 ih2
->tag
= LIO_CONTROL
;
670 ih2
->grp
= CFG_GET_CTRL_Q_GRP(oct_cfg
);
673 ih2
->dlengsz
= sc
->datasize
;
677 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd2
.irh
;
678 irh
->opcode
= opcode
;
679 irh
->subcode
= subcode
;
681 /* opcode/subcode specific parameters (ossp) */
682 irh
->ossp
= irh_ossp
;
683 sc
->cmd
.cmd2
.ossp
[0] = ossp0
;
684 sc
->cmd
.cmd2
.ossp
[1] = ossp1
;
687 rdp
= (struct octeon_instr_rdp
*)&sc
->cmd
.cmd2
.rdp
;
688 rdp
->pcie_port
= oct
->pcie_port
;
689 rdp
->rlen
= sc
->rdatasize
;
692 /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
693 ih2
->fsz
= LIO_SOFTCMDRESP_IH2
;
696 /* irh + ossp[0] + ossp[1] = 24 bytes */
697 ih2
->fsz
= LIO_PCICMD_O2
;
702 int octeon_send_soft_command(struct octeon_device
*oct
,
703 struct octeon_soft_command
*sc
)
705 struct octeon_instr_ih2
*ih2
;
706 struct octeon_instr_ih3
*ih3
;
707 struct octeon_instr_irh
*irh
;
710 if (OCTEON_CN23XX_PF(oct
) || OCTEON_CN23XX_VF(oct
)) {
711 ih3
= (struct octeon_instr_ih3
*)&sc
->cmd
.cmd3
.ih3
;
713 WARN_ON(!sc
->dmadptr
);
714 sc
->cmd
.cmd3
.dptr
= sc
->dmadptr
;
716 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd3
.irh
;
718 WARN_ON(!sc
->dmarptr
);
719 WARN_ON(!sc
->status_word
);
720 *sc
->status_word
= COMPLETION_WORD_INIT
;
721 sc
->cmd
.cmd3
.rptr
= sc
->dmarptr
;
723 len
= (u32
)ih3
->dlengsz
;
725 ih2
= (struct octeon_instr_ih2
*)&sc
->cmd
.cmd2
.ih2
;
727 WARN_ON(!sc
->dmadptr
);
728 sc
->cmd
.cmd2
.dptr
= sc
->dmadptr
;
730 irh
= (struct octeon_instr_irh
*)&sc
->cmd
.cmd2
.irh
;
732 WARN_ON(!sc
->dmarptr
);
733 WARN_ON(!sc
->status_word
);
734 *sc
->status_word
= COMPLETION_WORD_INIT
;
735 sc
->cmd
.cmd2
.rptr
= sc
->dmarptr
;
737 len
= (u32
)ih2
->dlengsz
;
741 sc
->timeout
= jiffies
+ sc
->wait_time
;
743 return (octeon_send_command(oct
, sc
->iq_no
, 1, &sc
->cmd
, sc
,
744 len
, REQTYPE_SOFT_COMMAND
));
747 int octeon_setup_sc_buffer_pool(struct octeon_device
*oct
)
751 struct octeon_soft_command
*sc
;
753 INIT_LIST_HEAD(&oct
->sc_buf_pool
.head
);
754 spin_lock_init(&oct
->sc_buf_pool
.lock
);
755 atomic_set(&oct
->sc_buf_pool
.alloc_buf_count
, 0);
757 for (i
= 0; i
< MAX_SOFT_COMMAND_BUFFERS
; i
++) {
758 sc
= (struct octeon_soft_command
*)
760 SOFT_COMMAND_BUFFER_SIZE
,
761 (dma_addr_t
*)&dma_addr
);
763 octeon_free_sc_buffer_pool(oct
);
767 sc
->dma_addr
= dma_addr
;
768 sc
->size
= SOFT_COMMAND_BUFFER_SIZE
;
770 list_add_tail(&sc
->node
, &oct
->sc_buf_pool
.head
);
776 int octeon_free_sc_buffer_pool(struct octeon_device
*oct
)
778 struct list_head
*tmp
, *tmp2
;
779 struct octeon_soft_command
*sc
;
781 spin_lock_bh(&oct
->sc_buf_pool
.lock
);
783 list_for_each_safe(tmp
, tmp2
, &oct
->sc_buf_pool
.head
) {
786 sc
= (struct octeon_soft_command
*)tmp
;
788 lio_dma_free(oct
, sc
->size
, sc
, sc
->dma_addr
);
791 INIT_LIST_HEAD(&oct
->sc_buf_pool
.head
);
793 spin_unlock_bh(&oct
->sc_buf_pool
.lock
);
798 struct octeon_soft_command
*octeon_alloc_soft_command(struct octeon_device
*oct
,
805 u32 offset
= sizeof(struct octeon_soft_command
);
806 struct octeon_soft_command
*sc
= NULL
;
807 struct list_head
*tmp
;
809 WARN_ON((offset
+ datasize
+ rdatasize
+ ctxsize
) >
810 SOFT_COMMAND_BUFFER_SIZE
);
812 spin_lock_bh(&oct
->sc_buf_pool
.lock
);
814 if (list_empty(&oct
->sc_buf_pool
.head
)) {
815 spin_unlock_bh(&oct
->sc_buf_pool
.lock
);
819 list_for_each(tmp
, &oct
->sc_buf_pool
.head
)
824 atomic_inc(&oct
->sc_buf_pool
.alloc_buf_count
);
826 spin_unlock_bh(&oct
->sc_buf_pool
.lock
);
828 sc
= (struct octeon_soft_command
*)tmp
;
830 dma_addr
= sc
->dma_addr
;
833 memset(sc
, 0, sc
->size
);
835 sc
->dma_addr
= dma_addr
;
839 sc
->ctxptr
= (u8
*)sc
+ offset
;
840 sc
->ctxsize
= ctxsize
;
843 /* Start data at 128 byte boundary */
844 offset
= (offset
+ ctxsize
+ 127) & 0xffffff80;
847 sc
->virtdptr
= (u8
*)sc
+ offset
;
848 sc
->dmadptr
= dma_addr
+ offset
;
849 sc
->datasize
= datasize
;
852 /* Start rdata at 128 byte boundary */
853 offset
= (offset
+ datasize
+ 127) & 0xffffff80;
856 WARN_ON(rdatasize
< 16);
857 sc
->virtrptr
= (u8
*)sc
+ offset
;
858 sc
->dmarptr
= dma_addr
+ offset
;
859 sc
->rdatasize
= rdatasize
;
860 sc
->status_word
= (u64
*)((u8
*)(sc
->virtrptr
) + rdatasize
- 8);
866 void octeon_free_soft_command(struct octeon_device
*oct
,
867 struct octeon_soft_command
*sc
)
869 spin_lock_bh(&oct
->sc_buf_pool
.lock
);
871 list_add_tail(&sc
->node
, &oct
->sc_buf_pool
.head
);
873 atomic_dec(&oct
->sc_buf_pool
.alloc_buf_count
);
875 spin_unlock_bh(&oct
->sc_buf_pool
.lock
);