2 * Copyright (C) 2005 - 2009 ServerEngines
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
10 * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
12 * Contact Information:
13 * linux-drivers@serverengines.com
16 * 209 N. Fair Oaks Ave
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/interrupt.h>
23 #include <linux/blkdev.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/kernel.h>
27 #include <linux/semaphore.h>
29 #include <scsi/libiscsi.h>
30 #include <scsi/scsi_transport_iscsi.h>
31 #include <scsi/scsi_transport.h>
32 #include <scsi/scsi_cmnd.h>
33 #include <scsi/scsi_device.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi.h>
40 static unsigned int be_iopoll_budget
= 10;
41 static unsigned int be_max_phys_size
= 64;
42 static unsigned int enable_msix
= 1;
43 static unsigned int ring_mode
;
45 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
46 MODULE_DESCRIPTION(DRV_DESC
" " BUILD_STR
);
47 MODULE_AUTHOR("ServerEngines Corporation");
48 MODULE_LICENSE("GPL");
49 module_param(be_iopoll_budget
, int, 0);
50 module_param(enable_msix
, int, 0);
51 module_param(be_max_phys_size
, uint
, S_IRUGO
);
52 MODULE_PARM_DESC(be_max_phys_size
, "Maximum Size (In Kilobytes) of physically"
53 "contiguous memory that can be allocated."
56 static int beiscsi_slave_configure(struct scsi_device
*sdev
)
58 blk_queue_max_segment_size(sdev
->request_queue
, 65536);
62 /*------------------- PCI Driver operations and data ----------------- */
63 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table
) = {
64 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID1
) },
65 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID1
) },
66 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID2
) },
67 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID3
) },
68 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID4
) },
71 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
73 static struct scsi_host_template beiscsi_sht
= {
74 .module
= THIS_MODULE
,
75 .name
= "ServerEngines 10Gbe open-iscsi Initiator Driver",
76 .proc_name
= DRV_NAME
,
77 .queuecommand
= iscsi_queuecommand
,
78 .eh_abort_handler
= iscsi_eh_abort
,
79 .change_queue_depth
= iscsi_change_queue_depth
,
80 .slave_configure
= beiscsi_slave_configure
,
81 .target_alloc
= iscsi_target_alloc
,
82 .eh_device_reset_handler
= iscsi_eh_device_reset
,
83 .eh_target_reset_handler
= iscsi_eh_target_reset
,
84 .sg_tablesize
= BEISCSI_SGLIST_ELEMENTS
,
85 .can_queue
= BE2_IO_DEPTH
,
87 .max_sectors
= BEISCSI_MAX_SECTORS
,
88 .cmd_per_lun
= BEISCSI_CMD_PER_LUN
,
89 .use_clustering
= ENABLE_CLUSTERING
,
92 static struct scsi_transport_template
*beiscsi_scsi_transport
;
94 static struct beiscsi_hba
*beiscsi_hba_alloc(struct pci_dev
*pcidev
)
96 struct beiscsi_hba
*phba
;
97 struct Scsi_Host
*shost
;
99 shost
= iscsi_host_alloc(&beiscsi_sht
, sizeof(*phba
), 0);
101 dev_err(&pcidev
->dev
, "beiscsi_hba_alloc -"
102 "iscsi_host_alloc failed \n");
105 shost
->dma_boundary
= pcidev
->dma_mask
;
106 shost
->max_id
= BE2_MAX_SESSIONS
;
107 shost
->max_channel
= 0;
108 shost
->max_cmd_len
= BEISCSI_MAX_CMD_LEN
;
109 shost
->max_lun
= BEISCSI_NUM_MAX_LUN
;
110 shost
->transportt
= beiscsi_scsi_transport
;
111 phba
= iscsi_host_priv(shost
);
112 memset(phba
, 0, sizeof(*phba
));
114 phba
->pcidev
= pci_dev_get(pcidev
);
115 pci_set_drvdata(pcidev
, phba
);
117 if (iscsi_host_add(shost
, &phba
->pcidev
->dev
))
122 pci_dev_put(phba
->pcidev
);
123 iscsi_host_free(phba
->shost
);
127 static void beiscsi_unmap_pci_function(struct beiscsi_hba
*phba
)
130 iounmap(phba
->csr_va
);
134 iounmap(phba
->db_va
);
138 iounmap(phba
->pci_va
);
143 static int beiscsi_map_pci_bars(struct beiscsi_hba
*phba
,
144 struct pci_dev
*pcidev
)
148 addr
= ioremap_nocache(pci_resource_start(pcidev
, 2),
149 pci_resource_len(pcidev
, 2));
152 phba
->ctrl
.csr
= addr
;
154 phba
->csr_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 2);
156 addr
= ioremap_nocache(pci_resource_start(pcidev
, 4), 128 * 1024);
159 phba
->ctrl
.db
= addr
;
161 phba
->db_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 4);
163 addr
= ioremap_nocache(pci_resource_start(pcidev
, 1),
164 pci_resource_len(pcidev
, 1));
167 phba
->ctrl
.pcicfg
= addr
;
169 phba
->pci_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 1);
173 beiscsi_unmap_pci_function(phba
);
177 static int beiscsi_enable_pci(struct pci_dev
*pcidev
)
181 ret
= pci_enable_device(pcidev
);
183 dev_err(&pcidev
->dev
, "beiscsi_enable_pci - enable device "
184 "failed. Returning -ENODEV\n");
188 pci_set_master(pcidev
);
189 if (pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(64))) {
190 ret
= pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(32));
192 dev_err(&pcidev
->dev
, "Could not set PCI DMA Mask\n");
193 pci_disable_device(pcidev
);
200 static int be_ctrl_init(struct beiscsi_hba
*phba
, struct pci_dev
*pdev
)
202 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
203 struct be_dma_mem
*mbox_mem_alloc
= &ctrl
->mbox_mem_alloced
;
204 struct be_dma_mem
*mbox_mem_align
= &ctrl
->mbox_mem
;
208 status
= beiscsi_map_pci_bars(phba
, pdev
);
211 mbox_mem_alloc
->size
= sizeof(struct be_mcc_mailbox
) + 16;
212 mbox_mem_alloc
->va
= pci_alloc_consistent(pdev
,
213 mbox_mem_alloc
->size
,
214 &mbox_mem_alloc
->dma
);
215 if (!mbox_mem_alloc
->va
) {
216 beiscsi_unmap_pci_function(phba
);
221 mbox_mem_align
->size
= sizeof(struct be_mcc_mailbox
);
222 mbox_mem_align
->va
= PTR_ALIGN(mbox_mem_alloc
->va
, 16);
223 mbox_mem_align
->dma
= PTR_ALIGN(mbox_mem_alloc
->dma
, 16);
224 memset(mbox_mem_align
->va
, 0, sizeof(struct be_mcc_mailbox
));
225 spin_lock_init(&ctrl
->mbox_lock
);
226 spin_lock_init(&phba
->ctrl
.mcc_lock
);
227 spin_lock_init(&phba
->ctrl
.mcc_cq_lock
);
232 static void beiscsi_get_params(struct beiscsi_hba
*phba
)
234 phba
->params
.ios_per_ctrl
= (phba
->fw_config
.iscsi_icd_count
235 - (phba
->fw_config
.iscsi_cid_count
238 phba
->params
.cxns_per_ctrl
= phba
->fw_config
.iscsi_cid_count
;
239 phba
->params
.asyncpdus_per_ctrl
= phba
->fw_config
.iscsi_cid_count
;;
240 phba
->params
.icds_per_ctrl
= phba
->fw_config
.iscsi_icd_count
;;
241 phba
->params
.num_sge_per_io
= BE2_SGE
;
242 phba
->params
.defpdu_hdr_sz
= BE2_DEFPDU_HDR_SZ
;
243 phba
->params
.defpdu_data_sz
= BE2_DEFPDU_DATA_SZ
;
244 phba
->params
.eq_timer
= 64;
245 phba
->params
.num_eq_entries
=
246 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
247 + BE2_TMFS
) / 512) + 1) * 512;
248 phba
->params
.num_eq_entries
= (phba
->params
.num_eq_entries
< 1024)
249 ? 1024 : phba
->params
.num_eq_entries
;
250 SE_DEBUG(DBG_LVL_8
, "phba->params.num_eq_entries=%d \n",
251 phba
->params
.num_eq_entries
);
252 phba
->params
.num_cq_entries
=
253 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
254 + BE2_TMFS
) / 512) + 1) * 512;
255 phba
->params
.wrbs_per_cxn
= 256;
258 static void hwi_ring_eq_db(struct beiscsi_hba
*phba
,
259 unsigned int id
, unsigned int clr_interrupt
,
260 unsigned int num_processed
,
261 unsigned char rearm
, unsigned char event
)
264 val
|= id
& DB_EQ_RING_ID_MASK
;
266 val
|= 1 << DB_EQ_REARM_SHIFT
;
268 val
|= 1 << DB_EQ_CLR_SHIFT
;
270 val
|= 1 << DB_EQ_EVNT_SHIFT
;
271 val
|= num_processed
<< DB_EQ_NUM_POPPED_SHIFT
;
272 iowrite32(val
, phba
->db_va
+ DB_EQ_OFFSET
);
276 * be_isr_mcc - The isr routine of the driver.
278 * @dev_id: Pointer to host adapter structure
280 static irqreturn_t
be_isr_mcc(int irq
, void *dev_id
)
282 struct beiscsi_hba
*phba
;
283 struct be_eq_entry
*eqe
= NULL
;
284 struct be_queue_info
*eq
;
285 struct be_queue_info
*mcc
;
286 unsigned int num_eq_processed
;
287 struct be_eq_obj
*pbe_eq
;
293 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
294 eqe
= queue_tail_node(eq
);
296 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
298 num_eq_processed
= 0;
300 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
302 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
304 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
305 spin_lock_irqsave(&phba
->isr_lock
, flags
);
306 phba
->todo_mcc_cq
= 1;
307 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
309 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
311 eqe
= queue_tail_node(eq
);
314 if (phba
->todo_mcc_cq
)
315 queue_work(phba
->wq
, &phba
->work_cqs
);
316 if (num_eq_processed
)
317 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
323 * be_isr_msix - The isr routine of the driver.
325 * @dev_id: Pointer to host adapter structure
327 static irqreturn_t
be_isr_msix(int irq
, void *dev_id
)
329 struct beiscsi_hba
*phba
;
330 struct be_eq_entry
*eqe
= NULL
;
331 struct be_queue_info
*eq
;
332 struct be_queue_info
*cq
;
333 unsigned int num_eq_processed
;
334 struct be_eq_obj
*pbe_eq
;
340 eqe
= queue_tail_node(eq
);
342 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
345 num_eq_processed
= 0;
346 if (blk_iopoll_enabled
) {
347 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
349 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
350 blk_iopoll_sched(&pbe_eq
->iopoll
);
352 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
354 eqe
= queue_tail_node(eq
);
357 if (num_eq_processed
)
358 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 0, 1);
362 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
364 spin_lock_irqsave(&phba
->isr_lock
, flags
);
366 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
367 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
369 eqe
= queue_tail_node(eq
);
373 queue_work(phba
->wq
, &phba
->work_cqs
);
375 if (num_eq_processed
)
376 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
383 * be_isr - The isr routine of the driver.
385 * @dev_id: Pointer to host adapter structure
387 static irqreturn_t
be_isr(int irq
, void *dev_id
)
389 struct beiscsi_hba
*phba
;
390 struct hwi_controller
*phwi_ctrlr
;
391 struct hwi_context_memory
*phwi_context
;
392 struct be_eq_entry
*eqe
= NULL
;
393 struct be_queue_info
*eq
;
394 struct be_queue_info
*cq
;
395 struct be_queue_info
*mcc
;
396 unsigned long flags
, index
;
397 unsigned int num_mcceq_processed
, num_ioeq_processed
;
398 struct be_ctrl_info
*ctrl
;
399 struct be_eq_obj
*pbe_eq
;
404 isr
= ioread32(ctrl
->csr
+ CEV_ISR0_OFFSET
+
405 (PCI_FUNC(ctrl
->pdev
->devfn
) * CEV_ISR_SIZE
));
409 phwi_ctrlr
= phba
->phwi_ctrlr
;
410 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
411 pbe_eq
= &phwi_context
->be_eq
[0];
413 eq
= &phwi_context
->be_eq
[0].q
;
414 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
416 eqe
= queue_tail_node(eq
);
418 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
420 num_ioeq_processed
= 0;
421 num_mcceq_processed
= 0;
422 if (blk_iopoll_enabled
) {
423 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
425 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
427 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
428 spin_lock_irqsave(&phba
->isr_lock
, flags
);
429 phba
->todo_mcc_cq
= 1;
430 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
431 num_mcceq_processed
++;
433 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
434 blk_iopoll_sched(&pbe_eq
->iopoll
);
435 num_ioeq_processed
++;
437 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
439 eqe
= queue_tail_node(eq
);
441 if (num_ioeq_processed
|| num_mcceq_processed
) {
442 if (phba
->todo_mcc_cq
)
443 queue_work(phba
->wq
, &phba
->work_cqs
);
445 if ((num_mcceq_processed
) && (!num_ioeq_processed
))
446 hwi_ring_eq_db(phba
, eq
->id
, 0,
447 (num_ioeq_processed
+
448 num_mcceq_processed
) , 1, 1);
450 hwi_ring_eq_db(phba
, eq
->id
, 0,
451 (num_ioeq_processed
+
452 num_mcceq_processed
), 0, 1);
458 cq
= &phwi_context
->be_cq
[0];
459 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
462 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
464 EQE_RESID_MASK
) >> 16) != cq
->id
) {
465 spin_lock_irqsave(&phba
->isr_lock
, flags
);
466 phba
->todo_mcc_cq
= 1;
467 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
469 spin_lock_irqsave(&phba
->isr_lock
, flags
);
471 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
473 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
475 eqe
= queue_tail_node(eq
);
476 num_ioeq_processed
++;
478 if (phba
->todo_cq
|| phba
->todo_mcc_cq
)
479 queue_work(phba
->wq
, &phba
->work_cqs
);
481 if (num_ioeq_processed
) {
482 hwi_ring_eq_db(phba
, eq
->id
, 0,
483 num_ioeq_processed
, 1, 1);
490 static int beiscsi_init_irqs(struct beiscsi_hba
*phba
)
492 struct pci_dev
*pcidev
= phba
->pcidev
;
493 struct hwi_controller
*phwi_ctrlr
;
494 struct hwi_context_memory
*phwi_context
;
495 int ret
, msix_vec
, i
= 0;
498 phwi_ctrlr
= phba
->phwi_ctrlr
;
499 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
501 if (phba
->msix_enabled
) {
502 for (i
= 0; i
< phba
->num_cpus
; i
++) {
503 sprintf(desc
, "beiscsi_msix_%04x", i
);
504 msix_vec
= phba
->msix_entries
[i
].vector
;
505 ret
= request_irq(msix_vec
, be_isr_msix
, 0, desc
,
506 &phwi_context
->be_eq
[i
]);
508 msix_vec
= phba
->msix_entries
[i
].vector
;
509 ret
= request_irq(msix_vec
, be_isr_mcc
, 0, "beiscsi_msix_mcc",
510 &phwi_context
->be_eq
[i
]);
512 ret
= request_irq(pcidev
->irq
, be_isr
, IRQF_SHARED
,
515 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_init_irqs-"
516 "Failed to register irq\\n");
523 static void hwi_ring_cq_db(struct beiscsi_hba
*phba
,
524 unsigned int id
, unsigned int num_processed
,
525 unsigned char rearm
, unsigned char event
)
528 val
|= id
& DB_CQ_RING_ID_MASK
;
530 val
|= 1 << DB_CQ_REARM_SHIFT
;
531 val
|= num_processed
<< DB_CQ_NUM_POPPED_SHIFT
;
532 iowrite32(val
, phba
->db_va
+ DB_CQ_OFFSET
);
536 beiscsi_process_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
537 struct beiscsi_hba
*phba
,
539 struct pdu_base
*ppdu
,
540 unsigned long pdu_len
,
541 void *pbuffer
, unsigned long buf_len
)
543 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
544 struct iscsi_session
*session
= conn
->session
;
545 struct iscsi_task
*task
;
546 struct beiscsi_io_task
*io_task
;
547 struct iscsi_hdr
*login_hdr
;
549 switch (ppdu
->dw
[offsetof(struct amap_pdu_base
, opcode
) / 32] &
550 PDUBASE_OPCODE_MASK
) {
551 case ISCSI_OP_NOOP_IN
:
555 case ISCSI_OP_ASYNC_EVENT
:
557 case ISCSI_OP_REJECT
:
559 WARN_ON(!(buf_len
== 48));
560 SE_DEBUG(DBG_LVL_1
, "In ISCSI_OP_REJECT\n");
562 case ISCSI_OP_LOGIN_RSP
:
563 case ISCSI_OP_TEXT_RSP
:
564 task
= conn
->login_task
;
565 io_task
= task
->dd_data
;
566 login_hdr
= (struct iscsi_hdr
*)ppdu
;
567 login_hdr
->itt
= io_task
->libiscsi_itt
;
570 shost_printk(KERN_WARNING
, phba
->shost
,
571 "Unrecognized opcode 0x%x in async msg \n",
573 dw
[offsetof(struct amap_pdu_base
, opcode
) / 32]
574 & PDUBASE_OPCODE_MASK
));
578 spin_lock_bh(&session
->lock
);
579 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)ppdu
, pbuffer
, buf_len
);
580 spin_unlock_bh(&session
->lock
);
584 static struct sgl_handle
*alloc_io_sgl_handle(struct beiscsi_hba
*phba
)
586 struct sgl_handle
*psgl_handle
;
588 if (phba
->io_sgl_hndl_avbl
) {
590 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
591 phba
->io_sgl_alloc_index
);
592 psgl_handle
= phba
->io_sgl_hndl_base
[phba
->
594 phba
->io_sgl_hndl_base
[phba
->io_sgl_alloc_index
] = NULL
;
595 phba
->io_sgl_hndl_avbl
--;
596 if (phba
->io_sgl_alloc_index
== (phba
->params
.
598 phba
->io_sgl_alloc_index
= 0;
600 phba
->io_sgl_alloc_index
++;
607 free_io_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
609 SE_DEBUG(DBG_LVL_8
, "In free_,io_sgl_free_index=%d \n",
610 phba
->io_sgl_free_index
);
611 if (phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]) {
613 * this can happen if clean_task is called on a task that
614 * failed in xmit_task or alloc_pdu.
617 "Double Free in IO SGL io_sgl_free_index=%d,"
618 "value there=%p \n", phba
->io_sgl_free_index
,
619 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]);
622 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
] = psgl_handle
;
623 phba
->io_sgl_hndl_avbl
++;
624 if (phba
->io_sgl_free_index
== (phba
->params
.ios_per_ctrl
- 1))
625 phba
->io_sgl_free_index
= 0;
627 phba
->io_sgl_free_index
++;
631 * alloc_wrb_handle - To allocate a wrb handle
632 * @phba: The hba pointer
633 * @cid: The cid to use for allocation
635 * This happens under session_lock until submission to chip
637 struct wrb_handle
*alloc_wrb_handle(struct beiscsi_hba
*phba
, unsigned int cid
)
639 struct hwi_wrb_context
*pwrb_context
;
640 struct hwi_controller
*phwi_ctrlr
;
641 struct wrb_handle
*pwrb_handle
, *pwrb_handle_tmp
;
643 phwi_ctrlr
= phba
->phwi_ctrlr
;
644 pwrb_context
= &phwi_ctrlr
->wrb_context
[cid
];
645 if (pwrb_context
->wrb_handles_available
>= 2) {
646 pwrb_handle
= pwrb_context
->pwrb_handle_base
[
647 pwrb_context
->alloc_index
];
648 pwrb_context
->wrb_handles_available
--;
649 if (pwrb_context
->alloc_index
==
650 (phba
->params
.wrbs_per_cxn
- 1))
651 pwrb_context
->alloc_index
= 0;
653 pwrb_context
->alloc_index
++;
654 pwrb_handle_tmp
= pwrb_context
->pwrb_handle_base
[
655 pwrb_context
->alloc_index
];
656 pwrb_handle
->nxt_wrb_index
= pwrb_handle_tmp
->wrb_index
;
663 * free_wrb_handle - To free the wrb handle back to pool
664 * @phba: The hba pointer
665 * @pwrb_context: The context to free from
666 * @pwrb_handle: The wrb_handle to free
668 * This happens under session_lock until submission to chip
671 free_wrb_handle(struct beiscsi_hba
*phba
, struct hwi_wrb_context
*pwrb_context
,
672 struct wrb_handle
*pwrb_handle
)
675 pwrb_context
->pwrb_handle_base
[pwrb_context
->free_index
] =
677 pwrb_context
->wrb_handles_available
++;
678 if (pwrb_context
->free_index
== (phba
->params
.wrbs_per_cxn
- 1))
679 pwrb_context
->free_index
= 0;
681 pwrb_context
->free_index
++;
684 "FREE WRB: pwrb_handle=%p free_index=0x%x"
685 "wrb_handles_available=%d \n",
686 pwrb_handle
, pwrb_context
->free_index
,
687 pwrb_context
->wrb_handles_available
);
690 static struct sgl_handle
*alloc_mgmt_sgl_handle(struct beiscsi_hba
*phba
)
692 struct sgl_handle
*psgl_handle
;
694 if (phba
->eh_sgl_hndl_avbl
) {
695 psgl_handle
= phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
];
696 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
] = NULL
;
697 SE_DEBUG(DBG_LVL_8
, "mgmt_sgl_alloc_index=%d=0x%x \n",
698 phba
->eh_sgl_alloc_index
, phba
->eh_sgl_alloc_index
);
699 phba
->eh_sgl_hndl_avbl
--;
700 if (phba
->eh_sgl_alloc_index
==
701 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
-
703 phba
->eh_sgl_alloc_index
= 0;
705 phba
->eh_sgl_alloc_index
++;
712 free_mgmt_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
715 SE_DEBUG(DBG_LVL_8
, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
716 phba
->eh_sgl_free_index
);
717 if (phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
]) {
719 * this can happen if clean_task is called on a task that
720 * failed in xmit_task or alloc_pdu.
723 "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
724 phba
->eh_sgl_free_index
);
727 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
] = psgl_handle
;
728 phba
->eh_sgl_hndl_avbl
++;
729 if (phba
->eh_sgl_free_index
==
730 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
- 1))
731 phba
->eh_sgl_free_index
= 0;
733 phba
->eh_sgl_free_index
++;
737 be_complete_io(struct beiscsi_conn
*beiscsi_conn
,
738 struct iscsi_task
*task
, struct sol_cqe
*psol
)
740 struct beiscsi_io_task
*io_task
= task
->dd_data
;
741 struct be_status_bhs
*sts_bhs
=
742 (struct be_status_bhs
*)io_task
->cmd_bhs
;
743 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
744 unsigned int sense_len
;
745 unsigned char *sense
;
746 u32 resid
= 0, exp_cmdsn
, max_cmdsn
;
747 u8 rsp
, status
, flags
;
750 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
751 & SOL_EXP_CMD_SN_MASK
);
753 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
754 & SOL_EXP_CMD_SN_MASK
) +
755 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
756 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
757 rsp
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) / 32]
758 & SOL_RESP_MASK
) >> 16);
759 status
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_sts
) / 32]
760 & SOL_STS_MASK
) >> 8);
761 flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
762 & SOL_FLAGS_MASK
) >> 24) | 0x80;
764 task
->sc
->result
= (DID_OK
<< 16) | status
;
765 if (rsp
!= ISCSI_STATUS_CMD_COMPLETED
) {
766 task
->sc
->result
= DID_ERROR
<< 16;
770 /* bidi not initially supported */
771 if (flags
& (ISCSI_FLAG_CMD_UNDERFLOW
| ISCSI_FLAG_CMD_OVERFLOW
)) {
772 resid
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) /
773 32] & SOL_RES_CNT_MASK
);
775 if (!status
&& (flags
& ISCSI_FLAG_CMD_OVERFLOW
))
776 task
->sc
->result
= DID_ERROR
<< 16;
778 if (flags
& ISCSI_FLAG_CMD_UNDERFLOW
) {
779 scsi_set_resid(task
->sc
, resid
);
780 if (!status
&& (scsi_bufflen(task
->sc
) - resid
<
781 task
->sc
->underflow
))
782 task
->sc
->result
= DID_ERROR
<< 16;
786 if (status
== SAM_STAT_CHECK_CONDITION
) {
787 unsigned short *slen
= (unsigned short *)sts_bhs
->sense_info
;
788 sense
= sts_bhs
->sense_info
+ sizeof(unsigned short);
789 sense_len
= cpu_to_be16(*slen
);
790 memcpy(task
->sc
->sense_buffer
, sense
,
791 min_t(u16
, sense_len
, SCSI_SENSE_BUFFERSIZE
));
794 if (io_task
->cmd_bhs
->iscsi_hdr
.flags
& ISCSI_FLAG_CMD_READ
) {
795 if (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
797 conn
->rxdata_octets
+= (psol
->
798 dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
802 scsi_dma_unmap(io_task
->scsi_cmnd
);
803 iscsi_complete_scsi_task(task
, exp_cmdsn
, max_cmdsn
);
807 be_complete_logout(struct beiscsi_conn
*beiscsi_conn
,
808 struct iscsi_task
*task
, struct sol_cqe
*psol
)
810 struct iscsi_logout_rsp
*hdr
;
811 struct beiscsi_io_task
*io_task
= task
->dd_data
;
812 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
814 hdr
= (struct iscsi_logout_rsp
*)task
->hdr
;
815 hdr
->opcode
= ISCSI_OP_LOGOUT_RSP
;
818 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
819 & SOL_FLAGS_MASK
) >> 24) | 0x80;
820 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
821 32] & SOL_RESP_MASK
);
822 hdr
->exp_cmdsn
= cpu_to_be32(psol
->
823 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
824 & SOL_EXP_CMD_SN_MASK
);
825 hdr
->max_cmdsn
= be32_to_cpu((psol
->
826 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
827 & SOL_EXP_CMD_SN_MASK
) +
828 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
829 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
834 hdr
->itt
= io_task
->libiscsi_itt
;
835 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
839 be_complete_tmf(struct beiscsi_conn
*beiscsi_conn
,
840 struct iscsi_task
*task
, struct sol_cqe
*psol
)
842 struct iscsi_tm_rsp
*hdr
;
843 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
844 struct beiscsi_io_task
*io_task
= task
->dd_data
;
846 hdr
= (struct iscsi_tm_rsp
*)task
->hdr
;
847 hdr
->opcode
= ISCSI_OP_SCSI_TMFUNC_RSP
;
848 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
849 & SOL_FLAGS_MASK
) >> 24) | 0x80;
850 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
851 32] & SOL_RESP_MASK
);
852 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
853 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
854 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
855 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
856 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
857 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
858 hdr
->itt
= io_task
->libiscsi_itt
;
859 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
863 hwi_complete_drvr_msgs(struct beiscsi_conn
*beiscsi_conn
,
864 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
866 struct hwi_wrb_context
*pwrb_context
;
867 struct wrb_handle
*pwrb_handle
= NULL
;
868 struct sgl_handle
*psgl_handle
= NULL
;
869 struct hwi_controller
*phwi_ctrlr
;
870 struct iscsi_task
*task
;
871 struct beiscsi_io_task
*io_task
;
872 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
873 struct iscsi_session
*session
= conn
->session
;
875 phwi_ctrlr
= phba
->phwi_ctrlr
;
877 psgl_handle
= phba
->sgl_hndl_array
[((psol
->
878 dw
[offsetof(struct amap_sol_cqe_ring
, icd_index
) /
879 32] & SOL_ICD_INDEX_MASK
) >> 6)];
880 pwrb_context
= &phwi_ctrlr
->wrb_context
[psgl_handle
->cid
];
881 task
= psgl_handle
->task
;
884 pwrb_context
= &phwi_ctrlr
->wrb_context
[((psol
->
885 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
886 SOL_CID_MASK
) >> 6) -
887 phba
->fw_config
.iscsi_cid_start
];
888 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
889 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
890 32] & SOL_WRB_INDEX_MASK
) >> 16)];
891 task
= pwrb_handle
->pio_handle
;
894 io_task
= task
->dd_data
;
895 spin_lock(&phba
->mgmt_sgl_lock
);
896 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
897 spin_unlock(&phba
->mgmt_sgl_lock
);
898 spin_lock_bh(&session
->lock
);
899 free_wrb_handle(phba
, pwrb_context
, pwrb_handle
);
900 spin_unlock_bh(&session
->lock
);
904 be_complete_nopin_resp(struct beiscsi_conn
*beiscsi_conn
,
905 struct iscsi_task
*task
, struct sol_cqe
*psol
)
907 struct iscsi_nopin
*hdr
;
908 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
909 struct beiscsi_io_task
*io_task
= task
->dd_data
;
911 hdr
= (struct iscsi_nopin
*)task
->hdr
;
912 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
913 & SOL_FLAGS_MASK
) >> 24) | 0x80;
914 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
915 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
916 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
917 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
918 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
919 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
920 hdr
->opcode
= ISCSI_OP_NOOP_IN
;
921 hdr
->itt
= io_task
->libiscsi_itt
;
922 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
925 static void hwi_complete_cmd(struct beiscsi_conn
*beiscsi_conn
,
926 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
928 struct hwi_wrb_context
*pwrb_context
;
929 struct wrb_handle
*pwrb_handle
;
930 struct iscsi_wrb
*pwrb
= NULL
;
931 struct hwi_controller
*phwi_ctrlr
;
932 struct iscsi_task
*task
;
933 struct sgl_handle
*psgl_handle
= NULL
;
935 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
936 struct iscsi_session
*session
= conn
->session
;
938 phwi_ctrlr
= phba
->phwi_ctrlr
;
940 psgl_handle
= phba
->sgl_hndl_array
[((psol
->
941 dw
[offsetof(struct amap_sol_cqe_ring
, icd_index
) /
942 32] & SOL_ICD_INDEX_MASK
) >> 6)];
943 task
= psgl_handle
->task
;
944 type
= psgl_handle
->type
;
946 pwrb_context
= &phwi_ctrlr
->
947 wrb_context
[((psol
->dw
[offsetof
948 (struct amap_sol_cqe
, cid
) / 32]
949 & SOL_CID_MASK
) >> 6) -
950 phba
->fw_config
.iscsi_cid_start
];
951 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
952 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
953 32] & SOL_WRB_INDEX_MASK
) >> 16)];
954 task
= pwrb_handle
->pio_handle
;
955 pwrb
= pwrb_handle
->pwrb
;
956 type
= (pwrb
->dw
[offsetof(struct amap_iscsi_wrb
, type
) / 32] &
957 WRB_TYPE_MASK
) >> 28;
959 spin_lock_bh(&session
->lock
);
963 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) ==
965 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
967 be_complete_io(beiscsi_conn
, task
, psol
);
970 case HWH_TYPE_LOGOUT
:
971 be_complete_logout(beiscsi_conn
, task
, psol
);
976 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
977 "- Solicited path \n");
981 be_complete_tmf(beiscsi_conn
, task
, psol
);
985 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
990 shost_printk(KERN_WARNING
, phba
->shost
,
991 "In hwi_complete_cmd, unknown type = %d"
992 "icd_index 0x%x CID 0x%x\n", type
,
993 ((psol
->dw
[offsetof(struct amap_sol_cqe_ring
,
994 icd_index
) / 32] & SOL_ICD_INDEX_MASK
) >> 6),
997 shost_printk(KERN_WARNING
, phba
->shost
,
998 "In hwi_complete_cmd, unknown type = %d"
999 "wrb_index 0x%x CID 0x%x\n", type
,
1000 ((psol
->dw
[offsetof(struct amap_iscsi_wrb
,
1001 type
) / 32] & SOL_WRB_INDEX_MASK
) >> 16),
1002 ((psol
->dw
[offsetof(struct amap_sol_cqe
,
1003 cid
) / 32] & SOL_CID_MASK
) >> 6));
1007 spin_unlock_bh(&session
->lock
);
1010 static struct list_head
*hwi_get_async_busy_list(struct hwi_async_pdu_context
1011 *pasync_ctx
, unsigned int is_header
,
1012 unsigned int host_write_ptr
)
1015 return &pasync_ctx
->async_entry
[host_write_ptr
].
1018 return &pasync_ctx
->async_entry
[host_write_ptr
].data_busy_list
;
1021 static struct async_pdu_handle
*
1022 hwi_get_async_handle(struct beiscsi_hba
*phba
,
1023 struct beiscsi_conn
*beiscsi_conn
,
1024 struct hwi_async_pdu_context
*pasync_ctx
,
1025 struct i_t_dpdu_cqe
*pdpdu_cqe
, unsigned int *pcq_index
)
1027 struct be_bus_address phys_addr
;
1028 struct list_head
*pbusy_list
;
1029 struct async_pdu_handle
*pasync_handle
= NULL
;
1031 unsigned char buffer_index
= -1;
1032 unsigned char is_header
= 0;
1034 phys_addr
.u
.a32
.address_lo
=
1035 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_lo
) / 32] -
1036 ((pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1037 & PDUCQE_DPL_MASK
) >> 16);
1038 phys_addr
.u
.a32
.address_hi
=
1039 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_hi
) / 32];
1041 phys_addr
.u
.a64
.address
=
1042 *((unsigned long long *)(&phys_addr
.u
.a64
.address
));
1044 switch (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, code
) / 32]
1045 & PDUCQE_CODE_MASK
) {
1046 case UNSOL_HDR_NOTIFY
:
1049 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 1,
1050 (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1051 index
) / 32] & PDUCQE_INDEX_MASK
));
1053 buffer_len
= (unsigned int)(phys_addr
.u
.a64
.address
-
1054 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
);
1056 buffer_index
= buffer_len
/
1057 pasync_ctx
->async_header
.buffer_size
;
1060 case UNSOL_DATA_NOTIFY
:
1061 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 0, (pdpdu_cqe
->
1062 dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1063 index
) / 32] & PDUCQE_INDEX_MASK
));
1064 buffer_len
= (unsigned long)(phys_addr
.u
.a64
.address
-
1065 pasync_ctx
->async_data
.pa_base
.u
.
1067 buffer_index
= buffer_len
/ pasync_ctx
->async_data
.buffer_size
;
1071 shost_printk(KERN_WARNING
, phba
->shost
,
1072 "Unexpected code=%d \n",
1073 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1074 code
) / 32] & PDUCQE_CODE_MASK
);
1078 WARN_ON(!(buffer_index
<= pasync_ctx
->async_data
.num_entries
));
1079 WARN_ON(list_empty(pbusy_list
));
1080 list_for_each_entry(pasync_handle
, pbusy_list
, link
) {
1081 WARN_ON(pasync_handle
->consumed
);
1082 if (pasync_handle
->index
== buffer_index
)
1086 WARN_ON(!pasync_handle
);
1088 pasync_handle
->cri
= (unsigned short)beiscsi_conn
->beiscsi_conn_cid
-
1089 phba
->fw_config
.iscsi_cid_start
;
1090 pasync_handle
->is_header
= is_header
;
1091 pasync_handle
->buffer_len
= ((pdpdu_cqe
->
1092 dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1093 & PDUCQE_DPL_MASK
) >> 16);
1095 *pcq_index
= (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1096 index
) / 32] & PDUCQE_INDEX_MASK
);
1097 return pasync_handle
;
1101 hwi_update_async_writables(struct hwi_async_pdu_context
*pasync_ctx
,
1102 unsigned int is_header
, unsigned int cq_index
)
1104 struct list_head
*pbusy_list
;
1105 struct async_pdu_handle
*pasync_handle
;
1106 unsigned int num_entries
, writables
= 0;
1107 unsigned int *pep_read_ptr
, *pwritables
;
1111 pep_read_ptr
= &pasync_ctx
->async_header
.ep_read_ptr
;
1112 pwritables
= &pasync_ctx
->async_header
.writables
;
1113 num_entries
= pasync_ctx
->async_header
.num_entries
;
1115 pep_read_ptr
= &pasync_ctx
->async_data
.ep_read_ptr
;
1116 pwritables
= &pasync_ctx
->async_data
.writables
;
1117 num_entries
= pasync_ctx
->async_data
.num_entries
;
1120 while ((*pep_read_ptr
) != cq_index
) {
1122 *pep_read_ptr
= (*pep_read_ptr
) % num_entries
;
1124 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, is_header
,
1127 WARN_ON(list_empty(pbusy_list
));
1129 if (!list_empty(pbusy_list
)) {
1130 pasync_handle
= list_entry(pbusy_list
->next
,
1131 struct async_pdu_handle
,
1133 WARN_ON(!pasync_handle
);
1134 pasync_handle
->consumed
= 1;
1142 "Duplicate notification received - index 0x%x!!\n",
1147 *pwritables
= *pwritables
+ writables
;
1151 static unsigned int hwi_free_async_msg(struct beiscsi_hba
*phba
,
1154 struct hwi_controller
*phwi_ctrlr
;
1155 struct hwi_async_pdu_context
*pasync_ctx
;
1156 struct async_pdu_handle
*pasync_handle
, *tmp_handle
;
1157 struct list_head
*plist
;
1160 phwi_ctrlr
= phba
->phwi_ctrlr
;
1161 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1163 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1165 list_for_each_entry_safe(pasync_handle
, tmp_handle
, plist
, link
) {
1166 list_del(&pasync_handle
->link
);
1169 list_add_tail(&pasync_handle
->link
,
1170 &pasync_ctx
->async_header
.free_list
);
1171 pasync_ctx
->async_header
.free_entries
++;
1174 list_add_tail(&pasync_handle
->link
,
1175 &pasync_ctx
->async_data
.free_list
);
1176 pasync_ctx
->async_data
.free_entries
++;
1181 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1182 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 0;
1183 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1187 static struct phys_addr
*
1188 hwi_get_ring_address(struct hwi_async_pdu_context
*pasync_ctx
,
1189 unsigned int is_header
, unsigned int host_write_ptr
)
1191 struct phys_addr
*pasync_sge
= NULL
;
1194 pasync_sge
= pasync_ctx
->async_header
.ring_base
;
1196 pasync_sge
= pasync_ctx
->async_data
.ring_base
;
1198 return pasync_sge
+ host_write_ptr
;
1201 static void hwi_post_async_buffers(struct beiscsi_hba
*phba
,
1202 unsigned int is_header
)
1204 struct hwi_controller
*phwi_ctrlr
;
1205 struct hwi_async_pdu_context
*pasync_ctx
;
1206 struct async_pdu_handle
*pasync_handle
;
1207 struct list_head
*pfree_link
, *pbusy_list
;
1208 struct phys_addr
*pasync_sge
;
1209 unsigned int ring_id
, num_entries
;
1210 unsigned int host_write_num
;
1211 unsigned int writables
;
1215 phwi_ctrlr
= phba
->phwi_ctrlr
;
1216 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1219 num_entries
= pasync_ctx
->async_header
.num_entries
;
1220 writables
= min(pasync_ctx
->async_header
.writables
,
1221 pasync_ctx
->async_header
.free_entries
);
1222 pfree_link
= pasync_ctx
->async_header
.free_list
.next
;
1223 host_write_num
= pasync_ctx
->async_header
.host_write_ptr
;
1224 ring_id
= phwi_ctrlr
->default_pdu_hdr
.id
;
1226 num_entries
= pasync_ctx
->async_data
.num_entries
;
1227 writables
= min(pasync_ctx
->async_data
.writables
,
1228 pasync_ctx
->async_data
.free_entries
);
1229 pfree_link
= pasync_ctx
->async_data
.free_list
.next
;
1230 host_write_num
= pasync_ctx
->async_data
.host_write_ptr
;
1231 ring_id
= phwi_ctrlr
->default_pdu_data
.id
;
1234 writables
= (writables
/ 8) * 8;
1236 for (i
= 0; i
< writables
; i
++) {
1238 hwi_get_async_busy_list(pasync_ctx
, is_header
,
1241 list_entry(pfree_link
, struct async_pdu_handle
,
1243 WARN_ON(!pasync_handle
);
1244 pasync_handle
->consumed
= 0;
1246 pfree_link
= pfree_link
->next
;
1248 pasync_sge
= hwi_get_ring_address(pasync_ctx
,
1249 is_header
, host_write_num
);
1251 pasync_sge
->hi
= pasync_handle
->pa
.u
.a32
.address_lo
;
1252 pasync_sge
->lo
= pasync_handle
->pa
.u
.a32
.address_hi
;
1254 list_move(&pasync_handle
->link
, pbusy_list
);
1257 host_write_num
= host_write_num
% num_entries
;
1261 pasync_ctx
->async_header
.host_write_ptr
=
1263 pasync_ctx
->async_header
.free_entries
-= writables
;
1264 pasync_ctx
->async_header
.writables
-= writables
;
1265 pasync_ctx
->async_header
.busy_entries
+= writables
;
1267 pasync_ctx
->async_data
.host_write_ptr
= host_write_num
;
1268 pasync_ctx
->async_data
.free_entries
-= writables
;
1269 pasync_ctx
->async_data
.writables
-= writables
;
1270 pasync_ctx
->async_data
.busy_entries
+= writables
;
1273 doorbell
|= ring_id
& DB_DEF_PDU_RING_ID_MASK
;
1274 doorbell
|= 1 << DB_DEF_PDU_REARM_SHIFT
;
1275 doorbell
|= 0 << DB_DEF_PDU_EVENT_SHIFT
;
1276 doorbell
|= (writables
& DB_DEF_PDU_CQPROC_MASK
)
1277 << DB_DEF_PDU_CQPROC_SHIFT
;
1279 iowrite32(doorbell
, phba
->db_va
+ DB_RXULP0_OFFSET
);
1283 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba
*phba
,
1284 struct beiscsi_conn
*beiscsi_conn
,
1285 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1287 struct hwi_controller
*phwi_ctrlr
;
1288 struct hwi_async_pdu_context
*pasync_ctx
;
1289 struct async_pdu_handle
*pasync_handle
= NULL
;
1290 unsigned int cq_index
= -1;
1292 phwi_ctrlr
= phba
->phwi_ctrlr
;
1293 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1295 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1296 pdpdu_cqe
, &cq_index
);
1297 BUG_ON(pasync_handle
->is_header
!= 0);
1298 if (pasync_handle
->consumed
== 0)
1299 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1302 hwi_free_async_msg(phba
, pasync_handle
->cri
);
1303 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1307 hwi_fwd_async_msg(struct beiscsi_conn
*beiscsi_conn
,
1308 struct beiscsi_hba
*phba
,
1309 struct hwi_async_pdu_context
*pasync_ctx
, unsigned short cri
)
1311 struct list_head
*plist
;
1312 struct async_pdu_handle
*pasync_handle
;
1314 unsigned int hdr_len
= 0, buf_len
= 0;
1315 unsigned int status
, index
= 0, offset
= 0;
1316 void *pfirst_buffer
= NULL
;
1317 unsigned int num_buf
= 0;
1319 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1321 list_for_each_entry(pasync_handle
, plist
, link
) {
1323 phdr
= pasync_handle
->pbuffer
;
1324 hdr_len
= pasync_handle
->buffer_len
;
1326 buf_len
= pasync_handle
->buffer_len
;
1328 pfirst_buffer
= pasync_handle
->pbuffer
;
1331 memcpy(pfirst_buffer
+ offset
,
1332 pasync_handle
->pbuffer
, buf_len
);
1338 status
= beiscsi_process_async_pdu(beiscsi_conn
, phba
,
1339 (beiscsi_conn
->beiscsi_conn_cid
-
1340 phba
->fw_config
.iscsi_cid_start
),
1341 phdr
, hdr_len
, pfirst_buffer
,
1345 hwi_free_async_msg(phba
, cri
);
1350 hwi_gather_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
1351 struct beiscsi_hba
*phba
,
1352 struct async_pdu_handle
*pasync_handle
)
1354 struct hwi_async_pdu_context
*pasync_ctx
;
1355 struct hwi_controller
*phwi_ctrlr
;
1356 unsigned int bytes_needed
= 0, status
= 0;
1357 unsigned short cri
= pasync_handle
->cri
;
1358 struct pdu_base
*ppdu
;
1360 phwi_ctrlr
= phba
->phwi_ctrlr
;
1361 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1363 list_del(&pasync_handle
->link
);
1364 if (pasync_handle
->is_header
) {
1365 pasync_ctx
->async_header
.busy_entries
--;
1366 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1367 hwi_free_async_msg(phba
, cri
);
1371 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1372 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 1;
1373 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_len
=
1374 (unsigned short)pasync_handle
->buffer_len
;
1375 list_add_tail(&pasync_handle
->link
,
1376 &pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1378 ppdu
= pasync_handle
->pbuffer
;
1379 bytes_needed
= ((((ppdu
->dw
[offsetof(struct amap_pdu_base
,
1380 data_len_hi
) / 32] & PDUBASE_DATALENHI_MASK
) << 8) &
1381 0xFFFF0000) | ((be16_to_cpu((ppdu
->
1382 dw
[offsetof(struct amap_pdu_base
, data_len_lo
) / 32]
1383 & PDUBASE_DATALENLO_MASK
) >> 16)) & 0x0000FFFF));
1386 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_needed
=
1389 if (bytes_needed
== 0)
1390 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1394 pasync_ctx
->async_data
.busy_entries
--;
1395 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1396 list_add_tail(&pasync_handle
->link
,
1397 &pasync_ctx
->async_entry
[cri
].wait_queue
.
1399 pasync_ctx
->async_entry
[cri
].wait_queue
.
1401 (unsigned short)pasync_handle
->buffer_len
;
1403 if (pasync_ctx
->async_entry
[cri
].wait_queue
.
1405 pasync_ctx
->async_entry
[cri
].wait_queue
.
1407 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1414 static void hwi_process_default_pdu_ring(struct beiscsi_conn
*beiscsi_conn
,
1415 struct beiscsi_hba
*phba
,
1416 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1418 struct hwi_controller
*phwi_ctrlr
;
1419 struct hwi_async_pdu_context
*pasync_ctx
;
1420 struct async_pdu_handle
*pasync_handle
= NULL
;
1421 unsigned int cq_index
= -1;
1423 phwi_ctrlr
= phba
->phwi_ctrlr
;
1424 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1425 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1426 pdpdu_cqe
, &cq_index
);
1428 if (pasync_handle
->consumed
== 0)
1429 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1431 hwi_gather_async_pdu(beiscsi_conn
, phba
, pasync_handle
);
1432 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1435 static void beiscsi_process_mcc_isr(struct beiscsi_hba
*phba
)
1437 struct be_queue_info
*mcc_cq
;
1438 struct be_mcc_compl
*mcc_compl
;
1439 unsigned int num_processed
= 0;
1441 mcc_cq
= &phba
->ctrl
.mcc_obj
.cq
;
1442 mcc_compl
= queue_tail_node(mcc_cq
);
1443 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1444 while (mcc_compl
->flags
& CQE_FLAGS_VALID_MASK
) {
1446 if (num_processed
>= 32) {
1447 hwi_ring_cq_db(phba
, mcc_cq
->id
,
1448 num_processed
, 0, 0);
1451 if (mcc_compl
->flags
& CQE_FLAGS_ASYNC_MASK
) {
1452 /* Interpret flags as an async trailer */
1453 if (is_link_state_evt(mcc_compl
->flags
))
1454 /* Interpret compl as a async link evt */
1455 beiscsi_async_link_state_process(phba
,
1456 (struct be_async_event_link_state
*) mcc_compl
);
1459 " Unsupported Async Event, flags"
1460 " = 0x%08x \n", mcc_compl
->flags
);
1461 } else if (mcc_compl
->flags
& CQE_FLAGS_COMPLETED_MASK
) {
1462 be_mcc_compl_process_isr(&phba
->ctrl
, mcc_compl
);
1463 atomic_dec(&phba
->ctrl
.mcc_obj
.q
.used
);
1466 mcc_compl
->flags
= 0;
1467 queue_tail_inc(mcc_cq
);
1468 mcc_compl
= queue_tail_node(mcc_cq
);
1469 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1473 if (num_processed
> 0)
1474 hwi_ring_cq_db(phba
, mcc_cq
->id
, num_processed
, 1, 0);
1478 static unsigned int beiscsi_process_cq(struct be_eq_obj
*pbe_eq
)
1480 struct be_queue_info
*cq
;
1481 struct sol_cqe
*sol
;
1482 struct dmsg_cqe
*dmsg
;
1483 unsigned int num_processed
= 0;
1484 unsigned int tot_nump
= 0;
1485 struct beiscsi_conn
*beiscsi_conn
;
1486 struct sgl_handle
*psgl_handle
= NULL
;
1487 struct beiscsi_endpoint
*beiscsi_ep
;
1488 struct iscsi_endpoint
*ep
;
1489 struct beiscsi_hba
*phba
;
1492 sol
= queue_tail_node(cq
);
1493 phba
= pbe_eq
->phba
;
1495 while (sol
->dw
[offsetof(struct amap_sol_cqe
, valid
) / 32] &
1497 be_dws_le_to_cpu(sol
, sizeof(struct sol_cqe
));
1500 psgl_handle
= phba
->sgl_hndl_array
[((sol
->
1501 dw
[offsetof(struct amap_sol_cqe_ring
,
1502 icd_index
) / 32] & SOL_ICD_INDEX_MASK
)
1504 ep
= phba
->ep_array
[psgl_handle
->cid
];
1506 ep
= phba
->ep_array
[(u32
) ((sol
->
1507 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
1508 SOL_CID_MASK
) >> 6) -
1509 phba
->fw_config
.iscsi_cid_start
];
1511 beiscsi_ep
= ep
->dd_data
;
1512 beiscsi_conn
= beiscsi_ep
->conn
;
1514 if (num_processed
>= 32) {
1515 hwi_ring_cq_db(phba
, cq
->id
,
1516 num_processed
, 0, 0);
1517 tot_nump
+= num_processed
;
1521 switch ((u32
) sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1522 32] & CQE_CODE_MASK
) {
1523 case SOL_CMD_COMPLETE
:
1524 hwi_complete_cmd(beiscsi_conn
, phba
, sol
);
1526 case DRIVERMSG_NOTIFY
:
1527 SE_DEBUG(DBG_LVL_8
, "Received DRIVERMSG_NOTIFY \n");
1528 dmsg
= (struct dmsg_cqe
*)sol
;
1529 hwi_complete_drvr_msgs(beiscsi_conn
, phba
, sol
);
1531 case UNSOL_HDR_NOTIFY
:
1532 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_HDR_ NOTIFY\n");
1533 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1534 (struct i_t_dpdu_cqe
*)sol
);
1536 case UNSOL_DATA_NOTIFY
:
1537 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_DATA_NOTIFY\n");
1538 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1539 (struct i_t_dpdu_cqe
*)sol
);
1541 case CXN_INVALIDATE_INDEX_NOTIFY
:
1542 case CMD_INVALIDATED_NOTIFY
:
1543 case CXN_INVALIDATE_NOTIFY
:
1545 "Ignoring CQ Error notification for cmd/cxn"
1548 case SOL_CMD_KILLED_DATA_DIGEST_ERR
:
1549 case CMD_KILLED_INVALID_STATSN_RCVD
:
1550 case CMD_KILLED_INVALID_R2T_RCVD
:
1551 case CMD_CXN_KILLED_LUN_INVALID
:
1552 case CMD_CXN_KILLED_ICD_INVALID
:
1553 case CMD_CXN_KILLED_ITT_INVALID
:
1554 case CMD_CXN_KILLED_SEQ_OUTOFORDER
:
1555 case CMD_CXN_KILLED_INVALID_DATASN_RCVD
:
1558 "CQ Error notification for cmd.. "
1559 "code %d cid 0x%x\n",
1560 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1561 32] & CQE_CODE_MASK
, psgl_handle
->cid
);
1564 "CQ Error notification for cmd.. "
1565 "code %d cid 0x%x\n",
1566 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1567 32] & CQE_CODE_MASK
,
1568 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1569 32] & SOL_CID_MASK
));
1572 case UNSOL_DATA_DIGEST_ERROR_NOTIFY
:
1574 "Digest error on def pdu ring, dropping..\n");
1575 hwi_flush_default_pdu_buffer(phba
, beiscsi_conn
,
1576 (struct i_t_dpdu_cqe
*) sol
);
1578 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL
:
1579 case CXN_KILLED_BURST_LEN_MISMATCH
:
1580 case CXN_KILLED_AHS_RCVD
:
1581 case CXN_KILLED_HDR_DIGEST_ERR
:
1582 case CXN_KILLED_UNKNOWN_HDR
:
1583 case CXN_KILLED_STALE_ITT_TTT_RCVD
:
1584 case CXN_KILLED_INVALID_ITT_TTT_RCVD
:
1585 case CXN_KILLED_TIMED_OUT
:
1586 case CXN_KILLED_FIN_RCVD
:
1587 case CXN_KILLED_BAD_UNSOL_PDU_RCVD
:
1588 case CXN_KILLED_BAD_WRB_INDEX_ERROR
:
1589 case CXN_KILLED_OVER_RUN_RESIDUAL
:
1590 case CXN_KILLED_UNDER_RUN_RESIDUAL
:
1591 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN
:
1593 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset CID "
1595 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1596 32] & CQE_CODE_MASK
, psgl_handle
->cid
);
1598 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset CID "
1600 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1601 32] & CQE_CODE_MASK
,
1602 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1603 32] & CQE_CID_MASK
));
1605 iscsi_conn_failure(beiscsi_conn
->conn
,
1606 ISCSI_ERR_CONN_FAILED
);
1608 case CXN_KILLED_RST_SENT
:
1609 case CXN_KILLED_RST_RCVD
:
1611 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset"
1612 "received/sent on CID 0x%x...\n",
1613 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1614 32] & CQE_CODE_MASK
, psgl_handle
->cid
);
1616 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset"
1617 "received/sent on CID 0x%x...\n",
1618 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1619 32] & CQE_CODE_MASK
,
1620 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1621 32] & CQE_CID_MASK
));
1623 iscsi_conn_failure(beiscsi_conn
->conn
,
1624 ISCSI_ERR_CONN_FAILED
);
1627 SE_DEBUG(DBG_LVL_1
, "CQ Error Invalid code= %d "
1628 "received on CID 0x%x...\n",
1629 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1630 32] & CQE_CODE_MASK
,
1631 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1632 32] & CQE_CID_MASK
));
1636 AMAP_SET_BITS(struct amap_sol_cqe
, valid
, sol
, 0);
1638 sol
= queue_tail_node(cq
);
1642 if (num_processed
> 0) {
1643 tot_nump
+= num_processed
;
1644 hwi_ring_cq_db(phba
, cq
->id
, num_processed
, 1, 0);
1649 void beiscsi_process_all_cqs(struct work_struct
*work
)
1651 unsigned long flags
;
1652 struct hwi_controller
*phwi_ctrlr
;
1653 struct hwi_context_memory
*phwi_context
;
1654 struct be_eq_obj
*pbe_eq
;
1655 struct beiscsi_hba
*phba
=
1656 container_of(work
, struct beiscsi_hba
, work_cqs
);
1658 phwi_ctrlr
= phba
->phwi_ctrlr
;
1659 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
1660 if (phba
->msix_enabled
)
1661 pbe_eq
= &phwi_context
->be_eq
[phba
->num_cpus
];
1663 pbe_eq
= &phwi_context
->be_eq
[0];
1665 if (phba
->todo_mcc_cq
) {
1666 spin_lock_irqsave(&phba
->isr_lock
, flags
);
1667 phba
->todo_mcc_cq
= 0;
1668 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
1669 beiscsi_process_mcc_isr(phba
);
1672 if (phba
->todo_cq
) {
1673 spin_lock_irqsave(&phba
->isr_lock
, flags
);
1675 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
1676 beiscsi_process_cq(pbe_eq
);
1680 static int be_iopoll(struct blk_iopoll
*iop
, int budget
)
1682 static unsigned int ret
;
1683 struct beiscsi_hba
*phba
;
1684 struct be_eq_obj
*pbe_eq
;
1686 pbe_eq
= container_of(iop
, struct be_eq_obj
, iopoll
);
1687 ret
= beiscsi_process_cq(pbe_eq
);
1689 phba
= pbe_eq
->phba
;
1690 blk_iopoll_complete(iop
);
1691 SE_DEBUG(DBG_LVL_8
, "rearm pbe_eq->q.id =%d\n", pbe_eq
->q
.id
);
1692 hwi_ring_eq_db(phba
, pbe_eq
->q
.id
, 0, 0, 1, 1);
1698 hwi_write_sgl(struct iscsi_wrb
*pwrb
, struct scatterlist
*sg
,
1699 unsigned int num_sg
, struct beiscsi_io_task
*io_task
)
1701 struct iscsi_sge
*psgl
;
1702 unsigned short sg_len
, index
;
1703 unsigned int sge_len
= 0;
1704 unsigned long long addr
;
1705 struct scatterlist
*l_sg
;
1706 unsigned int offset
;
1708 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
1709 io_task
->bhs_pa
.u
.a32
.address_lo
);
1710 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
1711 io_task
->bhs_pa
.u
.a32
.address_hi
);
1714 for (index
= 0; (index
< num_sg
) && (index
< 2); index
++,
1717 sg_len
= sg_dma_len(sg
);
1718 addr
= (u64
) sg_dma_address(sg
);
1719 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
1720 (addr
& 0xFFFFFFFF));
1721 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
1723 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
1726 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
1729 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
1731 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_r2t_offset
,
1733 sg_len
= sg_dma_len(sg
);
1734 addr
= (u64
) sg_dma_address(sg
);
1735 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_lo
, pwrb
,
1736 (addr
& 0xFFFFFFFF));
1737 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_hi
, pwrb
,
1739 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_len
, pwrb
,
1743 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
1744 memset(psgl
, 0, sizeof(*psgl
) * BE2_SGE
);
1746 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
- 2);
1748 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
1749 io_task
->bhs_pa
.u
.a32
.address_hi
);
1750 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
1751 io_task
->bhs_pa
.u
.a32
.address_lo
);
1754 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
, 1);
1759 for (index
= 0; index
< num_sg
; index
++, sg
= sg_next(sg
), psgl
++) {
1760 sg_len
= sg_dma_len(sg
);
1761 addr
= (u64
) sg_dma_address(sg
);
1762 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
1763 (addr
& 0xFFFFFFFF));
1764 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
1766 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, sg_len
);
1767 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, offset
);
1768 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
1772 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
1775 static void hwi_write_buffer(struct iscsi_wrb
*pwrb
, struct iscsi_task
*task
)
1777 struct iscsi_sge
*psgl
;
1778 unsigned long long addr
;
1779 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1780 struct beiscsi_conn
*beiscsi_conn
= io_task
->conn
;
1781 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
1783 io_task
->bhs_len
= sizeof(struct be_nonio_bhs
) - 2;
1784 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
1785 io_task
->bhs_pa
.u
.a32
.address_lo
);
1786 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
1787 io_task
->bhs_pa
.u
.a32
.address_hi
);
1790 if (task
->data_count
) {
1791 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
1792 addr
= (u64
) pci_map_single(phba
->pcidev
,
1794 task
->data_count
, 1);
1796 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
1799 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
1800 (addr
& 0xFFFFFFFF));
1801 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
1803 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
1806 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
, 1);
1808 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
1812 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
1814 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
);
1816 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
1817 io_task
->bhs_pa
.u
.a32
.address_hi
);
1818 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
1819 io_task
->bhs_pa
.u
.a32
.address_lo
);
1822 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
, 0);
1823 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
, 0);
1824 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0);
1825 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, 0);
1826 AMAP_SET_BITS(struct amap_iscsi_sge
, rsvd0
, psgl
, 0);
1827 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
1831 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
1832 (addr
& 0xFFFFFFFF));
1833 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
1836 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0x106);
1838 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
1841 static void beiscsi_find_mem_req(struct beiscsi_hba
*phba
)
1843 unsigned int num_cq_pages
, num_async_pdu_buf_pages
;
1844 unsigned int num_async_pdu_data_pages
, wrb_sz_per_cxn
;
1845 unsigned int num_async_pdu_buf_sgl_pages
, num_async_pdu_data_sgl_pages
;
1847 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
1848 sizeof(struct sol_cqe
));
1849 num_async_pdu_buf_pages
=
1850 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
1851 phba
->params
.defpdu_hdr_sz
);
1852 num_async_pdu_buf_sgl_pages
=
1853 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
1854 sizeof(struct phys_addr
));
1855 num_async_pdu_data_pages
=
1856 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
1857 phba
->params
.defpdu_data_sz
);
1858 num_async_pdu_data_sgl_pages
=
1859 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
1860 sizeof(struct phys_addr
));
1862 phba
->params
.hwi_ws_sz
= sizeof(struct hwi_controller
);
1864 phba
->mem_req
[ISCSI_MEM_GLOBAL_HEADER
] = 2 *
1865 BE_ISCSI_PDU_HEADER_SIZE
;
1866 phba
->mem_req
[HWI_MEM_ADDN_CONTEXT
] =
1867 sizeof(struct hwi_context_memory
);
1870 phba
->mem_req
[HWI_MEM_WRB
] = sizeof(struct iscsi_wrb
)
1871 * (phba
->params
.wrbs_per_cxn
)
1872 * phba
->params
.cxns_per_ctrl
;
1873 wrb_sz_per_cxn
= sizeof(struct wrb_handle
) *
1874 (phba
->params
.wrbs_per_cxn
);
1875 phba
->mem_req
[HWI_MEM_WRBH
] = roundup_pow_of_two((wrb_sz_per_cxn
) *
1876 phba
->params
.cxns_per_ctrl
);
1878 phba
->mem_req
[HWI_MEM_SGLH
] = sizeof(struct sgl_handle
) *
1879 phba
->params
.icds_per_ctrl
;
1880 phba
->mem_req
[HWI_MEM_SGE
] = sizeof(struct iscsi_sge
) *
1881 phba
->params
.num_sge_per_io
* phba
->params
.icds_per_ctrl
;
1883 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_BUF
] =
1884 num_async_pdu_buf_pages
* PAGE_SIZE
;
1885 phba
->mem_req
[HWI_MEM_ASYNC_DATA_BUF
] =
1886 num_async_pdu_data_pages
* PAGE_SIZE
;
1887 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_RING
] =
1888 num_async_pdu_buf_sgl_pages
* PAGE_SIZE
;
1889 phba
->mem_req
[HWI_MEM_ASYNC_DATA_RING
] =
1890 num_async_pdu_data_sgl_pages
* PAGE_SIZE
;
1891 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_HANDLE
] =
1892 phba
->params
.asyncpdus_per_ctrl
*
1893 sizeof(struct async_pdu_handle
);
1894 phba
->mem_req
[HWI_MEM_ASYNC_DATA_HANDLE
] =
1895 phba
->params
.asyncpdus_per_ctrl
*
1896 sizeof(struct async_pdu_handle
);
1897 phba
->mem_req
[HWI_MEM_ASYNC_PDU_CONTEXT
] =
1898 sizeof(struct hwi_async_pdu_context
) +
1899 (phba
->params
.cxns_per_ctrl
* sizeof(struct hwi_async_entry
));
1902 static int beiscsi_alloc_mem(struct beiscsi_hba
*phba
)
1904 struct be_mem_descriptor
*mem_descr
;
1906 struct mem_array
*mem_arr
, *mem_arr_orig
;
1907 unsigned int i
, j
, alloc_size
, curr_alloc_size
;
1909 phba
->phwi_ctrlr
= kmalloc(phba
->params
.hwi_ws_sz
, GFP_KERNEL
);
1910 if (!phba
->phwi_ctrlr
)
1913 phba
->init_mem
= kcalloc(SE_MEM_MAX
, sizeof(*mem_descr
),
1915 if (!phba
->init_mem
) {
1916 kfree(phba
->phwi_ctrlr
);
1920 mem_arr_orig
= kmalloc(sizeof(*mem_arr_orig
) * BEISCSI_MAX_FRAGS_INIT
,
1922 if (!mem_arr_orig
) {
1923 kfree(phba
->init_mem
);
1924 kfree(phba
->phwi_ctrlr
);
1928 mem_descr
= phba
->init_mem
;
1929 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
1931 mem_arr
= mem_arr_orig
;
1932 alloc_size
= phba
->mem_req
[i
];
1933 memset(mem_arr
, 0, sizeof(struct mem_array
) *
1934 BEISCSI_MAX_FRAGS_INIT
);
1935 curr_alloc_size
= min(be_max_phys_size
* 1024, alloc_size
);
1937 mem_arr
->virtual_address
= pci_alloc_consistent(
1941 if (!mem_arr
->virtual_address
) {
1942 if (curr_alloc_size
<= BE_MIN_MEM_SIZE
)
1944 if (curr_alloc_size
-
1945 rounddown_pow_of_two(curr_alloc_size
))
1946 curr_alloc_size
= rounddown_pow_of_two
1949 curr_alloc_size
= curr_alloc_size
/ 2;
1951 mem_arr
->bus_address
.u
.
1952 a64
.address
= (__u64
) bus_add
;
1953 mem_arr
->size
= curr_alloc_size
;
1954 alloc_size
-= curr_alloc_size
;
1955 curr_alloc_size
= min(be_max_phys_size
*
1960 } while (alloc_size
);
1961 mem_descr
->num_elements
= j
;
1962 mem_descr
->size_in_bytes
= phba
->mem_req
[i
];
1963 mem_descr
->mem_array
= kmalloc(sizeof(*mem_arr
) * j
,
1965 if (!mem_descr
->mem_array
)
1968 memcpy(mem_descr
->mem_array
, mem_arr_orig
,
1969 sizeof(struct mem_array
) * j
);
1972 kfree(mem_arr_orig
);
1975 mem_descr
->num_elements
= j
;
1976 while ((i
) || (j
)) {
1977 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
1978 pci_free_consistent(phba
->pcidev
,
1979 mem_descr
->mem_array
[j
- 1].size
,
1980 mem_descr
->mem_array
[j
- 1].
1982 mem_descr
->mem_array
[j
- 1].
1983 bus_address
.u
.a64
.address
);
1987 kfree(mem_descr
->mem_array
);
1991 kfree(mem_arr_orig
);
1992 kfree(phba
->init_mem
);
1993 kfree(phba
->phwi_ctrlr
);
1997 static int beiscsi_get_memory(struct beiscsi_hba
*phba
)
1999 beiscsi_find_mem_req(phba
);
2000 return beiscsi_alloc_mem(phba
);
2003 static void iscsi_init_global_templates(struct beiscsi_hba
*phba
)
2005 struct pdu_data_out
*pdata_out
;
2006 struct pdu_nop_out
*pnop_out
;
2007 struct be_mem_descriptor
*mem_descr
;
2009 mem_descr
= phba
->init_mem
;
2010 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
2012 (struct pdu_data_out
*)mem_descr
->mem_array
[0].virtual_address
;
2013 memset(pdata_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2015 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
, pdata_out
,
2019 (struct pdu_nop_out
*)((unsigned char *)mem_descr
->mem_array
[0].
2020 virtual_address
+ BE_ISCSI_PDU_HEADER_SIZE
);
2022 memset(pnop_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2023 AMAP_SET_BITS(struct amap_pdu_nop_out
, ttt
, pnop_out
, 0xFFFFFFFF);
2024 AMAP_SET_BITS(struct amap_pdu_nop_out
, f_bit
, pnop_out
, 1);
2025 AMAP_SET_BITS(struct amap_pdu_nop_out
, i_bit
, pnop_out
, 0);
2028 static void beiscsi_init_wrb_handle(struct beiscsi_hba
*phba
)
2030 struct be_mem_descriptor
*mem_descr_wrbh
, *mem_descr_wrb
;
2031 struct wrb_handle
*pwrb_handle
;
2032 struct hwi_controller
*phwi_ctrlr
;
2033 struct hwi_wrb_context
*pwrb_context
;
2034 struct iscsi_wrb
*pwrb
;
2035 unsigned int num_cxn_wrbh
;
2036 unsigned int num_cxn_wrb
, j
, idx
, index
;
2038 mem_descr_wrbh
= phba
->init_mem
;
2039 mem_descr_wrbh
+= HWI_MEM_WRBH
;
2041 mem_descr_wrb
= phba
->init_mem
;
2042 mem_descr_wrb
+= HWI_MEM_WRB
;
2045 pwrb_handle
= mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2046 num_cxn_wrbh
= ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2047 ((sizeof(struct wrb_handle
)) *
2048 phba
->params
.wrbs_per_cxn
));
2049 phwi_ctrlr
= phba
->phwi_ctrlr
;
2051 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2052 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2053 pwrb_context
->pwrb_handle_base
=
2054 kzalloc(sizeof(struct wrb_handle
*) *
2055 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2056 pwrb_context
->pwrb_handle_basestd
=
2057 kzalloc(sizeof(struct wrb_handle
*) *
2058 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2060 pwrb_context
->alloc_index
= 0;
2061 pwrb_context
->wrb_handles_available
= 0;
2062 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2063 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2064 pwrb_context
->pwrb_handle_basestd
[j
] =
2066 pwrb_context
->wrb_handles_available
++;
2067 pwrb_handle
->wrb_index
= j
;
2070 pwrb_context
->free_index
= 0;
2075 mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2077 ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2078 ((sizeof(struct wrb_handle
)) *
2079 phba
->params
.wrbs_per_cxn
));
2080 pwrb_context
->alloc_index
= 0;
2081 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2082 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2083 pwrb_context
->pwrb_handle_basestd
[j
] =
2085 pwrb_context
->wrb_handles_available
++;
2086 pwrb_handle
->wrb_index
= j
;
2089 pwrb_context
->free_index
= 0;
2094 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2096 ((mem_descr_wrb
->mem_array
[idx
].size
) / (sizeof(struct iscsi_wrb
)) *
2097 phba
->params
.wrbs_per_cxn
);
2099 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
; index
+= 2) {
2100 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2102 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2103 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2104 pwrb_handle
->pwrb
= pwrb
;
2110 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2111 num_cxn_wrb
= ((mem_descr_wrb
->mem_array
[idx
].size
) /
2112 (sizeof(struct iscsi_wrb
)) *
2113 phba
->params
.wrbs_per_cxn
);
2114 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2115 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2116 pwrb_handle
->pwrb
= pwrb
;
2124 static void hwi_init_async_pdu_ctx(struct beiscsi_hba
*phba
)
2126 struct hwi_controller
*phwi_ctrlr
;
2127 struct hba_parameters
*p
= &phba
->params
;
2128 struct hwi_async_pdu_context
*pasync_ctx
;
2129 struct async_pdu_handle
*pasync_header_h
, *pasync_data_h
;
2131 struct be_mem_descriptor
*mem_descr
;
2133 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2134 mem_descr
+= HWI_MEM_ASYNC_PDU_CONTEXT
;
2136 phwi_ctrlr
= phba
->phwi_ctrlr
;
2137 phwi_ctrlr
->phwi_ctxt
->pasync_ctx
= (struct hwi_async_pdu_context
*)
2138 mem_descr
->mem_array
[0].virtual_address
;
2139 pasync_ctx
= phwi_ctrlr
->phwi_ctxt
->pasync_ctx
;
2140 memset(pasync_ctx
, 0, sizeof(*pasync_ctx
));
2142 pasync_ctx
->async_header
.num_entries
= p
->asyncpdus_per_ctrl
;
2143 pasync_ctx
->async_header
.buffer_size
= p
->defpdu_hdr_sz
;
2144 pasync_ctx
->async_data
.buffer_size
= p
->defpdu_data_sz
;
2145 pasync_ctx
->async_data
.num_entries
= p
->asyncpdus_per_ctrl
;
2147 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2148 mem_descr
+= HWI_MEM_ASYNC_HEADER_BUF
;
2149 if (mem_descr
->mem_array
[0].virtual_address
) {
2151 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2152 "va=%p \n", mem_descr
->mem_array
[0].virtual_address
);
2154 shost_printk(KERN_WARNING
, phba
->shost
,
2155 "No Virtual address \n");
2157 pasync_ctx
->async_header
.va_base
=
2158 mem_descr
->mem_array
[0].virtual_address
;
2160 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
=
2161 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2163 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2164 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2165 if (mem_descr
->mem_array
[0].virtual_address
) {
2167 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2168 "va=%p \n", mem_descr
->mem_array
[0].virtual_address
);
2170 shost_printk(KERN_WARNING
, phba
->shost
,
2171 "No Virtual address \n");
2172 pasync_ctx
->async_header
.ring_base
=
2173 mem_descr
->mem_array
[0].virtual_address
;
2175 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2176 mem_descr
+= HWI_MEM_ASYNC_HEADER_HANDLE
;
2177 if (mem_descr
->mem_array
[0].virtual_address
) {
2179 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2180 "va=%p \n", mem_descr
->mem_array
[0].virtual_address
);
2182 shost_printk(KERN_WARNING
, phba
->shost
,
2183 "No Virtual address \n");
2185 pasync_ctx
->async_header
.handle_base
=
2186 mem_descr
->mem_array
[0].virtual_address
;
2187 pasync_ctx
->async_header
.writables
= 0;
2188 INIT_LIST_HEAD(&pasync_ctx
->async_header
.free_list
);
2190 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2191 mem_descr
+= HWI_MEM_ASYNC_DATA_BUF
;
2192 if (mem_descr
->mem_array
[0].virtual_address
) {
2194 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2195 "va=%p \n", mem_descr
->mem_array
[0].virtual_address
);
2197 shost_printk(KERN_WARNING
, phba
->shost
,
2198 "No Virtual address \n");
2199 pasync_ctx
->async_data
.va_base
=
2200 mem_descr
->mem_array
[0].virtual_address
;
2201 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
=
2202 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2204 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2205 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2206 if (mem_descr
->mem_array
[0].virtual_address
) {
2208 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2209 "va=%p \n", mem_descr
->mem_array
[0].virtual_address
);
2211 shost_printk(KERN_WARNING
, phba
->shost
,
2212 "No Virtual address \n");
2214 pasync_ctx
->async_data
.ring_base
=
2215 mem_descr
->mem_array
[0].virtual_address
;
2217 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2218 mem_descr
+= HWI_MEM_ASYNC_DATA_HANDLE
;
2219 if (!mem_descr
->mem_array
[0].virtual_address
)
2220 shost_printk(KERN_WARNING
, phba
->shost
,
2221 "No Virtual address \n");
2223 pasync_ctx
->async_data
.handle_base
=
2224 mem_descr
->mem_array
[0].virtual_address
;
2225 pasync_ctx
->async_data
.writables
= 0;
2226 INIT_LIST_HEAD(&pasync_ctx
->async_data
.free_list
);
2229 (struct async_pdu_handle
*)pasync_ctx
->async_header
.handle_base
;
2231 (struct async_pdu_handle
*)pasync_ctx
->async_data
.handle_base
;
2233 for (index
= 0; index
< p
->asyncpdus_per_ctrl
; index
++) {
2234 pasync_header_h
->cri
= -1;
2235 pasync_header_h
->index
= (char)index
;
2236 INIT_LIST_HEAD(&pasync_header_h
->link
);
2237 pasync_header_h
->pbuffer
=
2238 (void *)((unsigned long)
2239 (pasync_ctx
->async_header
.va_base
) +
2240 (p
->defpdu_hdr_sz
* index
));
2242 pasync_header_h
->pa
.u
.a64
.address
=
2243 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
+
2244 (p
->defpdu_hdr_sz
* index
);
2246 list_add_tail(&pasync_header_h
->link
,
2247 &pasync_ctx
->async_header
.free_list
);
2249 pasync_ctx
->async_header
.free_entries
++;
2250 pasync_ctx
->async_header
.writables
++;
2252 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].wait_queue
.list
);
2253 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].
2255 pasync_data_h
->cri
= -1;
2256 pasync_data_h
->index
= (char)index
;
2257 INIT_LIST_HEAD(&pasync_data_h
->link
);
2258 pasync_data_h
->pbuffer
=
2259 (void *)((unsigned long)
2260 (pasync_ctx
->async_data
.va_base
) +
2261 (p
->defpdu_data_sz
* index
));
2263 pasync_data_h
->pa
.u
.a64
.address
=
2264 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
+
2265 (p
->defpdu_data_sz
* index
);
2267 list_add_tail(&pasync_data_h
->link
,
2268 &pasync_ctx
->async_data
.free_list
);
2270 pasync_ctx
->async_data
.free_entries
++;
2271 pasync_ctx
->async_data
.writables
++;
2273 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].data_busy_list
);
2276 pasync_ctx
->async_header
.host_write_ptr
= 0;
2277 pasync_ctx
->async_header
.ep_read_ptr
= -1;
2278 pasync_ctx
->async_data
.host_write_ptr
= 0;
2279 pasync_ctx
->async_data
.ep_read_ptr
= -1;
2283 be_sgl_create_contiguous(void *virtual_address
,
2284 u64 physical_address
, u32 length
,
2285 struct be_dma_mem
*sgl
)
2287 WARN_ON(!virtual_address
);
2288 WARN_ON(!physical_address
);
2289 WARN_ON(!length
> 0);
2292 sgl
->va
= virtual_address
;
2293 sgl
->dma
= physical_address
;
2299 static void be_sgl_destroy_contiguous(struct be_dma_mem
*sgl
)
2301 memset(sgl
, 0, sizeof(*sgl
));
2305 hwi_build_be_sgl_arr(struct beiscsi_hba
*phba
,
2306 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2309 be_sgl_destroy_contiguous(sgl
);
2311 be_sgl_create_contiguous(pmem
->virtual_address
,
2312 pmem
->bus_address
.u
.a64
.address
,
2317 hwi_build_be_sgl_by_offset(struct beiscsi_hba
*phba
,
2318 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2321 be_sgl_destroy_contiguous(sgl
);
2323 be_sgl_create_contiguous((unsigned char *)pmem
->virtual_address
,
2324 pmem
->bus_address
.u
.a64
.address
,
2328 static int be_fill_queue(struct be_queue_info
*q
,
2329 u16 len
, u16 entry_size
, void *vaddress
)
2331 struct be_dma_mem
*mem
= &q
->dma_mem
;
2333 memset(q
, 0, sizeof(*q
));
2335 q
->entry_size
= entry_size
;
2336 mem
->size
= len
* entry_size
;
2340 memset(mem
->va
, 0, mem
->size
);
2344 static int beiscsi_create_eqs(struct beiscsi_hba
*phba
,
2345 struct hwi_context_memory
*phwi_context
)
2347 unsigned int i
, num_eq_pages
;
2348 int ret
, eq_for_mcc
;
2349 struct be_queue_info
*eq
;
2350 struct be_dma_mem
*mem
;
2354 num_eq_pages
= PAGES_REQUIRED(phba
->params
.num_eq_entries
* \
2355 sizeof(struct be_eq_entry
));
2357 if (phba
->msix_enabled
)
2361 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
2362 eq
= &phwi_context
->be_eq
[i
].q
;
2364 phwi_context
->be_eq
[i
].phba
= phba
;
2365 eq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2366 num_eq_pages
* PAGE_SIZE
,
2369 goto create_eq_error
;
2371 mem
->va
= eq_vaddress
;
2372 ret
= be_fill_queue(eq
, phba
->params
.num_eq_entries
,
2373 sizeof(struct be_eq_entry
), eq_vaddress
);
2375 shost_printk(KERN_ERR
, phba
->shost
,
2376 "be_fill_queue Failed for EQ \n");
2377 goto create_eq_error
;
2381 ret
= beiscsi_cmd_eq_create(&phba
->ctrl
, eq
,
2382 phwi_context
->cur_eqd
);
2384 shost_printk(KERN_ERR
, phba
->shost
,
2385 "beiscsi_cmd_eq_create"
2387 goto create_eq_error
;
2389 SE_DEBUG(DBG_LVL_8
, "eqid = %d\n", phwi_context
->be_eq
[i
].q
.id
);
2393 for (i
= 0; i
< (phba
->num_cpus
+ 1); i
++) {
2394 eq
= &phwi_context
->be_eq
[i
].q
;
2397 pci_free_consistent(phba
->pcidev
, num_eq_pages
2404 static int beiscsi_create_cqs(struct beiscsi_hba
*phba
,
2405 struct hwi_context_memory
*phwi_context
)
2407 unsigned int i
, num_cq_pages
;
2409 struct be_queue_info
*cq
, *eq
;
2410 struct be_dma_mem
*mem
;
2411 struct be_eq_obj
*pbe_eq
;
2415 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
2416 sizeof(struct sol_cqe
));
2418 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2419 cq
= &phwi_context
->be_cq
[i
];
2420 eq
= &phwi_context
->be_eq
[i
].q
;
2421 pbe_eq
= &phwi_context
->be_eq
[i
];
2423 pbe_eq
->phba
= phba
;
2425 cq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2426 num_cq_pages
* PAGE_SIZE
,
2429 goto create_cq_error
;
2430 ret
= be_fill_queue(cq
, phba
->params
.num_cq_entries
,
2431 sizeof(struct sol_cqe
), cq_vaddress
);
2433 shost_printk(KERN_ERR
, phba
->shost
,
2434 "be_fill_queue Failed for ISCSI CQ \n");
2435 goto create_cq_error
;
2439 ret
= beiscsi_cmd_cq_create(&phba
->ctrl
, cq
, eq
, false,
2442 shost_printk(KERN_ERR
, phba
->shost
,
2443 "beiscsi_cmd_eq_create"
2444 "Failed for ISCSI CQ \n");
2445 goto create_cq_error
;
2447 SE_DEBUG(DBG_LVL_8
, "iscsi cq_id is %d for eq_id %d\n",
2449 SE_DEBUG(DBG_LVL_8
, "ISCSI CQ CREATED\n");
2454 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2455 cq
= &phwi_context
->be_cq
[i
];
2458 pci_free_consistent(phba
->pcidev
, num_cq_pages
2467 beiscsi_create_def_hdr(struct beiscsi_hba
*phba
,
2468 struct hwi_context_memory
*phwi_context
,
2469 struct hwi_controller
*phwi_ctrlr
,
2470 unsigned int def_pdu_ring_sz
)
2474 struct be_queue_info
*dq
, *cq
;
2475 struct be_dma_mem
*mem
;
2476 struct be_mem_descriptor
*mem_descr
;
2480 dq
= &phwi_context
->be_def_hdrq
;
2481 cq
= &phwi_context
->be_cq
[0];
2483 mem_descr
= phba
->init_mem
;
2484 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2485 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2486 ret
= be_fill_queue(dq
, mem_descr
->mem_array
[0].size
/
2487 sizeof(struct phys_addr
),
2488 sizeof(struct phys_addr
), dq_vaddress
);
2490 shost_printk(KERN_ERR
, phba
->shost
,
2491 "be_fill_queue Failed for DEF PDU HDR\n");
2494 mem
->dma
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
2495 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dq
,
2497 phba
->params
.defpdu_hdr_sz
);
2499 shost_printk(KERN_ERR
, phba
->shost
,
2500 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2503 phwi_ctrlr
->default_pdu_hdr
.id
= phwi_context
->be_def_hdrq
.id
;
2504 SE_DEBUG(DBG_LVL_8
, "iscsi def pdu id is %d\n",
2505 phwi_context
->be_def_hdrq
.id
);
2506 hwi_post_async_buffers(phba
, 1);
2511 beiscsi_create_def_data(struct beiscsi_hba
*phba
,
2512 struct hwi_context_memory
*phwi_context
,
2513 struct hwi_controller
*phwi_ctrlr
,
2514 unsigned int def_pdu_ring_sz
)
2518 struct be_queue_info
*dataq
, *cq
;
2519 struct be_dma_mem
*mem
;
2520 struct be_mem_descriptor
*mem_descr
;
2524 dataq
= &phwi_context
->be_def_dataq
;
2525 cq
= &phwi_context
->be_cq
[0];
2526 mem
= &dataq
->dma_mem
;
2527 mem_descr
= phba
->init_mem
;
2528 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2529 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2530 ret
= be_fill_queue(dataq
, mem_descr
->mem_array
[0].size
/
2531 sizeof(struct phys_addr
),
2532 sizeof(struct phys_addr
), dq_vaddress
);
2534 shost_printk(KERN_ERR
, phba
->shost
,
2535 "be_fill_queue Failed for DEF PDU DATA\n");
2538 mem
->dma
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
2539 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dataq
,
2541 phba
->params
.defpdu_data_sz
);
2543 shost_printk(KERN_ERR
, phba
->shost
,
2544 "be_cmd_create_default_pdu_queue Failed"
2545 " for DEF PDU DATA\n");
2548 phwi_ctrlr
->default_pdu_data
.id
= phwi_context
->be_def_dataq
.id
;
2549 SE_DEBUG(DBG_LVL_8
, "iscsi def data id is %d\n",
2550 phwi_context
->be_def_dataq
.id
);
2551 hwi_post_async_buffers(phba
, 0);
2552 SE_DEBUG(DBG_LVL_8
, "DEFAULT PDU DATA RING CREATED \n");
2557 beiscsi_post_pages(struct beiscsi_hba
*phba
)
2559 struct be_mem_descriptor
*mem_descr
;
2560 struct mem_array
*pm_arr
;
2561 unsigned int page_offset
, i
;
2562 struct be_dma_mem sgl
;
2565 mem_descr
= phba
->init_mem
;
2566 mem_descr
+= HWI_MEM_SGE
;
2567 pm_arr
= mem_descr
->mem_array
;
2569 page_offset
= (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
*
2570 phba
->fw_config
.iscsi_icd_start
) / PAGE_SIZE
;
2571 for (i
= 0; i
< mem_descr
->num_elements
; i
++) {
2572 hwi_build_be_sgl_arr(phba
, pm_arr
, &sgl
);
2573 status
= be_cmd_iscsi_post_sgl_pages(&phba
->ctrl
, &sgl
,
2575 (pm_arr
->size
/ PAGE_SIZE
));
2576 page_offset
+= pm_arr
->size
/ PAGE_SIZE
;
2578 shost_printk(KERN_ERR
, phba
->shost
,
2579 "post sgl failed.\n");
2584 SE_DEBUG(DBG_LVL_8
, "POSTED PAGES \n");
2588 static void be_queue_free(struct beiscsi_hba
*phba
, struct be_queue_info
*q
)
2590 struct be_dma_mem
*mem
= &q
->dma_mem
;
2592 pci_free_consistent(phba
->pcidev
, mem
->size
,
2596 static int be_queue_alloc(struct beiscsi_hba
*phba
, struct be_queue_info
*q
,
2597 u16 len
, u16 entry_size
)
2599 struct be_dma_mem
*mem
= &q
->dma_mem
;
2601 memset(q
, 0, sizeof(*q
));
2603 q
->entry_size
= entry_size
;
2604 mem
->size
= len
* entry_size
;
2605 mem
->va
= pci_alloc_consistent(phba
->pcidev
, mem
->size
, &mem
->dma
);
2608 memset(mem
->va
, 0, mem
->size
);
2613 beiscsi_create_wrb_rings(struct beiscsi_hba
*phba
,
2614 struct hwi_context_memory
*phwi_context
,
2615 struct hwi_controller
*phwi_ctrlr
)
2617 unsigned int wrb_mem_index
, offset
, size
, num_wrb_rings
;
2619 unsigned int idx
, num
, i
;
2620 struct mem_array
*pwrb_arr
;
2622 struct be_dma_mem sgl
;
2623 struct be_mem_descriptor
*mem_descr
;
2627 mem_descr
= phba
->init_mem
;
2628 mem_descr
+= HWI_MEM_WRB
;
2629 pwrb_arr
= kmalloc(sizeof(*pwrb_arr
) * phba
->params
.cxns_per_ctrl
,
2632 shost_printk(KERN_ERR
, phba
->shost
,
2633 "Memory alloc failed in create wrb ring.\n");
2636 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
2637 pa_addr_lo
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
2638 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
2639 (phba
->params
.wrbs_per_cxn
* sizeof(struct iscsi_wrb
));
2641 for (num
= 0; num
< phba
->params
.cxns_per_ctrl
; num
++) {
2642 if (num_wrb_rings
) {
2643 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
2644 pwrb_arr
[num
].bus_address
.u
.a64
.address
= pa_addr_lo
;
2645 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
2646 sizeof(struct iscsi_wrb
);
2647 wrb_vaddr
+= pwrb_arr
[num
].size
;
2648 pa_addr_lo
+= pwrb_arr
[num
].size
;
2652 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
2653 pa_addr_lo
= mem_descr
->mem_array
[idx
].\
2654 bus_address
.u
.a64
.address
;
2655 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
2656 (phba
->params
.wrbs_per_cxn
*
2657 sizeof(struct iscsi_wrb
));
2658 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
2659 pwrb_arr
[num
].bus_address
.u
.a64
.address\
2661 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
2662 sizeof(struct iscsi_wrb
);
2663 wrb_vaddr
+= pwrb_arr
[num
].size
;
2664 pa_addr_lo
+= pwrb_arr
[num
].size
;
2668 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
2673 hwi_build_be_sgl_by_offset(phba
, &pwrb_arr
[i
], &sgl
);
2674 status
= be_cmd_wrbq_create(&phba
->ctrl
, &sgl
,
2675 &phwi_context
->be_wrbq
[i
]);
2677 shost_printk(KERN_ERR
, phba
->shost
,
2678 "wrbq create failed.");
2681 phwi_ctrlr
->wrb_context
[i
* 2].cid
= phwi_context
->be_wrbq
[i
].
2688 static void free_wrb_handles(struct beiscsi_hba
*phba
)
2691 struct hwi_controller
*phwi_ctrlr
;
2692 struct hwi_wrb_context
*pwrb_context
;
2694 phwi_ctrlr
= phba
->phwi_ctrlr
;
2695 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2696 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2697 kfree(pwrb_context
->pwrb_handle_base
);
2698 kfree(pwrb_context
->pwrb_handle_basestd
);
2702 static void be_mcc_queues_destroy(struct beiscsi_hba
*phba
)
2704 struct be_queue_info
*q
;
2705 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
2707 q
= &phba
->ctrl
.mcc_obj
.q
;
2709 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_MCCQ
);
2710 be_queue_free(phba
, q
);
2712 q
= &phba
->ctrl
.mcc_obj
.cq
;
2714 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
2715 be_queue_free(phba
, q
);
2718 static void hwi_cleanup(struct beiscsi_hba
*phba
)
2720 struct be_queue_info
*q
;
2721 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
2722 struct hwi_controller
*phwi_ctrlr
;
2723 struct hwi_context_memory
*phwi_context
;
2726 phwi_ctrlr
= phba
->phwi_ctrlr
;
2727 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
2728 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
2729 q
= &phwi_context
->be_wrbq
[i
];
2731 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_WRBQ
);
2733 free_wrb_handles(phba
);
2735 q
= &phwi_context
->be_def_hdrq
;
2737 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
2739 q
= &phwi_context
->be_def_dataq
;
2741 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
2743 beiscsi_cmd_q_destroy(ctrl
, NULL
, QTYPE_SGL
);
2745 for (i
= 0; i
< (phba
->num_cpus
); i
++) {
2746 q
= &phwi_context
->be_cq
[i
];
2748 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
2750 if (phba
->msix_enabled
)
2754 for (i
= 0; i
< (phba
->num_cpus
+ eq_num
); i
++) {
2755 q
= &phwi_context
->be_eq
[i
].q
;
2757 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_EQ
);
2759 be_mcc_queues_destroy(phba
);
2762 static int be_mcc_queues_create(struct beiscsi_hba
*phba
,
2763 struct hwi_context_memory
*phwi_context
)
2765 struct be_queue_info
*q
, *cq
;
2766 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
2768 /* Alloc MCC compl queue */
2769 cq
= &phba
->ctrl
.mcc_obj
.cq
;
2770 if (be_queue_alloc(phba
, cq
, MCC_CQ_LEN
,
2771 sizeof(struct be_mcc_compl
)))
2773 /* Ask BE to create MCC compl queue; */
2774 if (phba
->msix_enabled
) {
2775 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
2776 [phba
->num_cpus
].q
, false, true, 0))
2779 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
[0].q
,
2784 /* Alloc MCC queue */
2785 q
= &phba
->ctrl
.mcc_obj
.q
;
2786 if (be_queue_alloc(phba
, q
, MCC_Q_LEN
, sizeof(struct be_mcc_wrb
)))
2787 goto mcc_cq_destroy
;
2789 /* Ask BE to create MCC queue */
2790 if (beiscsi_cmd_mccq_create(phba
, q
, cq
))
2796 be_queue_free(phba
, q
);
2798 beiscsi_cmd_q_destroy(ctrl
, cq
, QTYPE_CQ
);
2800 be_queue_free(phba
, cq
);
2805 static int find_num_cpus(void)
2809 num_cpus
= num_online_cpus();
2810 if (num_cpus
>= MAX_CPUS
)
2811 num_cpus
= MAX_CPUS
- 1;
2813 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d \n", num_cpus
);
2817 static int hwi_init_port(struct beiscsi_hba
*phba
)
2819 struct hwi_controller
*phwi_ctrlr
;
2820 struct hwi_context_memory
*phwi_context
;
2821 unsigned int def_pdu_ring_sz
;
2822 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
2826 phba
->params
.asyncpdus_per_ctrl
* sizeof(struct phys_addr
);
2827 phwi_ctrlr
= phba
->phwi_ctrlr
;
2828 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
2829 phwi_context
->max_eqd
= 0;
2830 phwi_context
->min_eqd
= 0;
2831 phwi_context
->cur_eqd
= 64;
2832 be_cmd_fw_initialize(&phba
->ctrl
);
2834 status
= beiscsi_create_eqs(phba
, phwi_context
);
2836 shost_printk(KERN_ERR
, phba
->shost
, "EQ not created \n");
2840 status
= be_mcc_queues_create(phba
, phwi_context
);
2844 status
= mgmt_check_supported_fw(ctrl
, phba
);
2846 shost_printk(KERN_ERR
, phba
->shost
,
2847 "Unsupported fw version \n");
2851 if (phba
->fw_config
.iscsi_features
== 0x1)
2856 status
= beiscsi_create_cqs(phba
, phwi_context
);
2858 shost_printk(KERN_ERR
, phba
->shost
, "CQ not created\n");
2862 status
= beiscsi_create_def_hdr(phba
, phwi_context
, phwi_ctrlr
,
2865 shost_printk(KERN_ERR
, phba
->shost
,
2866 "Default Header not created\n");
2870 status
= beiscsi_create_def_data(phba
, phwi_context
,
2871 phwi_ctrlr
, def_pdu_ring_sz
);
2873 shost_printk(KERN_ERR
, phba
->shost
,
2874 "Default Data not created\n");
2878 status
= beiscsi_post_pages(phba
);
2880 shost_printk(KERN_ERR
, phba
->shost
, "Post SGL Pages Failed\n");
2884 status
= beiscsi_create_wrb_rings(phba
, phwi_context
, phwi_ctrlr
);
2886 shost_printk(KERN_ERR
, phba
->shost
,
2887 "WRB Rings not created\n");
2891 SE_DEBUG(DBG_LVL_8
, "hwi_init_port success\n");
2895 shost_printk(KERN_ERR
, phba
->shost
, "hwi_init_port failed");
2900 static int hwi_init_controller(struct beiscsi_hba
*phba
)
2902 struct hwi_controller
*phwi_ctrlr
;
2904 phwi_ctrlr
= phba
->phwi_ctrlr
;
2905 if (1 == phba
->init_mem
[HWI_MEM_ADDN_CONTEXT
].num_elements
) {
2906 phwi_ctrlr
->phwi_ctxt
= (struct hwi_context_memory
*)phba
->
2907 init_mem
[HWI_MEM_ADDN_CONTEXT
].mem_array
[0].virtual_address
;
2908 SE_DEBUG(DBG_LVL_8
, " phwi_ctrlr->phwi_ctxt=%p \n",
2909 phwi_ctrlr
->phwi_ctxt
);
2911 shost_printk(KERN_ERR
, phba
->shost
,
2912 "HWI_MEM_ADDN_CONTEXT is more than one element."
2913 "Failing to load\n");
2917 iscsi_init_global_templates(phba
);
2918 beiscsi_init_wrb_handle(phba
);
2919 hwi_init_async_pdu_ctx(phba
);
2920 if (hwi_init_port(phba
) != 0) {
2921 shost_printk(KERN_ERR
, phba
->shost
,
2922 "hwi_init_controller failed\n");
2928 static void beiscsi_free_mem(struct beiscsi_hba
*phba
)
2930 struct be_mem_descriptor
*mem_descr
;
2933 mem_descr
= phba
->init_mem
;
2936 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
2937 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
2938 pci_free_consistent(phba
->pcidev
,
2939 mem_descr
->mem_array
[j
- 1].size
,
2940 mem_descr
->mem_array
[j
- 1].virtual_address
,
2941 mem_descr
->mem_array
[j
- 1].bus_address
.
2944 kfree(mem_descr
->mem_array
);
2947 kfree(phba
->init_mem
);
2948 kfree(phba
->phwi_ctrlr
);
2951 static int beiscsi_init_controller(struct beiscsi_hba
*phba
)
2955 ret
= beiscsi_get_memory(phba
);
2957 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe -"
2958 "Failed in beiscsi_alloc_memory \n");
2962 ret
= hwi_init_controller(phba
);
2965 SE_DEBUG(DBG_LVL_8
, "Return success from beiscsi_init_controller");
2969 beiscsi_free_mem(phba
);
2973 static int beiscsi_init_sgl_handle(struct beiscsi_hba
*phba
)
2975 struct be_mem_descriptor
*mem_descr_sglh
, *mem_descr_sg
;
2976 struct sgl_handle
*psgl_handle
;
2977 struct iscsi_sge
*pfrag
;
2978 unsigned int arr_index
, i
, idx
;
2980 phba
->io_sgl_hndl_avbl
= 0;
2981 phba
->eh_sgl_hndl_avbl
= 0;
2984 phba
->sgl_hndl_array
= kzalloc(sizeof(struct sgl_handle
*) *
2985 phba
->params
.icds_per_ctrl
,
2987 if (!phba
->sgl_hndl_array
) {
2988 shost_printk(KERN_ERR
, phba
->shost
,
2989 "Mem Alloc Failed. Failing to load\n");
2994 mem_descr_sglh
= phba
->init_mem
;
2995 mem_descr_sglh
+= HWI_MEM_SGLH
;
2996 if (1 == mem_descr_sglh
->num_elements
) {
2997 phba
->io_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
2998 phba
->params
.ios_per_ctrl
,
3000 if (!phba
->io_sgl_hndl_base
) {
3002 kfree(phba
->sgl_hndl_array
);
3003 shost_printk(KERN_ERR
, phba
->shost
,
3004 "Mem Alloc Failed. Failing to load\n");
3007 phba
->eh_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
3008 (phba
->params
.icds_per_ctrl
-
3009 phba
->params
.ios_per_ctrl
),
3011 if (!phba
->eh_sgl_hndl_base
) {
3012 kfree(phba
->io_sgl_hndl_base
);
3013 shost_printk(KERN_ERR
, phba
->shost
,
3014 "Mem Alloc Failed. Failing to load\n");
3018 shost_printk(KERN_ERR
, phba
->shost
,
3019 "HWI_MEM_SGLH is more than one element."
3020 "Failing to load\n");
3026 while (idx
< mem_descr_sglh
->num_elements
) {
3027 psgl_handle
= mem_descr_sglh
->mem_array
[idx
].virtual_address
;
3029 for (i
= 0; i
< (mem_descr_sglh
->mem_array
[idx
].size
/
3030 sizeof(struct sgl_handle
)); i
++) {
3031 if (arr_index
< phba
->params
.ios_per_ctrl
) {
3032 phba
->io_sgl_hndl_base
[arr_index
] = psgl_handle
;
3033 phba
->io_sgl_hndl_avbl
++;
3036 phba
->eh_sgl_hndl_base
[arr_index
-
3037 phba
->params
.ios_per_ctrl
] =
3040 phba
->eh_sgl_hndl_avbl
++;
3047 "phba->io_sgl_hndl_avbl=%d"
3048 "phba->eh_sgl_hndl_avbl=%d \n",
3049 phba
->io_sgl_hndl_avbl
,
3050 phba
->eh_sgl_hndl_avbl
);
3051 mem_descr_sg
= phba
->init_mem
;
3052 mem_descr_sg
+= HWI_MEM_SGE
;
3053 SE_DEBUG(DBG_LVL_8
, "\n mem_descr_sg->num_elements=%d \n",
3054 mem_descr_sg
->num_elements
);
3057 while (idx
< mem_descr_sg
->num_elements
) {
3058 pfrag
= mem_descr_sg
->mem_array
[idx
].virtual_address
;
3061 i
< (mem_descr_sg
->mem_array
[idx
].size
) /
3062 (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
);
3064 if (arr_index
< phba
->params
.ios_per_ctrl
)
3065 psgl_handle
= phba
->io_sgl_hndl_base
[arr_index
];
3067 psgl_handle
= phba
->eh_sgl_hndl_base
[arr_index
-
3068 phba
->params
.ios_per_ctrl
];
3069 psgl_handle
->pfrag
= pfrag
;
3070 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, pfrag
, 0);
3071 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, pfrag
, 0);
3072 pfrag
+= phba
->params
.num_sge_per_io
;
3073 psgl_handle
->sgl_index
=
3074 phba
->fw_config
.iscsi_icd_start
+ arr_index
++;
3078 phba
->io_sgl_free_index
= 0;
3079 phba
->io_sgl_alloc_index
= 0;
3080 phba
->eh_sgl_free_index
= 0;
3081 phba
->eh_sgl_alloc_index
= 0;
3085 static int hba_setup_cid_tbls(struct beiscsi_hba
*phba
)
3089 phba
->cid_array
= kzalloc(sizeof(void *) * phba
->params
.cxns_per_ctrl
,
3091 if (!phba
->cid_array
) {
3092 shost_printk(KERN_ERR
, phba
->shost
,
3093 "Failed to allocate memory in "
3094 "hba_setup_cid_tbls\n");
3097 phba
->ep_array
= kzalloc(sizeof(struct iscsi_endpoint
*) *
3098 phba
->params
.cxns_per_ctrl
* 2, GFP_KERNEL
);
3099 if (!phba
->ep_array
) {
3100 shost_printk(KERN_ERR
, phba
->shost
,
3101 "Failed to allocate memory in "
3102 "hba_setup_cid_tbls \n");
3103 kfree(phba
->cid_array
);
3106 new_cid
= phba
->fw_config
.iscsi_cid_start
;
3107 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3108 phba
->cid_array
[i
] = new_cid
;
3111 phba
->avlbl_cids
= phba
->params
.cxns_per_ctrl
;
3115 static unsigned char hwi_enable_intr(struct beiscsi_hba
*phba
)
3117 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3118 struct hwi_controller
*phwi_ctrlr
;
3119 struct hwi_context_memory
*phwi_context
;
3120 struct be_queue_info
*eq
;
3125 phwi_ctrlr
= phba
->phwi_ctrlr
;
3126 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3128 addr
= (u8 __iomem
*) ((u8 __iomem
*) ctrl
->pcicfg
+
3129 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
);
3130 reg
= ioread32(addr
);
3131 SE_DEBUG(DBG_LVL_8
, "reg =x%08x \n", reg
);
3133 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3135 reg
|= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3136 SE_DEBUG(DBG_LVL_8
, "reg =x%08x addr=%p \n", reg
, addr
);
3137 iowrite32(reg
, addr
);
3138 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
3139 eq
= &phwi_context
->be_eq
[i
].q
;
3140 SE_DEBUG(DBG_LVL_8
, "eq->id=%d \n", eq
->id
);
3141 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
3144 shost_printk(KERN_WARNING
, phba
->shost
,
3145 "In hwi_enable_intr, Not Enabled \n");
3149 static void hwi_disable_intr(struct beiscsi_hba
*phba
)
3151 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3153 u8 __iomem
*addr
= ctrl
->pcicfg
+ PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
;
3154 u32 reg
= ioread32(addr
);
3156 u32 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3158 reg
&= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3159 iowrite32(reg
, addr
);
3161 shost_printk(KERN_WARNING
, phba
->shost
,
3162 "In hwi_disable_intr, Already Disabled \n");
3165 static int beiscsi_init_port(struct beiscsi_hba
*phba
)
3169 ret
= beiscsi_init_controller(phba
);
3171 shost_printk(KERN_ERR
, phba
->shost
,
3172 "beiscsi_dev_probe - Failed in"
3173 "beiscsi_init_controller \n");
3176 ret
= beiscsi_init_sgl_handle(phba
);
3178 shost_printk(KERN_ERR
, phba
->shost
,
3179 "beiscsi_dev_probe - Failed in"
3180 "beiscsi_init_sgl_handle \n");
3181 goto do_cleanup_ctrlr
;
3184 if (hba_setup_cid_tbls(phba
)) {
3185 shost_printk(KERN_ERR
, phba
->shost
,
3186 "Failed in hba_setup_cid_tbls\n");
3188 kfree(phba
->sgl_hndl_array
);
3189 kfree(phba
->io_sgl_hndl_base
);
3190 kfree(phba
->eh_sgl_hndl_base
);
3191 goto do_cleanup_ctrlr
;
3201 static void hwi_purge_eq(struct beiscsi_hba
*phba
)
3203 struct hwi_controller
*phwi_ctrlr
;
3204 struct hwi_context_memory
*phwi_context
;
3205 struct be_queue_info
*eq
;
3206 struct be_eq_entry
*eqe
= NULL
;
3208 unsigned int num_processed
;
3210 phwi_ctrlr
= phba
->phwi_ctrlr
;
3211 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3212 if (phba
->msix_enabled
)
3217 for (i
= 0; i
< (phba
->num_cpus
+ eq_msix
); i
++) {
3218 eq
= &phwi_context
->be_eq
[i
].q
;
3219 eqe
= queue_tail_node(eq
);
3221 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
3223 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
3225 eqe
= queue_tail_node(eq
);
3230 hwi_ring_eq_db(phba
, eq
->id
, 1, num_processed
, 1, 1);
3234 static void beiscsi_clean_port(struct beiscsi_hba
*phba
)
3236 unsigned char mgmt_status
;
3238 mgmt_status
= mgmt_epfw_cleanup(phba
, CMD_CONNECTION_CHUTE_0
);
3240 shost_printk(KERN_WARNING
, phba
->shost
,
3241 "mgmt_epfw_cleanup FAILED \n");
3246 kfree(phba
->sgl_hndl_array
);
3247 kfree(phba
->io_sgl_hndl_base
);
3248 kfree(phba
->eh_sgl_hndl_base
);
3249 kfree(phba
->cid_array
);
3250 kfree(phba
->ep_array
);
3254 beiscsi_offload_connection(struct beiscsi_conn
*beiscsi_conn
,
3255 struct beiscsi_offload_params
*params
)
3257 struct wrb_handle
*pwrb_handle
;
3258 struct iscsi_target_context_update_wrb
*pwrb
= NULL
;
3259 struct be_mem_descriptor
*mem_descr
;
3260 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3264 * We can always use 0 here because it is reserved by libiscsi for
3265 * login/startup related tasks.
3267 pwrb_handle
= alloc_wrb_handle(phba
, (beiscsi_conn
->beiscsi_conn_cid
-
3268 phba
->fw_config
.iscsi_cid_start
));
3269 pwrb
= (struct iscsi_target_context_update_wrb
*)pwrb_handle
->pwrb
;
3270 memset(pwrb
, 0, sizeof(*pwrb
));
3271 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3272 max_burst_length
, pwrb
, params
->dw
[offsetof
3273 (struct amap_beiscsi_offload_params
,
3274 max_burst_length
) / 32]);
3275 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3276 max_send_data_segment_length
, pwrb
,
3277 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3278 max_send_data_segment_length
) / 32]);
3279 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3282 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3283 first_burst_length
) / 32]);
3285 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, erl
, pwrb
,
3286 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3287 erl
) / 32] & OFFLD_PARAMS_ERL
));
3288 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, dde
, pwrb
,
3289 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3290 dde
) / 32] & OFFLD_PARAMS_DDE
) >> 2);
3291 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, hde
, pwrb
,
3292 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3293 hde
) / 32] & OFFLD_PARAMS_HDE
) >> 3);
3294 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ir2t
, pwrb
,
3295 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3296 ir2t
) / 32] & OFFLD_PARAMS_IR2T
) >> 4);
3297 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, imd
, pwrb
,
3298 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3299 imd
) / 32] & OFFLD_PARAMS_IMD
) >> 5);
3300 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, stat_sn
,
3302 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3303 exp_statsn
) / 32] + 1));
3304 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, type
, pwrb
,
3306 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, wrb_idx
,
3307 pwrb
, pwrb_handle
->wrb_index
);
3308 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ptr2nextwrb
,
3309 pwrb
, pwrb_handle
->nxt_wrb_index
);
3310 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3311 session_state
, pwrb
, 0);
3312 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, compltonack
,
3314 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, notpredblq
,
3316 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, mode
, pwrb
,
3319 mem_descr
= phba
->init_mem
;
3320 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
3322 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3323 pad_buffer_addr_hi
, pwrb
,
3324 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_hi
);
3325 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3326 pad_buffer_addr_lo
, pwrb
,
3327 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_lo
);
3329 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_target_context_update_wrb
));
3331 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
3333 doorbell
|= (pwrb_handle
->wrb_index
& DB_DEF_PDU_WRB_INDEX_MASK
)
3334 << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3335 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3337 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3340 static void beiscsi_parse_pdu(struct iscsi_conn
*conn
, itt_t itt
,
3341 int *index
, int *age
)
3345 *age
= conn
->session
->age
;
3349 * beiscsi_alloc_pdu - allocates pdu and related resources
3350 * @task: libiscsi task
3351 * @opcode: opcode of pdu for task
3353 * This is called with the session lock held. It will allocate
3354 * the wrb and sgl if needed for the command. And it will prep
3355 * the pdu's itt. beiscsi_parse_pdu will later translate
3356 * the pdu itt to the libiscsi task itt.
3358 static int beiscsi_alloc_pdu(struct iscsi_task
*task
, uint8_t opcode
)
3360 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3361 struct iscsi_conn
*conn
= task
->conn
;
3362 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3363 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3364 struct hwi_wrb_context
*pwrb_context
;
3365 struct hwi_controller
*phwi_ctrlr
;
3367 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3370 io_task
->cmd_bhs
= pci_pool_alloc(beiscsi_sess
->bhs_pool
,
3371 GFP_KERNEL
, &paddr
);
3372 if (!io_task
->cmd_bhs
)
3374 io_task
->bhs_pa
.u
.a64
.address
= paddr
;
3375 io_task
->libiscsi_itt
= (itt_t
)task
->itt
;
3376 io_task
->pwrb_handle
= alloc_wrb_handle(phba
,
3377 beiscsi_conn
->beiscsi_conn_cid
-
3378 phba
->fw_config
.iscsi_cid_start
3380 io_task
->conn
= beiscsi_conn
;
3382 task
->hdr
= (struct iscsi_hdr
*)&io_task
->cmd_bhs
->iscsi_hdr
;
3383 task
->hdr_max
= sizeof(struct be_cmd_bhs
);
3386 spin_lock(&phba
->io_sgl_lock
);
3387 io_task
->psgl_handle
= alloc_io_sgl_handle(phba
);
3388 spin_unlock(&phba
->io_sgl_lock
);
3389 if (!io_task
->psgl_handle
)
3392 io_task
->scsi_cmnd
= NULL
;
3393 if ((opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
) {
3394 if (!beiscsi_conn
->login_in_progress
) {
3395 spin_lock(&phba
->mgmt_sgl_lock
);
3396 io_task
->psgl_handle
= (struct sgl_handle
*)
3397 alloc_mgmt_sgl_handle(phba
);
3398 spin_unlock(&phba
->mgmt_sgl_lock
);
3399 if (!io_task
->psgl_handle
)
3402 beiscsi_conn
->login_in_progress
= 1;
3403 beiscsi_conn
->plogin_sgl_handle
=
3404 io_task
->psgl_handle
;
3406 io_task
->psgl_handle
=
3407 beiscsi_conn
->plogin_sgl_handle
;
3410 spin_lock(&phba
->mgmt_sgl_lock
);
3411 io_task
->psgl_handle
= alloc_mgmt_sgl_handle(phba
);
3412 spin_unlock(&phba
->mgmt_sgl_lock
);
3413 if (!io_task
->psgl_handle
)
3417 itt
= (itt_t
) cpu_to_be32(((unsigned int)io_task
->pwrb_handle
->
3418 wrb_index
<< 16) | (unsigned int)
3419 (io_task
->psgl_handle
->sgl_index
));
3421 phba
->sgl_hndl_array
[io_task
->psgl_handle
->sgl_index
-
3422 phba
->fw_config
.iscsi_icd_start
] =
3423 io_task
->psgl_handle
;
3424 io_task
->psgl_handle
->task
= task
;
3425 io_task
->psgl_handle
->cid
= beiscsi_conn
->beiscsi_conn_cid
-
3426 phba
->fw_config
.iscsi_cid_start
;
3428 io_task
->pwrb_handle
->pio_handle
= task
;
3430 io_task
->cmd_bhs
->iscsi_hdr
.itt
= itt
;
3434 phwi_ctrlr
= phba
->phwi_ctrlr
;
3435 pwrb_context
= &phwi_ctrlr
->wrb_context
[
3436 beiscsi_conn
->beiscsi_conn_cid
-
3437 phba
->fw_config
.iscsi_cid_start
];
3438 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3439 io_task
->pwrb_handle
= NULL
;
3440 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3441 io_task
->bhs_pa
.u
.a64
.address
);
3442 SE_DEBUG(DBG_LVL_1
, "Alloc of SGL_ICD Failed \n");
3446 static void beiscsi_cleanup_task(struct iscsi_task
*task
)
3448 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3449 struct iscsi_conn
*conn
= task
->conn
;
3450 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3451 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3452 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3453 struct hwi_wrb_context
*pwrb_context
;
3454 struct hwi_controller
*phwi_ctrlr
;
3456 phwi_ctrlr
= phba
->phwi_ctrlr
;
3457 pwrb_context
= &phwi_ctrlr
->wrb_context
[beiscsi_conn
->beiscsi_conn_cid
3458 - phba
->fw_config
.iscsi_cid_start
];
3459 if (io_task
->pwrb_handle
) {
3460 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3461 io_task
->pwrb_handle
= NULL
;
3464 if (io_task
->cmd_bhs
) {
3465 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3466 io_task
->bhs_pa
.u
.a64
.address
);
3470 if (io_task
->psgl_handle
) {
3471 spin_lock(&phba
->io_sgl_lock
);
3472 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
3473 spin_unlock(&phba
->io_sgl_lock
);
3474 io_task
->psgl_handle
= NULL
;
3477 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
)
3479 if (io_task
->psgl_handle
) {
3480 spin_lock(&phba
->mgmt_sgl_lock
);
3481 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
3482 spin_unlock(&phba
->mgmt_sgl_lock
);
3483 io_task
->psgl_handle
= NULL
;
3488 static int beiscsi_iotask(struct iscsi_task
*task
, struct scatterlist
*sg
,
3489 unsigned int num_sg
, unsigned int xferlen
,
3490 unsigned int writedir
)
3493 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3494 struct iscsi_conn
*conn
= task
->conn
;
3495 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3496 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3497 struct iscsi_wrb
*pwrb
= NULL
;
3498 unsigned int doorbell
= 0;
3500 pwrb
= io_task
->pwrb_handle
->pwrb
;
3501 io_task
->cmd_bhs
->iscsi_hdr
.exp_statsn
= 0;
3502 io_task
->bhs_len
= sizeof(struct be_cmd_bhs
);
3505 memset(&io_task
->cmd_bhs
->iscsi_data_pdu
, 0, 48);
3506 AMAP_SET_BITS(struct amap_pdu_data_out
, itt
,
3507 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3508 (unsigned int)io_task
->cmd_bhs
->iscsi_hdr
.itt
);
3509 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
,
3510 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3511 ISCSI_OPCODE_SCSI_DATA_OUT
);
3512 AMAP_SET_BITS(struct amap_pdu_data_out
, final_bit
,
3513 &io_task
->cmd_bhs
->iscsi_data_pdu
, 1);
3515 io_task
->psgl_handle
->type
= INI_WR_CMD
;
3517 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3519 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
3522 io_task
->psgl_handle
->type
= INI_RD_CMD
;
3524 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3526 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
3528 memcpy(&io_task
->cmd_bhs
->iscsi_data_pdu
.
3529 dw
[offsetof(struct amap_pdu_data_out
, lun
) / 32],
3530 io_task
->cmd_bhs
->iscsi_hdr
.lun
, sizeof(struct scsi_lun
));
3532 AMAP_SET_BITS(struct amap_iscsi_wrb
, lun
, pwrb
,
3533 cpu_to_be16((unsigned short)io_task
->cmd_bhs
->iscsi_hdr
.
3535 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
, xferlen
);
3536 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
3537 io_task
->pwrb_handle
->wrb_index
);
3538 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
3539 be32_to_cpu(task
->cmdsn
));
3540 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
3541 io_task
->psgl_handle
->sgl_index
);
3543 hwi_write_sgl(pwrb
, sg
, num_sg
, io_task
);
3545 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
3546 io_task
->pwrb_handle
->nxt_wrb_index
);
3547 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
3549 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
3551 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
3552 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3553 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3555 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3559 static int beiscsi_mtask(struct iscsi_task
*task
)
3561 struct beiscsi_io_task
*aborted_io_task
, *io_task
= task
->dd_data
;
3562 struct iscsi_conn
*conn
= task
->conn
;
3563 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3564 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3565 struct iscsi_session
*session
;
3566 struct iscsi_wrb
*pwrb
= NULL
;
3567 struct hwi_controller
*phwi_ctrlr
;
3568 struct hwi_wrb_context
*pwrb_context
;
3569 struct wrb_handle
*pwrb_handle
;
3570 unsigned int doorbell
= 0;
3571 unsigned int i
, cid
;
3572 struct iscsi_task
*aborted_task
;
3575 cid
= beiscsi_conn
->beiscsi_conn_cid
;
3576 pwrb
= io_task
->pwrb_handle
->pwrb
;
3577 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
3578 be32_to_cpu(task
->cmdsn
));
3579 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
3580 io_task
->pwrb_handle
->wrb_index
);
3581 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
3582 io_task
->psgl_handle
->sgl_index
);
3584 switch (task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) {
3585 case ISCSI_OP_LOGIN
:
3587 io_task
->psgl_handle
->type
= TGT_DM_CMD
;
3589 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3591 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
3592 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
, 1);
3593 hwi_write_buffer(pwrb
, task
);
3595 case ISCSI_OP_NOOP_OUT
:
3597 io_task
->psgl_handle
->type
= INI_RD_CMD
;
3599 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3601 if (task
->hdr
->ttt
== ISCSI_RESERVED_TAG
)
3602 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
3604 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 1);
3605 hwi_write_buffer(pwrb
, task
);
3609 io_task
->psgl_handle
->type
= INI_WR_CMD
;
3611 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3613 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
3614 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
3615 hwi_write_buffer(pwrb
, task
);
3617 case ISCSI_OP_SCSI_TMFUNC
:
3618 session
= conn
->session
;
3619 i
= ((struct iscsi_tm
*)task
->hdr
)->rtt
;
3620 phwi_ctrlr
= phba
->phwi_ctrlr
;
3621 pwrb_context
= &phwi_ctrlr
->wrb_context
[cid
-
3622 phba
->fw_config
.iscsi_cid_start
];
3623 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[be32_to_cpu(i
)
3625 aborted_task
= pwrb_handle
->pio_handle
;
3629 aborted_io_task
= aborted_task
->dd_data
;
3630 if (!aborted_io_task
->scsi_cmnd
)
3633 tag
= mgmt_invalidate_icds(phba
,
3634 aborted_io_task
->psgl_handle
->sgl_index
,
3637 shost_printk(KERN_WARNING
, phba
->shost
,
3638 "mgmt_invalidate_icds could not be"
3641 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
3642 phba
->ctrl
.mcc_numtag
[tag
]);
3643 free_mcc_tag(&phba
->ctrl
, tag
);
3646 io_task
->psgl_handle
->type
= INI_TMF_CMD
;
3648 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3650 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
3651 hwi_write_buffer(pwrb
, task
);
3653 case ISCSI_OP_LOGOUT
:
3654 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
3656 io_task
->psgl_handle
->type
= HWH_TYPE_LOGOUT
;
3658 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3660 hwi_write_buffer(pwrb
, task
);
3664 SE_DEBUG(DBG_LVL_1
, "opcode =%d Not supported \n",
3665 task
->hdr
->opcode
& ISCSI_OPCODE_MASK
);
3669 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
,
3671 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
3672 io_task
->pwrb_handle
->nxt_wrb_index
);
3673 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
3675 doorbell
|= cid
& DB_WRB_POST_CID_MASK
;
3677 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
3678 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3679 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3680 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3684 static int beiscsi_task_xmit(struct iscsi_task
*task
)
3686 struct iscsi_conn
*conn
= task
->conn
;
3687 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3688 struct scsi_cmnd
*sc
= task
->sc
;
3689 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3690 struct scatterlist
*sg
;
3692 unsigned int writedir
= 0, xferlen
= 0;
3694 SE_DEBUG(DBG_LVL_4
, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3695 "beiscsi_conn=%p \n", beiscsi_conn
->beiscsi_conn_cid
,
3696 task
, conn
, beiscsi_conn
);
3698 return beiscsi_mtask(task
);
3700 io_task
->scsi_cmnd
= sc
;
3701 num_sg
= scsi_dma_map(sc
);
3703 SE_DEBUG(DBG_LVL_1
, " scsi_dma_map Failed\n")
3706 SE_DEBUG(DBG_LVL_4
, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3707 (scsi_bufflen(sc
)), sc
, num_sg
, sc
->serial_number
);
3708 xferlen
= scsi_bufflen(sc
);
3709 sg
= scsi_sglist(sc
);
3710 if (sc
->sc_data_direction
== DMA_TO_DEVICE
) {
3712 SE_DEBUG(DBG_LVL_4
, "task->imm_count=0x%08x \n",
3716 return beiscsi_iotask(task
, sg
, num_sg
, xferlen
, writedir
);
3719 static void beiscsi_remove(struct pci_dev
*pcidev
)
3721 struct beiscsi_hba
*phba
= NULL
;
3722 struct hwi_controller
*phwi_ctrlr
;
3723 struct hwi_context_memory
*phwi_context
;
3724 struct be_eq_obj
*pbe_eq
;
3725 unsigned int i
, msix_vec
;
3727 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pcidev
);
3729 dev_err(&pcidev
->dev
, "beiscsi_remove called with no phba \n");
3733 phwi_ctrlr
= phba
->phwi_ctrlr
;
3734 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3735 hwi_disable_intr(phba
);
3736 if (phba
->msix_enabled
) {
3737 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
3738 msix_vec
= phba
->msix_entries
[i
].vector
;
3739 free_irq(msix_vec
, &phwi_context
->be_eq
[i
]);
3742 if (phba
->pcidev
->irq
)
3743 free_irq(phba
->pcidev
->irq
, phba
);
3744 pci_disable_msix(phba
->pcidev
);
3745 destroy_workqueue(phba
->wq
);
3746 if (blk_iopoll_enabled
)
3747 for (i
= 0; i
< phba
->num_cpus
; i
++) {
3748 pbe_eq
= &phwi_context
->be_eq
[i
];
3749 blk_iopoll_disable(&pbe_eq
->iopoll
);
3752 beiscsi_clean_port(phba
);
3753 beiscsi_free_mem(phba
);
3754 beiscsi_unmap_pci_function(phba
);
3755 pci_free_consistent(phba
->pcidev
,
3756 phba
->ctrl
.mbox_mem_alloced
.size
,
3757 phba
->ctrl
.mbox_mem_alloced
.va
,
3758 phba
->ctrl
.mbox_mem_alloced
.dma
);
3759 iscsi_host_remove(phba
->shost
);
3760 pci_dev_put(phba
->pcidev
);
3761 iscsi_host_free(phba
->shost
);
3764 static void beiscsi_msix_enable(struct beiscsi_hba
*phba
)
3768 for (i
= 0; i
<= phba
->num_cpus
; i
++)
3769 phba
->msix_entries
[i
].entry
= i
;
3771 status
= pci_enable_msix(phba
->pcidev
, phba
->msix_entries
,
3772 (phba
->num_cpus
+ 1));
3774 phba
->msix_enabled
= true;
3779 static int __devinit
beiscsi_dev_probe(struct pci_dev
*pcidev
,
3780 const struct pci_device_id
*id
)
3782 struct beiscsi_hba
*phba
= NULL
;
3783 struct hwi_controller
*phwi_ctrlr
;
3784 struct hwi_context_memory
*phwi_context
;
3785 struct be_eq_obj
*pbe_eq
;
3786 int ret
, msix_vec
, num_cpus
, i
;
3788 ret
= beiscsi_enable_pci(pcidev
);
3790 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3791 "Failed to enable pci device \n");
3795 phba
= beiscsi_hba_alloc(pcidev
);
3797 dev_err(&pcidev
->dev
, "beiscsi_dev_probe-"
3798 " Failed in beiscsi_hba_alloc \n");
3801 SE_DEBUG(DBG_LVL_8
, " phba = %p \n", phba
);
3804 num_cpus
= find_num_cpus();
3807 phba
->num_cpus
= num_cpus
;
3808 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d \n", phba
->num_cpus
);
3811 beiscsi_msix_enable(phba
);
3812 ret
= be_ctrl_init(phba
, pcidev
);
3814 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3815 "Failed in be_ctrl_init\n");
3819 spin_lock_init(&phba
->io_sgl_lock
);
3820 spin_lock_init(&phba
->mgmt_sgl_lock
);
3821 spin_lock_init(&phba
->isr_lock
);
3822 ret
= mgmt_get_fw_config(&phba
->ctrl
, phba
);
3824 shost_printk(KERN_ERR
, phba
->shost
,
3825 "Error getting fw config\n");
3828 phba
->shost
->max_id
= phba
->fw_config
.iscsi_cid_count
;
3829 beiscsi_get_params(phba
);
3830 phba
->shost
->can_queue
= phba
->params
.ios_per_ctrl
;
3831 ret
= beiscsi_init_port(phba
);
3833 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3834 "Failed in beiscsi_init_port\n");
3838 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
3839 init_waitqueue_head(&phba
->ctrl
.mcc_wait
[i
+ 1]);
3840 phba
->ctrl
.mcc_tag
[i
] = i
+ 1;
3841 phba
->ctrl
.mcc_numtag
[i
+ 1] = 0;
3842 phba
->ctrl
.mcc_tag_available
++;
3845 phba
->ctrl
.mcc_alloc_index
= phba
->ctrl
.mcc_free_index
= 0;
3847 snprintf(phba
->wq_name
, sizeof(phba
->wq_name
), "beiscsi_q_irq%u",
3848 phba
->shost
->host_no
);
3849 phba
->wq
= create_workqueue(phba
->wq_name
);
3851 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3852 "Failed to allocate work queue\n");
3856 INIT_WORK(&phba
->work_cqs
, beiscsi_process_all_cqs
);
3858 phwi_ctrlr
= phba
->phwi_ctrlr
;
3859 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3860 if (blk_iopoll_enabled
) {
3861 for (i
= 0; i
< phba
->num_cpus
; i
++) {
3862 pbe_eq
= &phwi_context
->be_eq
[i
];
3863 blk_iopoll_init(&pbe_eq
->iopoll
, be_iopoll_budget
,
3865 blk_iopoll_enable(&pbe_eq
->iopoll
);
3868 ret
= beiscsi_init_irqs(phba
);
3870 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3871 "Failed to beiscsi_init_irqs\n");
3874 ret
= hwi_enable_intr(phba
);
3876 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
3877 "Failed to hwi_enable_intr\n");
3880 SE_DEBUG(DBG_LVL_8
, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3884 if (phba
->msix_enabled
) {
3885 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
3886 msix_vec
= phba
->msix_entries
[i
].vector
;
3887 free_irq(msix_vec
, &phwi_context
->be_eq
[i
]);
3890 if (phba
->pcidev
->irq
)
3891 free_irq(phba
->pcidev
->irq
, phba
);
3892 pci_disable_msix(phba
->pcidev
);
3894 destroy_workqueue(phba
->wq
);
3895 if (blk_iopoll_enabled
)
3896 for (i
= 0; i
< phba
->num_cpus
; i
++) {
3897 pbe_eq
= &phwi_context
->be_eq
[i
];
3898 blk_iopoll_disable(&pbe_eq
->iopoll
);
3901 beiscsi_clean_port(phba
);
3902 beiscsi_free_mem(phba
);
3904 pci_free_consistent(phba
->pcidev
,
3905 phba
->ctrl
.mbox_mem_alloced
.size
,
3906 phba
->ctrl
.mbox_mem_alloced
.va
,
3907 phba
->ctrl
.mbox_mem_alloced
.dma
);
3908 beiscsi_unmap_pci_function(phba
);
3910 iscsi_host_remove(phba
->shost
);
3911 pci_dev_put(phba
->pcidev
);
3912 iscsi_host_free(phba
->shost
);
3914 pci_disable_device(pcidev
);
3918 struct iscsi_transport beiscsi_iscsi_transport
= {
3919 .owner
= THIS_MODULE
,
3921 .caps
= CAP_RECOVERY_L0
| CAP_HDRDGST
| CAP_TEXT_NEGO
|
3922 CAP_MULTI_R2T
| CAP_DATADGST
| CAP_DATA_PATH_OFFLOAD
,
3923 .param_mask
= ISCSI_MAX_RECV_DLENGTH
|
3924 ISCSI_MAX_XMIT_DLENGTH
|
3927 ISCSI_INITIAL_R2T_EN
|
3932 ISCSI_PDU_INORDER_EN
|
3933 ISCSI_DATASEQ_INORDER_EN
|
3936 ISCSI_CONN_ADDRESS
|
3938 ISCSI_PERSISTENT_PORT
|
3939 ISCSI_PERSISTENT_ADDRESS
|
3940 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
3941 ISCSI_USERNAME
| ISCSI_PASSWORD
|
3942 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
3943 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
3944 ISCSI_LU_RESET_TMO
|
3945 ISCSI_PING_TMO
| ISCSI_RECV_TMO
|
3946 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
3947 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
3948 ISCSI_HOST_INITIATOR_NAME
,
3949 .create_session
= beiscsi_session_create
,
3950 .destroy_session
= beiscsi_session_destroy
,
3951 .create_conn
= beiscsi_conn_create
,
3952 .bind_conn
= beiscsi_conn_bind
,
3953 .destroy_conn
= iscsi_conn_teardown
,
3954 .set_param
= beiscsi_set_param
,
3955 .get_conn_param
= beiscsi_conn_get_param
,
3956 .get_session_param
= iscsi_session_get_param
,
3957 .get_host_param
= beiscsi_get_host_param
,
3958 .start_conn
= beiscsi_conn_start
,
3959 .stop_conn
= beiscsi_conn_stop
,
3960 .send_pdu
= iscsi_conn_send_pdu
,
3961 .xmit_task
= beiscsi_task_xmit
,
3962 .cleanup_task
= beiscsi_cleanup_task
,
3963 .alloc_pdu
= beiscsi_alloc_pdu
,
3964 .parse_pdu_itt
= beiscsi_parse_pdu
,
3965 .get_stats
= beiscsi_conn_get_stats
,
3966 .ep_connect
= beiscsi_ep_connect
,
3967 .ep_poll
= beiscsi_ep_poll
,
3968 .ep_disconnect
= beiscsi_ep_disconnect
,
3969 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
3972 static struct pci_driver beiscsi_pci_driver
= {
3974 .probe
= beiscsi_dev_probe
,
3975 .remove
= beiscsi_remove
,
3976 .id_table
= beiscsi_pci_id_table
3980 static int __init
beiscsi_module_init(void)
3984 beiscsi_scsi_transport
=
3985 iscsi_register_transport(&beiscsi_iscsi_transport
);
3986 if (!beiscsi_scsi_transport
) {
3988 "beiscsi_module_init - Unable to register beiscsi"
3992 SE_DEBUG(DBG_LVL_8
, "In beiscsi_module_init, tt=%p \n",
3993 &beiscsi_iscsi_transport
);
3995 ret
= pci_register_driver(&beiscsi_pci_driver
);
3998 "beiscsi_module_init - Unable to register"
3999 "beiscsi pci driver.\n");
4000 goto unregister_iscsi_transport
;
4005 unregister_iscsi_transport
:
4006 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4010 static void __exit
beiscsi_module_exit(void)
4012 pci_unregister_driver(&beiscsi_pci_driver
);
4013 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4016 module_init(beiscsi_module_init
);
4017 module_exit(beiscsi_module_exit
);