?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
iommu-api: Add ->{un}map callbacks to iommu_ops
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / scsi / be2iscsi / be_main.c
blob7c22616ab141795a70e070e074ab4855e59df0cc
1 /**
2 * Copyright (C) 2005 - 2010 ServerEngines
3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
10 * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
11 *
12 * Contact Information:
13 * linux-drivers@serverengines.com
14 *
15 * ServerEngines
16 * 209 N. Fair Oaks Ave
17 * Sunnyvale, CA 94085
18 *
19 */
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/interrupt.h>
23 #include <linux/blkdev.h>
24 #include <linux/pci.h>
25 #include <linux/string.h>
26 #include <linux/kernel.h>
27 #include <linux/semaphore.h>
28
29 #include <scsi/libiscsi.h>
30 #include <scsi/scsi_transport_iscsi.h>
31 #include <scsi/scsi_transport.h>
32 #include <scsi/scsi_cmnd.h>
33 #include <scsi/scsi_device.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi.h>
36 #include "be_main.h"
37 #include "be_iscsi.h"
38 #include "be_mgmt.h"
39
40 static unsigned int be_iopoll_budget = 10;
41 static unsigned int be_max_phys_size = 64;
42 static unsigned int enable_msix = 1;
43
44 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
45 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
46 MODULE_AUTHOR("ServerEngines Corporation");
47 MODULE_LICENSE("GPL");
48 module_param(be_iopoll_budget, int, 0);
49 module_param(enable_msix, int, 0);
50 module_param(be_max_phys_size, uint, S_IRUGO);
51 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
52 "contiguous memory that can be allocated."
53 "Range is 16 - 128");
54
55 static int beiscsi_slave_configure(struct scsi_device *sdev)
56 {
57 blk_queue_max_segment_size(sdev->request_queue, 65536);
58 return 0;
59 }
60
61 /*------------------- PCI Driver operations and data ----------------- */
62 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
63 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
64 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
65 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
66 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
67 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
68 { 0 }
69 };
70 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
71
72 static struct scsi_host_template beiscsi_sht = {
73 .module = THIS_MODULE,
74 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
75 .proc_name = DRV_NAME,
76 .queuecommand = iscsi_queuecommand,
77 .eh_abort_handler = iscsi_eh_abort,
78 .change_queue_depth = iscsi_change_queue_depth,
79 .slave_configure = beiscsi_slave_configure,
80 .target_alloc = iscsi_target_alloc,
81 .eh_device_reset_handler = iscsi_eh_device_reset,
82 .eh_target_reset_handler = iscsi_eh_target_reset,
83 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
84 .can_queue = BE2_IO_DEPTH,
85 .this_id = -1,
86 .max_sectors = BEISCSI_MAX_SECTORS,
87 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
88 .use_clustering = ENABLE_CLUSTERING,
89 };
90
91 static struct scsi_transport_template *beiscsi_scsi_transport;
92
93 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
94 {
95 struct beiscsi_hba *phba;
96 struct Scsi_Host *shost;
97
98 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
99 if (!shost) {
100 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
101 "iscsi_host_alloc failed \n");
102 return NULL;
103 }
104 shost->dma_boundary = pcidev->dma_mask;
105 shost->max_id = BE2_MAX_SESSIONS;
106 shost->max_channel = 0;
107 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
108 shost->max_lun = BEISCSI_NUM_MAX_LUN;
109 shost->transportt = beiscsi_scsi_transport;
110 phba = iscsi_host_priv(shost);
111 memset(phba, 0, sizeof(*phba));
112 phba->shost = shost;
113 phba->pcidev = pci_dev_get(pcidev);
114 pci_set_drvdata(pcidev, phba);
115
116 if (iscsi_host_add(shost, &phba->pcidev->dev))
117 goto free_devices;
118 return phba;
119
120 free_devices:
121 pci_dev_put(phba->pcidev);
122 iscsi_host_free(phba->shost);
123 return NULL;
124 }
125
126 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
127 {
128 if (phba->csr_va) {
129 iounmap(phba->csr_va);
130 phba->csr_va = NULL;
131 }
132 if (phba->db_va) {
133 iounmap(phba->db_va);
134 phba->db_va = NULL;
135 }
136 if (phba->pci_va) {
137 iounmap(phba->pci_va);
138 phba->pci_va = NULL;
139 }
140 }
141
142 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
143 struct pci_dev *pcidev)
144 {
145 u8 __iomem *addr;
146 int pcicfg_reg;
147
148 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
149 pci_resource_len(pcidev, 2));
150 if (addr == NULL)
151 return -ENOMEM;
152 phba->ctrl.csr = addr;
153 phba->csr_va = addr;
154 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
155
156 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
157 if (addr == NULL)
158 goto pci_map_err;
159 phba->ctrl.db = addr;
160 phba->db_va = addr;
161 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
162
163 if (phba->generation == BE_GEN2)
164 pcicfg_reg = 1;
165 else
166 pcicfg_reg = 0;
167
168 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
169 pci_resource_len(pcidev, pcicfg_reg));
170
171 if (addr == NULL)
172 goto pci_map_err;
173 phba->ctrl.pcicfg = addr;
174 phba->pci_va = addr;
175 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
176 return 0;
177
178 pci_map_err:
179 beiscsi_unmap_pci_function(phba);
180 return -ENOMEM;
181 }
182
183 static int beiscsi_enable_pci(struct pci_dev *pcidev)
184 {
185 int ret;
186
187 ret = pci_enable_device(pcidev);
188 if (ret) {
189 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
190 "failed. Returning -ENODEV\n");
191 return ret;
192 }
193
194 pci_set_master(pcidev);
195 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
196 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
197 if (ret) {
198 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
199 pci_disable_device(pcidev);
200 return ret;
201 }
202 }
203 return 0;
204 }
205
206 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
207 {
208 struct be_ctrl_info *ctrl = &phba->ctrl;
209 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
210 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
211 int status = 0;
212
213 ctrl->pdev = pdev;
214 status = beiscsi_map_pci_bars(phba, pdev);
215 if (status)
216 return status;
217 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
218 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
219 mbox_mem_alloc->size,
220 &mbox_mem_alloc->dma);
221 if (!mbox_mem_alloc->va) {
222 beiscsi_unmap_pci_function(phba);
223 status = -ENOMEM;
224 return status;
225 }
226
227 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
228 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
229 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
230 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
231 spin_lock_init(&ctrl->mbox_lock);
232 spin_lock_init(&phba->ctrl.mcc_lock);
233 spin_lock_init(&phba->ctrl.mcc_cq_lock);
234
235 return status;
236 }
237
238 static void beiscsi_get_params(struct beiscsi_hba *phba)
239 {
240 phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
241 - (phba->fw_config.iscsi_cid_count
242 + BE2_TMFS
243 + BE2_NOPOUT_REQ));
244 phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
245 phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count;;
246 phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;;
247 phba->params.num_sge_per_io = BE2_SGE;
248 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
249 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
250 phba->params.eq_timer = 64;
251 phba->params.num_eq_entries =
252 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
253 + BE2_TMFS) / 512) + 1) * 512;
254 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
255 ? 1024 : phba->params.num_eq_entries;
256 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
257 phba->params.num_eq_entries);
258 phba->params.num_cq_entries =
259 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
260 + BE2_TMFS) / 512) + 1) * 512;
261 phba->params.wrbs_per_cxn = 256;
262 }
263
264 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
265 unsigned int id, unsigned int clr_interrupt,
266 unsigned int num_processed,
267 unsigned char rearm, unsigned char event)
268 {
269 u32 val = 0;
270 val |= id & DB_EQ_RING_ID_MASK;
271 if (rearm)
272 val |= 1 << DB_EQ_REARM_SHIFT;
273 if (clr_interrupt)
274 val |= 1 << DB_EQ_CLR_SHIFT;
275 if (event)
276 val |= 1 << DB_EQ_EVNT_SHIFT;
277 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
278 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
279 }
280
281 /**
282 * be_isr_mcc - The isr routine of the driver.
283 * @irq: Not used
284 * @dev_id: Pointer to host adapter structure
285 */
286 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
287 {
288 struct beiscsi_hba *phba;
289 struct be_eq_entry *eqe = NULL;
290 struct be_queue_info *eq;
291 struct be_queue_info *mcc;
292 unsigned int num_eq_processed;
293 struct be_eq_obj *pbe_eq;
294 unsigned long flags;
295
296 pbe_eq = dev_id;
297 eq = &pbe_eq->q;
298 phba = pbe_eq->phba;
299 mcc = &phba->ctrl.mcc_obj.cq;
300 eqe = queue_tail_node(eq);
301 if (!eqe)
302 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
303
304 num_eq_processed = 0;
305
306 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
307 & EQE_VALID_MASK) {
308 if (((eqe->dw[offsetof(struct amap_eq_entry,
309 resource_id) / 32] &
310 EQE_RESID_MASK) >> 16) == mcc->id) {
311 spin_lock_irqsave(&phba->isr_lock, flags);
312 phba->todo_mcc_cq = 1;
313 spin_unlock_irqrestore(&phba->isr_lock, flags);
314 }
315 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
316 queue_tail_inc(eq);
317 eqe = queue_tail_node(eq);
318 num_eq_processed++;
319 }
320 if (phba->todo_mcc_cq)
321 queue_work(phba->wq, &phba->work_cqs);
322 if (num_eq_processed)
323 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
324
325 return IRQ_HANDLED;
326 }
327
328 /**
329 * be_isr_msix - The isr routine of the driver.
330 * @irq: Not used
331 * @dev_id: Pointer to host adapter structure
332 */
333 static irqreturn_t be_isr_msix(int irq, void *dev_id)
334 {
335 struct beiscsi_hba *phba;
336 struct be_eq_entry *eqe = NULL;
337 struct be_queue_info *eq;
338 struct be_queue_info *cq;
339 unsigned int num_eq_processed;
340 struct be_eq_obj *pbe_eq;
341 unsigned long flags;
342
343 pbe_eq = dev_id;
344 eq = &pbe_eq->q;
345 cq = pbe_eq->cq;
346 eqe = queue_tail_node(eq);
347 if (!eqe)
348 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
349
350 phba = pbe_eq->phba;
351 num_eq_processed = 0;
352 if (blk_iopoll_enabled) {
353 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
354 & EQE_VALID_MASK) {
355 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
356 blk_iopoll_sched(&pbe_eq->iopoll);
357
358 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
359 queue_tail_inc(eq);
360 eqe = queue_tail_node(eq);
361 num_eq_processed++;
362 }
363 if (num_eq_processed)
364 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
365
366 return IRQ_HANDLED;
367 } else {
368 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
369 & EQE_VALID_MASK) {
370 spin_lock_irqsave(&phba->isr_lock, flags);
371 phba->todo_cq = 1;
372 spin_unlock_irqrestore(&phba->isr_lock, flags);
373 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
374 queue_tail_inc(eq);
375 eqe = queue_tail_node(eq);
376 num_eq_processed++;
377 }
378 if (phba->todo_cq)
379 queue_work(phba->wq, &phba->work_cqs);
380
381 if (num_eq_processed)
382 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
383
384 return IRQ_HANDLED;
385 }
386 }
387
388 /**
389 * be_isr - The isr routine of the driver.
390 * @irq: Not used
391 * @dev_id: Pointer to host adapter structure
392 */
393 static irqreturn_t be_isr(int irq, void *dev_id)
394 {
395 struct beiscsi_hba *phba;
396 struct hwi_controller *phwi_ctrlr;
397 struct hwi_context_memory *phwi_context;
398 struct be_eq_entry *eqe = NULL;
399 struct be_queue_info *eq;
400 struct be_queue_info *cq;
401 struct be_queue_info *mcc;
402 unsigned long flags, index;
403 unsigned int num_mcceq_processed, num_ioeq_processed;
404 struct be_ctrl_info *ctrl;
405 struct be_eq_obj *pbe_eq;
406 int isr;
407
408 phba = dev_id;
409 ctrl = &phba->ctrl;;
410 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
411 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
412 if (!isr)
413 return IRQ_NONE;
414
415 phwi_ctrlr = phba->phwi_ctrlr;
416 phwi_context = phwi_ctrlr->phwi_ctxt;
417 pbe_eq = &phwi_context->be_eq[0];
418
419 eq = &phwi_context->be_eq[0].q;
420 mcc = &phba->ctrl.mcc_obj.cq;
421 index = 0;
422 eqe = queue_tail_node(eq);
423 if (!eqe)
424 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
425
426 num_ioeq_processed = 0;
427 num_mcceq_processed = 0;
428 if (blk_iopoll_enabled) {
429 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
430 & EQE_VALID_MASK) {
431 if (((eqe->dw[offsetof(struct amap_eq_entry,
432 resource_id) / 32] &
433 EQE_RESID_MASK) >> 16) == mcc->id) {
434 spin_lock_irqsave(&phba->isr_lock, flags);
435 phba->todo_mcc_cq = 1;
436 spin_unlock_irqrestore(&phba->isr_lock, flags);
437 num_mcceq_processed++;
438 } else {
439 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
440 blk_iopoll_sched(&pbe_eq->iopoll);
441 num_ioeq_processed++;
442 }
443 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
444 queue_tail_inc(eq);
445 eqe = queue_tail_node(eq);
446 }
447 if (num_ioeq_processed || num_mcceq_processed) {
448 if (phba->todo_mcc_cq)
449 queue_work(phba->wq, &phba->work_cqs);
450
451 if ((num_mcceq_processed) && (!num_ioeq_processed))
452 hwi_ring_eq_db(phba, eq->id, 0,
453 (num_ioeq_processed +
454 num_mcceq_processed) , 1, 1);
455 else
456 hwi_ring_eq_db(phba, eq->id, 0,
457 (num_ioeq_processed +
458 num_mcceq_processed), 0, 1);
459
460 return IRQ_HANDLED;
461 } else
462 return IRQ_NONE;
463 } else {
464 cq = &phwi_context->be_cq[0];
465 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
466 & EQE_VALID_MASK) {
467
468 if (((eqe->dw[offsetof(struct amap_eq_entry,
469 resource_id) / 32] &
470 EQE_RESID_MASK) >> 16) != cq->id) {
471 spin_lock_irqsave(&phba->isr_lock, flags);
472 phba->todo_mcc_cq = 1;
473 spin_unlock_irqrestore(&phba->isr_lock, flags);
474 } else {
475 spin_lock_irqsave(&phba->isr_lock, flags);
476 phba->todo_cq = 1;
477 spin_unlock_irqrestore(&phba->isr_lock, flags);
478 }
479 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
480 queue_tail_inc(eq);
481 eqe = queue_tail_node(eq);
482 num_ioeq_processed++;
483 }
484 if (phba->todo_cq || phba->todo_mcc_cq)
485 queue_work(phba->wq, &phba->work_cqs);
486
487 if (num_ioeq_processed) {
488 hwi_ring_eq_db(phba, eq->id, 0,
489 num_ioeq_processed, 1, 1);
490 return IRQ_HANDLED;
491 } else
492 return IRQ_NONE;
493 }
494 }
495
496 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
497 {
498 struct pci_dev *pcidev = phba->pcidev;
499 struct hwi_controller *phwi_ctrlr;
500 struct hwi_context_memory *phwi_context;
501 int ret, msix_vec, i = 0;
502 char desc[32];
503
504 phwi_ctrlr = phba->phwi_ctrlr;
505 phwi_context = phwi_ctrlr->phwi_ctxt;
506
507 if (phba->msix_enabled) {
508 for (i = 0; i < phba->num_cpus; i++) {
509 sprintf(desc, "beiscsi_msix_%04x", i);
510 msix_vec = phba->msix_entries[i].vector;
511 ret = request_irq(msix_vec, be_isr_msix, 0, desc,
512 &phwi_context->be_eq[i]);
513 }
514 msix_vec = phba->msix_entries[i].vector;
515 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
516 &phwi_context->be_eq[i]);
517 } else {
518 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
519 "beiscsi", phba);
520 if (ret) {
521 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
522 "Failed to register irq\\n");
523 return ret;
524 }
525 }
526 return 0;
527 }
528
529 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
530 unsigned int id, unsigned int num_processed,
531 unsigned char rearm, unsigned char event)
532 {
533 u32 val = 0;
534 val |= id & DB_CQ_RING_ID_MASK;
535 if (rearm)
536 val |= 1 << DB_CQ_REARM_SHIFT;
537 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
538 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
539 }
540
541 static unsigned int
542 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
543 struct beiscsi_hba *phba,
544 unsigned short cid,
545 struct pdu_base *ppdu,
546 unsigned long pdu_len,
547 void *pbuffer, unsigned long buf_len)
548 {
549 struct iscsi_conn *conn = beiscsi_conn->conn;
550 struct iscsi_session *session = conn->session;
551 struct iscsi_task *task;
552 struct beiscsi_io_task *io_task;
553 struct iscsi_hdr *login_hdr;
554
555 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
556 PDUBASE_OPCODE_MASK) {
557 case ISCSI_OP_NOOP_IN:
558 pbuffer = NULL;
559 buf_len = 0;
560 break;
561 case ISCSI_OP_ASYNC_EVENT:
562 break;
563 case ISCSI_OP_REJECT:
564 WARN_ON(!pbuffer);
565 WARN_ON(!(buf_len == 48));
566 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
567 break;
568 case ISCSI_OP_LOGIN_RSP:
569 case ISCSI_OP_TEXT_RSP:
570 task = conn->login_task;
571 io_task = task->dd_data;
572 login_hdr = (struct iscsi_hdr *)ppdu;
573 login_hdr->itt = io_task->libiscsi_itt;
574 break;
575 default:
576 shost_printk(KERN_WARNING, phba->shost,
577 "Unrecognized opcode 0x%x in async msg \n",
578 (ppdu->
579 dw[offsetof(struct amap_pdu_base, opcode) / 32]
580 & PDUBASE_OPCODE_MASK));
581 return 1;
582 }
583
584 spin_lock_bh(&session->lock);
585 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
586 spin_unlock_bh(&session->lock);
587 return 0;
588 }
589
590 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
591 {
592 struct sgl_handle *psgl_handle;
593
594 if (phba->io_sgl_hndl_avbl) {
595 SE_DEBUG(DBG_LVL_8,
596 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
597 phba->io_sgl_alloc_index);
598 psgl_handle = phba->io_sgl_hndl_base[phba->
599 io_sgl_alloc_index];
600 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
601 phba->io_sgl_hndl_avbl--;
602 if (phba->io_sgl_alloc_index == (phba->params.
603 ios_per_ctrl - 1))
604 phba->io_sgl_alloc_index = 0;
605 else
606 phba->io_sgl_alloc_index++;
607 } else
608 psgl_handle = NULL;
609 return psgl_handle;
610 }
611
612 static void
613 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
614 {
615 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
616 phba->io_sgl_free_index);
617 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
618 /*
619 * this can happen if clean_task is called on a task that
620 * failed in xmit_task or alloc_pdu.
621 */
622 SE_DEBUG(DBG_LVL_8,
623 "Double Free in IO SGL io_sgl_free_index=%d,"
624 "value there=%p \n", phba->io_sgl_free_index,
625 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
626 return;
627 }
628 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
629 phba->io_sgl_hndl_avbl++;
630 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
631 phba->io_sgl_free_index = 0;
632 else
633 phba->io_sgl_free_index++;
634 }
635
636 /**
637 * alloc_wrb_handle - To allocate a wrb handle
638 * @phba: The hba pointer
639 * @cid: The cid to use for allocation
640 *
641 * This happens under session_lock until submission to chip
642 */
643 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
644 {
645 struct hwi_wrb_context *pwrb_context;
646 struct hwi_controller *phwi_ctrlr;
647 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
648
649 phwi_ctrlr = phba->phwi_ctrlr;
650 pwrb_context = &phwi_ctrlr->wrb_context[cid];
651 if (pwrb_context->wrb_handles_available >= 2) {
652 pwrb_handle = pwrb_context->pwrb_handle_base[
653 pwrb_context->alloc_index];
654 pwrb_context->wrb_handles_available--;
655 if (pwrb_context->alloc_index ==
656 (phba->params.wrbs_per_cxn - 1))
657 pwrb_context->alloc_index = 0;
658 else
659 pwrb_context->alloc_index++;
660 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
661 pwrb_context->alloc_index];
662 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
663 } else
664 pwrb_handle = NULL;
665 return pwrb_handle;
666 }
667
668 /**
669 * free_wrb_handle - To free the wrb handle back to pool
670 * @phba: The hba pointer
671 * @pwrb_context: The context to free from
672 * @pwrb_handle: The wrb_handle to free
673 *
674 * This happens under session_lock until submission to chip
675 */
676 static void
677 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
678 struct wrb_handle *pwrb_handle)
679 {
680 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
681 pwrb_context->wrb_handles_available++;
682 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
683 pwrb_context->free_index = 0;
684 else
685 pwrb_context->free_index++;
686
687 SE_DEBUG(DBG_LVL_8,
688 "FREE WRB: pwrb_handle=%p free_index=0x%x"
689 "wrb_handles_available=%d \n",
690 pwrb_handle, pwrb_context->free_index,
691 pwrb_context->wrb_handles_available);
692 }
693
694 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
695 {
696 struct sgl_handle *psgl_handle;
697
698 if (phba->eh_sgl_hndl_avbl) {
699 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
700 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
701 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
702 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
703 phba->eh_sgl_hndl_avbl--;
704 if (phba->eh_sgl_alloc_index ==
705 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
706 1))
707 phba->eh_sgl_alloc_index = 0;
708 else
709 phba->eh_sgl_alloc_index++;
710 } else
711 psgl_handle = NULL;
712 return psgl_handle;
713 }
714
715 void
716 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
717 {
718
719 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
720 phba->eh_sgl_free_index);
721 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
722 /*
723 * this can happen if clean_task is called on a task that
724 * failed in xmit_task or alloc_pdu.
725 */
726 SE_DEBUG(DBG_LVL_8,
727 "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
728 phba->eh_sgl_free_index);
729 return;
730 }
731 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
732 phba->eh_sgl_hndl_avbl++;
733 if (phba->eh_sgl_free_index ==
734 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
735 phba->eh_sgl_free_index = 0;
736 else
737 phba->eh_sgl_free_index++;
738 }
739
740 static void
741 be_complete_io(struct beiscsi_conn *beiscsi_conn,
742 struct iscsi_task *task, struct sol_cqe *psol)
743 {
744 struct beiscsi_io_task *io_task = task->dd_data;
745 struct be_status_bhs *sts_bhs =
746 (struct be_status_bhs *)io_task->cmd_bhs;
747 struct iscsi_conn *conn = beiscsi_conn->conn;
748 unsigned int sense_len;
749 unsigned char *sense;
750 u32 resid = 0, exp_cmdsn, max_cmdsn;
751 u8 rsp, status, flags;
752
753 exp_cmdsn = (psol->
754 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
755 & SOL_EXP_CMD_SN_MASK);
756 max_cmdsn = ((psol->
757 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
758 & SOL_EXP_CMD_SN_MASK) +
759 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
760 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
761 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
762 & SOL_RESP_MASK) >> 16);
763 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
764 & SOL_STS_MASK) >> 8);
765 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
766 & SOL_FLAGS_MASK) >> 24) | 0x80;
767
768 task->sc->result = (DID_OK << 16) | status;
769 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
770 task->sc->result = DID_ERROR << 16;
771 goto unmap;
772 }
773
774 /* bidi not initially supported */
775 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
776 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
777 32] & SOL_RES_CNT_MASK);
778
779 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
780 task->sc->result = DID_ERROR << 16;
781
782 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
783 scsi_set_resid(task->sc, resid);
784 if (!status && (scsi_bufflen(task->sc) - resid <
785 task->sc->underflow))
786 task->sc->result = DID_ERROR << 16;
787 }
788 }
789
790 if (status == SAM_STAT_CHECK_CONDITION) {
791 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
792 sense = sts_bhs->sense_info + sizeof(unsigned short);
793 sense_len = cpu_to_be16(*slen);
794 memcpy(task->sc->sense_buffer, sense,
795 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
796 }
797
798 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
799 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
800 & SOL_RES_CNT_MASK)
801 conn->rxdata_octets += (psol->
802 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
803 & SOL_RES_CNT_MASK);
804 }
805 unmap:
806 scsi_dma_unmap(io_task->scsi_cmnd);
807 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
808 }
809
810 static void
811 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
812 struct iscsi_task *task, struct sol_cqe *psol)
813 {
814 struct iscsi_logout_rsp *hdr;
815 struct beiscsi_io_task *io_task = task->dd_data;
816 struct iscsi_conn *conn = beiscsi_conn->conn;
817
818 hdr = (struct iscsi_logout_rsp *)task->hdr;
819 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
820 hdr->t2wait = 5;
821 hdr->t2retain = 0;
822 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
823 & SOL_FLAGS_MASK) >> 24) | 0x80;
824 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
825 32] & SOL_RESP_MASK);
826 hdr->exp_cmdsn = cpu_to_be32(psol->
827 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
828 & SOL_EXP_CMD_SN_MASK);
829 hdr->max_cmdsn = be32_to_cpu((psol->
830 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
831 & SOL_EXP_CMD_SN_MASK) +
832 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
833 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
834 hdr->dlength[0] = 0;
835 hdr->dlength[1] = 0;
836 hdr->dlength[2] = 0;
837 hdr->hlength = 0;
838 hdr->itt = io_task->libiscsi_itt;
839 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
840 }
841
842 static void
843 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
844 struct iscsi_task *task, struct sol_cqe *psol)
845 {
846 struct iscsi_tm_rsp *hdr;
847 struct iscsi_conn *conn = beiscsi_conn->conn;
848 struct beiscsi_io_task *io_task = task->dd_data;
849
850 hdr = (struct iscsi_tm_rsp *)task->hdr;
851 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
852 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
853 & SOL_FLAGS_MASK) >> 24) | 0x80;
854 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
855 32] & SOL_RESP_MASK);
856 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
857 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
858 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
859 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
860 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
861 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
862 hdr->itt = io_task->libiscsi_itt;
863 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
864 }
865
866 static void
867 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
868 struct beiscsi_hba *phba, struct sol_cqe *psol)
869 {
870 struct hwi_wrb_context *pwrb_context;
871 struct wrb_handle *pwrb_handle = NULL;
872 struct hwi_controller *phwi_ctrlr;
873 struct iscsi_task *task;
874 struct beiscsi_io_task *io_task;
875 struct iscsi_conn *conn = beiscsi_conn->conn;
876 struct iscsi_session *session = conn->session;
877
878 phwi_ctrlr = phba->phwi_ctrlr;
879 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
880 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
881 SOL_CID_MASK) >> 6) -
882 phba->fw_config.iscsi_cid_start];
883 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
884 dw[offsetof(struct amap_sol_cqe, wrb_index) /
885 32] & SOL_WRB_INDEX_MASK) >> 16)];
886 task = pwrb_handle->pio_handle;
887
888 io_task = task->dd_data;
889 spin_lock(&phba->mgmt_sgl_lock);
890 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
891 spin_unlock(&phba->mgmt_sgl_lock);
892 spin_lock_bh(&session->lock);
893 free_wrb_handle(phba, pwrb_context, pwrb_handle);
894 spin_unlock_bh(&session->lock);
895 }
896
897 static void
898 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
899 struct iscsi_task *task, struct sol_cqe *psol)
900 {
901 struct iscsi_nopin *hdr;
902 struct iscsi_conn *conn = beiscsi_conn->conn;
903 struct beiscsi_io_task *io_task = task->dd_data;
904
905 hdr = (struct iscsi_nopin *)task->hdr;
906 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
907 & SOL_FLAGS_MASK) >> 24) | 0x80;
908 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
909 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
910 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
911 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
912 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
913 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
914 hdr->opcode = ISCSI_OP_NOOP_IN;
915 hdr->itt = io_task->libiscsi_itt;
916 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
917 }
918
919 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
920 struct beiscsi_hba *phba, struct sol_cqe *psol)
921 {
922 struct hwi_wrb_context *pwrb_context;
923 struct wrb_handle *pwrb_handle;
924 struct iscsi_wrb *pwrb = NULL;
925 struct hwi_controller *phwi_ctrlr;
926 struct iscsi_task *task;
927 unsigned int type;
928 struct iscsi_conn *conn = beiscsi_conn->conn;
929 struct iscsi_session *session = conn->session;
930
931 phwi_ctrlr = phba->phwi_ctrlr;
932 pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
933 (struct amap_sol_cqe, cid) / 32]
934 & SOL_CID_MASK) >> 6) -
935 phba->fw_config.iscsi_cid_start];
936 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
937 dw[offsetof(struct amap_sol_cqe, wrb_index) /
938 32] & SOL_WRB_INDEX_MASK) >> 16)];
939 task = pwrb_handle->pio_handle;
940 pwrb = pwrb_handle->pwrb;
941 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
942 WRB_TYPE_MASK) >> 28;
943
944 spin_lock_bh(&session->lock);
945 switch (type) {
946 case HWH_TYPE_IO:
947 case HWH_TYPE_IO_RD:
948 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
949 ISCSI_OP_NOOP_OUT) {
950 be_complete_nopin_resp(beiscsi_conn, task, psol);
951 } else
952 be_complete_io(beiscsi_conn, task, psol);
953 break;
954
955 case HWH_TYPE_LOGOUT:
956 be_complete_logout(beiscsi_conn, task, psol);
957 break;
958
959 case HWH_TYPE_LOGIN:
960 SE_DEBUG(DBG_LVL_1,
961 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
962 "- Solicited path \n");
963 break;
964
965 case HWH_TYPE_TMF:
966 be_complete_tmf(beiscsi_conn, task, psol);
967 break;
968
969 case HWH_TYPE_NOP:
970 be_complete_nopin_resp(beiscsi_conn, task, psol);
971 break;
972
973 default:
974 shost_printk(KERN_WARNING, phba->shost,
975 "In hwi_complete_cmd, unknown type = %d"
976 "wrb_index 0x%x CID 0x%x\n", type,
977 ((psol->dw[offsetof(struct amap_iscsi_wrb,
978 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
979 ((psol->dw[offsetof(struct amap_sol_cqe,
980 cid) / 32] & SOL_CID_MASK) >> 6));
981 break;
982 }
983
984 spin_unlock_bh(&session->lock);
985 }
986
987 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
988 *pasync_ctx, unsigned int is_header,
989 unsigned int host_write_ptr)
990 {
991 if (is_header)
992 return &pasync_ctx->async_entry[host_write_ptr].
993 header_busy_list;
994 else
995 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
996 }
997
998 static struct async_pdu_handle *
999 hwi_get_async_handle(struct beiscsi_hba *phba,
1000 struct beiscsi_conn *beiscsi_conn,
1001 struct hwi_async_pdu_context *pasync_ctx,
1002 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1003 {
1004 struct be_bus_address phys_addr;
1005 struct list_head *pbusy_list;
1006 struct async_pdu_handle *pasync_handle = NULL;
1007 int buffer_len = 0;
1008 unsigned char buffer_index = -1;
1009 unsigned char is_header = 0;
1010
1011 phys_addr.u.a32.address_lo =
1012 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1013 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1014 & PDUCQE_DPL_MASK) >> 16);
1015 phys_addr.u.a32.address_hi =
1016 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1017
1018 phys_addr.u.a64.address =
1019 *((unsigned long long *)(&phys_addr.u.a64.address));
1020
1021 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1022 & PDUCQE_CODE_MASK) {
1023 case UNSOL_HDR_NOTIFY:
1024 is_header = 1;
1025
1026 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1027 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1028 index) / 32] & PDUCQE_INDEX_MASK));
1029
1030 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1031 pasync_ctx->async_header.pa_base.u.a64.address);
1032
1033 buffer_index = buffer_len /
1034 pasync_ctx->async_header.buffer_size;
1035
1036 break;
1037 case UNSOL_DATA_NOTIFY:
1038 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1039 dw[offsetof(struct amap_i_t_dpdu_cqe,
1040 index) / 32] & PDUCQE_INDEX_MASK));
1041 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1042 pasync_ctx->async_data.pa_base.u.
1043 a64.address);
1044 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1045 break;
1046 default:
1047 pbusy_list = NULL;
1048 shost_printk(KERN_WARNING, phba->shost,
1049 "Unexpected code=%d \n",
1050 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1051 code) / 32] & PDUCQE_CODE_MASK);
1052 return NULL;
1053 }
1054
1055 WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1056 WARN_ON(list_empty(pbusy_list));
1057 list_for_each_entry(pasync_handle, pbusy_list, link) {
1058 WARN_ON(pasync_handle->consumed);
1059 if (pasync_handle->index == buffer_index)
1060 break;
1061 }
1062
1063 WARN_ON(!pasync_handle);
1064
1065 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1066 phba->fw_config.iscsi_cid_start;
1067 pasync_handle->is_header = is_header;
1068 pasync_handle->buffer_len = ((pdpdu_cqe->
1069 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1070 & PDUCQE_DPL_MASK) >> 16);
1071
1072 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1073 index) / 32] & PDUCQE_INDEX_MASK);
1074 return pasync_handle;
1075 }
1076
1077 static unsigned int
1078 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1079 unsigned int is_header, unsigned int cq_index)
1080 {
1081 struct list_head *pbusy_list;
1082 struct async_pdu_handle *pasync_handle;
1083 unsigned int num_entries, writables = 0;
1084 unsigned int *pep_read_ptr, *pwritables;
1085
1086
1087 if (is_header) {
1088 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1089 pwritables = &pasync_ctx->async_header.writables;
1090 num_entries = pasync_ctx->async_header.num_entries;
1091 } else {
1092 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1093 pwritables = &pasync_ctx->async_data.writables;
1094 num_entries = pasync_ctx->async_data.num_entries;
1095 }
1096
1097 while ((*pep_read_ptr) != cq_index) {
1098 (*pep_read_ptr)++;
1099 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1100
1101 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1102 *pep_read_ptr);
1103 if (writables == 0)
1104 WARN_ON(list_empty(pbusy_list));
1105
1106 if (!list_empty(pbusy_list)) {
1107 pasync_handle = list_entry(pbusy_list->next,
1108 struct async_pdu_handle,
1109 link);
1110 WARN_ON(!pasync_handle);
1111 pasync_handle->consumed = 1;
1112 }
1113
1114 writables++;
1115 }
1116
1117 if (!writables) {
1118 SE_DEBUG(DBG_LVL_1,
1119 "Duplicate notification received - index 0x%x!!\n",
1120 cq_index);
1121 WARN_ON(1);
1122 }
1123
1124 *pwritables = *pwritables + writables;
1125 return 0;
1126 }
1127
1128 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1129 unsigned int cri)
1130 {
1131 struct hwi_controller *phwi_ctrlr;
1132 struct hwi_async_pdu_context *pasync_ctx;
1133 struct async_pdu_handle *pasync_handle, *tmp_handle;
1134 struct list_head *plist;
1135 unsigned int i = 0;
1136
1137 phwi_ctrlr = phba->phwi_ctrlr;
1138 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1139
1140 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1141
1142 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1143 list_del(&pasync_handle->link);
1144
1145 if (i == 0) {
1146 list_add_tail(&pasync_handle->link,
1147 &pasync_ctx->async_header.free_list);
1148 pasync_ctx->async_header.free_entries++;
1149 i++;
1150 } else {
1151 list_add_tail(&pasync_handle->link,
1152 &pasync_ctx->async_data.free_list);
1153 pasync_ctx->async_data.free_entries++;
1154 i++;
1155 }
1156 }
1157
1158 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1159 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1160 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1161 return 0;
1162 }
1163
1164 static struct phys_addr *
1165 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1166 unsigned int is_header, unsigned int host_write_ptr)
1167 {
1168 struct phys_addr *pasync_sge = NULL;
1169
1170 if (is_header)
1171 pasync_sge = pasync_ctx->async_header.ring_base;
1172 else
1173 pasync_sge = pasync_ctx->async_data.ring_base;
1174
1175 return pasync_sge + host_write_ptr;
1176 }
1177
1178 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1179 unsigned int is_header)
1180 {
1181 struct hwi_controller *phwi_ctrlr;
1182 struct hwi_async_pdu_context *pasync_ctx;
1183 struct async_pdu_handle *pasync_handle;
1184 struct list_head *pfree_link, *pbusy_list;
1185 struct phys_addr *pasync_sge;
1186 unsigned int ring_id, num_entries;
1187 unsigned int host_write_num;
1188 unsigned int writables;
1189 unsigned int i = 0;
1190 u32 doorbell = 0;
1191
1192 phwi_ctrlr = phba->phwi_ctrlr;
1193 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1194
1195 if (is_header) {
1196 num_entries = pasync_ctx->async_header.num_entries;
1197 writables = min(pasync_ctx->async_header.writables,
1198 pasync_ctx->async_header.free_entries);
1199 pfree_link = pasync_ctx->async_header.free_list.next;
1200 host_write_num = pasync_ctx->async_header.host_write_ptr;
1201 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1202 } else {
1203 num_entries = pasync_ctx->async_data.num_entries;
1204 writables = min(pasync_ctx->async_data.writables,
1205 pasync_ctx->async_data.free_entries);
1206 pfree_link = pasync_ctx->async_data.free_list.next;
1207 host_write_num = pasync_ctx->async_data.host_write_ptr;
1208 ring_id = phwi_ctrlr->default_pdu_data.id;
1209 }
1210
1211 writables = (writables / 8) * 8;
1212 if (writables) {
1213 for (i = 0; i < writables; i++) {
1214 pbusy_list =
1215 hwi_get_async_busy_list(pasync_ctx, is_header,
1216 host_write_num);
1217 pasync_handle =
1218 list_entry(pfree_link, struct async_pdu_handle,
1219 link);
1220 WARN_ON(!pasync_handle);
1221 pasync_handle->consumed = 0;
1222
1223 pfree_link = pfree_link->next;
1224
1225 pasync_sge = hwi_get_ring_address(pasync_ctx,
1226 is_header, host_write_num);
1227
1228 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1229 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1230
1231 list_move(&pasync_handle->link, pbusy_list);
1232
1233 host_write_num++;
1234 host_write_num = host_write_num % num_entries;
1235 }
1236
1237 if (is_header) {
1238 pasync_ctx->async_header.host_write_ptr =
1239 host_write_num;
1240 pasync_ctx->async_header.free_entries -= writables;
1241 pasync_ctx->async_header.writables -= writables;
1242 pasync_ctx->async_header.busy_entries += writables;
1243 } else {
1244 pasync_ctx->async_data.host_write_ptr = host_write_num;
1245 pasync_ctx->async_data.free_entries -= writables;
1246 pasync_ctx->async_data.writables -= writables;
1247 pasync_ctx->async_data.busy_entries += writables;
1248 }
1249
1250 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1251 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1252 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1253 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1254 << DB_DEF_PDU_CQPROC_SHIFT;
1255
1256 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1257 }
1258 }
1259
1260 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1261 struct beiscsi_conn *beiscsi_conn,
1262 struct i_t_dpdu_cqe *pdpdu_cqe)
1263 {
1264 struct hwi_controller *phwi_ctrlr;
1265 struct hwi_async_pdu_context *pasync_ctx;
1266 struct async_pdu_handle *pasync_handle = NULL;
1267 unsigned int cq_index = -1;
1268
1269 phwi_ctrlr = phba->phwi_ctrlr;
1270 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1271
1272 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1273 pdpdu_cqe, &cq_index);
1274 BUG_ON(pasync_handle->is_header != 0);
1275 if (pasync_handle->consumed == 0)
1276 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1277 cq_index);
1278
1279 hwi_free_async_msg(phba, pasync_handle->cri);
1280 hwi_post_async_buffers(phba, pasync_handle->is_header);
1281 }
1282
1283 static unsigned int
1284 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1285 struct beiscsi_hba *phba,
1286 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1287 {
1288 struct list_head *plist;
1289 struct async_pdu_handle *pasync_handle;
1290 void *phdr = NULL;
1291 unsigned int hdr_len = 0, buf_len = 0;
1292 unsigned int status, index = 0, offset = 0;
1293 void *pfirst_buffer = NULL;
1294 unsigned int num_buf = 0;
1295
1296 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1297
1298 list_for_each_entry(pasync_handle, plist, link) {
1299 if (index == 0) {
1300 phdr = pasync_handle->pbuffer;
1301 hdr_len = pasync_handle->buffer_len;
1302 } else {
1303 buf_len = pasync_handle->buffer_len;
1304 if (!num_buf) {
1305 pfirst_buffer = pasync_handle->pbuffer;
1306 num_buf++;
1307 }
1308 memcpy(pfirst_buffer + offset,
1309 pasync_handle->pbuffer, buf_len);
1310 offset = buf_len;
1311 }
1312 index++;
1313 }
1314
1315 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1316 (beiscsi_conn->beiscsi_conn_cid -
1317 phba->fw_config.iscsi_cid_start),
1318 phdr, hdr_len, pfirst_buffer,
1319 buf_len);
1320
1321 if (status == 0)
1322 hwi_free_async_msg(phba, cri);
1323 return 0;
1324 }
1325
1326 static unsigned int
1327 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1328 struct beiscsi_hba *phba,
1329 struct async_pdu_handle *pasync_handle)
1330 {
1331 struct hwi_async_pdu_context *pasync_ctx;
1332 struct hwi_controller *phwi_ctrlr;
1333 unsigned int bytes_needed = 0, status = 0;
1334 unsigned short cri = pasync_handle->cri;
1335 struct pdu_base *ppdu;
1336
1337 phwi_ctrlr = phba->phwi_ctrlr;
1338 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1339
1340 list_del(&pasync_handle->link);
1341 if (pasync_handle->is_header) {
1342 pasync_ctx->async_header.busy_entries--;
1343 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1344 hwi_free_async_msg(phba, cri);
1345 BUG();
1346 }
1347
1348 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1349 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1350 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1351 (unsigned short)pasync_handle->buffer_len;
1352 list_add_tail(&pasync_handle->link,
1353 &pasync_ctx->async_entry[cri].wait_queue.list);
1354
1355 ppdu = pasync_handle->pbuffer;
1356 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1357 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1358 0xFFFF0000) | ((be16_to_cpu((ppdu->
1359 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1360 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1361
1362 if (status == 0) {
1363 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1364 bytes_needed;
1365
1366 if (bytes_needed == 0)
1367 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1368 pasync_ctx, cri);
1369 }
1370 } else {
1371 pasync_ctx->async_data.busy_entries--;
1372 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1373 list_add_tail(&pasync_handle->link,
1374 &pasync_ctx->async_entry[cri].wait_queue.
1375 list);
1376 pasync_ctx->async_entry[cri].wait_queue.
1377 bytes_received +=
1378 (unsigned short)pasync_handle->buffer_len;
1379
1380 if (pasync_ctx->async_entry[cri].wait_queue.
1381 bytes_received >=
1382 pasync_ctx->async_entry[cri].wait_queue.
1383 bytes_needed)
1384 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1385 pasync_ctx, cri);
1386 }
1387 }
1388 return status;
1389 }
1390
1391 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1392 struct beiscsi_hba *phba,
1393 struct i_t_dpdu_cqe *pdpdu_cqe)
1394 {
1395 struct hwi_controller *phwi_ctrlr;
1396 struct hwi_async_pdu_context *pasync_ctx;
1397 struct async_pdu_handle *pasync_handle = NULL;
1398 unsigned int cq_index = -1;
1399
1400 phwi_ctrlr = phba->phwi_ctrlr;
1401 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1402 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1403 pdpdu_cqe, &cq_index);
1404
1405 if (pasync_handle->consumed == 0)
1406 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1407 cq_index);
1408 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1409 hwi_post_async_buffers(phba, pasync_handle->is_header);
1410 }
1411
1412 static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1413 {
1414 struct be_queue_info *mcc_cq;
1415 struct be_mcc_compl *mcc_compl;
1416 unsigned int num_processed = 0;
1417
1418 mcc_cq = &phba->ctrl.mcc_obj.cq;
1419 mcc_compl = queue_tail_node(mcc_cq);
1420 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1421 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1422
1423 if (num_processed >= 32) {
1424 hwi_ring_cq_db(phba, mcc_cq->id,
1425 num_processed, 0, 0);
1426 num_processed = 0;
1427 }
1428 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1429 /* Interpret flags as an async trailer */
1430 if (is_link_state_evt(mcc_compl->flags))
1431 /* Interpret compl as a async link evt */
1432 beiscsi_async_link_state_process(phba,
1433 (struct be_async_event_link_state *) mcc_compl);
1434 else
1435 SE_DEBUG(DBG_LVL_1,
1436 " Unsupported Async Event, flags"
1437 " = 0x%08x \n", mcc_compl->flags);
1438 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1439 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1440 atomic_dec(&phba->ctrl.mcc_obj.q.used);
1441 }
1442
1443 mcc_compl->flags = 0;
1444 queue_tail_inc(mcc_cq);
1445 mcc_compl = queue_tail_node(mcc_cq);
1446 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1447 num_processed++;
1448 }
1449
1450 if (num_processed > 0)
1451 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1452
1453 }
1454
1455 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1456 {
1457 struct be_queue_info *cq;
1458 struct sol_cqe *sol;
1459 struct dmsg_cqe *dmsg;
1460 unsigned int num_processed = 0;
1461 unsigned int tot_nump = 0;
1462 struct beiscsi_conn *beiscsi_conn;
1463 struct beiscsi_endpoint *beiscsi_ep;
1464 struct iscsi_endpoint *ep;
1465 struct beiscsi_hba *phba;
1466
1467 cq = pbe_eq->cq;
1468 sol = queue_tail_node(cq);
1469 phba = pbe_eq->phba;
1470
1471 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1472 CQE_VALID_MASK) {
1473 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1474
1475 ep = phba->ep_array[(u32) ((sol->
1476 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1477 SOL_CID_MASK) >> 6) -
1478 phba->fw_config.iscsi_cid_start];
1479
1480 beiscsi_ep = ep->dd_data;
1481 beiscsi_conn = beiscsi_ep->conn;
1482
1483 if (num_processed >= 32) {
1484 hwi_ring_cq_db(phba, cq->id,
1485 num_processed, 0, 0);
1486 tot_nump += num_processed;
1487 num_processed = 0;
1488 }
1489
1490 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1491 32] & CQE_CODE_MASK) {
1492 case SOL_CMD_COMPLETE:
1493 hwi_complete_cmd(beiscsi_conn, phba, sol);
1494 break;
1495 case DRIVERMSG_NOTIFY:
1496 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1497 dmsg = (struct dmsg_cqe *)sol;
1498 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1499 break;
1500 case UNSOL_HDR_NOTIFY:
1501 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1502 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1503 (struct i_t_dpdu_cqe *)sol);
1504 break;
1505 case UNSOL_DATA_NOTIFY:
1506 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1507 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1508 (struct i_t_dpdu_cqe *)sol);
1509 break;
1510 case CXN_INVALIDATE_INDEX_NOTIFY:
1511 case CMD_INVALIDATED_NOTIFY:
1512 case CXN_INVALIDATE_NOTIFY:
1513 SE_DEBUG(DBG_LVL_1,
1514 "Ignoring CQ Error notification for cmd/cxn"
1515 "invalidate\n");
1516 break;
1517 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1518 case CMD_KILLED_INVALID_STATSN_RCVD:
1519 case CMD_KILLED_INVALID_R2T_RCVD:
1520 case CMD_CXN_KILLED_LUN_INVALID:
1521 case CMD_CXN_KILLED_ICD_INVALID:
1522 case CMD_CXN_KILLED_ITT_INVALID:
1523 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1524 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1525 SE_DEBUG(DBG_LVL_1,
1526 "CQ Error notification for cmd.. "
1527 "code %d cid 0x%x\n",
1528 sol->dw[offsetof(struct amap_sol_cqe, code) /
1529 32] & CQE_CODE_MASK,
1530 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1531 32] & SOL_CID_MASK));
1532 break;
1533 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1534 SE_DEBUG(DBG_LVL_1,
1535 "Digest error on def pdu ring, dropping..\n");
1536 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1537 (struct i_t_dpdu_cqe *) sol);
1538 break;
1539 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1540 case CXN_KILLED_BURST_LEN_MISMATCH:
1541 case CXN_KILLED_AHS_RCVD:
1542 case CXN_KILLED_HDR_DIGEST_ERR:
1543 case CXN_KILLED_UNKNOWN_HDR:
1544 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1545 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1546 case CXN_KILLED_TIMED_OUT:
1547 case CXN_KILLED_FIN_RCVD:
1548 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1549 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1550 case CXN_KILLED_OVER_RUN_RESIDUAL:
1551 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1552 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1553 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1554 "0x%x...\n",
1555 sol->dw[offsetof(struct amap_sol_cqe, code) /
1556 32] & CQE_CODE_MASK,
1557 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1558 32] & CQE_CID_MASK));
1559 iscsi_conn_failure(beiscsi_conn->conn,
1560 ISCSI_ERR_CONN_FAILED);
1561 break;
1562 case CXN_KILLED_RST_SENT:
1563 case CXN_KILLED_RST_RCVD:
1564 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1565 "received/sent on CID 0x%x...\n",
1566 sol->dw[offsetof(struct amap_sol_cqe, code) /
1567 32] & CQE_CODE_MASK,
1568 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1569 32] & CQE_CID_MASK));
1570 iscsi_conn_failure(beiscsi_conn->conn,
1571 ISCSI_ERR_CONN_FAILED);
1572 break;
1573 default:
1574 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1575 "received on CID 0x%x...\n",
1576 sol->dw[offsetof(struct amap_sol_cqe, code) /
1577 32] & CQE_CODE_MASK,
1578 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1579 32] & CQE_CID_MASK));
1580 break;
1581 }
1582
1583 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1584 queue_tail_inc(cq);
1585 sol = queue_tail_node(cq);
1586 num_processed++;
1587 }
1588
1589 if (num_processed > 0) {
1590 tot_nump += num_processed;
1591 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1592 }
1593 return tot_nump;
1594 }
1595
1596 void beiscsi_process_all_cqs(struct work_struct *work)
1597 {
1598 unsigned long flags;
1599 struct hwi_controller *phwi_ctrlr;
1600 struct hwi_context_memory *phwi_context;
1601 struct be_eq_obj *pbe_eq;
1602 struct beiscsi_hba *phba =
1603 container_of(work, struct beiscsi_hba, work_cqs);
1604
1605 phwi_ctrlr = phba->phwi_ctrlr;
1606 phwi_context = phwi_ctrlr->phwi_ctxt;
1607 if (phba->msix_enabled)
1608 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1609 else
1610 pbe_eq = &phwi_context->be_eq[0];
1611
1612 if (phba->todo_mcc_cq) {
1613 spin_lock_irqsave(&phba->isr_lock, flags);
1614 phba->todo_mcc_cq = 0;
1615 spin_unlock_irqrestore(&phba->isr_lock, flags);
1616 beiscsi_process_mcc_isr(phba);
1617 }
1618
1619 if (phba->todo_cq) {
1620 spin_lock_irqsave(&phba->isr_lock, flags);
1621 phba->todo_cq = 0;
1622 spin_unlock_irqrestore(&phba->isr_lock, flags);
1623 beiscsi_process_cq(pbe_eq);
1624 }
1625 }
1626
1627 static int be_iopoll(struct blk_iopoll *iop, int budget)
1628 {
1629 static unsigned int ret;
1630 struct beiscsi_hba *phba;
1631 struct be_eq_obj *pbe_eq;
1632
1633 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1634 ret = beiscsi_process_cq(pbe_eq);
1635 if (ret < budget) {
1636 phba = pbe_eq->phba;
1637 blk_iopoll_complete(iop);
1638 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1639 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1640 }
1641 return ret;
1642 }
1643
1644 static void
1645 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1646 unsigned int num_sg, struct beiscsi_io_task *io_task)
1647 {
1648 struct iscsi_sge *psgl;
1649 unsigned short sg_len, index;
1650 unsigned int sge_len = 0;
1651 unsigned long long addr;
1652 struct scatterlist *l_sg;
1653 unsigned int offset;
1654
1655 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1656 io_task->bhs_pa.u.a32.address_lo);
1657 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1658 io_task->bhs_pa.u.a32.address_hi);
1659
1660 l_sg = sg;
1661 for (index = 0; (index < num_sg) && (index < 2); index++,
1662 sg = sg_next(sg)) {
1663 if (index == 0) {
1664 sg_len = sg_dma_len(sg);
1665 addr = (u64) sg_dma_address(sg);
1666 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1667 (addr & 0xFFFFFFFF));
1668 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1669 (addr >> 32));
1670 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1671 sg_len);
1672 sge_len = sg_len;
1673 } else {
1674 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1675 pwrb, sge_len);
1676 sg_len = sg_dma_len(sg);
1677 addr = (u64) sg_dma_address(sg);
1678 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1679 (addr & 0xFFFFFFFF));
1680 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1681 (addr >> 32));
1682 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1683 sg_len);
1684 }
1685 }
1686 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1687 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1688
1689 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1690
1691 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1692 io_task->bhs_pa.u.a32.address_hi);
1693 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1694 io_task->bhs_pa.u.a32.address_lo);
1695
1696 if (num_sg == 1) {
1697 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1698 1);
1699 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1700 0);
1701 } else if (num_sg == 2) {
1702 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1703 0);
1704 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1705 1);
1706 } else {
1707 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1708 0);
1709 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1710 0);
1711 }
1712 sg = l_sg;
1713 psgl++;
1714 psgl++;
1715 offset = 0;
1716 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
1717 sg_len = sg_dma_len(sg);
1718 addr = (u64) sg_dma_address(sg);
1719 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1720 (addr & 0xFFFFFFFF));
1721 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1722 (addr >> 32));
1723 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1724 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1725 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1726 offset += sg_len;
1727 }
1728 psgl--;
1729 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1730 }
1731
1732 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1733 {
1734 struct iscsi_sge *psgl;
1735 unsigned long long addr;
1736 struct beiscsi_io_task *io_task = task->dd_data;
1737 struct beiscsi_conn *beiscsi_conn = io_task->conn;
1738 struct beiscsi_hba *phba = beiscsi_conn->phba;
1739
1740 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1741 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1742 io_task->bhs_pa.u.a32.address_lo);
1743 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1744 io_task->bhs_pa.u.a32.address_hi);
1745
1746 if (task->data) {
1747 if (task->data_count) {
1748 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1749 addr = (u64) pci_map_single(phba->pcidev,
1750 task->data,
1751 task->data_count, 1);
1752 } else {
1753 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1754 addr = 0;
1755 }
1756 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1757 (addr & 0xFFFFFFFF));
1758 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1759 (addr >> 32));
1760 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1761 task->data_count);
1762
1763 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1764 } else {
1765 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1766 addr = 0;
1767 }
1768
1769 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1770
1771 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1772
1773 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1774 io_task->bhs_pa.u.a32.address_hi);
1775 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1776 io_task->bhs_pa.u.a32.address_lo);
1777 if (task->data) {
1778 psgl++;
1779 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1780 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1781 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1782 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1783 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1784 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1785
1786 psgl++;
1787 if (task->data) {
1788 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1789 (addr & 0xFFFFFFFF));
1790 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1791 (addr >> 32));
1792 }
1793 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1794 }
1795 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1796 }
1797
1798 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1799 {
1800 unsigned int num_cq_pages, num_async_pdu_buf_pages;
1801 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1802 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1803
1804 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1805 sizeof(struct sol_cqe));
1806 num_async_pdu_buf_pages =
1807 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1808 phba->params.defpdu_hdr_sz);
1809 num_async_pdu_buf_sgl_pages =
1810 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1811 sizeof(struct phys_addr));
1812 num_async_pdu_data_pages =
1813 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1814 phba->params.defpdu_data_sz);
1815 num_async_pdu_data_sgl_pages =
1816 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1817 sizeof(struct phys_addr));
1818
1819 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1820
1821 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1822 BE_ISCSI_PDU_HEADER_SIZE;
1823 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1824 sizeof(struct hwi_context_memory);
1825
1826
1827 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1828 * (phba->params.wrbs_per_cxn)
1829 * phba->params.cxns_per_ctrl;
1830 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
1831 (phba->params.wrbs_per_cxn);
1832 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1833 phba->params.cxns_per_ctrl);
1834
1835 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1836 phba->params.icds_per_ctrl;
1837 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1838 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1839
1840 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1841 num_async_pdu_buf_pages * PAGE_SIZE;
1842 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1843 num_async_pdu_data_pages * PAGE_SIZE;
1844 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1845 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1846 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1847 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1848 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1849 phba->params.asyncpdus_per_ctrl *
1850 sizeof(struct async_pdu_handle);
1851 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1852 phba->params.asyncpdus_per_ctrl *
1853 sizeof(struct async_pdu_handle);
1854 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1855 sizeof(struct hwi_async_pdu_context) +
1856 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1857 }
1858
1859 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1860 {
1861 struct be_mem_descriptor *mem_descr;
1862 dma_addr_t bus_add;
1863 struct mem_array *mem_arr, *mem_arr_orig;
1864 unsigned int i, j, alloc_size, curr_alloc_size;
1865
1866 phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1867 if (!phba->phwi_ctrlr)
1868 return -ENOMEM;
1869
1870 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1871 GFP_KERNEL);
1872 if (!phba->init_mem) {
1873 kfree(phba->phwi_ctrlr);
1874 return -ENOMEM;
1875 }
1876
1877 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1878 GFP_KERNEL);
1879 if (!mem_arr_orig) {
1880 kfree(phba->init_mem);
1881 kfree(phba->phwi_ctrlr);
1882 return -ENOMEM;
1883 }
1884
1885 mem_descr = phba->init_mem;
1886 for (i = 0; i < SE_MEM_MAX; i++) {
1887 j = 0;
1888 mem_arr = mem_arr_orig;
1889 alloc_size = phba->mem_req[i];
1890 memset(mem_arr, 0, sizeof(struct mem_array) *
1891 BEISCSI_MAX_FRAGS_INIT);
1892 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1893 do {
1894 mem_arr->virtual_address = pci_alloc_consistent(
1895 phba->pcidev,
1896 curr_alloc_size,
1897 &bus_add);
1898 if (!mem_arr->virtual_address) {
1899 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1900 goto free_mem;
1901 if (curr_alloc_size -
1902 rounddown_pow_of_two(curr_alloc_size))
1903 curr_alloc_size = rounddown_pow_of_two
1904 (curr_alloc_size);
1905 else
1906 curr_alloc_size = curr_alloc_size / 2;
1907 } else {
1908 mem_arr->bus_address.u.
1909 a64.address = (__u64) bus_add;
1910 mem_arr->size = curr_alloc_size;
1911 alloc_size -= curr_alloc_size;
1912 curr_alloc_size = min(be_max_phys_size *
1913 1024, alloc_size);
1914 j++;
1915 mem_arr++;
1916 }
1917 } while (alloc_size);
1918 mem_descr->num_elements = j;
1919 mem_descr->size_in_bytes = phba->mem_req[i];
1920 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1921 GFP_KERNEL);
1922 if (!mem_descr->mem_array)
1923 goto free_mem;
1924
1925 memcpy(mem_descr->mem_array, mem_arr_orig,
1926 sizeof(struct mem_array) * j);
1927 mem_descr++;
1928 }
1929 kfree(mem_arr_orig);
1930 return 0;
1931 free_mem:
1932 mem_descr->num_elements = j;
1933 while ((i) || (j)) {
1934 for (j = mem_descr->num_elements; j > 0; j--) {
1935 pci_free_consistent(phba->pcidev,
1936 mem_descr->mem_array[j - 1].size,
1937 mem_descr->mem_array[j - 1].
1938 virtual_address,
1939 mem_descr->mem_array[j - 1].
1940 bus_address.u.a64.address);
1941 }
1942 if (i) {
1943 i--;
1944 kfree(mem_descr->mem_array);
1945 mem_descr--;
1946 }
1947 }
1948 kfree(mem_arr_orig);
1949 kfree(phba->init_mem);
1950 kfree(phba->phwi_ctrlr);
1951 return -ENOMEM;
1952 }
1953
1954 static int beiscsi_get_memory(struct beiscsi_hba *phba)
1955 {
1956 beiscsi_find_mem_req(phba);
1957 return beiscsi_alloc_mem(phba);
1958 }
1959
1960 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1961 {
1962 struct pdu_data_out *pdata_out;
1963 struct pdu_nop_out *pnop_out;
1964 struct be_mem_descriptor *mem_descr;
1965
1966 mem_descr = phba->init_mem;
1967 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1968 pdata_out =
1969 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1970 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1971
1972 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1973 IIOC_SCSI_DATA);
1974
1975 pnop_out =
1976 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1977 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1978
1979 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1980 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1981 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1982 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1983 }
1984
1985 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1986 {
1987 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1988 struct wrb_handle *pwrb_handle;
1989 struct hwi_controller *phwi_ctrlr;
1990 struct hwi_wrb_context *pwrb_context;
1991 struct iscsi_wrb *pwrb;
1992 unsigned int num_cxn_wrbh;
1993 unsigned int num_cxn_wrb, j, idx, index;
1994
1995 mem_descr_wrbh = phba->init_mem;
1996 mem_descr_wrbh += HWI_MEM_WRBH;
1997
1998 mem_descr_wrb = phba->init_mem;
1999 mem_descr_wrb += HWI_MEM_WRB;
2000
2001 idx = 0;
2002 pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2003 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2004 ((sizeof(struct wrb_handle)) *
2005 phba->params.wrbs_per_cxn));
2006 phwi_ctrlr = phba->phwi_ctrlr;
2007
2008 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2009 pwrb_context = &phwi_ctrlr->wrb_context[index];
2010 pwrb_context->pwrb_handle_base =
2011 kzalloc(sizeof(struct wrb_handle *) *
2012 phba->params.wrbs_per_cxn, GFP_KERNEL);
2013 pwrb_context->pwrb_handle_basestd =
2014 kzalloc(sizeof(struct wrb_handle *) *
2015 phba->params.wrbs_per_cxn, GFP_KERNEL);
2016 if (num_cxn_wrbh) {
2017 pwrb_context->alloc_index = 0;
2018 pwrb_context->wrb_handles_available = 0;
2019 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2020 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2021 pwrb_context->pwrb_handle_basestd[j] =
2022 pwrb_handle;
2023 pwrb_context->wrb_handles_available++;
2024 pwrb_handle->wrb_index = j;
2025 pwrb_handle++;
2026 }
2027 pwrb_context->free_index = 0;
2028 num_cxn_wrbh--;
2029 } else {
2030 idx++;
2031 pwrb_handle =
2032 mem_descr_wrbh->mem_array[idx].virtual_address;
2033 num_cxn_wrbh =
2034 ((mem_descr_wrbh->mem_array[idx].size) /
2035 ((sizeof(struct wrb_handle)) *
2036 phba->params.wrbs_per_cxn));
2037 pwrb_context->alloc_index = 0;
2038 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2039 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2040 pwrb_context->pwrb_handle_basestd[j] =
2041 pwrb_handle;
2042 pwrb_context->wrb_handles_available++;
2043 pwrb_handle->wrb_index = j;
2044 pwrb_handle++;
2045 }
2046 pwrb_context->free_index = 0;
2047 num_cxn_wrbh--;
2048 }
2049 }
2050 idx = 0;
2051 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2052 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2053 ((sizeof(struct iscsi_wrb) *
2054 phba->params.wrbs_per_cxn));
2055 for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
2056 pwrb_context = &phwi_ctrlr->wrb_context[index];
2057 if (num_cxn_wrb) {
2058 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2059 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2060 pwrb_handle->pwrb = pwrb;
2061 pwrb++;
2062 }
2063 num_cxn_wrb--;
2064 } else {
2065 idx++;
2066 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2067 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2068 ((sizeof(struct iscsi_wrb) *
2069 phba->params.wrbs_per_cxn));
2070 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2071 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2072 pwrb_handle->pwrb = pwrb;
2073 pwrb++;
2074 }
2075 num_cxn_wrb--;
2076 }
2077 }
2078 }
2079
2080 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2081 {
2082 struct hwi_controller *phwi_ctrlr;
2083 struct hba_parameters *p = &phba->params;
2084 struct hwi_async_pdu_context *pasync_ctx;
2085 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2086 unsigned int index;
2087 struct be_mem_descriptor *mem_descr;
2088
2089 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2090 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2091
2092 phwi_ctrlr = phba->phwi_ctrlr;
2093 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2094 mem_descr->mem_array[0].virtual_address;
2095 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2096 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2097
2098 pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2099 pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2100 pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2101 pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2102
2103 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2104 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2105 if (mem_descr->mem_array[0].virtual_address) {
2106 SE_DEBUG(DBG_LVL_8,
2107 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2108 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2109 } else
2110 shost_printk(KERN_WARNING, phba->shost,
2111 "No Virtual address \n");
2112
2113 pasync_ctx->async_header.va_base =
2114 mem_descr->mem_array[0].virtual_address;
2115
2116 pasync_ctx->async_header.pa_base.u.a64.address =
2117 mem_descr->mem_array[0].bus_address.u.a64.address;
2118
2119 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2120 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2121 if (mem_descr->mem_array[0].virtual_address) {
2122 SE_DEBUG(DBG_LVL_8,
2123 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2124 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2125 } else
2126 shost_printk(KERN_WARNING, phba->shost,
2127 "No Virtual address \n");
2128 pasync_ctx->async_header.ring_base =
2129 mem_descr->mem_array[0].virtual_address;
2130
2131 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2132 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2133 if (mem_descr->mem_array[0].virtual_address) {
2134 SE_DEBUG(DBG_LVL_8,
2135 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2136 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2137 } else
2138 shost_printk(KERN_WARNING, phba->shost,
2139 "No Virtual address \n");
2140
2141 pasync_ctx->async_header.handle_base =
2142 mem_descr->mem_array[0].virtual_address;
2143 pasync_ctx->async_header.writables = 0;
2144 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2145
2146 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2147 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2148 if (mem_descr->mem_array[0].virtual_address) {
2149 SE_DEBUG(DBG_LVL_8,
2150 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2151 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2152 } else
2153 shost_printk(KERN_WARNING, phba->shost,
2154 "No Virtual address \n");
2155 pasync_ctx->async_data.va_base =
2156 mem_descr->mem_array[0].virtual_address;
2157 pasync_ctx->async_data.pa_base.u.a64.address =
2158 mem_descr->mem_array[0].bus_address.u.a64.address;
2159
2160 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2161 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2162 if (mem_descr->mem_array[0].virtual_address) {
2163 SE_DEBUG(DBG_LVL_8,
2164 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2165 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2166 } else
2167 shost_printk(KERN_WARNING, phba->shost,
2168 "No Virtual address \n");
2169
2170 pasync_ctx->async_data.ring_base =
2171 mem_descr->mem_array[0].virtual_address;
2172
2173 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2174 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2175 if (!mem_descr->mem_array[0].virtual_address)
2176 shost_printk(KERN_WARNING, phba->shost,
2177 "No Virtual address \n");
2178
2179 pasync_ctx->async_data.handle_base =
2180 mem_descr->mem_array[0].virtual_address;
2181 pasync_ctx->async_data.writables = 0;
2182 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2183
2184 pasync_header_h =
2185 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2186 pasync_data_h =
2187 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2188
2189 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2190 pasync_header_h->cri = -1;
2191 pasync_header_h->index = (char)index;
2192 INIT_LIST_HEAD(&pasync_header_h->link);
2193 pasync_header_h->pbuffer =
2194 (void *)((unsigned long)
2195 (pasync_ctx->async_header.va_base) +
2196 (p->defpdu_hdr_sz * index));
2197
2198 pasync_header_h->pa.u.a64.address =
2199 pasync_ctx->async_header.pa_base.u.a64.address +
2200 (p->defpdu_hdr_sz * index);
2201
2202 list_add_tail(&pasync_header_h->link,
2203 &pasync_ctx->async_header.free_list);
2204 pasync_header_h++;
2205 pasync_ctx->async_header.free_entries++;
2206 pasync_ctx->async_header.writables++;
2207
2208 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2209 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2210 header_busy_list);
2211 pasync_data_h->cri = -1;
2212 pasync_data_h->index = (char)index;
2213 INIT_LIST_HEAD(&pasync_data_h->link);
2214 pasync_data_h->pbuffer =
2215 (void *)((unsigned long)
2216 (pasync_ctx->async_data.va_base) +
2217 (p->defpdu_data_sz * index));
2218
2219 pasync_data_h->pa.u.a64.address =
2220 pasync_ctx->async_data.pa_base.u.a64.address +
2221 (p->defpdu_data_sz * index);
2222
2223 list_add_tail(&pasync_data_h->link,
2224 &pasync_ctx->async_data.free_list);
2225 pasync_data_h++;
2226 pasync_ctx->async_data.free_entries++;
2227 pasync_ctx->async_data.writables++;
2228
2229 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2230 }
2231
2232 pasync_ctx->async_header.host_write_ptr = 0;
2233 pasync_ctx->async_header.ep_read_ptr = -1;
2234 pasync_ctx->async_data.host_write_ptr = 0;
2235 pasync_ctx->async_data.ep_read_ptr = -1;
2236 }
2237
2238 static int
2239 be_sgl_create_contiguous(void *virtual_address,
2240 u64 physical_address, u32 length,
2241 struct be_dma_mem *sgl)
2242 {
2243 WARN_ON(!virtual_address);
2244 WARN_ON(!physical_address);
2245 WARN_ON(!length > 0);
2246 WARN_ON(!sgl);
2247
2248 sgl->va = virtual_address;
2249 sgl->dma = physical_address;
2250 sgl->size = length;
2251
2252 return 0;
2253 }
2254
2255 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2256 {
2257 memset(sgl, 0, sizeof(*sgl));
2258 }
2259
2260 static void
2261 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2262 struct mem_array *pmem, struct be_dma_mem *sgl)
2263 {
2264 if (sgl->va)
2265 be_sgl_destroy_contiguous(sgl);
2266
2267 be_sgl_create_contiguous(pmem->virtual_address,
2268 pmem->bus_address.u.a64.address,
2269 pmem->size, sgl);
2270 }
2271
2272 static void
2273 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2274 struct mem_array *pmem, struct be_dma_mem *sgl)
2275 {
2276 if (sgl->va)
2277 be_sgl_destroy_contiguous(sgl);
2278
2279 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2280 pmem->bus_address.u.a64.address,
2281 pmem->size, sgl);
2282 }
2283
2284 static int be_fill_queue(struct be_queue_info *q,
2285 u16 len, u16 entry_size, void *vaddress)
2286 {
2287 struct be_dma_mem *mem = &q->dma_mem;
2288
2289 memset(q, 0, sizeof(*q));
2290 q->len = len;
2291 q->entry_size = entry_size;
2292 mem->size = len * entry_size;
2293 mem->va = vaddress;
2294 if (!mem->va)
2295 return -ENOMEM;
2296 memset(mem->va, 0, mem->size);
2297 return 0;
2298 }
2299
2300 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2301 struct hwi_context_memory *phwi_context)
2302 {
2303 unsigned int i, num_eq_pages;
2304 int ret, eq_for_mcc;
2305 struct be_queue_info *eq;
2306 struct be_dma_mem *mem;
2307 void *eq_vaddress;
2308 dma_addr_t paddr;
2309
2310 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2311 sizeof(struct be_eq_entry));
2312
2313 if (phba->msix_enabled)
2314 eq_for_mcc = 1;
2315 else
2316 eq_for_mcc = 0;
2317 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2318 eq = &phwi_context->be_eq[i].q;
2319 mem = &eq->dma_mem;
2320 phwi_context->be_eq[i].phba = phba;
2321 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2322 num_eq_pages * PAGE_SIZE,
2323 &paddr);
2324 if (!eq_vaddress)
2325 goto create_eq_error;
2326
2327 mem->va = eq_vaddress;
2328 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2329 sizeof(struct be_eq_entry), eq_vaddress);
2330 if (ret) {
2331 shost_printk(KERN_ERR, phba->shost,
2332 "be_fill_queue Failed for EQ \n");
2333 goto create_eq_error;
2334 }
2335
2336 mem->dma = paddr;
2337 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2338 phwi_context->cur_eqd);
2339 if (ret) {
2340 shost_printk(KERN_ERR, phba->shost,
2341 "beiscsi_cmd_eq_create"
2342 "Failedfor EQ \n");
2343 goto create_eq_error;
2344 }
2345 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2346 }
2347 return 0;
2348 create_eq_error:
2349 for (i = 0; i < (phba->num_cpus + 1); i++) {
2350 eq = &phwi_context->be_eq[i].q;
2351 mem = &eq->dma_mem;
2352 if (mem->va)
2353 pci_free_consistent(phba->pcidev, num_eq_pages
2354 * PAGE_SIZE,
2355 mem->va, mem->dma);
2356 }
2357 return ret;
2358 }
2359
2360 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2361 struct hwi_context_memory *phwi_context)
2362 {
2363 unsigned int i, num_cq_pages;
2364 int ret;
2365 struct be_queue_info *cq, *eq;
2366 struct be_dma_mem *mem;
2367 struct be_eq_obj *pbe_eq;
2368 void *cq_vaddress;
2369 dma_addr_t paddr;
2370
2371 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2372 sizeof(struct sol_cqe));
2373
2374 for (i = 0; i < phba->num_cpus; i++) {
2375 cq = &phwi_context->be_cq[i];
2376 eq = &phwi_context->be_eq[i].q;
2377 pbe_eq = &phwi_context->be_eq[i];
2378 pbe_eq->cq = cq;
2379 pbe_eq->phba = phba;
2380 mem = &cq->dma_mem;
2381 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2382 num_cq_pages * PAGE_SIZE,
2383 &paddr);
2384 if (!cq_vaddress)
2385 goto create_cq_error;
2386 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2387 sizeof(struct sol_cqe), cq_vaddress);
2388 if (ret) {
2389 shost_printk(KERN_ERR, phba->shost,
2390 "be_fill_queue Failed for ISCSI CQ \n");
2391 goto create_cq_error;
2392 }
2393
2394 mem->dma = paddr;
2395 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2396 false, 0);
2397 if (ret) {
2398 shost_printk(KERN_ERR, phba->shost,
2399 "beiscsi_cmd_eq_create"
2400 "Failed for ISCSI CQ \n");
2401 goto create_cq_error;
2402 }
2403 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2404 cq->id, eq->id);
2405 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2406 }
2407 return 0;
2408
2409 create_cq_error:
2410 for (i = 0; i < phba->num_cpus; i++) {
2411 cq = &phwi_context->be_cq[i];
2412 mem = &cq->dma_mem;
2413 if (mem->va)
2414 pci_free_consistent(phba->pcidev, num_cq_pages
2415 * PAGE_SIZE,
2416 mem->va, mem->dma);
2417 }
2418 return ret;
2419
2420 }
2421
2422 static int
2423 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2424 struct hwi_context_memory *phwi_context,
2425 struct hwi_controller *phwi_ctrlr,
2426 unsigned int def_pdu_ring_sz)
2427 {
2428 unsigned int idx;
2429 int ret;
2430 struct be_queue_info *dq, *cq;
2431 struct be_dma_mem *mem;
2432 struct be_mem_descriptor *mem_descr;
2433 void *dq_vaddress;
2434
2435 idx = 0;
2436 dq = &phwi_context->be_def_hdrq;
2437 cq = &phwi_context->be_cq[0];
2438 mem = &dq->dma_mem;
2439 mem_descr = phba->init_mem;
2440 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2441 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2442 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2443 sizeof(struct phys_addr),
2444 sizeof(struct phys_addr), dq_vaddress);
2445 if (ret) {
2446 shost_printk(KERN_ERR, phba->shost,
2447 "be_fill_queue Failed for DEF PDU HDR\n");
2448 return ret;
2449 }
2450 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2451 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2452 def_pdu_ring_sz,
2453 phba->params.defpdu_hdr_sz);
2454 if (ret) {
2455 shost_printk(KERN_ERR, phba->shost,
2456 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2457 return ret;
2458 }
2459 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2460 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2461 phwi_context->be_def_hdrq.id);
2462 hwi_post_async_buffers(phba, 1);
2463 return 0;
2464 }
2465
2466 static int
2467 beiscsi_create_def_data(struct beiscsi_hba *phba,
2468 struct hwi_context_memory *phwi_context,
2469 struct hwi_controller *phwi_ctrlr,
2470 unsigned int def_pdu_ring_sz)
2471 {
2472 unsigned int idx;
2473 int ret;
2474 struct be_queue_info *dataq, *cq;
2475 struct be_dma_mem *mem;
2476 struct be_mem_descriptor *mem_descr;
2477 void *dq_vaddress;
2478
2479 idx = 0;
2480 dataq = &phwi_context->be_def_dataq;
2481 cq = &phwi_context->be_cq[0];
2482 mem = &dataq->dma_mem;
2483 mem_descr = phba->init_mem;
2484 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2485 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2486 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2487 sizeof(struct phys_addr),
2488 sizeof(struct phys_addr), dq_vaddress);
2489 if (ret) {
2490 shost_printk(KERN_ERR, phba->shost,
2491 "be_fill_queue Failed for DEF PDU DATA\n");
2492 return ret;
2493 }
2494 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2495 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2496 def_pdu_ring_sz,
2497 phba->params.defpdu_data_sz);
2498 if (ret) {
2499 shost_printk(KERN_ERR, phba->shost,
2500 "be_cmd_create_default_pdu_queue Failed"
2501 " for DEF PDU DATA\n");
2502 return ret;
2503 }
2504 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2505 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2506 phwi_context->be_def_dataq.id);
2507 hwi_post_async_buffers(phba, 0);
2508 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2509 return 0;
2510 }
2511
2512 static int
2513 beiscsi_post_pages(struct beiscsi_hba *phba)
2514 {
2515 struct be_mem_descriptor *mem_descr;
2516 struct mem_array *pm_arr;
2517 unsigned int page_offset, i;
2518 struct be_dma_mem sgl;
2519 int status;
2520
2521 mem_descr = phba->init_mem;
2522 mem_descr += HWI_MEM_SGE;
2523 pm_arr = mem_descr->mem_array;
2524
2525 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2526 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2527 for (i = 0; i < mem_descr->num_elements; i++) {
2528 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2529 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2530 page_offset,
2531 (pm_arr->size / PAGE_SIZE));
2532 page_offset += pm_arr->size / PAGE_SIZE;
2533 if (status != 0) {
2534 shost_printk(KERN_ERR, phba->shost,
2535 "post sgl failed.\n");
2536 return status;
2537 }
2538 pm_arr++;
2539 }
2540 SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2541 return 0;
2542 }
2543
2544 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2545 {
2546 struct be_dma_mem *mem = &q->dma_mem;
2547 if (mem->va)
2548 pci_free_consistent(phba->pcidev, mem->size,
2549 mem->va, mem->dma);
2550 }
2551
2552 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2553 u16 len, u16 entry_size)
2554 {
2555 struct be_dma_mem *mem = &q->dma_mem;
2556
2557 memset(q, 0, sizeof(*q));
2558 q->len = len;
2559 q->entry_size = entry_size;
2560 mem->size = len * entry_size;
2561 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2562 if (!mem->va)
2563 return -1;
2564 memset(mem->va, 0, mem->size);
2565 return 0;
2566 }
2567
2568 static int
2569 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2570 struct hwi_context_memory *phwi_context,
2571 struct hwi_controller *phwi_ctrlr)
2572 {
2573 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2574 u64 pa_addr_lo;
2575 unsigned int idx, num, i;
2576 struct mem_array *pwrb_arr;
2577 void *wrb_vaddr;
2578 struct be_dma_mem sgl;
2579 struct be_mem_descriptor *mem_descr;
2580 int status;
2581
2582 idx = 0;
2583 mem_descr = phba->init_mem;
2584 mem_descr += HWI_MEM_WRB;
2585 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2586 GFP_KERNEL);
2587 if (!pwrb_arr) {
2588 shost_printk(KERN_ERR, phba->shost,
2589 "Memory alloc failed in create wrb ring.\n");
2590 return -ENOMEM;
2591 }
2592 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2593 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2594 num_wrb_rings = mem_descr->mem_array[idx].size /
2595 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2596
2597 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2598 if (num_wrb_rings) {
2599 pwrb_arr[num].virtual_address = wrb_vaddr;
2600 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2601 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2602 sizeof(struct iscsi_wrb);
2603 wrb_vaddr += pwrb_arr[num].size;
2604 pa_addr_lo += pwrb_arr[num].size;
2605 num_wrb_rings--;
2606 } else {
2607 idx++;
2608 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2609 pa_addr_lo = mem_descr->mem_array[idx].\
2610 bus_address.u.a64.address;
2611 num_wrb_rings = mem_descr->mem_array[idx].size /
2612 (phba->params.wrbs_per_cxn *
2613 sizeof(struct iscsi_wrb));
2614 pwrb_arr[num].virtual_address = wrb_vaddr;
2615 pwrb_arr[num].bus_address.u.a64.address\
2616 = pa_addr_lo;
2617 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2618 sizeof(struct iscsi_wrb);
2619 wrb_vaddr += pwrb_arr[num].size;
2620 pa_addr_lo += pwrb_arr[num].size;
2621 num_wrb_rings--;
2622 }
2623 }
2624 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2625 wrb_mem_index = 0;
2626 offset = 0;
2627 size = 0;
2628
2629 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2630 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2631 &phwi_context->be_wrbq[i]);
2632 if (status != 0) {
2633 shost_printk(KERN_ERR, phba->shost,
2634 "wrbq create failed.");
2635 return status;
2636 }
2637 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2638 id;
2639 }
2640 kfree(pwrb_arr);
2641 return 0;
2642 }
2643
2644 static void free_wrb_handles(struct beiscsi_hba *phba)
2645 {
2646 unsigned int index;
2647 struct hwi_controller *phwi_ctrlr;
2648 struct hwi_wrb_context *pwrb_context;
2649
2650 phwi_ctrlr = phba->phwi_ctrlr;
2651 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2652 pwrb_context = &phwi_ctrlr->wrb_context[index];
2653 kfree(pwrb_context->pwrb_handle_base);
2654 kfree(pwrb_context->pwrb_handle_basestd);
2655 }
2656 }
2657
2658 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2659 {
2660 struct be_queue_info *q;
2661 struct be_ctrl_info *ctrl = &phba->ctrl;
2662
2663 q = &phba->ctrl.mcc_obj.q;
2664 if (q->created)
2665 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2666 be_queue_free(phba, q);
2667
2668 q = &phba->ctrl.mcc_obj.cq;
2669 if (q->created)
2670 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2671 be_queue_free(phba, q);
2672 }
2673
2674 static void hwi_cleanup(struct beiscsi_hba *phba)
2675 {
2676 struct be_queue_info *q;
2677 struct be_ctrl_info *ctrl = &phba->ctrl;
2678 struct hwi_controller *phwi_ctrlr;
2679 struct hwi_context_memory *phwi_context;
2680 int i, eq_num;
2681
2682 phwi_ctrlr = phba->phwi_ctrlr;
2683 phwi_context = phwi_ctrlr->phwi_ctxt;
2684 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2685 q = &phwi_context->be_wrbq[i];
2686 if (q->created)
2687 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2688 }
2689 free_wrb_handles(phba);
2690
2691 q = &phwi_context->be_def_hdrq;
2692 if (q->created)
2693 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2694
2695 q = &phwi_context->be_def_dataq;
2696 if (q->created)
2697 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2698
2699 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2700
2701 for (i = 0; i < (phba->num_cpus); i++) {
2702 q = &phwi_context->be_cq[i];
2703 if (q->created)
2704 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2705 }
2706 if (phba->msix_enabled)
2707 eq_num = 1;
2708 else
2709 eq_num = 0;
2710 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2711 q = &phwi_context->be_eq[i].q;
2712 if (q->created)
2713 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2714 }
2715 be_mcc_queues_destroy(phba);
2716 }
2717
2718 static int be_mcc_queues_create(struct beiscsi_hba *phba,
2719 struct hwi_context_memory *phwi_context)
2720 {
2721 struct be_queue_info *q, *cq;
2722 struct be_ctrl_info *ctrl = &phba->ctrl;
2723
2724 /* Alloc MCC compl queue */
2725 cq = &phba->ctrl.mcc_obj.cq;
2726 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2727 sizeof(struct be_mcc_compl)))
2728 goto err;
2729 /* Ask BE to create MCC compl queue; */
2730 if (phba->msix_enabled) {
2731 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2732 [phba->num_cpus].q, false, true, 0))
2733 goto mcc_cq_free;
2734 } else {
2735 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2736 false, true, 0))
2737 goto mcc_cq_free;
2738 }
2739
2740 /* Alloc MCC queue */
2741 q = &phba->ctrl.mcc_obj.q;
2742 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2743 goto mcc_cq_destroy;
2744
2745 /* Ask BE to create MCC queue */
2746 if (beiscsi_cmd_mccq_create(phba, q, cq))
2747 goto mcc_q_free;
2748
2749 return 0;
2750
2751 mcc_q_free:
2752 be_queue_free(phba, q);
2753 mcc_cq_destroy:
2754 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2755 mcc_cq_free:
2756 be_queue_free(phba, cq);
2757 err:
2758 return -1;
2759 }
2760
2761 static int find_num_cpus(void)
2762 {
2763 int num_cpus = 0;
2764
2765 num_cpus = num_online_cpus();
2766 if (num_cpus >= MAX_CPUS)
2767 num_cpus = MAX_CPUS - 1;
2768
2769 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2770 return num_cpus;
2771 }
2772
2773 static int hwi_init_port(struct beiscsi_hba *phba)
2774 {
2775 struct hwi_controller *phwi_ctrlr;
2776 struct hwi_context_memory *phwi_context;
2777 unsigned int def_pdu_ring_sz;
2778 struct be_ctrl_info *ctrl = &phba->ctrl;
2779 int status;
2780
2781 def_pdu_ring_sz =
2782 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2783 phwi_ctrlr = phba->phwi_ctrlr;
2784 phwi_context = phwi_ctrlr->phwi_ctxt;
2785 phwi_context->max_eqd = 0;
2786 phwi_context->min_eqd = 0;
2787 phwi_context->cur_eqd = 64;
2788 be_cmd_fw_initialize(&phba->ctrl);
2789
2790 status = beiscsi_create_eqs(phba, phwi_context);
2791 if (status != 0) {
2792 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2793 goto error;
2794 }
2795
2796 status = be_mcc_queues_create(phba, phwi_context);
2797 if (status != 0)
2798 goto error;
2799
2800 status = mgmt_check_supported_fw(ctrl, phba);
2801 if (status != 0) {
2802 shost_printk(KERN_ERR, phba->shost,
2803 "Unsupported fw version \n");
2804 goto error;
2805 }
2806
2807 status = beiscsi_create_cqs(phba, phwi_context);
2808 if (status != 0) {
2809 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2810 goto error;
2811 }
2812
2813 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2814 def_pdu_ring_sz);
2815 if (status != 0) {
2816 shost_printk(KERN_ERR, phba->shost,
2817 "Default Header not created\n");
2818 goto error;
2819 }
2820
2821 status = beiscsi_create_def_data(phba, phwi_context,
2822 phwi_ctrlr, def_pdu_ring_sz);
2823 if (status != 0) {
2824 shost_printk(KERN_ERR, phba->shost,
2825 "Default Data not created\n");
2826 goto error;
2827 }
2828
2829 status = beiscsi_post_pages(phba);
2830 if (status != 0) {
2831 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2832 goto error;
2833 }
2834
2835 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2836 if (status != 0) {
2837 shost_printk(KERN_ERR, phba->shost,
2838 "WRB Rings not created\n");
2839 goto error;
2840 }
2841
2842 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2843 return 0;
2844
2845 error:
2846 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2847 hwi_cleanup(phba);
2848 return -ENOMEM;
2849 }
2850
2851 static int hwi_init_controller(struct beiscsi_hba *phba)
2852 {
2853 struct hwi_controller *phwi_ctrlr;
2854
2855 phwi_ctrlr = phba->phwi_ctrlr;
2856 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2857 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2858 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2859 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2860 phwi_ctrlr->phwi_ctxt);
2861 } else {
2862 shost_printk(KERN_ERR, phba->shost,
2863 "HWI_MEM_ADDN_CONTEXT is more than one element."
2864 "Failing to load\n");
2865 return -ENOMEM;
2866 }
2867
2868 iscsi_init_global_templates(phba);
2869 beiscsi_init_wrb_handle(phba);
2870 hwi_init_async_pdu_ctx(phba);
2871 if (hwi_init_port(phba) != 0) {
2872 shost_printk(KERN_ERR, phba->shost,
2873 "hwi_init_controller failed\n");
2874 return -ENOMEM;
2875 }
2876 return 0;
2877 }
2878
2879 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2880 {
2881 struct be_mem_descriptor *mem_descr;
2882 int i, j;
2883
2884 mem_descr = phba->init_mem;
2885 i = 0;
2886 j = 0;
2887 for (i = 0; i < SE_MEM_MAX; i++) {
2888 for (j = mem_descr->num_elements; j > 0; j--) {
2889 pci_free_consistent(phba->pcidev,
2890 mem_descr->mem_array[j - 1].size,
2891 mem_descr->mem_array[j - 1].virtual_address,
2892 mem_descr->mem_array[j - 1].bus_address.
2893 u.a64.address);
2894 }
2895 kfree(mem_descr->mem_array);
2896 mem_descr++;
2897 }
2898 kfree(phba->init_mem);
2899 kfree(phba->phwi_ctrlr);
2900 }
2901
2902 static int beiscsi_init_controller(struct beiscsi_hba *phba)
2903 {
2904 int ret = -ENOMEM;
2905
2906 ret = beiscsi_get_memory(phba);
2907 if (ret < 0) {
2908 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2909 "Failed in beiscsi_alloc_memory \n");
2910 return ret;
2911 }
2912
2913 ret = hwi_init_controller(phba);
2914 if (ret)
2915 goto free_init;
2916 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2917 return 0;
2918
2919 free_init:
2920 beiscsi_free_mem(phba);
2921 return -ENOMEM;
2922 }
2923
2924 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2925 {
2926 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2927 struct sgl_handle *psgl_handle;
2928 struct iscsi_sge *pfrag;
2929 unsigned int arr_index, i, idx;
2930
2931 phba->io_sgl_hndl_avbl = 0;
2932 phba->eh_sgl_hndl_avbl = 0;
2933
2934 mem_descr_sglh = phba->init_mem;
2935 mem_descr_sglh += HWI_MEM_SGLH;
2936 if (1 == mem_descr_sglh->num_elements) {
2937 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2938 phba->params.ios_per_ctrl,
2939 GFP_KERNEL);
2940 if (!phba->io_sgl_hndl_base) {
2941 shost_printk(KERN_ERR, phba->shost,
2942 "Mem Alloc Failed. Failing to load\n");
2943 return -ENOMEM;
2944 }
2945 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2946 (phba->params.icds_per_ctrl -
2947 phba->params.ios_per_ctrl),
2948 GFP_KERNEL);
2949 if (!phba->eh_sgl_hndl_base) {
2950 kfree(phba->io_sgl_hndl_base);
2951 shost_printk(KERN_ERR, phba->shost,
2952 "Mem Alloc Failed. Failing to load\n");
2953 return -ENOMEM;
2954 }
2955 } else {
2956 shost_printk(KERN_ERR, phba->shost,
2957 "HWI_MEM_SGLH is more than one element."
2958 "Failing to load\n");
2959 return -ENOMEM;
2960 }
2961
2962 arr_index = 0;
2963 idx = 0;
2964 while (idx < mem_descr_sglh->num_elements) {
2965 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2966
2967 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2968 sizeof(struct sgl_handle)); i++) {
2969 if (arr_index < phba->params.ios_per_ctrl) {
2970 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2971 phba->io_sgl_hndl_avbl++;
2972 arr_index++;
2973 } else {
2974 phba->eh_sgl_hndl_base[arr_index -
2975 phba->params.ios_per_ctrl] =
2976 psgl_handle;
2977 arr_index++;
2978 phba->eh_sgl_hndl_avbl++;
2979 }
2980 psgl_handle++;
2981 }
2982 idx++;
2983 }
2984 SE_DEBUG(DBG_LVL_8,
2985 "phba->io_sgl_hndl_avbl=%d"
2986 "phba->eh_sgl_hndl_avbl=%d \n",
2987 phba->io_sgl_hndl_avbl,
2988 phba->eh_sgl_hndl_avbl);
2989 mem_descr_sg = phba->init_mem;
2990 mem_descr_sg += HWI_MEM_SGE;
2991 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
2992 mem_descr_sg->num_elements);
2993 arr_index = 0;
2994 idx = 0;
2995 while (idx < mem_descr_sg->num_elements) {
2996 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
2997
2998 for (i = 0;
2999 i < (mem_descr_sg->mem_array[idx].size) /
3000 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3001 i++) {
3002 if (arr_index < phba->params.ios_per_ctrl)
3003 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3004 else
3005 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3006 phba->params.ios_per_ctrl];
3007 psgl_handle->pfrag = pfrag;
3008 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3009 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3010 pfrag += phba->params.num_sge_per_io;
3011 psgl_handle->sgl_index =
3012 phba->fw_config.iscsi_icd_start + arr_index++;
3013 }
3014 idx++;
3015 }
3016 phba->io_sgl_free_index = 0;
3017 phba->io_sgl_alloc_index = 0;
3018 phba->eh_sgl_free_index = 0;
3019 phba->eh_sgl_alloc_index = 0;
3020 return 0;
3021 }
3022
3023 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3024 {
3025 int i, new_cid;
3026
3027 phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3028 GFP_KERNEL);
3029 if (!phba->cid_array) {
3030 shost_printk(KERN_ERR, phba->shost,
3031 "Failed to allocate memory in "
3032 "hba_setup_cid_tbls\n");
3033 return -ENOMEM;
3034 }
3035 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3036 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3037 if (!phba->ep_array) {
3038 shost_printk(KERN_ERR, phba->shost,
3039 "Failed to allocate memory in "
3040 "hba_setup_cid_tbls \n");
3041 kfree(phba->cid_array);
3042 return -ENOMEM;
3043 }
3044 new_cid = phba->fw_config.iscsi_cid_start;
3045 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3046 phba->cid_array[i] = new_cid;
3047 new_cid += 2;
3048 }
3049 phba->avlbl_cids = phba->params.cxns_per_ctrl;
3050 return 0;
3051 }
3052
3053 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3054 {
3055 struct be_ctrl_info *ctrl = &phba->ctrl;
3056 struct hwi_controller *phwi_ctrlr;
3057 struct hwi_context_memory *phwi_context;
3058 struct be_queue_info *eq;
3059 u8 __iomem *addr;
3060 u32 reg, i;
3061 u32 enabled;
3062
3063 phwi_ctrlr = phba->phwi_ctrlr;
3064 phwi_context = phwi_ctrlr->phwi_ctxt;
3065
3066 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3067 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3068 reg = ioread32(addr);
3069 SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3070
3071 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3072 if (!enabled) {
3073 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3074 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3075 iowrite32(reg, addr);
3076 for (i = 0; i <= phba->num_cpus; i++) {
3077 eq = &phwi_context->be_eq[i].q;
3078 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3079 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3080 }
3081 } else
3082 shost_printk(KERN_WARNING, phba->shost,
3083 "In hwi_enable_intr, Not Enabled \n");
3084 return true;
3085 }
3086
3087 static void hwi_disable_intr(struct beiscsi_hba *phba)
3088 {
3089 struct be_ctrl_info *ctrl = &phba->ctrl;
3090
3091 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3092 u32 reg = ioread32(addr);
3093
3094 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3095 if (enabled) {
3096 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3097 iowrite32(reg, addr);
3098 } else
3099 shost_printk(KERN_WARNING, phba->shost,
3100 "In hwi_disable_intr, Already Disabled \n");
3101 }
3102
3103 static int beiscsi_init_port(struct beiscsi_hba *phba)
3104 {
3105 int ret;
3106
3107 ret = beiscsi_init_controller(phba);
3108 if (ret < 0) {
3109 shost_printk(KERN_ERR, phba->shost,
3110 "beiscsi_dev_probe - Failed in"
3111 "beiscsi_init_controller \n");
3112 return ret;
3113 }
3114 ret = beiscsi_init_sgl_handle(phba);
3115 if (ret < 0) {
3116 shost_printk(KERN_ERR, phba->shost,
3117 "beiscsi_dev_probe - Failed in"
3118 "beiscsi_init_sgl_handle \n");
3119 goto do_cleanup_ctrlr;
3120 }
3121
3122 if (hba_setup_cid_tbls(phba)) {
3123 shost_printk(KERN_ERR, phba->shost,
3124 "Failed in hba_setup_cid_tbls\n");
3125 kfree(phba->io_sgl_hndl_base);
3126 kfree(phba->eh_sgl_hndl_base);
3127 goto do_cleanup_ctrlr;
3128 }
3129
3130 return ret;
3131
3132 do_cleanup_ctrlr:
3133 hwi_cleanup(phba);
3134 return ret;
3135 }
3136
3137 static void hwi_purge_eq(struct beiscsi_hba *phba)
3138 {
3139 struct hwi_controller *phwi_ctrlr;
3140 struct hwi_context_memory *phwi_context;
3141 struct be_queue_info *eq;
3142 struct be_eq_entry *eqe = NULL;
3143 int i, eq_msix;
3144 unsigned int num_processed;
3145
3146 phwi_ctrlr = phba->phwi_ctrlr;
3147 phwi_context = phwi_ctrlr->phwi_ctxt;
3148 if (phba->msix_enabled)
3149 eq_msix = 1;
3150 else
3151 eq_msix = 0;
3152
3153 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3154 eq = &phwi_context->be_eq[i].q;
3155 eqe = queue_tail_node(eq);
3156 num_processed = 0;
3157 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3158 & EQE_VALID_MASK) {
3159 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3160 queue_tail_inc(eq);
3161 eqe = queue_tail_node(eq);
3162 num_processed++;
3163 }
3164
3165 if (num_processed)
3166 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3167 }
3168 }
3169
3170 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3171 {
3172 unsigned char mgmt_status;
3173
3174 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3175 if (mgmt_status)
3176 shost_printk(KERN_WARNING, phba->shost,
3177 "mgmt_epfw_cleanup FAILED \n");
3178
3179 hwi_purge_eq(phba);
3180 hwi_cleanup(phba);
3181 kfree(phba->io_sgl_hndl_base);
3182 kfree(phba->eh_sgl_hndl_base);
3183 kfree(phba->cid_array);
3184 kfree(phba->ep_array);
3185 }
3186
3187 void
3188 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3189 struct beiscsi_offload_params *params)
3190 {
3191 struct wrb_handle *pwrb_handle;
3192 struct iscsi_target_context_update_wrb *pwrb = NULL;
3193 struct be_mem_descriptor *mem_descr;
3194 struct beiscsi_hba *phba = beiscsi_conn->phba;
3195 u32 doorbell = 0;
3196
3197 /*
3198 * We can always use 0 here because it is reserved by libiscsi for
3199 * login/startup related tasks.
3200 */
3201 pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3202 phba->fw_config.iscsi_cid_start));
3203 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3204 memset(pwrb, 0, sizeof(*pwrb));
3205 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3206 max_burst_length, pwrb, params->dw[offsetof
3207 (struct amap_beiscsi_offload_params,
3208 max_burst_length) / 32]);
3209 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3210 max_send_data_segment_length, pwrb,
3211 params->dw[offsetof(struct amap_beiscsi_offload_params,
3212 max_send_data_segment_length) / 32]);
3213 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3214 first_burst_length,
3215 pwrb,
3216 params->dw[offsetof(struct amap_beiscsi_offload_params,
3217 first_burst_length) / 32]);
3218
3219 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3220 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3221 erl) / 32] & OFFLD_PARAMS_ERL));
3222 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3223 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3224 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3225 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3226 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3227 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3228 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3229 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3230 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3231 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3232 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3233 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3234 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3235 pwrb,
3236 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3237 exp_statsn) / 32] + 1));
3238 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3239 0x7);
3240 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3241 pwrb, pwrb_handle->wrb_index);
3242 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3243 pwrb, pwrb_handle->nxt_wrb_index);
3244 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3245 session_state, pwrb, 0);
3246 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3247 pwrb, 1);
3248 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3249 pwrb, 0);
3250 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3251 0);
3252
3253 mem_descr = phba->init_mem;
3254 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3255
3256 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3257 pad_buffer_addr_hi, pwrb,
3258 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3259 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3260 pad_buffer_addr_lo, pwrb,
3261 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3262
3263 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3264
3265 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3266 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3267 << DB_DEF_PDU_WRB_INDEX_SHIFT;
3268 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3269
3270 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3271 }
3272
3273 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3274 int *index, int *age)
3275 {
3276 *index = (int)itt;
3277 if (age)
3278 *age = conn->session->age;
3279 }
3280
3281 /**
3282 * beiscsi_alloc_pdu - allocates pdu and related resources
3283 * @task: libiscsi task
3284 * @opcode: opcode of pdu for task
3285 *
3286 * This is called with the session lock held. It will allocate
3287 * the wrb and sgl if needed for the command. And it will prep
3288 * the pdu's itt. beiscsi_parse_pdu will later translate
3289 * the pdu itt to the libiscsi task itt.
3290 */
3291 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3292 {
3293 struct beiscsi_io_task *io_task = task->dd_data;
3294 struct iscsi_conn *conn = task->conn;
3295 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3296 struct beiscsi_hba *phba = beiscsi_conn->phba;
3297 struct hwi_wrb_context *pwrb_context;
3298 struct hwi_controller *phwi_ctrlr;
3299 itt_t itt;
3300 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3301 dma_addr_t paddr;
3302
3303 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3304 GFP_KERNEL, &paddr);
3305 if (!io_task->cmd_bhs)
3306 return -ENOMEM;
3307 io_task->bhs_pa.u.a64.address = paddr;
3308 io_task->libiscsi_itt = (itt_t)task->itt;
3309 io_task->pwrb_handle = alloc_wrb_handle(phba,
3310 beiscsi_conn->beiscsi_conn_cid -
3311 phba->fw_config.iscsi_cid_start
3312 );
3313 io_task->conn = beiscsi_conn;
3314
3315 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3316 task->hdr_max = sizeof(struct be_cmd_bhs);
3317
3318 if (task->sc) {
3319 spin_lock(&phba->io_sgl_lock);
3320 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3321 spin_unlock(&phba->io_sgl_lock);
3322 if (!io_task->psgl_handle)
3323 goto free_hndls;
3324 } else {
3325 io_task->scsi_cmnd = NULL;
3326 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3327 if (!beiscsi_conn->login_in_progress) {
3328 spin_lock(&phba->mgmt_sgl_lock);
3329 io_task->psgl_handle = (struct sgl_handle *)
3330 alloc_mgmt_sgl_handle(phba);
3331 spin_unlock(&phba->mgmt_sgl_lock);
3332 if (!io_task->psgl_handle)
3333 goto free_hndls;
3334
3335 beiscsi_conn->login_in_progress = 1;
3336 beiscsi_conn->plogin_sgl_handle =
3337 io_task->psgl_handle;
3338 } else {
3339 io_task->psgl_handle =
3340 beiscsi_conn->plogin_sgl_handle;
3341 }
3342 } else {
3343 spin_lock(&phba->mgmt_sgl_lock);
3344 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3345 spin_unlock(&phba->mgmt_sgl_lock);
3346 if (!io_task->psgl_handle)
3347 goto free_hndls;
3348 }
3349 }
3350 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3351 wrb_index << 16) | (unsigned int)
3352 (io_task->psgl_handle->sgl_index));
3353 io_task->pwrb_handle->pio_handle = task;
3354
3355 io_task->cmd_bhs->iscsi_hdr.itt = itt;
3356 return 0;
3357
3358 free_hndls:
3359 phwi_ctrlr = phba->phwi_ctrlr;
3360 pwrb_context = &phwi_ctrlr->wrb_context[
3361 beiscsi_conn->beiscsi_conn_cid -
3362 phba->fw_config.iscsi_cid_start];
3363 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3364 io_task->pwrb_handle = NULL;
3365 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3366 io_task->bhs_pa.u.a64.address);
3367 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3368 return -ENOMEM;
3369 }
3370
3371 static void beiscsi_cleanup_task(struct iscsi_task *task)
3372 {
3373 struct beiscsi_io_task *io_task = task->dd_data;
3374 struct iscsi_conn *conn = task->conn;
3375 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3376 struct beiscsi_hba *phba = beiscsi_conn->phba;
3377 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3378 struct hwi_wrb_context *pwrb_context;
3379 struct hwi_controller *phwi_ctrlr;
3380
3381 phwi_ctrlr = phba->phwi_ctrlr;
3382 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3383 - phba->fw_config.iscsi_cid_start];
3384 if (io_task->pwrb_handle) {
3385 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3386 io_task->pwrb_handle = NULL;
3387 }
3388
3389 if (io_task->cmd_bhs) {
3390 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3391 io_task->bhs_pa.u.a64.address);
3392 }
3393
3394 if (task->sc) {
3395 if (io_task->psgl_handle) {
3396 spin_lock(&phba->io_sgl_lock);
3397 free_io_sgl_handle(phba, io_task->psgl_handle);
3398 spin_unlock(&phba->io_sgl_lock);
3399 io_task->psgl_handle = NULL;
3400 }
3401 } else {
3402 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3403 return;
3404 if (io_task->psgl_handle) {
3405 spin_lock(&phba->mgmt_sgl_lock);
3406 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3407 spin_unlock(&phba->mgmt_sgl_lock);
3408 io_task->psgl_handle = NULL;
3409 }
3410 }
3411 }
3412
3413 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3414 unsigned int num_sg, unsigned int xferlen,
3415 unsigned int writedir)
3416 {
3417
3418 struct beiscsi_io_task *io_task = task->dd_data;
3419 struct iscsi_conn *conn = task->conn;
3420 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3421 struct beiscsi_hba *phba = beiscsi_conn->phba;
3422 struct iscsi_wrb *pwrb = NULL;
3423 unsigned int doorbell = 0;
3424
3425 pwrb = io_task->pwrb_handle->pwrb;
3426 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3427 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3428
3429 if (writedir) {
3430 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3431 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3432 &io_task->cmd_bhs->iscsi_data_pdu,
3433 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3434 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3435 &io_task->cmd_bhs->iscsi_data_pdu,
3436 ISCSI_OPCODE_SCSI_DATA_OUT);
3437 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3438 &io_task->cmd_bhs->iscsi_data_pdu, 1);
3439 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3440 INI_WR_CMD);
3441 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3442 } else {
3443 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3444 INI_RD_CMD);
3445 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3446 }
3447 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3448 dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3449 io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3450
3451 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3452 cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3453 lun[0]));
3454 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3455 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3456 io_task->pwrb_handle->wrb_index);
3457 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3458 be32_to_cpu(task->cmdsn));
3459 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3460 io_task->psgl_handle->sgl_index);
3461
3462 hwi_write_sgl(pwrb, sg, num_sg, io_task);
3463
3464 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3465 io_task->pwrb_handle->nxt_wrb_index);
3466 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3467
3468 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3469 doorbell |= (io_task->pwrb_handle->wrb_index &
3470 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3471 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3472
3473 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3474 return 0;
3475 }
3476
3477 static int beiscsi_mtask(struct iscsi_task *task)
3478 {
3479 struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3480 struct iscsi_conn *conn = task->conn;
3481 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3482 struct beiscsi_hba *phba = beiscsi_conn->phba;
3483 struct iscsi_session *session;
3484 struct iscsi_wrb *pwrb = NULL;
3485 struct hwi_controller *phwi_ctrlr;
3486 struct hwi_wrb_context *pwrb_context;
3487 struct wrb_handle *pwrb_handle;
3488 unsigned int doorbell = 0;
3489 unsigned int i, cid;
3490 struct iscsi_task *aborted_task;
3491 unsigned int tag;
3492
3493 cid = beiscsi_conn->beiscsi_conn_cid;
3494 pwrb = io_task->pwrb_handle->pwrb;
3495 memset(pwrb, 0, sizeof(*pwrb));
3496 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3497 be32_to_cpu(task->cmdsn));
3498 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3499 io_task->pwrb_handle->wrb_index);
3500 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3501 io_task->psgl_handle->sgl_index);
3502 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3503 case ISCSI_OP_LOGIN:
3504 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3505 TGT_DM_CMD);
3506 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3507 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3508 hwi_write_buffer(pwrb, task);
3509 break;
3510 case ISCSI_OP_NOOP_OUT:
3511 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3512 INI_RD_CMD);
3513 if (task->hdr->ttt == ISCSI_RESERVED_TAG)
3514 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3515 else
3516 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
3517 hwi_write_buffer(pwrb, task);
3518 break;
3519 case ISCSI_OP_TEXT:
3520 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3521 TGT_DM_CMD);
3522 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3523 hwi_write_buffer(pwrb, task);
3524 break;
3525 case ISCSI_OP_SCSI_TMFUNC:
3526 session = conn->session;
3527 i = ((struct iscsi_tm *)task->hdr)->rtt;
3528 phwi_ctrlr = phba->phwi_ctrlr;
3529 pwrb_context = &phwi_ctrlr->wrb_context[cid -
3530 phba->fw_config.iscsi_cid_start];
3531 pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i)
3532 >> 16];
3533 aborted_task = pwrb_handle->pio_handle;
3534 if (!aborted_task)
3535 return 0;
3536
3537 aborted_io_task = aborted_task->dd_data;
3538 if (!aborted_io_task->scsi_cmnd)
3539 return 0;
3540
3541 tag = mgmt_invalidate_icds(phba,
3542 aborted_io_task->psgl_handle->sgl_index,
3543 cid);
3544 if (!tag) {
3545 shost_printk(KERN_WARNING, phba->shost,
3546 "mgmt_invalidate_icds could not be"
3547 " submitted\n");
3548 } else {
3549 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3550 phba->ctrl.mcc_numtag[tag]);
3551 free_mcc_tag(&phba->ctrl, tag);
3552 }
3553 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3554 INI_TMF_CMD);
3555 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3556 hwi_write_buffer(pwrb, task);
3557 break;
3558 case ISCSI_OP_LOGOUT:
3559 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3560 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3561 HWH_TYPE_LOGOUT);
3562 hwi_write_buffer(pwrb, task);
3563 break;
3564
3565 default:
3566 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3567 task->hdr->opcode & ISCSI_OPCODE_MASK);
3568 return -EINVAL;
3569 }
3570
3571 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3572 task->data_count);
3573 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3574 io_task->pwrb_handle->nxt_wrb_index);
3575 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3576
3577 doorbell |= cid & DB_WRB_POST_CID_MASK;
3578 doorbell |= (io_task->pwrb_handle->wrb_index &
3579 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3580 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3581 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3582 return 0;
3583 }
3584
3585 static int beiscsi_task_xmit(struct iscsi_task *task)
3586 {
3587 struct iscsi_conn *conn = task->conn;
3588 struct beiscsi_io_task *io_task = task->dd_data;
3589 struct scsi_cmnd *sc = task->sc;
3590 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3591 struct scatterlist *sg;
3592 int num_sg;
3593 unsigned int writedir = 0, xferlen = 0;
3594
3595 SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3596 "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3597 task, conn, beiscsi_conn);
3598 if (!sc)
3599 return beiscsi_mtask(task);
3600
3601 io_task->scsi_cmnd = sc;
3602 num_sg = scsi_dma_map(sc);
3603 if (num_sg < 0) {
3604 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3605 return num_sg;
3606 }
3607 SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3608 (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3609 xferlen = scsi_bufflen(sc);
3610 sg = scsi_sglist(sc);
3611 if (sc->sc_data_direction == DMA_TO_DEVICE) {
3612 writedir = 1;
3613 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3614 task->imm_count);
3615 } else
3616 writedir = 0;
3617 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3618 }
3619
3620 static void beiscsi_remove(struct pci_dev *pcidev)
3621 {
3622 struct beiscsi_hba *phba = NULL;
3623 struct hwi_controller *phwi_ctrlr;
3624 struct hwi_context_memory *phwi_context;
3625 struct be_eq_obj *pbe_eq;
3626 unsigned int i, msix_vec;
3627
3628 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3629 if (!phba) {
3630 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3631 return;
3632 }
3633
3634 phwi_ctrlr = phba->phwi_ctrlr;
3635 phwi_context = phwi_ctrlr->phwi_ctxt;
3636 hwi_disable_intr(phba);
3637 if (phba->msix_enabled) {
3638 for (i = 0; i <= phba->num_cpus; i++) {
3639 msix_vec = phba->msix_entries[i].vector;
3640 free_irq(msix_vec, &phwi_context->be_eq[i]);
3641 }
3642 } else
3643 if (phba->pcidev->irq)
3644 free_irq(phba->pcidev->irq, phba);
3645 pci_disable_msix(phba->pcidev);
3646 destroy_workqueue(phba->wq);
3647 if (blk_iopoll_enabled)
3648 for (i = 0; i < phba->num_cpus; i++) {
3649 pbe_eq = &phwi_context->be_eq[i];
3650 blk_iopoll_disable(&pbe_eq->iopoll);
3651 }
3652
3653 beiscsi_clean_port(phba);
3654 beiscsi_free_mem(phba);
3655 beiscsi_unmap_pci_function(phba);
3656 pci_free_consistent(phba->pcidev,
3657 phba->ctrl.mbox_mem_alloced.size,
3658 phba->ctrl.mbox_mem_alloced.va,
3659 phba->ctrl.mbox_mem_alloced.dma);
3660 iscsi_host_remove(phba->shost);
3661 pci_dev_put(phba->pcidev);
3662 iscsi_host_free(phba->shost);
3663 }
3664
3665 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3666 {
3667 int i, status;
3668
3669 for (i = 0; i <= phba->num_cpus; i++)
3670 phba->msix_entries[i].entry = i;
3671
3672 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3673 (phba->num_cpus + 1));
3674 if (!status)
3675 phba->msix_enabled = true;
3676
3677 return;
3678 }
3679
3680 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3681 const struct pci_device_id *id)
3682 {
3683 struct beiscsi_hba *phba = NULL;
3684 struct hwi_controller *phwi_ctrlr;
3685 struct hwi_context_memory *phwi_context;
3686 struct be_eq_obj *pbe_eq;
3687 int ret, msix_vec, num_cpus, i;
3688
3689 ret = beiscsi_enable_pci(pcidev);
3690 if (ret < 0) {
3691 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3692 "Failed to enable pci device \n");
3693 return ret;
3694 }
3695
3696 phba = beiscsi_hba_alloc(pcidev);
3697 if (!phba) {
3698 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3699 " Failed in beiscsi_hba_alloc \n");
3700 goto disable_pci;
3701 }
3702 SE_DEBUG(DBG_LVL_8, " phba = %p \n", phba);
3703
3704 switch (pcidev->device) {
3705 case BE_DEVICE_ID1:
3706 case OC_DEVICE_ID1:
3707 case OC_DEVICE_ID2:
3708 phba->generation = BE_GEN2;
3709 break;
3710 case BE_DEVICE_ID2:
3711 case OC_DEVICE_ID3:
3712 phba->generation = BE_GEN3;
3713 break;
3714 default:
3715 phba->generation = 0;
3716 }
3717
3718 if (enable_msix)
3719 num_cpus = find_num_cpus();
3720 else
3721 num_cpus = 1;
3722 phba->num_cpus = num_cpus;
3723 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3724
3725 if (enable_msix)
3726 beiscsi_msix_enable(phba);
3727 ret = be_ctrl_init(phba, pcidev);
3728 if (ret) {
3729 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3730 "Failed in be_ctrl_init\n");
3731 goto hba_free;
3732 }
3733
3734 spin_lock_init(&phba->io_sgl_lock);
3735 spin_lock_init(&phba->mgmt_sgl_lock);
3736 spin_lock_init(&phba->isr_lock);
3737 ret = mgmt_get_fw_config(&phba->ctrl, phba);
3738 if (ret != 0) {
3739 shost_printk(KERN_ERR, phba->shost,
3740 "Error getting fw config\n");
3741 goto free_port;
3742 }
3743 phba->shost->max_id = phba->fw_config.iscsi_cid_count;
3744 beiscsi_get_params(phba);
3745 phba->shost->can_queue = phba->params.ios_per_ctrl;
3746 ret = beiscsi_init_port(phba);
3747 if (ret < 0) {
3748 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3749 "Failed in beiscsi_init_port\n");
3750 goto free_port;
3751 }
3752
3753 for (i = 0; i < MAX_MCC_CMD ; i++) {
3754 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
3755 phba->ctrl.mcc_tag[i] = i + 1;
3756 phba->ctrl.mcc_numtag[i + 1] = 0;
3757 phba->ctrl.mcc_tag_available++;
3758 }
3759
3760 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
3761
3762 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3763 phba->shost->host_no);
3764 phba->wq = create_workqueue(phba->wq_name);
3765 if (!phba->wq) {
3766 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3767 "Failed to allocate work queue\n");
3768 goto free_twq;
3769 }
3770
3771 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3772
3773 phwi_ctrlr = phba->phwi_ctrlr;
3774 phwi_context = phwi_ctrlr->phwi_ctxt;
3775 if (blk_iopoll_enabled) {
3776 for (i = 0; i < phba->num_cpus; i++) {
3777 pbe_eq = &phwi_context->be_eq[i];
3778 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3779 be_iopoll);
3780 blk_iopoll_enable(&pbe_eq->iopoll);
3781 }
3782 }
3783 ret = beiscsi_init_irqs(phba);
3784 if (ret < 0) {
3785 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3786 "Failed to beiscsi_init_irqs\n");
3787 goto free_blkenbld;
3788 }
3789 ret = hwi_enable_intr(phba);
3790 if (ret < 0) {
3791 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3792 "Failed to hwi_enable_intr\n");
3793 goto free_ctrlr;
3794 }
3795 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3796 return 0;
3797
3798 free_ctrlr:
3799 if (phba->msix_enabled) {
3800 for (i = 0; i <= phba->num_cpus; i++) {
3801 msix_vec = phba->msix_entries[i].vector;
3802 free_irq(msix_vec, &phwi_context->be_eq[i]);
3803 }
3804 } else
3805 if (phba->pcidev->irq)
3806 free_irq(phba->pcidev->irq, phba);
3807 pci_disable_msix(phba->pcidev);
3808 free_blkenbld:
3809 destroy_workqueue(phba->wq);
3810 if (blk_iopoll_enabled)
3811 for (i = 0; i < phba->num_cpus; i++) {
3812 pbe_eq = &phwi_context->be_eq[i];
3813 blk_iopoll_disable(&pbe_eq->iopoll);
3814 }
3815 free_twq:
3816 beiscsi_clean_port(phba);
3817 beiscsi_free_mem(phba);
3818 free_port:
3819 pci_free_consistent(phba->pcidev,
3820 phba->ctrl.mbox_mem_alloced.size,
3821 phba->ctrl.mbox_mem_alloced.va,
3822 phba->ctrl.mbox_mem_alloced.dma);
3823 beiscsi_unmap_pci_function(phba);
3824 hba_free:
3825 iscsi_host_remove(phba->shost);
3826 pci_dev_put(phba->pcidev);
3827 iscsi_host_free(phba->shost);
3828 disable_pci:
3829 pci_disable_device(pcidev);
3830 return ret;
3831 }
3832
3833 struct iscsi_transport beiscsi_iscsi_transport = {
3834 .owner = THIS_MODULE,
3835 .name = DRV_NAME,
3836 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
3837 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3838 .param_mask = ISCSI_MAX_RECV_DLENGTH |
3839 ISCSI_MAX_XMIT_DLENGTH |
3840 ISCSI_HDRDGST_EN |
3841 ISCSI_DATADGST_EN |
3842 ISCSI_INITIAL_R2T_EN |
3843 ISCSI_MAX_R2T |
3844 ISCSI_IMM_DATA_EN |
3845 ISCSI_FIRST_BURST |
3846 ISCSI_MAX_BURST |
3847 ISCSI_PDU_INORDER_EN |
3848 ISCSI_DATASEQ_INORDER_EN |
3849 ISCSI_ERL |
3850 ISCSI_CONN_PORT |
3851 ISCSI_CONN_ADDRESS |
3852 ISCSI_EXP_STATSN |
3853 ISCSI_PERSISTENT_PORT |
3854 ISCSI_PERSISTENT_ADDRESS |
3855 ISCSI_TARGET_NAME | ISCSI_TPGT |
3856 ISCSI_USERNAME | ISCSI_PASSWORD |
3857 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3858 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3859 ISCSI_LU_RESET_TMO |
3860 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3861 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3862 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3863 ISCSI_HOST_INITIATOR_NAME,
3864 .create_session = beiscsi_session_create,
3865 .destroy_session = beiscsi_session_destroy,
3866 .create_conn = beiscsi_conn_create,
3867 .bind_conn = beiscsi_conn_bind,
3868 .destroy_conn = iscsi_conn_teardown,
3869 .set_param = beiscsi_set_param,
3870 .get_conn_param = beiscsi_conn_get_param,
3871 .get_session_param = iscsi_session_get_param,
3872 .get_host_param = beiscsi_get_host_param,
3873 .start_conn = beiscsi_conn_start,
3874 .stop_conn = beiscsi_conn_stop,
3875 .send_pdu = iscsi_conn_send_pdu,
3876 .xmit_task = beiscsi_task_xmit,
3877 .cleanup_task = beiscsi_cleanup_task,
3878 .alloc_pdu = beiscsi_alloc_pdu,
3879 .parse_pdu_itt = beiscsi_parse_pdu,
3880 .get_stats = beiscsi_conn_get_stats,
3881 .ep_connect = beiscsi_ep_connect,
3882 .ep_poll = beiscsi_ep_poll,
3883 .ep_disconnect = beiscsi_ep_disconnect,
3884 .session_recovery_timedout = iscsi_session_recovery_timedout,
3885 };
3886
3887 static struct pci_driver beiscsi_pci_driver = {
3888 .name = DRV_NAME,
3889 .probe = beiscsi_dev_probe,
3890 .remove = beiscsi_remove,
3891 .id_table = beiscsi_pci_id_table
3892 };
3893
3894
3895 static int __init beiscsi_module_init(void)
3896 {
3897 int ret;
3898
3899 beiscsi_scsi_transport =
3900 iscsi_register_transport(&beiscsi_iscsi_transport);
3901 if (!beiscsi_scsi_transport) {
3902 SE_DEBUG(DBG_LVL_1,
3903 "beiscsi_module_init - Unable to register beiscsi"
3904 "transport.\n");
3905 return -ENOMEM;
3906 }
3907 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3908 &beiscsi_iscsi_transport);
3909
3910 ret = pci_register_driver(&beiscsi_pci_driver);
3911 if (ret) {
3912 SE_DEBUG(DBG_LVL_1,
3913 "beiscsi_module_init - Unable to register"
3914 "beiscsi pci driver.\n");
3915 goto unregister_iscsi_transport;
3916 }
3917 return 0;
3918
3919 unregister_iscsi_transport:
3920 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3921 return ret;
3922 }
3923
3924 static void __exit beiscsi_module_exit(void)
3925 {
3926 pci_unregister_driver(&beiscsi_pci_driver);
3927 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3928 }
3929
3930 module_init(beiscsi_module_init);
3931 module_exit(beiscsi_module_exit);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree