?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
vmwgfx: Fix assignment in vmw_framebuffer_create_handle
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / scsi / be2iscsi / be_main.c
blob379c696dac19cb31f6c2cb43f9bcc0e647a89725
1 /**
2 * Copyright (C) 2005 - 2011 Emulex
3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
11 *
12 * Contact Information:
13 * linux-drivers@emulex.com
14 *
15 * Emulex
16 * 3333 Susan Street
17 * Costa Mesa, CA 92626
18 */
19
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/string.h>
27 #include <linux/kernel.h>
28 #include <linux/semaphore.h>
29 #include <linux/iscsi_boot_sysfs.h>
30 #include <linux/module.h>
31
32 #include <scsi/libiscsi.h>
33 #include <scsi/scsi_transport_iscsi.h>
34 #include <scsi/scsi_transport.h>
35 #include <scsi/scsi_cmnd.h>
36 #include <scsi/scsi_device.h>
37 #include <scsi/scsi_host.h>
38 #include <scsi/scsi.h>
39 #include "be_main.h"
40 #include "be_iscsi.h"
41 #include "be_mgmt.h"
42
43 static unsigned int be_iopoll_budget = 10;
44 static unsigned int be_max_phys_size = 64;
45 static unsigned int enable_msix = 1;
46 static unsigned int gcrashmode = 0;
47 static unsigned int num_hba = 0;
48
49 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
50 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
51 MODULE_AUTHOR("ServerEngines Corporation");
52 MODULE_LICENSE("GPL");
53 module_param(be_iopoll_budget, int, 0);
54 module_param(enable_msix, int, 0);
55 module_param(be_max_phys_size, uint, S_IRUGO);
56 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
57 "contiguous memory that can be allocated."
58 "Range is 16 - 128");
59
60 static int beiscsi_slave_configure(struct scsi_device *sdev)
61 {
62 blk_queue_max_segment_size(sdev->request_queue, 65536);
63 return 0;
64 }
65
66 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
67 {
68 struct iscsi_cls_session *cls_session;
69 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
70 struct beiscsi_io_task *aborted_io_task;
71 struct iscsi_conn *conn;
72 struct beiscsi_conn *beiscsi_conn;
73 struct beiscsi_hba *phba;
74 struct iscsi_session *session;
75 struct invalidate_command_table *inv_tbl;
76 struct be_dma_mem nonemb_cmd;
77 unsigned int cid, tag, num_invalidate;
78
79 cls_session = starget_to_session(scsi_target(sc->device));
80 session = cls_session->dd_data;
81
82 spin_lock_bh(&session->lock);
83 if (!aborted_task || !aborted_task->sc) {
84 /* we raced */
85 spin_unlock_bh(&session->lock);
86 return SUCCESS;
87 }
88
89 aborted_io_task = aborted_task->dd_data;
90 if (!aborted_io_task->scsi_cmnd) {
91 /* raced or invalid command */
92 spin_unlock_bh(&session->lock);
93 return SUCCESS;
94 }
95 spin_unlock_bh(&session->lock);
96 conn = aborted_task->conn;
97 beiscsi_conn = conn->dd_data;
98 phba = beiscsi_conn->phba;
99
100 /* invalidate iocb */
101 cid = beiscsi_conn->beiscsi_conn_cid;
102 inv_tbl = phba->inv_tbl;
103 memset(inv_tbl, 0x0, sizeof(*inv_tbl));
104 inv_tbl->cid = cid;
105 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
106 num_invalidate = 1;
107 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
108 sizeof(struct invalidate_commands_params_in),
109 &nonemb_cmd.dma);
110 if (nonemb_cmd.va == NULL) {
111 SE_DEBUG(DBG_LVL_1,
112 "Failed to allocate memory for"
113 "mgmt_invalidate_icds\n");
114 return FAILED;
115 }
116 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
117
118 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
119 cid, &nonemb_cmd);
120 if (!tag) {
121 shost_printk(KERN_WARNING, phba->shost,
122 "mgmt_invalidate_icds could not be"
123 " submitted\n");
124 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
125 nonemb_cmd.va, nonemb_cmd.dma);
126
127 return FAILED;
128 } else {
129 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
130 phba->ctrl.mcc_numtag[tag]);
131 free_mcc_tag(&phba->ctrl, tag);
132 }
133 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
134 nonemb_cmd.va, nonemb_cmd.dma);
135 return iscsi_eh_abort(sc);
136 }
137
138 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
139 {
140 struct iscsi_task *abrt_task;
141 struct beiscsi_io_task *abrt_io_task;
142 struct iscsi_conn *conn;
143 struct beiscsi_conn *beiscsi_conn;
144 struct beiscsi_hba *phba;
145 struct iscsi_session *session;
146 struct iscsi_cls_session *cls_session;
147 struct invalidate_command_table *inv_tbl;
148 struct be_dma_mem nonemb_cmd;
149 unsigned int cid, tag, i, num_invalidate;
150 int rc = FAILED;
151
152 /* invalidate iocbs */
153 cls_session = starget_to_session(scsi_target(sc->device));
154 session = cls_session->dd_data;
155 spin_lock_bh(&session->lock);
156 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
157 goto unlock;
158
159 conn = session->leadconn;
160 beiscsi_conn = conn->dd_data;
161 phba = beiscsi_conn->phba;
162 cid = beiscsi_conn->beiscsi_conn_cid;
163 inv_tbl = phba->inv_tbl;
164 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
165 num_invalidate = 0;
166 for (i = 0; i < conn->session->cmds_max; i++) {
167 abrt_task = conn->session->cmds[i];
168 abrt_io_task = abrt_task->dd_data;
169 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
170 continue;
171
172 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
173 continue;
174
175 inv_tbl->cid = cid;
176 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
177 num_invalidate++;
178 inv_tbl++;
179 }
180 spin_unlock_bh(&session->lock);
181 inv_tbl = phba->inv_tbl;
182
183 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
184 sizeof(struct invalidate_commands_params_in),
185 &nonemb_cmd.dma);
186 if (nonemb_cmd.va == NULL) {
187 SE_DEBUG(DBG_LVL_1,
188 "Failed to allocate memory for"
189 "mgmt_invalidate_icds\n");
190 return FAILED;
191 }
192 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
193 memset(nonemb_cmd.va, 0, nonemb_cmd.size);
194 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
195 cid, &nonemb_cmd);
196 if (!tag) {
197 shost_printk(KERN_WARNING, phba->shost,
198 "mgmt_invalidate_icds could not be"
199 " submitted\n");
200 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
201 nonemb_cmd.va, nonemb_cmd.dma);
202 return FAILED;
203 } else {
204 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
205 phba->ctrl.mcc_numtag[tag]);
206 free_mcc_tag(&phba->ctrl, tag);
207 }
208 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
209 nonemb_cmd.va, nonemb_cmd.dma);
210 return iscsi_eh_device_reset(sc);
211 unlock:
212 spin_unlock_bh(&session->lock);
213 return rc;
214 }
215
216 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
217 {
218 struct beiscsi_hba *phba = data;
219 struct mgmt_session_info *boot_sess = &phba->boot_sess;
220 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
221 char *str = buf;
222 int rc;
223
224 switch (type) {
225 case ISCSI_BOOT_TGT_NAME:
226 rc = sprintf(buf, "%.*s\n",
227 (int)strlen(boot_sess->target_name),
228 (char *)&boot_sess->target_name);
229 break;
230 case ISCSI_BOOT_TGT_IP_ADDR:
231 if (boot_conn->dest_ipaddr.ip_type == 0x1)
232 rc = sprintf(buf, "%pI4\n",
233 (char *)&boot_conn->dest_ipaddr.ip_address);
234 else
235 rc = sprintf(str, "%pI6\n",
236 (char *)&boot_conn->dest_ipaddr.ip_address);
237 break;
238 case ISCSI_BOOT_TGT_PORT:
239 rc = sprintf(str, "%d\n", boot_conn->dest_port);
240 break;
241
242 case ISCSI_BOOT_TGT_CHAP_NAME:
243 rc = sprintf(str, "%.*s\n",
244 boot_conn->negotiated_login_options.auth_data.chap.
245 target_chap_name_length,
246 (char *)&boot_conn->negotiated_login_options.
247 auth_data.chap.target_chap_name);
248 break;
249 case ISCSI_BOOT_TGT_CHAP_SECRET:
250 rc = sprintf(str, "%.*s\n",
251 boot_conn->negotiated_login_options.auth_data.chap.
252 target_secret_length,
253 (char *)&boot_conn->negotiated_login_options.
254 auth_data.chap.target_secret);
255 break;
256 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
257 rc = sprintf(str, "%.*s\n",
258 boot_conn->negotiated_login_options.auth_data.chap.
259 intr_chap_name_length,
260 (char *)&boot_conn->negotiated_login_options.
261 auth_data.chap.intr_chap_name);
262 break;
263 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
264 rc = sprintf(str, "%.*s\n",
265 boot_conn->negotiated_login_options.auth_data.chap.
266 intr_secret_length,
267 (char *)&boot_conn->negotiated_login_options.
268 auth_data.chap.intr_secret);
269 break;
270 case ISCSI_BOOT_TGT_FLAGS:
271 rc = sprintf(str, "2\n");
272 break;
273 case ISCSI_BOOT_TGT_NIC_ASSOC:
274 rc = sprintf(str, "0\n");
275 break;
276 default:
277 rc = -ENOSYS;
278 break;
279 }
280 return rc;
281 }
282
283 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
284 {
285 struct beiscsi_hba *phba = data;
286 char *str = buf;
287 int rc;
288
289 switch (type) {
290 case ISCSI_BOOT_INI_INITIATOR_NAME:
291 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
292 break;
293 default:
294 rc = -ENOSYS;
295 break;
296 }
297 return rc;
298 }
299
300 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
301 {
302 struct beiscsi_hba *phba = data;
303 char *str = buf;
304 int rc;
305
306 switch (type) {
307 case ISCSI_BOOT_ETH_FLAGS:
308 rc = sprintf(str, "2\n");
309 break;
310 case ISCSI_BOOT_ETH_INDEX:
311 rc = sprintf(str, "0\n");
312 break;
313 case ISCSI_BOOT_ETH_MAC:
314 rc = beiscsi_get_macaddr(buf, phba);
315 if (rc < 0) {
316 SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
317 return rc;
318 }
319 break;
320 default:
321 rc = -ENOSYS;
322 break;
323 }
324 return rc;
325 }
326
327
328 static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
329 {
330 int rc;
331
332 switch (type) {
333 case ISCSI_BOOT_TGT_NAME:
334 case ISCSI_BOOT_TGT_IP_ADDR:
335 case ISCSI_BOOT_TGT_PORT:
336 case ISCSI_BOOT_TGT_CHAP_NAME:
337 case ISCSI_BOOT_TGT_CHAP_SECRET:
338 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
339 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
340 case ISCSI_BOOT_TGT_NIC_ASSOC:
341 case ISCSI_BOOT_TGT_FLAGS:
342 rc = S_IRUGO;
343 break;
344 default:
345 rc = 0;
346 break;
347 }
348 return rc;
349 }
350
351 static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
352 {
353 int rc;
354
355 switch (type) {
356 case ISCSI_BOOT_INI_INITIATOR_NAME:
357 rc = S_IRUGO;
358 break;
359 default:
360 rc = 0;
361 break;
362 }
363 return rc;
364 }
365
366
367 static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
368 {
369 int rc;
370
371 switch (type) {
372 case ISCSI_BOOT_ETH_FLAGS:
373 case ISCSI_BOOT_ETH_MAC:
374 case ISCSI_BOOT_ETH_INDEX:
375 rc = S_IRUGO;
376 break;
377 default:
378 rc = 0;
379 break;
380 }
381 return rc;
382 }
383
384 /*------------------- PCI Driver operations and data ----------------- */
385 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
386 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
387 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
388 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
389 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
390 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
391 { 0 }
392 };
393 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
394
395 static struct scsi_host_template beiscsi_sht = {
396 .module = THIS_MODULE,
397 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
398 .proc_name = DRV_NAME,
399 .queuecommand = iscsi_queuecommand,
400 .change_queue_depth = iscsi_change_queue_depth,
401 .slave_configure = beiscsi_slave_configure,
402 .target_alloc = iscsi_target_alloc,
403 .eh_abort_handler = beiscsi_eh_abort,
404 .eh_device_reset_handler = beiscsi_eh_device_reset,
405 .eh_target_reset_handler = iscsi_eh_session_reset,
406 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
407 .can_queue = BE2_IO_DEPTH,
408 .this_id = -1,
409 .max_sectors = BEISCSI_MAX_SECTORS,
410 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
411 .use_clustering = ENABLE_CLUSTERING,
412 };
413
414 static struct scsi_transport_template *beiscsi_scsi_transport;
415
416 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
417 {
418 struct beiscsi_hba *phba;
419 struct Scsi_Host *shost;
420
421 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
422 if (!shost) {
423 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
424 "iscsi_host_alloc failed\n");
425 return NULL;
426 }
427 shost->dma_boundary = pcidev->dma_mask;
428 shost->max_id = BE2_MAX_SESSIONS;
429 shost->max_channel = 0;
430 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
431 shost->max_lun = BEISCSI_NUM_MAX_LUN;
432 shost->transportt = beiscsi_scsi_transport;
433 phba = iscsi_host_priv(shost);
434 memset(phba, 0, sizeof(*phba));
435 phba->shost = shost;
436 phba->pcidev = pci_dev_get(pcidev);
437 pci_set_drvdata(pcidev, phba);
438
439 if (iscsi_host_add(shost, &phba->pcidev->dev))
440 goto free_devices;
441
442 return phba;
443
444 free_devices:
445 pci_dev_put(phba->pcidev);
446 iscsi_host_free(phba->shost);
447 return NULL;
448 }
449
450 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
451 {
452 if (phba->csr_va) {
453 iounmap(phba->csr_va);
454 phba->csr_va = NULL;
455 }
456 if (phba->db_va) {
457 iounmap(phba->db_va);
458 phba->db_va = NULL;
459 }
460 if (phba->pci_va) {
461 iounmap(phba->pci_va);
462 phba->pci_va = NULL;
463 }
464 }
465
466 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
467 struct pci_dev *pcidev)
468 {
469 u8 __iomem *addr;
470 int pcicfg_reg;
471
472 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
473 pci_resource_len(pcidev, 2));
474 if (addr == NULL)
475 return -ENOMEM;
476 phba->ctrl.csr = addr;
477 phba->csr_va = addr;
478 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
479
480 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
481 if (addr == NULL)
482 goto pci_map_err;
483 phba->ctrl.db = addr;
484 phba->db_va = addr;
485 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
486
487 if (phba->generation == BE_GEN2)
488 pcicfg_reg = 1;
489 else
490 pcicfg_reg = 0;
491
492 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
493 pci_resource_len(pcidev, pcicfg_reg));
494
495 if (addr == NULL)
496 goto pci_map_err;
497 phba->ctrl.pcicfg = addr;
498 phba->pci_va = addr;
499 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
500 return 0;
501
502 pci_map_err:
503 beiscsi_unmap_pci_function(phba);
504 return -ENOMEM;
505 }
506
507 static int beiscsi_enable_pci(struct pci_dev *pcidev)
508 {
509 int ret;
510
511 ret = pci_enable_device(pcidev);
512 if (ret) {
513 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
514 "failed. Returning -ENODEV\n");
515 return ret;
516 }
517
518 pci_set_master(pcidev);
519 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
520 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
521 if (ret) {
522 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
523 pci_disable_device(pcidev);
524 return ret;
525 }
526 }
527 return 0;
528 }
529
530 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
531 {
532 struct be_ctrl_info *ctrl = &phba->ctrl;
533 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
534 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
535 int status = 0;
536
537 ctrl->pdev = pdev;
538 status = beiscsi_map_pci_bars(phba, pdev);
539 if (status)
540 return status;
541 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
542 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
543 mbox_mem_alloc->size,
544 &mbox_mem_alloc->dma);
545 if (!mbox_mem_alloc->va) {
546 beiscsi_unmap_pci_function(phba);
547 status = -ENOMEM;
548 return status;
549 }
550
551 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
552 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
553 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
554 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
555 spin_lock_init(&ctrl->mbox_lock);
556 spin_lock_init(&phba->ctrl.mcc_lock);
557 spin_lock_init(&phba->ctrl.mcc_cq_lock);
558
559 return status;
560 }
561
562 static void beiscsi_get_params(struct beiscsi_hba *phba)
563 {
564 phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
565 - (phba->fw_config.iscsi_cid_count
566 + BE2_TMFS
567 + BE2_NOPOUT_REQ));
568 phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
569 phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
570 phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
571 phba->params.num_sge_per_io = BE2_SGE;
572 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
573 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
574 phba->params.eq_timer = 64;
575 phba->params.num_eq_entries =
576 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
577 + BE2_TMFS) / 512) + 1) * 512;
578 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
579 ? 1024 : phba->params.num_eq_entries;
580 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
581 phba->params.num_eq_entries);
582 phba->params.num_cq_entries =
583 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
584 + BE2_TMFS) / 512) + 1) * 512;
585 phba->params.wrbs_per_cxn = 256;
586 }
587
588 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
589 unsigned int id, unsigned int clr_interrupt,
590 unsigned int num_processed,
591 unsigned char rearm, unsigned char event)
592 {
593 u32 val = 0;
594 val |= id & DB_EQ_RING_ID_MASK;
595 if (rearm)
596 val |= 1 << DB_EQ_REARM_SHIFT;
597 if (clr_interrupt)
598 val |= 1 << DB_EQ_CLR_SHIFT;
599 if (event)
600 val |= 1 << DB_EQ_EVNT_SHIFT;
601 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
602 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
603 }
604
605 /**
606 * be_isr_mcc - The isr routine of the driver.
607 * @irq: Not used
608 * @dev_id: Pointer to host adapter structure
609 */
610 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
611 {
612 struct beiscsi_hba *phba;
613 struct be_eq_entry *eqe = NULL;
614 struct be_queue_info *eq;
615 struct be_queue_info *mcc;
616 unsigned int num_eq_processed;
617 struct be_eq_obj *pbe_eq;
618 unsigned long flags;
619
620 pbe_eq = dev_id;
621 eq = &pbe_eq->q;
622 phba = pbe_eq->phba;
623 mcc = &phba->ctrl.mcc_obj.cq;
624 eqe = queue_tail_node(eq);
625 if (!eqe)
626 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
627
628 num_eq_processed = 0;
629
630 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
631 & EQE_VALID_MASK) {
632 if (((eqe->dw[offsetof(struct amap_eq_entry,
633 resource_id) / 32] &
634 EQE_RESID_MASK) >> 16) == mcc->id) {
635 spin_lock_irqsave(&phba->isr_lock, flags);
636 phba->todo_mcc_cq = 1;
637 spin_unlock_irqrestore(&phba->isr_lock, flags);
638 }
639 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
640 queue_tail_inc(eq);
641 eqe = queue_tail_node(eq);
642 num_eq_processed++;
643 }
644 if (phba->todo_mcc_cq)
645 queue_work(phba->wq, &phba->work_cqs);
646 if (num_eq_processed)
647 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
648
649 return IRQ_HANDLED;
650 }
651
652 /**
653 * be_isr_msix - The isr routine of the driver.
654 * @irq: Not used
655 * @dev_id: Pointer to host adapter structure
656 */
657 static irqreturn_t be_isr_msix(int irq, void *dev_id)
658 {
659 struct beiscsi_hba *phba;
660 struct be_eq_entry *eqe = NULL;
661 struct be_queue_info *eq;
662 struct be_queue_info *cq;
663 unsigned int num_eq_processed;
664 struct be_eq_obj *pbe_eq;
665 unsigned long flags;
666
667 pbe_eq = dev_id;
668 eq = &pbe_eq->q;
669 cq = pbe_eq->cq;
670 eqe = queue_tail_node(eq);
671 if (!eqe)
672 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
673
674 phba = pbe_eq->phba;
675 num_eq_processed = 0;
676 if (blk_iopoll_enabled) {
677 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
678 & EQE_VALID_MASK) {
679 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
680 blk_iopoll_sched(&pbe_eq->iopoll);
681
682 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
683 queue_tail_inc(eq);
684 eqe = queue_tail_node(eq);
685 num_eq_processed++;
686 }
687 if (num_eq_processed)
688 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
689
690 return IRQ_HANDLED;
691 } else {
692 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
693 & EQE_VALID_MASK) {
694 spin_lock_irqsave(&phba->isr_lock, flags);
695 phba->todo_cq = 1;
696 spin_unlock_irqrestore(&phba->isr_lock, flags);
697 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
698 queue_tail_inc(eq);
699 eqe = queue_tail_node(eq);
700 num_eq_processed++;
701 }
702 if (phba->todo_cq)
703 queue_work(phba->wq, &phba->work_cqs);
704
705 if (num_eq_processed)
706 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
707
708 return IRQ_HANDLED;
709 }
710 }
711
712 /**
713 * be_isr - The isr routine of the driver.
714 * @irq: Not used
715 * @dev_id: Pointer to host adapter structure
716 */
717 static irqreturn_t be_isr(int irq, void *dev_id)
718 {
719 struct beiscsi_hba *phba;
720 struct hwi_controller *phwi_ctrlr;
721 struct hwi_context_memory *phwi_context;
722 struct be_eq_entry *eqe = NULL;
723 struct be_queue_info *eq;
724 struct be_queue_info *cq;
725 struct be_queue_info *mcc;
726 unsigned long flags, index;
727 unsigned int num_mcceq_processed, num_ioeq_processed;
728 struct be_ctrl_info *ctrl;
729 struct be_eq_obj *pbe_eq;
730 int isr;
731
732 phba = dev_id;
733 ctrl = &phba->ctrl;
734 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
735 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
736 if (!isr)
737 return IRQ_NONE;
738
739 phwi_ctrlr = phba->phwi_ctrlr;
740 phwi_context = phwi_ctrlr->phwi_ctxt;
741 pbe_eq = &phwi_context->be_eq[0];
742
743 eq = &phwi_context->be_eq[0].q;
744 mcc = &phba->ctrl.mcc_obj.cq;
745 index = 0;
746 eqe = queue_tail_node(eq);
747 if (!eqe)
748 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
749
750 num_ioeq_processed = 0;
751 num_mcceq_processed = 0;
752 if (blk_iopoll_enabled) {
753 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
754 & EQE_VALID_MASK) {
755 if (((eqe->dw[offsetof(struct amap_eq_entry,
756 resource_id) / 32] &
757 EQE_RESID_MASK) >> 16) == mcc->id) {
758 spin_lock_irqsave(&phba->isr_lock, flags);
759 phba->todo_mcc_cq = 1;
760 spin_unlock_irqrestore(&phba->isr_lock, flags);
761 num_mcceq_processed++;
762 } else {
763 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
764 blk_iopoll_sched(&pbe_eq->iopoll);
765 num_ioeq_processed++;
766 }
767 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
768 queue_tail_inc(eq);
769 eqe = queue_tail_node(eq);
770 }
771 if (num_ioeq_processed || num_mcceq_processed) {
772 if (phba->todo_mcc_cq)
773 queue_work(phba->wq, &phba->work_cqs);
774
775 if ((num_mcceq_processed) && (!num_ioeq_processed))
776 hwi_ring_eq_db(phba, eq->id, 0,
777 (num_ioeq_processed +
778 num_mcceq_processed) , 1, 1);
779 else
780 hwi_ring_eq_db(phba, eq->id, 0,
781 (num_ioeq_processed +
782 num_mcceq_processed), 0, 1);
783
784 return IRQ_HANDLED;
785 } else
786 return IRQ_NONE;
787 } else {
788 cq = &phwi_context->be_cq[0];
789 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
790 & EQE_VALID_MASK) {
791
792 if (((eqe->dw[offsetof(struct amap_eq_entry,
793 resource_id) / 32] &
794 EQE_RESID_MASK) >> 16) != cq->id) {
795 spin_lock_irqsave(&phba->isr_lock, flags);
796 phba->todo_mcc_cq = 1;
797 spin_unlock_irqrestore(&phba->isr_lock, flags);
798 } else {
799 spin_lock_irqsave(&phba->isr_lock, flags);
800 phba->todo_cq = 1;
801 spin_unlock_irqrestore(&phba->isr_lock, flags);
802 }
803 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
804 queue_tail_inc(eq);
805 eqe = queue_tail_node(eq);
806 num_ioeq_processed++;
807 }
808 if (phba->todo_cq || phba->todo_mcc_cq)
809 queue_work(phba->wq, &phba->work_cqs);
810
811 if (num_ioeq_processed) {
812 hwi_ring_eq_db(phba, eq->id, 0,
813 num_ioeq_processed, 1, 1);
814 return IRQ_HANDLED;
815 } else
816 return IRQ_NONE;
817 }
818 }
819
820 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
821 {
822 struct pci_dev *pcidev = phba->pcidev;
823 struct hwi_controller *phwi_ctrlr;
824 struct hwi_context_memory *phwi_context;
825 int ret, msix_vec, i, j;
826
827 phwi_ctrlr = phba->phwi_ctrlr;
828 phwi_context = phwi_ctrlr->phwi_ctxt;
829
830 if (phba->msix_enabled) {
831 for (i = 0; i < phba->num_cpus; i++) {
832 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
833 GFP_KERNEL);
834 if (!phba->msi_name[i]) {
835 ret = -ENOMEM;
836 goto free_msix_irqs;
837 }
838
839 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
840 phba->shost->host_no, i);
841 msix_vec = phba->msix_entries[i].vector;
842 ret = request_irq(msix_vec, be_isr_msix, 0,
843 phba->msi_name[i],
844 &phwi_context->be_eq[i]);
845 if (ret) {
846 shost_printk(KERN_ERR, phba->shost,
847 "beiscsi_init_irqs-Failed to"
848 "register msix for i = %d\n", i);
849 kfree(phba->msi_name[i]);
850 goto free_msix_irqs;
851 }
852 }
853 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
854 if (!phba->msi_name[i]) {
855 ret = -ENOMEM;
856 goto free_msix_irqs;
857 }
858 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
859 phba->shost->host_no);
860 msix_vec = phba->msix_entries[i].vector;
861 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
862 &phwi_context->be_eq[i]);
863 if (ret) {
864 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
865 "Failed to register beiscsi_msix_mcc\n");
866 kfree(phba->msi_name[i]);
867 goto free_msix_irqs;
868 }
869
870 } else {
871 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
872 "beiscsi", phba);
873 if (ret) {
874 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
875 "Failed to register irq\\n");
876 return ret;
877 }
878 }
879 return 0;
880 free_msix_irqs:
881 for (j = i - 1; j >= 0; j--) {
882 kfree(phba->msi_name[j]);
883 msix_vec = phba->msix_entries[j].vector;
884 free_irq(msix_vec, &phwi_context->be_eq[j]);
885 }
886 return ret;
887 }
888
889 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
890 unsigned int id, unsigned int num_processed,
891 unsigned char rearm, unsigned char event)
892 {
893 u32 val = 0;
894 val |= id & DB_CQ_RING_ID_MASK;
895 if (rearm)
896 val |= 1 << DB_CQ_REARM_SHIFT;
897 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
898 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
899 }
900
901 static unsigned int
902 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
903 struct beiscsi_hba *phba,
904 unsigned short cid,
905 struct pdu_base *ppdu,
906 unsigned long pdu_len,
907 void *pbuffer, unsigned long buf_len)
908 {
909 struct iscsi_conn *conn = beiscsi_conn->conn;
910 struct iscsi_session *session = conn->session;
911 struct iscsi_task *task;
912 struct beiscsi_io_task *io_task;
913 struct iscsi_hdr *login_hdr;
914
915 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
916 PDUBASE_OPCODE_MASK) {
917 case ISCSI_OP_NOOP_IN:
918 pbuffer = NULL;
919 buf_len = 0;
920 break;
921 case ISCSI_OP_ASYNC_EVENT:
922 break;
923 case ISCSI_OP_REJECT:
924 WARN_ON(!pbuffer);
925 WARN_ON(!(buf_len == 48));
926 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
927 break;
928 case ISCSI_OP_LOGIN_RSP:
929 case ISCSI_OP_TEXT_RSP:
930 task = conn->login_task;
931 io_task = task->dd_data;
932 login_hdr = (struct iscsi_hdr *)ppdu;
933 login_hdr->itt = io_task->libiscsi_itt;
934 break;
935 default:
936 shost_printk(KERN_WARNING, phba->shost,
937 "Unrecognized opcode 0x%x in async msg\n",
938 (ppdu->
939 dw[offsetof(struct amap_pdu_base, opcode) / 32]
940 & PDUBASE_OPCODE_MASK));
941 return 1;
942 }
943
944 spin_lock_bh(&session->lock);
945 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
946 spin_unlock_bh(&session->lock);
947 return 0;
948 }
949
950 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
951 {
952 struct sgl_handle *psgl_handle;
953
954 if (phba->io_sgl_hndl_avbl) {
955 SE_DEBUG(DBG_LVL_8,
956 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
957 phba->io_sgl_alloc_index);
958 psgl_handle = phba->io_sgl_hndl_base[phba->
959 io_sgl_alloc_index];
960 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
961 phba->io_sgl_hndl_avbl--;
962 if (phba->io_sgl_alloc_index == (phba->params.
963 ios_per_ctrl - 1))
964 phba->io_sgl_alloc_index = 0;
965 else
966 phba->io_sgl_alloc_index++;
967 } else
968 psgl_handle = NULL;
969 return psgl_handle;
970 }
971
972 static void
973 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
974 {
975 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
976 phba->io_sgl_free_index);
977 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
978 /*
979 * this can happen if clean_task is called on a task that
980 * failed in xmit_task or alloc_pdu.
981 */
982 SE_DEBUG(DBG_LVL_8,
983 "Double Free in IO SGL io_sgl_free_index=%d,"
984 "value there=%p\n", phba->io_sgl_free_index,
985 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
986 return;
987 }
988 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
989 phba->io_sgl_hndl_avbl++;
990 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
991 phba->io_sgl_free_index = 0;
992 else
993 phba->io_sgl_free_index++;
994 }
995
996 /**
997 * alloc_wrb_handle - To allocate a wrb handle
998 * @phba: The hba pointer
999 * @cid: The cid to use for allocation
1000 *
1001 * This happens under session_lock until submission to chip
1002 */
1003 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1004 {
1005 struct hwi_wrb_context *pwrb_context;
1006 struct hwi_controller *phwi_ctrlr;
1007 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1008
1009 phwi_ctrlr = phba->phwi_ctrlr;
1010 pwrb_context = &phwi_ctrlr->wrb_context[cid];
1011 if (pwrb_context->wrb_handles_available >= 2) {
1012 pwrb_handle = pwrb_context->pwrb_handle_base[
1013 pwrb_context->alloc_index];
1014 pwrb_context->wrb_handles_available--;
1015 if (pwrb_context->alloc_index ==
1016 (phba->params.wrbs_per_cxn - 1))
1017 pwrb_context->alloc_index = 0;
1018 else
1019 pwrb_context->alloc_index++;
1020 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1021 pwrb_context->alloc_index];
1022 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1023 } else
1024 pwrb_handle = NULL;
1025 return pwrb_handle;
1026 }
1027
1028 /**
1029 * free_wrb_handle - To free the wrb handle back to pool
1030 * @phba: The hba pointer
1031 * @pwrb_context: The context to free from
1032 * @pwrb_handle: The wrb_handle to free
1033 *
1034 * This happens under session_lock until submission to chip
1035 */
1036 static void
1037 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1038 struct wrb_handle *pwrb_handle)
1039 {
1040 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1041 pwrb_context->wrb_handles_available++;
1042 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1043 pwrb_context->free_index = 0;
1044 else
1045 pwrb_context->free_index++;
1046
1047 SE_DEBUG(DBG_LVL_8,
1048 "FREE WRB: pwrb_handle=%p free_index=0x%x"
1049 "wrb_handles_available=%d\n",
1050 pwrb_handle, pwrb_context->free_index,
1051 pwrb_context->wrb_handles_available);
1052 }
1053
1054 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1055 {
1056 struct sgl_handle *psgl_handle;
1057
1058 if (phba->eh_sgl_hndl_avbl) {
1059 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1060 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1061 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
1062 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1063 phba->eh_sgl_hndl_avbl--;
1064 if (phba->eh_sgl_alloc_index ==
1065 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1066 1))
1067 phba->eh_sgl_alloc_index = 0;
1068 else
1069 phba->eh_sgl_alloc_index++;
1070 } else
1071 psgl_handle = NULL;
1072 return psgl_handle;
1073 }
1074
1075 void
1076 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1077 {
1078
1079 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1080 phba->eh_sgl_free_index);
1081 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1082 /*
1083 * this can happen if clean_task is called on a task that
1084 * failed in xmit_task or alloc_pdu.
1085 */
1086 SE_DEBUG(DBG_LVL_8,
1087 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1088 phba->eh_sgl_free_index);
1089 return;
1090 }
1091 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1092 phba->eh_sgl_hndl_avbl++;
1093 if (phba->eh_sgl_free_index ==
1094 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1095 phba->eh_sgl_free_index = 0;
1096 else
1097 phba->eh_sgl_free_index++;
1098 }
1099
1100 static void
1101 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1102 struct iscsi_task *task, struct sol_cqe *psol)
1103 {
1104 struct beiscsi_io_task *io_task = task->dd_data;
1105 struct be_status_bhs *sts_bhs =
1106 (struct be_status_bhs *)io_task->cmd_bhs;
1107 struct iscsi_conn *conn = beiscsi_conn->conn;
1108 unsigned int sense_len;
1109 unsigned char *sense;
1110 u32 resid = 0, exp_cmdsn, max_cmdsn;
1111 u8 rsp, status, flags;
1112
1113 exp_cmdsn = (psol->
1114 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1115 & SOL_EXP_CMD_SN_MASK);
1116 max_cmdsn = ((psol->
1117 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1118 & SOL_EXP_CMD_SN_MASK) +
1119 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1120 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1121 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1122 & SOL_RESP_MASK) >> 16);
1123 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1124 & SOL_STS_MASK) >> 8);
1125 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1126 & SOL_FLAGS_MASK) >> 24) | 0x80;
1127 if (!task->sc) {
1128 if (io_task->scsi_cmnd)
1129 scsi_dma_unmap(io_task->scsi_cmnd);
1130
1131 return;
1132 }
1133 task->sc->result = (DID_OK << 16) | status;
1134 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1135 task->sc->result = DID_ERROR << 16;
1136 goto unmap;
1137 }
1138
1139 /* bidi not initially supported */
1140 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1141 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1142 32] & SOL_RES_CNT_MASK);
1143
1144 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1145 task->sc->result = DID_ERROR << 16;
1146
1147 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1148 scsi_set_resid(task->sc, resid);
1149 if (!status && (scsi_bufflen(task->sc) - resid <
1150 task->sc->underflow))
1151 task->sc->result = DID_ERROR << 16;
1152 }
1153 }
1154
1155 if (status == SAM_STAT_CHECK_CONDITION) {
1156 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1157 sense = sts_bhs->sense_info + sizeof(unsigned short);
1158 sense_len = cpu_to_be16(*slen);
1159 memcpy(task->sc->sense_buffer, sense,
1160 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1161 }
1162
1163 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1164 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1165 & SOL_RES_CNT_MASK)
1166 conn->rxdata_octets += (psol->
1167 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1168 & SOL_RES_CNT_MASK);
1169 }
1170 unmap:
1171 scsi_dma_unmap(io_task->scsi_cmnd);
1172 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1173 }
1174
1175 static void
1176 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1177 struct iscsi_task *task, struct sol_cqe *psol)
1178 {
1179 struct iscsi_logout_rsp *hdr;
1180 struct beiscsi_io_task *io_task = task->dd_data;
1181 struct iscsi_conn *conn = beiscsi_conn->conn;
1182
1183 hdr = (struct iscsi_logout_rsp *)task->hdr;
1184 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1185 hdr->t2wait = 5;
1186 hdr->t2retain = 0;
1187 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1188 & SOL_FLAGS_MASK) >> 24) | 0x80;
1189 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1190 32] & SOL_RESP_MASK);
1191 hdr->exp_cmdsn = cpu_to_be32(psol->
1192 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1193 & SOL_EXP_CMD_SN_MASK);
1194 hdr->max_cmdsn = be32_to_cpu((psol->
1195 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1196 & SOL_EXP_CMD_SN_MASK) +
1197 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1198 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1199 hdr->dlength[0] = 0;
1200 hdr->dlength[1] = 0;
1201 hdr->dlength[2] = 0;
1202 hdr->hlength = 0;
1203 hdr->itt = io_task->libiscsi_itt;
1204 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1205 }
1206
1207 static void
1208 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1209 struct iscsi_task *task, struct sol_cqe *psol)
1210 {
1211 struct iscsi_tm_rsp *hdr;
1212 struct iscsi_conn *conn = beiscsi_conn->conn;
1213 struct beiscsi_io_task *io_task = task->dd_data;
1214
1215 hdr = (struct iscsi_tm_rsp *)task->hdr;
1216 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1217 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1218 & SOL_FLAGS_MASK) >> 24) | 0x80;
1219 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1220 32] & SOL_RESP_MASK);
1221 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1222 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1223 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1224 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1225 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1226 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1227 hdr->itt = io_task->libiscsi_itt;
1228 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1229 }
1230
1231 static void
1232 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1233 struct beiscsi_hba *phba, struct sol_cqe *psol)
1234 {
1235 struct hwi_wrb_context *pwrb_context;
1236 struct wrb_handle *pwrb_handle = NULL;
1237 struct hwi_controller *phwi_ctrlr;
1238 struct iscsi_task *task;
1239 struct beiscsi_io_task *io_task;
1240 struct iscsi_conn *conn = beiscsi_conn->conn;
1241 struct iscsi_session *session = conn->session;
1242
1243 phwi_ctrlr = phba->phwi_ctrlr;
1244 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1245 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1246 SOL_CID_MASK) >> 6) -
1247 phba->fw_config.iscsi_cid_start];
1248 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1249 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1250 32] & SOL_WRB_INDEX_MASK) >> 16)];
1251 task = pwrb_handle->pio_handle;
1252
1253 io_task = task->dd_data;
1254 spin_lock(&phba->mgmt_sgl_lock);
1255 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1256 spin_unlock(&phba->mgmt_sgl_lock);
1257 spin_lock_bh(&session->lock);
1258 free_wrb_handle(phba, pwrb_context, pwrb_handle);
1259 spin_unlock_bh(&session->lock);
1260 }
1261
1262 static void
1263 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1264 struct iscsi_task *task, struct sol_cqe *psol)
1265 {
1266 struct iscsi_nopin *hdr;
1267 struct iscsi_conn *conn = beiscsi_conn->conn;
1268 struct beiscsi_io_task *io_task = task->dd_data;
1269
1270 hdr = (struct iscsi_nopin *)task->hdr;
1271 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1272 & SOL_FLAGS_MASK) >> 24) | 0x80;
1273 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1274 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1275 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1276 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1277 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1278 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1279 hdr->opcode = ISCSI_OP_NOOP_IN;
1280 hdr->itt = io_task->libiscsi_itt;
1281 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1282 }
1283
1284 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1285 struct beiscsi_hba *phba, struct sol_cqe *psol)
1286 {
1287 struct hwi_wrb_context *pwrb_context;
1288 struct wrb_handle *pwrb_handle;
1289 struct iscsi_wrb *pwrb = NULL;
1290 struct hwi_controller *phwi_ctrlr;
1291 struct iscsi_task *task;
1292 unsigned int type;
1293 struct iscsi_conn *conn = beiscsi_conn->conn;
1294 struct iscsi_session *session = conn->session;
1295
1296 phwi_ctrlr = phba->phwi_ctrlr;
1297 pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1298 (struct amap_sol_cqe, cid) / 32]
1299 & SOL_CID_MASK) >> 6) -
1300 phba->fw_config.iscsi_cid_start];
1301 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1302 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1303 32] & SOL_WRB_INDEX_MASK) >> 16)];
1304 task = pwrb_handle->pio_handle;
1305 pwrb = pwrb_handle->pwrb;
1306 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1307 WRB_TYPE_MASK) >> 28;
1308
1309 spin_lock_bh(&session->lock);
1310 switch (type) {
1311 case HWH_TYPE_IO:
1312 case HWH_TYPE_IO_RD:
1313 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1314 ISCSI_OP_NOOP_OUT)
1315 be_complete_nopin_resp(beiscsi_conn, task, psol);
1316 else
1317 be_complete_io(beiscsi_conn, task, psol);
1318 break;
1319
1320 case HWH_TYPE_LOGOUT:
1321 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1322 be_complete_logout(beiscsi_conn, task, psol);
1323 else
1324 be_complete_tmf(beiscsi_conn, task, psol);
1325
1326 break;
1327
1328 case HWH_TYPE_LOGIN:
1329 SE_DEBUG(DBG_LVL_1,
1330 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1331 "- Solicited path\n");
1332 break;
1333
1334 case HWH_TYPE_NOP:
1335 be_complete_nopin_resp(beiscsi_conn, task, psol);
1336 break;
1337
1338 default:
1339 shost_printk(KERN_WARNING, phba->shost,
1340 "In hwi_complete_cmd, unknown type = %d"
1341 "wrb_index 0x%x CID 0x%x\n", type,
1342 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1343 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1344 ((psol->dw[offsetof(struct amap_sol_cqe,
1345 cid) / 32] & SOL_CID_MASK) >> 6));
1346 break;
1347 }
1348
1349 spin_unlock_bh(&session->lock);
1350 }
1351
1352 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1353 *pasync_ctx, unsigned int is_header,
1354 unsigned int host_write_ptr)
1355 {
1356 if (is_header)
1357 return &pasync_ctx->async_entry[host_write_ptr].
1358 header_busy_list;
1359 else
1360 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1361 }
1362
1363 static struct async_pdu_handle *
1364 hwi_get_async_handle(struct beiscsi_hba *phba,
1365 struct beiscsi_conn *beiscsi_conn,
1366 struct hwi_async_pdu_context *pasync_ctx,
1367 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1368 {
1369 struct be_bus_address phys_addr;
1370 struct list_head *pbusy_list;
1371 struct async_pdu_handle *pasync_handle = NULL;
1372 int buffer_len = 0;
1373 unsigned char buffer_index = -1;
1374 unsigned char is_header = 0;
1375
1376 phys_addr.u.a32.address_lo =
1377 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1378 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1379 & PDUCQE_DPL_MASK) >> 16);
1380 phys_addr.u.a32.address_hi =
1381 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1382
1383 phys_addr.u.a64.address =
1384 *((unsigned long long *)(&phys_addr.u.a64.address));
1385
1386 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1387 & PDUCQE_CODE_MASK) {
1388 case UNSOL_HDR_NOTIFY:
1389 is_header = 1;
1390
1391 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1392 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1393 index) / 32] & PDUCQE_INDEX_MASK));
1394
1395 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1396 pasync_ctx->async_header.pa_base.u.a64.address);
1397
1398 buffer_index = buffer_len /
1399 pasync_ctx->async_header.buffer_size;
1400
1401 break;
1402 case UNSOL_DATA_NOTIFY:
1403 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1404 dw[offsetof(struct amap_i_t_dpdu_cqe,
1405 index) / 32] & PDUCQE_INDEX_MASK));
1406 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1407 pasync_ctx->async_data.pa_base.u.
1408 a64.address);
1409 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1410 break;
1411 default:
1412 pbusy_list = NULL;
1413 shost_printk(KERN_WARNING, phba->shost,
1414 "Unexpected code=%d\n",
1415 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1416 code) / 32] & PDUCQE_CODE_MASK);
1417 return NULL;
1418 }
1419
1420 WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1421 WARN_ON(list_empty(pbusy_list));
1422 list_for_each_entry(pasync_handle, pbusy_list, link) {
1423 WARN_ON(pasync_handle->consumed);
1424 if (pasync_handle->index == buffer_index)
1425 break;
1426 }
1427
1428 WARN_ON(!pasync_handle);
1429
1430 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1431 phba->fw_config.iscsi_cid_start;
1432 pasync_handle->is_header = is_header;
1433 pasync_handle->buffer_len = ((pdpdu_cqe->
1434 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1435 & PDUCQE_DPL_MASK) >> 16);
1436
1437 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1438 index) / 32] & PDUCQE_INDEX_MASK);
1439 return pasync_handle;
1440 }
1441
1442 static unsigned int
1443 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1444 unsigned int is_header, unsigned int cq_index)
1445 {
1446 struct list_head *pbusy_list;
1447 struct async_pdu_handle *pasync_handle;
1448 unsigned int num_entries, writables = 0;
1449 unsigned int *pep_read_ptr, *pwritables;
1450
1451
1452 if (is_header) {
1453 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1454 pwritables = &pasync_ctx->async_header.writables;
1455 num_entries = pasync_ctx->async_header.num_entries;
1456 } else {
1457 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1458 pwritables = &pasync_ctx->async_data.writables;
1459 num_entries = pasync_ctx->async_data.num_entries;
1460 }
1461
1462 while ((*pep_read_ptr) != cq_index) {
1463 (*pep_read_ptr)++;
1464 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1465
1466 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1467 *pep_read_ptr);
1468 if (writables == 0)
1469 WARN_ON(list_empty(pbusy_list));
1470
1471 if (!list_empty(pbusy_list)) {
1472 pasync_handle = list_entry(pbusy_list->next,
1473 struct async_pdu_handle,
1474 link);
1475 WARN_ON(!pasync_handle);
1476 pasync_handle->consumed = 1;
1477 }
1478
1479 writables++;
1480 }
1481
1482 if (!writables) {
1483 SE_DEBUG(DBG_LVL_1,
1484 "Duplicate notification received - index 0x%x!!\n",
1485 cq_index);
1486 WARN_ON(1);
1487 }
1488
1489 *pwritables = *pwritables + writables;
1490 return 0;
1491 }
1492
1493 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1494 unsigned int cri)
1495 {
1496 struct hwi_controller *phwi_ctrlr;
1497 struct hwi_async_pdu_context *pasync_ctx;
1498 struct async_pdu_handle *pasync_handle, *tmp_handle;
1499 struct list_head *plist;
1500 unsigned int i = 0;
1501
1502 phwi_ctrlr = phba->phwi_ctrlr;
1503 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1504
1505 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1506
1507 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1508 list_del(&pasync_handle->link);
1509
1510 if (i == 0) {
1511 list_add_tail(&pasync_handle->link,
1512 &pasync_ctx->async_header.free_list);
1513 pasync_ctx->async_header.free_entries++;
1514 i++;
1515 } else {
1516 list_add_tail(&pasync_handle->link,
1517 &pasync_ctx->async_data.free_list);
1518 pasync_ctx->async_data.free_entries++;
1519 i++;
1520 }
1521 }
1522
1523 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1524 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1525 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1526 return 0;
1527 }
1528
1529 static struct phys_addr *
1530 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1531 unsigned int is_header, unsigned int host_write_ptr)
1532 {
1533 struct phys_addr *pasync_sge = NULL;
1534
1535 if (is_header)
1536 pasync_sge = pasync_ctx->async_header.ring_base;
1537 else
1538 pasync_sge = pasync_ctx->async_data.ring_base;
1539
1540 return pasync_sge + host_write_ptr;
1541 }
1542
1543 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1544 unsigned int is_header)
1545 {
1546 struct hwi_controller *phwi_ctrlr;
1547 struct hwi_async_pdu_context *pasync_ctx;
1548 struct async_pdu_handle *pasync_handle;
1549 struct list_head *pfree_link, *pbusy_list;
1550 struct phys_addr *pasync_sge;
1551 unsigned int ring_id, num_entries;
1552 unsigned int host_write_num;
1553 unsigned int writables;
1554 unsigned int i = 0;
1555 u32 doorbell = 0;
1556
1557 phwi_ctrlr = phba->phwi_ctrlr;
1558 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1559
1560 if (is_header) {
1561 num_entries = pasync_ctx->async_header.num_entries;
1562 writables = min(pasync_ctx->async_header.writables,
1563 pasync_ctx->async_header.free_entries);
1564 pfree_link = pasync_ctx->async_header.free_list.next;
1565 host_write_num = pasync_ctx->async_header.host_write_ptr;
1566 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1567 } else {
1568 num_entries = pasync_ctx->async_data.num_entries;
1569 writables = min(pasync_ctx->async_data.writables,
1570 pasync_ctx->async_data.free_entries);
1571 pfree_link = pasync_ctx->async_data.free_list.next;
1572 host_write_num = pasync_ctx->async_data.host_write_ptr;
1573 ring_id = phwi_ctrlr->default_pdu_data.id;
1574 }
1575
1576 writables = (writables / 8) * 8;
1577 if (writables) {
1578 for (i = 0; i < writables; i++) {
1579 pbusy_list =
1580 hwi_get_async_busy_list(pasync_ctx, is_header,
1581 host_write_num);
1582 pasync_handle =
1583 list_entry(pfree_link, struct async_pdu_handle,
1584 link);
1585 WARN_ON(!pasync_handle);
1586 pasync_handle->consumed = 0;
1587
1588 pfree_link = pfree_link->next;
1589
1590 pasync_sge = hwi_get_ring_address(pasync_ctx,
1591 is_header, host_write_num);
1592
1593 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1594 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1595
1596 list_move(&pasync_handle->link, pbusy_list);
1597
1598 host_write_num++;
1599 host_write_num = host_write_num % num_entries;
1600 }
1601
1602 if (is_header) {
1603 pasync_ctx->async_header.host_write_ptr =
1604 host_write_num;
1605 pasync_ctx->async_header.free_entries -= writables;
1606 pasync_ctx->async_header.writables -= writables;
1607 pasync_ctx->async_header.busy_entries += writables;
1608 } else {
1609 pasync_ctx->async_data.host_write_ptr = host_write_num;
1610 pasync_ctx->async_data.free_entries -= writables;
1611 pasync_ctx->async_data.writables -= writables;
1612 pasync_ctx->async_data.busy_entries += writables;
1613 }
1614
1615 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1616 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1617 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1618 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1619 << DB_DEF_PDU_CQPROC_SHIFT;
1620
1621 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1622 }
1623 }
1624
1625 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1626 struct beiscsi_conn *beiscsi_conn,
1627 struct i_t_dpdu_cqe *pdpdu_cqe)
1628 {
1629 struct hwi_controller *phwi_ctrlr;
1630 struct hwi_async_pdu_context *pasync_ctx;
1631 struct async_pdu_handle *pasync_handle = NULL;
1632 unsigned int cq_index = -1;
1633
1634 phwi_ctrlr = phba->phwi_ctrlr;
1635 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1636
1637 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1638 pdpdu_cqe, &cq_index);
1639 BUG_ON(pasync_handle->is_header != 0);
1640 if (pasync_handle->consumed == 0)
1641 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1642 cq_index);
1643
1644 hwi_free_async_msg(phba, pasync_handle->cri);
1645 hwi_post_async_buffers(phba, pasync_handle->is_header);
1646 }
1647
1648 static unsigned int
1649 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1650 struct beiscsi_hba *phba,
1651 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1652 {
1653 struct list_head *plist;
1654 struct async_pdu_handle *pasync_handle;
1655 void *phdr = NULL;
1656 unsigned int hdr_len = 0, buf_len = 0;
1657 unsigned int status, index = 0, offset = 0;
1658 void *pfirst_buffer = NULL;
1659 unsigned int num_buf = 0;
1660
1661 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1662
1663 list_for_each_entry(pasync_handle, plist, link) {
1664 if (index == 0) {
1665 phdr = pasync_handle->pbuffer;
1666 hdr_len = pasync_handle->buffer_len;
1667 } else {
1668 buf_len = pasync_handle->buffer_len;
1669 if (!num_buf) {
1670 pfirst_buffer = pasync_handle->pbuffer;
1671 num_buf++;
1672 }
1673 memcpy(pfirst_buffer + offset,
1674 pasync_handle->pbuffer, buf_len);
1675 offset = buf_len;
1676 }
1677 index++;
1678 }
1679
1680 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1681 (beiscsi_conn->beiscsi_conn_cid -
1682 phba->fw_config.iscsi_cid_start),
1683 phdr, hdr_len, pfirst_buffer,
1684 buf_len);
1685
1686 if (status == 0)
1687 hwi_free_async_msg(phba, cri);
1688 return 0;
1689 }
1690
1691 static unsigned int
1692 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1693 struct beiscsi_hba *phba,
1694 struct async_pdu_handle *pasync_handle)
1695 {
1696 struct hwi_async_pdu_context *pasync_ctx;
1697 struct hwi_controller *phwi_ctrlr;
1698 unsigned int bytes_needed = 0, status = 0;
1699 unsigned short cri = pasync_handle->cri;
1700 struct pdu_base *ppdu;
1701
1702 phwi_ctrlr = phba->phwi_ctrlr;
1703 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1704
1705 list_del(&pasync_handle->link);
1706 if (pasync_handle->is_header) {
1707 pasync_ctx->async_header.busy_entries--;
1708 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1709 hwi_free_async_msg(phba, cri);
1710 BUG();
1711 }
1712
1713 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1714 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1715 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1716 (unsigned short)pasync_handle->buffer_len;
1717 list_add_tail(&pasync_handle->link,
1718 &pasync_ctx->async_entry[cri].wait_queue.list);
1719
1720 ppdu = pasync_handle->pbuffer;
1721 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1722 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1723 0xFFFF0000) | ((be16_to_cpu((ppdu->
1724 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1725 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1726
1727 if (status == 0) {
1728 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1729 bytes_needed;
1730
1731 if (bytes_needed == 0)
1732 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1733 pasync_ctx, cri);
1734 }
1735 } else {
1736 pasync_ctx->async_data.busy_entries--;
1737 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1738 list_add_tail(&pasync_handle->link,
1739 &pasync_ctx->async_entry[cri].wait_queue.
1740 list);
1741 pasync_ctx->async_entry[cri].wait_queue.
1742 bytes_received +=
1743 (unsigned short)pasync_handle->buffer_len;
1744
1745 if (pasync_ctx->async_entry[cri].wait_queue.
1746 bytes_received >=
1747 pasync_ctx->async_entry[cri].wait_queue.
1748 bytes_needed)
1749 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1750 pasync_ctx, cri);
1751 }
1752 }
1753 return status;
1754 }
1755
1756 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1757 struct beiscsi_hba *phba,
1758 struct i_t_dpdu_cqe *pdpdu_cqe)
1759 {
1760 struct hwi_controller *phwi_ctrlr;
1761 struct hwi_async_pdu_context *pasync_ctx;
1762 struct async_pdu_handle *pasync_handle = NULL;
1763 unsigned int cq_index = -1;
1764
1765 phwi_ctrlr = phba->phwi_ctrlr;
1766 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1767 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1768 pdpdu_cqe, &cq_index);
1769
1770 if (pasync_handle->consumed == 0)
1771 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1772 cq_index);
1773 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1774 hwi_post_async_buffers(phba, pasync_handle->is_header);
1775 }
1776
1777 static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1778 {
1779 struct be_queue_info *mcc_cq;
1780 struct be_mcc_compl *mcc_compl;
1781 unsigned int num_processed = 0;
1782
1783 mcc_cq = &phba->ctrl.mcc_obj.cq;
1784 mcc_compl = queue_tail_node(mcc_cq);
1785 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1786 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1787
1788 if (num_processed >= 32) {
1789 hwi_ring_cq_db(phba, mcc_cq->id,
1790 num_processed, 0, 0);
1791 num_processed = 0;
1792 }
1793 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1794 /* Interpret flags as an async trailer */
1795 if (is_link_state_evt(mcc_compl->flags))
1796 /* Interpret compl as a async link evt */
1797 beiscsi_async_link_state_process(phba,
1798 (struct be_async_event_link_state *) mcc_compl);
1799 else
1800 SE_DEBUG(DBG_LVL_1,
1801 " Unsupported Async Event, flags"
1802 " = 0x%08x\n", mcc_compl->flags);
1803 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1804 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1805 atomic_dec(&phba->ctrl.mcc_obj.q.used);
1806 }
1807
1808 mcc_compl->flags = 0;
1809 queue_tail_inc(mcc_cq);
1810 mcc_compl = queue_tail_node(mcc_cq);
1811 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1812 num_processed++;
1813 }
1814
1815 if (num_processed > 0)
1816 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1817
1818 }
1819
1820 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1821 {
1822 struct be_queue_info *cq;
1823 struct sol_cqe *sol;
1824 struct dmsg_cqe *dmsg;
1825 unsigned int num_processed = 0;
1826 unsigned int tot_nump = 0;
1827 struct beiscsi_conn *beiscsi_conn;
1828 struct beiscsi_endpoint *beiscsi_ep;
1829 struct iscsi_endpoint *ep;
1830 struct beiscsi_hba *phba;
1831
1832 cq = pbe_eq->cq;
1833 sol = queue_tail_node(cq);
1834 phba = pbe_eq->phba;
1835
1836 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1837 CQE_VALID_MASK) {
1838 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1839
1840 ep = phba->ep_array[(u32) ((sol->
1841 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1842 SOL_CID_MASK) >> 6) -
1843 phba->fw_config.iscsi_cid_start];
1844
1845 beiscsi_ep = ep->dd_data;
1846 beiscsi_conn = beiscsi_ep->conn;
1847
1848 if (num_processed >= 32) {
1849 hwi_ring_cq_db(phba, cq->id,
1850 num_processed, 0, 0);
1851 tot_nump += num_processed;
1852 num_processed = 0;
1853 }
1854
1855 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1856 32] & CQE_CODE_MASK) {
1857 case SOL_CMD_COMPLETE:
1858 hwi_complete_cmd(beiscsi_conn, phba, sol);
1859 break;
1860 case DRIVERMSG_NOTIFY:
1861 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
1862 dmsg = (struct dmsg_cqe *)sol;
1863 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1864 break;
1865 case UNSOL_HDR_NOTIFY:
1866 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1867 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1868 (struct i_t_dpdu_cqe *)sol);
1869 break;
1870 case UNSOL_DATA_NOTIFY:
1871 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1872 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1873 (struct i_t_dpdu_cqe *)sol);
1874 break;
1875 case CXN_INVALIDATE_INDEX_NOTIFY:
1876 case CMD_INVALIDATED_NOTIFY:
1877 case CXN_INVALIDATE_NOTIFY:
1878 SE_DEBUG(DBG_LVL_1,
1879 "Ignoring CQ Error notification for cmd/cxn"
1880 "invalidate\n");
1881 break;
1882 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1883 case CMD_KILLED_INVALID_STATSN_RCVD:
1884 case CMD_KILLED_INVALID_R2T_RCVD:
1885 case CMD_CXN_KILLED_LUN_INVALID:
1886 case CMD_CXN_KILLED_ICD_INVALID:
1887 case CMD_CXN_KILLED_ITT_INVALID:
1888 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1889 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1890 SE_DEBUG(DBG_LVL_1,
1891 "CQ Error notification for cmd.. "
1892 "code %d cid 0x%x\n",
1893 sol->dw[offsetof(struct amap_sol_cqe, code) /
1894 32] & CQE_CODE_MASK,
1895 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1896 32] & SOL_CID_MASK));
1897 break;
1898 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1899 SE_DEBUG(DBG_LVL_1,
1900 "Digest error on def pdu ring, dropping..\n");
1901 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1902 (struct i_t_dpdu_cqe *) sol);
1903 break;
1904 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1905 case CXN_KILLED_BURST_LEN_MISMATCH:
1906 case CXN_KILLED_AHS_RCVD:
1907 case CXN_KILLED_HDR_DIGEST_ERR:
1908 case CXN_KILLED_UNKNOWN_HDR:
1909 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1910 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1911 case CXN_KILLED_TIMED_OUT:
1912 case CXN_KILLED_FIN_RCVD:
1913 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1914 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1915 case CXN_KILLED_OVER_RUN_RESIDUAL:
1916 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1917 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1918 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1919 "0x%x...\n",
1920 sol->dw[offsetof(struct amap_sol_cqe, code) /
1921 32] & CQE_CODE_MASK,
1922 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1923 32] & CQE_CID_MASK));
1924 iscsi_conn_failure(beiscsi_conn->conn,
1925 ISCSI_ERR_CONN_FAILED);
1926 break;
1927 case CXN_KILLED_RST_SENT:
1928 case CXN_KILLED_RST_RCVD:
1929 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1930 "received/sent on CID 0x%x...\n",
1931 sol->dw[offsetof(struct amap_sol_cqe, code) /
1932 32] & CQE_CODE_MASK,
1933 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1934 32] & CQE_CID_MASK));
1935 iscsi_conn_failure(beiscsi_conn->conn,
1936 ISCSI_ERR_CONN_FAILED);
1937 break;
1938 default:
1939 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1940 "received on CID 0x%x...\n",
1941 sol->dw[offsetof(struct amap_sol_cqe, code) /
1942 32] & CQE_CODE_MASK,
1943 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1944 32] & CQE_CID_MASK));
1945 break;
1946 }
1947
1948 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1949 queue_tail_inc(cq);
1950 sol = queue_tail_node(cq);
1951 num_processed++;
1952 }
1953
1954 if (num_processed > 0) {
1955 tot_nump += num_processed;
1956 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1957 }
1958 return tot_nump;
1959 }
1960
1961 void beiscsi_process_all_cqs(struct work_struct *work)
1962 {
1963 unsigned long flags;
1964 struct hwi_controller *phwi_ctrlr;
1965 struct hwi_context_memory *phwi_context;
1966 struct be_eq_obj *pbe_eq;
1967 struct beiscsi_hba *phba =
1968 container_of(work, struct beiscsi_hba, work_cqs);
1969
1970 phwi_ctrlr = phba->phwi_ctrlr;
1971 phwi_context = phwi_ctrlr->phwi_ctxt;
1972 if (phba->msix_enabled)
1973 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1974 else
1975 pbe_eq = &phwi_context->be_eq[0];
1976
1977 if (phba->todo_mcc_cq) {
1978 spin_lock_irqsave(&phba->isr_lock, flags);
1979 phba->todo_mcc_cq = 0;
1980 spin_unlock_irqrestore(&phba->isr_lock, flags);
1981 beiscsi_process_mcc_isr(phba);
1982 }
1983
1984 if (phba->todo_cq) {
1985 spin_lock_irqsave(&phba->isr_lock, flags);
1986 phba->todo_cq = 0;
1987 spin_unlock_irqrestore(&phba->isr_lock, flags);
1988 beiscsi_process_cq(pbe_eq);
1989 }
1990 }
1991
1992 static int be_iopoll(struct blk_iopoll *iop, int budget)
1993 {
1994 static unsigned int ret;
1995 struct beiscsi_hba *phba;
1996 struct be_eq_obj *pbe_eq;
1997
1998 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1999 ret = beiscsi_process_cq(pbe_eq);
2000 if (ret < budget) {
2001 phba = pbe_eq->phba;
2002 blk_iopoll_complete(iop);
2003 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
2004 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2005 }
2006 return ret;
2007 }
2008
2009 static void
2010 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2011 unsigned int num_sg, struct beiscsi_io_task *io_task)
2012 {
2013 struct iscsi_sge *psgl;
2014 unsigned int sg_len, index;
2015 unsigned int sge_len = 0;
2016 unsigned long long addr;
2017 struct scatterlist *l_sg;
2018 unsigned int offset;
2019
2020 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2021 io_task->bhs_pa.u.a32.address_lo);
2022 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2023 io_task->bhs_pa.u.a32.address_hi);
2024
2025 l_sg = sg;
2026 for (index = 0; (index < num_sg) && (index < 2); index++,
2027 sg = sg_next(sg)) {
2028 if (index == 0) {
2029 sg_len = sg_dma_len(sg);
2030 addr = (u64) sg_dma_address(sg);
2031 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2032 ((u32)(addr & 0xFFFFFFFF)));
2033 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2034 ((u32)(addr >> 32)));
2035 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2036 sg_len);
2037 sge_len = sg_len;
2038 } else {
2039 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2040 pwrb, sge_len);
2041 sg_len = sg_dma_len(sg);
2042 addr = (u64) sg_dma_address(sg);
2043 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2044 ((u32)(addr & 0xFFFFFFFF)));
2045 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2046 ((u32)(addr >> 32)));
2047 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2048 sg_len);
2049 }
2050 }
2051 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2052 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2053
2054 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2055
2056 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2057 io_task->bhs_pa.u.a32.address_hi);
2058 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2059 io_task->bhs_pa.u.a32.address_lo);
2060
2061 if (num_sg == 1) {
2062 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2063 1);
2064 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2065 0);
2066 } else if (num_sg == 2) {
2067 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2068 0);
2069 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2070 1);
2071 } else {
2072 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2073 0);
2074 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2075 0);
2076 }
2077 sg = l_sg;
2078 psgl++;
2079 psgl++;
2080 offset = 0;
2081 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2082 sg_len = sg_dma_len(sg);
2083 addr = (u64) sg_dma_address(sg);
2084 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2085 (addr & 0xFFFFFFFF));
2086 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2087 (addr >> 32));
2088 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2089 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2090 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2091 offset += sg_len;
2092 }
2093 psgl--;
2094 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2095 }
2096
2097 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2098 {
2099 struct iscsi_sge *psgl;
2100 unsigned long long addr;
2101 struct beiscsi_io_task *io_task = task->dd_data;
2102 struct beiscsi_conn *beiscsi_conn = io_task->conn;
2103 struct beiscsi_hba *phba = beiscsi_conn->phba;
2104
2105 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2106 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2107 io_task->bhs_pa.u.a32.address_lo);
2108 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2109 io_task->bhs_pa.u.a32.address_hi);
2110
2111 if (task->data) {
2112 if (task->data_count) {
2113 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2114 addr = (u64) pci_map_single(phba->pcidev,
2115 task->data,
2116 task->data_count, 1);
2117 } else {
2118 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2119 addr = 0;
2120 }
2121 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2122 ((u32)(addr & 0xFFFFFFFF)));
2123 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2124 ((u32)(addr >> 32)));
2125 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2126 task->data_count);
2127
2128 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2129 } else {
2130 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2131 addr = 0;
2132 }
2133
2134 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2135
2136 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2137
2138 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2139 io_task->bhs_pa.u.a32.address_hi);
2140 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2141 io_task->bhs_pa.u.a32.address_lo);
2142 if (task->data) {
2143 psgl++;
2144 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2145 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2146 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2147 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2148 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2149 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2150
2151 psgl++;
2152 if (task->data) {
2153 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2154 ((u32)(addr & 0xFFFFFFFF)));
2155 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2156 ((u32)(addr >> 32)));
2157 }
2158 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2159 }
2160 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2161 }
2162
2163 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2164 {
2165 unsigned int num_cq_pages, num_async_pdu_buf_pages;
2166 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2167 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2168
2169 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2170 sizeof(struct sol_cqe));
2171 num_async_pdu_buf_pages =
2172 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2173 phba->params.defpdu_hdr_sz);
2174 num_async_pdu_buf_sgl_pages =
2175 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2176 sizeof(struct phys_addr));
2177 num_async_pdu_data_pages =
2178 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2179 phba->params.defpdu_data_sz);
2180 num_async_pdu_data_sgl_pages =
2181 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2182 sizeof(struct phys_addr));
2183
2184 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2185
2186 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2187 BE_ISCSI_PDU_HEADER_SIZE;
2188 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2189 sizeof(struct hwi_context_memory);
2190
2191
2192 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2193 * (phba->params.wrbs_per_cxn)
2194 * phba->params.cxns_per_ctrl;
2195 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
2196 (phba->params.wrbs_per_cxn);
2197 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2198 phba->params.cxns_per_ctrl);
2199
2200 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2201 phba->params.icds_per_ctrl;
2202 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2203 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2204
2205 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2206 num_async_pdu_buf_pages * PAGE_SIZE;
2207 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2208 num_async_pdu_data_pages * PAGE_SIZE;
2209 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2210 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2211 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2212 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2213 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2214 phba->params.asyncpdus_per_ctrl *
2215 sizeof(struct async_pdu_handle);
2216 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2217 phba->params.asyncpdus_per_ctrl *
2218 sizeof(struct async_pdu_handle);
2219 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2220 sizeof(struct hwi_async_pdu_context) +
2221 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2222 }
2223
2224 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2225 {
2226 struct be_mem_descriptor *mem_descr;
2227 dma_addr_t bus_add;
2228 struct mem_array *mem_arr, *mem_arr_orig;
2229 unsigned int i, j, alloc_size, curr_alloc_size;
2230
2231 phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2232 if (!phba->phwi_ctrlr)
2233 return -ENOMEM;
2234
2235 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2236 GFP_KERNEL);
2237 if (!phba->init_mem) {
2238 kfree(phba->phwi_ctrlr);
2239 return -ENOMEM;
2240 }
2241
2242 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2243 GFP_KERNEL);
2244 if (!mem_arr_orig) {
2245 kfree(phba->init_mem);
2246 kfree(phba->phwi_ctrlr);
2247 return -ENOMEM;
2248 }
2249
2250 mem_descr = phba->init_mem;
2251 for (i = 0; i < SE_MEM_MAX; i++) {
2252 j = 0;
2253 mem_arr = mem_arr_orig;
2254 alloc_size = phba->mem_req[i];
2255 memset(mem_arr, 0, sizeof(struct mem_array) *
2256 BEISCSI_MAX_FRAGS_INIT);
2257 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2258 do {
2259 mem_arr->virtual_address = pci_alloc_consistent(
2260 phba->pcidev,
2261 curr_alloc_size,
2262 &bus_add);
2263 if (!mem_arr->virtual_address) {
2264 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2265 goto free_mem;
2266 if (curr_alloc_size -
2267 rounddown_pow_of_two(curr_alloc_size))
2268 curr_alloc_size = rounddown_pow_of_two
2269 (curr_alloc_size);
2270 else
2271 curr_alloc_size = curr_alloc_size / 2;
2272 } else {
2273 mem_arr->bus_address.u.
2274 a64.address = (__u64) bus_add;
2275 mem_arr->size = curr_alloc_size;
2276 alloc_size -= curr_alloc_size;
2277 curr_alloc_size = min(be_max_phys_size *
2278 1024, alloc_size);
2279 j++;
2280 mem_arr++;
2281 }
2282 } while (alloc_size);
2283 mem_descr->num_elements = j;
2284 mem_descr->size_in_bytes = phba->mem_req[i];
2285 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2286 GFP_KERNEL);
2287 if (!mem_descr->mem_array)
2288 goto free_mem;
2289
2290 memcpy(mem_descr->mem_array, mem_arr_orig,
2291 sizeof(struct mem_array) * j);
2292 mem_descr++;
2293 }
2294 kfree(mem_arr_orig);
2295 return 0;
2296 free_mem:
2297 mem_descr->num_elements = j;
2298 while ((i) || (j)) {
2299 for (j = mem_descr->num_elements; j > 0; j--) {
2300 pci_free_consistent(phba->pcidev,
2301 mem_descr->mem_array[j - 1].size,
2302 mem_descr->mem_array[j - 1].
2303 virtual_address,
2304 (unsigned long)mem_descr->
2305 mem_array[j - 1].
2306 bus_address.u.a64.address);
2307 }
2308 if (i) {
2309 i--;
2310 kfree(mem_descr->mem_array);
2311 mem_descr--;
2312 }
2313 }
2314 kfree(mem_arr_orig);
2315 kfree(phba->init_mem);
2316 kfree(phba->phwi_ctrlr);
2317 return -ENOMEM;
2318 }
2319
2320 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2321 {
2322 beiscsi_find_mem_req(phba);
2323 return beiscsi_alloc_mem(phba);
2324 }
2325
2326 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2327 {
2328 struct pdu_data_out *pdata_out;
2329 struct pdu_nop_out *pnop_out;
2330 struct be_mem_descriptor *mem_descr;
2331
2332 mem_descr = phba->init_mem;
2333 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2334 pdata_out =
2335 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2336 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2337
2338 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2339 IIOC_SCSI_DATA);
2340
2341 pnop_out =
2342 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2343 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2344
2345 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2346 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2347 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2348 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2349 }
2350
2351 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2352 {
2353 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2354 struct wrb_handle *pwrb_handle;
2355 struct hwi_controller *phwi_ctrlr;
2356 struct hwi_wrb_context *pwrb_context;
2357 struct iscsi_wrb *pwrb;
2358 unsigned int num_cxn_wrbh;
2359 unsigned int num_cxn_wrb, j, idx, index;
2360
2361 mem_descr_wrbh = phba->init_mem;
2362 mem_descr_wrbh += HWI_MEM_WRBH;
2363
2364 mem_descr_wrb = phba->init_mem;
2365 mem_descr_wrb += HWI_MEM_WRB;
2366
2367 idx = 0;
2368 pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2369 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2370 ((sizeof(struct wrb_handle)) *
2371 phba->params.wrbs_per_cxn));
2372 phwi_ctrlr = phba->phwi_ctrlr;
2373
2374 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2375 pwrb_context = &phwi_ctrlr->wrb_context[index];
2376 pwrb_context->pwrb_handle_base =
2377 kzalloc(sizeof(struct wrb_handle *) *
2378 phba->params.wrbs_per_cxn, GFP_KERNEL);
2379 pwrb_context->pwrb_handle_basestd =
2380 kzalloc(sizeof(struct wrb_handle *) *
2381 phba->params.wrbs_per_cxn, GFP_KERNEL);
2382 if (num_cxn_wrbh) {
2383 pwrb_context->alloc_index = 0;
2384 pwrb_context->wrb_handles_available = 0;
2385 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2386 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2387 pwrb_context->pwrb_handle_basestd[j] =
2388 pwrb_handle;
2389 pwrb_context->wrb_handles_available++;
2390 pwrb_handle->wrb_index = j;
2391 pwrb_handle++;
2392 }
2393 pwrb_context->free_index = 0;
2394 num_cxn_wrbh--;
2395 } else {
2396 idx++;
2397 pwrb_handle =
2398 mem_descr_wrbh->mem_array[idx].virtual_address;
2399 num_cxn_wrbh =
2400 ((mem_descr_wrbh->mem_array[idx].size) /
2401 ((sizeof(struct wrb_handle)) *
2402 phba->params.wrbs_per_cxn));
2403 pwrb_context->alloc_index = 0;
2404 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2405 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2406 pwrb_context->pwrb_handle_basestd[j] =
2407 pwrb_handle;
2408 pwrb_context->wrb_handles_available++;
2409 pwrb_handle->wrb_index = j;
2410 pwrb_handle++;
2411 }
2412 pwrb_context->free_index = 0;
2413 num_cxn_wrbh--;
2414 }
2415 }
2416 idx = 0;
2417 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2418 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2419 ((sizeof(struct iscsi_wrb) *
2420 phba->params.wrbs_per_cxn));
2421 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2422 pwrb_context = &phwi_ctrlr->wrb_context[index];
2423 if (num_cxn_wrb) {
2424 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2425 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2426 pwrb_handle->pwrb = pwrb;
2427 pwrb++;
2428 }
2429 num_cxn_wrb--;
2430 } else {
2431 idx++;
2432 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2433 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2434 ((sizeof(struct iscsi_wrb) *
2435 phba->params.wrbs_per_cxn));
2436 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2437 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2438 pwrb_handle->pwrb = pwrb;
2439 pwrb++;
2440 }
2441 num_cxn_wrb--;
2442 }
2443 }
2444 }
2445
2446 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2447 {
2448 struct hwi_controller *phwi_ctrlr;
2449 struct hba_parameters *p = &phba->params;
2450 struct hwi_async_pdu_context *pasync_ctx;
2451 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2452 unsigned int index;
2453 struct be_mem_descriptor *mem_descr;
2454
2455 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2456 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2457
2458 phwi_ctrlr = phba->phwi_ctrlr;
2459 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2460 mem_descr->mem_array[0].virtual_address;
2461 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2462 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2463
2464 pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2465 pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2466 pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2467 pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2468
2469 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2470 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2471 if (mem_descr->mem_array[0].virtual_address) {
2472 SE_DEBUG(DBG_LVL_8,
2473 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2474 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2475 } else
2476 shost_printk(KERN_WARNING, phba->shost,
2477 "No Virtual address\n");
2478
2479 pasync_ctx->async_header.va_base =
2480 mem_descr->mem_array[0].virtual_address;
2481
2482 pasync_ctx->async_header.pa_base.u.a64.address =
2483 mem_descr->mem_array[0].bus_address.u.a64.address;
2484
2485 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2486 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2487 if (mem_descr->mem_array[0].virtual_address) {
2488 SE_DEBUG(DBG_LVL_8,
2489 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2490 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2491 } else
2492 shost_printk(KERN_WARNING, phba->shost,
2493 "No Virtual address\n");
2494 pasync_ctx->async_header.ring_base =
2495 mem_descr->mem_array[0].virtual_address;
2496
2497 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2498 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2499 if (mem_descr->mem_array[0].virtual_address) {
2500 SE_DEBUG(DBG_LVL_8,
2501 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2502 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2503 } else
2504 shost_printk(KERN_WARNING, phba->shost,
2505 "No Virtual address\n");
2506
2507 pasync_ctx->async_header.handle_base =
2508 mem_descr->mem_array[0].virtual_address;
2509 pasync_ctx->async_header.writables = 0;
2510 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2511
2512 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2513 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2514 if (mem_descr->mem_array[0].virtual_address) {
2515 SE_DEBUG(DBG_LVL_8,
2516 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2517 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2518 } else
2519 shost_printk(KERN_WARNING, phba->shost,
2520 "No Virtual address\n");
2521 pasync_ctx->async_data.va_base =
2522 mem_descr->mem_array[0].virtual_address;
2523 pasync_ctx->async_data.pa_base.u.a64.address =
2524 mem_descr->mem_array[0].bus_address.u.a64.address;
2525
2526 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2527 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2528 if (mem_descr->mem_array[0].virtual_address) {
2529 SE_DEBUG(DBG_LVL_8,
2530 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2531 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2532 } else
2533 shost_printk(KERN_WARNING, phba->shost,
2534 "No Virtual address\n");
2535
2536 pasync_ctx->async_data.ring_base =
2537 mem_descr->mem_array[0].virtual_address;
2538
2539 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2540 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2541 if (!mem_descr->mem_array[0].virtual_address)
2542 shost_printk(KERN_WARNING, phba->shost,
2543 "No Virtual address\n");
2544
2545 pasync_ctx->async_data.handle_base =
2546 mem_descr->mem_array[0].virtual_address;
2547 pasync_ctx->async_data.writables = 0;
2548 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2549
2550 pasync_header_h =
2551 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2552 pasync_data_h =
2553 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2554
2555 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2556 pasync_header_h->cri = -1;
2557 pasync_header_h->index = (char)index;
2558 INIT_LIST_HEAD(&pasync_header_h->link);
2559 pasync_header_h->pbuffer =
2560 (void *)((unsigned long)
2561 (pasync_ctx->async_header.va_base) +
2562 (p->defpdu_hdr_sz * index));
2563
2564 pasync_header_h->pa.u.a64.address =
2565 pasync_ctx->async_header.pa_base.u.a64.address +
2566 (p->defpdu_hdr_sz * index);
2567
2568 list_add_tail(&pasync_header_h->link,
2569 &pasync_ctx->async_header.free_list);
2570 pasync_header_h++;
2571 pasync_ctx->async_header.free_entries++;
2572 pasync_ctx->async_header.writables++;
2573
2574 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2575 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2576 header_busy_list);
2577 pasync_data_h->cri = -1;
2578 pasync_data_h->index = (char)index;
2579 INIT_LIST_HEAD(&pasync_data_h->link);
2580 pasync_data_h->pbuffer =
2581 (void *)((unsigned long)
2582 (pasync_ctx->async_data.va_base) +
2583 (p->defpdu_data_sz * index));
2584
2585 pasync_data_h->pa.u.a64.address =
2586 pasync_ctx->async_data.pa_base.u.a64.address +
2587 (p->defpdu_data_sz * index);
2588
2589 list_add_tail(&pasync_data_h->link,
2590 &pasync_ctx->async_data.free_list);
2591 pasync_data_h++;
2592 pasync_ctx->async_data.free_entries++;
2593 pasync_ctx->async_data.writables++;
2594
2595 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2596 }
2597
2598 pasync_ctx->async_header.host_write_ptr = 0;
2599 pasync_ctx->async_header.ep_read_ptr = -1;
2600 pasync_ctx->async_data.host_write_ptr = 0;
2601 pasync_ctx->async_data.ep_read_ptr = -1;
2602 }
2603
2604 static int
2605 be_sgl_create_contiguous(void *virtual_address,
2606 u64 physical_address, u32 length,
2607 struct be_dma_mem *sgl)
2608 {
2609 WARN_ON(!virtual_address);
2610 WARN_ON(!physical_address);
2611 WARN_ON(!length > 0);
2612 WARN_ON(!sgl);
2613
2614 sgl->va = virtual_address;
2615 sgl->dma = (unsigned long)physical_address;
2616 sgl->size = length;
2617
2618 return 0;
2619 }
2620
2621 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2622 {
2623 memset(sgl, 0, sizeof(*sgl));
2624 }
2625
2626 static void
2627 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2628 struct mem_array *pmem, struct be_dma_mem *sgl)
2629 {
2630 if (sgl->va)
2631 be_sgl_destroy_contiguous(sgl);
2632
2633 be_sgl_create_contiguous(pmem->virtual_address,
2634 pmem->bus_address.u.a64.address,
2635 pmem->size, sgl);
2636 }
2637
2638 static void
2639 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2640 struct mem_array *pmem, struct be_dma_mem *sgl)
2641 {
2642 if (sgl->va)
2643 be_sgl_destroy_contiguous(sgl);
2644
2645 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2646 pmem->bus_address.u.a64.address,
2647 pmem->size, sgl);
2648 }
2649
2650 static int be_fill_queue(struct be_queue_info *q,
2651 u16 len, u16 entry_size, void *vaddress)
2652 {
2653 struct be_dma_mem *mem = &q->dma_mem;
2654
2655 memset(q, 0, sizeof(*q));
2656 q->len = len;
2657 q->entry_size = entry_size;
2658 mem->size = len * entry_size;
2659 mem->va = vaddress;
2660 if (!mem->va)
2661 return -ENOMEM;
2662 memset(mem->va, 0, mem->size);
2663 return 0;
2664 }
2665
2666 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2667 struct hwi_context_memory *phwi_context)
2668 {
2669 unsigned int i, num_eq_pages;
2670 int ret, eq_for_mcc;
2671 struct be_queue_info *eq;
2672 struct be_dma_mem *mem;
2673 void *eq_vaddress;
2674 dma_addr_t paddr;
2675
2676 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2677 sizeof(struct be_eq_entry));
2678
2679 if (phba->msix_enabled)
2680 eq_for_mcc = 1;
2681 else
2682 eq_for_mcc = 0;
2683 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2684 eq = &phwi_context->be_eq[i].q;
2685 mem = &eq->dma_mem;
2686 phwi_context->be_eq[i].phba = phba;
2687 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2688 num_eq_pages * PAGE_SIZE,
2689 &paddr);
2690 if (!eq_vaddress)
2691 goto create_eq_error;
2692
2693 mem->va = eq_vaddress;
2694 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2695 sizeof(struct be_eq_entry), eq_vaddress);
2696 if (ret) {
2697 shost_printk(KERN_ERR, phba->shost,
2698 "be_fill_queue Failed for EQ\n");
2699 goto create_eq_error;
2700 }
2701
2702 mem->dma = paddr;
2703 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2704 phwi_context->cur_eqd);
2705 if (ret) {
2706 shost_printk(KERN_ERR, phba->shost,
2707 "beiscsi_cmd_eq_create"
2708 "Failedfor EQ\n");
2709 goto create_eq_error;
2710 }
2711 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2712 }
2713 return 0;
2714 create_eq_error:
2715 for (i = 0; i < (phba->num_cpus + 1); i++) {
2716 eq = &phwi_context->be_eq[i].q;
2717 mem = &eq->dma_mem;
2718 if (mem->va)
2719 pci_free_consistent(phba->pcidev, num_eq_pages
2720 * PAGE_SIZE,
2721 mem->va, mem->dma);
2722 }
2723 return ret;
2724 }
2725
2726 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2727 struct hwi_context_memory *phwi_context)
2728 {
2729 unsigned int i, num_cq_pages;
2730 int ret;
2731 struct be_queue_info *cq, *eq;
2732 struct be_dma_mem *mem;
2733 struct be_eq_obj *pbe_eq;
2734 void *cq_vaddress;
2735 dma_addr_t paddr;
2736
2737 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2738 sizeof(struct sol_cqe));
2739
2740 for (i = 0; i < phba->num_cpus; i++) {
2741 cq = &phwi_context->be_cq[i];
2742 eq = &phwi_context->be_eq[i].q;
2743 pbe_eq = &phwi_context->be_eq[i];
2744 pbe_eq->cq = cq;
2745 pbe_eq->phba = phba;
2746 mem = &cq->dma_mem;
2747 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2748 num_cq_pages * PAGE_SIZE,
2749 &paddr);
2750 if (!cq_vaddress)
2751 goto create_cq_error;
2752 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2753 sizeof(struct sol_cqe), cq_vaddress);
2754 if (ret) {
2755 shost_printk(KERN_ERR, phba->shost,
2756 "be_fill_queue Failed for ISCSI CQ\n");
2757 goto create_cq_error;
2758 }
2759
2760 mem->dma = paddr;
2761 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2762 false, 0);
2763 if (ret) {
2764 shost_printk(KERN_ERR, phba->shost,
2765 "beiscsi_cmd_eq_create"
2766 "Failed for ISCSI CQ\n");
2767 goto create_cq_error;
2768 }
2769 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2770 cq->id, eq->id);
2771 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2772 }
2773 return 0;
2774
2775 create_cq_error:
2776 for (i = 0; i < phba->num_cpus; i++) {
2777 cq = &phwi_context->be_cq[i];
2778 mem = &cq->dma_mem;
2779 if (mem->va)
2780 pci_free_consistent(phba->pcidev, num_cq_pages
2781 * PAGE_SIZE,
2782 mem->va, mem->dma);
2783 }
2784 return ret;
2785
2786 }
2787
2788 static int
2789 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2790 struct hwi_context_memory *phwi_context,
2791 struct hwi_controller *phwi_ctrlr,
2792 unsigned int def_pdu_ring_sz)
2793 {
2794 unsigned int idx;
2795 int ret;
2796 struct be_queue_info *dq, *cq;
2797 struct be_dma_mem *mem;
2798 struct be_mem_descriptor *mem_descr;
2799 void *dq_vaddress;
2800
2801 idx = 0;
2802 dq = &phwi_context->be_def_hdrq;
2803 cq = &phwi_context->be_cq[0];
2804 mem = &dq->dma_mem;
2805 mem_descr = phba->init_mem;
2806 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2807 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2808 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2809 sizeof(struct phys_addr),
2810 sizeof(struct phys_addr), dq_vaddress);
2811 if (ret) {
2812 shost_printk(KERN_ERR, phba->shost,
2813 "be_fill_queue Failed for DEF PDU HDR\n");
2814 return ret;
2815 }
2816 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2817 bus_address.u.a64.address;
2818 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2819 def_pdu_ring_sz,
2820 phba->params.defpdu_hdr_sz);
2821 if (ret) {
2822 shost_printk(KERN_ERR, phba->shost,
2823 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2824 return ret;
2825 }
2826 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2827 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2828 phwi_context->be_def_hdrq.id);
2829 hwi_post_async_buffers(phba, 1);
2830 return 0;
2831 }
2832
2833 static int
2834 beiscsi_create_def_data(struct beiscsi_hba *phba,
2835 struct hwi_context_memory *phwi_context,
2836 struct hwi_controller *phwi_ctrlr,
2837 unsigned int def_pdu_ring_sz)
2838 {
2839 unsigned int idx;
2840 int ret;
2841 struct be_queue_info *dataq, *cq;
2842 struct be_dma_mem *mem;
2843 struct be_mem_descriptor *mem_descr;
2844 void *dq_vaddress;
2845
2846 idx = 0;
2847 dataq = &phwi_context->be_def_dataq;
2848 cq = &phwi_context->be_cq[0];
2849 mem = &dataq->dma_mem;
2850 mem_descr = phba->init_mem;
2851 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2852 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2853 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2854 sizeof(struct phys_addr),
2855 sizeof(struct phys_addr), dq_vaddress);
2856 if (ret) {
2857 shost_printk(KERN_ERR, phba->shost,
2858 "be_fill_queue Failed for DEF PDU DATA\n");
2859 return ret;
2860 }
2861 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2862 bus_address.u.a64.address;
2863 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2864 def_pdu_ring_sz,
2865 phba->params.defpdu_data_sz);
2866 if (ret) {
2867 shost_printk(KERN_ERR, phba->shost,
2868 "be_cmd_create_default_pdu_queue Failed"
2869 " for DEF PDU DATA\n");
2870 return ret;
2871 }
2872 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2873 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2874 phwi_context->be_def_dataq.id);
2875 hwi_post_async_buffers(phba, 0);
2876 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
2877 return 0;
2878 }
2879
2880 static int
2881 beiscsi_post_pages(struct beiscsi_hba *phba)
2882 {
2883 struct be_mem_descriptor *mem_descr;
2884 struct mem_array *pm_arr;
2885 unsigned int page_offset, i;
2886 struct be_dma_mem sgl;
2887 int status;
2888
2889 mem_descr = phba->init_mem;
2890 mem_descr += HWI_MEM_SGE;
2891 pm_arr = mem_descr->mem_array;
2892
2893 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2894 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2895 for (i = 0; i < mem_descr->num_elements; i++) {
2896 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2897 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2898 page_offset,
2899 (pm_arr->size / PAGE_SIZE));
2900 page_offset += pm_arr->size / PAGE_SIZE;
2901 if (status != 0) {
2902 shost_printk(KERN_ERR, phba->shost,
2903 "post sgl failed.\n");
2904 return status;
2905 }
2906 pm_arr++;
2907 }
2908 SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
2909 return 0;
2910 }
2911
2912 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2913 {
2914 struct be_dma_mem *mem = &q->dma_mem;
2915 if (mem->va)
2916 pci_free_consistent(phba->pcidev, mem->size,
2917 mem->va, mem->dma);
2918 }
2919
2920 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2921 u16 len, u16 entry_size)
2922 {
2923 struct be_dma_mem *mem = &q->dma_mem;
2924
2925 memset(q, 0, sizeof(*q));
2926 q->len = len;
2927 q->entry_size = entry_size;
2928 mem->size = len * entry_size;
2929 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2930 if (!mem->va)
2931 return -ENOMEM;
2932 memset(mem->va, 0, mem->size);
2933 return 0;
2934 }
2935
2936 static int
2937 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2938 struct hwi_context_memory *phwi_context,
2939 struct hwi_controller *phwi_ctrlr)
2940 {
2941 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2942 u64 pa_addr_lo;
2943 unsigned int idx, num, i;
2944 struct mem_array *pwrb_arr;
2945 void *wrb_vaddr;
2946 struct be_dma_mem sgl;
2947 struct be_mem_descriptor *mem_descr;
2948 int status;
2949
2950 idx = 0;
2951 mem_descr = phba->init_mem;
2952 mem_descr += HWI_MEM_WRB;
2953 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2954 GFP_KERNEL);
2955 if (!pwrb_arr) {
2956 shost_printk(KERN_ERR, phba->shost,
2957 "Memory alloc failed in create wrb ring.\n");
2958 return -ENOMEM;
2959 }
2960 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2961 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2962 num_wrb_rings = mem_descr->mem_array[idx].size /
2963 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2964
2965 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2966 if (num_wrb_rings) {
2967 pwrb_arr[num].virtual_address = wrb_vaddr;
2968 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2969 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2970 sizeof(struct iscsi_wrb);
2971 wrb_vaddr += pwrb_arr[num].size;
2972 pa_addr_lo += pwrb_arr[num].size;
2973 num_wrb_rings--;
2974 } else {
2975 idx++;
2976 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2977 pa_addr_lo = mem_descr->mem_array[idx].\
2978 bus_address.u.a64.address;
2979 num_wrb_rings = mem_descr->mem_array[idx].size /
2980 (phba->params.wrbs_per_cxn *
2981 sizeof(struct iscsi_wrb));
2982 pwrb_arr[num].virtual_address = wrb_vaddr;
2983 pwrb_arr[num].bus_address.u.a64.address\
2984 = pa_addr_lo;
2985 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2986 sizeof(struct iscsi_wrb);
2987 wrb_vaddr += pwrb_arr[num].size;
2988 pa_addr_lo += pwrb_arr[num].size;
2989 num_wrb_rings--;
2990 }
2991 }
2992 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2993 wrb_mem_index = 0;
2994 offset = 0;
2995 size = 0;
2996
2997 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2998 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2999 &phwi_context->be_wrbq[i]);
3000 if (status != 0) {
3001 shost_printk(KERN_ERR, phba->shost,
3002 "wrbq create failed.");
3003 kfree(pwrb_arr);
3004 return status;
3005 }
3006 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3007 id;
3008 }
3009 kfree(pwrb_arr);
3010 return 0;
3011 }
3012
3013 static void free_wrb_handles(struct beiscsi_hba *phba)
3014 {
3015 unsigned int index;
3016 struct hwi_controller *phwi_ctrlr;
3017 struct hwi_wrb_context *pwrb_context;
3018
3019 phwi_ctrlr = phba->phwi_ctrlr;
3020 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3021 pwrb_context = &phwi_ctrlr->wrb_context[index];
3022 kfree(pwrb_context->pwrb_handle_base);
3023 kfree(pwrb_context->pwrb_handle_basestd);
3024 }
3025 }
3026
3027 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3028 {
3029 struct be_queue_info *q;
3030 struct be_ctrl_info *ctrl = &phba->ctrl;
3031
3032 q = &phba->ctrl.mcc_obj.q;
3033 if (q->created)
3034 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3035 be_queue_free(phba, q);
3036
3037 q = &phba->ctrl.mcc_obj.cq;
3038 if (q->created)
3039 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3040 be_queue_free(phba, q);
3041 }
3042
3043 static void hwi_cleanup(struct beiscsi_hba *phba)
3044 {
3045 struct be_queue_info *q;
3046 struct be_ctrl_info *ctrl = &phba->ctrl;
3047 struct hwi_controller *phwi_ctrlr;
3048 struct hwi_context_memory *phwi_context;
3049 int i, eq_num;
3050
3051 phwi_ctrlr = phba->phwi_ctrlr;
3052 phwi_context = phwi_ctrlr->phwi_ctxt;
3053 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3054 q = &phwi_context->be_wrbq[i];
3055 if (q->created)
3056 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3057 }
3058 free_wrb_handles(phba);
3059
3060 q = &phwi_context->be_def_hdrq;
3061 if (q->created)
3062 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3063
3064 q = &phwi_context->be_def_dataq;
3065 if (q->created)
3066 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3067
3068 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3069
3070 for (i = 0; i < (phba->num_cpus); i++) {
3071 q = &phwi_context->be_cq[i];
3072 if (q->created)
3073 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3074 }
3075 if (phba->msix_enabled)
3076 eq_num = 1;
3077 else
3078 eq_num = 0;
3079 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3080 q = &phwi_context->be_eq[i].q;
3081 if (q->created)
3082 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3083 }
3084 be_mcc_queues_destroy(phba);
3085 }
3086
3087 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3088 struct hwi_context_memory *phwi_context)
3089 {
3090 struct be_queue_info *q, *cq;
3091 struct be_ctrl_info *ctrl = &phba->ctrl;
3092
3093 /* Alloc MCC compl queue */
3094 cq = &phba->ctrl.mcc_obj.cq;
3095 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3096 sizeof(struct be_mcc_compl)))
3097 goto err;
3098 /* Ask BE to create MCC compl queue; */
3099 if (phba->msix_enabled) {
3100 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3101 [phba->num_cpus].q, false, true, 0))
3102 goto mcc_cq_free;
3103 } else {
3104 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3105 false, true, 0))
3106 goto mcc_cq_free;
3107 }
3108
3109 /* Alloc MCC queue */
3110 q = &phba->ctrl.mcc_obj.q;
3111 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3112 goto mcc_cq_destroy;
3113
3114 /* Ask BE to create MCC queue */
3115 if (beiscsi_cmd_mccq_create(phba, q, cq))
3116 goto mcc_q_free;
3117
3118 return 0;
3119
3120 mcc_q_free:
3121 be_queue_free(phba, q);
3122 mcc_cq_destroy:
3123 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3124 mcc_cq_free:
3125 be_queue_free(phba, cq);
3126 err:
3127 return -ENOMEM;
3128 }
3129
3130 static int find_num_cpus(void)
3131 {
3132 int num_cpus = 0;
3133
3134 num_cpus = num_online_cpus();
3135 if (num_cpus >= MAX_CPUS)
3136 num_cpus = MAX_CPUS - 1;
3137
3138 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
3139 return num_cpus;
3140 }
3141
3142 static int hwi_init_port(struct beiscsi_hba *phba)
3143 {
3144 struct hwi_controller *phwi_ctrlr;
3145 struct hwi_context_memory *phwi_context;
3146 unsigned int def_pdu_ring_sz;
3147 struct be_ctrl_info *ctrl = &phba->ctrl;
3148 int status;
3149
3150 def_pdu_ring_sz =
3151 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3152 phwi_ctrlr = phba->phwi_ctrlr;
3153 phwi_context = phwi_ctrlr->phwi_ctxt;
3154 phwi_context->max_eqd = 0;
3155 phwi_context->min_eqd = 0;
3156 phwi_context->cur_eqd = 64;
3157 be_cmd_fw_initialize(&phba->ctrl);
3158
3159 status = beiscsi_create_eqs(phba, phwi_context);
3160 if (status != 0) {
3161 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
3162 goto error;
3163 }
3164
3165 status = be_mcc_queues_create(phba, phwi_context);
3166 if (status != 0)
3167 goto error;
3168
3169 status = mgmt_check_supported_fw(ctrl, phba);
3170 if (status != 0) {
3171 shost_printk(KERN_ERR, phba->shost,
3172 "Unsupported fw version\n");
3173 goto error;
3174 }
3175
3176 status = beiscsi_create_cqs(phba, phwi_context);
3177 if (status != 0) {
3178 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3179 goto error;
3180 }
3181
3182 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3183 def_pdu_ring_sz);
3184 if (status != 0) {
3185 shost_printk(KERN_ERR, phba->shost,
3186 "Default Header not created\n");
3187 goto error;
3188 }
3189
3190 status = beiscsi_create_def_data(phba, phwi_context,
3191 phwi_ctrlr, def_pdu_ring_sz);
3192 if (status != 0) {
3193 shost_printk(KERN_ERR, phba->shost,
3194 "Default Data not created\n");
3195 goto error;
3196 }
3197
3198 status = beiscsi_post_pages(phba);
3199 if (status != 0) {
3200 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3201 goto error;
3202 }
3203
3204 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3205 if (status != 0) {
3206 shost_printk(KERN_ERR, phba->shost,
3207 "WRB Rings not created\n");
3208 goto error;
3209 }
3210
3211 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3212 return 0;
3213
3214 error:
3215 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3216 hwi_cleanup(phba);
3217 return -ENOMEM;
3218 }
3219
3220 static int hwi_init_controller(struct beiscsi_hba *phba)
3221 {
3222 struct hwi_controller *phwi_ctrlr;
3223
3224 phwi_ctrlr = phba->phwi_ctrlr;
3225 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3226 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3227 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3228 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
3229 phwi_ctrlr->phwi_ctxt);
3230 } else {
3231 shost_printk(KERN_ERR, phba->shost,
3232 "HWI_MEM_ADDN_CONTEXT is more than one element."
3233 "Failing to load\n");
3234 return -ENOMEM;
3235 }
3236
3237 iscsi_init_global_templates(phba);
3238 beiscsi_init_wrb_handle(phba);
3239 hwi_init_async_pdu_ctx(phba);
3240 if (hwi_init_port(phba) != 0) {
3241 shost_printk(KERN_ERR, phba->shost,
3242 "hwi_init_controller failed\n");
3243 return -ENOMEM;
3244 }
3245 return 0;
3246 }
3247
3248 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3249 {
3250 struct be_mem_descriptor *mem_descr;
3251 int i, j;
3252
3253 mem_descr = phba->init_mem;
3254 i = 0;
3255 j = 0;
3256 for (i = 0; i < SE_MEM_MAX; i++) {
3257 for (j = mem_descr->num_elements; j > 0; j--) {
3258 pci_free_consistent(phba->pcidev,
3259 mem_descr->mem_array[j - 1].size,
3260 mem_descr->mem_array[j - 1].virtual_address,
3261 (unsigned long)mem_descr->mem_array[j - 1].
3262 bus_address.u.a64.address);
3263 }
3264 kfree(mem_descr->mem_array);
3265 mem_descr++;
3266 }
3267 kfree(phba->init_mem);
3268 kfree(phba->phwi_ctrlr);
3269 }
3270
3271 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3272 {
3273 int ret = -ENOMEM;
3274
3275 ret = beiscsi_get_memory(phba);
3276 if (ret < 0) {
3277 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3278 "Failed in beiscsi_alloc_memory\n");
3279 return ret;
3280 }
3281
3282 ret = hwi_init_controller(phba);
3283 if (ret)
3284 goto free_init;
3285 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3286 return 0;
3287
3288 free_init:
3289 beiscsi_free_mem(phba);
3290 return -ENOMEM;
3291 }
3292
3293 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3294 {
3295 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3296 struct sgl_handle *psgl_handle;
3297 struct iscsi_sge *pfrag;
3298 unsigned int arr_index, i, idx;
3299
3300 phba->io_sgl_hndl_avbl = 0;
3301 phba->eh_sgl_hndl_avbl = 0;
3302
3303 mem_descr_sglh = phba->init_mem;
3304 mem_descr_sglh += HWI_MEM_SGLH;
3305 if (1 == mem_descr_sglh->num_elements) {
3306 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3307 phba->params.ios_per_ctrl,
3308 GFP_KERNEL);
3309 if (!phba->io_sgl_hndl_base) {
3310 shost_printk(KERN_ERR, phba->shost,
3311 "Mem Alloc Failed. Failing to load\n");
3312 return -ENOMEM;
3313 }
3314 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3315 (phba->params.icds_per_ctrl -
3316 phba->params.ios_per_ctrl),
3317 GFP_KERNEL);
3318 if (!phba->eh_sgl_hndl_base) {
3319 kfree(phba->io_sgl_hndl_base);
3320 shost_printk(KERN_ERR, phba->shost,
3321 "Mem Alloc Failed. Failing to load\n");
3322 return -ENOMEM;
3323 }
3324 } else {
3325 shost_printk(KERN_ERR, phba->shost,
3326 "HWI_MEM_SGLH is more than one element."
3327 "Failing to load\n");
3328 return -ENOMEM;
3329 }
3330
3331 arr_index = 0;
3332 idx = 0;
3333 while (idx < mem_descr_sglh->num_elements) {
3334 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3335
3336 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3337 sizeof(struct sgl_handle)); i++) {
3338 if (arr_index < phba->params.ios_per_ctrl) {
3339 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3340 phba->io_sgl_hndl_avbl++;
3341 arr_index++;
3342 } else {
3343 phba->eh_sgl_hndl_base[arr_index -
3344 phba->params.ios_per_ctrl] =
3345 psgl_handle;
3346 arr_index++;
3347 phba->eh_sgl_hndl_avbl++;
3348 }
3349 psgl_handle++;
3350 }
3351 idx++;
3352 }
3353 SE_DEBUG(DBG_LVL_8,
3354 "phba->io_sgl_hndl_avbl=%d"
3355 "phba->eh_sgl_hndl_avbl=%d\n",
3356 phba->io_sgl_hndl_avbl,
3357 phba->eh_sgl_hndl_avbl);
3358 mem_descr_sg = phba->init_mem;
3359 mem_descr_sg += HWI_MEM_SGE;
3360 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
3361 mem_descr_sg->num_elements);
3362 arr_index = 0;
3363 idx = 0;
3364 while (idx < mem_descr_sg->num_elements) {
3365 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3366
3367 for (i = 0;
3368 i < (mem_descr_sg->mem_array[idx].size) /
3369 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3370 i++) {
3371 if (arr_index < phba->params.ios_per_ctrl)
3372 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3373 else
3374 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3375 phba->params.ios_per_ctrl];
3376 psgl_handle->pfrag = pfrag;
3377 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3378 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3379 pfrag += phba->params.num_sge_per_io;
3380 psgl_handle->sgl_index =
3381 phba->fw_config.iscsi_icd_start + arr_index++;
3382 }
3383 idx++;
3384 }
3385 phba->io_sgl_free_index = 0;
3386 phba->io_sgl_alloc_index = 0;
3387 phba->eh_sgl_free_index = 0;
3388 phba->eh_sgl_alloc_index = 0;
3389 return 0;
3390 }
3391
3392 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3393 {
3394 int i, new_cid;
3395
3396 phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3397 GFP_KERNEL);
3398 if (!phba->cid_array) {
3399 shost_printk(KERN_ERR, phba->shost,
3400 "Failed to allocate memory in "
3401 "hba_setup_cid_tbls\n");
3402 return -ENOMEM;
3403 }
3404 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3405 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3406 if (!phba->ep_array) {
3407 shost_printk(KERN_ERR, phba->shost,
3408 "Failed to allocate memory in "
3409 "hba_setup_cid_tbls\n");
3410 kfree(phba->cid_array);
3411 return -ENOMEM;
3412 }
3413 new_cid = phba->fw_config.iscsi_cid_start;
3414 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3415 phba->cid_array[i] = new_cid;
3416 new_cid += 2;
3417 }
3418 phba->avlbl_cids = phba->params.cxns_per_ctrl;
3419 return 0;
3420 }
3421
3422 static void hwi_enable_intr(struct beiscsi_hba *phba)
3423 {
3424 struct be_ctrl_info *ctrl = &phba->ctrl;
3425 struct hwi_controller *phwi_ctrlr;
3426 struct hwi_context_memory *phwi_context;
3427 struct be_queue_info *eq;
3428 u8 __iomem *addr;
3429 u32 reg, i;
3430 u32 enabled;
3431
3432 phwi_ctrlr = phba->phwi_ctrlr;
3433 phwi_context = phwi_ctrlr->phwi_ctxt;
3434
3435 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3436 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3437 reg = ioread32(addr);
3438
3439 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3440 if (!enabled) {
3441 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3442 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
3443 iowrite32(reg, addr);
3444 }
3445
3446 if (!phba->msix_enabled) {
3447 eq = &phwi_context->be_eq[0].q;
3448 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3449 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3450 } else {
3451 for (i = 0; i <= phba->num_cpus; i++) {
3452 eq = &phwi_context->be_eq[i].q;
3453 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3454 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3455 }
3456 }
3457 }
3458
3459 static void hwi_disable_intr(struct beiscsi_hba *phba)
3460 {
3461 struct be_ctrl_info *ctrl = &phba->ctrl;
3462
3463 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3464 u32 reg = ioread32(addr);
3465
3466 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3467 if (enabled) {
3468 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3469 iowrite32(reg, addr);
3470 } else
3471 shost_printk(KERN_WARNING, phba->shost,
3472 "In hwi_disable_intr, Already Disabled\n");
3473 }
3474
3475 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3476 {
3477 struct be_cmd_resp_get_boot_target *boot_resp;
3478 struct be_cmd_resp_get_session *session_resp;
3479 struct be_mcc_wrb *wrb;
3480 struct be_dma_mem nonemb_cmd;
3481 unsigned int tag, wrb_num;
3482 unsigned short status, extd_status;
3483 struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3484 int ret = -ENOMEM;
3485
3486 tag = beiscsi_get_boot_target(phba);
3487 if (!tag) {
3488 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
3489 return -EAGAIN;
3490 } else
3491 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3492 phba->ctrl.mcc_numtag[tag]);
3493
3494 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3495 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3496 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3497 if (status || extd_status) {
3498 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
3499 " status = %d extd_status = %d\n",
3500 status, extd_status);
3501 free_mcc_tag(&phba->ctrl, tag);
3502 return -EBUSY;
3503 }
3504 wrb = queue_get_wrb(mccq, wrb_num);
3505 free_mcc_tag(&phba->ctrl, tag);
3506 boot_resp = embedded_payload(wrb);
3507
3508 if (boot_resp->boot_session_handle < 0) {
3509 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
3510 return -ENXIO;
3511 }
3512
3513 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3514 sizeof(*session_resp),
3515 &nonemb_cmd.dma);
3516 if (nonemb_cmd.va == NULL) {
3517 SE_DEBUG(DBG_LVL_1,
3518 "Failed to allocate memory for"
3519 "beiscsi_get_session_info\n");
3520 return -ENOMEM;
3521 }
3522
3523 memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3524 tag = beiscsi_get_session_info(phba,
3525 boot_resp->boot_session_handle, &nonemb_cmd);
3526 if (!tag) {
3527 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3528 " Failed\n");
3529 goto boot_freemem;
3530 } else
3531 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3532 phba->ctrl.mcc_numtag[tag]);
3533
3534 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3535 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3536 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3537 if (status || extd_status) {
3538 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3539 " status = %d extd_status = %d\n",
3540 status, extd_status);
3541 free_mcc_tag(&phba->ctrl, tag);
3542 goto boot_freemem;
3543 }
3544 wrb = queue_get_wrb(mccq, wrb_num);
3545 free_mcc_tag(&phba->ctrl, tag);
3546 session_resp = nonemb_cmd.va ;
3547
3548 memcpy(&phba->boot_sess, &session_resp->session_info,
3549 sizeof(struct mgmt_session_info));
3550 ret = 0;
3551
3552 boot_freemem:
3553 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3554 nonemb_cmd.va, nonemb_cmd.dma);
3555 return ret;
3556 }
3557
3558 static void beiscsi_boot_release(void *data)
3559 {
3560 struct beiscsi_hba *phba = data;
3561
3562 scsi_host_put(phba->shost);
3563 }
3564
3565 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3566 {
3567 struct iscsi_boot_kobj *boot_kobj;
3568
3569 /* get boot info using mgmt cmd */
3570 if (beiscsi_get_boot_info(phba))
3571 /* Try to see if we can carry on without this */
3572 return 0;
3573
3574 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3575 if (!phba->boot_kset)
3576 return -ENOMEM;
3577
3578 /* get a ref because the show function will ref the phba */
3579 if (!scsi_host_get(phba->shost))
3580 goto free_kset;
3581 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3582 beiscsi_show_boot_tgt_info,
3583 beiscsi_tgt_get_attr_visibility,
3584 beiscsi_boot_release);
3585 if (!boot_kobj)
3586 goto put_shost;
3587
3588 if (!scsi_host_get(phba->shost))
3589 goto free_kset;
3590 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3591 beiscsi_show_boot_ini_info,
3592 beiscsi_ini_get_attr_visibility,
3593 beiscsi_boot_release);
3594 if (!boot_kobj)
3595 goto put_shost;
3596
3597 if (!scsi_host_get(phba->shost))
3598 goto free_kset;
3599 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3600 beiscsi_show_boot_eth_info,
3601 beiscsi_eth_get_attr_visibility,
3602 beiscsi_boot_release);
3603 if (!boot_kobj)
3604 goto put_shost;
3605 return 0;
3606
3607 put_shost:
3608 scsi_host_put(phba->shost);
3609 free_kset:
3610 iscsi_boot_destroy_kset(phba->boot_kset);
3611 return -ENOMEM;
3612 }
3613
3614 static int beiscsi_init_port(struct beiscsi_hba *phba)
3615 {
3616 int ret;
3617
3618 ret = beiscsi_init_controller(phba);
3619 if (ret < 0) {
3620 shost_printk(KERN_ERR, phba->shost,
3621 "beiscsi_dev_probe - Failed in"
3622 "beiscsi_init_controller\n");
3623 return ret;
3624 }
3625 ret = beiscsi_init_sgl_handle(phba);
3626 if (ret < 0) {
3627 shost_printk(KERN_ERR, phba->shost,
3628 "beiscsi_dev_probe - Failed in"
3629 "beiscsi_init_sgl_handle\n");
3630 goto do_cleanup_ctrlr;
3631 }
3632
3633 if (hba_setup_cid_tbls(phba)) {
3634 shost_printk(KERN_ERR, phba->shost,
3635 "Failed in hba_setup_cid_tbls\n");
3636 kfree(phba->io_sgl_hndl_base);
3637 kfree(phba->eh_sgl_hndl_base);
3638 goto do_cleanup_ctrlr;
3639 }
3640
3641 return ret;
3642
3643 do_cleanup_ctrlr:
3644 hwi_cleanup(phba);
3645 return ret;
3646 }
3647
3648 static void hwi_purge_eq(struct beiscsi_hba *phba)
3649 {
3650 struct hwi_controller *phwi_ctrlr;
3651 struct hwi_context_memory *phwi_context;
3652 struct be_queue_info *eq;
3653 struct be_eq_entry *eqe = NULL;
3654 int i, eq_msix;
3655 unsigned int num_processed;
3656
3657 phwi_ctrlr = phba->phwi_ctrlr;
3658 phwi_context = phwi_ctrlr->phwi_ctxt;
3659 if (phba->msix_enabled)
3660 eq_msix = 1;
3661 else
3662 eq_msix = 0;
3663
3664 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3665 eq = &phwi_context->be_eq[i].q;
3666 eqe = queue_tail_node(eq);
3667 num_processed = 0;
3668 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3669 & EQE_VALID_MASK) {
3670 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3671 queue_tail_inc(eq);
3672 eqe = queue_tail_node(eq);
3673 num_processed++;
3674 }
3675
3676 if (num_processed)
3677 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3678 }
3679 }
3680
3681 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3682 {
3683 int mgmt_status;
3684
3685 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3686 if (mgmt_status)
3687 shost_printk(KERN_WARNING, phba->shost,
3688 "mgmt_epfw_cleanup FAILED\n");
3689
3690 hwi_purge_eq(phba);
3691 hwi_cleanup(phba);
3692 kfree(phba->io_sgl_hndl_base);
3693 kfree(phba->eh_sgl_hndl_base);
3694 kfree(phba->cid_array);
3695 kfree(phba->ep_array);
3696 }
3697
3698 void
3699 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3700 struct beiscsi_offload_params *params)
3701 {
3702 struct wrb_handle *pwrb_handle;
3703 struct iscsi_target_context_update_wrb *pwrb = NULL;
3704 struct be_mem_descriptor *mem_descr;
3705 struct beiscsi_hba *phba = beiscsi_conn->phba;
3706 u32 doorbell = 0;
3707
3708 /*
3709 * We can always use 0 here because it is reserved by libiscsi for
3710 * login/startup related tasks.
3711 */
3712 pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3713 phba->fw_config.iscsi_cid_start));
3714 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3715 memset(pwrb, 0, sizeof(*pwrb));
3716 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3717 max_burst_length, pwrb, params->dw[offsetof
3718 (struct amap_beiscsi_offload_params,
3719 max_burst_length) / 32]);
3720 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3721 max_send_data_segment_length, pwrb,
3722 params->dw[offsetof(struct amap_beiscsi_offload_params,
3723 max_send_data_segment_length) / 32]);
3724 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3725 first_burst_length,
3726 pwrb,
3727 params->dw[offsetof(struct amap_beiscsi_offload_params,
3728 first_burst_length) / 32]);
3729
3730 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3731 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3732 erl) / 32] & OFFLD_PARAMS_ERL));
3733 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3734 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3735 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3736 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3737 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3738 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3739 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3740 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3741 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3742 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3743 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3744 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3745 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3746 pwrb,
3747 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3748 exp_statsn) / 32] + 1));
3749 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3750 0x7);
3751 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3752 pwrb, pwrb_handle->wrb_index);
3753 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3754 pwrb, pwrb_handle->nxt_wrb_index);
3755 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3756 session_state, pwrb, 0);
3757 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3758 pwrb, 1);
3759 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3760 pwrb, 0);
3761 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3762 0);
3763
3764 mem_descr = phba->init_mem;
3765 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3766
3767 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3768 pad_buffer_addr_hi, pwrb,
3769 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3770 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3771 pad_buffer_addr_lo, pwrb,
3772 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3773
3774 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3775
3776 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3777 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3778 << DB_DEF_PDU_WRB_INDEX_SHIFT;
3779 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3780
3781 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3782 }
3783
3784 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3785 int *index, int *age)
3786 {
3787 *index = (int)itt;
3788 if (age)
3789 *age = conn->session->age;
3790 }
3791
3792 /**
3793 * beiscsi_alloc_pdu - allocates pdu and related resources
3794 * @task: libiscsi task
3795 * @opcode: opcode of pdu for task
3796 *
3797 * This is called with the session lock held. It will allocate
3798 * the wrb and sgl if needed for the command. And it will prep
3799 * the pdu's itt. beiscsi_parse_pdu will later translate
3800 * the pdu itt to the libiscsi task itt.
3801 */
3802 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3803 {
3804 struct beiscsi_io_task *io_task = task->dd_data;
3805 struct iscsi_conn *conn = task->conn;
3806 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3807 struct beiscsi_hba *phba = beiscsi_conn->phba;
3808 struct hwi_wrb_context *pwrb_context;
3809 struct hwi_controller *phwi_ctrlr;
3810 itt_t itt;
3811 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3812 dma_addr_t paddr;
3813
3814 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3815 GFP_ATOMIC, &paddr);
3816 if (!io_task->cmd_bhs)
3817 return -ENOMEM;
3818 io_task->bhs_pa.u.a64.address = paddr;
3819 io_task->libiscsi_itt = (itt_t)task->itt;
3820 io_task->conn = beiscsi_conn;
3821
3822 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3823 task->hdr_max = sizeof(struct be_cmd_bhs);
3824 io_task->psgl_handle = NULL;
3825 io_task->psgl_handle = NULL;
3826
3827 if (task->sc) {
3828 spin_lock(&phba->io_sgl_lock);
3829 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3830 spin_unlock(&phba->io_sgl_lock);
3831 if (!io_task->psgl_handle)
3832 goto free_hndls;
3833 io_task->pwrb_handle = alloc_wrb_handle(phba,
3834 beiscsi_conn->beiscsi_conn_cid -
3835 phba->fw_config.iscsi_cid_start);
3836 if (!io_task->pwrb_handle)
3837 goto free_io_hndls;
3838 } else {
3839 io_task->scsi_cmnd = NULL;
3840 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3841 if (!beiscsi_conn->login_in_progress) {
3842 spin_lock(&phba->mgmt_sgl_lock);
3843 io_task->psgl_handle = (struct sgl_handle *)
3844 alloc_mgmt_sgl_handle(phba);
3845 spin_unlock(&phba->mgmt_sgl_lock);
3846 if (!io_task->psgl_handle)
3847 goto free_hndls;
3848
3849 beiscsi_conn->login_in_progress = 1;
3850 beiscsi_conn->plogin_sgl_handle =
3851 io_task->psgl_handle;
3852 io_task->pwrb_handle =
3853 alloc_wrb_handle(phba,
3854 beiscsi_conn->beiscsi_conn_cid -
3855 phba->fw_config.iscsi_cid_start);
3856 if (!io_task->pwrb_handle)
3857 goto free_io_hndls;
3858 beiscsi_conn->plogin_wrb_handle =
3859 io_task->pwrb_handle;
3860
3861 } else {
3862 io_task->psgl_handle =
3863 beiscsi_conn->plogin_sgl_handle;
3864 io_task->pwrb_handle =
3865 beiscsi_conn->plogin_wrb_handle;
3866 }
3867 } else {
3868 spin_lock(&phba->mgmt_sgl_lock);
3869 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3870 spin_unlock(&phba->mgmt_sgl_lock);
3871 if (!io_task->psgl_handle)
3872 goto free_hndls;
3873 io_task->pwrb_handle =
3874 alloc_wrb_handle(phba,
3875 beiscsi_conn->beiscsi_conn_cid -
3876 phba->fw_config.iscsi_cid_start);
3877 if (!io_task->pwrb_handle)
3878 goto free_mgmt_hndls;
3879
3880 }
3881 }
3882 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3883 wrb_index << 16) | (unsigned int)
3884 (io_task->psgl_handle->sgl_index));
3885 io_task->pwrb_handle->pio_handle = task;
3886
3887 io_task->cmd_bhs->iscsi_hdr.itt = itt;
3888 return 0;
3889
3890 free_io_hndls:
3891 spin_lock(&phba->io_sgl_lock);
3892 free_io_sgl_handle(phba, io_task->psgl_handle);
3893 spin_unlock(&phba->io_sgl_lock);
3894 goto free_hndls;
3895 free_mgmt_hndls:
3896 spin_lock(&phba->mgmt_sgl_lock);
3897 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3898 spin_unlock(&phba->mgmt_sgl_lock);
3899 free_hndls:
3900 phwi_ctrlr = phba->phwi_ctrlr;
3901 pwrb_context = &phwi_ctrlr->wrb_context[
3902 beiscsi_conn->beiscsi_conn_cid -
3903 phba->fw_config.iscsi_cid_start];
3904 if (io_task->pwrb_handle)
3905 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3906 io_task->pwrb_handle = NULL;
3907 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3908 io_task->bhs_pa.u.a64.address);
3909 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
3910 return -ENOMEM;
3911 }
3912
3913 static void beiscsi_cleanup_task(struct iscsi_task *task)
3914 {
3915 struct beiscsi_io_task *io_task = task->dd_data;
3916 struct iscsi_conn *conn = task->conn;
3917 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3918 struct beiscsi_hba *phba = beiscsi_conn->phba;
3919 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3920 struct hwi_wrb_context *pwrb_context;
3921 struct hwi_controller *phwi_ctrlr;
3922
3923 phwi_ctrlr = phba->phwi_ctrlr;
3924 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3925 - phba->fw_config.iscsi_cid_start];
3926 if (io_task->pwrb_handle) {
3927 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3928 io_task->pwrb_handle = NULL;
3929 }
3930
3931 if (io_task->cmd_bhs) {
3932 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3933 io_task->bhs_pa.u.a64.address);
3934 }
3935
3936 if (task->sc) {
3937 if (io_task->psgl_handle) {
3938 spin_lock(&phba->io_sgl_lock);
3939 free_io_sgl_handle(phba, io_task->psgl_handle);
3940 spin_unlock(&phba->io_sgl_lock);
3941 io_task->psgl_handle = NULL;
3942 }
3943 } else {
3944 if (task->hdr &&
3945 ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN))
3946 return;
3947 if (io_task->psgl_handle) {
3948 spin_lock(&phba->mgmt_sgl_lock);
3949 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3950 spin_unlock(&phba->mgmt_sgl_lock);
3951 io_task->psgl_handle = NULL;
3952 }
3953 }
3954 }
3955
3956 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3957 unsigned int num_sg, unsigned int xferlen,
3958 unsigned int writedir)
3959 {
3960
3961 struct beiscsi_io_task *io_task = task->dd_data;
3962 struct iscsi_conn *conn = task->conn;
3963 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3964 struct beiscsi_hba *phba = beiscsi_conn->phba;
3965 struct iscsi_wrb *pwrb = NULL;
3966 unsigned int doorbell = 0;
3967
3968 pwrb = io_task->pwrb_handle->pwrb;
3969 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3970 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3971
3972 if (writedir) {
3973 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3974 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3975 &io_task->cmd_bhs->iscsi_data_pdu,
3976 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3977 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3978 &io_task->cmd_bhs->iscsi_data_pdu,
3979 ISCSI_OPCODE_SCSI_DATA_OUT);
3980 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3981 &io_task->cmd_bhs->iscsi_data_pdu, 1);
3982 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3983 INI_WR_CMD);
3984 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3985 } else {
3986 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3987 INI_RD_CMD);
3988 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3989 }
3990 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3991 dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3992 &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3993
3994 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3995 cpu_to_be16(*(unsigned short *)&io_task->cmd_bhs->iscsi_hdr.lun));
3996 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3997 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3998 io_task->pwrb_handle->wrb_index);
3999 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4000 be32_to_cpu(task->cmdsn));
4001 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4002 io_task->psgl_handle->sgl_index);
4003
4004 hwi_write_sgl(pwrb, sg, num_sg, io_task);
4005
4006 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4007 io_task->pwrb_handle->nxt_wrb_index);
4008 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4009
4010 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4011 doorbell |= (io_task->pwrb_handle->wrb_index &
4012 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4013 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4014
4015 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4016 return 0;
4017 }
4018
4019 static int beiscsi_mtask(struct iscsi_task *task)
4020 {
4021 struct beiscsi_io_task *io_task = task->dd_data;
4022 struct iscsi_conn *conn = task->conn;
4023 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4024 struct beiscsi_hba *phba = beiscsi_conn->phba;
4025 struct iscsi_wrb *pwrb = NULL;
4026 unsigned int doorbell = 0;
4027 unsigned int cid;
4028
4029 cid = beiscsi_conn->beiscsi_conn_cid;
4030 pwrb = io_task->pwrb_handle->pwrb;
4031 memset(pwrb, 0, sizeof(*pwrb));
4032 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4033 be32_to_cpu(task->cmdsn));
4034 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4035 io_task->pwrb_handle->wrb_index);
4036 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4037 io_task->psgl_handle->sgl_index);
4038
4039 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4040 case ISCSI_OP_LOGIN:
4041 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4042 TGT_DM_CMD);
4043 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4044 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4045 hwi_write_buffer(pwrb, task);
4046 break;
4047 case ISCSI_OP_NOOP_OUT:
4048 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4049 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4050 TGT_DM_CMD);
4051 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4052 pwrb, 0);
4053 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
4054 } else {
4055 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4056 INI_RD_CMD);
4057 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4058 }
4059 hwi_write_buffer(pwrb, task);
4060 break;
4061 case ISCSI_OP_TEXT:
4062 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4063 TGT_DM_CMD);
4064 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4065 hwi_write_buffer(pwrb, task);
4066 break;
4067 case ISCSI_OP_SCSI_TMFUNC:
4068 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4069 INI_TMF_CMD);
4070 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4071 hwi_write_buffer(pwrb, task);
4072 break;
4073 case ISCSI_OP_LOGOUT:
4074 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4075 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4076 HWH_TYPE_LOGOUT);
4077 hwi_write_buffer(pwrb, task);
4078 break;
4079
4080 default:
4081 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
4082 task->hdr->opcode & ISCSI_OPCODE_MASK);
4083 return -EINVAL;
4084 }
4085
4086 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4087 task->data_count);
4088 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4089 io_task->pwrb_handle->nxt_wrb_index);
4090 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4091
4092 doorbell |= cid & DB_WRB_POST_CID_MASK;
4093 doorbell |= (io_task->pwrb_handle->wrb_index &
4094 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4095 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4096 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4097 return 0;
4098 }
4099
4100 static int beiscsi_task_xmit(struct iscsi_task *task)
4101 {
4102 struct beiscsi_io_task *io_task = task->dd_data;
4103 struct scsi_cmnd *sc = task->sc;
4104 struct scatterlist *sg;
4105 int num_sg;
4106 unsigned int writedir = 0, xferlen = 0;
4107
4108 if (!sc)
4109 return beiscsi_mtask(task);
4110
4111 io_task->scsi_cmnd = sc;
4112 num_sg = scsi_dma_map(sc);
4113 if (num_sg < 0) {
4114 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4115 return num_sg;
4116 }
4117 xferlen = scsi_bufflen(sc);
4118 sg = scsi_sglist(sc);
4119 if (sc->sc_data_direction == DMA_TO_DEVICE) {
4120 writedir = 1;
4121 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
4122 task->imm_count);
4123 } else
4124 writedir = 0;
4125 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4126 }
4127
4128 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4129 {
4130 struct hwi_controller *phwi_ctrlr;
4131 struct hwi_context_memory *phwi_context;
4132 struct be_eq_obj *pbe_eq;
4133 unsigned int i, msix_vec;
4134 u8 *real_offset = 0;
4135 u32 value = 0;
4136
4137 phwi_ctrlr = phba->phwi_ctrlr;
4138 phwi_context = phwi_ctrlr->phwi_ctxt;
4139 hwi_disable_intr(phba);
4140 if (phba->msix_enabled) {
4141 for (i = 0; i <= phba->num_cpus; i++) {
4142 msix_vec = phba->msix_entries[i].vector;
4143 free_irq(msix_vec, &phwi_context->be_eq[i]);
4144 kfree(phba->msi_name[i]);
4145 }
4146 } else
4147 if (phba->pcidev->irq)
4148 free_irq(phba->pcidev->irq, phba);
4149 pci_disable_msix(phba->pcidev);
4150 destroy_workqueue(phba->wq);
4151 if (blk_iopoll_enabled)
4152 for (i = 0; i < phba->num_cpus; i++) {
4153 pbe_eq = &phwi_context->be_eq[i];
4154 blk_iopoll_disable(&pbe_eq->iopoll);
4155 }
4156
4157 beiscsi_clean_port(phba);
4158 beiscsi_free_mem(phba);
4159 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4160
4161 value = readl((void *)real_offset);
4162
4163 if (value & 0x00010000) {
4164 value &= 0xfffeffff;
4165 writel(value, (void *)real_offset);
4166 }
4167 beiscsi_unmap_pci_function(phba);
4168 pci_free_consistent(phba->pcidev,
4169 phba->ctrl.mbox_mem_alloced.size,
4170 phba->ctrl.mbox_mem_alloced.va,
4171 phba->ctrl.mbox_mem_alloced.dma);
4172 }
4173
4174 static void beiscsi_remove(struct pci_dev *pcidev)
4175 {
4176
4177 struct beiscsi_hba *phba = NULL;
4178
4179 phba = pci_get_drvdata(pcidev);
4180 if (!phba) {
4181 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4182 return;
4183 }
4184
4185 beiscsi_quiesce(phba);
4186 iscsi_boot_destroy_kset(phba->boot_kset);
4187 iscsi_host_remove(phba->shost);
4188 pci_dev_put(phba->pcidev);
4189 iscsi_host_free(phba->shost);
4190 pci_disable_device(pcidev);
4191 }
4192
4193 static void beiscsi_shutdown(struct pci_dev *pcidev)
4194 {
4195
4196 struct beiscsi_hba *phba = NULL;
4197
4198 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4199 if (!phba) {
4200 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4201 return;
4202 }
4203
4204 beiscsi_quiesce(phba);
4205 pci_disable_device(pcidev);
4206 }
4207
4208 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4209 {
4210 int i, status;
4211
4212 for (i = 0; i <= phba->num_cpus; i++)
4213 phba->msix_entries[i].entry = i;
4214
4215 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4216 (phba->num_cpus + 1));
4217 if (!status)
4218 phba->msix_enabled = true;
4219
4220 return;
4221 }
4222
4223 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4224 const struct pci_device_id *id)
4225 {
4226 struct beiscsi_hba *phba = NULL;
4227 struct hwi_controller *phwi_ctrlr;
4228 struct hwi_context_memory *phwi_context;
4229 struct be_eq_obj *pbe_eq;
4230 int ret, num_cpus, i;
4231 u8 *real_offset = 0;
4232 u32 value = 0;
4233
4234 ret = beiscsi_enable_pci(pcidev);
4235 if (ret < 0) {
4236 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4237 " Failed to enable pci device\n");
4238 return ret;
4239 }
4240
4241 phba = beiscsi_hba_alloc(pcidev);
4242 if (!phba) {
4243 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4244 " Failed in beiscsi_hba_alloc\n");
4245 goto disable_pci;
4246 }
4247
4248 switch (pcidev->device) {
4249 case BE_DEVICE_ID1:
4250 case OC_DEVICE_ID1:
4251 case OC_DEVICE_ID2:
4252 phba->generation = BE_GEN2;
4253 break;
4254 case BE_DEVICE_ID2:
4255 case OC_DEVICE_ID3:
4256 phba->generation = BE_GEN3;
4257 break;
4258 default:
4259 phba->generation = 0;
4260 }
4261
4262 if (enable_msix)
4263 num_cpus = find_num_cpus();
4264 else
4265 num_cpus = 1;
4266 phba->num_cpus = num_cpus;
4267 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
4268
4269 if (enable_msix)
4270 beiscsi_msix_enable(phba);
4271 ret = be_ctrl_init(phba, pcidev);
4272 if (ret) {
4273 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4274 "Failed in be_ctrl_init\n");
4275 goto hba_free;
4276 }
4277
4278 if (!num_hba) {
4279 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4280 value = readl((void *)real_offset);
4281 if (value & 0x00010000) {
4282 gcrashmode++;
4283 shost_printk(KERN_ERR, phba->shost,
4284 "Loading Driver in crashdump mode\n");
4285 ret = beiscsi_cmd_reset_function(phba);
4286 if (ret) {
4287 shost_printk(KERN_ERR, phba->shost,
4288 "Reset Failed. Aborting Crashdump\n");
4289 goto hba_free;
4290 }
4291 ret = be_chk_reset_complete(phba);
4292 if (ret) {
4293 shost_printk(KERN_ERR, phba->shost,
4294 "Failed to get out of reset."
4295 "Aborting Crashdump\n");
4296 goto hba_free;
4297 }
4298 } else {
4299 value |= 0x00010000;
4300 writel(value, (void *)real_offset);
4301 num_hba++;
4302 }
4303 }
4304
4305 spin_lock_init(&phba->io_sgl_lock);
4306 spin_lock_init(&phba->mgmt_sgl_lock);
4307 spin_lock_init(&phba->isr_lock);
4308 ret = mgmt_get_fw_config(&phba->ctrl, phba);
4309 if (ret != 0) {
4310 shost_printk(KERN_ERR, phba->shost,
4311 "Error getting fw config\n");
4312 goto free_port;
4313 }
4314 phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4315 beiscsi_get_params(phba);
4316 phba->shost->can_queue = phba->params.ios_per_ctrl;
4317 ret = beiscsi_init_port(phba);
4318 if (ret < 0) {
4319 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4320 "Failed in beiscsi_init_port\n");
4321 goto free_port;
4322 }
4323
4324 for (i = 0; i < MAX_MCC_CMD ; i++) {
4325 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4326 phba->ctrl.mcc_tag[i] = i + 1;
4327 phba->ctrl.mcc_numtag[i + 1] = 0;
4328 phba->ctrl.mcc_tag_available++;
4329 }
4330
4331 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4332
4333 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4334 phba->shost->host_no);
4335 phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4336 if (!phba->wq) {
4337 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4338 "Failed to allocate work queue\n");
4339 goto free_twq;
4340 }
4341
4342 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4343
4344 phwi_ctrlr = phba->phwi_ctrlr;
4345 phwi_context = phwi_ctrlr->phwi_ctxt;
4346 if (blk_iopoll_enabled) {
4347 for (i = 0; i < phba->num_cpus; i++) {
4348 pbe_eq = &phwi_context->be_eq[i];
4349 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4350 be_iopoll);
4351 blk_iopoll_enable(&pbe_eq->iopoll);
4352 }
4353 }
4354 ret = beiscsi_init_irqs(phba);
4355 if (ret < 0) {
4356 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4357 "Failed to beiscsi_init_irqs\n");
4358 goto free_blkenbld;
4359 }
4360 hwi_enable_intr(phba);
4361
4362 if (beiscsi_setup_boot_info(phba))
4363 /*
4364 * log error but continue, because we may not be using
4365 * iscsi boot.
4366 */
4367 shost_printk(KERN_ERR, phba->shost, "Could not set up "
4368 "iSCSI boot info.");
4369
4370 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4371 return 0;
4372
4373 free_blkenbld:
4374 destroy_workqueue(phba->wq);
4375 if (blk_iopoll_enabled)
4376 for (i = 0; i < phba->num_cpus; i++) {
4377 pbe_eq = &phwi_context->be_eq[i];
4378 blk_iopoll_disable(&pbe_eq->iopoll);
4379 }
4380 free_twq:
4381 beiscsi_clean_port(phba);
4382 beiscsi_free_mem(phba);
4383 free_port:
4384 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4385
4386 value = readl((void *)real_offset);
4387
4388 if (value & 0x00010000) {
4389 value &= 0xfffeffff;
4390 writel(value, (void *)real_offset);
4391 }
4392
4393 pci_free_consistent(phba->pcidev,
4394 phba->ctrl.mbox_mem_alloced.size,
4395 phba->ctrl.mbox_mem_alloced.va,
4396 phba->ctrl.mbox_mem_alloced.dma);
4397 beiscsi_unmap_pci_function(phba);
4398 hba_free:
4399 if (phba->msix_enabled)
4400 pci_disable_msix(phba->pcidev);
4401 iscsi_host_remove(phba->shost);
4402 pci_dev_put(phba->pcidev);
4403 iscsi_host_free(phba->shost);
4404 disable_pci:
4405 pci_disable_device(pcidev);
4406 return ret;
4407 }
4408
4409 struct iscsi_transport beiscsi_iscsi_transport = {
4410 .owner = THIS_MODULE,
4411 .name = DRV_NAME,
4412 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4413 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4414 .create_session = beiscsi_session_create,
4415 .destroy_session = beiscsi_session_destroy,
4416 .create_conn = beiscsi_conn_create,
4417 .bind_conn = beiscsi_conn_bind,
4418 .destroy_conn = iscsi_conn_teardown,
4419 .attr_is_visible = be2iscsi_attr_is_visible,
4420 .set_param = beiscsi_set_param,
4421 .get_conn_param = iscsi_conn_get_param,
4422 .get_session_param = iscsi_session_get_param,
4423 .get_host_param = beiscsi_get_host_param,
4424 .start_conn = beiscsi_conn_start,
4425 .stop_conn = iscsi_conn_stop,
4426 .send_pdu = iscsi_conn_send_pdu,
4427 .xmit_task = beiscsi_task_xmit,
4428 .cleanup_task = beiscsi_cleanup_task,
4429 .alloc_pdu = beiscsi_alloc_pdu,
4430 .parse_pdu_itt = beiscsi_parse_pdu,
4431 .get_stats = beiscsi_conn_get_stats,
4432 .get_ep_param = beiscsi_ep_get_param,
4433 .ep_connect = beiscsi_ep_connect,
4434 .ep_poll = beiscsi_ep_poll,
4435 .ep_disconnect = beiscsi_ep_disconnect,
4436 .session_recovery_timedout = iscsi_session_recovery_timedout,
4437 };
4438
4439 static struct pci_driver beiscsi_pci_driver = {
4440 .name = DRV_NAME,
4441 .probe = beiscsi_dev_probe,
4442 .remove = beiscsi_remove,
4443 .shutdown = beiscsi_shutdown,
4444 .id_table = beiscsi_pci_id_table
4445 };
4446
4447
4448 static int __init beiscsi_module_init(void)
4449 {
4450 int ret;
4451
4452 beiscsi_scsi_transport =
4453 iscsi_register_transport(&beiscsi_iscsi_transport);
4454 if (!beiscsi_scsi_transport) {
4455 SE_DEBUG(DBG_LVL_1,
4456 "beiscsi_module_init - Unable to register beiscsi"
4457 "transport.\n");
4458 return -ENOMEM;
4459 }
4460 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
4461 &beiscsi_iscsi_transport);
4462
4463 ret = pci_register_driver(&beiscsi_pci_driver);
4464 if (ret) {
4465 SE_DEBUG(DBG_LVL_1,
4466 "beiscsi_module_init - Unable to register"
4467 "beiscsi pci driver.\n");
4468 goto unregister_iscsi_transport;
4469 }
4470 return 0;
4471
4472 unregister_iscsi_transport:
4473 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4474 return ret;
4475 }
4476
4477 static void __exit beiscsi_module_exit(void)
4478 {
4479 pci_unregister_driver(&beiscsi_pci_driver);
4480 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4481 }
4482
4483 module_init(beiscsi_module_init);
4484 module_exit(beiscsi_module_exit);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree