2 * Copyright (c) 2006 - 2009 Intel Corporation. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/random.h>
37 #include <linux/highmem.h>
38 #include <linux/slab.h>
39 #include <asm/byteorder.h>
41 #include <rdma/ib_verbs.h>
42 #include <rdma/iw_cm.h>
43 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
49 atomic_t mod_qp_timouts
;
51 atomic_t sw_qps_destroyed
;
53 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
);
58 static struct ib_mw
*nes_alloc_mw(struct ib_pd
*ibpd
) {
59 struct nes_pd
*nespd
= to_nespd(ibpd
);
60 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
61 struct nes_device
*nesdev
= nesvnic
->nesdev
;
62 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
63 struct nes_cqp_request
*cqp_request
;
66 struct nes_hw_cqp_wqe
*cqp_wqe
;
70 u32 next_stag_index
= 0;
74 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
75 stag_key
= (u8
)next_stag_index
;
79 next_stag_index
>>= 8;
80 next_stag_index
%= nesadapter
->max_mr
;
82 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
83 nesadapter
->max_mr
, &stag_index
, &next_stag_index
);
88 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
90 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
91 return ERR_PTR(-ENOMEM
);
94 stag
= stag_index
<< 8;
96 stag
+= (u32
)stag_key
;
98 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, index = 0x%08X\n",
101 /* Register the region with the adapter */
102 cqp_request
= nes_get_cqp_request(nesdev
);
103 if (cqp_request
== NULL
) {
105 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
106 return ERR_PTR(-ENOMEM
);
109 cqp_request
->waiting
= 1;
110 cqp_wqe
= &cqp_request
->cqp_wqe
;
112 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] =
113 cpu_to_le32( NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_RIGHTS_REMOTE_READ
|
114 NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_VA_TO
|
115 NES_CQP_STAG_REM_ACC_EN
);
117 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
118 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
, (nespd
->pd_id
& 0x00007fff));
119 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
121 atomic_set(&cqp_request
->refcount
, 2);
122 nes_post_cqp_request(nesdev
, cqp_request
);
125 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
127 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
128 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
129 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
130 if ((!ret
) || (cqp_request
->major_code
)) {
131 nes_put_cqp_request(nesdev
, cqp_request
);
133 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
135 return ERR_PTR(-ETIME
);
137 return ERR_PTR(-ENOMEM
);
140 nes_put_cqp_request(nesdev
, cqp_request
);
142 nesmr
->ibmw
.rkey
= stag
;
143 nesmr
->mode
= IWNES_MEMREG_TYPE_MW
;
146 nesmr
->pbls_used
= 0;
155 static int nes_dealloc_mw(struct ib_mw
*ibmw
)
157 struct nes_mr
*nesmr
= to_nesmw(ibmw
);
158 struct nes_vnic
*nesvnic
= to_nesvnic(ibmw
->device
);
159 struct nes_device
*nesdev
= nesvnic
->nesdev
;
160 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
161 struct nes_hw_cqp_wqe
*cqp_wqe
;
162 struct nes_cqp_request
*cqp_request
;
166 /* Deallocate the window with the adapter */
167 cqp_request
= nes_get_cqp_request(nesdev
);
168 if (cqp_request
== NULL
) {
169 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
172 cqp_request
->waiting
= 1;
173 cqp_wqe
= &cqp_request
->cqp_wqe
;
174 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
175 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, NES_CQP_DEALLOCATE_STAG
);
176 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ibmw
->rkey
);
178 atomic_set(&cqp_request
->refcount
, 2);
179 nes_post_cqp_request(nesdev
, cqp_request
);
182 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X to complete.\n",
184 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
186 nes_debug(NES_DBG_MR
, "Deallocate STag completed, wait_event_timeout ret = %u,"
187 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
188 ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
191 else if (cqp_request
->major_code
)
194 nes_put_cqp_request(nesdev
, cqp_request
);
196 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
197 (ibmw
->rkey
& 0x0fffff00) >> 8);
207 static int nes_bind_mw(struct ib_qp
*ibqp
, struct ib_mw
*ibmw
,
208 struct ib_mw_bind
*ibmw_bind
)
211 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
212 struct nes_device
*nesdev
= nesvnic
->nesdev
;
213 /* struct nes_mr *nesmr = to_nesmw(ibmw); */
214 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
215 struct nes_hw_qp_wqe
*wqe
;
216 unsigned long flags
= 0;
221 if (nesqp
->ibqp_state
> IB_QPS_RTS
)
224 spin_lock_irqsave(&nesqp
->lock
, flags
);
226 head
= nesqp
->hwqp
.sq_head
;
227 qsize
= nesqp
->hwqp
.sq_tail
;
229 /* Check for SQ overflow */
230 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.sq_tail
) % qsize
) == (qsize
- 1)) {
231 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
235 wqe
= &nesqp
->hwqp
.sq_vbase
[head
];
236 /* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */
237 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
238 u64temp
= ibmw_bind
->wr_id
;
239 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
, u64temp
);
240 wqe_misc
= NES_IWARP_SQ_OP_BIND
;
242 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
244 if (ibmw_bind
->send_flags
& IB_SEND_SIGNALED
)
245 wqe_misc
|= NES_IWARP_SQ_WQE_SIGNALED_COMPL
;
247 if (ibmw_bind
->mw_access_flags
& IB_ACCESS_REMOTE_WRITE
) {
248 wqe_misc
|= NES_CQP_STAG_RIGHTS_REMOTE_WRITE
;
250 if (ibmw_bind
->mw_access_flags
& IB_ACCESS_REMOTE_READ
) {
251 wqe_misc
|= NES_CQP_STAG_RIGHTS_REMOTE_READ
;
254 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_MISC_IDX
, wqe_misc
);
255 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_MR_IDX
, ibmw_bind
->mr
->lkey
);
256 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_MW_IDX
, ibmw
->rkey
);
257 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX
,
259 wqe
->wqe_words
[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX
] = 0;
260 u64temp
= (u64
)ibmw_bind
->addr
;
261 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX
, u64temp
);
267 nesqp
->hwqp
.sq_head
= head
;
270 nes_write32(nesdev
->regs
+NES_WQE_ALLOC
,
271 (1 << 24) | 0x00800000 | nesqp
->hwqp
.qp_id
);
273 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
282 static int alloc_fast_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
283 u32 stag
, u32 page_count
)
285 struct nes_hw_cqp_wqe
*cqp_wqe
;
286 struct nes_cqp_request
*cqp_request
;
289 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
292 u64 region_length
= page_count
* PAGE_SIZE
;
295 cqp_request
= nes_get_cqp_request(nesdev
);
296 if (cqp_request
== NULL
) {
297 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
300 nes_debug(NES_DBG_MR
, "alloc_fast_reg_mr: page_count = %d, "
301 "region_length = %llu\n",
302 page_count
, region_length
);
303 cqp_request
->waiting
= 1;
304 cqp_wqe
= &cqp_request
->cqp_wqe
;
306 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
307 if (nesadapter
->free_4kpbl
> 0) {
308 nesadapter
->free_4kpbl
--;
309 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
311 /* No 4kpbl's available: */
312 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
313 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
314 nes_free_cqp_request(nesdev
, cqp_request
);
318 opcode
= NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_MR
|
319 NES_CQP_STAG_PBL_BLK_SIZE
| NES_CQP_STAG_VA_TO
|
320 NES_CQP_STAG_REM_ACC_EN
;
322 * The current OFED API does not support the zero based TO option.
323 * If added then need to changed the NES_CQP_STAG_VA* option. Also,
324 * the API does not support that ability to have the MR set for local
325 * access only when created and not allow the SQ op to override. Given
326 * this the remote enable must be set here.
329 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
330 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
331 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, 1);
333 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
334 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
335 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
336 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
338 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
339 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, 0);
340 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, 0);
341 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, 0);
342 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (page_count
* 8));
343 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
346 atomic_set(&cqp_request
->refcount
, 2);
347 nes_post_cqp_request(nesdev
, cqp_request
);
350 ret
= wait_event_timeout(cqp_request
->waitq
,
351 (0 != cqp_request
->request_done
),
354 nes_debug(NES_DBG_MR
, "Allocate STag 0x%08X completed, "
355 "wait_event_timeout ret = %u, CQP Major:Minor codes = "
356 "0x%04X:0x%04X.\n", stag
, ret
, cqp_request
->major_code
,
357 cqp_request
->minor_code
);
358 major_code
= cqp_request
->major_code
;
359 nes_put_cqp_request(nesdev
, cqp_request
);
361 if (!ret
|| major_code
) {
362 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
363 nesadapter
->free_4kpbl
++;
364 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
375 * nes_alloc_fast_reg_mr
377 static struct ib_mr
*nes_alloc_fast_reg_mr(struct ib_pd
*ibpd
, int max_page_list_len
)
379 struct nes_pd
*nespd
= to_nespd(ibpd
);
380 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
381 struct nes_device
*nesdev
= nesvnic
->nesdev
;
382 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
389 struct nes_mr
*nesmr
;
394 * Note: Set to always use a fixed length single page entry PBL. This is to allow
395 * for the fast_reg_mr operation to always know the size of the PBL.
397 if (max_page_list_len
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
398 return ERR_PTR(-E2BIG
);
400 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
401 stag_key
= (u8
)next_stag_index
;
402 next_stag_index
>>= 8;
403 next_stag_index
%= nesadapter
->max_mr
;
405 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
406 nesadapter
->max_mr
, &stag_index
,
411 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
413 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
414 return ERR_PTR(-ENOMEM
);
417 stag
= stag_index
<< 8;
419 stag
+= (u32
)stag_key
;
421 nes_debug(NES_DBG_MR
, "Allocating STag 0x%08X index = 0x%08X\n",
424 ret
= alloc_fast_reg_mr(nesdev
, nespd
, stag
, max_page_list_len
);
427 nesmr
->ibmr
.rkey
= stag
;
428 nesmr
->ibmr
.lkey
= stag
;
429 nesmr
->mode
= IWNES_MEMREG_TYPE_FMEM
;
433 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
434 ibmr
= ERR_PTR(-ENOMEM
);
440 * nes_alloc_fast_reg_page_list
442 static struct ib_fast_reg_page_list
*nes_alloc_fast_reg_page_list(
443 struct ib_device
*ibdev
,
446 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
447 struct nes_device
*nesdev
= nesvnic
->nesdev
;
448 struct ib_fast_reg_page_list
*pifrpl
;
449 struct nes_ib_fast_reg_page_list
*pnesfrpl
;
451 if (page_list_len
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
452 return ERR_PTR(-E2BIG
);
454 * Allocate the ib_fast_reg_page_list structure, the
455 * nes_fast_bpl structure, and the PLB table.
457 pnesfrpl
= kmalloc(sizeof(struct nes_ib_fast_reg_page_list
) +
458 page_list_len
* sizeof(u64
), GFP_KERNEL
);
461 return ERR_PTR(-ENOMEM
);
463 pifrpl
= &pnesfrpl
->ibfrpl
;
464 pifrpl
->page_list
= &pnesfrpl
->pbl
;
465 pifrpl
->max_page_list_len
= page_list_len
;
467 * Allocate the WQE PBL
469 pnesfrpl
->nes_wqe_pbl
.kva
= pci_alloc_consistent(nesdev
->pcidev
,
470 page_list_len
* sizeof(u64
),
471 &pnesfrpl
->nes_wqe_pbl
.paddr
);
473 if (!pnesfrpl
->nes_wqe_pbl
.kva
) {
475 return ERR_PTR(-ENOMEM
);
477 nes_debug(NES_DBG_MR
, "nes_alloc_fast_reg_pbl: nes_frpl = %p, "
478 "ibfrpl = %p, ibfrpl.page_list = %p, pbl.kva = %p, "
479 "pbl.paddr= %p\n", pnesfrpl
, &pnesfrpl
->ibfrpl
,
480 pnesfrpl
->ibfrpl
.page_list
, pnesfrpl
->nes_wqe_pbl
.kva
,
481 (void *)pnesfrpl
->nes_wqe_pbl
.paddr
);
487 * nes_free_fast_reg_page_list
489 static void nes_free_fast_reg_page_list(struct ib_fast_reg_page_list
*pifrpl
)
491 struct nes_vnic
*nesvnic
= to_nesvnic(pifrpl
->device
);
492 struct nes_device
*nesdev
= nesvnic
->nesdev
;
493 struct nes_ib_fast_reg_page_list
*pnesfrpl
;
495 pnesfrpl
= container_of(pifrpl
, struct nes_ib_fast_reg_page_list
, ibfrpl
);
499 pci_free_consistent(nesdev
->pcidev
,
500 pifrpl
->max_page_list_len
* sizeof(u64
),
501 pnesfrpl
->nes_wqe_pbl
.kva
,
502 pnesfrpl
->nes_wqe_pbl
.paddr
);
504 * Free the PBL structure
512 static int nes_query_device(struct ib_device
*ibdev
, struct ib_device_attr
*props
)
514 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
515 struct nes_device
*nesdev
= nesvnic
->nesdev
;
516 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
518 memset(props
, 0, sizeof(*props
));
519 memcpy(&props
->sys_image_guid
, nesvnic
->netdev
->dev_addr
, 6);
521 props
->fw_ver
= nesdev
->nesadapter
->firmware_version
;
522 props
->device_cap_flags
= nesdev
->nesadapter
->device_cap_flags
;
523 props
->vendor_id
= nesdev
->nesadapter
->vendor_id
;
524 props
->vendor_part_id
= nesdev
->nesadapter
->vendor_part_id
;
525 props
->hw_ver
= nesdev
->nesadapter
->hw_rev
;
526 props
->max_mr_size
= 0x80000000;
527 props
->max_qp
= nesibdev
->max_qp
;
528 props
->max_qp_wr
= nesdev
->nesadapter
->max_qp_wr
- 2;
529 props
->max_sge
= nesdev
->nesadapter
->max_sge
;
530 props
->max_cq
= nesibdev
->max_cq
;
531 props
->max_cqe
= nesdev
->nesadapter
->max_cqe
;
532 props
->max_mr
= nesibdev
->max_mr
;
533 props
->max_mw
= nesibdev
->max_mr
;
534 props
->max_pd
= nesibdev
->max_pd
;
535 props
->max_sge_rd
= 1;
536 switch (nesdev
->nesadapter
->max_irrq_wr
) {
538 props
->max_qp_rd_atom
= 2;
541 props
->max_qp_rd_atom
= 8;
544 props
->max_qp_rd_atom
= 32;
547 props
->max_qp_rd_atom
= 64;
550 props
->max_qp_rd_atom
= 0;
552 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
553 props
->atomic_cap
= IB_ATOMIC_NONE
;
554 props
->max_map_per_fmr
= 1;
563 static int nes_query_port(struct ib_device
*ibdev
, u8 port
, struct ib_port_attr
*props
)
565 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
566 struct net_device
*netdev
= nesvnic
->netdev
;
568 memset(props
, 0, sizeof(*props
));
570 props
->max_mtu
= IB_MTU_4096
;
572 if (netdev
->mtu
>= 4096)
573 props
->active_mtu
= IB_MTU_4096
;
574 else if (netdev
->mtu
>= 2048)
575 props
->active_mtu
= IB_MTU_2048
;
576 else if (netdev
->mtu
>= 1024)
577 props
->active_mtu
= IB_MTU_1024
;
578 else if (netdev
->mtu
>= 512)
579 props
->active_mtu
= IB_MTU_512
;
581 props
->active_mtu
= IB_MTU_256
;
588 props
->state
= IB_PORT_ACTIVE
;
590 props
->state
= IB_PORT_DOWN
;
591 props
->phys_state
= 0;
592 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
593 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
594 props
->gid_tbl_len
= 1;
595 props
->pkey_tbl_len
= 1;
596 props
->qkey_viol_cntr
= 0;
597 props
->active_width
= IB_WIDTH_4X
;
598 props
->active_speed
= 1;
599 props
->max_msg_sz
= 0x80000000;
608 static int nes_modify_port(struct ib_device
*ibdev
, u8 port
,
609 int port_modify_mask
, struct ib_port_modify
*props
)
618 static int nes_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
, u16
*pkey
)
628 static int nes_query_gid(struct ib_device
*ibdev
, u8 port
,
629 int index
, union ib_gid
*gid
)
631 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
633 memset(&(gid
->raw
[0]), 0, sizeof(gid
->raw
));
634 memcpy(&(gid
->raw
[0]), nesvnic
->netdev
->dev_addr
, 6);
641 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
642 * of all objects associated with a particular user-mode client.
644 static struct ib_ucontext
*nes_alloc_ucontext(struct ib_device
*ibdev
,
645 struct ib_udata
*udata
)
647 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
648 struct nes_device
*nesdev
= nesvnic
->nesdev
;
649 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
650 struct nes_alloc_ucontext_req req
;
651 struct nes_alloc_ucontext_resp uresp
;
652 struct nes_ucontext
*nes_ucontext
;
653 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
656 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_alloc_ucontext_req
))) {
657 printk(KERN_ERR PFX
"Invalid structure size on allocate user context.\n");
658 return ERR_PTR(-EINVAL
);
661 if (req
.userspace_ver
!= NES_ABI_USERSPACE_VER
) {
662 printk(KERN_ERR PFX
"Invalid userspace driver version detected. Detected version %d, should be %d\n",
663 req
.userspace_ver
, NES_ABI_USERSPACE_VER
);
664 return ERR_PTR(-EINVAL
);
668 memset(&uresp
, 0, sizeof uresp
);
670 uresp
.max_qps
= nesibdev
->max_qp
;
671 uresp
.max_pds
= nesibdev
->max_pd
;
672 uresp
.wq_size
= nesdev
->nesadapter
->max_qp_wr
* 2;
673 uresp
.virtwq
= nesadapter
->virtwq
;
674 uresp
.kernel_ver
= NES_ABI_KERNEL_VER
;
676 nes_ucontext
= kzalloc(sizeof *nes_ucontext
, GFP_KERNEL
);
678 return ERR_PTR(-ENOMEM
);
680 nes_ucontext
->nesdev
= nesdev
;
681 nes_ucontext
->mmap_wq_offset
= uresp
.max_pds
;
682 nes_ucontext
->mmap_cq_offset
= nes_ucontext
->mmap_wq_offset
+
683 ((sizeof(struct nes_hw_qp_wqe
) * uresp
.max_qps
* 2) + PAGE_SIZE
-1) /
687 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
689 return ERR_PTR(-EFAULT
);
692 INIT_LIST_HEAD(&nes_ucontext
->cq_reg_mem_list
);
693 INIT_LIST_HEAD(&nes_ucontext
->qp_reg_mem_list
);
694 atomic_set(&nes_ucontext
->usecnt
, 1);
695 return &nes_ucontext
->ibucontext
;
700 * nes_dealloc_ucontext
702 static int nes_dealloc_ucontext(struct ib_ucontext
*context
)
704 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */
705 /* struct nes_device *nesdev = nesvnic->nesdev; */
706 struct nes_ucontext
*nes_ucontext
= to_nesucontext(context
);
708 if (!atomic_dec_and_test(&nes_ucontext
->usecnt
))
718 static int nes_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
721 struct nes_vnic
*nesvnic
= to_nesvnic(context
->device
);
722 struct nes_device
*nesdev
= nesvnic
->nesdev
;
723 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */
724 struct nes_ucontext
*nes_ucontext
;
725 struct nes_qp
*nesqp
;
727 nes_ucontext
= to_nesucontext(context
);
730 if (vma
->vm_pgoff
>= nes_ucontext
->mmap_wq_offset
) {
731 index
= (vma
->vm_pgoff
- nes_ucontext
->mmap_wq_offset
) * PAGE_SIZE
;
732 index
/= ((sizeof(struct nes_hw_qp_wqe
) * nesdev
->nesadapter
->max_qp_wr
* 2) +
733 PAGE_SIZE
-1) & (~(PAGE_SIZE
-1));
734 if (!test_bit(index
, nes_ucontext
->allocated_wqs
)) {
735 nes_debug(NES_DBG_MMAP
, "wq %lu not allocated\n", index
);
738 nesqp
= nes_ucontext
->mmap_nesqp
[index
];
740 nes_debug(NES_DBG_MMAP
, "wq %lu has a NULL QP base.\n", index
);
743 if (remap_pfn_range(vma
, vma
->vm_start
,
744 virt_to_phys(nesqp
->hwqp
.sq_vbase
) >> PAGE_SHIFT
,
745 vma
->vm_end
- vma
->vm_start
,
746 vma
->vm_page_prot
)) {
747 nes_debug(NES_DBG_MMAP
, "remap_pfn_range failed.\n");
750 vma
->vm_private_data
= nesqp
;
753 index
= vma
->vm_pgoff
;
754 if (!test_bit(index
, nes_ucontext
->allocated_doorbells
))
757 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
758 if (io_remap_pfn_range(vma
, vma
->vm_start
,
759 (nesdev
->doorbell_start
+
760 ((nes_ucontext
->mmap_db_index
[index
] - nesdev
->base_doorbell_index
) * 4096))
761 >> PAGE_SHIFT
, PAGE_SIZE
, vma
->vm_page_prot
))
763 vma
->vm_private_data
= nes_ucontext
;
774 static struct ib_pd
*nes_alloc_pd(struct ib_device
*ibdev
,
775 struct ib_ucontext
*context
, struct ib_udata
*udata
)
777 struct nes_pd
*nespd
;
778 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
779 struct nes_device
*nesdev
= nesvnic
->nesdev
;
780 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
781 struct nes_ucontext
*nesucontext
;
782 struct nes_alloc_pd_resp uresp
;
786 nes_debug(NES_DBG_PD
, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
787 nesvnic
, nesdev
->netdev
[0], nesdev
->netdev
[0]->name
, ibdev
, context
,
788 atomic_read(&nesvnic
->netdev
->refcnt
));
790 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_pds
,
791 nesadapter
->max_pd
, &pd_num
, &nesadapter
->next_pd
);
796 nespd
= kzalloc(sizeof (struct nes_pd
), GFP_KERNEL
);
798 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
799 return ERR_PTR(-ENOMEM
);
802 nes_debug(NES_DBG_PD
, "Allocating PD (%p) for ib device %s\n",
803 nespd
, nesvnic
->nesibdev
->ibdev
.name
);
805 nespd
->pd_id
= (pd_num
<< (PAGE_SHIFT
-12)) + nesadapter
->base_pd
;
808 nesucontext
= to_nesucontext(context
);
809 nespd
->mmap_db_index
= find_next_zero_bit(nesucontext
->allocated_doorbells
,
810 NES_MAX_USER_DB_REGIONS
, nesucontext
->first_free_db
);
811 nes_debug(NES_DBG_PD
, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
812 nespd
->mmap_db_index
, nespd
->pd_id
);
813 if (nespd
->mmap_db_index
>= NES_MAX_USER_DB_REGIONS
) {
814 nes_debug(NES_DBG_PD
, "mmap_db_index > MAX\n");
815 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
817 return ERR_PTR(-ENOMEM
);
820 uresp
.pd_id
= nespd
->pd_id
;
821 uresp
.mmap_db_index
= nespd
->mmap_db_index
;
822 if (ib_copy_to_udata(udata
, &uresp
, sizeof (struct nes_alloc_pd_resp
))) {
823 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
825 return ERR_PTR(-EFAULT
);
828 set_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
829 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = nespd
->pd_id
;
830 nesucontext
->first_free_db
= nespd
->mmap_db_index
+ 1;
833 nes_debug(NES_DBG_PD
, "PD%u structure located @%p.\n", nespd
->pd_id
, nespd
);
841 static int nes_dealloc_pd(struct ib_pd
*ibpd
)
843 struct nes_ucontext
*nesucontext
;
844 struct nes_pd
*nespd
= to_nespd(ibpd
);
845 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
846 struct nes_device
*nesdev
= nesvnic
->nesdev
;
847 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
849 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
850 nesucontext
= to_nesucontext(ibpd
->uobject
->context
);
851 nes_debug(NES_DBG_PD
, "Clearing bit %u from allocated doorbells\n",
852 nespd
->mmap_db_index
);
853 clear_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
854 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = 0;
855 if (nesucontext
->first_free_db
> nespd
->mmap_db_index
) {
856 nesucontext
->first_free_db
= nespd
->mmap_db_index
;
860 nes_debug(NES_DBG_PD
, "Deallocating PD%u structure located @%p.\n",
861 nespd
->pd_id
, nespd
);
862 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
,
863 (nespd
->pd_id
-nesadapter
->base_pd
)>>(PAGE_SHIFT
-12));
873 static struct ib_ah
*nes_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
)
875 return ERR_PTR(-ENOSYS
);
882 static int nes_destroy_ah(struct ib_ah
*ah
)
889 * nes_get_encoded_size
891 static inline u8
nes_get_encoded_size(int *size
)
897 } else if (*size
<= 128) {
900 } else if (*size
<= 512) {
904 return (encoded_size
);
912 static int nes_setup_virt_qp(struct nes_qp
*nesqp
, struct nes_pbl
*nespbl
,
913 struct nes_vnic
*nesvnic
, int sq_size
, int rq_size
)
920 struct nes_device
*nesdev
= nesvnic
->nesdev
;
921 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
926 pbl_entries
= nespbl
->pbl_size
>> 3;
927 nes_debug(NES_DBG_QP
, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
928 nespbl
->pbl_size
, pbl_entries
,
929 (void *)nespbl
->pbl_vbase
,
930 (unsigned long) nespbl
->pbl_pbase
);
931 pbl
= (__le64
*) nespbl
->pbl_vbase
; /* points to first pbl entry */
932 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
933 /* the first pbl to be fro the rq_vbase... */
934 rq_pbl_entries
= (rq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
935 sq_pbl_entries
= (sq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
936 nesqp
->hwqp
.sq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
938 nes_debug(NES_DBG_QP
, "QP nespbl->page is NULL \n");
943 nesqp
->hwqp
.sq_vbase
= kmap(nespbl
->page
);
944 nesqp
->page
= nespbl
->page
;
945 if (!nesqp
->hwqp
.sq_vbase
) {
946 nes_debug(NES_DBG_QP
, "QP sq_vbase kmap failed\n");
951 /* Now to get to sq.. we need to calculate how many */
952 /* PBL entries were used by the rq.. */
953 pbl
+= sq_pbl_entries
;
954 nesqp
->hwqp
.rq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
955 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
956 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
958 nes_debug(NES_DBG_QP
, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
959 nesqp
->hwqp
.sq_vbase
, (unsigned long) nesqp
->hwqp
.sq_pbase
,
960 nesqp
->hwqp
.rq_vbase
, (unsigned long) nesqp
->hwqp
.rq_pbase
);
961 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
962 if (!nesadapter
->free_256pbl
) {
963 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
965 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
970 nesadapter
->free_256pbl
--;
971 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
973 nesqp
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 256, &nesqp
->pbl_pbase
);
974 pblbuffer
= nesqp
->pbl_vbase
;
975 if (!nesqp
->pbl_vbase
) {
976 /* memory allocated during nes_reg_user_mr() */
977 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
980 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
981 nesadapter
->free_256pbl
++;
982 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
986 memset(nesqp
->pbl_vbase
, 0, 256);
987 /* fill in the page address in the pbl buffer.. */
988 tpbl
= pblbuffer
+ 16;
989 pbl
= (__le64
*)nespbl
->pbl_vbase
;
990 while (sq_pbl_entries
--)
993 while (rq_pbl_entries
--)
996 /* done with memory allocated during nes_reg_user_mr() */
997 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1001 nesqp
->qp_mem_size
=
1002 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) + 256; /* this is Q2 */
1003 /* Round up to a multiple of a page */
1004 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
1005 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
1007 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1008 &nesqp
->hwqp
.q2_pbase
);
1011 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
1012 nesqp
->pbl_vbase
= NULL
;
1013 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1014 nesadapter
->free_256pbl
++;
1015 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1016 kunmap(nesqp
->page
);
1019 nesqp
->sq_kmapped
= 1;
1020 nesqp
->hwqp
.q2_vbase
= mem
;
1022 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
1023 nesqp
->nesqp_context
= mem
;
1024 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
1025 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
1034 static int nes_setup_mmap_qp(struct nes_qp
*nesqp
, struct nes_vnic
*nesvnic
,
1035 int sq_size
, int rq_size
)
1038 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1040 nesqp
->qp_mem_size
= (sizeof(struct nes_hw_qp_wqe
) * sq_size
) +
1041 (sizeof(struct nes_hw_qp_wqe
) * rq_size
) +
1042 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) +
1043 256; /* this is Q2 */
1044 /* Round up to a multiple of a page */
1045 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
1046 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
1048 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1049 &nesqp
->hwqp
.sq_pbase
);
1052 nes_debug(NES_DBG_QP
, "PCI consistent memory for "
1053 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
1054 mem
, (unsigned long)nesqp
->hwqp
.sq_pbase
, nesqp
->qp_mem_size
);
1056 memset(mem
, 0, nesqp
->qp_mem_size
);
1058 nesqp
->hwqp
.sq_vbase
= mem
;
1059 mem
+= sizeof(struct nes_hw_qp_wqe
) * sq_size
;
1061 nesqp
->hwqp
.rq_vbase
= mem
;
1062 nesqp
->hwqp
.rq_pbase
= nesqp
->hwqp
.sq_pbase
+
1063 sizeof(struct nes_hw_qp_wqe
) * sq_size
;
1064 mem
+= sizeof(struct nes_hw_qp_wqe
) * rq_size
;
1066 nesqp
->hwqp
.q2_vbase
= mem
;
1067 nesqp
->hwqp
.q2_pbase
= nesqp
->hwqp
.rq_pbase
+
1068 sizeof(struct nes_hw_qp_wqe
) * rq_size
;
1070 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
1072 nesqp
->nesqp_context
= mem
;
1073 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
1074 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
1080 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
1082 static inline void nes_free_qp_mem(struct nes_device
*nesdev
,
1083 struct nes_qp
*nesqp
, int virt_wqs
)
1085 unsigned long flags
;
1086 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1088 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
1089 nesqp
->hwqp
.sq_vbase
, nesqp
->hwqp
.sq_pbase
);
1091 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1092 nesadapter
->free_256pbl
++;
1093 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1094 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
, nesqp
->hwqp
.q2_vbase
, nesqp
->hwqp
.q2_pbase
);
1095 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
1096 nesqp
->pbl_vbase
= NULL
;
1097 if (nesqp
->sq_kmapped
) {
1098 nesqp
->sq_kmapped
= 0;
1099 kunmap(nesqp
->page
);
1108 static struct ib_qp
*nes_create_qp(struct ib_pd
*ibpd
,
1109 struct ib_qp_init_attr
*init_attr
, struct ib_udata
*udata
)
1113 struct nes_pd
*nespd
= to_nespd(ibpd
);
1114 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
1115 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1116 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1117 struct nes_qp
*nesqp
;
1118 struct nes_cq
*nescq
;
1119 struct nes_ucontext
*nes_ucontext
;
1120 struct nes_hw_cqp_wqe
*cqp_wqe
;
1121 struct nes_cqp_request
*cqp_request
;
1122 struct nes_create_qp_req req
;
1123 struct nes_create_qp_resp uresp
;
1124 struct nes_pbl
*nespbl
= NULL
;
1127 /* u32 counter = 0; */
1129 unsigned long flags
;
1139 if (init_attr
->create_flags
)
1140 return ERR_PTR(-EINVAL
);
1142 atomic_inc(&qps_created
);
1143 switch (init_attr
->qp_type
) {
1145 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) {
1146 init_attr
->cap
.max_inline_data
= 0;
1148 init_attr
->cap
.max_inline_data
= 64;
1150 sq_size
= init_attr
->cap
.max_send_wr
;
1151 rq_size
= init_attr
->cap
.max_recv_wr
;
1153 /* check if the encoded sizes are OK or not... */
1154 sq_encoded_size
= nes_get_encoded_size(&sq_size
);
1155 rq_encoded_size
= nes_get_encoded_size(&rq_size
);
1157 if ((!sq_encoded_size
) || (!rq_encoded_size
)) {
1158 nes_debug(NES_DBG_QP
, "ERROR bad rq (%u) or sq (%u) size\n",
1160 return ERR_PTR(-EINVAL
);
1163 init_attr
->cap
.max_send_wr
= sq_size
-2;
1164 init_attr
->cap
.max_recv_wr
= rq_size
-1;
1165 nes_debug(NES_DBG_QP
, "RQ size=%u, SQ Size=%u\n", rq_size
, sq_size
);
1167 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_qps
,
1168 nesadapter
->max_qp
, &qp_num
, &nesadapter
->next_qp
);
1170 return ERR_PTR(ret
);
1173 /* Need 512 (actually now 1024) byte alignment on this structure */
1174 mem
= kzalloc(sizeof(*nesqp
)+NES_SW_CONTEXT_ALIGN
-1, GFP_KERNEL
);
1176 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1177 nes_debug(NES_DBG_QP
, "Unable to allocate QP\n");
1178 return ERR_PTR(-ENOMEM
);
1180 u64nesqp
= (unsigned long)mem
;
1181 u64nesqp
+= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1182 u64temp
= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1183 u64nesqp
&= ~u64temp
;
1184 nesqp
= (struct nes_qp
*)(unsigned long)u64nesqp
;
1185 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n",
1186 nesqp, mem, NES_SW_CONTEXT_ALIGN); */
1187 nesqp
->allocated_buffer
= mem
;
1190 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_create_qp_req
))) {
1191 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1192 kfree(nesqp
->allocated_buffer
);
1193 nes_debug(NES_DBG_QP
, "ib_copy_from_udata() Failed \n");
1196 if (req
.user_wqe_buffers
) {
1199 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
1200 nesqp
->user_mode
= 1;
1201 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1204 list_for_each_entry(nespbl
, &nes_ucontext
->qp_reg_mem_list
, list
) {
1205 if (nespbl
->user_base
== (unsigned long )req
.user_wqe_buffers
) {
1206 list_del(&nespbl
->list
);
1208 nes_debug(NES_DBG_QP
, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
1209 nespbl
, nespbl
->user_base
);
1214 nes_debug(NES_DBG_QP
, "Didn't Find PBL for virtual QP. address = %llx.\n",
1215 (long long unsigned int)req
.user_wqe_buffers
);
1216 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1217 kfree(nesqp
->allocated_buffer
);
1218 return ERR_PTR(-EFAULT
);
1222 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1223 nesqp
->mmap_sq_db_index
=
1224 find_next_zero_bit(nes_ucontext
->allocated_wqs
,
1225 NES_MAX_USER_WQ_REGIONS
, nes_ucontext
->first_free_wq
);
1226 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
1227 nespd->mmap_db_index); */
1228 if (nesqp
->mmap_sq_db_index
>= NES_MAX_USER_WQ_REGIONS
) {
1229 nes_debug(NES_DBG_QP
,
1230 "db index > max user regions, failing create QP\n");
1231 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1233 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1237 kfree(nesqp
->allocated_buffer
);
1238 return ERR_PTR(-ENOMEM
);
1240 set_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1241 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = nesqp
;
1242 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
+ 1;
1244 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1245 kfree(nesqp
->allocated_buffer
);
1246 return ERR_PTR(-EFAULT
);
1249 err
= (!virt_wqs
) ? nes_setup_mmap_qp(nesqp
, nesvnic
, sq_size
, rq_size
) :
1250 nes_setup_virt_qp(nesqp
, nespbl
, nesvnic
, sq_size
, rq_size
);
1252 nes_debug(NES_DBG_QP
,
1253 "error geting qp mem code = %d\n", err
);
1254 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1255 kfree(nesqp
->allocated_buffer
);
1256 return ERR_PTR(-ENOMEM
);
1259 nesqp
->hwqp
.sq_size
= sq_size
;
1260 nesqp
->hwqp
.sq_encoded_size
= sq_encoded_size
;
1261 nesqp
->hwqp
.sq_head
= 1;
1262 nesqp
->hwqp
.rq_size
= rq_size
;
1263 nesqp
->hwqp
.rq_encoded_size
= rq_encoded_size
;
1264 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n",
1265 (void *)nesqp->nesqp_context_pbase);
1267 nesqp
->hwqp
.qp_id
= qp_num
;
1268 nesqp
->ibqp
.qp_num
= nesqp
->hwqp
.qp_id
;
1269 nesqp
->nespd
= nespd
;
1271 nescq
= to_nescq(init_attr
->send_cq
);
1272 nesqp
->nesscq
= nescq
;
1273 nescq
= to_nescq(init_attr
->recv_cq
);
1274 nesqp
->nesrcq
= nescq
;
1276 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) <<
1277 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT
);
1278 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.rq_encoded_size
<<
1279 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT
);
1280 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.sq_encoded_size
<<
1281 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT
);
1283 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN
);
1284 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN
);
1286 nesqp
->nesqp_context
->cqs
= cpu_to_le32(nesqp
->nesscq
->hw_cq
.cq_number
+
1287 ((u32
)nesqp
->nesrcq
->hw_cq
.cq_number
<< 16));
1288 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1289 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1290 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1294 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1295 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1296 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1297 u64temp
= (u64
)nesqp
->hwqp
.rq_pbase
;
1298 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1299 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1301 u64temp
= (u64
)nesqp
->pbl_pbase
;
1302 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1303 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1306 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n",
1307 nesvnic->next_qp_nic_index,
1308 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */
1309 spin_lock_irqsave(&nesdev
->cqp
.lock
, flags
);
1310 nesqp
->nesqp_context
->misc2
|= cpu_to_le32(
1311 (u32
)nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] <<
1312 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT
);
1313 nesvnic
->next_qp_nic_index
++;
1314 if ((nesvnic
->next_qp_nic_index
> 3) ||
1315 (nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] == 0xf)) {
1316 nesvnic
->next_qp_nic_index
= 0;
1318 spin_unlock_irqrestore(&nesdev
->cqp
.lock
, flags
);
1320 nesqp
->nesqp_context
->pd_index_wscale
|= cpu_to_le32((u32
)nesqp
->nespd
->pd_id
<< 16);
1321 u64temp
= (u64
)nesqp
->hwqp
.q2_pbase
;
1322 nesqp
->nesqp_context
->q2_addr_low
= cpu_to_le32((u32
)u64temp
);
1323 nesqp
->nesqp_context
->q2_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1324 nesqp
->nesqp_context
->aeq_token_low
= cpu_to_le32((u32
)((unsigned long)(nesqp
)));
1325 nesqp
->nesqp_context
->aeq_token_high
= cpu_to_le32((u32
)(upper_32_bits((unsigned long)(nesqp
))));
1326 nesqp
->nesqp_context
->ird_ord_sizes
= cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM
|
1327 NES_QPCONTEXT_ORDIRD_AAH
|
1328 ((((u32
)nesadapter
->max_irrq_wr
) <<
1329 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT
) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK
));
1330 if (disable_mpa_crc
) {
1331 nes_debug(NES_DBG_QP
, "Disabling MPA crc checking due to module option.\n");
1332 nesqp
->nesqp_context
->ird_ord_sizes
|= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC
);
1337 cqp_request
= nes_get_cqp_request(nesdev
);
1338 if (cqp_request
== NULL
) {
1339 nes_debug(NES_DBG_QP
, "Failed to get a cqp_request\n");
1340 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1341 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1342 kfree(nesqp
->allocated_buffer
);
1343 return ERR_PTR(-ENOMEM
);
1345 cqp_request
->waiting
= 1;
1346 cqp_wqe
= &cqp_request
->cqp_wqe
;
1349 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
|
1350 NES_CQP_QP_IWARP_STATE_IDLE
;
1352 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
| NES_CQP_QP_VIRT_WQS
|
1353 NES_CQP_QP_IWARP_STATE_IDLE
;
1355 opcode
|= NES_CQP_QP_CQS_VALID
;
1356 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1357 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1358 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
1360 u64temp
= (u64
)nesqp
->nesqp_context_pbase
;
1361 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, u64temp
);
1363 atomic_set(&cqp_request
->refcount
, 2);
1364 nes_post_cqp_request(nesdev
, cqp_request
);
1367 nes_debug(NES_DBG_QP
, "Waiting for create iWARP QP%u to complete.\n",
1369 ret
= wait_event_timeout(cqp_request
->waitq
,
1370 (cqp_request
->request_done
!= 0), NES_EVENT_TIMEOUT
);
1371 nes_debug(NES_DBG_QP
, "Create iwarp QP%u completed, wait_event_timeout ret=%u,"
1372 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u,"
1373 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1374 nesqp
->hwqp
.qp_id
, ret
, nesdev
->cqp
.sq_head
, nesdev
->cqp
.sq_tail
,
1375 cqp_request
->major_code
, cqp_request
->minor_code
);
1376 if ((!ret
) || (cqp_request
->major_code
)) {
1377 nes_put_cqp_request(nesdev
, cqp_request
);
1378 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1379 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1380 kfree(nesqp
->allocated_buffer
);
1382 return ERR_PTR(-ETIME
);
1384 return ERR_PTR(-EIO
);
1388 nes_put_cqp_request(nesdev
, cqp_request
);
1390 if (ibpd
->uobject
) {
1391 uresp
.mmap_sq_db_index
= nesqp
->mmap_sq_db_index
;
1392 uresp
.actual_sq_size
= sq_size
;
1393 uresp
.actual_rq_size
= rq_size
;
1394 uresp
.qp_id
= nesqp
->hwqp
.qp_id
;
1395 uresp
.nes_drv_opt
= nes_drv_opt
;
1396 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
1397 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1398 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1399 kfree(nesqp
->allocated_buffer
);
1400 return ERR_PTR(-EFAULT
);
1404 nes_debug(NES_DBG_QP
, "QP%u structure located @%p.Size = %u.\n",
1405 nesqp
->hwqp
.qp_id
, nesqp
, (u32
)sizeof(*nesqp
));
1406 spin_lock_init(&nesqp
->lock
);
1407 nes_add_ref(&nesqp
->ibqp
);
1410 nes_debug(NES_DBG_QP
, "Invalid QP type: %d\n", init_attr
->qp_type
);
1411 return ERR_PTR(-EINVAL
);
1414 nesqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
);
1416 /* update the QP table */
1417 nesdev
->nesadapter
->qp_table
[nesqp
->hwqp
.qp_id
-NES_FIRST_QPN
] = nesqp
;
1418 nes_debug(NES_DBG_QP
, "netdev refcnt=%u\n",
1419 atomic_read(&nesvnic
->netdev
->refcnt
));
1421 return &nesqp
->ibqp
;
1428 static void nes_clean_cq(struct nes_qp
*nesqp
, struct nes_cq
*nescq
)
1434 unsigned long flags
= 0;
1436 spin_lock_irqsave(&nescq
->lock
, flags
);
1438 cq_head
= nescq
->hw_cq
.cq_head
;
1439 while (le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_VALID
) {
1441 lo
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
1442 hi
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
]);
1443 u64temp
= (((u64
)hi
) << 32) | ((u64
)lo
);
1444 u64temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
1445 if (u64temp
== (u64
)(unsigned long)nesqp
) {
1446 /* Zero the context value so cqe will be ignored */
1447 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] = 0;
1448 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
] = 0;
1451 if (++cq_head
>= nescq
->hw_cq
.cq_size
)
1455 spin_unlock_irqrestore(&nescq
->lock
, flags
);
1462 static int nes_destroy_qp(struct ib_qp
*ibqp
)
1464 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
1465 struct nes_ucontext
*nes_ucontext
;
1466 struct ib_qp_attr attr
;
1467 struct iw_cm_id
*cm_id
;
1468 struct iw_cm_event cm_event
;
1471 atomic_inc(&sw_qps_destroyed
);
1472 nesqp
->destroyed
= 1;
1474 /* Blow away the connection if it exists. */
1475 if (nesqp
->ibqp_state
>= IB_QPS_INIT
&& nesqp
->ibqp_state
<= IB_QPS_RTS
) {
1476 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */
1477 attr
.qp_state
= IB_QPS_ERR
;
1478 nes_modify_qp(&nesqp
->ibqp
, &attr
, IB_QP_STATE
, NULL
);
1481 if (((nesqp
->ibqp_state
== IB_QPS_INIT
) ||
1482 (nesqp
->ibqp_state
== IB_QPS_RTR
)) && (nesqp
->cm_id
)) {
1483 cm_id
= nesqp
->cm_id
;
1484 cm_event
.event
= IW_CM_EVENT_CONNECT_REPLY
;
1485 cm_event
.status
= IW_CM_EVENT_STATUS_TIMEOUT
;
1486 cm_event
.local_addr
= cm_id
->local_addr
;
1487 cm_event
.remote_addr
= cm_id
->remote_addr
;
1488 cm_event
.private_data
= NULL
;
1489 cm_event
.private_data_len
= 0;
1491 nes_debug(NES_DBG_QP
, "Generating a CM Timeout Event for "
1492 "QP%u. cm_id = %p, refcount = %u. \n",
1493 nesqp
->hwqp
.qp_id
, cm_id
, atomic_read(&nesqp
->refcount
));
1495 cm_id
->rem_ref(cm_id
);
1496 ret
= cm_id
->event_handler(cm_id
, &cm_event
);
1498 nes_debug(NES_DBG_QP
, "OFA CM event_handler returned, ret=%d\n", ret
);
1501 if (nesqp
->user_mode
) {
1502 if ((ibqp
->uobject
)&&(ibqp
->uobject
->context
)) {
1503 nes_ucontext
= to_nesucontext(ibqp
->uobject
->context
);
1504 clear_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1505 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = NULL
;
1506 if (nes_ucontext
->first_free_wq
> nesqp
->mmap_sq_db_index
) {
1507 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
;
1510 if (nesqp
->pbl_pbase
&& nesqp
->sq_kmapped
) {
1511 nesqp
->sq_kmapped
= 0;
1512 kunmap(nesqp
->page
);
1515 /* Clean any pending completions from the cq(s) */
1517 nes_clean_cq(nesqp
, nesqp
->nesscq
);
1519 if ((nesqp
->nesrcq
) && (nesqp
->nesrcq
!= nesqp
->nesscq
))
1520 nes_clean_cq(nesqp
, nesqp
->nesrcq
);
1523 nes_rem_ref(&nesqp
->ibqp
);
1531 static struct ib_cq
*nes_create_cq(struct ib_device
*ibdev
, int entries
,
1533 struct ib_ucontext
*context
, struct ib_udata
*udata
)
1536 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
1537 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1538 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1539 struct nes_cq
*nescq
;
1540 struct nes_ucontext
*nes_ucontext
= NULL
;
1541 struct nes_cqp_request
*cqp_request
;
1543 struct nes_hw_cqp_wqe
*cqp_wqe
;
1544 struct nes_pbl
*nespbl
= NULL
;
1545 struct nes_create_cq_req req
;
1546 struct nes_create_cq_resp resp
;
1549 u32 pbl_entries
= 1;
1551 unsigned long flags
;
1554 if (entries
> nesadapter
->max_cqe
)
1555 return ERR_PTR(-EINVAL
);
1557 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_cqs
,
1558 nesadapter
->max_cq
, &cq_num
, &nesadapter
->next_cq
);
1560 return ERR_PTR(err
);
1563 nescq
= kzalloc(sizeof(struct nes_cq
), GFP_KERNEL
);
1565 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1566 nes_debug(NES_DBG_CQ
, "Unable to allocate nes_cq struct\n");
1567 return ERR_PTR(-ENOMEM
);
1570 nescq
->hw_cq
.cq_size
= max(entries
+ 1, 5);
1571 nescq
->hw_cq
.cq_number
= cq_num
;
1572 nescq
->ibcq
.cqe
= nescq
->hw_cq
.cq_size
- 1;
1576 nes_ucontext
= to_nesucontext(context
);
1577 if (ib_copy_from_udata(&req
, udata
, sizeof (struct nes_create_cq_req
))) {
1578 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1580 return ERR_PTR(-EFAULT
);
1582 nesvnic
->mcrq_ucontext
= nes_ucontext
;
1583 nes_ucontext
->mcrqf
= req
.mcrqf
;
1584 if (nes_ucontext
->mcrqf
) {
1585 if (nes_ucontext
->mcrqf
& 0x80000000)
1586 nescq
->hw_cq
.cq_number
= nesvnic
->nic
.qp_id
+ 28 + 2 * ((nes_ucontext
->mcrqf
& 0xf) - 1);
1587 else if (nes_ucontext
->mcrqf
& 0x40000000)
1588 nescq
->hw_cq
.cq_number
= nes_ucontext
->mcrqf
& 0xffff;
1590 nescq
->hw_cq
.cq_number
= nesvnic
->mcrq_qp_id
+ nes_ucontext
->mcrqf
-1;
1591 nescq
->mcrqf
= nes_ucontext
->mcrqf
;
1592 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1594 nes_debug(NES_DBG_CQ
, "CQ Virtual Address = %08lX, size = %u.\n",
1595 (unsigned long)req
.user_cq_buffer
, entries
);
1597 list_for_each_entry(nespbl
, &nes_ucontext
->cq_reg_mem_list
, list
) {
1598 if (nespbl
->user_base
== (unsigned long )req
.user_cq_buffer
) {
1599 list_del(&nespbl
->list
);
1601 nes_debug(NES_DBG_CQ
, "Found PBL for virtual CQ. nespbl=%p.\n",
1607 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1609 return ERR_PTR(-EFAULT
);
1612 pbl_entries
= nespbl
->pbl_size
>> 3;
1613 nescq
->cq_mem_size
= 0;
1615 nescq
->cq_mem_size
= nescq
->hw_cq
.cq_size
* sizeof(struct nes_hw_cqe
);
1616 nes_debug(NES_DBG_CQ
, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n",
1617 entries
, nescq
->cq_mem_size
, nescq
->hw_cq
.cq_number
);
1619 /* allocate the physical buffer space */
1620 mem
= pci_alloc_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1621 &nescq
->hw_cq
.cq_pbase
);
1623 printk(KERN_ERR PFX
"Unable to allocate pci memory for cq\n");
1624 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1626 return ERR_PTR(-ENOMEM
);
1629 memset(mem
, 0, nescq
->cq_mem_size
);
1630 nescq
->hw_cq
.cq_vbase
= mem
;
1631 nescq
->hw_cq
.cq_head
= 0;
1632 nes_debug(NES_DBG_CQ
, "CQ%u virtual address @ %p, phys = 0x%08X\n",
1633 nescq
->hw_cq
.cq_number
, nescq
->hw_cq
.cq_vbase
,
1634 (u32
)nescq
->hw_cq
.cq_pbase
);
1637 nescq
->hw_cq
.ce_handler
= nes_iwarp_ce_handler
;
1638 spin_lock_init(&nescq
->lock
);
1640 /* send CreateCQ request to CQP */
1641 cqp_request
= nes_get_cqp_request(nesdev
);
1642 if (cqp_request
== NULL
) {
1643 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1645 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1646 nescq
->hw_cq
.cq_pbase
);
1648 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1649 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1653 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1655 return ERR_PTR(-ENOMEM
);
1657 cqp_request
->waiting
= 1;
1658 cqp_wqe
= &cqp_request
->cqp_wqe
;
1660 opcode
= NES_CQP_CREATE_CQ
| NES_CQP_CQ_CEQ_VALID
|
1661 NES_CQP_CQ_CHK_OVERFLOW
|
1662 NES_CQP_CQ_CEQE_MASK
| ((u32
)nescq
->hw_cq
.cq_size
<< 16);
1664 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1666 if (pbl_entries
!= 1) {
1667 if (pbl_entries
> 32) {
1669 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 4k PBL\n", pbl_entries
);
1670 if (nesadapter
->free_4kpbl
== 0) {
1671 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1672 nes_free_cqp_request(nesdev
, cqp_request
);
1674 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1675 nescq
->hw_cq
.cq_pbase
);
1677 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1678 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1681 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1683 return ERR_PTR(-ENOMEM
);
1685 opcode
|= (NES_CQP_CQ_VIRT
| NES_CQP_CQ_4KB_CHUNK
);
1686 nescq
->virtual_cq
= 2;
1687 nesadapter
->free_4kpbl
--;
1690 /* use 256 byte pbl */
1691 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries
);
1692 if (nesadapter
->free_256pbl
== 0) {
1693 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1694 nes_free_cqp_request(nesdev
, cqp_request
);
1696 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1697 nescq
->hw_cq
.cq_pbase
);
1699 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1700 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1703 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1705 return ERR_PTR(-ENOMEM
);
1707 opcode
|= NES_CQP_CQ_VIRT
;
1708 nescq
->virtual_cq
= 1;
1709 nesadapter
->free_256pbl
--;
1714 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1716 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1717 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1718 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1719 (nescq
->hw_cq
.cq_number
| ((u32
)nesdev
->ceq_index
<< 16)));
1722 if (pbl_entries
!= 1)
1723 u64temp
= (u64
)nespbl
->pbl_pbase
;
1725 u64temp
= le64_to_cpu(nespbl
->pbl_vbase
[0]);
1726 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
,
1727 nes_ucontext
->mmap_db_index
[0]);
1729 u64temp
= (u64
)nescq
->hw_cq
.cq_pbase
;
1730 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
] = 0;
1732 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_PBL_LOW_IDX
, u64temp
);
1733 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] = 0;
1734 u64temp
= (u64
)(unsigned long)&nescq
->hw_cq
;
1735 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX
] =
1736 cpu_to_le32((u32
)(u64temp
>> 1));
1737 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] =
1738 cpu_to_le32(((u32
)((u64temp
) >> 33)) & 0x7FFFFFFF);
1740 atomic_set(&cqp_request
->refcount
, 2);
1741 nes_post_cqp_request(nesdev
, cqp_request
);
1744 nes_debug(NES_DBG_CQ
, "Waiting for create iWARP CQ%u to complete.\n",
1745 nescq
->hw_cq
.cq_number
);
1746 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1747 NES_EVENT_TIMEOUT
* 2);
1748 nes_debug(NES_DBG_CQ
, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n",
1749 nescq
->hw_cq
.cq_number
, ret
);
1750 if ((!ret
) || (cqp_request
->major_code
)) {
1751 nes_put_cqp_request(nesdev
, cqp_request
);
1753 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1754 nescq
->hw_cq
.cq_pbase
);
1756 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1757 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1760 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1762 return ERR_PTR(-EIO
);
1764 nes_put_cqp_request(nesdev
, cqp_request
);
1767 /* free the nespbl */
1768 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1771 resp
.cq_id
= nescq
->hw_cq
.cq_number
;
1772 resp
.cq_size
= nescq
->hw_cq
.cq_size
;
1773 resp
.mmap_db_index
= 0;
1774 if (ib_copy_to_udata(udata
, &resp
, sizeof resp
)) {
1775 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1777 return ERR_PTR(-EFAULT
);
1781 return &nescq
->ibcq
;
1788 static int nes_destroy_cq(struct ib_cq
*ib_cq
)
1790 struct nes_cq
*nescq
;
1791 struct nes_device
*nesdev
;
1792 struct nes_vnic
*nesvnic
;
1793 struct nes_adapter
*nesadapter
;
1794 struct nes_hw_cqp_wqe
*cqp_wqe
;
1795 struct nes_cqp_request
*cqp_request
;
1796 unsigned long flags
;
1803 nescq
= to_nescq(ib_cq
);
1804 nesvnic
= to_nesvnic(ib_cq
->device
);
1805 nesdev
= nesvnic
->nesdev
;
1806 nesadapter
= nesdev
->nesadapter
;
1808 nes_debug(NES_DBG_CQ
, "Destroy CQ%u\n", nescq
->hw_cq
.cq_number
);
1810 /* Send DestroyCQ request to CQP */
1811 cqp_request
= nes_get_cqp_request(nesdev
);
1812 if (cqp_request
== NULL
) {
1813 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1816 cqp_request
->waiting
= 1;
1817 cqp_wqe
= &cqp_request
->cqp_wqe
;
1818 opcode
= NES_CQP_DESTROY_CQ
| (nescq
->hw_cq
.cq_size
<< 16);
1819 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1820 if (nescq
->virtual_cq
== 1) {
1821 nesadapter
->free_256pbl
++;
1822 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
) {
1823 printk(KERN_ERR PFX
"%s: free 256B PBLs(%u) has exceeded the max(%u)\n",
1824 __func__
, nesadapter
->free_256pbl
, nesadapter
->max_256pbl
);
1826 } else if (nescq
->virtual_cq
== 2) {
1827 nesadapter
->free_4kpbl
++;
1828 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
) {
1829 printk(KERN_ERR PFX
"%s: free 4K PBLs(%u) has exceeded the max(%u)\n",
1830 __func__
, nesadapter
->free_4kpbl
, nesadapter
->max_4kpbl
);
1832 opcode
|= NES_CQP_CQ_4KB_CHUNK
;
1835 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1837 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1838 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1839 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1840 (nescq
->hw_cq
.cq_number
| ((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) << 16)));
1842 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, nescq
->hw_cq
.cq_number
);
1844 atomic_set(&cqp_request
->refcount
, 2);
1845 nes_post_cqp_request(nesdev
, cqp_request
);
1848 nes_debug(NES_DBG_CQ
, "Waiting for destroy iWARP CQ%u to complete.\n",
1849 nescq
->hw_cq
.cq_number
);
1850 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1852 nes_debug(NES_DBG_CQ
, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u,"
1853 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1854 nescq
->hw_cq
.cq_number
, ret
, cqp_request
->major_code
,
1855 cqp_request
->minor_code
);
1857 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy timeout expired\n",
1858 nescq
->hw_cq
.cq_number
);
1860 } else if (cqp_request
->major_code
) {
1861 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy failed\n",
1862 nescq
->hw_cq
.cq_number
);
1867 nes_put_cqp_request(nesdev
, cqp_request
);
1869 if (nescq
->cq_mem_size
)
1870 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1871 nescq
->hw_cq
.cq_vbase
, nescq
->hw_cq
.cq_pbase
);
1880 static u32
root_256(struct nes_device
*nesdev
,
1881 struct nes_root_vpbl
*root_vpbl
,
1882 struct nes_root_vpbl
*new_root
,
1888 if (pbl_count_4k
== 1) {
1889 new_root
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
1890 512, &new_root
->pbl_pbase
);
1892 if (new_root
->pbl_vbase
== NULL
)
1895 leaf_pbl
= (u64
)root_vpbl
->pbl_pbase
;
1896 for (i
= 0; i
< 16; i
++) {
1897 new_root
->pbl_vbase
[i
].pa_low
=
1898 cpu_to_le32((u32
)leaf_pbl
);
1899 new_root
->pbl_vbase
[i
].pa_high
=
1900 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1904 for (i
= 3; i
>= 0; i
--) {
1906 root_vpbl
->pbl_vbase
[j
] = root_vpbl
->pbl_vbase
[i
];
1907 leaf_pbl
= le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_low
) +
1908 (((u64
)le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_high
))
1910 for (k
= 1; k
< 16; k
++) {
1912 root_vpbl
->pbl_vbase
[j
+ k
].pa_low
=
1913 cpu_to_le32((u32
)leaf_pbl
);
1914 root_vpbl
->pbl_vbase
[j
+ k
].pa_high
=
1915 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1927 static int nes_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
1928 u32 stag
, u64 region_length
, struct nes_root_vpbl
*root_vpbl
,
1929 dma_addr_t single_buffer
, u16 pbl_count_4k
,
1930 u16 residual_page_count_4k
, int acc
, u64
*iova_start
,
1931 u16
*actual_pbl_cnt
, u8
*used_4k_pbls
)
1933 struct nes_hw_cqp_wqe
*cqp_wqe
;
1934 struct nes_cqp_request
*cqp_request
;
1935 unsigned long flags
;
1937 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1939 u16 pbl_count_256
= 0;
1941 u8 use_256_pbls
= 0;
1943 u16 use_two_level
= (pbl_count_4k
> 1) ? 1 : 0;
1944 struct nes_root_vpbl new_root
= { 0, NULL
, NULL
};
1948 /* Register the region with the adapter */
1949 cqp_request
= nes_get_cqp_request(nesdev
);
1950 if (cqp_request
== NULL
) {
1951 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
1954 cqp_request
->waiting
= 1;
1955 cqp_wqe
= &cqp_request
->cqp_wqe
;
1958 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1960 pg_cnt
= ((pbl_count_4k
- 1) * 512) + residual_page_count_4k
;
1961 pbl_count_256
= (pg_cnt
+ 31) / 32;
1963 if (pbl_count_256
<= nesadapter
->free_256pbl
)
1965 else if (pbl_count_4k
<= nesadapter
->free_4kpbl
)
1967 } else if (pg_cnt
<= 2048) {
1968 if (((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) &&
1969 (nesadapter
->free_4kpbl
> (nesadapter
->max_4kpbl
>> 1))) {
1971 } else if ((pbl_count_256
+ 1) <= nesadapter
->free_256pbl
) {
1974 } else if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
1978 if ((pbl_count_4k
+ 1) <= nesadapter
->free_4kpbl
)
1983 pbl_count
= pbl_count_256
;
1984 nesadapter
->free_256pbl
-= pbl_count
+ use_two_level
;
1985 } else if (use_4k_pbls
) {
1986 pbl_count
= pbl_count_4k
;
1987 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
1989 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1990 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
1991 nes_free_cqp_request(nesdev
, cqp_request
);
1995 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1998 if (use_256_pbls
&& use_two_level
) {
1999 if (root_256(nesdev
, root_vpbl
, &new_root
, pbl_count_4k
) == 1) {
2000 if (new_root
.pbl_pbase
!= 0)
2001 root_vpbl
= &new_root
;
2003 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2004 nesadapter
->free_256pbl
+= pbl_count_256
+ use_two_level
;
2007 if (pbl_count_4k
== 1)
2009 pbl_count
= pbl_count_4k
;
2011 if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
2012 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
2015 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2017 if (use_4k_pbls
== 0)
2022 opcode
= NES_CQP_REGISTER_STAG
| NES_CQP_STAG_RIGHTS_LOCAL_READ
|
2023 NES_CQP_STAG_VA_TO
| NES_CQP_STAG_MR
;
2024 if (acc
& IB_ACCESS_LOCAL_WRITE
)
2025 opcode
|= NES_CQP_STAG_RIGHTS_LOCAL_WRITE
;
2026 if (acc
& IB_ACCESS_REMOTE_WRITE
)
2027 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_REM_ACC_EN
;
2028 if (acc
& IB_ACCESS_REMOTE_READ
)
2029 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_READ
| NES_CQP_STAG_REM_ACC_EN
;
2030 if (acc
& IB_ACCESS_MW_BIND
)
2031 opcode
|= NES_CQP_STAG_RIGHTS_WINDOW_BIND
| NES_CQP_STAG_REM_ACC_EN
;
2033 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2034 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
2035 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, *iova_start
);
2036 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, region_length
);
2038 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
2039 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
2040 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
2041 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
2042 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
2044 if (pbl_count
== 0) {
2045 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, single_buffer
);
2047 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, root_vpbl
->pbl_pbase
);
2048 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, pbl_count
);
2049 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (pg_cnt
* 8));
2052 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
2056 atomic_set(&cqp_request
->refcount
, 2);
2057 nes_post_cqp_request(nesdev
, cqp_request
);
2060 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
2062 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
2063 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2064 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2065 major_code
= cqp_request
->major_code
;
2066 nes_put_cqp_request(nesdev
, cqp_request
);
2068 if ((!ret
|| major_code
) && pbl_count
!= 0) {
2069 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2071 nesadapter
->free_256pbl
+= pbl_count
+ use_two_level
;
2072 else if (use_4k_pbls
)
2073 nesadapter
->free_4kpbl
+= pbl_count
+ use_two_level
;
2074 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2076 if (new_root
.pbl_pbase
)
2077 pci_free_consistent(nesdev
->pcidev
, 512, new_root
.pbl_vbase
,
2078 new_root
.pbl_pbase
);
2082 else if (major_code
)
2085 *actual_pbl_cnt
= pbl_count
+ use_two_level
;
2086 *used_4k_pbls
= use_4k_pbls
;
2094 static struct ib_mr
*nes_reg_phys_mr(struct ib_pd
*ib_pd
,
2095 struct ib_phys_buf
*buffer_list
, int num_phys_buf
, int acc
,
2099 struct nes_pd
*nespd
= to_nespd(ib_pd
);
2100 struct nes_vnic
*nesvnic
= to_nesvnic(ib_pd
->device
);
2101 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2102 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2103 struct nes_mr
*nesmr
;
2105 struct nes_vpbl vpbl
;
2106 struct nes_root_vpbl root_vpbl
;
2111 u32 next_stag_index
= 0;
2113 u32 root_pbl_index
= 0;
2114 u32 cur_pbl_index
= 0;
2122 vpbl
.pbl_vbase
= NULL
;
2123 root_vpbl
.pbl_vbase
= NULL
;
2124 root_vpbl
.pbl_pbase
= 0;
2126 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2127 stag_key
= (u8
)next_stag_index
;
2131 next_stag_index
>>= 8;
2132 next_stag_index
%= nesadapter
->max_mr
;
2133 if (num_phys_buf
> (1024*512)) {
2134 return ERR_PTR(-E2BIG
);
2137 if ((buffer_list
[0].addr
^ *iova_start
) & ~PAGE_MASK
)
2138 return ERR_PTR(-EINVAL
);
2140 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
, nesadapter
->max_mr
,
2141 &stag_index
, &next_stag_index
);
2143 return ERR_PTR(err
);
2146 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2148 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2149 return ERR_PTR(-ENOMEM
);
2152 for (i
= 0; i
< num_phys_buf
; i
++) {
2154 if ((i
& 0x01FF) == 0) {
2155 if (root_pbl_index
== 1) {
2156 /* Allocate the root PBL */
2157 root_vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 8192,
2158 &root_vpbl
.pbl_pbase
);
2159 nes_debug(NES_DBG_MR
, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2160 root_vpbl
.pbl_vbase
, (unsigned int)root_vpbl
.pbl_pbase
);
2161 if (!root_vpbl
.pbl_vbase
) {
2162 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2164 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2166 return ERR_PTR(-ENOMEM
);
2168 root_vpbl
.leaf_vpbl
= kzalloc(sizeof(*root_vpbl
.leaf_vpbl
)*1024, GFP_KERNEL
);
2169 if (!root_vpbl
.leaf_vpbl
) {
2170 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2171 root_vpbl
.pbl_pbase
);
2172 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2174 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2176 return ERR_PTR(-ENOMEM
);
2178 root_vpbl
.pbl_vbase
[0].pa_low
= cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2179 root_vpbl
.pbl_vbase
[0].pa_high
=
2180 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2181 root_vpbl
.leaf_vpbl
[0] = vpbl
;
2183 /* Allocate a 4K buffer for the PBL */
2184 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2186 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
2187 vpbl
.pbl_vbase
, (unsigned long)vpbl
.pbl_pbase
);
2188 if (!vpbl
.pbl_vbase
) {
2189 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2190 ibmr
= ERR_PTR(-ENOMEM
);
2194 /* Fill in the root table */
2195 if (1 <= root_pbl_index
) {
2196 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_low
=
2197 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2198 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_high
=
2199 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2200 root_vpbl
.leaf_vpbl
[root_pbl_index
] = vpbl
;
2206 mask
= !buffer_list
[i
].size
;
2208 mask
|= buffer_list
[i
].addr
;
2209 if (i
!= num_phys_buf
- 1)
2210 mask
|= buffer_list
[i
].addr
+ buffer_list
[i
].size
;
2212 if (mask
& ~PAGE_MASK
) {
2213 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2214 nes_debug(NES_DBG_MR
, "Invalid buffer addr or size\n");
2215 ibmr
= ERR_PTR(-EINVAL
);
2220 region_length
+= buffer_list
[i
].size
;
2221 if ((i
!= 0) && (single_page
)) {
2222 if ((buffer_list
[i
-1].addr
+PAGE_SIZE
) != buffer_list
[i
].addr
)
2225 vpbl
.pbl_vbase
[cur_pbl_index
].pa_low
= cpu_to_le32((u32
)buffer_list
[i
].addr
& PAGE_MASK
);
2226 vpbl
.pbl_vbase
[cur_pbl_index
++].pa_high
=
2227 cpu_to_le32((u32
)((((u64
)buffer_list
[i
].addr
) >> 32)));
2230 stag
= stag_index
<< 8;
2232 stag
+= (u32
)stag_key
;
2234 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%016lX,"
2235 " length = 0x%016lX, index = 0x%08X\n",
2236 stag
, (unsigned long)*iova_start
, (unsigned long)region_length
, stag_index
);
2238 /* Make the leaf PBL the root if only one PBL */
2239 if (root_pbl_index
== 1) {
2240 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2246 pbl_count
= root_pbl_index
;
2248 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region_length
, &root_vpbl
,
2249 buffer_list
[0].addr
, pbl_count
, (u16
)cur_pbl_index
, acc
, iova_start
,
2250 &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2253 nesmr
->ibmr
.rkey
= stag
;
2254 nesmr
->ibmr
.lkey
= stag
;
2255 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2256 ibmr
= &nesmr
->ibmr
;
2259 ibmr
= ERR_PTR(-ENOMEM
);
2263 /* free the resources */
2264 if (root_pbl_index
== 1) {
2265 /* single PBL case */
2266 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
, vpbl
.pbl_pbase
);
2268 for (i
=0; i
<root_pbl_index
; i
++) {
2269 pci_free_consistent(nesdev
->pcidev
, 4096, root_vpbl
.leaf_vpbl
[i
].pbl_vbase
,
2270 root_vpbl
.leaf_vpbl
[i
].pbl_pbase
);
2272 kfree(root_vpbl
.leaf_vpbl
);
2273 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2274 root_vpbl
.pbl_pbase
);
2284 static struct ib_mr
*nes_get_dma_mr(struct ib_pd
*pd
, int acc
)
2286 struct ib_phys_buf bl
;
2289 nes_debug(NES_DBG_MR
, "\n");
2291 bl
.size
= (u64
)0xffffffffffULL
;
2293 return nes_reg_phys_mr(pd
, &bl
, 1, acc
, &kva
);
2300 static struct ib_mr
*nes_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
2301 u64 virt
, int acc
, struct ib_udata
*udata
)
2306 dma_addr_t last_dma_addr
= 0;
2307 dma_addr_t first_dma_addr
= 0;
2308 struct nes_pd
*nespd
= to_nespd(pd
);
2309 struct nes_vnic
*nesvnic
= to_nesvnic(pd
->device
);
2310 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2311 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2312 struct ib_mr
*ibmr
= ERR_PTR(-EINVAL
);
2313 struct ib_umem_chunk
*chunk
;
2314 struct nes_ucontext
*nes_ucontext
;
2315 struct nes_pbl
*nespbl
;
2316 struct nes_mr
*nesmr
;
2317 struct ib_umem
*region
;
2318 struct nes_mem_reg_req req
;
2319 struct nes_vpbl vpbl
;
2320 struct nes_root_vpbl root_vpbl
;
2321 int nmap_index
, page_index
;
2323 int err
, pbl_depth
= 0;
2328 u32 next_stag_index
;
2330 u32 root_pbl_index
= 0;
2331 u32 cur_pbl_index
= 0;
2337 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
2338 if (IS_ERR(region
)) {
2339 return (struct ib_mr
*)region
;
2342 nes_debug(NES_DBG_MR
, "User base = 0x%lX, Virt base = 0x%lX, length = %u,"
2343 " offset = %u, page size = %u.\n",
2344 (unsigned long int)start
, (unsigned long int)virt
, (u32
)length
,
2345 region
->offset
, region
->page_size
);
2347 skip_pages
= ((u32
)region
->offset
) >> 12;
2349 if (ib_copy_from_udata(&req
, udata
, sizeof(req
)))
2350 return ERR_PTR(-EFAULT
);
2351 nes_debug(NES_DBG_MR
, "Memory Registration type = %08X.\n", req
.reg_type
);
2353 switch (req
.reg_type
) {
2354 case IWNES_MEMREG_TYPE_MEM
:
2357 vpbl
.pbl_vbase
= NULL
;
2358 root_vpbl
.pbl_vbase
= NULL
;
2359 root_vpbl
.pbl_pbase
= 0;
2361 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2362 stag_key
= (u8
)next_stag_index
;
2364 driver_key
= next_stag_index
& 0x70000000;
2366 next_stag_index
>>= 8;
2367 next_stag_index
%= nesadapter
->max_mr
;
2369 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
2370 nesadapter
->max_mr
, &stag_index
, &next_stag_index
);
2372 ib_umem_release(region
);
2373 return ERR_PTR(err
);
2376 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2378 ib_umem_release(region
);
2379 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2380 return ERR_PTR(-ENOMEM
);
2382 nesmr
->region
= region
;
2384 list_for_each_entry(chunk
, ®ion
->chunk_list
, list
) {
2385 nes_debug(NES_DBG_MR
, "Chunk: nents = %u, nmap = %u .\n",
2386 chunk
->nents
, chunk
->nmap
);
2387 for (nmap_index
= 0; nmap_index
< chunk
->nmap
; ++nmap_index
) {
2388 if (sg_dma_address(&chunk
->page_list
[nmap_index
]) & ~PAGE_MASK
) {
2389 ib_umem_release(region
);
2390 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2391 nes_debug(NES_DBG_MR
, "Unaligned Memory Buffer: 0x%x\n",
2392 (unsigned int) sg_dma_address(&chunk
->page_list
[nmap_index
]));
2393 ibmr
= ERR_PTR(-EINVAL
);
2395 goto reg_user_mr_err
;
2398 if (!sg_dma_len(&chunk
->page_list
[nmap_index
])) {
2399 ib_umem_release(region
);
2400 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2402 nes_debug(NES_DBG_MR
, "Invalid Buffer Size\n");
2403 ibmr
= ERR_PTR(-EINVAL
);
2405 goto reg_user_mr_err
;
2408 region_length
+= sg_dma_len(&chunk
->page_list
[nmap_index
]);
2409 chunk_pages
= sg_dma_len(&chunk
->page_list
[nmap_index
]) >> 12;
2410 region_length
-= skip_pages
<< 12;
2411 for (page_index
=skip_pages
; page_index
< chunk_pages
; page_index
++) {
2413 if ((page_count
!=0)&&(page_count
<<12)-(region
->offset
&(4096-1))>=region
->length
)
2415 if ((page_count
&0x01FF) == 0) {
2416 if (page_count
>= 1024 * 512) {
2417 ib_umem_release(region
);
2418 nes_free_resource(nesadapter
,
2419 nesadapter
->allocated_mrs
, stag_index
);
2421 ibmr
= ERR_PTR(-E2BIG
);
2422 goto reg_user_mr_err
;
2424 if (root_pbl_index
== 1) {
2425 root_vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
2426 8192, &root_vpbl
.pbl_pbase
);
2427 nes_debug(NES_DBG_MR
, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2428 root_vpbl
.pbl_vbase
, (unsigned int)root_vpbl
.pbl_pbase
);
2429 if (!root_vpbl
.pbl_vbase
) {
2430 ib_umem_release(region
);
2431 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2433 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2436 ibmr
= ERR_PTR(-ENOMEM
);
2437 goto reg_user_mr_err
;
2439 root_vpbl
.leaf_vpbl
= kzalloc(sizeof(*root_vpbl
.leaf_vpbl
)*1024,
2441 if (!root_vpbl
.leaf_vpbl
) {
2442 ib_umem_release(region
);
2443 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2444 root_vpbl
.pbl_pbase
);
2445 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2447 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2450 ibmr
= ERR_PTR(-ENOMEM
);
2451 goto reg_user_mr_err
;
2453 root_vpbl
.pbl_vbase
[0].pa_low
=
2454 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2455 root_vpbl
.pbl_vbase
[0].pa_high
=
2456 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2457 root_vpbl
.leaf_vpbl
[0] = vpbl
;
2459 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2461 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2462 vpbl
.pbl_vbase
, (unsigned int)vpbl
.pbl_pbase
);
2463 if (!vpbl
.pbl_vbase
) {
2464 ib_umem_release(region
);
2465 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2466 ibmr
= ERR_PTR(-ENOMEM
);
2468 goto reg_user_mr_err
;
2470 if (1 <= root_pbl_index
) {
2471 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_low
=
2472 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2473 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_high
=
2474 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
)>>32)));
2475 root_vpbl
.leaf_vpbl
[root_pbl_index
] = vpbl
;
2481 if (page_count
!= 0) {
2482 if ((last_dma_addr
+4096) !=
2483 (sg_dma_address(&chunk
->page_list
[nmap_index
])+
2486 last_dma_addr
= sg_dma_address(&chunk
->page_list
[nmap_index
])+
2489 first_dma_addr
= sg_dma_address(&chunk
->page_list
[nmap_index
])+
2491 last_dma_addr
= first_dma_addr
;
2495 vpbl
.pbl_vbase
[cur_pbl_index
].pa_low
=
2496 cpu_to_le32((u32
)(sg_dma_address(&chunk
->page_list
[nmap_index
])+
2497 (page_index
*4096)));
2498 vpbl
.pbl_vbase
[cur_pbl_index
].pa_high
=
2499 cpu_to_le32((u32
)((((u64
)(sg_dma_address(&chunk
->page_list
[nmap_index
])+
2500 (page_index
*4096))) >> 32)));
2507 nes_debug(NES_DBG_MR
, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2508 " stag_key=0x%08x\n",
2509 stag_index
, driver_key
, stag_key
);
2510 stag
= stag_index
<< 8;
2512 stag
+= (u32
)stag_key
;
2515 /* Make the leaf PBL the root if only one PBL */
2516 if (root_pbl_index
== 1) {
2517 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2523 pbl_count
= root_pbl_index
;
2526 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X,"
2527 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n",
2528 stag
, (unsigned int)iova_start
,
2529 (unsigned int)region_length
, stag_index
,
2530 (unsigned long long)region
->length
, pbl_count
);
2531 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region
->length
, &root_vpbl
,
2532 first_dma_addr
, pbl_count
, (u16
)cur_pbl_index
, acc
,
2533 &iova_start
, &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2535 nes_debug(NES_DBG_MR
, "ret=%d\n", ret
);
2538 nesmr
->ibmr
.rkey
= stag
;
2539 nesmr
->ibmr
.lkey
= stag
;
2540 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2541 ibmr
= &nesmr
->ibmr
;
2543 ib_umem_release(region
);
2545 ibmr
= ERR_PTR(-ENOMEM
);
2549 /* free the resources */
2550 if (root_pbl_index
== 1) {
2551 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2554 for (page_index
=0; page_index
<root_pbl_index
; page_index
++) {
2555 pci_free_consistent(nesdev
->pcidev
, 4096,
2556 root_vpbl
.leaf_vpbl
[page_index
].pbl_vbase
,
2557 root_vpbl
.leaf_vpbl
[page_index
].pbl_pbase
);
2559 kfree(root_vpbl
.leaf_vpbl
);
2560 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2561 root_vpbl
.pbl_pbase
);
2564 nes_debug(NES_DBG_MR
, "Leaving, ibmr=%p", ibmr
);
2567 case IWNES_MEMREG_TYPE_QP
:
2568 case IWNES_MEMREG_TYPE_CQ
:
2569 nespbl
= kzalloc(sizeof(*nespbl
), GFP_KERNEL
);
2571 nes_debug(NES_DBG_MR
, "Unable to allocate PBL\n");
2572 ib_umem_release(region
);
2573 return ERR_PTR(-ENOMEM
);
2575 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2577 ib_umem_release(region
);
2579 nes_debug(NES_DBG_MR
, "Unable to allocate nesmr\n");
2580 return ERR_PTR(-ENOMEM
);
2582 nesmr
->region
= region
;
2583 nes_ucontext
= to_nesucontext(pd
->uobject
->context
);
2584 pbl_depth
= region
->length
>> 12;
2585 pbl_depth
+= (region
->length
& (4096-1)) ? 1 : 0;
2586 nespbl
->pbl_size
= pbl_depth
*sizeof(u64
);
2587 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2588 nes_debug(NES_DBG_MR
, "Attempting to allocate QP PBL memory");
2590 nes_debug(NES_DBG_MR
, "Attempting to allocate CP PBL memory");
2593 nes_debug(NES_DBG_MR
, " %u bytes, %u entries.\n",
2594 nespbl
->pbl_size
, pbl_depth
);
2595 pbl
= pci_alloc_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
2596 &nespbl
->pbl_pbase
);
2598 ib_umem_release(region
);
2601 nes_debug(NES_DBG_MR
, "Unable to allocate PBL memory\n");
2602 return ERR_PTR(-ENOMEM
);
2605 nespbl
->pbl_vbase
= (u64
*)pbl
;
2606 nespbl
->user_base
= start
;
2607 nes_debug(NES_DBG_MR
, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
2608 " pbl_vbase=%p user_base=0x%lx\n",
2609 nespbl
->pbl_size
, (unsigned long) nespbl
->pbl_pbase
,
2610 (void *) nespbl
->pbl_vbase
, nespbl
->user_base
);
2612 list_for_each_entry(chunk
, ®ion
->chunk_list
, list
) {
2613 for (nmap_index
= 0; nmap_index
< chunk
->nmap
; ++nmap_index
) {
2614 chunk_pages
= sg_dma_len(&chunk
->page_list
[nmap_index
]) >> 12;
2615 chunk_pages
+= (sg_dma_len(&chunk
->page_list
[nmap_index
]) & (4096-1)) ? 1 : 0;
2616 nespbl
->page
= sg_page(&chunk
->page_list
[0]);
2617 for (page_index
=0; page_index
<chunk_pages
; page_index
++) {
2618 ((__le32
*)pbl
)[0] = cpu_to_le32((u32
)
2619 (sg_dma_address(&chunk
->page_list
[nmap_index
])+
2620 (page_index
*4096)));
2621 ((__le32
*)pbl
)[1] = cpu_to_le32(((u64
)
2622 (sg_dma_address(&chunk
->page_list
[nmap_index
])+
2623 (page_index
*4096)))>>32);
2624 nes_debug(NES_DBG_MR
, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl
,
2625 (unsigned long long)*pbl
,
2626 le32_to_cpu(((__le32
*)pbl
)[1]), le32_to_cpu(((__le32
*)pbl
)[0]));
2631 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2632 list_add_tail(&nespbl
->list
, &nes_ucontext
->qp_reg_mem_list
);
2634 list_add_tail(&nespbl
->list
, &nes_ucontext
->cq_reg_mem_list
);
2636 nesmr
->ibmr
.rkey
= -1;
2637 nesmr
->ibmr
.lkey
= -1;
2638 nesmr
->mode
= req
.reg_type
;
2639 return &nesmr
->ibmr
;
2642 return ERR_PTR(-ENOSYS
);
2649 static int nes_dereg_mr(struct ib_mr
*ib_mr
)
2651 struct nes_mr
*nesmr
= to_nesmr(ib_mr
);
2652 struct nes_vnic
*nesvnic
= to_nesvnic(ib_mr
->device
);
2653 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2654 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2655 struct nes_hw_cqp_wqe
*cqp_wqe
;
2656 struct nes_cqp_request
*cqp_request
;
2657 unsigned long flags
;
2662 if (nesmr
->region
) {
2663 ib_umem_release(nesmr
->region
);
2665 if (nesmr
->mode
!= IWNES_MEMREG_TYPE_MEM
) {
2670 /* Deallocate the region with the adapter */
2672 cqp_request
= nes_get_cqp_request(nesdev
);
2673 if (cqp_request
== NULL
) {
2674 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
2677 cqp_request
->waiting
= 1;
2678 cqp_wqe
= &cqp_request
->cqp_wqe
;
2680 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2681 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2682 NES_CQP_DEALLOCATE_STAG
| NES_CQP_STAG_VA_TO
|
2683 NES_CQP_STAG_DEALLOC_PBLS
| NES_CQP_STAG_MR
);
2684 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ib_mr
->rkey
);
2686 atomic_set(&cqp_request
->refcount
, 2);
2687 nes_post_cqp_request(nesdev
, cqp_request
);
2690 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X completed\n", ib_mr
->rkey
);
2691 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2693 nes_debug(NES_DBG_MR
, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u,"
2694 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
2695 ib_mr
->rkey
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2697 major_code
= cqp_request
->major_code
;
2698 minor_code
= cqp_request
->minor_code
;
2700 nes_put_cqp_request(nesdev
, cqp_request
);
2703 nes_debug(NES_DBG_MR
, "Timeout waiting to destroy STag,"
2704 " ib_mr=%p, rkey = 0x%08X\n",
2705 ib_mr
, ib_mr
->rkey
);
2707 } else if (major_code
) {
2708 nes_debug(NES_DBG_MR
, "Error (0x%04X:0x%04X) while attempting"
2709 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n",
2710 major_code
, minor_code
, ib_mr
, ib_mr
->rkey
);
2714 if (nesmr
->pbls_used
!= 0) {
2715 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2716 if (nesmr
->pbl_4k
) {
2717 nesadapter
->free_4kpbl
+= nesmr
->pbls_used
;
2718 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
)
2719 printk(KERN_ERR PFX
"free 4KB PBLs(%u) has "
2720 "exceeded the max(%u)\n",
2721 nesadapter
->free_4kpbl
,
2722 nesadapter
->max_4kpbl
);
2724 nesadapter
->free_256pbl
+= nesmr
->pbls_used
;
2725 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
)
2726 printk(KERN_ERR PFX
"free 256B PBLs(%u) has "
2727 "exceeded the max(%u)\n",
2728 nesadapter
->free_256pbl
,
2729 nesadapter
->max_256pbl
);
2731 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2733 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2734 (ib_mr
->rkey
& 0x0fffff00) >> 8);
2745 static ssize_t
show_rev(struct device
*dev
, struct device_attribute
*attr
,
2748 struct nes_ib_device
*nesibdev
=
2749 container_of(dev
, struct nes_ib_device
, ibdev
.dev
);
2750 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
2752 nes_debug(NES_DBG_INIT
, "\n");
2753 return sprintf(buf
, "%x\n", nesvnic
->nesdev
->nesadapter
->hw_rev
);
2760 static ssize_t
show_fw_ver(struct device
*dev
, struct device_attribute
*attr
,
2763 struct nes_ib_device
*nesibdev
=
2764 container_of(dev
, struct nes_ib_device
, ibdev
.dev
);
2765 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
2767 nes_debug(NES_DBG_INIT
, "\n");
2768 return sprintf(buf
, "%u.%u\n",
2769 (nesvnic
->nesdev
->nesadapter
->firmware_version
>> 16),
2770 (nesvnic
->nesdev
->nesadapter
->firmware_version
& 0x000000ff));
2777 static ssize_t
show_hca(struct device
*dev
, struct device_attribute
*attr
,
2780 nes_debug(NES_DBG_INIT
, "\n");
2781 return sprintf(buf
, "NES020\n");
2788 static ssize_t
show_board(struct device
*dev
, struct device_attribute
*attr
,
2791 nes_debug(NES_DBG_INIT
, "\n");
2792 return sprintf(buf
, "%.*s\n", 32, "NES020 Board ID");
2796 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
2797 static DEVICE_ATTR(fw_ver
, S_IRUGO
, show_fw_ver
, NULL
);
2798 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
2799 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
2801 static struct device_attribute
*nes_dev_attributes
[] = {
2812 static int nes_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2813 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
2815 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2817 nes_debug(NES_DBG_QP
, "\n");
2819 attr
->qp_access_flags
= 0;
2820 attr
->cap
.max_send_wr
= nesqp
->hwqp
.sq_size
;
2821 attr
->cap
.max_recv_wr
= nesqp
->hwqp
.rq_size
;
2822 attr
->cap
.max_recv_sge
= 1;
2823 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
)
2824 attr
->cap
.max_inline_data
= 0;
2826 attr
->cap
.max_inline_data
= 64;
2828 init_attr
->event_handler
= nesqp
->ibqp
.event_handler
;
2829 init_attr
->qp_context
= nesqp
->ibqp
.qp_context
;
2830 init_attr
->send_cq
= nesqp
->ibqp
.send_cq
;
2831 init_attr
->recv_cq
= nesqp
->ibqp
.recv_cq
;
2832 init_attr
->srq
= nesqp
->ibqp
.srq
= nesqp
->ibqp
.srq
;
2833 init_attr
->cap
= attr
->cap
;
2842 int nes_hw_modify_qp(struct nes_device
*nesdev
, struct nes_qp
*nesqp
,
2843 u32 next_iwarp_state
, u32 termlen
, u32 wait_completion
)
2845 struct nes_hw_cqp_wqe
*cqp_wqe
;
2846 /* struct iw_cm_id *cm_id = nesqp->cm_id; */
2847 /* struct iw_cm_event cm_event; */
2848 struct nes_cqp_request
*cqp_request
;
2852 nes_debug(NES_DBG_MOD_QP
, "QP%u, refcount=%d\n",
2853 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
2855 cqp_request
= nes_get_cqp_request(nesdev
);
2856 if (cqp_request
== NULL
) {
2857 nes_debug(NES_DBG_MOD_QP
, "Failed to get a cqp_request.\n");
2860 if (wait_completion
) {
2861 cqp_request
->waiting
= 1;
2863 cqp_request
->waiting
= 0;
2865 cqp_wqe
= &cqp_request
->cqp_wqe
;
2867 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2868 NES_CQP_MODIFY_QP
| NES_CQP_QP_TYPE_IWARP
| next_iwarp_state
);
2869 nes_debug(NES_DBG_MOD_QP
, "using next_iwarp_state=%08x, wqe_words=%08x\n",
2870 next_iwarp_state
, le32_to_cpu(cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
]));
2871 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2872 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
2873 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, (u64
)nesqp
->nesqp_context_pbase
);
2875 /* If sending a terminate message, fill in the length (in words) */
2876 if (((next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
) == NES_CQP_QP_IWARP_STATE_TERMINATE
) &&
2877 !(next_iwarp_state
& NES_CQP_QP_TERM_DONT_SEND_TERM_MSG
)) {
2878 termlen
= ((termlen
+ 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT
;
2879 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_NEW_MSS_IDX
, termlen
);
2882 atomic_set(&cqp_request
->refcount
, 2);
2883 nes_post_cqp_request(nesdev
, cqp_request
);
2886 if (wait_completion
) {
2887 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n",
2888 nesqp->hwqp.qp_id); */
2889 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2891 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, "
2892 "CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2893 nesqp
->hwqp
.qp_id
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2894 major_code
= cqp_request
->major_code
;
2896 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u failed"
2897 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n",
2898 nesqp
->hwqp
.qp_id
, cqp_request
->major_code
,
2899 cqp_request
->minor_code
, next_iwarp_state
);
2902 nes_put_cqp_request(nesdev
, cqp_request
);
2906 else if (major_code
)
2919 int nes_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2920 int attr_mask
, struct ib_udata
*udata
)
2922 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2923 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
2924 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2927 u32 next_iwarp_state
= 0;
2929 unsigned long qplockflags
;
2931 u16 original_last_aeq
;
2932 u8 issue_modify_qp
= 0;
2935 nes_debug(NES_DBG_MOD_QP
, "QP%u: QP State=%u, cur QP State=%u,"
2936 " iwarp_state=0x%X, refcount=%d\n",
2937 nesqp
->hwqp
.qp_id
, attr
->qp_state
, nesqp
->ibqp_state
,
2938 nesqp
->iwarp_state
, atomic_read(&nesqp
->refcount
));
2940 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
2942 nes_debug(NES_DBG_MOD_QP
, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X,"
2943 " QP Access Flags=0x%X, attr_mask = 0x%0x\n",
2944 nesqp
->hwqp
.qp_id
, nesqp
->hw_iwarp_state
,
2945 nesqp
->hw_tcp_state
, attr
->qp_access_flags
, attr_mask
);
2947 if (attr_mask
& IB_QP_STATE
) {
2948 switch (attr
->qp_state
) {
2950 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = init\n",
2952 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
2953 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2956 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
2957 issue_modify_qp
= 1;
2960 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rtr\n",
2962 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
2963 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2966 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
2967 issue_modify_qp
= 1;
2970 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rts\n",
2972 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_RTS
) {
2973 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2976 if (nesqp
->cm_id
== NULL
) {
2977 nes_debug(NES_DBG_MOD_QP
, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n",
2978 nesqp
->hwqp
.qp_id
);
2979 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2982 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_RTS
;
2983 if (nesqp
->iwarp_state
!= NES_CQP_QP_IWARP_STATE_RTS
)
2984 next_iwarp_state
|= NES_CQP_QP_CONTEXT_VALID
|
2985 NES_CQP_QP_ARP_VALID
| NES_CQP_QP_ORD_VALID
;
2986 issue_modify_qp
= 1;
2987 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_ESTABLISHED
;
2988 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_RTS
;
2989 nesqp
->hte_added
= 1;
2992 issue_modify_qp
= 1;
2993 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n",
2994 nesqp
->hwqp
.qp_id
, nesqp
->hwqp
.sq_head
, nesqp
->hwqp
.sq_tail
);
2995 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
2996 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2999 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
3000 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
3001 " ignored due to current iWARP state\n",
3003 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3006 if (nesqp
->hw_iwarp_state
!= NES_AEQE_IWARP_STATE_RTS
) {
3007 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
3008 " already done based on hw state.\n",
3010 issue_modify_qp
= 0;
3012 switch (nesqp
->hw_iwarp_state
) {
3013 case NES_AEQE_IWARP_STATE_CLOSING
:
3014 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
3015 case NES_AEQE_IWARP_STATE_TERMINATE
:
3016 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
3018 case NES_AEQE_IWARP_STATE_ERROR
:
3019 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
3022 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
3023 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_CLOSING
;
3029 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = terminate\n",
3031 if (nesqp
->iwarp_state
>=(u32
)NES_CQP_QP_IWARP_STATE_TERMINATE
) {
3032 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3035 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3036 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
3037 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_TERMINATE
;
3038 issue_modify_qp
= 1;
3042 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_ERROR
) {
3043 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3046 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = error\n",
3048 if (nesqp
->term_flags
)
3049 del_timer(&nesqp
->terminate_timer
);
3051 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
3052 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
3053 if (nesqp
->hte_added
) {
3054 nes_debug(NES_DBG_MOD_QP
, "set CQP_QP_DEL_HTE\n");
3055 next_iwarp_state
|= NES_CQP_QP_DEL_HTE
;
3056 nesqp
->hte_added
= 0;
3058 if ((nesqp
->hw_tcp_state
> NES_AEQE_TCP_STATE_CLOSED
) &&
3059 (nesdev
->iw_status
) &&
3060 (nesqp
->hw_tcp_state
!= NES_AEQE_TCP_STATE_TIME_WAIT
)) {
3061 next_iwarp_state
|= NES_CQP_QP_RESET
;
3063 nes_debug(NES_DBG_MOD_QP
, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
3064 nesqp
->hwqp
.qp_id
, nesqp
->hw_tcp_state
);
3067 issue_modify_qp
= 1;
3068 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_ERROR
;
3071 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3076 nesqp
->ibqp_state
= attr
->qp_state
;
3077 if (((nesqp
->iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
) ==
3078 (u32
)NES_CQP_QP_IWARP_STATE_RTS
) &&
3079 ((next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
) >
3080 (u32
)NES_CQP_QP_IWARP_STATE_RTS
)) {
3081 nesqp
->iwarp_state
= next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
;
3082 nes_debug(NES_DBG_MOD_QP
, "Change nesqp->iwarp_state=%08x\n",
3083 nesqp
->iwarp_state
);
3085 nesqp
->iwarp_state
= next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
;
3086 nes_debug(NES_DBG_MOD_QP
, "Change nesqp->iwarp_state=%08x\n",
3087 nesqp
->iwarp_state
);
3091 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
3092 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
) {
3093 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
3094 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3095 issue_modify_qp
= 1;
3097 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
) {
3098 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
);
3099 issue_modify_qp
= 1;
3101 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
) {
3102 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3103 issue_modify_qp
= 1;
3105 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) {
3106 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN
);
3107 issue_modify_qp
= 1;
3110 if (nesqp
->user_mode
) {
3111 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
3112 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
3113 issue_modify_qp
= 1;
3117 original_last_aeq
= nesqp
->last_aeq
;
3118 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3120 nes_debug(NES_DBG_MOD_QP
, "issue_modify_qp=%u\n", issue_modify_qp
);
3125 if (issue_modify_qp
) {
3126 nes_debug(NES_DBG_MOD_QP
, "call nes_hw_modify_qp\n");
3127 ret
= nes_hw_modify_qp(nesdev
, nesqp
, next_iwarp_state
, 0, 1);
3129 nes_debug(NES_DBG_MOD_QP
, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)"
3130 " failed for QP%u.\n",
3131 next_iwarp_state
, nesqp
->hwqp
.qp_id
);
3135 if ((issue_modify_qp
) && (nesqp
->ibqp_state
> IB_QPS_RTS
)) {
3136 nes_debug(NES_DBG_MOD_QP
, "QP%u Issued ModifyQP refcount (%d),"
3137 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3138 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3139 original_last_aeq
, nesqp
->last_aeq
);
3141 ((original_last_aeq
!= NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE
) &&
3144 if (nesqp
->cm_id
&& nesqp
->hw_tcp_state
!= 0) {
3145 nes_debug(NES_DBG_MOD_QP
, "QP%u Queuing fake disconnect for QP refcount (%d),"
3146 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3147 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3148 original_last_aeq
, nesqp
->last_aeq
);
3149 /* this one is for the cm_disconnect thread */
3150 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
3151 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_CLOSED
;
3152 nesqp
->last_aeq
= NES_AEQE_AEID_RESET_SENT
;
3153 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3154 nes_cm_disconn(nesqp
);
3156 nes_debug(NES_DBG_MOD_QP
, "QP%u No fake disconnect, QP refcount=%d\n",
3157 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
3160 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
3162 /* These two are for the timer thread */
3163 if (atomic_inc_return(&nesqp
->close_timer_started
) == 1) {
3164 nesqp
->cm_id
->add_ref(nesqp
->cm_id
);
3165 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
3166 " need ae to finish up, original_last_aeq = 0x%04X."
3167 " last_aeq = 0x%04X, scheduling timer.\n",
3168 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3169 original_last_aeq
, nesqp
->last_aeq
);
3170 schedule_nes_timer(nesqp
->cm_node
, (struct sk_buff
*) nesqp
, NES_TIMER_TYPE_CLOSE
, 1, 0);
3172 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3174 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
3175 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
3176 " need ae to finish up, original_last_aeq = 0x%04X."
3177 " last_aeq = 0x%04X.\n",
3178 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3179 original_last_aeq
, nesqp
->last_aeq
);
3183 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3184 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3185 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3186 original_last_aeq
, nesqp
->last_aeq
);
3189 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
3190 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3191 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3192 original_last_aeq
, nesqp
->last_aeq
);
3197 nes_debug(NES_DBG_MOD_QP
, "QP%u Leaving, refcount=%d\n",
3198 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
3205 * nes_muticast_attach
3207 static int nes_multicast_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3209 nes_debug(NES_DBG_INIT
, "\n");
3215 * nes_multicast_detach
3217 static int nes_multicast_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3219 nes_debug(NES_DBG_INIT
, "\n");
3227 static int nes_process_mad(struct ib_device
*ibdev
, int mad_flags
,
3228 u8 port_num
, struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
3229 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
3231 nes_debug(NES_DBG_INIT
, "\n");
3236 fill_wqe_sg_send(struct nes_hw_qp_wqe
*wqe
, struct ib_send_wr
*ib_wr
, u32 uselkey
)
3239 int total_payload_length
= 0;
3240 for (sge_index
= 0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3241 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3242 ib_wr
->sg_list
[sge_index
].addr
);
3243 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_LENGTH0_IDX
+ (sge_index
*4),
3244 ib_wr
->sg_list
[sge_index
].length
);
3246 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4),
3247 (ib_wr
->sg_list
[sge_index
].lkey
));
3249 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4), 0);
3251 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3253 nes_debug(NES_DBG_IW_TX
, "UC UC UC, sending total_payload_length=%u \n",
3254 total_payload_length
);
3255 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3256 total_payload_length
);
3262 static int nes_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*ib_wr
,
3263 struct ib_send_wr
**bad_wr
)
3266 unsigned long flags
= 0;
3267 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3268 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3269 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3270 struct nes_hw_qp_wqe
*wqe
;
3272 u32 qsize
= nesqp
->hwqp
.sq_size
;
3278 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3283 spin_lock_irqsave(&nesqp
->lock
, flags
);
3285 head
= nesqp
->hwqp
.sq_head
;
3288 /* Check for QP error */
3289 if (nesqp
->term_flags
) {
3294 /* Check for SQ overflow */
3295 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.sq_tail
) % qsize
) == (qsize
- 1)) {
3300 wqe
= &nesqp
->hwqp
.sq_vbase
[head
];
3301 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n",
3302 nesqp->hwqp.qp_id, wqe, head); */
3303 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3304 u64temp
= (u64
)(ib_wr
->wr_id
);
3305 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3307 switch (ib_wr
->opcode
) {
3309 case IB_WR_SEND_WITH_INV
:
3310 if (IB_WR_SEND
== ib_wr
->opcode
) {
3311 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3312 wqe_misc
= NES_IWARP_SQ_OP_SENDSE
;
3314 wqe_misc
= NES_IWARP_SQ_OP_SEND
;
3316 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3317 wqe_misc
= NES_IWARP_SQ_OP_SENDSEINV
;
3319 wqe_misc
= NES_IWARP_SQ_OP_SENDINV
;
3321 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3322 ib_wr
->ex
.invalidate_rkey
);
3325 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3330 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3331 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3333 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3334 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3335 (ib_wr
->sg_list
[0].length
<= 64)) {
3336 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3337 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3338 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3339 ib_wr
->sg_list
[0].length
);
3340 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3342 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3346 case IB_WR_RDMA_WRITE
:
3347 wqe_misc
= NES_IWARP_SQ_OP_RDMAW
;
3348 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3349 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=%u\n",
3350 ib_wr
->num_sge
, nesdev
->nesadapter
->max_sge
);
3355 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3356 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3358 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3359 ib_wr
->wr
.rdma
.rkey
);
3360 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3361 ib_wr
->wr
.rdma
.remote_addr
);
3363 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3364 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3365 (ib_wr
->sg_list
[0].length
<= 64)) {
3366 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3367 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3368 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3369 ib_wr
->sg_list
[0].length
);
3370 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3372 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3375 wqe
->wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
] =
3376 wqe
->wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
];
3378 case IB_WR_RDMA_READ
:
3379 case IB_WR_RDMA_READ_WITH_INV
:
3380 /* iWARP only supports 1 sge for RDMA reads */
3381 if (ib_wr
->num_sge
> 1) {
3382 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=1\n",
3387 if (ib_wr
->opcode
== IB_WR_RDMA_READ
) {
3388 wqe_misc
= NES_IWARP_SQ_OP_RDMAR
;
3390 wqe_misc
= NES_IWARP_SQ_OP_RDMAR_LOCINV
;
3391 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3392 ib_wr
->ex
.invalidate_rkey
);
3395 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3396 ib_wr
->wr
.rdma
.remote_addr
);
3397 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3398 ib_wr
->wr
.rdma
.rkey
);
3399 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
,
3400 ib_wr
->sg_list
->length
);
3401 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
,
3402 ib_wr
->sg_list
->addr
);
3403 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
,
3404 ib_wr
->sg_list
->lkey
);
3406 case IB_WR_LOCAL_INV
:
3407 wqe_misc
= NES_IWARP_SQ_OP_LOCINV
;
3408 set_wqe_32bit_value(wqe
->wqe_words
,
3409 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX
,
3410 ib_wr
->ex
.invalidate_rkey
);
3412 case IB_WR_FAST_REG_MR
:
3415 int flags
= ib_wr
->wr
.fast_reg
.access_flags
;
3416 struct nes_ib_fast_reg_page_list
*pnesfrpl
=
3417 container_of(ib_wr
->wr
.fast_reg
.page_list
,
3418 struct nes_ib_fast_reg_page_list
,
3420 u64
*src_page_list
= pnesfrpl
->ibfrpl
.page_list
;
3421 u64
*dst_page_list
= pnesfrpl
->nes_wqe_pbl
.kva
;
3423 if (ib_wr
->wr
.fast_reg
.page_list_len
>
3424 (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
))) {
3425 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: bad page_list_len\n");
3429 wqe_misc
= NES_IWARP_SQ_OP_FAST_REG
;
3430 set_wqe_64bit_value(wqe
->wqe_words
,
3431 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX
,
3432 ib_wr
->wr
.fast_reg
.iova_start
);
3433 set_wqe_32bit_value(wqe
->wqe_words
,
3434 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX
,
3435 ib_wr
->wr
.fast_reg
.length
);
3436 set_wqe_32bit_value(wqe
->wqe_words
,
3437 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX
,
3438 ib_wr
->wr
.fast_reg
.rkey
);
3439 /* Set page size: */
3440 if (ib_wr
->wr
.fast_reg
.page_shift
== 12) {
3441 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K
;
3442 } else if (ib_wr
->wr
.fast_reg
.page_shift
== 21) {
3443 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M
;
3445 nes_debug(NES_DBG_IW_TX
, "Invalid page shift,"
3446 " ib_wr=%u, max=1\n", ib_wr
->num_sge
);
3450 /* Set access_flags */
3451 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ
;
3452 if (flags
& IB_ACCESS_LOCAL_WRITE
)
3453 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE
;
3455 if (flags
& IB_ACCESS_REMOTE_WRITE
)
3456 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE
;
3458 if (flags
& IB_ACCESS_REMOTE_READ
)
3459 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ
;
3461 if (flags
& IB_ACCESS_MW_BIND
)
3462 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND
;
3464 /* Fill in PBL info: */
3465 if (ib_wr
->wr
.fast_reg
.page_list_len
>
3466 pnesfrpl
->ibfrpl
.max_page_list_len
) {
3467 nes_debug(NES_DBG_IW_TX
, "Invalid page list length,"
3468 " ib_wr=%p, value=%u, max=%u\n",
3469 ib_wr
, ib_wr
->wr
.fast_reg
.page_list_len
,
3470 pnesfrpl
->ibfrpl
.max_page_list_len
);
3475 set_wqe_64bit_value(wqe
->wqe_words
,
3476 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX
,
3477 pnesfrpl
->nes_wqe_pbl
.paddr
);
3479 set_wqe_32bit_value(wqe
->wqe_words
,
3480 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX
,
3481 ib_wr
->wr
.fast_reg
.page_list_len
* 8);
3483 for (i
= 0; i
< ib_wr
->wr
.fast_reg
.page_list_len
; i
++)
3484 dst_page_list
[i
] = cpu_to_le64(src_page_list
[i
]);
3486 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: iova_start: %p, "
3487 "length: %d, rkey: %0x, pgl_paddr: %p, "
3488 "page_list_len: %u, wqe_misc: %x\n",
3489 (void *)ib_wr
->wr
.fast_reg
.iova_start
,
3490 ib_wr
->wr
.fast_reg
.length
,
3491 ib_wr
->wr
.fast_reg
.rkey
,
3492 (void *)pnesfrpl
->nes_wqe_pbl
.paddr
,
3493 ib_wr
->wr
.fast_reg
.page_list_len
,
3506 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || nesqp
->sig_all
)
3507 wqe_misc
|= NES_IWARP_SQ_WQE_SIGNALED_COMPL
;
3509 wqe
->wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
] = cpu_to_le32(wqe_misc
);
3511 ib_wr
= ib_wr
->next
;
3519 nesqp
->hwqp
.sq_head
= head
;
3522 counter
= min(wqe_count
, ((u32
)255));
3523 wqe_count
-= counter
;
3524 nes_write32(nesdev
->regs
+ NES_WQE_ALLOC
,
3525 (counter
<< 24) | 0x00800000 | nesqp
->hwqp
.qp_id
);
3528 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3540 static int nes_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*ib_wr
,
3541 struct ib_recv_wr
**bad_wr
)
3544 unsigned long flags
= 0;
3545 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3546 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3547 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3548 struct nes_hw_qp_wqe
*wqe
;
3551 u32 qsize
= nesqp
->hwqp
.rq_size
;
3555 u32 total_payload_length
;
3557 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3562 spin_lock_irqsave(&nesqp
->lock
, flags
);
3564 head
= nesqp
->hwqp
.rq_head
;
3567 /* Check for QP error */
3568 if (nesqp
->term_flags
) {
3573 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3577 /* Check for RQ overflow */
3578 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.rq_tail
) % qsize
) == (qsize
- 1)) {
3583 nes_debug(NES_DBG_IW_RX
, "ibwr sge count = %u.\n", ib_wr
->num_sge
);
3584 wqe
= &nesqp
->hwqp
.rq_vbase
[head
];
3586 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n",
3587 nesqp->hwqp.qp_id, wqe, head); */
3588 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3589 u64temp
= (u64
)(ib_wr
->wr_id
);
3590 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3592 total_payload_length
= 0;
3593 for (sge_index
=0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3594 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3595 ib_wr
->sg_list
[sge_index
].addr
);
3596 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_LENGTH0_IDX
+(sge_index
*4),
3597 ib_wr
->sg_list
[sge_index
].length
);
3598 set_wqe_32bit_value(wqe
->wqe_words
,NES_IWARP_RQ_WQE_STAG0_IDX
+(sge_index
*4),
3599 ib_wr
->sg_list
[sge_index
].lkey
);
3601 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3603 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX
,
3604 total_payload_length
);
3606 ib_wr
= ib_wr
->next
;
3613 nesqp
->hwqp
.rq_head
= head
;
3616 counter
= min(wqe_count
, ((u32
)255));
3617 wqe_count
-= counter
;
3618 nes_write32(nesdev
->regs
+NES_WQE_ALLOC
, (counter
<<24) | nesqp
->hwqp
.qp_id
);
3621 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3633 static int nes_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*entry
)
3637 unsigned long flags
= 0;
3638 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3639 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3640 struct nes_cq
*nescq
= to_nescq(ibcq
);
3641 struct nes_qp
*nesqp
;
3642 struct nes_hw_cqe cqe
;
3649 u32 move_cq_head
= 1;
3652 nes_debug(NES_DBG_CQ
, "\n");
3654 spin_lock_irqsave(&nescq
->lock
, flags
);
3656 head
= nescq
->hw_cq
.cq_head
;
3657 cq_size
= nescq
->hw_cq
.cq_size
;
3659 while (cqe_count
< num_entries
) {
3660 if ((le32_to_cpu(nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
]) &
3661 NES_CQE_VALID
) == 0)
3665 * Make sure we read CQ entry contents *after*
3666 * we've checked the valid bit.
3670 cqe
= nescq
->hw_cq
.cq_vbase
[head
];
3671 u32temp
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3672 wqe_index
= u32temp
& (nesdev
->nesadapter
->max_qp_wr
- 1);
3673 u32temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
3674 /* parse CQE, get completion context from WQE (either rq or sq) */
3675 u64temp
= (((u64
)(le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
])))<<32) |
3679 nesqp
= (struct nes_qp
*)(unsigned long)u64temp
;
3680 memset(entry
, 0, sizeof *entry
);
3681 if (cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
] == 0) {
3682 entry
->status
= IB_WC_SUCCESS
;
3684 err_code
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
]);
3685 if (NES_IWARP_CQE_MAJOR_DRV
== (err_code
>> 16)) {
3686 entry
->status
= err_code
& 0x0000ffff;
3688 /* The rest of the cqe's will be marked as flushed */
3689 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_ERROR_CODE_IDX
] =
3690 cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH
<< 16) |
3691 NES_IWARP_CQE_MINOR_FLUSH
);
3693 entry
->status
= IB_WC_WR_FLUSH_ERR
;
3696 entry
->qp
= &nesqp
->ibqp
;
3697 entry
->src_qp
= nesqp
->hwqp
.qp_id
;
3699 if (le32_to_cpu(cqe
.cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_SQ
) {
3700 if (nesqp
->skip_lsmm
) {
3701 nesqp
->skip_lsmm
= 0;
3702 nesqp
->hwqp
.sq_tail
++;
3705 /* Working on a SQ Completion*/
3706 wrid
= (((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3707 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX
]))) << 32) |
3708 ((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3709 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
])));
3710 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3711 wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
]);
3713 switch (le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3714 wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
]) & 0x3f) {
3715 case NES_IWARP_SQ_OP_RDMAW
:
3716 nes_debug(NES_DBG_CQ
, "Operation = RDMA WRITE.\n");
3717 entry
->opcode
= IB_WC_RDMA_WRITE
;
3719 case NES_IWARP_SQ_OP_RDMAR
:
3720 nes_debug(NES_DBG_CQ
, "Operation = RDMA READ.\n");
3721 entry
->opcode
= IB_WC_RDMA_READ
;
3722 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3723 wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
]);
3725 case NES_IWARP_SQ_OP_SENDINV
:
3726 case NES_IWARP_SQ_OP_SENDSEINV
:
3727 case NES_IWARP_SQ_OP_SEND
:
3728 case NES_IWARP_SQ_OP_SENDSE
:
3729 nes_debug(NES_DBG_CQ
, "Operation = Send.\n");
3730 entry
->opcode
= IB_WC_SEND
;
3732 case NES_IWARP_SQ_OP_LOCINV
:
3733 entry
->opcode
= IB_WR_LOCAL_INV
;
3735 case NES_IWARP_SQ_OP_FAST_REG
:
3736 entry
->opcode
= IB_WC_FAST_REG_MR
;
3740 nesqp
->hwqp
.sq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.sq_size
- 1);
3741 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.sq_tail
!= nesqp
->hwqp
.sq_head
)) {
3743 wq_tail
= nesqp
->hwqp
.sq_tail
;
3746 /* Working on a RQ Completion*/
3747 entry
->byte_len
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_PAYLOAD_LENGTH_IDX
]);
3748 wrid
= ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX
]))) |
3749 ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX
]))<<32);
3750 entry
->opcode
= IB_WC_RECV
;
3752 nesqp
->hwqp
.rq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.rq_size
- 1);
3753 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.rq_tail
!= nesqp
->hwqp
.rq_head
)) {
3755 wq_tail
= nesqp
->hwqp
.rq_tail
;
3759 entry
->wr_id
= wrid
;
3765 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
] = 0;
3766 if (++head
>= cq_size
)
3768 nescq
->polled_completions
++;
3770 if ((nescq
->polled_completions
> (cq_size
/ 2)) ||
3771 (nescq
->polled_completions
== 255)) {
3772 nes_debug(NES_DBG_CQ
, "CQ%u Issuing CQE Allocate since more than half of cqes"
3773 " are pending %u of %u.\n",
3774 nescq
->hw_cq
.cq_number
, nescq
->polled_completions
, cq_size
);
3775 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3776 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3777 nescq
->polled_completions
= 0;
3780 /* Update the wqe index and set status to flush */
3781 wqe_index
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3782 wqe_index
= (wqe_index
& (~(nesdev
->nesadapter
->max_qp_wr
- 1))) | wq_tail
;
3783 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] =
3784 cpu_to_le32(wqe_index
);
3785 move_cq_head
= 1; /* ready for next pass */
3789 if (nescq
->polled_completions
) {
3790 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3791 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3792 nescq
->polled_completions
= 0;
3795 nescq
->hw_cq
.cq_head
= head
;
3796 nes_debug(NES_DBG_CQ
, "Reporting %u completions for CQ%u.\n",
3797 cqe_count
, nescq
->hw_cq
.cq_number
);
3799 spin_unlock_irqrestore(&nescq
->lock
, flags
);
3808 static int nes_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags notify_flags
)
3810 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3811 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3812 struct nes_cq
*nescq
= to_nescq(ibcq
);
3815 nes_debug(NES_DBG_CQ
, "Requesting notification for CQ%u.\n",
3816 nescq
->hw_cq
.cq_number
);
3818 cq_arm
= nescq
->hw_cq
.cq_number
;
3819 if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_NEXT_COMP
)
3820 cq_arm
|= NES_CQE_ALLOC_NOTIFY_NEXT
;
3821 else if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
)
3822 cq_arm
|= NES_CQE_ALLOC_NOTIFY_SE
;
3826 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
, cq_arm
);
3827 nes_read32(nesdev
->regs
+NES_CQE_ALLOC
);
3834 * nes_init_ofa_device
3836 struct nes_ib_device
*nes_init_ofa_device(struct net_device
*netdev
)
3838 struct nes_ib_device
*nesibdev
;
3839 struct nes_vnic
*nesvnic
= netdev_priv(netdev
);
3840 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3842 nesibdev
= (struct nes_ib_device
*)ib_alloc_device(sizeof(struct nes_ib_device
));
3843 if (nesibdev
== NULL
) {
3846 strlcpy(nesibdev
->ibdev
.name
, "nes%d", IB_DEVICE_NAME_MAX
);
3847 nesibdev
->ibdev
.owner
= THIS_MODULE
;
3849 nesibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
3850 memset(&nesibdev
->ibdev
.node_guid
, 0, sizeof(nesibdev
->ibdev
.node_guid
));
3851 memcpy(&nesibdev
->ibdev
.node_guid
, netdev
->dev_addr
, 6);
3853 nesibdev
->ibdev
.uverbs_cmd_mask
=
3854 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
3855 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
3856 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
3857 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
3858 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
3859 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
3860 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
3861 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
3862 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
3863 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
3864 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
3865 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
3866 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
3867 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
3868 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
3869 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
3870 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
3871 (1ull << IB_USER_VERBS_CMD_ALLOC_MW
) |
3872 (1ull << IB_USER_VERBS_CMD_BIND_MW
) |
3873 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW
) |
3874 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
3875 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
3877 nesibdev
->ibdev
.phys_port_cnt
= 1;
3878 nesibdev
->ibdev
.num_comp_vectors
= 1;
3879 nesibdev
->ibdev
.dma_device
= &nesdev
->pcidev
->dev
;
3880 nesibdev
->ibdev
.dev
.parent
= &nesdev
->pcidev
->dev
;
3881 nesibdev
->ibdev
.query_device
= nes_query_device
;
3882 nesibdev
->ibdev
.query_port
= nes_query_port
;
3883 nesibdev
->ibdev
.modify_port
= nes_modify_port
;
3884 nesibdev
->ibdev
.query_pkey
= nes_query_pkey
;
3885 nesibdev
->ibdev
.query_gid
= nes_query_gid
;
3886 nesibdev
->ibdev
.alloc_ucontext
= nes_alloc_ucontext
;
3887 nesibdev
->ibdev
.dealloc_ucontext
= nes_dealloc_ucontext
;
3888 nesibdev
->ibdev
.mmap
= nes_mmap
;
3889 nesibdev
->ibdev
.alloc_pd
= nes_alloc_pd
;
3890 nesibdev
->ibdev
.dealloc_pd
= nes_dealloc_pd
;
3891 nesibdev
->ibdev
.create_ah
= nes_create_ah
;
3892 nesibdev
->ibdev
.destroy_ah
= nes_destroy_ah
;
3893 nesibdev
->ibdev
.create_qp
= nes_create_qp
;
3894 nesibdev
->ibdev
.modify_qp
= nes_modify_qp
;
3895 nesibdev
->ibdev
.query_qp
= nes_query_qp
;
3896 nesibdev
->ibdev
.destroy_qp
= nes_destroy_qp
;
3897 nesibdev
->ibdev
.create_cq
= nes_create_cq
;
3898 nesibdev
->ibdev
.destroy_cq
= nes_destroy_cq
;
3899 nesibdev
->ibdev
.poll_cq
= nes_poll_cq
;
3900 nesibdev
->ibdev
.get_dma_mr
= nes_get_dma_mr
;
3901 nesibdev
->ibdev
.reg_phys_mr
= nes_reg_phys_mr
;
3902 nesibdev
->ibdev
.reg_user_mr
= nes_reg_user_mr
;
3903 nesibdev
->ibdev
.dereg_mr
= nes_dereg_mr
;
3904 nesibdev
->ibdev
.alloc_mw
= nes_alloc_mw
;
3905 nesibdev
->ibdev
.dealloc_mw
= nes_dealloc_mw
;
3906 nesibdev
->ibdev
.bind_mw
= nes_bind_mw
;
3908 nesibdev
->ibdev
.alloc_fast_reg_mr
= nes_alloc_fast_reg_mr
;
3909 nesibdev
->ibdev
.alloc_fast_reg_page_list
= nes_alloc_fast_reg_page_list
;
3910 nesibdev
->ibdev
.free_fast_reg_page_list
= nes_free_fast_reg_page_list
;
3912 nesibdev
->ibdev
.attach_mcast
= nes_multicast_attach
;
3913 nesibdev
->ibdev
.detach_mcast
= nes_multicast_detach
;
3914 nesibdev
->ibdev
.process_mad
= nes_process_mad
;
3916 nesibdev
->ibdev
.req_notify_cq
= nes_req_notify_cq
;
3917 nesibdev
->ibdev
.post_send
= nes_post_send
;
3918 nesibdev
->ibdev
.post_recv
= nes_post_recv
;
3920 nesibdev
->ibdev
.iwcm
= kzalloc(sizeof(*nesibdev
->ibdev
.iwcm
), GFP_KERNEL
);
3921 if (nesibdev
->ibdev
.iwcm
== NULL
) {
3922 ib_dealloc_device(&nesibdev
->ibdev
);
3925 nesibdev
->ibdev
.iwcm
->add_ref
= nes_add_ref
;
3926 nesibdev
->ibdev
.iwcm
->rem_ref
= nes_rem_ref
;
3927 nesibdev
->ibdev
.iwcm
->get_qp
= nes_get_qp
;
3928 nesibdev
->ibdev
.iwcm
->connect
= nes_connect
;
3929 nesibdev
->ibdev
.iwcm
->accept
= nes_accept
;
3930 nesibdev
->ibdev
.iwcm
->reject
= nes_reject
;
3931 nesibdev
->ibdev
.iwcm
->create_listen
= nes_create_listen
;
3932 nesibdev
->ibdev
.iwcm
->destroy_listen
= nes_destroy_listen
;
3937 void nes_port_ibevent(struct nes_vnic
*nesvnic
)
3939 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
3940 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3941 struct ib_event event
;
3942 event
.device
= &nesibdev
->ibdev
;
3943 event
.element
.port_num
= nesvnic
->logical_port
+ 1;
3944 event
.event
= nesdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
3945 ib_dispatch_event(&event
);
3950 * nes_destroy_ofa_device
3952 void nes_destroy_ofa_device(struct nes_ib_device
*nesibdev
)
3954 if (nesibdev
== NULL
)
3957 nes_unregister_ofa_device(nesibdev
);
3959 kfree(nesibdev
->ibdev
.iwcm
);
3960 ib_dealloc_device(&nesibdev
->ibdev
);
3965 * nes_register_ofa_device
3967 int nes_register_ofa_device(struct nes_ib_device
*nesibdev
)
3969 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
3970 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3971 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
3974 ret
= ib_register_device(&nesvnic
->nesibdev
->ibdev
, NULL
);
3979 /* Get the resources allocated to this device */
3980 nesibdev
->max_cq
= (nesadapter
->max_cq
-NES_FIRST_QPN
) / nesadapter
->port_count
;
3981 nesibdev
->max_mr
= nesadapter
->max_mr
/ nesadapter
->port_count
;
3982 nesibdev
->max_qp
= (nesadapter
->max_qp
-NES_FIRST_QPN
) / nesadapter
->port_count
;
3983 nesibdev
->max_pd
= nesadapter
->max_pd
/ nesadapter
->port_count
;
3985 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
3986 ret
= device_create_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
3990 device_remove_file(&nesibdev
->ibdev
.dev
,
3991 nes_dev_attributes
[i
]);
3993 ib_unregister_device(&nesibdev
->ibdev
);
3998 nesvnic
->of_device_registered
= 1;
4005 * nes_unregister_ofa_device
4007 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
)
4009 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
4012 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
4013 device_remove_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
4016 if (nesvnic
->of_device_registered
) {
4017 ib_unregister_device(&nesibdev
->ibdev
);
4020 nesvnic
->of_device_registered
= 0;