2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 #include <linux/delay.h>
42 #include "c2_status.h"
44 #define C2_MAX_ORD_PER_QP 128
45 #define C2_MAX_IRD_PER_QP 128
47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
52 static const u8 c2_opcode
[] = {
53 [IB_WR_SEND
] = C2_WR_TYPE_SEND
,
54 [IB_WR_SEND_WITH_IMM
] = NO_SUPPORT
,
55 [IB_WR_RDMA_WRITE
] = C2_WR_TYPE_RDMA_WRITE
,
56 [IB_WR_RDMA_WRITE_WITH_IMM
] = NO_SUPPORT
,
57 [IB_WR_RDMA_READ
] = C2_WR_TYPE_RDMA_READ
,
58 [IB_WR_ATOMIC_CMP_AND_SWP
] = NO_SUPPORT
,
59 [IB_WR_ATOMIC_FETCH_AND_ADD
] = NO_SUPPORT
,
62 static int to_c2_state(enum ib_qp_state ib_state
)
66 return C2_QP_STATE_IDLE
;
68 return C2_QP_STATE_RTS
;
70 return C2_QP_STATE_CLOSING
;
72 return C2_QP_STATE_CLOSING
;
74 return C2_QP_STATE_ERROR
;
80 static int to_ib_state(enum c2_qp_state c2_state
)
83 case C2_QP_STATE_IDLE
:
85 case C2_QP_STATE_CONNECTING
:
89 case C2_QP_STATE_CLOSING
:
91 case C2_QP_STATE_ERROR
:
93 case C2_QP_STATE_TERMINATE
:
100 static const char *to_ib_state_str(int ib_state
)
102 static const char *state_str
[] = {
111 if (ib_state
< IB_QPS_RESET
||
112 ib_state
> IB_QPS_ERR
)
113 return "<invalid IB QP state>";
115 ib_state
-= IB_QPS_RESET
;
116 return state_str
[ib_state
];
119 void c2_set_qp_state(struct c2_qp
*qp
, int c2_state
)
121 int new_state
= to_ib_state(c2_state
);
123 pr_debug("%s: qp[%p] state modify %s --> %s\n",
126 to_ib_state_str(qp
->state
),
127 to_ib_state_str(new_state
));
128 qp
->state
= new_state
;
131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
133 int c2_qp_modify(struct c2_dev
*c2dev
, struct c2_qp
*qp
,
134 struct ib_qp_attr
*attr
, int attr_mask
)
136 struct c2wr_qp_modify_req wr
;
137 struct c2wr_qp_modify_rep
*reply
;
138 struct c2_vq_req
*vq_req
;
143 pr_debug("%s:%d qp=%p, %s --> %s\n",
144 __FUNCTION__
, __LINE__
,
146 to_ib_state_str(qp
->state
),
147 to_ib_state_str(attr
->qp_state
));
149 vq_req
= vq_req_alloc(c2dev
);
153 c2_wr_set_id(&wr
, CCWR_QP_MODIFY
);
154 wr
.hdr
.context
= (unsigned long) vq_req
;
155 wr
.rnic_handle
= c2dev
->adapter_handle
;
156 wr
.qp_handle
= qp
->adapter_handle
;
157 wr
.ord
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
158 wr
.ird
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
159 wr
.sq_depth
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
160 wr
.rq_depth
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
162 if (attr_mask
& IB_QP_STATE
) {
163 /* Ensure the state is valid */
164 if (attr
->qp_state
< 0 || attr
->qp_state
> IB_QPS_ERR
)
167 wr
.next_qp_state
= cpu_to_be32(to_c2_state(attr
->qp_state
));
169 if (attr
->qp_state
== IB_QPS_ERR
) {
170 spin_lock_irqsave(&qp
->lock
, flags
);
171 if (qp
->cm_id
&& qp
->state
== IB_QPS_RTS
) {
172 pr_debug("Generating CLOSE event for QP-->ERR, "
173 "qp=%p, cm_id=%p\n",qp
,qp
->cm_id
);
174 /* Generate an CLOSE event */
175 vq_req
->cm_id
= qp
->cm_id
;
176 vq_req
->event
= IW_CM_EVENT_CLOSE
;
178 spin_unlock_irqrestore(&qp
->lock
, flags
);
180 next_state
= attr
->qp_state
;
182 } else if (attr_mask
& IB_QP_CUR_STATE
) {
184 if (attr
->cur_qp_state
!= IB_QPS_RTR
&&
185 attr
->cur_qp_state
!= IB_QPS_RTS
&&
186 attr
->cur_qp_state
!= IB_QPS_SQD
&&
187 attr
->cur_qp_state
!= IB_QPS_SQE
)
191 cpu_to_be32(to_c2_state(attr
->cur_qp_state
));
193 next_state
= attr
->cur_qp_state
;
200 /* reference the request struct */
201 vq_req_get(c2dev
, vq_req
);
203 err
= vq_send_wr(c2dev
, (union c2wr
*) & wr
);
205 vq_req_put(c2dev
, vq_req
);
209 err
= vq_wait_for_reply(c2dev
, vq_req
);
213 reply
= (struct c2wr_qp_modify_rep
*) (unsigned long) vq_req
->reply_msg
;
219 err
= c2_errno(reply
);
221 qp
->state
= next_state
;
224 pr_debug("%s: c2_errno=%d\n", __FUNCTION__
, err
);
227 * If we're going to error and generating the event here, then
228 * we need to remove the reference because there will be no
229 * close event generated by the adapter
231 spin_lock_irqsave(&qp
->lock
, flags
);
232 if (vq_req
->event
==IW_CM_EVENT_CLOSE
&& qp
->cm_id
) {
233 qp
->cm_id
->rem_ref(qp
->cm_id
);
236 spin_unlock_irqrestore(&qp
->lock
, flags
);
238 vq_repbuf_free(c2dev
, reply
);
240 vq_req_free(c2dev
, vq_req
);
242 pr_debug("%s:%d qp=%p, cur_state=%s\n",
243 __FUNCTION__
, __LINE__
,
245 to_ib_state_str(qp
->state
));
249 int c2_qp_set_read_limits(struct c2_dev
*c2dev
, struct c2_qp
*qp
,
252 struct c2wr_qp_modify_req wr
;
253 struct c2wr_qp_modify_rep
*reply
;
254 struct c2_vq_req
*vq_req
;
257 vq_req
= vq_req_alloc(c2dev
);
261 c2_wr_set_id(&wr
, CCWR_QP_MODIFY
);
262 wr
.hdr
.context
= (unsigned long) vq_req
;
263 wr
.rnic_handle
= c2dev
->adapter_handle
;
264 wr
.qp_handle
= qp
->adapter_handle
;
265 wr
.ord
= cpu_to_be32(ord
);
266 wr
.ird
= cpu_to_be32(ird
);
267 wr
.sq_depth
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
268 wr
.rq_depth
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
269 wr
.next_qp_state
= cpu_to_be32(C2_QP_NO_ATTR_CHANGE
);
271 /* reference the request struct */
272 vq_req_get(c2dev
, vq_req
);
274 err
= vq_send_wr(c2dev
, (union c2wr
*) & wr
);
276 vq_req_put(c2dev
, vq_req
);
280 err
= vq_wait_for_reply(c2dev
, vq_req
);
284 reply
= (struct c2wr_qp_modify_rep
*) (unsigned long)
291 err
= c2_errno(reply
);
292 vq_repbuf_free(c2dev
, reply
);
294 vq_req_free(c2dev
, vq_req
);
298 static int destroy_qp(struct c2_dev
*c2dev
, struct c2_qp
*qp
)
300 struct c2_vq_req
*vq_req
;
301 struct c2wr_qp_destroy_req wr
;
302 struct c2wr_qp_destroy_rep
*reply
;
307 * Allocate a verb request message
309 vq_req
= vq_req_alloc(c2dev
);
317 c2_wr_set_id(&wr
, CCWR_QP_DESTROY
);
318 wr
.hdr
.context
= (unsigned long) vq_req
;
319 wr
.rnic_handle
= c2dev
->adapter_handle
;
320 wr
.qp_handle
= qp
->adapter_handle
;
323 * reference the request struct. dereferenced in the int handler.
325 vq_req_get(c2dev
, vq_req
);
327 spin_lock_irqsave(&qp
->lock
, flags
);
328 if (qp
->cm_id
&& qp
->state
== IB_QPS_RTS
) {
329 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
330 "qp=%p, cm_id=%p\n",qp
,qp
->cm_id
);
331 /* Generate an CLOSE event */
333 vq_req
->cm_id
= qp
->cm_id
;
334 vq_req
->event
= IW_CM_EVENT_CLOSE
;
336 spin_unlock_irqrestore(&qp
->lock
, flags
);
341 err
= vq_send_wr(c2dev
, (union c2wr
*) & wr
);
343 vq_req_put(c2dev
, vq_req
);
348 * Wait for reply from adapter
350 err
= vq_wait_for_reply(c2dev
, vq_req
);
358 reply
= (struct c2wr_qp_destroy_rep
*) (unsigned long) (vq_req
->reply_msg
);
364 spin_lock_irqsave(&qp
->lock
, flags
);
366 qp
->cm_id
->rem_ref(qp
->cm_id
);
369 spin_unlock_irqrestore(&qp
->lock
, flags
);
371 vq_repbuf_free(c2dev
, reply
);
373 vq_req_free(c2dev
, vq_req
);
377 static int c2_alloc_qpn(struct c2_dev
*c2dev
, struct c2_qp
*qp
)
382 spin_lock_irq(&c2dev
->qp_table
.lock
);
383 ret
= idr_get_new_above(&c2dev
->qp_table
.idr
, qp
,
384 c2dev
->qp_table
.last
++, &qp
->qpn
);
385 spin_unlock_irq(&c2dev
->qp_table
.lock
);
386 } while ((ret
== -EAGAIN
) &&
387 idr_pre_get(&c2dev
->qp_table
.idr
, GFP_KERNEL
));
391 static void c2_free_qpn(struct c2_dev
*c2dev
, int qpn
)
393 spin_lock_irq(&c2dev
->qp_table
.lock
);
394 idr_remove(&c2dev
->qp_table
.idr
, qpn
);
395 spin_unlock_irq(&c2dev
->qp_table
.lock
);
398 struct c2_qp
*c2_find_qpn(struct c2_dev
*c2dev
, int qpn
)
403 spin_lock_irqsave(&c2dev
->qp_table
.lock
, flags
);
404 qp
= idr_find(&c2dev
->qp_table
.idr
, qpn
);
405 spin_unlock_irqrestore(&c2dev
->qp_table
.lock
, flags
);
409 int c2_alloc_qp(struct c2_dev
*c2dev
,
411 struct ib_qp_init_attr
*qp_attrs
, struct c2_qp
*qp
)
413 struct c2wr_qp_create_req wr
;
414 struct c2wr_qp_create_rep
*reply
;
415 struct c2_vq_req
*vq_req
;
416 struct c2_cq
*send_cq
= to_c2cq(qp_attrs
->send_cq
);
417 struct c2_cq
*recv_cq
= to_c2cq(qp_attrs
->recv_cq
);
418 unsigned long peer_pa
;
419 u32 q_size
, msg_size
, mmap_size
;
423 err
= c2_alloc_qpn(c2dev
, qp
);
426 qp
->ibqp
.qp_num
= qp
->qpn
;
427 qp
->ibqp
.qp_type
= IB_QPT_RC
;
429 /* Allocate the SQ and RQ shared pointers */
430 qp
->sq_mq
.shared
= c2_alloc_mqsp(c2dev
, c2dev
->kern_mqsp_pool
,
431 &qp
->sq_mq
.shared_dma
, GFP_KERNEL
);
432 if (!qp
->sq_mq
.shared
) {
437 qp
->rq_mq
.shared
= c2_alloc_mqsp(c2dev
, c2dev
->kern_mqsp_pool
,
438 &qp
->rq_mq
.shared_dma
, GFP_KERNEL
);
439 if (!qp
->rq_mq
.shared
) {
444 /* Allocate the verbs request */
445 vq_req
= vq_req_alloc(c2dev
);
446 if (vq_req
== NULL
) {
451 /* Initialize the work request */
452 memset(&wr
, 0, sizeof(wr
));
453 c2_wr_set_id(&wr
, CCWR_QP_CREATE
);
454 wr
.hdr
.context
= (unsigned long) vq_req
;
455 wr
.rnic_handle
= c2dev
->adapter_handle
;
456 wr
.sq_cq_handle
= send_cq
->adapter_handle
;
457 wr
.rq_cq_handle
= recv_cq
->adapter_handle
;
458 wr
.sq_depth
= cpu_to_be32(qp_attrs
->cap
.max_send_wr
+ 1);
459 wr
.rq_depth
= cpu_to_be32(qp_attrs
->cap
.max_recv_wr
+ 1);
461 wr
.flags
= cpu_to_be32(QP_RDMA_READ
| QP_RDMA_WRITE
| QP_MW_BIND
|
462 QP_ZERO_STAG
| QP_RDMA_READ_RESPONSE
);
463 wr
.send_sgl_depth
= cpu_to_be32(qp_attrs
->cap
.max_send_sge
);
464 wr
.recv_sgl_depth
= cpu_to_be32(qp_attrs
->cap
.max_recv_sge
);
465 wr
.rdma_write_sgl_depth
= cpu_to_be32(qp_attrs
->cap
.max_send_sge
);
466 wr
.shared_sq_ht
= cpu_to_be64(qp
->sq_mq
.shared_dma
);
467 wr
.shared_rq_ht
= cpu_to_be64(qp
->rq_mq
.shared_dma
);
468 wr
.ord
= cpu_to_be32(C2_MAX_ORD_PER_QP
);
469 wr
.ird
= cpu_to_be32(C2_MAX_IRD_PER_QP
);
470 wr
.pd_id
= pd
->pd_id
;
471 wr
.user_context
= (unsigned long) qp
;
473 vq_req_get(c2dev
, vq_req
);
475 /* Send the WR to the adapter */
476 err
= vq_send_wr(c2dev
, (union c2wr
*) & wr
);
478 vq_req_put(c2dev
, vq_req
);
482 /* Wait for the verb reply */
483 err
= vq_wait_for_reply(c2dev
, vq_req
);
488 /* Process the reply */
489 reply
= (struct c2wr_qp_create_rep
*) (unsigned long) (vq_req
->reply_msg
);
495 if ((err
= c2_wr_get_result(reply
)) != 0) {
499 /* Fill in the kernel QP struct */
500 atomic_set(&qp
->refcount
, 1);
501 qp
->adapter_handle
= reply
->qp_handle
;
502 qp
->state
= IB_QPS_RESET
;
503 qp
->send_sgl_depth
= qp_attrs
->cap
.max_send_sge
;
504 qp
->rdma_write_sgl_depth
= qp_attrs
->cap
.max_send_sge
;
505 qp
->recv_sgl_depth
= qp_attrs
->cap
.max_recv_sge
;
507 /* Initialize the SQ MQ */
508 q_size
= be32_to_cpu(reply
->sq_depth
);
509 msg_size
= be32_to_cpu(reply
->sq_msg_size
);
510 peer_pa
= c2dev
->pa
+ be32_to_cpu(reply
->sq_mq_start
);
511 mmap_size
= PAGE_ALIGN(sizeof(struct c2_mq_shared
) + msg_size
* q_size
);
512 mmap
= ioremap_nocache(peer_pa
, mmap_size
);
518 c2_mq_req_init(&qp
->sq_mq
,
519 be32_to_cpu(reply
->sq_mq_index
),
522 mmap
+ sizeof(struct c2_mq_shared
), /* pool start */
524 C2_MQ_ADAPTER_TARGET
);
526 /* Initialize the RQ mq */
527 q_size
= be32_to_cpu(reply
->rq_depth
);
528 msg_size
= be32_to_cpu(reply
->rq_msg_size
);
529 peer_pa
= c2dev
->pa
+ be32_to_cpu(reply
->rq_mq_start
);
530 mmap_size
= PAGE_ALIGN(sizeof(struct c2_mq_shared
) + msg_size
* q_size
);
531 mmap
= ioremap_nocache(peer_pa
, mmap_size
);
537 c2_mq_req_init(&qp
->rq_mq
,
538 be32_to_cpu(reply
->rq_mq_index
),
541 mmap
+ sizeof(struct c2_mq_shared
), /* pool start */
543 C2_MQ_ADAPTER_TARGET
);
545 vq_repbuf_free(c2dev
, reply
);
546 vq_req_free(c2dev
, vq_req
);
551 iounmap(qp
->sq_mq
.peer
);
553 destroy_qp(c2dev
, qp
);
555 vq_repbuf_free(c2dev
, reply
);
557 vq_req_free(c2dev
, vq_req
);
559 c2_free_mqsp(qp
->rq_mq
.shared
);
561 c2_free_mqsp(qp
->sq_mq
.shared
);
563 c2_free_qpn(c2dev
, qp
->qpn
);
567 void c2_free_qp(struct c2_dev
*c2dev
, struct c2_qp
*qp
)
569 struct c2_cq
*send_cq
;
570 struct c2_cq
*recv_cq
;
572 send_cq
= to_c2cq(qp
->ibqp
.send_cq
);
573 recv_cq
= to_c2cq(qp
->ibqp
.recv_cq
);
576 * Lock CQs here, so that CQ polling code can do QP lookup
577 * without taking a lock.
579 spin_lock_irq(&send_cq
->lock
);
580 if (send_cq
!= recv_cq
)
581 spin_lock(&recv_cq
->lock
);
583 c2_free_qpn(c2dev
, qp
->qpn
);
585 if (send_cq
!= recv_cq
)
586 spin_unlock(&recv_cq
->lock
);
587 spin_unlock_irq(&send_cq
->lock
);
590 * Destory qp in the rnic...
592 destroy_qp(c2dev
, qp
);
595 * Mark any unreaped CQEs as null and void.
597 c2_cq_clean(c2dev
, qp
, send_cq
->cqn
);
598 if (send_cq
!= recv_cq
)
599 c2_cq_clean(c2dev
, qp
, recv_cq
->cqn
);
601 * Unmap the MQs and return the shared pointers
602 * to the message pool.
604 iounmap(qp
->sq_mq
.peer
);
605 iounmap(qp
->rq_mq
.peer
);
606 c2_free_mqsp(qp
->sq_mq
.shared
);
607 c2_free_mqsp(qp
->rq_mq
.shared
);
609 atomic_dec(&qp
->refcount
);
610 wait_event(qp
->wait
, !atomic_read(&qp
->refcount
));
617 * Move an SGL from the user's work request struct into a CCIL Work Request
618 * message, swapping to WR byte order and ensure the total length doesn't
622 * dst - ptr to CCIL Work Request message SGL memory.
623 * src - ptr to the consumers SGL memory.
631 move_sgl(struct c2_data_addr
* dst
, struct ib_sge
*src
, int count
, u32
* p_len
,
634 u32 tot
= 0; /* running total */
635 u8 acount
= 0; /* running total non-0 len sge's */
639 * If the addition of this SGE causes the
640 * total SGL length to exceed 2^32-1, then
643 * If the current total plus the next element length
644 * wraps, then it will go negative and be less than the
647 if ((tot
+ src
->length
) < tot
) {
651 * Bug: 1456 (as well as 1498 & 1643)
652 * Skip over any sge's supplied with len=0
656 dst
->stag
= cpu_to_be32(src
->lkey
);
657 dst
->to
= cpu_to_be64(src
->addr
);
658 dst
->length
= cpu_to_be32(src
->length
);
668 * Bug: 1476 (as well as 1498, 1456 and 1643)
669 * Setup the SGL in the WR to make it easier for the RNIC.
670 * This way, the FW doesn't have to deal with special cases.
671 * Setting length=0 should be sufficient.
679 *actual_count
= acount
;
684 * Function: c2_activity (private function)
687 * Post an mq index to the host->adapter activity fifo.
690 * c2dev - ptr to c2dev structure
691 * mq_index - mq index to post
692 * shared - value most recently written to shared
699 static inline void c2_activity(struct c2_dev
*c2dev
, u32 mq_index
, u16 shared
)
702 * First read the register to see if the FIFO is full, and if so,
703 * spin until it's not. This isn't perfect -- there is no
704 * synchronization among the clients of the register, but in
705 * practice it prevents multiple CPU from hammering the bus
706 * with PCI RETRY. Note that when this does happen, the card
707 * cannot get on the bus and the card and system hang in a
708 * deadlock -- thus the need for this code. [TOT]
710 while (readl(c2dev
->regs
+ PCI_BAR0_ADAPTER_HINT
) & 0x80000000)
713 __raw_writel(C2_HINT_MAKE(mq_index
, shared
),
714 c2dev
->regs
+ PCI_BAR0_ADAPTER_HINT
);
718 * Function: qp_wr_post
721 * This in-line function allocates a MQ msg, then moves the host-copy of
722 * the completed WR into msg. Then it posts the message.
725 * q - ptr to user MQ.
726 * wr - ptr to host-copy of the WR.
727 * qp - ptr to user qp
728 * size - Number of bytes to post. Assumed to be divisible by 4.
735 static int qp_wr_post(struct c2_mq
*q
, union c2wr
* wr
, struct c2_qp
*qp
, u32 size
)
739 msg
= c2_mq_alloc(q
);
744 ((c2wr_hdr_t
*) wr
)->magic
= cpu_to_be32(CCWR_MAGIC
);
748 * Since all header fields in the WR are the same as the
749 * CQE, set the following so the adapter need not.
751 c2_wr_set_result(wr
, CCERR_PENDING
);
754 * Copy the wr down to the adapter
756 memcpy((void *) msg
, (void *) wr
, size
);
763 int c2_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*ib_wr
,
764 struct ib_send_wr
**bad_wr
)
766 struct c2_dev
*c2dev
= to_c2dev(ibqp
->device
);
767 struct c2_qp
*qp
= to_c2qp(ibqp
);
769 unsigned long lock_flags
;
777 if (qp
->state
> IB_QPS_RTS
)
783 wr
.sqwr
.sq_hdr
.user_hdr
.hdr
.context
= ib_wr
->wr_id
;
784 if (ib_wr
->send_flags
& IB_SEND_SIGNALED
) {
785 flags
|= SQ_SIGNALED
;
788 switch (ib_wr
->opcode
) {
790 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
) {
791 c2_wr_set_id(&wr
, C2_WR_TYPE_SEND_SE
);
792 msg_size
= sizeof(struct c2wr_send_req
);
794 c2_wr_set_id(&wr
, C2_WR_TYPE_SEND
);
795 msg_size
= sizeof(struct c2wr_send_req
);
798 wr
.sqwr
.send
.remote_stag
= 0;
799 msg_size
+= sizeof(struct c2_data_addr
) * ib_wr
->num_sge
;
800 if (ib_wr
->num_sge
> qp
->send_sgl_depth
) {
804 if (ib_wr
->send_flags
& IB_SEND_FENCE
) {
805 flags
|= SQ_READ_FENCE
;
807 err
= move_sgl((struct c2_data_addr
*) & (wr
.sqwr
.send
.data
),
810 &tot_len
, &actual_sge_count
);
811 wr
.sqwr
.send
.sge_len
= cpu_to_be32(tot_len
);
812 c2_wr_set_sge_count(&wr
, actual_sge_count
);
814 case IB_WR_RDMA_WRITE
:
815 c2_wr_set_id(&wr
, C2_WR_TYPE_RDMA_WRITE
);
816 msg_size
= sizeof(struct c2wr_rdma_write_req
) +
817 (sizeof(struct c2_data_addr
) * ib_wr
->num_sge
);
818 if (ib_wr
->num_sge
> qp
->rdma_write_sgl_depth
) {
822 if (ib_wr
->send_flags
& IB_SEND_FENCE
) {
823 flags
|= SQ_READ_FENCE
;
825 wr
.sqwr
.rdma_write
.remote_stag
=
826 cpu_to_be32(ib_wr
->wr
.rdma
.rkey
);
827 wr
.sqwr
.rdma_write
.remote_to
=
828 cpu_to_be64(ib_wr
->wr
.rdma
.remote_addr
);
829 err
= move_sgl((struct c2_data_addr
*)
830 & (wr
.sqwr
.rdma_write
.data
),
833 &tot_len
, &actual_sge_count
);
834 wr
.sqwr
.rdma_write
.sge_len
= cpu_to_be32(tot_len
);
835 c2_wr_set_sge_count(&wr
, actual_sge_count
);
837 case IB_WR_RDMA_READ
:
838 c2_wr_set_id(&wr
, C2_WR_TYPE_RDMA_READ
);
839 msg_size
= sizeof(struct c2wr_rdma_read_req
);
841 /* IWarp only suppots 1 sge for RDMA reads */
842 if (ib_wr
->num_sge
> 1) {
848 * Move the local and remote stag/to/len into the WR.
850 wr
.sqwr
.rdma_read
.local_stag
=
851 cpu_to_be32(ib_wr
->sg_list
->lkey
);
852 wr
.sqwr
.rdma_read
.local_to
=
853 cpu_to_be64(ib_wr
->sg_list
->addr
);
854 wr
.sqwr
.rdma_read
.remote_stag
=
855 cpu_to_be32(ib_wr
->wr
.rdma
.rkey
);
856 wr
.sqwr
.rdma_read
.remote_to
=
857 cpu_to_be64(ib_wr
->wr
.rdma
.remote_addr
);
858 wr
.sqwr
.rdma_read
.length
=
859 cpu_to_be32(ib_wr
->sg_list
->length
);
869 * If we had an error on the last wr build, then
870 * break out. Possible errors include bogus WR
871 * type, and a bogus SGL length...
880 c2_wr_set_flags(&wr
, flags
);
885 spin_lock_irqsave(&qp
->lock
, lock_flags
);
886 err
= qp_wr_post(&qp
->sq_mq
, &wr
, qp
, msg_size
);
888 spin_unlock_irqrestore(&qp
->lock
, lock_flags
);
893 * Enqueue mq index to activity FIFO.
895 c2_activity(c2dev
, qp
->sq_mq
.index
, qp
->sq_mq
.hint_count
);
896 spin_unlock_irqrestore(&qp
->lock
, lock_flags
);
906 int c2_post_receive(struct ib_qp
*ibqp
, struct ib_recv_wr
*ib_wr
,
907 struct ib_recv_wr
**bad_wr
)
909 struct c2_dev
*c2dev
= to_c2dev(ibqp
->device
);
910 struct c2_qp
*qp
= to_c2qp(ibqp
);
912 unsigned long lock_flags
;
915 if (qp
->state
> IB_QPS_RTS
)
919 * Try and post each work request
925 if (ib_wr
->num_sge
> qp
->recv_sgl_depth
) {
931 * Create local host-copy of the WR
933 wr
.rqwr
.rq_hdr
.user_hdr
.hdr
.context
= ib_wr
->wr_id
;
934 c2_wr_set_id(&wr
, CCWR_RECV
);
935 c2_wr_set_flags(&wr
, 0);
937 /* sge_count is limited to eight bits. */
938 BUG_ON(ib_wr
->num_sge
>= 256);
939 err
= move_sgl((struct c2_data_addr
*) & (wr
.rqwr
.data
),
941 ib_wr
->num_sge
, &tot_len
, &actual_sge_count
);
942 c2_wr_set_sge_count(&wr
, actual_sge_count
);
945 * If we had an error on the last wr build, then
946 * break out. Possible errors include bogus WR
947 * type, and a bogus SGL length...
953 spin_lock_irqsave(&qp
->lock
, lock_flags
);
954 err
= qp_wr_post(&qp
->rq_mq
, &wr
, qp
, qp
->rq_mq
.msg_size
);
956 spin_unlock_irqrestore(&qp
->lock
, lock_flags
);
961 * Enqueue mq index to activity FIFO
963 c2_activity(c2dev
, qp
->rq_mq
.index
, qp
->rq_mq
.hint_count
);
964 spin_unlock_irqrestore(&qp
->lock
, lock_flags
);
974 void __devinit
c2_init_qp_table(struct c2_dev
*c2dev
)
976 spin_lock_init(&c2dev
->qp_table
.lock
);
977 idr_init(&c2dev
->qp_table
.idr
);
980 void __devexit
c2_cleanup_qp_table(struct c2_dev
*c2dev
)
982 idr_destroy(&c2dev
->qp_table
.idr
);