2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/sched.h>
35 #include <rdma/ib_smi.h>
37 #include "ipath_verbs.h"
38 #include "ipath_kernel.h"
41 * ipath_ud_loopback - handle send on loopback QPs
42 * @sqp: the sending QP
43 * @swqe: the send work request
45 * This is called from ipath_make_ud_req() to forward a WQE addressed
47 * Note that the receive interrupt handler may be calling ipath_ud_rcv()
48 * while this is being called.
50 static void ipath_ud_loopback(struct ipath_qp
*sqp
, struct ipath_swqe
*swqe
)
52 struct ipath_ibdev
*dev
= to_idev(sqp
->ibqp
.device
);
54 struct ib_ah_attr
*ah_attr
;
57 struct ipath_srq
*srq
;
58 struct ipath_sge_state rsge
;
59 struct ipath_sge
*sge
;
61 struct ipath_rwqe
*wqe
;
62 void (*handler
)(struct ib_event
*, void *);
68 qp
= ipath_lookup_qpn(&dev
->qp_table
, swqe
->wr
.wr
.ud
.remote_qpn
);
69 if (!qp
|| !(ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_RECV_OK
)) {
75 * Check that the qkey matches (except for QP0, see 9.6.1.4.1).
76 * Qkeys with the high order bit set mean use the
77 * qkey from the QP context instead of the WR (see 10.2.5).
79 if (unlikely(qp
->ibqp
.qp_num
&&
80 ((int) swqe
->wr
.wr
.ud
.remote_qkey
< 0 ?
81 sqp
->qkey
: swqe
->wr
.wr
.ud
.remote_qkey
) != qp
->qkey
)) {
82 dev
->qkey_violations
++;
88 * A GRH is expected to preceed the data even if not
89 * present on the wire.
91 length
= swqe
->length
;
92 memset(&wc
, 0, sizeof wc
);
93 wc
.byte_len
= length
+ sizeof(struct ib_grh
);
95 if (swqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
96 wc
.wc_flags
= IB_WC_WITH_IMM
;
97 wc
.ex
.imm_data
= swqe
->wr
.ex
.imm_data
;
101 * This would be a lot simpler if we could call ipath_get_rwqe()
102 * but that uses state that the receive interrupt handler uses
103 * so we would need to lock out receive interrupts while doing
107 srq
= to_isrq(qp
->ibqp
.srq
);
108 handler
= srq
->ibsrq
.event_handler
;
117 * Get the next work request entry to find where to put the data.
118 * Note that it is safe to drop the lock after changing rq->tail
119 * since ipath_post_receive() won't fill the empty slot.
121 spin_lock_irqsave(&rq
->lock
, flags
);
124 /* Validate tail before using it since it is user writable. */
125 if (tail
>= rq
->size
)
127 if (unlikely(tail
== wq
->head
)) {
128 spin_unlock_irqrestore(&rq
->lock
, flags
);
132 wqe
= get_rwqe_ptr(rq
, tail
);
133 rsge
.sg_list
= qp
->r_ud_sg_list
;
134 if (!ipath_init_sge(qp
, wqe
, &rlen
, &rsge
)) {
135 spin_unlock_irqrestore(&rq
->lock
, flags
);
139 /* Silently drop packets which are too big. */
140 if (wc
.byte_len
> rlen
) {
141 spin_unlock_irqrestore(&rq
->lock
, flags
);
145 if (++tail
>= rq
->size
)
148 wc
.wr_id
= wqe
->wr_id
;
153 * validate head pointer value and compute
154 * the number of remaining WQEs.
160 n
+= rq
->size
- tail
;
163 if (n
< srq
->limit
) {
167 spin_unlock_irqrestore(&rq
->lock
, flags
);
168 ev
.device
= qp
->ibqp
.device
;
169 ev
.element
.srq
= qp
->ibqp
.srq
;
170 ev
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
171 handler(&ev
, srq
->ibsrq
.srq_context
);
173 spin_unlock_irqrestore(&rq
->lock
, flags
);
175 spin_unlock_irqrestore(&rq
->lock
, flags
);
177 ah_attr
= &to_iah(swqe
->wr
.wr
.ud
.ah
)->attr
;
178 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
179 ipath_copy_sge(&rsge
, &ah_attr
->grh
, sizeof(struct ib_grh
));
180 wc
.wc_flags
|= IB_WC_GRH
;
182 ipath_skip_sge(&rsge
, sizeof(struct ib_grh
));
185 u32 len
= sge
->length
;
189 if (len
> sge
->sge_length
)
190 len
= sge
->sge_length
;
192 ipath_copy_sge(&rsge
, sge
->vaddr
, len
);
195 sge
->sge_length
-= len
;
196 if (sge
->sge_length
== 0) {
197 if (--swqe
->wr
.num_sge
)
199 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
200 if (++sge
->n
>= IPATH_SEGSZ
) {
201 if (++sge
->m
>= sge
->mr
->mapsz
)
206 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
208 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
212 wc
.status
= IB_WC_SUCCESS
;
213 wc
.opcode
= IB_WC_RECV
;
215 wc
.src_qp
= sqp
->ibqp
.qp_num
;
217 wc
.slid
= dev
->dd
->ipath_lid
|
218 (ah_attr
->src_path_bits
&
219 ((1 << dev
->dd
->ipath_lmc
) - 1));
222 ah_attr
->dlid
& ((1 << dev
->dd
->ipath_lmc
) - 1);
224 /* Signal completion event if the solicited bit is set. */
225 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
226 swqe
->wr
.send_flags
& IB_SEND_SOLICITED
);
228 if (atomic_dec_and_test(&qp
->refcount
))
234 * ipath_make_ud_req - construct a UD request packet
237 * Return 1 if constructed; otherwise, return 0.
239 int ipath_make_ud_req(struct ipath_qp
*qp
)
241 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
242 struct ipath_other_headers
*ohdr
;
243 struct ib_ah_attr
*ah_attr
;
244 struct ipath_swqe
*wqe
;
254 spin_lock_irqsave(&qp
->s_lock
, flags
);
256 if (!(ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_NEXT_SEND_OK
)) {
257 if (!(ib_ipath_state_ops
[qp
->state
] & IPATH_FLUSH_SEND
))
259 /* We are in the error state, flush the work request. */
260 if (qp
->s_last
== qp
->s_head
)
262 /* If DMAs are in progress, we can't flush immediately. */
263 if (atomic_read(&qp
->s_dma_busy
)) {
264 qp
->s_flags
|= IPATH_S_WAIT_DMA
;
267 wqe
= get_swqe_ptr(qp
, qp
->s_last
);
268 ipath_send_complete(qp
, wqe
, IB_WC_WR_FLUSH_ERR
);
272 if (qp
->s_cur
== qp
->s_head
)
275 wqe
= get_swqe_ptr(qp
, qp
->s_cur
);
276 next_cur
= qp
->s_cur
+ 1;
277 if (next_cur
>= qp
->s_size
)
280 /* Construct the header. */
281 ah_attr
= &to_iah(wqe
->wr
.wr
.ud
.ah
)->attr
;
282 if (ah_attr
->dlid
>= IPATH_MULTICAST_LID_BASE
) {
283 if (ah_attr
->dlid
!= IPATH_PERMISSIVE_LID
)
284 dev
->n_multicast_xmit
++;
286 dev
->n_unicast_xmit
++;
288 dev
->n_unicast_xmit
++;
289 lid
= ah_attr
->dlid
& ~((1 << dev
->dd
->ipath_lmc
) - 1);
290 if (unlikely(lid
== dev
->dd
->ipath_lid
)) {
291 if (atomic_read(&qp
->s_dma_busy
)) {
292 qp
->s_flags
|= IPATH_S_WAIT_DMA
;
295 qp
->s_cur
= next_cur
;
296 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
297 ipath_ud_loopback(qp
, wqe
);
298 spin_lock_irqsave(&qp
->s_lock
, flags
);
299 ipath_send_complete(qp
, wqe
, IB_WC_SUCCESS
);
304 qp
->s_cur
= next_cur
;
305 extra_bytes
= -wqe
->length
& 3;
306 nwords
= (wqe
->length
+ extra_bytes
) >> 2;
308 /* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
310 qp
->s_cur_size
= wqe
->length
;
311 qp
->s_cur_sge
= &qp
->s_sge
;
312 qp
->s_dmult
= ah_attr
->static_rate
;
314 qp
->s_sge
.sge
= wqe
->sg_list
[0];
315 qp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
316 qp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
318 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
319 /* Header size in 32-bit words. */
320 qp
->s_hdrwords
+= ipath_make_grh(dev
, &qp
->s_hdr
.u
.l
.grh
,
322 qp
->s_hdrwords
, nwords
);
323 lrh0
= IPATH_LRH_GRH
;
324 ohdr
= &qp
->s_hdr
.u
.l
.oth
;
326 * Don't worry about sending to locally attached multicast
327 * QPs. It is unspecified by the spec. what happens.
330 /* Header size in 32-bit words. */
331 lrh0
= IPATH_LRH_BTH
;
332 ohdr
= &qp
->s_hdr
.u
.oth
;
334 if (wqe
->wr
.opcode
== IB_WR_SEND_WITH_IMM
) {
336 ohdr
->u
.ud
.imm_data
= wqe
->wr
.ex
.imm_data
;
337 bth0
= IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
<< 24;
339 bth0
= IB_OPCODE_UD_SEND_ONLY
<< 24;
340 lrh0
|= ah_attr
->sl
<< 4;
341 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
)
342 lrh0
|= 0xF000; /* Set VL (see ch. 13.5.3.1) */
343 qp
->s_hdr
.lrh
[0] = cpu_to_be16(lrh0
);
344 qp
->s_hdr
.lrh
[1] = cpu_to_be16(ah_attr
->dlid
); /* DEST LID */
345 qp
->s_hdr
.lrh
[2] = cpu_to_be16(qp
->s_hdrwords
+ nwords
+
347 lid
= dev
->dd
->ipath_lid
;
349 lid
|= ah_attr
->src_path_bits
&
350 ((1 << dev
->dd
->ipath_lmc
) - 1);
351 qp
->s_hdr
.lrh
[3] = cpu_to_be16(lid
);
353 qp
->s_hdr
.lrh
[3] = IB_LID_PERMISSIVE
;
354 if (wqe
->wr
.send_flags
& IB_SEND_SOLICITED
)
356 bth0
|= extra_bytes
<< 20;
357 bth0
|= qp
->ibqp
.qp_type
== IB_QPT_SMI
? IPATH_DEFAULT_P_KEY
:
358 ipath_get_pkey(dev
->dd
, qp
->s_pkey_index
);
359 ohdr
->bth
[0] = cpu_to_be32(bth0
);
361 * Use the multicast QP if the destination LID is a multicast LID.
363 ohdr
->bth
[1] = ah_attr
->dlid
>= IPATH_MULTICAST_LID_BASE
&&
364 ah_attr
->dlid
!= IPATH_PERMISSIVE_LID
?
365 cpu_to_be32(IPATH_MULTICAST_QPN
) :
366 cpu_to_be32(wqe
->wr
.wr
.ud
.remote_qpn
);
367 ohdr
->bth
[2] = cpu_to_be32(qp
->s_next_psn
++ & IPATH_PSN_MASK
);
369 * Qkeys with the high order bit set mean use the
370 * qkey from the QP context instead of the WR (see 10.2.5).
372 ohdr
->u
.ud
.deth
[0] = cpu_to_be32((int)wqe
->wr
.wr
.ud
.remote_qkey
< 0 ?
373 qp
->qkey
: wqe
->wr
.wr
.ud
.remote_qkey
);
374 ohdr
->u
.ud
.deth
[1] = cpu_to_be32(qp
->ibqp
.qp_num
);
381 qp
->s_flags
&= ~IPATH_S_BUSY
;
383 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
388 * ipath_ud_rcv - receive an incoming UD packet
389 * @dev: the device the packet came in on
390 * @hdr: the packet header
391 * @has_grh: true if the packet has a GRH
392 * @data: the packet data
393 * @tlen: the packet length
394 * @qp: the QP the packet came on
396 * This is called from ipath_qp_rcv() to process an incoming UD packet
398 * Called at interrupt level.
400 void ipath_ud_rcv(struct ipath_ibdev
*dev
, struct ipath_ib_header
*hdr
,
401 int has_grh
, void *data
, u32 tlen
, struct ipath_qp
*qp
)
403 struct ipath_other_headers
*ohdr
;
416 hdrsize
= 8 + 12 + 8; /* LRH + BTH + DETH */
417 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
418 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]);
421 ohdr
= &hdr
->u
.l
.oth
;
422 hdrsize
= 8 + 40 + 12 + 8; /* LRH + GRH + BTH + DETH */
424 * The header with GRH is 68 bytes and the core driver sets
425 * the eager header buffer size to 56 bytes so the last 12
426 * bytes of the IB header is in the data buffer.
428 header_in_data
= dev
->dd
->ipath_rcvhdrentsize
== 16;
429 if (header_in_data
) {
430 qkey
= be32_to_cpu(((__be32
*) data
)[1]);
431 src_qp
= be32_to_cpu(((__be32
*) data
)[2]);
434 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
435 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]);
438 src_qp
&= IPATH_QPN_MASK
;
441 * Check that the permissive LID is only used on QP0
442 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
444 if (qp
->ibqp
.qp_num
) {
445 if (unlikely(hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
446 hdr
->lrh
[3] == IB_LID_PERMISSIVE
)) {
450 if (unlikely(qkey
!= qp
->qkey
)) {
451 dev
->qkey_violations
++;
455 } else if (hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
456 hdr
->lrh
[3] == IB_LID_PERMISSIVE
) {
457 struct ib_smp
*smp
= (struct ib_smp
*) data
;
459 if (smp
->mgmt_class
!= IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) {
466 * The opcode is in the low byte when its in network order
467 * (top byte when in host order).
469 opcode
= be32_to_cpu(ohdr
->bth
[0]) >> 24;
470 if (qp
->ibqp
.qp_num
> 1 &&
471 opcode
== IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
) {
472 if (header_in_data
) {
473 wc
.ex
.imm_data
= *(__be32
*) data
;
474 data
+= sizeof(__be32
);
476 wc
.ex
.imm_data
= ohdr
->u
.ud
.imm_data
;
477 wc
.wc_flags
= IB_WC_WITH_IMM
;
478 hdrsize
+= sizeof(u32
);
479 } else if (opcode
== IB_OPCODE_UD_SEND_ONLY
) {
487 /* Get the number of bytes the message was padded by. */
488 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
489 if (unlikely(tlen
< (hdrsize
+ pad
+ 4))) {
490 /* Drop incomplete packets. */
494 tlen
-= hdrsize
+ pad
+ 4;
496 /* Drop invalid MAD packets (see 13.5.3.1). */
497 if (unlikely((qp
->ibqp
.qp_num
== 0 &&
499 (be16_to_cpu(hdr
->lrh
[0]) >> 12) != 15)) ||
500 (qp
->ibqp
.qp_num
== 1 &&
502 (be16_to_cpu(hdr
->lrh
[0]) >> 12) == 15)))) {
508 * A GRH is expected to preceed the data even if not
509 * present on the wire.
511 wc
.byte_len
= tlen
+ sizeof(struct ib_grh
);
514 * Get the next work request entry to find where to put the data.
516 if (qp
->r_flags
& IPATH_R_REUSE_SGE
)
517 qp
->r_flags
&= ~IPATH_R_REUSE_SGE
;
518 else if (!ipath_get_rwqe(qp
, 0)) {
520 * Count VL15 packets dropped due to no receive buffer.
521 * Otherwise, count them as buffer overruns since usually,
522 * the HW will be able to receive packets even if there are
523 * no QPs with posted receive buffers.
525 if (qp
->ibqp
.qp_num
== 0)
526 dev
->n_vl15_dropped
++;
531 /* Silently drop packets which are too big. */
532 if (wc
.byte_len
> qp
->r_len
) {
533 qp
->r_flags
|= IPATH_R_REUSE_SGE
;
538 ipath_copy_sge(&qp
->r_sge
, &hdr
->u
.l
.grh
,
539 sizeof(struct ib_grh
));
540 wc
.wc_flags
|= IB_WC_GRH
;
542 ipath_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
));
543 ipath_copy_sge(&qp
->r_sge
, data
,
544 wc
.byte_len
- sizeof(struct ib_grh
));
545 if (!test_and_clear_bit(IPATH_R_WRID_VALID
, &qp
->r_aflags
))
547 wc
.wr_id
= qp
->r_wr_id
;
548 wc
.status
= IB_WC_SUCCESS
;
549 wc
.opcode
= IB_WC_RECV
;
554 wc
.slid
= be16_to_cpu(hdr
->lrh
[3]);
555 wc
.sl
= (be16_to_cpu(hdr
->lrh
[0]) >> 4) & 0xF;
556 dlid
= be16_to_cpu(hdr
->lrh
[1]);
558 * Save the LMC lower bits if the destination LID is a unicast LID.
560 wc
.dlid_path_bits
= dlid
>= IPATH_MULTICAST_LID_BASE
? 0 :
561 dlid
& ((1 << dev
->dd
->ipath_lmc
) - 1);
563 /* Signal completion event if the solicited bit is set. */
564 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
566 cpu_to_be32(1 << 23)) != 0);