2 * Copyright (c) 2006 QLogic, Inc. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <rdma/ib_smi.h>
36 #include "ipath_verbs.h"
37 #include "ipath_common.h"
40 * ipath_ud_loopback - handle send on loopback QPs
43 * @length: the length of the data to send
44 * @wr: the work request
45 * @wc: the work completion entry
47 * This is called from ipath_post_ud_send() to forward a WQE addressed
50 static void ipath_ud_loopback(struct ipath_qp
*sqp
,
51 struct ipath_sge_state
*ss
,
52 u32 length
, struct ib_send_wr
*wr
,
55 struct ipath_ibdev
*dev
= to_idev(sqp
->ibqp
.device
);
57 struct ib_ah_attr
*ah_attr
;
60 struct ipath_srq
*srq
;
61 struct ipath_sge_state rsge
;
62 struct ipath_sge
*sge
;
63 struct ipath_rwqe
*wqe
;
65 qp
= ipath_lookup_qpn(&dev
->qp_table
, wr
->wr
.ud
.remote_qpn
);
70 * Check that the qkey matches (except for QP0, see 9.6.1.4.1).
71 * Qkeys with the high order bit set mean use the
72 * qkey from the QP context instead of the WR (see 10.2.5).
74 if (unlikely(qp
->ibqp
.qp_num
&&
75 ((int) wr
->wr
.ud
.remote_qkey
< 0
76 ? qp
->qkey
: wr
->wr
.ud
.remote_qkey
) != qp
->qkey
)) {
77 /* XXX OK to lose a count once in a while. */
78 dev
->qkey_violations
++;
84 * A GRH is expected to preceed the data even if not
85 * present on the wire.
87 wc
->byte_len
= length
+ sizeof(struct ib_grh
);
89 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
) {
90 wc
->wc_flags
= IB_WC_WITH_IMM
;
91 wc
->imm_data
= wr
->imm_data
;
98 * Get the next work request entry to find where to put the data.
99 * Note that it is safe to drop the lock after changing rq->tail
100 * since ipath_post_receive() won't fill the empty slot.
103 srq
= to_isrq(qp
->ibqp
.srq
);
109 spin_lock_irqsave(&rq
->lock
, flags
);
110 if (rq
->tail
== rq
->head
) {
111 spin_unlock_irqrestore(&rq
->lock
, flags
);
115 /* Silently drop packets which are too big. */
116 wqe
= get_rwqe_ptr(rq
, rq
->tail
);
117 if (wc
->byte_len
> wqe
->length
) {
118 spin_unlock_irqrestore(&rq
->lock
, flags
);
122 wc
->wr_id
= wqe
->wr_id
;
123 rsge
.sge
= wqe
->sg_list
[0];
124 rsge
.sg_list
= wqe
->sg_list
+ 1;
125 rsge
.num_sge
= wqe
->num_sge
;
126 if (++rq
->tail
>= rq
->size
)
128 if (srq
&& srq
->ibsrq
.event_handler
) {
131 if (rq
->head
< rq
->tail
)
132 n
= rq
->size
+ rq
->head
- rq
->tail
;
134 n
= rq
->head
- rq
->tail
;
135 if (n
< srq
->limit
) {
139 spin_unlock_irqrestore(&rq
->lock
, flags
);
140 ev
.device
= qp
->ibqp
.device
;
141 ev
.element
.srq
= qp
->ibqp
.srq
;
142 ev
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
143 srq
->ibsrq
.event_handler(&ev
,
144 srq
->ibsrq
.srq_context
);
146 spin_unlock_irqrestore(&rq
->lock
, flags
);
148 spin_unlock_irqrestore(&rq
->lock
, flags
);
149 ah_attr
= &to_iah(wr
->wr
.ud
.ah
)->attr
;
150 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
151 ipath_copy_sge(&rsge
, &ah_attr
->grh
, sizeof(struct ib_grh
));
152 wc
->wc_flags
|= IB_WC_GRH
;
154 ipath_skip_sge(&rsge
, sizeof(struct ib_grh
));
157 u32 len
= sge
->length
;
162 ipath_copy_sge(&rsge
, sge
->vaddr
, len
);
165 sge
->sge_length
-= len
;
166 if (sge
->sge_length
== 0) {
168 *sge
= *ss
->sg_list
++;
169 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
170 if (++sge
->n
>= IPATH_SEGSZ
) {
171 if (++sge
->m
>= sge
->mr
->mapsz
)
176 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
178 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
182 wc
->status
= IB_WC_SUCCESS
;
183 wc
->opcode
= IB_WC_RECV
;
185 wc
->qp_num
= qp
->ibqp
.qp_num
;
186 wc
->src_qp
= sqp
->ibqp
.qp_num
;
187 /* XXX do we know which pkey matched? Only needed for GSI. */
189 wc
->slid
= ipath_layer_get_lid(dev
->dd
) |
190 (ah_attr
->src_path_bits
&
191 ((1 << (dev
->mkeyprot_resv_lmc
& 7)) - 1));
192 wc
->sl
= ah_attr
->sl
;
194 ah_attr
->dlid
& ((1 << (dev
->mkeyprot_resv_lmc
& 7)) - 1);
195 /* Signal completion event if the solicited bit is set. */
196 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), wc
,
197 wr
->send_flags
& IB_SEND_SOLICITED
);
200 if (atomic_dec_and_test(&qp
->refcount
))
205 * ipath_post_ud_send - post a UD send on QP
207 * @wr: the work request
209 * Note that we actually send the data as it is posted instead of putting
210 * the request into a ring buffer. If we wanted to use a ring buffer,
211 * we would need to save a reference to the destination address in the SWQE.
213 int ipath_post_ud_send(struct ipath_qp
*qp
, struct ib_send_wr
*wr
)
215 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
216 struct ipath_other_headers
*ohdr
;
217 struct ib_ah_attr
*ah_attr
;
218 struct ipath_sge_state ss
;
219 struct ipath_sge
*sg_list
;
231 if (!(ib_ipath_state_ops
[qp
->state
] & IPATH_PROCESS_SEND_OK
)) {
236 /* IB spec says that num_sge == 0 is OK. */
237 if (wr
->num_sge
> qp
->s_max_sge
) {
242 if (wr
->num_sge
> 1) {
243 sg_list
= kmalloc((qp
->s_max_sge
- 1) * sizeof(*sg_list
),
252 /* Check the buffer to send. */
253 ss
.sg_list
= sg_list
;
257 ss
.sge
.sge_length
= 0;
260 for (i
= 0; i
< wr
->num_sge
; i
++) {
262 if (to_ipd(qp
->ibqp
.pd
)->user
&& wr
->sg_list
[i
].lkey
== 0) {
267 if (wr
->sg_list
[i
].length
== 0)
269 if (!ipath_lkey_ok(&dev
->lk_table
, ss
.num_sge
?
270 sg_list
+ ss
.num_sge
- 1 : &ss
.sge
,
271 &wr
->sg_list
[i
], 0)) {
275 len
+= wr
->sg_list
[i
].length
;
278 /* Check for invalid packet size. */
279 if (len
> ipath_layer_get_ibmtu(dev
->dd
)) {
283 extra_bytes
= (4 - len
) & 3;
284 nwords
= (len
+ extra_bytes
) >> 2;
286 /* Construct the header. */
287 ah_attr
= &to_iah(wr
->wr
.ud
.ah
)->attr
;
288 if (ah_attr
->dlid
== 0) {
292 if (ah_attr
->dlid
>= IPATH_MULTICAST_LID_BASE
) {
293 if (ah_attr
->dlid
!= IPATH_PERMISSIVE_LID
)
294 dev
->n_multicast_xmit
++;
296 dev
->n_unicast_xmit
++;
298 dev
->n_unicast_xmit
++;
299 lid
= ah_attr
->dlid
&
300 ~((1 << (dev
->mkeyprot_resv_lmc
& 7)) - 1);
301 if (unlikely(lid
== ipath_layer_get_lid(dev
->dd
))) {
303 * Pass in an uninitialized ib_wc to save stack
306 ipath_ud_loopback(qp
, &ss
, len
, wr
, &wc
);
310 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
311 /* Header size in 32-bit words. */
313 lrh0
= IPATH_LRH_GRH
;
314 ohdr
= &qp
->s_hdr
.u
.l
.oth
;
315 qp
->s_hdr
.u
.l
.grh
.version_tclass_flow
=
316 cpu_to_be32((6 << 28) |
317 (ah_attr
->grh
.traffic_class
<< 20) |
318 ah_attr
->grh
.flow_label
);
319 qp
->s_hdr
.u
.l
.grh
.paylen
=
320 cpu_to_be16(((wr
->opcode
==
321 IB_WR_SEND_WITH_IMM
? 6 : 5) +
322 nwords
+ SIZE_OF_CRC
) << 2);
323 /* next_hdr is defined by C8-7 in ch. 8.4.1 */
324 qp
->s_hdr
.u
.l
.grh
.next_hdr
= 0x1B;
325 qp
->s_hdr
.u
.l
.grh
.hop_limit
= ah_attr
->grh
.hop_limit
;
326 /* The SGID is 32-bit aligned. */
327 qp
->s_hdr
.u
.l
.grh
.sgid
.global
.subnet_prefix
=
329 qp
->s_hdr
.u
.l
.grh
.sgid
.global
.interface_id
=
330 ipath_layer_get_guid(dev
->dd
);
331 qp
->s_hdr
.u
.l
.grh
.dgid
= ah_attr
->grh
.dgid
;
333 * Don't worry about sending to locally attached multicast
334 * QPs. It is unspecified by the spec. what happens.
337 /* Header size in 32-bit words. */
339 lrh0
= IPATH_LRH_BTH
;
340 ohdr
= &qp
->s_hdr
.u
.oth
;
342 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
) {
343 ohdr
->u
.ud
.imm_data
= wr
->imm_data
;
344 wc
.imm_data
= wr
->imm_data
;
346 bth0
= IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
<< 24;
347 } else if (wr
->opcode
== IB_WR_SEND
) {
349 bth0
= IB_OPCODE_UD_SEND_ONLY
<< 24;
354 lrh0
|= ah_attr
->sl
<< 4;
355 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
)
356 lrh0
|= 0xF000; /* Set VL (see ch. 13.5.3.1) */
357 qp
->s_hdr
.lrh
[0] = cpu_to_be16(lrh0
);
358 qp
->s_hdr
.lrh
[1] = cpu_to_be16(ah_attr
->dlid
); /* DEST LID */
359 qp
->s_hdr
.lrh
[2] = cpu_to_be16(hwords
+ nwords
+ SIZE_OF_CRC
);
360 lid
= ipath_layer_get_lid(dev
->dd
);
362 lid
|= ah_attr
->src_path_bits
&
363 ((1 << (dev
->mkeyprot_resv_lmc
& 7)) - 1);
364 qp
->s_hdr
.lrh
[3] = cpu_to_be16(lid
);
366 qp
->s_hdr
.lrh
[3] = IB_LID_PERMISSIVE
;
367 if (wr
->send_flags
& IB_SEND_SOLICITED
)
369 bth0
|= extra_bytes
<< 20;
370 bth0
|= qp
->ibqp
.qp_type
== IB_QPT_SMI
? IPATH_DEFAULT_P_KEY
:
371 ipath_layer_get_pkey(dev
->dd
, qp
->s_pkey_index
);
372 ohdr
->bth
[0] = cpu_to_be32(bth0
);
374 * Use the multicast QP if the destination LID is a multicast LID.
376 ohdr
->bth
[1] = ah_attr
->dlid
>= IPATH_MULTICAST_LID_BASE
&&
377 ah_attr
->dlid
!= IPATH_PERMISSIVE_LID
?
378 __constant_cpu_to_be32(IPATH_MULTICAST_QPN
) :
379 cpu_to_be32(wr
->wr
.ud
.remote_qpn
);
380 /* XXX Could lose a PSN count but not worth locking */
381 ohdr
->bth
[2] = cpu_to_be32(qp
->s_next_psn
++ & IPATH_PSN_MASK
);
383 * Qkeys with the high order bit set mean use the
384 * qkey from the QP context instead of the WR (see 10.2.5).
386 ohdr
->u
.ud
.deth
[0] = cpu_to_be32((int)wr
->wr
.ud
.remote_qkey
< 0 ?
387 qp
->qkey
: wr
->wr
.ud
.remote_qkey
);
388 ohdr
->u
.ud
.deth
[1] = cpu_to_be32(qp
->ibqp
.qp_num
);
389 if (ipath_verbs_send(dev
->dd
, hwords
, (u32
*) &qp
->s_hdr
,
394 /* Queue the completion status entry. */
395 if (!test_bit(IPATH_S_SIGNAL_REQ_WR
, &qp
->s_flags
) ||
396 (wr
->send_flags
& IB_SEND_SIGNALED
)) {
397 wc
.wr_id
= wr
->wr_id
;
398 wc
.status
= IB_WC_SUCCESS
;
400 wc
.opcode
= IB_WC_SEND
;
402 wc
.qp_num
= qp
->ibqp
.qp_num
;
405 /* XXX initialize other fields? */
406 ipath_cq_enter(to_icq(qp
->ibqp
.send_cq
), &wc
, 0);
417 * ipath_ud_rcv - receive an incoming UD packet
418 * @dev: the device the packet came in on
419 * @hdr: the packet header
420 * @has_grh: true if the packet has a GRH
421 * @data: the packet data
422 * @tlen: the packet length
423 * @qp: the QP the packet came on
425 * This is called from ipath_qp_rcv() to process an incoming UD packet
427 * Called at interrupt level.
429 void ipath_ud_rcv(struct ipath_ibdev
*dev
, struct ipath_ib_header
*hdr
,
430 int has_grh
, void *data
, u32 tlen
, struct ipath_qp
*qp
)
432 struct ipath_other_headers
*ohdr
;
441 struct ipath_srq
*srq
;
442 struct ipath_rwqe
*wqe
;
449 hdrsize
= 8 + 12 + 8; /* LRH + BTH + DETH */
450 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
451 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]);
454 ohdr
= &hdr
->u
.l
.oth
;
455 hdrsize
= 8 + 40 + 12 + 8; /* LRH + GRH + BTH + DETH */
457 * The header with GRH is 68 bytes and the core driver sets
458 * the eager header buffer size to 56 bytes so the last 12
459 * bytes of the IB header is in the data buffer.
462 ipath_layer_get_rcvhdrentsize(dev
->dd
) == 16;
463 if (header_in_data
) {
464 qkey
= be32_to_cpu(((__be32
*) data
)[1]);
465 src_qp
= be32_to_cpu(((__be32
*) data
)[2]);
468 qkey
= be32_to_cpu(ohdr
->u
.ud
.deth
[0]);
469 src_qp
= be32_to_cpu(ohdr
->u
.ud
.deth
[1]);
472 src_qp
&= IPATH_QPN_MASK
;
475 * Check that the permissive LID is only used on QP0
476 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
478 if (qp
->ibqp
.qp_num
) {
479 if (unlikely(hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
480 hdr
->lrh
[3] == IB_LID_PERMISSIVE
)) {
484 if (unlikely(qkey
!= qp
->qkey
)) {
485 /* XXX OK to lose a count once in a while. */
486 dev
->qkey_violations
++;
490 } else if (hdr
->lrh
[1] == IB_LID_PERMISSIVE
||
491 hdr
->lrh
[3] == IB_LID_PERMISSIVE
) {
492 struct ib_smp
*smp
= (struct ib_smp
*) data
;
494 if (smp
->mgmt_class
!= IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) {
500 /* Get the number of bytes the message was padded by. */
501 pad
= (be32_to_cpu(ohdr
->bth
[0]) >> 20) & 3;
502 if (unlikely(tlen
< (hdrsize
+ pad
+ 4))) {
503 /* Drop incomplete packets. */
507 tlen
-= hdrsize
+ pad
+ 4;
509 /* Drop invalid MAD packets (see 13.5.3.1). */
510 if (unlikely((qp
->ibqp
.qp_num
== 0 &&
512 (be16_to_cpu(hdr
->lrh
[0]) >> 12) != 15)) ||
513 (qp
->ibqp
.qp_num
== 1 &&
515 (be16_to_cpu(hdr
->lrh
[0]) >> 12) == 15)))) {
521 * A GRH is expected to preceed the data even if not
522 * present on the wire.
524 wc
.byte_len
= tlen
+ sizeof(struct ib_grh
);
527 * The opcode is in the low byte when its in network order
528 * (top byte when in host order).
530 opcode
= be32_to_cpu(ohdr
->bth
[0]) >> 24;
531 if (qp
->ibqp
.qp_num
> 1 &&
532 opcode
== IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
) {
533 if (header_in_data
) {
534 wc
.imm_data
= *(__be32
*) data
;
535 data
+= sizeof(__be32
);
537 wc
.imm_data
= ohdr
->u
.ud
.imm_data
;
538 wc
.wc_flags
= IB_WC_WITH_IMM
;
539 hdrsize
+= sizeof(u32
);
540 } else if (opcode
== IB_OPCODE_UD_SEND_ONLY
) {
549 * Get the next work request entry to find where to put the data.
550 * Note that it is safe to drop the lock after changing rq->tail
551 * since ipath_post_receive() won't fill the empty slot.
554 srq
= to_isrq(qp
->ibqp
.srq
);
560 spin_lock_irqsave(&rq
->lock
, flags
);
561 if (rq
->tail
== rq
->head
) {
562 spin_unlock_irqrestore(&rq
->lock
, flags
);
564 * Count VL15 packets dropped due to no receive buffer.
565 * Otherwise, count them as buffer overruns since usually,
566 * the HW will be able to receive packets even if there are
567 * no QPs with posted receive buffers.
569 if (qp
->ibqp
.qp_num
== 0)
570 dev
->n_vl15_dropped
++;
575 /* Silently drop packets which are too big. */
576 wqe
= get_rwqe_ptr(rq
, rq
->tail
);
577 if (wc
.byte_len
> wqe
->length
) {
578 spin_unlock_irqrestore(&rq
->lock
, flags
);
582 wc
.wr_id
= wqe
->wr_id
;
583 qp
->r_sge
.sge
= wqe
->sg_list
[0];
584 qp
->r_sge
.sg_list
= wqe
->sg_list
+ 1;
585 qp
->r_sge
.num_sge
= wqe
->num_sge
;
586 if (++rq
->tail
>= rq
->size
)
588 if (srq
&& srq
->ibsrq
.event_handler
) {
591 if (rq
->head
< rq
->tail
)
592 n
= rq
->size
+ rq
->head
- rq
->tail
;
594 n
= rq
->head
- rq
->tail
;
595 if (n
< srq
->limit
) {
599 spin_unlock_irqrestore(&rq
->lock
, flags
);
600 ev
.device
= qp
->ibqp
.device
;
601 ev
.element
.srq
= qp
->ibqp
.srq
;
602 ev
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
603 srq
->ibsrq
.event_handler(&ev
,
604 srq
->ibsrq
.srq_context
);
606 spin_unlock_irqrestore(&rq
->lock
, flags
);
608 spin_unlock_irqrestore(&rq
->lock
, flags
);
610 ipath_copy_sge(&qp
->r_sge
, &hdr
->u
.l
.grh
,
611 sizeof(struct ib_grh
));
612 wc
.wc_flags
|= IB_WC_GRH
;
614 ipath_skip_sge(&qp
->r_sge
, sizeof(struct ib_grh
));
615 ipath_copy_sge(&qp
->r_sge
, data
,
616 wc
.byte_len
- sizeof(struct ib_grh
));
617 wc
.status
= IB_WC_SUCCESS
;
618 wc
.opcode
= IB_WC_RECV
;
620 wc
.qp_num
= qp
->ibqp
.qp_num
;
622 /* XXX do we know which pkey matched? Only needed for GSI. */
624 wc
.slid
= be16_to_cpu(hdr
->lrh
[3]);
625 wc
.sl
= (be16_to_cpu(hdr
->lrh
[0]) >> 4) & 0xF;
626 dlid
= be16_to_cpu(hdr
->lrh
[1]);
628 * Save the LMC lower bits if the destination LID is a unicast LID.
630 wc
.dlid_path_bits
= dlid
>= IPATH_MULTICAST_LID_BASE
? 0 :
631 dlid
& ((1 << (dev
->mkeyprot_resv_lmc
& 7)) - 1);
632 /* Signal completion event if the solicited bit is set. */
633 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
635 __constant_cpu_to_be32(1 << 23)) != 0);