2 * Copyright (c) 2006 QLogic, Inc. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include "ipath_verbs.h"
35 #include "ipath_kernel.h"
38 * Convert the AETH RNR timeout code into the number of milliseconds.
40 const u32 ib_ipath_rnr_table
[32] = {
76 * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device
79 * XXX Use a simple list for now. We might need a priority
80 * queue if we have lots of QPs waiting for RNR timeouts
81 * but that should be rare.
83 void ipath_insert_rnr_queue(struct ipath_qp
*qp
)
85 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
88 spin_lock_irqsave(&dev
->pending_lock
, flags
);
89 if (list_empty(&dev
->rnrwait
))
90 list_add(&qp
->timerwait
, &dev
->rnrwait
);
92 struct list_head
*l
= &dev
->rnrwait
;
93 struct ipath_qp
*nqp
= list_entry(l
->next
, struct ipath_qp
,
96 while (qp
->s_rnr_timeout
>= nqp
->s_rnr_timeout
) {
97 qp
->s_rnr_timeout
-= nqp
->s_rnr_timeout
;
99 if (l
->next
== &dev
->rnrwait
)
101 nqp
= list_entry(l
->next
, struct ipath_qp
,
104 list_add(&qp
->timerwait
, l
);
106 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
109 static int init_sge(struct ipath_qp
*qp
, struct ipath_rwqe
*wqe
)
111 int user
= to_ipd(qp
->ibqp
.pd
)->user
;
116 for (i
= j
= 0; i
< wqe
->num_sge
; i
++) {
117 if (wqe
->sg_list
[i
].length
== 0)
120 if ((user
&& wqe
->sg_list
[i
].lkey
== 0) ||
121 !ipath_lkey_ok(qp
, &qp
->r_sg_list
[j
], &wqe
->sg_list
[i
],
122 IB_ACCESS_LOCAL_WRITE
))
124 qp
->r_len
+= wqe
->sg_list
[i
].length
;
127 qp
->r_sge
.sge
= qp
->r_sg_list
[0];
128 qp
->r_sge
.sg_list
= qp
->r_sg_list
+ 1;
129 qp
->r_sge
.num_sge
= j
;
134 wc
.wr_id
= wqe
->wr_id
;
135 wc
.status
= IB_WC_LOC_PROT_ERR
;
136 wc
.opcode
= IB_WC_RECV
;
140 wc
.qp_num
= qp
->ibqp
.qp_num
;
146 wc
.dlid_path_bits
= 0;
148 /* Signal solicited completion event. */
149 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
, 1);
156 * ipath_get_rwqe - copy the next RWQE into the QP's RWQE
158 * @wr_id_only: update wr_id only, not SGEs
160 * Return 0 if no RWQE is available, otherwise return 1.
162 * Can be called from interrupt level.
164 int ipath_get_rwqe(struct ipath_qp
*qp
, int wr_id_only
)
168 struct ipath_rwq
*wq
;
169 struct ipath_srq
*srq
;
170 struct ipath_rwqe
*wqe
;
171 void (*handler
)(struct ib_event
*, void *);
176 srq
= to_isrq(qp
->ibqp
.srq
);
177 handler
= srq
->ibsrq
.event_handler
;
185 spin_lock_irqsave(&rq
->lock
, flags
);
188 /* Validate tail before using it since it is user writable. */
189 if (tail
>= rq
->size
)
192 if (unlikely(tail
== wq
->head
)) {
193 spin_unlock_irqrestore(&rq
->lock
, flags
);
197 wqe
= get_rwqe_ptr(rq
, tail
);
198 if (++tail
>= rq
->size
)
200 } while (!wr_id_only
&& !init_sge(qp
, wqe
));
201 qp
->r_wr_id
= wqe
->wr_id
;
209 * validate head pointer value and compute
210 * the number of remaining WQEs.
216 n
+= rq
->size
- tail
;
219 if (n
< srq
->limit
) {
223 spin_unlock_irqrestore(&rq
->lock
, flags
);
224 ev
.device
= qp
->ibqp
.device
;
225 ev
.element
.srq
= qp
->ibqp
.srq
;
226 ev
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
227 handler(&ev
, srq
->ibsrq
.srq_context
);
231 spin_unlock_irqrestore(&rq
->lock
, flags
);
238 * ipath_ruc_loopback - handle UC and RC lookback requests
239 * @sqp: the loopback QP
241 * This is called from ipath_do_uc_send() or ipath_do_rc_send() to
242 * forward a WQE addressed to the same HCA.
243 * Note that although we are single threaded due to the tasklet, we still
244 * have to protect against post_send(). We don't have to worry about
245 * receive interrupts since this is a connected protocol and all packets
246 * will pass through here.
248 static void ipath_ruc_loopback(struct ipath_qp
*sqp
)
250 struct ipath_ibdev
*dev
= to_idev(sqp
->ibqp
.device
);
252 struct ipath_swqe
*wqe
;
253 struct ipath_sge
*sge
;
258 qp
= ipath_lookup_qpn(&dev
->qp_table
, sqp
->remote_qpn
);
265 spin_lock_irqsave(&sqp
->s_lock
, flags
);
267 if (!(ib_ipath_state_ops
[sqp
->state
] & IPATH_PROCESS_SEND_OK
)) {
268 spin_unlock_irqrestore(&sqp
->s_lock
, flags
);
272 /* Get the next send request. */
273 if (sqp
->s_last
== sqp
->s_head
) {
274 /* Send work queue is empty. */
275 spin_unlock_irqrestore(&sqp
->s_lock
, flags
);
280 * We can rely on the entry not changing without the s_lock
281 * being held until we update s_last.
283 wqe
= get_swqe_ptr(sqp
, sqp
->s_last
);
284 spin_unlock_irqrestore(&sqp
->s_lock
, flags
);
289 sqp
->s_sge
.sge
= wqe
->sg_list
[0];
290 sqp
->s_sge
.sg_list
= wqe
->sg_list
+ 1;
291 sqp
->s_sge
.num_sge
= wqe
->wr
.num_sge
;
292 sqp
->s_len
= wqe
->length
;
293 switch (wqe
->wr
.opcode
) {
294 case IB_WR_SEND_WITH_IMM
:
295 wc
.wc_flags
= IB_WC_WITH_IMM
;
296 wc
.imm_data
= wqe
->wr
.imm_data
;
299 if (!ipath_get_rwqe(qp
, 0)) {
302 if (qp
->ibqp
.qp_type
== IB_QPT_UC
)
304 if (sqp
->s_rnr_retry
== 0) {
305 wc
.status
= IB_WC_RNR_RETRY_EXC_ERR
;
308 if (sqp
->s_rnr_retry_cnt
< 7)
312 ib_ipath_rnr_table
[sqp
->r_min_rnr_timer
];
313 ipath_insert_rnr_queue(sqp
);
318 case IB_WR_RDMA_WRITE_WITH_IMM
:
319 wc
.wc_flags
= IB_WC_WITH_IMM
;
320 wc
.imm_data
= wqe
->wr
.imm_data
;
321 if (!ipath_get_rwqe(qp
, 1))
324 case IB_WR_RDMA_WRITE
:
325 if (wqe
->length
== 0)
327 if (unlikely(!ipath_rkey_ok(qp
, &qp
->r_sge
, wqe
->length
,
328 wqe
->wr
.wr
.rdma
.remote_addr
,
329 wqe
->wr
.wr
.rdma
.rkey
,
330 IB_ACCESS_REMOTE_WRITE
))) {
332 wc
.status
= IB_WC_REM_ACCESS_ERR
;
334 wc
.wr_id
= wqe
->wr
.wr_id
;
335 wc
.opcode
= ib_ipath_wc_opcode
[wqe
->wr
.opcode
];
338 wc
.qp_num
= sqp
->ibqp
.qp_num
;
339 wc
.src_qp
= sqp
->remote_qpn
;
341 wc
.slid
= sqp
->remote_ah_attr
.dlid
;
342 wc
.sl
= sqp
->remote_ah_attr
.sl
;
343 wc
.dlid_path_bits
= 0;
345 ipath_sqerror_qp(sqp
, &wc
);
350 case IB_WR_RDMA_READ
:
351 if (unlikely(!ipath_rkey_ok(qp
, &sqp
->s_sge
, wqe
->length
,
352 wqe
->wr
.wr
.rdma
.remote_addr
,
353 wqe
->wr
.wr
.rdma
.rkey
,
354 IB_ACCESS_REMOTE_READ
)))
356 if (unlikely(!(qp
->qp_access_flags
&
357 IB_ACCESS_REMOTE_READ
)))
359 qp
->r_sge
.sge
= wqe
->sg_list
[0];
360 qp
->r_sge
.sg_list
= wqe
->sg_list
+ 1;
361 qp
->r_sge
.num_sge
= wqe
->wr
.num_sge
;
364 case IB_WR_ATOMIC_CMP_AND_SWP
:
365 case IB_WR_ATOMIC_FETCH_AND_ADD
:
366 if (unlikely(!ipath_rkey_ok(qp
, &qp
->r_sge
, sizeof(u64
),
367 wqe
->wr
.wr
.rdma
.remote_addr
,
368 wqe
->wr
.wr
.rdma
.rkey
,
369 IB_ACCESS_REMOTE_ATOMIC
)))
371 /* Perform atomic OP and save result. */
372 sdata
= wqe
->wr
.wr
.atomic
.swap
;
373 spin_lock_irqsave(&dev
->pending_lock
, flags
);
374 qp
->r_atomic_data
= *(u64
*) qp
->r_sge
.sge
.vaddr
;
375 if (wqe
->wr
.opcode
== IB_WR_ATOMIC_FETCH_AND_ADD
)
376 *(u64
*) qp
->r_sge
.sge
.vaddr
=
377 qp
->r_atomic_data
+ sdata
;
378 else if (qp
->r_atomic_data
== wqe
->wr
.wr
.atomic
.compare_add
)
379 *(u64
*) qp
->r_sge
.sge
.vaddr
= sdata
;
380 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
381 *(u64
*) sqp
->s_sge
.sge
.vaddr
= qp
->r_atomic_data
;
388 sge
= &sqp
->s_sge
.sge
;
390 u32 len
= sqp
->s_len
;
392 if (len
> sge
->length
)
395 ipath_copy_sge(&qp
->r_sge
, sge
->vaddr
, len
);
398 sge
->sge_length
-= len
;
399 if (sge
->sge_length
== 0) {
400 if (--sqp
->s_sge
.num_sge
)
401 *sge
= *sqp
->s_sge
.sg_list
++;
402 } else if (sge
->length
== 0 && sge
->mr
!= NULL
) {
403 if (++sge
->n
>= IPATH_SEGSZ
) {
404 if (++sge
->m
>= sge
->mr
->mapsz
)
409 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
411 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
416 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE
||
417 wqe
->wr
.opcode
== IB_WR_RDMA_READ
)
420 if (wqe
->wr
.opcode
== IB_WR_RDMA_WRITE_WITH_IMM
)
421 wc
.opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
423 wc
.opcode
= IB_WC_RECV
;
424 wc
.wr_id
= qp
->r_wr_id
;
425 wc
.status
= IB_WC_SUCCESS
;
427 wc
.byte_len
= wqe
->length
;
428 wc
.qp_num
= qp
->ibqp
.qp_num
;
429 wc
.src_qp
= qp
->remote_qpn
;
430 /* XXX do we know which pkey matched? Only needed for GSI. */
432 wc
.slid
= qp
->remote_ah_attr
.dlid
;
433 wc
.sl
= qp
->remote_ah_attr
.sl
;
434 wc
.dlid_path_bits
= 0;
435 /* Signal completion event if the solicited bit is set. */
436 ipath_cq_enter(to_icq(qp
->ibqp
.recv_cq
), &wc
,
437 wqe
->wr
.send_flags
& IB_SEND_SOLICITED
);
440 sqp
->s_rnr_retry
= sqp
->s_rnr_retry_cnt
;
442 if (!test_bit(IPATH_S_SIGNAL_REQ_WR
, &sqp
->s_flags
) ||
443 (wqe
->wr
.send_flags
& IB_SEND_SIGNALED
)) {
444 wc
.wr_id
= wqe
->wr
.wr_id
;
445 wc
.status
= IB_WC_SUCCESS
;
446 wc
.opcode
= ib_ipath_wc_opcode
[wqe
->wr
.opcode
];
448 wc
.byte_len
= wqe
->length
;
449 wc
.qp_num
= sqp
->ibqp
.qp_num
;
454 wc
.dlid_path_bits
= 0;
456 ipath_cq_enter(to_icq(sqp
->ibqp
.send_cq
), &wc
, 0);
459 /* Update s_last now that we are finished with the SWQE */
460 spin_lock_irqsave(&sqp
->s_lock
, flags
);
461 if (++sqp
->s_last
>= sqp
->s_size
)
463 spin_unlock_irqrestore(&sqp
->s_lock
, flags
);
467 if (atomic_dec_and_test(&qp
->refcount
))
471 static int want_buffer(struct ipath_devdata
*dd
)
473 set_bit(IPATH_S_PIOINTBUFAVAIL
, &dd
->ipath_sendctrl
);
474 ipath_write_kreg(dd
, dd
->ipath_kregs
->kr_sendctrl
,
481 * ipath_no_bufs_available - tell the layer driver we need buffers
482 * @qp: the QP that caused the problem
483 * @dev: the device we ran out of buffers on
485 * Called when we run out of PIO buffers.
487 void ipath_no_bufs_available(struct ipath_qp
*qp
, struct ipath_ibdev
*dev
)
491 spin_lock_irqsave(&dev
->pending_lock
, flags
);
492 if (list_empty(&qp
->piowait
))
493 list_add_tail(&qp
->piowait
, &dev
->piowait
);
494 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
496 * Note that as soon as want_buffer() is called and
497 * possibly before it returns, ipath_ib_piobufavail()
498 * could be called. If we are still in the tasklet function,
499 * tasklet_hi_schedule() will not call us until the next time
500 * tasklet_hi_schedule() is called.
501 * We clear the tasklet flag now since we are committing to return
502 * from the tasklet function.
504 clear_bit(IPATH_S_BUSY
, &qp
->s_flags
);
505 tasklet_unlock(&qp
->s_task
);
506 want_buffer(dev
->dd
);
511 * ipath_post_ruc_send - post RC and UC sends
512 * @qp: the QP to post on
513 * @wr: the work request to send
515 int ipath_post_ruc_send(struct ipath_qp
*qp
, struct ib_send_wr
*wr
)
517 struct ipath_swqe
*wqe
;
525 * Don't allow RDMA reads or atomic operations on UC or
526 * undefined operations.
527 * Make sure buffer is large enough to hold the result for atomics.
529 if (qp
->ibqp
.qp_type
== IB_QPT_UC
) {
530 if ((unsigned) wr
->opcode
>= IB_WR_RDMA_READ
) {
534 } else if ((unsigned) wr
->opcode
> IB_WR_ATOMIC_FETCH_AND_ADD
) {
537 } else if (wr
->opcode
>= IB_WR_ATOMIC_CMP_AND_SWP
&&
539 wr
->sg_list
[0].length
< sizeof(u64
) ||
540 wr
->sg_list
[0].addr
& (sizeof(u64
) - 1))) {
544 /* IB spec says that num_sge == 0 is OK. */
545 if (wr
->num_sge
> qp
->s_max_sge
) {
549 spin_lock_irqsave(&qp
->s_lock
, flags
);
550 next
= qp
->s_head
+ 1;
551 if (next
>= qp
->s_size
)
553 if (next
== qp
->s_last
) {
554 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
559 wqe
= get_swqe_ptr(qp
, qp
->s_head
);
561 wqe
->ssn
= qp
->s_ssn
++;
562 wqe
->sg_list
[0].mr
= NULL
;
563 wqe
->sg_list
[0].vaddr
= NULL
;
564 wqe
->sg_list
[0].length
= 0;
565 wqe
->sg_list
[0].sge_length
= 0;
567 acc
= wr
->opcode
>= IB_WR_RDMA_READ
? IB_ACCESS_LOCAL_WRITE
: 0;
568 for (i
= 0, j
= 0; i
< wr
->num_sge
; i
++) {
569 if (to_ipd(qp
->ibqp
.pd
)->user
&& wr
->sg_list
[i
].lkey
== 0) {
570 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
574 if (wr
->sg_list
[i
].length
== 0)
576 if (!ipath_lkey_ok(qp
, &wqe
->sg_list
[j
], &wr
->sg_list
[i
],
578 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
582 wqe
->length
+= wr
->sg_list
[i
].length
;
587 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
589 ipath_do_ruc_send((unsigned long) qp
);
598 * ipath_make_grh - construct a GRH header
599 * @dev: a pointer to the ipath device
600 * @hdr: a pointer to the GRH header being constructed
601 * @grh: the global route address to send to
602 * @hwords: the number of 32 bit words of header being sent
603 * @nwords: the number of 32 bit words of data being sent
605 * Return the size of the header in 32 bit words.
607 u32
ipath_make_grh(struct ipath_ibdev
*dev
, struct ib_grh
*hdr
,
608 struct ib_global_route
*grh
, u32 hwords
, u32 nwords
)
610 hdr
->version_tclass_flow
=
611 cpu_to_be32((6 << 28) |
612 (grh
->traffic_class
<< 20) |
614 hdr
->paylen
= cpu_to_be16((hwords
- 2 + nwords
+ SIZE_OF_CRC
) << 2);
615 /* next_hdr is defined by C8-7 in ch. 8.4.1 */
616 hdr
->next_hdr
= 0x1B;
617 hdr
->hop_limit
= grh
->hop_limit
;
618 /* The SGID is 32-bit aligned. */
619 hdr
->sgid
.global
.subnet_prefix
= dev
->gid_prefix
;
620 hdr
->sgid
.global
.interface_id
= dev
->dd
->ipath_guid
;
621 hdr
->dgid
= grh
->dgid
;
623 /* GRH header size in 32-bit words. */
624 return sizeof(struct ib_grh
) / sizeof(u32
);
628 * ipath_do_ruc_send - perform a send on an RC or UC QP
629 * @data: contains a pointer to the QP
631 * Process entries in the send work queue until credit or queue is
632 * exhausted. Only allow one CPU to send a packet per QP (tasklet).
633 * Otherwise, after we drop the QP s_lock, two threads could send
634 * packets out of order.
636 void ipath_do_ruc_send(unsigned long data
)
638 struct ipath_qp
*qp
= (struct ipath_qp
*)data
;
639 struct ipath_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
646 u32 pmtu
= ib_mtu_enum_to_int(qp
->path_mtu
);
647 struct ipath_other_headers
*ohdr
;
649 if (test_and_set_bit(IPATH_S_BUSY
, &qp
->s_flags
))
652 if (unlikely(qp
->remote_ah_attr
.dlid
== dev
->dd
->ipath_lid
)) {
653 ipath_ruc_loopback(qp
);
657 ohdr
= &qp
->s_hdr
.u
.oth
;
658 if (qp
->remote_ah_attr
.ah_flags
& IB_AH_GRH
)
659 ohdr
= &qp
->s_hdr
.u
.l
.oth
;
662 /* Check for a constructed packet to be sent. */
663 if (qp
->s_hdrwords
!= 0) {
665 * If no PIO bufs are available, return. An interrupt will
666 * call ipath_ib_piobufavail() when one is available.
668 if (ipath_verbs_send(dev
->dd
, qp
->s_hdrwords
,
669 (u32
*) &qp
->s_hdr
, qp
->s_cur_size
,
671 ipath_no_bufs_available(qp
, dev
);
674 dev
->n_unicast_xmit
++;
675 /* Record that we sent the packet and s_hdr is empty. */
680 * The lock is needed to synchronize between setting
681 * qp->s_ack_state, resend timer, and post_send().
683 spin_lock_irqsave(&qp
->s_lock
, flags
);
685 /* Sending responses has higher priority over sending requests. */
686 if (qp
->s_ack_state
!= IB_OPCODE_RC_ACKNOWLEDGE
&&
687 (bth0
= ipath_make_rc_ack(qp
, ohdr
, pmtu
)) != 0)
688 bth2
= qp
->s_ack_psn
++ & IPATH_PSN_MASK
;
689 else if (!((qp
->ibqp
.qp_type
== IB_QPT_RC
) ?
690 ipath_make_rc_req(qp
, ohdr
, pmtu
, &bth0
, &bth2
) :
691 ipath_make_uc_req(qp
, ohdr
, pmtu
, &bth0
, &bth2
))) {
693 * Clear the busy bit before unlocking to avoid races with
694 * adding new work queue items and then failing to process
697 clear_bit(IPATH_S_BUSY
, &qp
->s_flags
);
698 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
702 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
704 /* Construct the header. */
705 extra_bytes
= (4 - qp
->s_cur_size
) & 3;
706 nwords
= (qp
->s_cur_size
+ extra_bytes
) >> 2;
707 lrh0
= IPATH_LRH_BTH
;
708 if (unlikely(qp
->remote_ah_attr
.ah_flags
& IB_AH_GRH
)) {
709 qp
->s_hdrwords
+= ipath_make_grh(dev
, &qp
->s_hdr
.u
.l
.grh
,
710 &qp
->remote_ah_attr
.grh
,
711 qp
->s_hdrwords
, nwords
);
712 lrh0
= IPATH_LRH_GRH
;
714 lrh0
|= qp
->remote_ah_attr
.sl
<< 4;
715 qp
->s_hdr
.lrh
[0] = cpu_to_be16(lrh0
);
716 qp
->s_hdr
.lrh
[1] = cpu_to_be16(qp
->remote_ah_attr
.dlid
);
717 qp
->s_hdr
.lrh
[2] = cpu_to_be16(qp
->s_hdrwords
+ nwords
+
719 qp
->s_hdr
.lrh
[3] = cpu_to_be16(dev
->dd
->ipath_lid
);
720 bth0
|= ipath_get_pkey(dev
->dd
, qp
->s_pkey_index
);
721 bth0
|= extra_bytes
<< 20;
722 ohdr
->bth
[0] = cpu_to_be32(bth0
);
723 ohdr
->bth
[1] = cpu_to_be32(qp
->remote_qpn
);
724 ohdr
->bth
[2] = cpu_to_be32(bth2
);
726 /* Check for more work to do. */
730 clear_bit(IPATH_S_BUSY
, &qp
->s_flags
);