2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: mthca_srq.c 3047 2005-08-10 03:59:35Z roland $
35 #include <linux/slab.h>
36 #include <linux/string.h>
38 #include "mthca_dev.h"
39 #include "mthca_cmd.h"
40 #include "mthca_memfree.h"
41 #include "mthca_wqe.h"
44 MTHCA_MAX_DIRECT_SRQ_SIZE
= 4 * PAGE_SIZE
47 struct mthca_tavor_srq_context
{
48 __be64 wqe_base_ds
; /* low 6 bits is descriptor size */
52 __be16 limit_watermark
;
57 struct mthca_arbel_srq_context
{
58 __be32 state_logsize_srqn
;
61 __be32 logstride_usrpage
;
64 __be16 limit_watermark
;
71 static void *get_wqe(struct mthca_srq
*srq
, int n
)
74 return srq
->queue
.direct
.buf
+ (n
<< srq
->wqe_shift
);
76 return srq
->queue
.page_list
[(n
<< srq
->wqe_shift
) >> PAGE_SHIFT
].buf
+
77 ((n
<< srq
->wqe_shift
) & (PAGE_SIZE
- 1));
81 * Return a pointer to the location within a WQE that we're using as a
82 * link when the WQE is in the free list. We use the imm field
83 * because in the Tavor case, posting a WQE may overwrite the next
84 * segment of the previous WQE, but a receive WQE will never touch the
85 * imm field. This avoids corrupting our free list if the previous
86 * WQE has already completed and been put on the free list when we
89 static inline int *wqe_to_link(void *wqe
)
91 return (int *) (wqe
+ offsetof(struct mthca_next_seg
, imm
));
94 static void mthca_tavor_init_srq_context(struct mthca_dev
*dev
,
96 struct mthca_srq
*srq
,
97 struct mthca_tavor_srq_context
*context
)
99 memset(context
, 0, sizeof *context
);
101 context
->wqe_base_ds
= cpu_to_be64(1 << (srq
->wqe_shift
- 4));
102 context
->state_pd
= cpu_to_be32(pd
->pd_num
);
103 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
105 if (pd
->ibpd
.uobject
)
107 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
109 context
->uar
= cpu_to_be32(dev
->driver_uar
.index
);
112 static void mthca_arbel_init_srq_context(struct mthca_dev
*dev
,
114 struct mthca_srq
*srq
,
115 struct mthca_arbel_srq_context
*context
)
119 memset(context
, 0, sizeof *context
);
121 logsize
= long_log2(srq
->max
) + srq
->wqe_shift
;
122 context
->state_logsize_srqn
= cpu_to_be32(logsize
<< 24 | srq
->srqn
);
123 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
124 context
->db_index
= cpu_to_be32(srq
->db_index
);
125 context
->logstride_usrpage
= cpu_to_be32((srq
->wqe_shift
- 4) << 29);
126 if (pd
->ibpd
.uobject
)
127 context
->logstride_usrpage
|=
128 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
130 context
->logstride_usrpage
|= cpu_to_be32(dev
->driver_uar
.index
);
131 context
->eq_pd
= cpu_to_be32(MTHCA_EQ_ASYNC
<< 24 | pd
->pd_num
);
134 static void mthca_free_srq_buf(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
136 mthca_buf_free(dev
, srq
->max
<< srq
->wqe_shift
, &srq
->queue
,
137 srq
->is_direct
, &srq
->mr
);
141 static int mthca_alloc_srq_buf(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
142 struct mthca_srq
*srq
)
144 struct mthca_data_seg
*scatter
;
149 if (pd
->ibpd
.uobject
)
152 srq
->wrid
= kmalloc(srq
->max
* sizeof (u64
), GFP_KERNEL
);
156 err
= mthca_buf_alloc(dev
, srq
->max
<< srq
->wqe_shift
,
157 MTHCA_MAX_DIRECT_SRQ_SIZE
,
158 &srq
->queue
, &srq
->is_direct
, pd
, 1, &srq
->mr
);
165 * Now initialize the SRQ buffer so that all of the WQEs are
166 * linked into the list of free WQEs. In addition, set the
167 * scatter list L_Keys to the sentry value of 0x100.
169 for (i
= 0; i
< srq
->max
; ++i
) {
170 wqe
= get_wqe(srq
, i
);
172 *wqe_to_link(wqe
) = i
< srq
->max
- 1 ? i
+ 1 : -1;
174 for (scatter
= wqe
+ sizeof (struct mthca_next_seg
);
175 (void *) scatter
< wqe
+ (1 << srq
->wqe_shift
);
177 scatter
->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
180 srq
->last
= get_wqe(srq
, srq
->max
- 1);
185 int mthca_alloc_srq(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
186 struct ib_srq_attr
*attr
, struct mthca_srq
*srq
)
188 struct mthca_mailbox
*mailbox
;
193 /* Sanity check SRQ size before proceeding */
194 if (attr
->max_wr
> dev
->limits
.max_srq_wqes
||
195 attr
->max_sge
> dev
->limits
.max_sg
)
198 srq
->max
= attr
->max_wr
;
199 srq
->max_gs
= attr
->max_sge
;
202 if (mthca_is_memfree(dev
))
203 srq
->max
= roundup_pow_of_two(srq
->max
+ 1);
206 roundup_pow_of_two(sizeof (struct mthca_next_seg
) +
207 srq
->max_gs
* sizeof (struct mthca_data_seg
)));
209 if (!mthca_is_memfree(dev
) && (ds
> dev
->limits
.max_desc_sz
))
212 srq
->wqe_shift
= long_log2(ds
);
214 srq
->srqn
= mthca_alloc(&dev
->srq_table
.alloc
);
218 if (mthca_is_memfree(dev
)) {
219 err
= mthca_table_get(dev
, dev
->srq_table
.table
, srq
->srqn
);
223 if (!pd
->ibpd
.uobject
) {
224 srq
->db_index
= mthca_alloc_db(dev
, MTHCA_DB_TYPE_SRQ
,
225 srq
->srqn
, &srq
->db
);
226 if (srq
->db_index
< 0) {
233 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
234 if (IS_ERR(mailbox
)) {
235 err
= PTR_ERR(mailbox
);
239 err
= mthca_alloc_srq_buf(dev
, pd
, srq
);
241 goto err_out_mailbox
;
243 spin_lock_init(&srq
->lock
);
244 atomic_set(&srq
->refcount
, 1);
245 init_waitqueue_head(&srq
->wait
);
247 if (mthca_is_memfree(dev
))
248 mthca_arbel_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
250 mthca_tavor_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
252 err
= mthca_SW2HW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
255 mthca_warn(dev
, "SW2HW_SRQ failed (%d)\n", err
);
256 goto err_out_free_buf
;
259 mthca_warn(dev
, "SW2HW_SRQ returned status 0x%02x\n",
262 goto err_out_free_buf
;
265 spin_lock_irq(&dev
->srq_table
.lock
);
266 if (mthca_array_set(&dev
->srq_table
.srq
,
267 srq
->srqn
& (dev
->limits
.num_srqs
- 1),
269 spin_unlock_irq(&dev
->srq_table
.lock
);
270 goto err_out_free_srq
;
272 spin_unlock_irq(&dev
->srq_table
.lock
);
274 mthca_free_mailbox(dev
, mailbox
);
277 srq
->last_free
= srq
->max
- 1;
279 attr
->max_wr
= (mthca_is_memfree(dev
)) ? srq
->max
- 1 : srq
->max
;
280 attr
->max_sge
= srq
->max_gs
;
285 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
287 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
289 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
292 if (!pd
->ibpd
.uobject
)
293 mthca_free_srq_buf(dev
, srq
);
296 mthca_free_mailbox(dev
, mailbox
);
299 if (!pd
->ibpd
.uobject
&& mthca_is_memfree(dev
))
300 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
303 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
306 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
311 void mthca_free_srq(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
313 struct mthca_mailbox
*mailbox
;
317 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
318 if (IS_ERR(mailbox
)) {
319 mthca_warn(dev
, "No memory for mailbox to free SRQ.\n");
323 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
325 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
327 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
329 spin_lock_irq(&dev
->srq_table
.lock
);
330 mthca_array_clear(&dev
->srq_table
.srq
,
331 srq
->srqn
& (dev
->limits
.num_srqs
- 1));
332 spin_unlock_irq(&dev
->srq_table
.lock
);
334 atomic_dec(&srq
->refcount
);
335 wait_event(srq
->wait
, !atomic_read(&srq
->refcount
));
337 if (!srq
->ibsrq
.uobject
) {
338 mthca_free_srq_buf(dev
, srq
);
339 if (mthca_is_memfree(dev
))
340 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
343 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
344 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
345 mthca_free_mailbox(dev
, mailbox
);
348 int mthca_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
349 enum ib_srq_attr_mask attr_mask
)
351 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
352 struct mthca_srq
*srq
= to_msrq(ibsrq
);
356 /* We don't support resizing SRQs (yet?) */
357 if (attr_mask
& IB_SRQ_MAX_WR
)
360 if (attr_mask
& IB_SRQ_LIMIT
) {
361 if (attr
->srq_limit
> srq
->max
)
363 ret
= mthca_ARM_SRQ(dev
, srq
->srqn
, attr
->srq_limit
, &status
);
373 int mthca_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
375 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
376 struct mthca_srq
*srq
= to_msrq(ibsrq
);
377 struct mthca_mailbox
*mailbox
;
378 struct mthca_arbel_srq_context
*arbel_ctx
;
379 struct mthca_tavor_srq_context
*tavor_ctx
;
383 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
385 return PTR_ERR(mailbox
);
387 err
= mthca_QUERY_SRQ(dev
, srq
->srqn
, mailbox
, &status
);
391 if (mthca_is_memfree(dev
)) {
392 arbel_ctx
= mailbox
->buf
;
393 srq_attr
->srq_limit
= be16_to_cpu(arbel_ctx
->limit_watermark
);
395 tavor_ctx
= mailbox
->buf
;
396 srq_attr
->srq_limit
= be16_to_cpu(tavor_ctx
->limit_watermark
);
399 srq_attr
->max_wr
= (mthca_is_memfree(dev
)) ? srq
->max
- 1 : srq
->max
;
400 srq_attr
->max_sge
= srq
->max_gs
;
403 mthca_free_mailbox(dev
, mailbox
);
408 void mthca_srq_event(struct mthca_dev
*dev
, u32 srqn
,
409 enum ib_event_type event_type
)
411 struct mthca_srq
*srq
;
412 struct ib_event event
;
414 spin_lock(&dev
->srq_table
.lock
);
415 srq
= mthca_array_get(&dev
->srq_table
.srq
, srqn
& (dev
->limits
.num_srqs
- 1));
417 atomic_inc(&srq
->refcount
);
418 spin_unlock(&dev
->srq_table
.lock
);
421 mthca_warn(dev
, "Async event for bogus SRQ %08x\n", srqn
);
425 if (!srq
->ibsrq
.event_handler
)
428 event
.device
= &dev
->ib_dev
;
429 event
.event
= event_type
;
430 event
.element
.srq
= &srq
->ibsrq
;
431 srq
->ibsrq
.event_handler(&event
, srq
->ibsrq
.srq_context
);
434 if (atomic_dec_and_test(&srq
->refcount
))
439 * This function must be called with IRQs disabled.
441 void mthca_free_srq_wqe(struct mthca_srq
*srq
, u32 wqe_addr
)
445 ind
= wqe_addr
>> srq
->wqe_shift
;
447 spin_lock(&srq
->lock
);
449 if (likely(srq
->first_free
>= 0))
450 *wqe_to_link(get_wqe(srq
, srq
->last_free
)) = ind
;
452 srq
->first_free
= ind
;
454 *wqe_to_link(get_wqe(srq
, ind
)) = -1;
455 srq
->last_free
= ind
;
457 spin_unlock(&srq
->lock
);
460 int mthca_tavor_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
461 struct ib_recv_wr
**bad_wr
)
463 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
464 struct mthca_srq
*srq
= to_msrq(ibsrq
);
476 spin_lock_irqsave(&srq
->lock
, flags
);
478 first_ind
= srq
->first_free
;
480 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
481 if (unlikely(nreq
== MTHCA_TAVOR_MAX_WQES_PER_RECV_DB
)) {
484 doorbell
[0] = cpu_to_be32(first_ind
<< srq
->wqe_shift
);
485 doorbell
[1] = cpu_to_be32(srq
->srqn
<< 8);
488 * Make sure that descriptors are written
489 * before doorbell is rung.
493 mthca_write64(doorbell
,
494 dev
->kar
+ MTHCA_RECEIVE_DOORBELL
,
495 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
497 first_ind
= srq
->first_free
;
500 ind
= srq
->first_free
;
503 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
509 wqe
= get_wqe(srq
, ind
);
510 next_ind
= *wqe_to_link(wqe
);
513 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
519 prev_wqe
= srq
->last
;
522 ((struct mthca_next_seg
*) wqe
)->nda_op
= 0;
523 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
524 /* flags field will always remain 0 */
526 wqe
+= sizeof (struct mthca_next_seg
);
528 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
531 srq
->last
= prev_wqe
;
535 for (i
= 0; i
< wr
->num_sge
; ++i
) {
536 ((struct mthca_data_seg
*) wqe
)->byte_count
=
537 cpu_to_be32(wr
->sg_list
[i
].length
);
538 ((struct mthca_data_seg
*) wqe
)->lkey
=
539 cpu_to_be32(wr
->sg_list
[i
].lkey
);
540 ((struct mthca_data_seg
*) wqe
)->addr
=
541 cpu_to_be64(wr
->sg_list
[i
].addr
);
542 wqe
+= sizeof (struct mthca_data_seg
);
545 if (i
< srq
->max_gs
) {
546 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
547 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
548 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
551 ((struct mthca_next_seg
*) prev_wqe
)->nda_op
=
552 cpu_to_be32((ind
<< srq
->wqe_shift
) | 1);
554 ((struct mthca_next_seg
*) prev_wqe
)->ee_nds
=
555 cpu_to_be32(MTHCA_NEXT_DBD
);
557 srq
->wrid
[ind
] = wr
->wr_id
;
558 srq
->first_free
= next_ind
;
562 doorbell
[0] = cpu_to_be32(first_ind
<< srq
->wqe_shift
);
563 doorbell
[1] = cpu_to_be32((srq
->srqn
<< 8) | nreq
);
566 * Make sure that descriptors are written before
571 mthca_write64(doorbell
,
572 dev
->kar
+ MTHCA_RECEIVE_DOORBELL
,
573 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
576 spin_unlock_irqrestore(&srq
->lock
, flags
);
580 int mthca_arbel_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
581 struct ib_recv_wr
**bad_wr
)
583 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
584 struct mthca_srq
*srq
= to_msrq(ibsrq
);
593 spin_lock_irqsave(&srq
->lock
, flags
);
595 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
596 ind
= srq
->first_free
;
599 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
605 wqe
= get_wqe(srq
, ind
);
606 next_ind
= *wqe_to_link(wqe
);
609 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
615 ((struct mthca_next_seg
*) wqe
)->nda_op
=
616 cpu_to_be32((next_ind
<< srq
->wqe_shift
) | 1);
617 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
618 /* flags field will always remain 0 */
620 wqe
+= sizeof (struct mthca_next_seg
);
622 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
628 for (i
= 0; i
< wr
->num_sge
; ++i
) {
629 ((struct mthca_data_seg
*) wqe
)->byte_count
=
630 cpu_to_be32(wr
->sg_list
[i
].length
);
631 ((struct mthca_data_seg
*) wqe
)->lkey
=
632 cpu_to_be32(wr
->sg_list
[i
].lkey
);
633 ((struct mthca_data_seg
*) wqe
)->addr
=
634 cpu_to_be64(wr
->sg_list
[i
].addr
);
635 wqe
+= sizeof (struct mthca_data_seg
);
638 if (i
< srq
->max_gs
) {
639 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
640 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
641 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
644 srq
->wrid
[ind
] = wr
->wr_id
;
645 srq
->first_free
= next_ind
;
649 srq
->counter
+= nreq
;
652 * Make sure that descriptors are written before
653 * we write doorbell record.
656 *srq
->db
= cpu_to_be32(srq
->counter
);
659 spin_unlock_irqrestore(&srq
->lock
, flags
);
663 int __devinit
mthca_init_srq_table(struct mthca_dev
*dev
)
667 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
670 spin_lock_init(&dev
->srq_table
.lock
);
672 err
= mthca_alloc_init(&dev
->srq_table
.alloc
,
673 dev
->limits
.num_srqs
,
674 dev
->limits
.num_srqs
- 1,
675 dev
->limits
.reserved_srqs
);
679 err
= mthca_array_init(&dev
->srq_table
.srq
,
680 dev
->limits
.num_srqs
);
682 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);
687 void __devexit
mthca_cleanup_srq_table(struct mthca_dev
*dev
)
689 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
692 mthca_array_cleanup(&dev
->srq_table
.srq
, dev
->limits
.num_srqs
);
693 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);