2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
35 #include <linux/slab.h>
36 #include <linux/init.h>
37 #include <linux/errno.h>
39 #include "mthca_dev.h"
40 #include "mthca_cmd.h"
41 #include "mthca_memfree.h"
44 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
46 struct mthca_mpt_entry
{
55 u32 window_count_limit
;
57 u32 mtt_sz
; /* Arbel only */
59 } __attribute__((packed
));
61 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
62 #define MTHCA_MPT_FLAG_MIO (1 << 17)
63 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
64 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
65 #define MTHCA_MPT_FLAG_REGION (1 << 8)
67 #define MTHCA_MTT_FLAG_PRESENT 1
70 * Buddy allocator for MTT segments (currently not very efficient
71 * since it doesn't keep a free list and just searches linearly
72 * through the bitmaps)
75 static u32
__mthca_alloc_mtt(struct mthca_dev
*dev
, int order
)
81 spin_lock(&dev
->mr_table
.mpt_alloc
.lock
);
83 for (o
= order
; o
<= dev
->mr_table
.max_mtt_order
; ++o
) {
84 m
= 1 << (dev
->mr_table
.max_mtt_order
- o
);
85 seg
= find_first_bit(dev
->mr_table
.mtt_buddy
[o
], m
);
90 spin_unlock(&dev
->mr_table
.mpt_alloc
.lock
);
94 clear_bit(seg
, dev
->mr_table
.mtt_buddy
[o
]);
99 set_bit(seg
^ 1, dev
->mr_table
.mtt_buddy
[o
]);
102 spin_unlock(&dev
->mr_table
.mpt_alloc
.lock
);
109 static void __mthca_free_mtt(struct mthca_dev
*dev
, u32 seg
, int order
)
113 spin_lock(&dev
->mr_table
.mpt_alloc
.lock
);
115 while (test_bit(seg
^ 1, dev
->mr_table
.mtt_buddy
[order
])) {
116 clear_bit(seg
^ 1, dev
->mr_table
.mtt_buddy
[order
]);
121 set_bit(seg
, dev
->mr_table
.mtt_buddy
[order
]);
123 spin_unlock(&dev
->mr_table
.mpt_alloc
.lock
);
126 static u32
mthca_alloc_mtt(struct mthca_dev
*dev
, int order
)
128 u32 seg
= __mthca_alloc_mtt(dev
, order
);
133 if (dev
->hca_type
== ARBEL_NATIVE
)
134 if (mthca_table_get_range(dev
, dev
->mr_table
.mtt_table
, seg
,
135 seg
+ (1 << order
) - 1)) {
136 __mthca_free_mtt(dev
, seg
, order
);
143 static void mthca_free_mtt(struct mthca_dev
*dev
, u32 seg
, int order
)
145 __mthca_free_mtt(dev
, seg
, order
);
147 if (dev
->hca_type
== ARBEL_NATIVE
)
148 mthca_table_put_range(dev
, dev
->mr_table
.mtt_table
, seg
,
149 seg
+ (1 << order
) - 1);
152 static inline u32
hw_index_to_key(struct mthca_dev
*dev
, u32 ind
)
154 if (dev
->hca_type
== ARBEL_NATIVE
)
155 return (ind
>> 24) | (ind
<< 8);
160 static inline u32
key_to_hw_index(struct mthca_dev
*dev
, u32 key
)
162 if (dev
->hca_type
== ARBEL_NATIVE
)
163 return (key
<< 24) | (key
>> 8);
168 int mthca_mr_alloc_notrans(struct mthca_dev
*dev
, u32 pd
,
169 u32 access
, struct mthca_mr
*mr
)
171 void *mailbox
= NULL
;
172 struct mthca_mpt_entry
*mpt_entry
;
180 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
183 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
185 if (dev
->hca_type
== ARBEL_NATIVE
) {
186 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
188 goto err_out_mpt_free
;
191 mailbox
= kmalloc(sizeof *mpt_entry
+ MTHCA_CMD_MAILBOX_EXTRA
,
197 mpt_entry
= MAILBOX_ALIGN(mailbox
);
199 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
201 MTHCA_MPT_FLAG_PHYSICAL
|
202 MTHCA_MPT_FLAG_REGION
|
204 mpt_entry
->page_size
= 0;
205 mpt_entry
->key
= cpu_to_be32(key
);
206 mpt_entry
->pd
= cpu_to_be32(pd
);
207 mpt_entry
->start
= 0;
208 mpt_entry
->length
= ~0ULL;
210 memset(&mpt_entry
->lkey
, 0,
211 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, lkey
));
213 err
= mthca_SW2HW_MPT(dev
, mpt_entry
,
214 key
& (dev
->limits
.num_mpts
- 1),
217 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
220 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
230 if (dev
->hca_type
== ARBEL_NATIVE
)
231 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
234 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
239 int mthca_mr_alloc_phys(struct mthca_dev
*dev
, u32 pd
,
240 u64
*buffer_list
, int buffer_size_shift
,
241 int list_len
, u64 iova
, u64 total_size
,
242 u32 access
, struct mthca_mr
*mr
)
246 struct mthca_mpt_entry
*mpt_entry
;
253 WARN_ON(buffer_size_shift
>= 32);
255 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
258 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
260 if (dev
->hca_type
== ARBEL_NATIVE
) {
261 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
263 goto err_out_mpt_free
;
266 for (i
= dev
->limits
.mtt_seg_size
/ 8, mr
->order
= 0;
268 i
<<= 1, ++mr
->order
)
271 mr
->first_seg
= mthca_alloc_mtt(dev
, mr
->order
);
272 if (mr
->first_seg
== -1)
276 * If list_len is odd, we add one more dummy entry for
277 * firmware efficiency.
279 mailbox
= kmalloc(max(sizeof *mpt_entry
,
280 (size_t) 8 * (list_len
+ (list_len
& 1) + 2)) +
281 MTHCA_CMD_MAILBOX_EXTRA
,
284 goto err_out_free_mtt
;
286 mtt_entry
= MAILBOX_ALIGN(mailbox
);
288 mtt_entry
[0] = cpu_to_be64(dev
->mr_table
.mtt_base
+
289 mr
->first_seg
* dev
->limits
.mtt_seg_size
);
291 for (i
= 0; i
< list_len
; ++i
)
292 mtt_entry
[i
+ 2] = cpu_to_be64(buffer_list
[i
] |
293 MTHCA_MTT_FLAG_PRESENT
);
295 mtt_entry
[i
+ 2] = 0;
300 mthca_dbg(dev
, "Dumping MPT entry\n");
301 for (i
= 0; i
< list_len
+ 2; ++i
)
302 printk(KERN_ERR
"[%2d] %016llx\n",
303 i
, (unsigned long long) be64_to_cpu(mtt_entry
[i
]));
306 err
= mthca_WRITE_MTT(dev
, mtt_entry
, list_len
, &status
);
308 mthca_warn(dev
, "WRITE_MTT failed (%d)\n", err
);
309 goto err_out_mailbox_free
;
312 mthca_warn(dev
, "WRITE_MTT returned status 0x%02x\n",
315 goto err_out_mailbox_free
;
318 mpt_entry
= MAILBOX_ALIGN(mailbox
);
320 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
322 MTHCA_MPT_FLAG_REGION
|
325 mpt_entry
->page_size
= cpu_to_be32(buffer_size_shift
- 12);
326 mpt_entry
->key
= cpu_to_be32(key
);
327 mpt_entry
->pd
= cpu_to_be32(pd
);
328 mpt_entry
->start
= cpu_to_be64(iova
);
329 mpt_entry
->length
= cpu_to_be64(total_size
);
330 memset(&mpt_entry
->lkey
, 0,
331 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, lkey
));
332 mpt_entry
->mtt_seg
= cpu_to_be64(dev
->mr_table
.mtt_base
+
333 mr
->first_seg
* dev
->limits
.mtt_seg_size
);
336 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
337 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
339 printk("[%02x] ", i
* 4);
340 printk(" %08x", be32_to_cpu(((u32
*) mpt_entry
)[i
]));
341 if ((i
+ 1) % 4 == 0)
346 err
= mthca_SW2HW_MPT(dev
, mpt_entry
,
347 key
& (dev
->limits
.num_mpts
- 1),
350 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
352 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
360 err_out_mailbox_free
:
364 mthca_free_mtt(dev
, mr
->first_seg
, mr
->order
);
367 if (dev
->hca_type
== ARBEL_NATIVE
)
368 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
371 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
375 void mthca_free_mr(struct mthca_dev
*dev
, struct mthca_mr
*mr
)
382 err
= mthca_HW2SW_MPT(dev
, NULL
,
383 key_to_hw_index(dev
, mr
->ibmr
.lkey
) &
384 (dev
->limits
.num_mpts
- 1),
387 mthca_warn(dev
, "HW2SW_MPT failed (%d)\n", err
);
389 mthca_warn(dev
, "HW2SW_MPT returned status 0x%02x\n",
393 mthca_free_mtt(dev
, mr
->first_seg
, mr
->order
);
395 if (dev
->hca_type
== ARBEL_NATIVE
)
396 mthca_table_put(dev
, dev
->mr_table
.mpt_table
,
397 key_to_hw_index(dev
, mr
->ibmr
.lkey
));
398 mthca_free(&dev
->mr_table
.mpt_alloc
, key_to_hw_index(dev
, mr
->ibmr
.lkey
));
401 int __devinit
mthca_init_mr_table(struct mthca_dev
*dev
)
406 err
= mthca_alloc_init(&dev
->mr_table
.mpt_alloc
,
407 dev
->limits
.num_mpts
,
408 ~0, dev
->limits
.reserved_mrws
);
414 for (i
= 1, dev
->mr_table
.max_mtt_order
= 0;
415 i
< dev
->limits
.num_mtt_segs
;
416 i
<<= 1, ++dev
->mr_table
.max_mtt_order
)
419 dev
->mr_table
.mtt_buddy
= kmalloc((dev
->mr_table
.max_mtt_order
+ 1) *
422 if (!dev
->mr_table
.mtt_buddy
)
425 for (i
= 0; i
<= dev
->mr_table
.max_mtt_order
; ++i
)
426 dev
->mr_table
.mtt_buddy
[i
] = NULL
;
428 for (i
= 0; i
<= dev
->mr_table
.max_mtt_order
; ++i
) {
429 s
= BITS_TO_LONGS(1 << (dev
->mr_table
.max_mtt_order
- i
));
430 dev
->mr_table
.mtt_buddy
[i
] = kmalloc(s
* sizeof (long),
432 if (!dev
->mr_table
.mtt_buddy
[i
])
434 bitmap_zero(dev
->mr_table
.mtt_buddy
[i
],
435 1 << (dev
->mr_table
.max_mtt_order
- i
));
438 set_bit(0, dev
->mr_table
.mtt_buddy
[dev
->mr_table
.max_mtt_order
]);
440 for (i
= 0; i
< dev
->mr_table
.max_mtt_order
; ++i
)
441 if (1 << i
>= dev
->limits
.reserved_mtts
)
444 if (i
== dev
->mr_table
.max_mtt_order
) {
445 mthca_err(dev
, "MTT table of order %d is "
450 (void) mthca_alloc_mtt(dev
, i
);
455 for (i
= 0; i
<= dev
->mr_table
.max_mtt_order
; ++i
)
456 kfree(dev
->mr_table
.mtt_buddy
[i
]);
459 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);
464 void __devexit
mthca_cleanup_mr_table(struct mthca_dev
*dev
)
468 /* XXX check if any MRs are still allocated? */
469 for (i
= 0; i
<= dev
->mr_table
.max_mtt_order
; ++i
)
470 kfree(dev
->mr_table
.mtt_buddy
[i
]);
471 kfree(dev
->mr_table
.mtt_buddy
);
472 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);