2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
36 #include <linux/slab.h>
37 #include <linux/init.h>
38 #include <linux/errno.h>
40 #include "mthca_dev.h"
41 #include "mthca_cmd.h"
42 #include "mthca_memfree.h"
45 struct mthca_buddy
*buddy
;
51 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
53 struct mthca_mpt_entry
{
62 __be32 window_count_limit
;
64 __be32 mtt_sz
; /* Arbel only */
66 } __attribute__((packed
));
68 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
69 #define MTHCA_MPT_FLAG_MIO (1 << 17)
70 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
71 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
72 #define MTHCA_MPT_FLAG_REGION (1 << 8)
74 #define MTHCA_MTT_FLAG_PRESENT 1
76 #define MTHCA_MPT_STATUS_SW 0xF0
77 #define MTHCA_MPT_STATUS_HW 0x00
79 #define SINAI_FMR_KEY_INC 0x1000000
82 * Buddy allocator for MTT segments (currently not very efficient
83 * since it doesn't keep a free list and just searches linearly
84 * through the bitmaps)
87 static u32
mthca_buddy_alloc(struct mthca_buddy
*buddy
, int order
)
93 spin_lock(&buddy
->lock
);
95 for (o
= order
; o
<= buddy
->max_order
; ++o
) {
96 m
= 1 << (buddy
->max_order
- o
);
97 seg
= find_first_bit(buddy
->bits
[o
], m
);
102 spin_unlock(&buddy
->lock
);
106 clear_bit(seg
, buddy
->bits
[o
]);
111 set_bit(seg
^ 1, buddy
->bits
[o
]);
114 spin_unlock(&buddy
->lock
);
121 static void mthca_buddy_free(struct mthca_buddy
*buddy
, u32 seg
, int order
)
125 spin_lock(&buddy
->lock
);
127 while (test_bit(seg
^ 1, buddy
->bits
[order
])) {
128 clear_bit(seg
^ 1, buddy
->bits
[order
]);
133 set_bit(seg
, buddy
->bits
[order
]);
135 spin_unlock(&buddy
->lock
);
138 static int __devinit
mthca_buddy_init(struct mthca_buddy
*buddy
, int max_order
)
142 buddy
->max_order
= max_order
;
143 spin_lock_init(&buddy
->lock
);
145 buddy
->bits
= kzalloc((buddy
->max_order
+ 1) * sizeof (long *),
150 for (i
= 0; i
<= buddy
->max_order
; ++i
) {
151 s
= BITS_TO_LONGS(1 << (buddy
->max_order
- i
));
152 buddy
->bits
[i
] = kmalloc(s
* sizeof (long), GFP_KERNEL
);
155 bitmap_zero(buddy
->bits
[i
],
156 1 << (buddy
->max_order
- i
));
159 set_bit(0, buddy
->bits
[buddy
->max_order
]);
164 for (i
= 0; i
<= buddy
->max_order
; ++i
)
165 kfree(buddy
->bits
[i
]);
173 static void __devexit
mthca_buddy_cleanup(struct mthca_buddy
*buddy
)
177 for (i
= 0; i
<= buddy
->max_order
; ++i
)
178 kfree(buddy
->bits
[i
]);
183 static u32
mthca_alloc_mtt_range(struct mthca_dev
*dev
, int order
,
184 struct mthca_buddy
*buddy
)
186 u32 seg
= mthca_buddy_alloc(buddy
, order
);
191 if (mthca_is_memfree(dev
))
192 if (mthca_table_get_range(dev
, dev
->mr_table
.mtt_table
, seg
,
193 seg
+ (1 << order
) - 1)) {
194 mthca_buddy_free(buddy
, seg
, order
);
201 static struct mthca_mtt
*__mthca_alloc_mtt(struct mthca_dev
*dev
, int size
,
202 struct mthca_buddy
*buddy
)
204 struct mthca_mtt
*mtt
;
208 return ERR_PTR(-EINVAL
);
210 mtt
= kmalloc(sizeof *mtt
, GFP_KERNEL
);
212 return ERR_PTR(-ENOMEM
);
216 for (i
= MTHCA_MTT_SEG_SIZE
/ 8; i
< size
; i
<<= 1)
219 mtt
->first_seg
= mthca_alloc_mtt_range(dev
, mtt
->order
, buddy
);
220 if (mtt
->first_seg
== -1) {
222 return ERR_PTR(-ENOMEM
);
228 struct mthca_mtt
*mthca_alloc_mtt(struct mthca_dev
*dev
, int size
)
230 return __mthca_alloc_mtt(dev
, size
, &dev
->mr_table
.mtt_buddy
);
233 void mthca_free_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
)
238 mthca_buddy_free(mtt
->buddy
, mtt
->first_seg
, mtt
->order
);
240 mthca_table_put_range(dev
, dev
->mr_table
.mtt_table
,
242 mtt
->first_seg
+ (1 << mtt
->order
) - 1);
247 int mthca_write_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
,
248 int start_index
, u64
*buffer_list
, int list_len
)
250 struct mthca_mailbox
*mailbox
;
256 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
258 return PTR_ERR(mailbox
);
259 mtt_entry
= mailbox
->buf
;
261 while (list_len
> 0) {
262 mtt_entry
[0] = cpu_to_be64(dev
->mr_table
.mtt_base
+
263 mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
+
266 for (i
= 0; i
< list_len
&& i
< MTHCA_MAILBOX_SIZE
/ 8 - 2; ++i
)
267 mtt_entry
[i
+ 2] = cpu_to_be64(buffer_list
[i
] |
268 MTHCA_MTT_FLAG_PRESENT
);
271 * If we have an odd number of entries to write, add
272 * one more dummy entry for firmware efficiency.
275 mtt_entry
[i
+ 2] = 0;
277 err
= mthca_WRITE_MTT(dev
, mailbox
, (i
+ 1) & ~1, &status
);
279 mthca_warn(dev
, "WRITE_MTT failed (%d)\n", err
);
283 mthca_warn(dev
, "WRITE_MTT returned status 0x%02x\n",
295 mthca_free_mailbox(dev
, mailbox
);
299 static inline u32
tavor_hw_index_to_key(u32 ind
)
304 static inline u32
tavor_key_to_hw_index(u32 key
)
309 static inline u32
arbel_hw_index_to_key(u32 ind
)
311 return (ind
>> 24) | (ind
<< 8);
314 static inline u32
arbel_key_to_hw_index(u32 key
)
316 return (key
<< 24) | (key
>> 8);
319 static inline u32
hw_index_to_key(struct mthca_dev
*dev
, u32 ind
)
321 if (mthca_is_memfree(dev
))
322 return arbel_hw_index_to_key(ind
);
324 return tavor_hw_index_to_key(ind
);
327 static inline u32
key_to_hw_index(struct mthca_dev
*dev
, u32 key
)
329 if (mthca_is_memfree(dev
))
330 return arbel_key_to_hw_index(key
);
332 return tavor_key_to_hw_index(key
);
335 static inline u32
adjust_key(struct mthca_dev
*dev
, u32 key
)
337 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
338 return ((key
<< 20) & 0x800000) | (key
& 0x7fffff);
343 int mthca_mr_alloc(struct mthca_dev
*dev
, u32 pd
, int buffer_size_shift
,
344 u64 iova
, u64 total_size
, u32 access
, struct mthca_mr
*mr
)
346 struct mthca_mailbox
*mailbox
;
347 struct mthca_mpt_entry
*mpt_entry
;
353 WARN_ON(buffer_size_shift
>= 32);
355 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
358 key
= adjust_key(dev
, key
);
359 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
361 if (mthca_is_memfree(dev
)) {
362 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
364 goto err_out_mpt_free
;
367 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
368 if (IS_ERR(mailbox
)) {
369 err
= PTR_ERR(mailbox
);
372 mpt_entry
= mailbox
->buf
;
374 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
376 MTHCA_MPT_FLAG_REGION
|
379 mpt_entry
->flags
|= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL
);
381 mpt_entry
->page_size
= cpu_to_be32(buffer_size_shift
- 12);
382 mpt_entry
->key
= cpu_to_be32(key
);
383 mpt_entry
->pd
= cpu_to_be32(pd
);
384 mpt_entry
->start
= cpu_to_be64(iova
);
385 mpt_entry
->length
= cpu_to_be64(total_size
);
387 memset(&mpt_entry
->lkey
, 0,
388 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, lkey
));
392 cpu_to_be64(dev
->mr_table
.mtt_base
+
393 mr
->mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
);
396 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
397 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
399 printk("[%02x] ", i
* 4);
400 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
401 if ((i
+ 1) % 4 == 0)
406 err
= mthca_SW2HW_MPT(dev
, mailbox
,
407 key
& (dev
->limits
.num_mpts
- 1),
410 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
411 goto err_out_mailbox
;
413 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
416 goto err_out_mailbox
;
419 mthca_free_mailbox(dev
, mailbox
);
423 mthca_free_mailbox(dev
, mailbox
);
426 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
429 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
433 int mthca_mr_alloc_notrans(struct mthca_dev
*dev
, u32 pd
,
434 u32 access
, struct mthca_mr
*mr
)
437 return mthca_mr_alloc(dev
, pd
, 12, 0, ~0ULL, access
, mr
);
440 int mthca_mr_alloc_phys(struct mthca_dev
*dev
, u32 pd
,
441 u64
*buffer_list
, int buffer_size_shift
,
442 int list_len
, u64 iova
, u64 total_size
,
443 u32 access
, struct mthca_mr
*mr
)
447 mr
->mtt
= mthca_alloc_mtt(dev
, list_len
);
449 return PTR_ERR(mr
->mtt
);
451 err
= mthca_write_mtt(dev
, mr
->mtt
, 0, buffer_list
, list_len
);
453 mthca_free_mtt(dev
, mr
->mtt
);
457 err
= mthca_mr_alloc(dev
, pd
, buffer_size_shift
, iova
,
458 total_size
, access
, mr
);
460 mthca_free_mtt(dev
, mr
->mtt
);
466 static void mthca_free_region(struct mthca_dev
*dev
, u32 lkey
)
468 mthca_table_put(dev
, dev
->mr_table
.mpt_table
,
469 key_to_hw_index(dev
, lkey
));
471 mthca_free(&dev
->mr_table
.mpt_alloc
, key_to_hw_index(dev
, lkey
));
474 void mthca_free_mr(struct mthca_dev
*dev
, struct mthca_mr
*mr
)
479 err
= mthca_HW2SW_MPT(dev
, NULL
,
480 key_to_hw_index(dev
, mr
->ibmr
.lkey
) &
481 (dev
->limits
.num_mpts
- 1),
484 mthca_warn(dev
, "HW2SW_MPT failed (%d)\n", err
);
486 mthca_warn(dev
, "HW2SW_MPT returned status 0x%02x\n",
489 mthca_free_region(dev
, mr
->ibmr
.lkey
);
490 mthca_free_mtt(dev
, mr
->mtt
);
493 int mthca_fmr_alloc(struct mthca_dev
*dev
, u32 pd
,
494 u32 access
, struct mthca_fmr
*mr
)
496 struct mthca_mpt_entry
*mpt_entry
;
497 struct mthca_mailbox
*mailbox
;
501 int list_len
= mr
->attr
.max_pages
;
505 if (mr
->attr
.page_shift
< 12 || mr
->attr
.page_shift
>= 32)
508 /* For Arbel, all MTTs must fit in the same page. */
509 if (mthca_is_memfree(dev
) &&
510 mr
->attr
.max_pages
* sizeof *mr
->mem
.arbel
.mtts
> PAGE_SIZE
)
515 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
518 key
= adjust_key(dev
, key
);
520 idx
= key
& (dev
->limits
.num_mpts
- 1);
521 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
523 if (mthca_is_memfree(dev
)) {
524 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
526 goto err_out_mpt_free
;
528 mr
->mem
.arbel
.mpt
= mthca_table_find(dev
->mr_table
.mpt_table
, key
);
529 BUG_ON(!mr
->mem
.arbel
.mpt
);
531 mr
->mem
.tavor
.mpt
= dev
->mr_table
.tavor_fmr
.mpt_base
+
532 sizeof *(mr
->mem
.tavor
.mpt
) * idx
;
534 mr
->mtt
= __mthca_alloc_mtt(dev
, list_len
, dev
->mr_table
.fmr_mtt_buddy
);
538 mtt_seg
= mr
->mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
;
540 if (mthca_is_memfree(dev
)) {
541 mr
->mem
.arbel
.mtts
= mthca_table_find(dev
->mr_table
.mtt_table
,
543 BUG_ON(!mr
->mem
.arbel
.mtts
);
545 mr
->mem
.tavor
.mtts
= dev
->mr_table
.tavor_fmr
.mtt_base
+ mtt_seg
;
547 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
549 goto err_out_free_mtt
;
551 mpt_entry
= mailbox
->buf
;
553 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
555 MTHCA_MPT_FLAG_REGION
|
558 mpt_entry
->page_size
= cpu_to_be32(mr
->attr
.page_shift
- 12);
559 mpt_entry
->key
= cpu_to_be32(key
);
560 mpt_entry
->pd
= cpu_to_be32(pd
);
561 memset(&mpt_entry
->start
, 0,
562 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, start
));
563 mpt_entry
->mtt_seg
= cpu_to_be64(dev
->mr_table
.mtt_base
+ mtt_seg
);
566 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
567 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
569 printk("[%02x] ", i
* 4);
570 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
571 if ((i
+ 1) % 4 == 0)
576 err
= mthca_SW2HW_MPT(dev
, mailbox
,
577 key
& (dev
->limits
.num_mpts
- 1),
580 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
581 goto err_out_mailbox_free
;
584 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
587 goto err_out_mailbox_free
;
590 mthca_free_mailbox(dev
, mailbox
);
593 err_out_mailbox_free
:
594 mthca_free_mailbox(dev
, mailbox
);
597 mthca_free_mtt(dev
, mr
->mtt
);
600 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
603 mthca_free(&dev
->mr_table
.mpt_alloc
, mr
->ibmr
.lkey
);
607 int mthca_free_fmr(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
612 mthca_free_region(dev
, fmr
->ibmr
.lkey
);
613 mthca_free_mtt(dev
, fmr
->mtt
);
618 static inline int mthca_check_fmr(struct mthca_fmr
*fmr
, u64
*page_list
,
619 int list_len
, u64 iova
)
623 if (list_len
> fmr
->attr
.max_pages
)
626 page_mask
= (1 << fmr
->attr
.page_shift
) - 1;
628 /* We are getting page lists, so va must be page aligned. */
629 if (iova
& page_mask
)
632 /* Trust the user not to pass misaligned data in page_list */
634 for (i
= 0; i
< list_len
; ++i
) {
635 if (page_list
[i
] & ~page_mask
)
639 if (fmr
->maps
>= fmr
->attr
.max_maps
)
646 int mthca_tavor_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
647 int list_len
, u64 iova
)
649 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
650 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
651 struct mthca_mpt_entry mpt_entry
;
655 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
661 key
= tavor_key_to_hw_index(fmr
->ibmr
.lkey
);
662 key
+= dev
->limits
.num_mpts
;
663 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= tavor_hw_index_to_key(key
);
665 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
667 for (i
= 0; i
< list_len
; ++i
) {
668 __be64 mtt_entry
= cpu_to_be64(page_list
[i
] |
669 MTHCA_MTT_FLAG_PRESENT
);
670 mthca_write64_raw(mtt_entry
, fmr
->mem
.tavor
.mtts
+ i
);
673 mpt_entry
.lkey
= cpu_to_be32(key
);
674 mpt_entry
.length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
675 mpt_entry
.start
= cpu_to_be64(iova
);
677 __raw_writel((__force u32
) mpt_entry
.lkey
, &fmr
->mem
.tavor
.mpt
->key
);
678 memcpy_toio(&fmr
->mem
.tavor
.mpt
->start
, &mpt_entry
.start
,
679 offsetof(struct mthca_mpt_entry
, window_count
) -
680 offsetof(struct mthca_mpt_entry
, start
));
682 writeb(MTHCA_MPT_STATUS_HW
, fmr
->mem
.tavor
.mpt
);
687 int mthca_arbel_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
688 int list_len
, u64 iova
)
690 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
691 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
695 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
701 key
= arbel_key_to_hw_index(fmr
->ibmr
.lkey
);
702 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
703 key
+= SINAI_FMR_KEY_INC
;
705 key
+= dev
->limits
.num_mpts
;
706 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= arbel_hw_index_to_key(key
);
708 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
712 for (i
= 0; i
< list_len
; ++i
)
713 fmr
->mem
.arbel
.mtts
[i
] = cpu_to_be64(page_list
[i
] |
714 MTHCA_MTT_FLAG_PRESENT
);
716 fmr
->mem
.arbel
.mpt
->key
= cpu_to_be32(key
);
717 fmr
->mem
.arbel
.mpt
->lkey
= cpu_to_be32(key
);
718 fmr
->mem
.arbel
.mpt
->length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
719 fmr
->mem
.arbel
.mpt
->start
= cpu_to_be64(iova
);
723 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_HW
;
730 void mthca_tavor_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
737 key
= tavor_key_to_hw_index(fmr
->ibmr
.lkey
);
738 key
&= dev
->limits
.num_mpts
- 1;
739 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= tavor_hw_index_to_key(key
);
743 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
746 void mthca_arbel_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
753 key
= arbel_key_to_hw_index(fmr
->ibmr
.lkey
);
754 key
&= dev
->limits
.num_mpts
- 1;
755 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= arbel_hw_index_to_key(key
);
759 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
762 int __devinit
mthca_init_mr_table(struct mthca_dev
*dev
)
766 err
= mthca_alloc_init(&dev
->mr_table
.mpt_alloc
,
767 dev
->limits
.num_mpts
,
768 ~0, dev
->limits
.reserved_mrws
);
772 if (!mthca_is_memfree(dev
) &&
773 (dev
->mthca_flags
& MTHCA_FLAG_DDR_HIDDEN
))
774 dev
->limits
.fmr_reserved_mtts
= 0;
776 dev
->mthca_flags
|= MTHCA_FLAG_FMR
;
778 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
779 mthca_dbg(dev
, "Memory key throughput optimization activated.\n");
781 err
= mthca_buddy_init(&dev
->mr_table
.mtt_buddy
,
782 fls(dev
->limits
.num_mtt_segs
- 1));
787 dev
->mr_table
.tavor_fmr
.mpt_base
= NULL
;
788 dev
->mr_table
.tavor_fmr
.mtt_base
= NULL
;
790 if (dev
->limits
.fmr_reserved_mtts
) {
791 i
= fls(dev
->limits
.fmr_reserved_mtts
- 1);
794 mthca_warn(dev
, "Unable to reserve 2^31 FMR MTTs.\n");
799 dev
->mr_table
.tavor_fmr
.mpt_base
=
800 ioremap(dev
->mr_table
.mpt_base
,
801 (1 << i
) * sizeof (struct mthca_mpt_entry
));
803 if (!dev
->mr_table
.tavor_fmr
.mpt_base
) {
804 mthca_warn(dev
, "MPT ioremap for FMR failed.\n");
809 dev
->mr_table
.tavor_fmr
.mtt_base
=
810 ioremap(dev
->mr_table
.mtt_base
,
811 (1 << i
) * MTHCA_MTT_SEG_SIZE
);
812 if (!dev
->mr_table
.tavor_fmr
.mtt_base
) {
813 mthca_warn(dev
, "MTT ioremap for FMR failed.\n");
818 err
= mthca_buddy_init(&dev
->mr_table
.tavor_fmr
.mtt_buddy
, i
);
820 goto err_fmr_mtt_buddy
;
822 /* Prevent regular MRs from using FMR keys */
823 err
= mthca_buddy_alloc(&dev
->mr_table
.mtt_buddy
, i
);
825 goto err_reserve_fmr
;
827 dev
->mr_table
.fmr_mtt_buddy
=
828 &dev
->mr_table
.tavor_fmr
.mtt_buddy
;
830 dev
->mr_table
.fmr_mtt_buddy
= &dev
->mr_table
.mtt_buddy
;
832 /* FMR table is always the first, take reserved MTTs out of there */
833 if (dev
->limits
.reserved_mtts
) {
834 i
= fls(dev
->limits
.reserved_mtts
- 1);
836 if (mthca_alloc_mtt_range(dev
, i
,
837 dev
->mr_table
.fmr_mtt_buddy
) == -1) {
838 mthca_warn(dev
, "MTT table of order %d is too small.\n",
839 dev
->mr_table
.fmr_mtt_buddy
->max_order
);
841 goto err_reserve_mtts
;
849 if (dev
->limits
.fmr_reserved_mtts
)
850 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
853 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
854 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
857 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
858 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
861 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
864 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);
869 void __devexit
mthca_cleanup_mr_table(struct mthca_dev
*dev
)
871 /* XXX check if any MRs are still allocated? */
872 if (dev
->limits
.fmr_reserved_mtts
)
873 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
875 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
877 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
878 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
879 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
880 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
882 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);