2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
36 #include <linux/slab.h>
37 #include <linux/errno.h>
39 #include "mthca_dev.h"
40 #include "mthca_cmd.h"
41 #include "mthca_memfree.h"
44 struct mthca_buddy
*buddy
;
50 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
52 struct mthca_mpt_entry
{
61 __be32 window_count_limit
;
63 __be32 mtt_sz
; /* Arbel only */
65 } __attribute__((packed
));
67 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
68 #define MTHCA_MPT_FLAG_MIO (1 << 17)
69 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
70 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
71 #define MTHCA_MPT_FLAG_REGION (1 << 8)
73 #define MTHCA_MTT_FLAG_PRESENT 1
75 #define MTHCA_MPT_STATUS_SW 0xF0
76 #define MTHCA_MPT_STATUS_HW 0x00
78 #define SINAI_FMR_KEY_INC 0x1000000
81 * Buddy allocator for MTT segments (currently not very efficient
82 * since it doesn't keep a free list and just searches linearly
83 * through the bitmaps)
86 static u32
mthca_buddy_alloc(struct mthca_buddy
*buddy
, int order
)
92 spin_lock(&buddy
->lock
);
94 for (o
= order
; o
<= buddy
->max_order
; ++o
) {
95 m
= 1 << (buddy
->max_order
- o
);
96 seg
= find_first_bit(buddy
->bits
[o
], m
);
101 spin_unlock(&buddy
->lock
);
105 clear_bit(seg
, buddy
->bits
[o
]);
110 set_bit(seg
^ 1, buddy
->bits
[o
]);
113 spin_unlock(&buddy
->lock
);
120 static void mthca_buddy_free(struct mthca_buddy
*buddy
, u32 seg
, int order
)
124 spin_lock(&buddy
->lock
);
126 while (test_bit(seg
^ 1, buddy
->bits
[order
])) {
127 clear_bit(seg
^ 1, buddy
->bits
[order
]);
132 set_bit(seg
, buddy
->bits
[order
]);
134 spin_unlock(&buddy
->lock
);
137 static int mthca_buddy_init(struct mthca_buddy
*buddy
, int max_order
)
141 buddy
->max_order
= max_order
;
142 spin_lock_init(&buddy
->lock
);
144 buddy
->bits
= kzalloc((buddy
->max_order
+ 1) * sizeof (long *),
149 for (i
= 0; i
<= buddy
->max_order
; ++i
) {
150 s
= BITS_TO_LONGS(1 << (buddy
->max_order
- i
));
151 buddy
->bits
[i
] = kmalloc(s
* sizeof (long), GFP_KERNEL
);
154 bitmap_zero(buddy
->bits
[i
],
155 1 << (buddy
->max_order
- i
));
158 set_bit(0, buddy
->bits
[buddy
->max_order
]);
163 for (i
= 0; i
<= buddy
->max_order
; ++i
)
164 kfree(buddy
->bits
[i
]);
172 static void mthca_buddy_cleanup(struct mthca_buddy
*buddy
)
176 for (i
= 0; i
<= buddy
->max_order
; ++i
)
177 kfree(buddy
->bits
[i
]);
182 static u32
mthca_alloc_mtt_range(struct mthca_dev
*dev
, int order
,
183 struct mthca_buddy
*buddy
)
185 u32 seg
= mthca_buddy_alloc(buddy
, order
);
190 if (mthca_is_memfree(dev
))
191 if (mthca_table_get_range(dev
, dev
->mr_table
.mtt_table
, seg
,
192 seg
+ (1 << order
) - 1)) {
193 mthca_buddy_free(buddy
, seg
, order
);
200 static struct mthca_mtt
*__mthca_alloc_mtt(struct mthca_dev
*dev
, int size
,
201 struct mthca_buddy
*buddy
)
203 struct mthca_mtt
*mtt
;
207 return ERR_PTR(-EINVAL
);
209 mtt
= kmalloc(sizeof *mtt
, GFP_KERNEL
);
211 return ERR_PTR(-ENOMEM
);
215 for (i
= MTHCA_MTT_SEG_SIZE
/ 8; i
< size
; i
<<= 1)
218 mtt
->first_seg
= mthca_alloc_mtt_range(dev
, mtt
->order
, buddy
);
219 if (mtt
->first_seg
== -1) {
221 return ERR_PTR(-ENOMEM
);
227 struct mthca_mtt
*mthca_alloc_mtt(struct mthca_dev
*dev
, int size
)
229 return __mthca_alloc_mtt(dev
, size
, &dev
->mr_table
.mtt_buddy
);
232 void mthca_free_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
)
237 mthca_buddy_free(mtt
->buddy
, mtt
->first_seg
, mtt
->order
);
239 mthca_table_put_range(dev
, dev
->mr_table
.mtt_table
,
241 mtt
->first_seg
+ (1 << mtt
->order
) - 1);
246 int mthca_write_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
,
247 int start_index
, u64
*buffer_list
, int list_len
)
249 struct mthca_mailbox
*mailbox
;
255 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
257 return PTR_ERR(mailbox
);
258 mtt_entry
= mailbox
->buf
;
260 while (list_len
> 0) {
261 mtt_entry
[0] = cpu_to_be64(dev
->mr_table
.mtt_base
+
262 mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
+
265 for (i
= 0; i
< list_len
&& i
< MTHCA_MAILBOX_SIZE
/ 8 - 2; ++i
)
266 mtt_entry
[i
+ 2] = cpu_to_be64(buffer_list
[i
] |
267 MTHCA_MTT_FLAG_PRESENT
);
270 * If we have an odd number of entries to write, add
271 * one more dummy entry for firmware efficiency.
274 mtt_entry
[i
+ 2] = 0;
276 err
= mthca_WRITE_MTT(dev
, mailbox
, (i
+ 1) & ~1, &status
);
278 mthca_warn(dev
, "WRITE_MTT failed (%d)\n", err
);
282 mthca_warn(dev
, "WRITE_MTT returned status 0x%02x\n",
294 mthca_free_mailbox(dev
, mailbox
);
298 static inline u32
tavor_hw_index_to_key(u32 ind
)
303 static inline u32
tavor_key_to_hw_index(u32 key
)
308 static inline u32
arbel_hw_index_to_key(u32 ind
)
310 return (ind
>> 24) | (ind
<< 8);
313 static inline u32
arbel_key_to_hw_index(u32 key
)
315 return (key
<< 24) | (key
>> 8);
318 static inline u32
hw_index_to_key(struct mthca_dev
*dev
, u32 ind
)
320 if (mthca_is_memfree(dev
))
321 return arbel_hw_index_to_key(ind
);
323 return tavor_hw_index_to_key(ind
);
326 static inline u32
key_to_hw_index(struct mthca_dev
*dev
, u32 key
)
328 if (mthca_is_memfree(dev
))
329 return arbel_key_to_hw_index(key
);
331 return tavor_key_to_hw_index(key
);
334 static inline u32
adjust_key(struct mthca_dev
*dev
, u32 key
)
336 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
337 return ((key
<< 20) & 0x800000) | (key
& 0x7fffff);
342 int mthca_mr_alloc(struct mthca_dev
*dev
, u32 pd
, int buffer_size_shift
,
343 u64 iova
, u64 total_size
, u32 access
, struct mthca_mr
*mr
)
345 struct mthca_mailbox
*mailbox
;
346 struct mthca_mpt_entry
*mpt_entry
;
352 WARN_ON(buffer_size_shift
>= 32);
354 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
357 key
= adjust_key(dev
, key
);
358 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
360 if (mthca_is_memfree(dev
)) {
361 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
363 goto err_out_mpt_free
;
366 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
367 if (IS_ERR(mailbox
)) {
368 err
= PTR_ERR(mailbox
);
371 mpt_entry
= mailbox
->buf
;
373 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
375 MTHCA_MPT_FLAG_REGION
|
378 mpt_entry
->flags
|= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL
);
380 mpt_entry
->page_size
= cpu_to_be32(buffer_size_shift
- 12);
381 mpt_entry
->key
= cpu_to_be32(key
);
382 mpt_entry
->pd
= cpu_to_be32(pd
);
383 mpt_entry
->start
= cpu_to_be64(iova
);
384 mpt_entry
->length
= cpu_to_be64(total_size
);
386 memset(&mpt_entry
->lkey
, 0,
387 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, lkey
));
391 cpu_to_be64(dev
->mr_table
.mtt_base
+
392 mr
->mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
);
395 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
396 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
398 printk("[%02x] ", i
* 4);
399 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
400 if ((i
+ 1) % 4 == 0)
405 err
= mthca_SW2HW_MPT(dev
, mailbox
,
406 key
& (dev
->limits
.num_mpts
- 1),
409 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
410 goto err_out_mailbox
;
412 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
415 goto err_out_mailbox
;
418 mthca_free_mailbox(dev
, mailbox
);
422 mthca_free_mailbox(dev
, mailbox
);
425 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
428 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
432 int mthca_mr_alloc_notrans(struct mthca_dev
*dev
, u32 pd
,
433 u32 access
, struct mthca_mr
*mr
)
436 return mthca_mr_alloc(dev
, pd
, 12, 0, ~0ULL, access
, mr
);
439 int mthca_mr_alloc_phys(struct mthca_dev
*dev
, u32 pd
,
440 u64
*buffer_list
, int buffer_size_shift
,
441 int list_len
, u64 iova
, u64 total_size
,
442 u32 access
, struct mthca_mr
*mr
)
446 mr
->mtt
= mthca_alloc_mtt(dev
, list_len
);
448 return PTR_ERR(mr
->mtt
);
450 err
= mthca_write_mtt(dev
, mr
->mtt
, 0, buffer_list
, list_len
);
452 mthca_free_mtt(dev
, mr
->mtt
);
456 err
= mthca_mr_alloc(dev
, pd
, buffer_size_shift
, iova
,
457 total_size
, access
, mr
);
459 mthca_free_mtt(dev
, mr
->mtt
);
465 static void mthca_free_region(struct mthca_dev
*dev
, u32 lkey
)
467 mthca_table_put(dev
, dev
->mr_table
.mpt_table
,
468 key_to_hw_index(dev
, lkey
));
470 mthca_free(&dev
->mr_table
.mpt_alloc
, key_to_hw_index(dev
, lkey
));
473 void mthca_free_mr(struct mthca_dev
*dev
, struct mthca_mr
*mr
)
478 err
= mthca_HW2SW_MPT(dev
, NULL
,
479 key_to_hw_index(dev
, mr
->ibmr
.lkey
) &
480 (dev
->limits
.num_mpts
- 1),
483 mthca_warn(dev
, "HW2SW_MPT failed (%d)\n", err
);
485 mthca_warn(dev
, "HW2SW_MPT returned status 0x%02x\n",
488 mthca_free_region(dev
, mr
->ibmr
.lkey
);
489 mthca_free_mtt(dev
, mr
->mtt
);
492 int mthca_fmr_alloc(struct mthca_dev
*dev
, u32 pd
,
493 u32 access
, struct mthca_fmr
*mr
)
495 struct mthca_mpt_entry
*mpt_entry
;
496 struct mthca_mailbox
*mailbox
;
500 int list_len
= mr
->attr
.max_pages
;
504 if (mr
->attr
.page_shift
< 12 || mr
->attr
.page_shift
>= 32)
507 /* For Arbel, all MTTs must fit in the same page. */
508 if (mthca_is_memfree(dev
) &&
509 mr
->attr
.max_pages
* sizeof *mr
->mem
.arbel
.mtts
> PAGE_SIZE
)
514 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
517 key
= adjust_key(dev
, key
);
519 idx
= key
& (dev
->limits
.num_mpts
- 1);
520 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
522 if (mthca_is_memfree(dev
)) {
523 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
525 goto err_out_mpt_free
;
527 mr
->mem
.arbel
.mpt
= mthca_table_find(dev
->mr_table
.mpt_table
, key
, NULL
);
528 BUG_ON(!mr
->mem
.arbel
.mpt
);
530 mr
->mem
.tavor
.mpt
= dev
->mr_table
.tavor_fmr
.mpt_base
+
531 sizeof *(mr
->mem
.tavor
.mpt
) * idx
;
533 mr
->mtt
= __mthca_alloc_mtt(dev
, list_len
, dev
->mr_table
.fmr_mtt_buddy
);
537 mtt_seg
= mr
->mtt
->first_seg
* MTHCA_MTT_SEG_SIZE
;
539 if (mthca_is_memfree(dev
)) {
540 mr
->mem
.arbel
.mtts
= mthca_table_find(dev
->mr_table
.mtt_table
,
542 &mr
->mem
.arbel
.dma_handle
);
543 BUG_ON(!mr
->mem
.arbel
.mtts
);
545 mr
->mem
.tavor
.mtts
= dev
->mr_table
.tavor_fmr
.mtt_base
+ mtt_seg
;
547 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
549 goto err_out_free_mtt
;
551 mpt_entry
= mailbox
->buf
;
553 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
555 MTHCA_MPT_FLAG_REGION
|
558 mpt_entry
->page_size
= cpu_to_be32(mr
->attr
.page_shift
- 12);
559 mpt_entry
->key
= cpu_to_be32(key
);
560 mpt_entry
->pd
= cpu_to_be32(pd
);
561 memset(&mpt_entry
->start
, 0,
562 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, start
));
563 mpt_entry
->mtt_seg
= cpu_to_be64(dev
->mr_table
.mtt_base
+ mtt_seg
);
566 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
567 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
569 printk("[%02x] ", i
* 4);
570 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
571 if ((i
+ 1) % 4 == 0)
576 err
= mthca_SW2HW_MPT(dev
, mailbox
,
577 key
& (dev
->limits
.num_mpts
- 1),
580 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
581 goto err_out_mailbox_free
;
584 mthca_warn(dev
, "SW2HW_MPT returned status 0x%02x\n",
587 goto err_out_mailbox_free
;
590 mthca_free_mailbox(dev
, mailbox
);
593 err_out_mailbox_free
:
594 mthca_free_mailbox(dev
, mailbox
);
597 mthca_free_mtt(dev
, mr
->mtt
);
600 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
603 mthca_free(&dev
->mr_table
.mpt_alloc
, mr
->ibmr
.lkey
);
607 int mthca_free_fmr(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
612 mthca_free_region(dev
, fmr
->ibmr
.lkey
);
613 mthca_free_mtt(dev
, fmr
->mtt
);
618 static inline int mthca_check_fmr(struct mthca_fmr
*fmr
, u64
*page_list
,
619 int list_len
, u64 iova
)
623 if (list_len
> fmr
->attr
.max_pages
)
626 page_mask
= (1 << fmr
->attr
.page_shift
) - 1;
628 /* We are getting page lists, so va must be page aligned. */
629 if (iova
& page_mask
)
632 /* Trust the user not to pass misaligned data in page_list */
634 for (i
= 0; i
< list_len
; ++i
) {
635 if (page_list
[i
] & ~page_mask
)
639 if (fmr
->maps
>= fmr
->attr
.max_maps
)
646 int mthca_tavor_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
647 int list_len
, u64 iova
)
649 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
650 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
651 struct mthca_mpt_entry mpt_entry
;
655 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
661 key
= tavor_key_to_hw_index(fmr
->ibmr
.lkey
);
662 key
+= dev
->limits
.num_mpts
;
663 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= tavor_hw_index_to_key(key
);
665 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
667 for (i
= 0; i
< list_len
; ++i
) {
668 __be64 mtt_entry
= cpu_to_be64(page_list
[i
] |
669 MTHCA_MTT_FLAG_PRESENT
);
670 mthca_write64_raw(mtt_entry
, fmr
->mem
.tavor
.mtts
+ i
);
673 mpt_entry
.lkey
= cpu_to_be32(key
);
674 mpt_entry
.length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
675 mpt_entry
.start
= cpu_to_be64(iova
);
677 __raw_writel((__force u32
) mpt_entry
.lkey
, &fmr
->mem
.tavor
.mpt
->key
);
678 memcpy_toio(&fmr
->mem
.tavor
.mpt
->start
, &mpt_entry
.start
,
679 offsetof(struct mthca_mpt_entry
, window_count
) -
680 offsetof(struct mthca_mpt_entry
, start
));
682 writeb(MTHCA_MPT_STATUS_HW
, fmr
->mem
.tavor
.mpt
);
687 int mthca_arbel_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
688 int list_len
, u64 iova
)
690 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
691 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
695 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
701 key
= arbel_key_to_hw_index(fmr
->ibmr
.lkey
);
702 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
703 key
+= SINAI_FMR_KEY_INC
;
705 key
+= dev
->limits
.num_mpts
;
706 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= arbel_hw_index_to_key(key
);
708 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
712 for (i
= 0; i
< list_len
; ++i
)
713 fmr
->mem
.arbel
.mtts
[i
] = cpu_to_be64(page_list
[i
] |
714 MTHCA_MTT_FLAG_PRESENT
);
716 dma_sync_single(&dev
->pdev
->dev
, fmr
->mem
.arbel
.dma_handle
,
717 list_len
* sizeof(u64
), DMA_TO_DEVICE
);
719 fmr
->mem
.arbel
.mpt
->key
= cpu_to_be32(key
);
720 fmr
->mem
.arbel
.mpt
->lkey
= cpu_to_be32(key
);
721 fmr
->mem
.arbel
.mpt
->length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
722 fmr
->mem
.arbel
.mpt
->start
= cpu_to_be64(iova
);
726 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_HW
;
733 void mthca_tavor_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
740 key
= tavor_key_to_hw_index(fmr
->ibmr
.lkey
);
741 key
&= dev
->limits
.num_mpts
- 1;
742 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= tavor_hw_index_to_key(key
);
746 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
749 void mthca_arbel_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
756 key
= arbel_key_to_hw_index(fmr
->ibmr
.lkey
);
757 key
&= dev
->limits
.num_mpts
- 1;
758 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= arbel_hw_index_to_key(key
);
762 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
765 int mthca_init_mr_table(struct mthca_dev
*dev
)
770 err
= mthca_alloc_init(&dev
->mr_table
.mpt_alloc
,
771 dev
->limits
.num_mpts
,
772 ~0, dev
->limits
.reserved_mrws
);
776 if (!mthca_is_memfree(dev
) &&
777 (dev
->mthca_flags
& MTHCA_FLAG_DDR_HIDDEN
))
778 dev
->limits
.fmr_reserved_mtts
= 0;
780 dev
->mthca_flags
|= MTHCA_FLAG_FMR
;
782 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
783 mthca_dbg(dev
, "Memory key throughput optimization activated.\n");
785 err
= mthca_buddy_init(&dev
->mr_table
.mtt_buddy
,
786 fls(dev
->limits
.num_mtt_segs
- 1));
791 dev
->mr_table
.tavor_fmr
.mpt_base
= NULL
;
792 dev
->mr_table
.tavor_fmr
.mtt_base
= NULL
;
794 if (dev
->limits
.fmr_reserved_mtts
) {
795 i
= fls(dev
->limits
.fmr_reserved_mtts
- 1);
798 mthca_warn(dev
, "Unable to reserve 2^31 FMR MTTs.\n");
803 addr
= pci_resource_start(dev
->pdev
, 4) +
804 ((pci_resource_len(dev
->pdev
, 4) - 1) &
805 dev
->mr_table
.mpt_base
);
807 dev
->mr_table
.tavor_fmr
.mpt_base
=
808 ioremap(addr
, (1 << i
) * sizeof(struct mthca_mpt_entry
));
810 if (!dev
->mr_table
.tavor_fmr
.mpt_base
) {
811 mthca_warn(dev
, "MPT ioremap for FMR failed.\n");
816 addr
= pci_resource_start(dev
->pdev
, 4) +
817 ((pci_resource_len(dev
->pdev
, 4) - 1) &
818 dev
->mr_table
.mtt_base
);
820 dev
->mr_table
.tavor_fmr
.mtt_base
=
821 ioremap(addr
, (1 << i
) * MTHCA_MTT_SEG_SIZE
);
822 if (!dev
->mr_table
.tavor_fmr
.mtt_base
) {
823 mthca_warn(dev
, "MTT ioremap for FMR failed.\n");
828 err
= mthca_buddy_init(&dev
->mr_table
.tavor_fmr
.mtt_buddy
, i
);
830 goto err_fmr_mtt_buddy
;
832 /* Prevent regular MRs from using FMR keys */
833 err
= mthca_buddy_alloc(&dev
->mr_table
.mtt_buddy
, i
);
835 goto err_reserve_fmr
;
837 dev
->mr_table
.fmr_mtt_buddy
=
838 &dev
->mr_table
.tavor_fmr
.mtt_buddy
;
840 dev
->mr_table
.fmr_mtt_buddy
= &dev
->mr_table
.mtt_buddy
;
842 /* FMR table is always the first, take reserved MTTs out of there */
843 if (dev
->limits
.reserved_mtts
) {
844 i
= fls(dev
->limits
.reserved_mtts
- 1);
846 if (mthca_alloc_mtt_range(dev
, i
,
847 dev
->mr_table
.fmr_mtt_buddy
) == -1) {
848 mthca_warn(dev
, "MTT table of order %d is too small.\n",
849 dev
->mr_table
.fmr_mtt_buddy
->max_order
);
851 goto err_reserve_mtts
;
859 if (dev
->limits
.fmr_reserved_mtts
)
860 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
863 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
864 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
867 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
868 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
871 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
874 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);
879 void mthca_cleanup_mr_table(struct mthca_dev
*dev
)
881 /* XXX check if any MRs are still allocated? */
882 if (dev
->limits
.fmr_reserved_mtts
)
883 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
885 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
887 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
888 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
889 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
890 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
892 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);