drivers/infiniband/hw/mthca/mthca_mr.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
  33  */
  34
  35 #include <linux/slab.h>
  36 #include <linux/init.h>
  37 #include <linux/errno.h>
  38
  39 #include "mthca_dev.h"
  40 #include "mthca_cmd.h"
  41 #include "mthca_memfree.h"
  42
  43 /*
  44  * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
  45  */
  46 struct mthca_mpt_entry {
  47         u32 flags;
  48         u32 page_size;
  49         u32 key;
  50         u32 pd;
  51         u64 start;
  52         u64 length;
  53         u32 lkey;
  54         u32 window_count;
  55         u32 window_count_limit;
  56         u64 mtt_seg;
  57         u32 mtt_sz;             /* Arbel only */
  58         u32 reserved[2];
  59 } __attribute__((packed));
  60
  61 #define MTHCA_MPT_FLAG_SW_OWNS       (0xfUL << 28)
  62 #define MTHCA_MPT_FLAG_MIO           (1 << 17)
  63 #define MTHCA_MPT_FLAG_BIND_ENABLE   (1 << 15)
  64 #define MTHCA_MPT_FLAG_PHYSICAL      (1 <<  9)
  65 #define MTHCA_MPT_FLAG_REGION        (1 <<  8)
  66
  67 #define MTHCA_MTT_FLAG_PRESENT       1
  68
  69 /*
  70  * Buddy allocator for MTT segments (currently not very efficient
  71  * since it doesn't keep a free list and just searches linearly
  72  * through the bitmaps)
  73  */
  74
  75 static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
  76 {
  77         int o;
  78         int m;
  79         u32 seg;
  80
  81         spin_lock(&buddy->lock);
  82
  83         for (o = order; o <= buddy->max_order; ++o) {
  84                 m = 1 << (buddy->max_order - o);
  85                 seg = find_first_bit(buddy->bits[o], m);
  86                 if (seg < m)
  87                         goto found;
  88         }
  89
  90         spin_unlock(&buddy->lock);
  91         return -1;
  92
  93  found:
  94         clear_bit(seg, buddy->bits[o]);
  95
  96         while (o > order) {
  97                 --o;
  98                 seg <<= 1;
  99                 set_bit(seg ^ 1, buddy->bits[o]);
 100         }
 101
 102         spin_unlock(&buddy->lock);
 103
 104         seg <<= order;
 105
 106         return seg;
 107 }
 108
 109 static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order)
 110 {
 111         seg >>= order;
 112
 113         spin_lock(&buddy->lock);
 114
 115         while (test_bit(seg ^ 1, buddy->bits[order])) {
 116                 clear_bit(seg ^ 1, buddy->bits[order]);
 117                 seg >>= 1;
 118                 ++order;
 119         }
 120
 121         set_bit(seg, buddy->bits[order]);
 122
 123         spin_unlock(&buddy->lock);
 124 }
 125
 126 static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order)
 127 {
 128         int i, s;
 129
 130         buddy->max_order = max_order;
 131         spin_lock_init(&buddy->lock);
 132
 133         buddy->bits = kmalloc((buddy->max_order + 1) * sizeof (long *),
 134                               GFP_KERNEL);
 135         if (!buddy->bits)
 136                 goto err_out;
 137
 138         memset(buddy->bits, 0, (buddy->max_order + 1) * sizeof (long *));
 139
 140         for (i = 0; i <= buddy->max_order; ++i) {
 141                 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
 142                 buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL);
 143                 if (!buddy->bits[i])
 144                         goto err_out_free;
 145                 bitmap_zero(buddy->bits[i],
 146                             1 << (buddy->max_order - i));
 147         }
 148
 149         set_bit(0, buddy->bits[buddy->max_order]);
 150
 151         return 0;
 152
 153 err_out_free:
 154         for (i = 0; i <= buddy->max_order; ++i)
 155                 kfree(buddy->bits[i]);
 156
 157         kfree(buddy->bits);
 158
 159 err_out:
 160         return -ENOMEM;
 161 }
 162
 163 static void __devexit mthca_buddy_cleanup(struct mthca_buddy *buddy)
 164 {
 165         int i;
 166
 167         for (i = 0; i <= buddy->max_order; ++i)
 168                 kfree(buddy->bits[i]);
 169
 170         kfree(buddy->bits);
 171 }
 172
 173 static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order,
 174                            struct mthca_buddy *buddy)
 175 {
 176         u32 seg = mthca_buddy_alloc(buddy, order);
 177
 178         if (seg == -1)
 179                 return -1;
 180
 181         if (dev->hca_type == ARBEL_NATIVE)
 182                 if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg,
 183                                           seg + (1 << order) - 1)) {
 184                         mthca_buddy_free(buddy, seg, order);
 185                         seg = -1;
 186                 }
 187
 188         return seg;
 189 }
 190
 191 static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order,
 192                            struct mthca_buddy* buddy)
 193 {
 194         mthca_buddy_free(buddy, seg, order);
 195
 196         if (dev->hca_type == ARBEL_NATIVE)
 197                 mthca_table_put_range(dev, dev->mr_table.mtt_table, seg,
 198                                       seg + (1 << order) - 1);
 199 }
 200
 201 static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
 202 {
 203         if (dev->hca_type == ARBEL_NATIVE)
 204                 return (ind >> 24) | (ind << 8);
 205         else
 206                 return ind;
 207 }
 208
 209 static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
 210 {
 211         if (dev->hca_type == ARBEL_NATIVE)
 212                 return (key << 24) | (key >> 8);
 213         else
 214                 return key;
 215 }
 216
 217 int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
 218                            u32 access, struct mthca_mr *mr)
 219 {
 220         void *mailbox = NULL;
 221         struct mthca_mpt_entry *mpt_entry;
 222         u32 key;
 223         int err;
 224         u8 status;
 225
 226         might_sleep();
 227
 228         mr->order = -1;
 229         key = mthca_alloc(&dev->mr_table.mpt_alloc);
 230         if (key == -1)
 231                 return -ENOMEM;
 232         mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 233
 234         if (dev->hca_type == ARBEL_NATIVE) {
 235                 err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
 236                 if (err)
 237                         goto err_out_mpt_free;
 238         }
 239
 240         mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
 241                           GFP_KERNEL);
 242         if (!mailbox) {
 243                 err = -ENOMEM;
 244                 goto err_out_table;
 245         }
 246         mpt_entry = MAILBOX_ALIGN(mailbox);
 247
 248         mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS     |
 249                                        MTHCA_MPT_FLAG_MIO         |
 250                                        MTHCA_MPT_FLAG_PHYSICAL    |
 251                                        MTHCA_MPT_FLAG_REGION      |
 252                                        access);
 253         mpt_entry->page_size = 0;
 254         mpt_entry->key       = cpu_to_be32(key);
 255         mpt_entry->pd        = cpu_to_be32(pd);
 256         mpt_entry->start     = 0;
 257         mpt_entry->length    = ~0ULL;
 258
 259         memset(&mpt_entry->lkey, 0,
 260                sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
 261
 262         err = mthca_SW2HW_MPT(dev, mpt_entry,
 263                               key & (dev->limits.num_mpts - 1),
 264                               &status);
 265         if (err) {
 266                 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
 267                 goto err_out_table;
 268         } else if (status) {
 269                 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
 270                            status);
 271                 err = -EINVAL;
 272                 goto err_out_table;
 273         }
 274
 275         kfree(mailbox);
 276         return err;
 277
 278 err_out_table:
 279         if (dev->hca_type == ARBEL_NATIVE)
 280                 mthca_table_put(dev, dev->mr_table.mpt_table, key);
 281
 282 err_out_mpt_free:
 283         mthca_free(&dev->mr_table.mpt_alloc, key);
 284         kfree(mailbox);
 285         return err;
 286 }
 287
 288 int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
 289                         u64 *buffer_list, int buffer_size_shift,
 290                         int list_len, u64 iova, u64 total_size,
 291                         u32 access, struct mthca_mr *mr)
 292 {
 293         void *mailbox;
 294         u64 *mtt_entry;
 295         struct mthca_mpt_entry *mpt_entry;
 296         u32 key;
 297         int err = -ENOMEM;
 298         u8 status;
 299         int i;
 300
 301         might_sleep();
 302         WARN_ON(buffer_size_shift >= 32);
 303
 304         key = mthca_alloc(&dev->mr_table.mpt_alloc);
 305         if (key == -1)
 306                 return -ENOMEM;
 307         mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 308
 309         if (dev->hca_type == ARBEL_NATIVE) {
 310                 err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
 311                 if (err)
 312                         goto err_out_mpt_free;
 313         }
 314
 315         for (i = MTHCA_MTT_SEG_SIZE / 8, mr->order = 0;
 316              i < list_len;
 317              i <<= 1, ++mr->order)
 318                 ; /* nothing */
 319
 320         mr->first_seg = mthca_alloc_mtt(dev, mr->order,
 321                                         &dev->mr_table.mtt_buddy);
 322         if (mr->first_seg == -1)
 323                 goto err_out_table;
 324
 325         /*
 326          * If list_len is odd, we add one more dummy entry for
 327          * firmware efficiency.
 328          */
 329         mailbox = kmalloc(max(sizeof *mpt_entry,
 330                               (size_t) 8 * (list_len + (list_len & 1) + 2)) +
 331                           MTHCA_CMD_MAILBOX_EXTRA,
 332                           GFP_KERNEL);
 333         if (!mailbox)
 334                 goto err_out_free_mtt;
 335
 336         mtt_entry = MAILBOX_ALIGN(mailbox);
 337
 338         mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
 339                                    mr->first_seg * MTHCA_MTT_SEG_SIZE);
 340         mtt_entry[1] = 0;
 341         for (i = 0; i < list_len; ++i)
 342                 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
 343                                                MTHCA_MTT_FLAG_PRESENT);
 344         if (list_len & 1) {
 345                 mtt_entry[i + 2] = 0;
 346                 ++list_len;
 347         }
 348
 349         if (0) {
 350                 mthca_dbg(dev, "Dumping MPT entry\n");
 351                 for (i = 0; i < list_len + 2; ++i)
 352                         printk(KERN_ERR "[%2d] %016llx\n",
 353                                i, (unsigned long long) be64_to_cpu(mtt_entry[i]));
 354         }
 355
 356         err = mthca_WRITE_MTT(dev, mtt_entry, list_len, &status);
 357         if (err) {
 358                 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
 359                 goto err_out_mailbox_free;
 360         }
 361         if (status) {
 362                 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
 363                            status);
 364                 err = -EINVAL;
 365                 goto err_out_mailbox_free;
 366         }
 367
 368         mpt_entry = MAILBOX_ALIGN(mailbox);
 369
 370         mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS     |
 371                                        MTHCA_MPT_FLAG_MIO         |
 372                                        MTHCA_MPT_FLAG_REGION      |
 373                                        access);
 374
 375         mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
 376         mpt_entry->key       = cpu_to_be32(key);
 377         mpt_entry->pd        = cpu_to_be32(pd);
 378         mpt_entry->start     = cpu_to_be64(iova);
 379         mpt_entry->length    = cpu_to_be64(total_size);
 380         memset(&mpt_entry->lkey, 0,
 381                sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
 382         mpt_entry->mtt_seg   = cpu_to_be64(dev->mr_table.mtt_base +
 383                                            mr->first_seg * MTHCA_MTT_SEG_SIZE);
 384
 385         if (0) {
 386                 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
 387                 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
 388                         if (i % 4 == 0)
 389                                 printk("[%02x] ", i * 4);
 390                         printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i]));
 391                         if ((i + 1) % 4 == 0)
 392                                 printk("\n");
 393                 }
 394         }
 395
 396         err = mthca_SW2HW_MPT(dev, mpt_entry,
 397                               key & (dev->limits.num_mpts - 1),
 398                               &status);
 399         if (err)
 400                 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
 401         else if (status) {
 402                 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
 403                            status);
 404                 err = -EINVAL;
 405         }
 406
 407         kfree(mailbox);
 408         return err;
 409
 410 err_out_mailbox_free:
 411         kfree(mailbox);
 412
 413 err_out_free_mtt:
 414         mthca_free_mtt(dev, mr->first_seg, mr->order, &dev->mr_table.mtt_buddy);
 415
 416 err_out_table:
 417         if (dev->hca_type == ARBEL_NATIVE)
 418                 mthca_table_put(dev, dev->mr_table.mpt_table, key);
 419
 420 err_out_mpt_free:
 421         mthca_free(&dev->mr_table.mpt_alloc, key);
 422         return err;
 423 }
 424
 425 void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
 426 {
 427         int err;
 428         u8 status;
 429
 430         might_sleep();
 431
 432         err = mthca_HW2SW_MPT(dev, NULL,
 433                               key_to_hw_index(dev, mr->ibmr.lkey) &
 434                               (dev->limits.num_mpts - 1),
 435                               &status);
 436         if (err)
 437                 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
 438         else if (status)
 439                 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
 440                            status);
 441
 442         if (mr->order >= 0)
 443                 mthca_free_mtt(dev, mr->first_seg, mr->order, &dev->mr_table.mtt_buddy);
 444
 445         if (dev->hca_type == ARBEL_NATIVE)
 446                 mthca_table_put(dev, dev->mr_table.mpt_table,
 447                                 key_to_hw_index(dev, mr->ibmr.lkey));
 448         mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey));
 449 }
 450
 451 int __devinit mthca_init_mr_table(struct mthca_dev *dev)
 452 {
 453         int err;
 454
 455         err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
 456                                dev->limits.num_mpts,
 457                                ~0, dev->limits.reserved_mrws);
 458         if (err)
 459                 return err;
 460
 461         err = mthca_buddy_init(&dev->mr_table.mtt_buddy,
 462                                fls(dev->limits.num_mtt_segs - 1));
 463         if (err)
 464                 goto err_mtt_buddy;
 465
 466         if (dev->limits.reserved_mtts) {
 467                 if (mthca_alloc_mtt(dev, fls(dev->limits.reserved_mtts - 1),
 468                                     &dev->mr_table.mtt_buddy) == -1) {
 469                         mthca_warn(dev, "MTT table of order %d is too small.\n",
 470                                   dev->mr_table.mtt_buddy.max_order);
 471                         err = -ENOMEM;
 472                         goto err_mtt_buddy;
 473                 }
 474         }
 475
 476         return 0;
 477
 478 err_mtt_buddy:
 479         mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
 480
 481         return err;
 482 }
 483
 484 void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev)
 485 {
 486         /* XXX check if any MRs are still allocated? */
 487         mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
 488         mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
 489 }