drivers/pci/msi.c

   1 /*
   2  * File:        msi.c
   3  * Purpose:     PCI Message Signaled Interrupt (MSI)
   4  *
   5  * Copyright (C) 2003-2004 Intel
   6  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
   7  */
   8
   9 #include <linux/err.h>
  10 #include <linux/mm.h>
  11 #include <linux/irq.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/init.h>
  14 #include <linux/ioport.h>
  15 #include <linux/pci.h>
  16 #include <linux/proc_fs.h>
  17 #include <linux/msi.h>
  18 #include <linux/smp.h>
  19 #include <linux/errno.h>
  20 #include <linux/io.h>
  21
  22 #include "pci.h"
  23 #include "msi.h"
  24
  25 static int pci_msi_enable = 1;
  26
  27 /* Arch hooks */
  28
  29 #ifndef arch_msi_check_device
  30 int arch_msi_check_device(struct pci_dev *dev, int nvec, int type)
  31 {
  32         return 0;
  33 }
  34 #endif
  35
  36 #ifndef arch_setup_msi_irqs
  37 int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
  38 {
  39         struct msi_desc *entry;
  40         int ret;
  41
  42         /*
  43          * If an architecture wants to support multiple MSI, it needs to
  44          * override arch_setup_msi_irqs()
  45          */
  46         if (type == PCI_CAP_ID_MSI && nvec > 1)
  47                 return 1;
  48
  49         list_for_each_entry(entry, &dev->msi_list, list) {
  50                 ret = arch_setup_msi_irq(dev, entry);
  51                 if (ret < 0)
  52                         return ret;
  53                 if (ret > 0)
  54                         return -ENOSPC;
  55         }
  56
  57         return 0;
  58 }
  59 #endif
  60
  61 #ifndef arch_teardown_msi_irqs
  62 void arch_teardown_msi_irqs(struct pci_dev *dev)
  63 {
  64         struct msi_desc *entry;
  65
  66         list_for_each_entry(entry, &dev->msi_list, list) {
  67                 int i, nvec;
  68                 if (entry->irq == 0)
  69                         continue;
  70                 nvec = 1 << entry->msi_attrib.multiple;
  71                 for (i = 0; i < nvec; i++)
  72                         arch_teardown_msi_irq(entry->irq + i);
  73         }
  74 }
  75 #endif
  76
  77 static void msi_set_enable(struct pci_dev *dev, int pos, int enable)
  78 {
  79         u16 control;
  80
  81         BUG_ON(!pos);
  82
  83         pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
  84         control &= ~PCI_MSI_FLAGS_ENABLE;
  85         if (enable)
  86                 control |= PCI_MSI_FLAGS_ENABLE;
  87         pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
  88 }
  89
  90 static void msix_set_enable(struct pci_dev *dev, int enable)
  91 {
  92         int pos;
  93         u16 control;
  94
  95         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
  96         if (pos) {
  97                 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
  98                 control &= ~PCI_MSIX_FLAGS_ENABLE;
  99                 if (enable)
 100                         control |= PCI_MSIX_FLAGS_ENABLE;
 101                 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 102         }
 103 }
 104
 105 static inline __attribute_const__ u32 msi_mask(unsigned x)
 106 {
 107         /* Don't shift by >= width of type */
 108         if (x >= 5)
 109                 return 0xffffffff;
 110         return (1 << (1 << x)) - 1;
 111 }
 112
 113 static inline __attribute_const__ u32 msi_capable_mask(u16 control)
 114 {
 115         return msi_mask((control >> 1) & 7);
 116 }
 117
 118 static inline __attribute_const__ u32 msi_enabled_mask(u16 control)
 119 {
 120         return msi_mask((control >> 4) & 7);
 121 }
 122
 123 /*
 124  * PCI 2.3 does not specify mask bits for each MSI interrupt.  Attempting to
 125  * mask all MSI interrupts by clearing the MSI enable bit does not work
 126  * reliably as devices without an INTx disable bit will then generate a
 127  * level IRQ which will never be cleared.
 128  */
 129 static u32 __msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
 130 {
 131         u32 mask_bits = desc->masked;
 132
 133         if (!desc->msi_attrib.maskbit)
 134                 return 0;
 135
 136         mask_bits &= ~mask;
 137         mask_bits |= flag;
 138         pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits);
 139
 140         return mask_bits;
 141 }
 142
 143 static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
 144 {
 145         desc->masked = __msi_mask_irq(desc, mask, flag);
 146 }
 147
 148 /*
 149  * This internal function does not flush PCI writes to the device.
 150  * All users must ensure that they read from the device before either
 151  * assuming that the device state is up to date, or returning out of this
 152  * file.  This saves a few milliseconds when initialising devices with lots
 153  * of MSI-X interrupts.
 154  */
 155 static u32 __msix_mask_irq(struct msi_desc *desc, u32 flag)
 156 {
 157         u32 mask_bits = desc->masked;
 158         unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
 159                                                 PCI_MSIX_ENTRY_VECTOR_CTRL;
 160         mask_bits &= ~1;
 161         mask_bits |= flag;
 162         writel(mask_bits, desc->mask_base + offset);
 163
 164         return mask_bits;
 165 }
 166
 167 static void msix_mask_irq(struct msi_desc *desc, u32 flag)
 168 {
 169         desc->masked = __msix_mask_irq(desc, flag);
 170 }
 171
 172 static void msi_set_mask_bit(unsigned irq, u32 flag)
 173 {
 174         struct msi_desc *desc = get_irq_msi(irq);
 175
 176         if (desc->msi_attrib.is_msix) {
 177                 msix_mask_irq(desc, flag);
 178                 readl(desc->mask_base);         /* Flush write to device */
 179         } else {
 180                 unsigned offset = irq - desc->dev->irq;
 181                 msi_mask_irq(desc, 1 << offset, flag << offset);
 182         }
 183 }
 184
 185 void mask_msi_irq(unsigned int irq)
 186 {
 187         msi_set_mask_bit(irq, 1);
 188 }
 189
 190 void unmask_msi_irq(unsigned int irq)
 191 {
 192         msi_set_mask_bit(irq, 0);
 193 }
 194
 195 void read_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg)
 196 {
 197         struct msi_desc *entry = get_irq_desc_msi(desc);
 198
 199         BUG_ON(entry->dev->current_state != PCI_D0);
 200
 201         if (entry->msi_attrib.is_msix) {
 202                 void __iomem *base = entry->mask_base +
 203                         entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
 204
 205                 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
 206                 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
 207                 msg->data = readl(base + PCI_MSIX_ENTRY_DATA);
 208         } else {
 209                 struct pci_dev *dev = entry->dev;
 210                 int pos = entry->msi_attrib.pos;
 211                 u16 data;
 212
 213                 pci_read_config_dword(dev, msi_lower_address_reg(pos),
 214                                         &msg->address_lo);
 215                 if (entry->msi_attrib.is_64) {
 216                         pci_read_config_dword(dev, msi_upper_address_reg(pos),
 217                                                 &msg->address_hi);
 218                         pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
 219                 } else {
 220                         msg->address_hi = 0;
 221                         pci_read_config_word(dev, msi_data_reg(pos, 0), &data);
 222                 }
 223                 msg->data = data;
 224         }
 225 }
 226
 227 void read_msi_msg(unsigned int irq, struct msi_msg *msg)
 228 {
 229         struct irq_desc *desc = irq_to_desc(irq);
 230
 231         read_msi_msg_desc(desc, msg);
 232 }
 233
 234 void get_cached_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg)
 235 {
 236         struct msi_desc *entry = get_irq_desc_msi(desc);
 237
 238         /* Assert that the cache is valid, assuming that
 239          * valid messages are not all-zeroes. */
 240         BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo |
 241                  entry->msg.data));
 242
 243         *msg = entry->msg;
 244 }
 245
 246 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
 247 {
 248         struct irq_desc *desc = irq_to_desc(irq);
 249
 250         get_cached_msi_msg_desc(desc, msg);
 251 }
 252
 253 void write_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg)
 254 {
 255         struct msi_desc *entry = get_irq_desc_msi(desc);
 256
 257         if (entry->dev->current_state != PCI_D0) {
 258                 /* Don't touch the hardware now */
 259         } else if (entry->msi_attrib.is_msix) {
 260                 void __iomem *base;
 261                 base = entry->mask_base +
 262                         entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
 263
 264                 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
 265                 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
 266                 writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
 267         } else {
 268                 struct pci_dev *dev = entry->dev;
 269                 int pos = entry->msi_attrib.pos;
 270                 u16 msgctl;
 271
 272                 pci_read_config_word(dev, msi_control_reg(pos), &msgctl);
 273                 msgctl &= ~PCI_MSI_FLAGS_QSIZE;
 274                 msgctl |= entry->msi_attrib.multiple << 4;
 275                 pci_write_config_word(dev, msi_control_reg(pos), msgctl);
 276
 277                 pci_write_config_dword(dev, msi_lower_address_reg(pos),
 278                                         msg->address_lo);
 279                 if (entry->msi_attrib.is_64) {
 280                         pci_write_config_dword(dev, msi_upper_address_reg(pos),
 281                                                 msg->address_hi);
 282                         pci_write_config_word(dev, msi_data_reg(pos, 1),
 283                                                 msg->data);
 284                 } else {
 285                         pci_write_config_word(dev, msi_data_reg(pos, 0),
 286                                                 msg->data);
 287                 }
 288         }
 289         entry->msg = *msg;
 290 }
 291
 292 void write_msi_msg(unsigned int irq, struct msi_msg *msg)
 293 {
 294         struct irq_desc *desc = irq_to_desc(irq);
 295
 296         write_msi_msg_desc(desc, msg);
 297 }
 298
 299 static void free_msi_irqs(struct pci_dev *dev)
 300 {
 301         struct msi_desc *entry, *tmp;
 302
 303         list_for_each_entry(entry, &dev->msi_list, list) {
 304                 int i, nvec;
 305                 if (!entry->irq)
 306                         continue;
 307                 nvec = 1 << entry->msi_attrib.multiple;
 308                 for (i = 0; i < nvec; i++)
 309                         BUG_ON(irq_has_action(entry->irq + i));
 310         }
 311
 312         arch_teardown_msi_irqs(dev);
 313
 314         list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) {
 315                 if (entry->msi_attrib.is_msix) {
 316                         if (list_is_last(&entry->list, &dev->msi_list))
 317                                 iounmap(entry->mask_base);
 318                 }
 319                 list_del(&entry->list);
 320                 kfree(entry);
 321         }
 322 }
 323
 324 static struct msi_desc *alloc_msi_entry(struct pci_dev *dev)
 325 {
 326         struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 327         if (!desc)
 328                 return NULL;
 329
 330         INIT_LIST_HEAD(&desc->list);
 331         desc->dev = dev;
 332
 333         return desc;
 334 }
 335
 336 static void pci_intx_for_msi(struct pci_dev *dev, int enable)
 337 {
 338         if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
 339                 pci_intx(dev, enable);
 340 }
 341
 342 static void __pci_restore_msi_state(struct pci_dev *dev)
 343 {
 344         int pos;
 345         u16 control;
 346         struct msi_desc *entry;
 347
 348         if (!dev->msi_enabled)
 349                 return;
 350
 351         entry = get_irq_msi(dev->irq);
 352         pos = entry->msi_attrib.pos;
 353
 354         pci_intx_for_msi(dev, 0);
 355         msi_set_enable(dev, pos, 0);
 356         write_msi_msg(dev->irq, &entry->msg);
 357
 358         pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
 359         msi_mask_irq(entry, msi_capable_mask(control), entry->masked);
 360         control &= ~PCI_MSI_FLAGS_QSIZE;
 361         control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
 362         pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
 363 }
 364
 365 static void __pci_restore_msix_state(struct pci_dev *dev)
 366 {
 367         int pos;
 368         struct msi_desc *entry;
 369         u16 control;
 370
 371         if (!dev->msix_enabled)
 372                 return;
 373         BUG_ON(list_empty(&dev->msi_list));
 374         entry = list_first_entry(&dev->msi_list, struct msi_desc, list);
 375         pos = entry->msi_attrib.pos;
 376         pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
 377
 378         /* route the table */
 379         pci_intx_for_msi(dev, 0);
 380         control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL;
 381         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 382
 383         list_for_each_entry(entry, &dev->msi_list, list) {
 384                 write_msi_msg(entry->irq, &entry->msg);
 385                 msix_mask_irq(entry, entry->masked);
 386         }
 387
 388         control &= ~PCI_MSIX_FLAGS_MASKALL;
 389         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 390 }
 391
 392 void pci_restore_msi_state(struct pci_dev *dev)
 393 {
 394         __pci_restore_msi_state(dev);
 395         __pci_restore_msix_state(dev);
 396 }
 397 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 398
 399 /**
 400  * msi_capability_init - configure device's MSI capability structure
 401  * @dev: pointer to the pci_dev data structure of MSI device function
 402  * @nvec: number of interrupts to allocate
 403  *
 404  * Setup the MSI capability structure of the device with the requested
 405  * number of interrupts.  A return value of zero indicates the successful
 406  * setup of an entry with the new MSI irq.  A negative return value indicates
 407  * an error, and a positive return value indicates the number of interrupts
 408  * which could have been allocated.
 409  */
 410 static int msi_capability_init(struct pci_dev *dev, int nvec)
 411 {
 412         struct msi_desc *entry;
 413         int pos, ret;
 414         u16 control;
 415         unsigned mask;
 416
 417         pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
 418         msi_set_enable(dev, pos, 0);    /* Disable MSI during set up */
 419
 420         pci_read_config_word(dev, msi_control_reg(pos), &control);
 421         /* MSI Entry Initialization */
 422         entry = alloc_msi_entry(dev);
 423         if (!entry)
 424                 return -ENOMEM;
 425
 426         entry->msi_attrib.is_msix       = 0;
 427         entry->msi_attrib.is_64         = is_64bit_address(control);
 428         entry->msi_attrib.entry_nr      = 0;
 429         entry->msi_attrib.maskbit       = is_mask_bit_support(control);
 430         entry->msi_attrib.default_irq   = dev->irq;     /* Save IOAPIC IRQ */
 431         entry->msi_attrib.pos           = pos;
 432
 433         entry->mask_pos = msi_mask_reg(pos, entry->msi_attrib.is_64);
 434         /* All MSIs are unmasked by default, Mask them all */
 435         if (entry->msi_attrib.maskbit)
 436                 pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
 437         mask = msi_capable_mask(control);
 438         msi_mask_irq(entry, mask, mask);
 439
 440         list_add_tail(&entry->list, &dev->msi_list);
 441
 442         /* Configure MSI capability structure */
 443         ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
 444         if (ret) {
 445                 msi_mask_irq(entry, mask, ~mask);
 446                 free_msi_irqs(dev);
 447                 return ret;
 448         }
 449
 450         /* Set MSI enabled bits  */
 451         pci_intx_for_msi(dev, 0);
 452         msi_set_enable(dev, pos, 1);
 453         dev->msi_enabled = 1;
 454
 455         dev->irq = entry->irq;
 456         return 0;
 457 }
 458
 459 static void __iomem *msix_map_region(struct pci_dev *dev, unsigned pos,
 460                                                         unsigned nr_entries)
 461 {
 462         unsigned long phys_addr;
 463         u32 table_offset;
 464         u8 bir;
 465
 466         pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
 467         bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
 468         table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
 469         phys_addr = pci_resource_start(dev, bir) + table_offset;
 470
 471         return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
 472 }
 473
 474 static int msix_setup_entries(struct pci_dev *dev, unsigned pos,
 475                                 void __iomem *base, struct msix_entry *entries,
 476                                 int nvec)
 477 {
 478         struct msi_desc *entry;
 479         int i;
 480
 481         for (i = 0; i < nvec; i++) {
 482                 entry = alloc_msi_entry(dev);
 483                 if (!entry) {
 484                         if (!i)
 485                                 iounmap(base);
 486                         else
 487                                 free_msi_irqs(dev);
 488                         /* No enough memory. Don't try again */
 489                         return -ENOMEM;
 490                 }
 491
 492                 entry->msi_attrib.is_msix       = 1;
 493                 entry->msi_attrib.is_64         = 1;
 494                 entry->msi_attrib.entry_nr      = entries[i].entry;
 495                 entry->msi_attrib.default_irq   = dev->irq;
 496                 entry->msi_attrib.pos           = pos;
 497                 entry->mask_base                = base;
 498
 499                 list_add_tail(&entry->list, &dev->msi_list);
 500         }
 501
 502         return 0;
 503 }
 504
 505 static void msix_program_entries(struct pci_dev *dev,
 506                                         struct msix_entry *entries)
 507 {
 508         struct msi_desc *entry;
 509         int i = 0;
 510
 511         list_for_each_entry(entry, &dev->msi_list, list) {
 512                 int offset = entries[i].entry * PCI_MSIX_ENTRY_SIZE +
 513                                                 PCI_MSIX_ENTRY_VECTOR_CTRL;
 514
 515                 entries[i].vector = entry->irq;
 516                 set_irq_msi(entry->irq, entry);
 517                 entry->masked = readl(entry->mask_base + offset);
 518                 msix_mask_irq(entry, 1);
 519                 i++;
 520         }
 521 }
 522
 523 /**
 524  * msix_capability_init - configure device's MSI-X capability
 525  * @dev: pointer to the pci_dev data structure of MSI-X device function
 526  * @entries: pointer to an array of struct msix_entry entries
 527  * @nvec: number of @entries
 528  *
 529  * Setup the MSI-X capability structure of device function with a
 530  * single MSI-X irq. A return of zero indicates the successful setup of
 531  * requested MSI-X entries with allocated irqs or non-zero for otherwise.
 532  **/
 533 static int msix_capability_init(struct pci_dev *dev,
 534                                 struct msix_entry *entries, int nvec)
 535 {
 536         int pos, ret;
 537         u16 control;
 538         void __iomem *base;
 539
 540         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
 541         pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
 542
 543         /* Ensure MSI-X is disabled while it is set up */
 544         control &= ~PCI_MSIX_FLAGS_ENABLE;
 545         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 546
 547         /* Request & Map MSI-X table region */
 548         base = msix_map_region(dev, pos, multi_msix_capable(control));
 549         if (!base)
 550                 return -ENOMEM;
 551
 552         ret = msix_setup_entries(dev, pos, base, entries, nvec);
 553         if (ret)
 554                 return ret;
 555
 556         ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
 557         if (ret)
 558                 goto error;
 559
 560         /*
 561          * Some devices require MSI-X to be enabled before we can touch the
 562          * MSI-X registers.  We need to mask all the vectors to prevent
 563          * interrupts coming in before they're fully set up.
 564          */
 565         control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE;
 566         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 567
 568         msix_program_entries(dev, entries);
 569
 570         /* Set MSI-X enabled bits and unmask the function */
 571         pci_intx_for_msi(dev, 0);
 572         dev->msix_enabled = 1;
 573
 574         control &= ~PCI_MSIX_FLAGS_MASKALL;
 575         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 576
 577         return 0;
 578
 579 error:
 580         if (ret < 0) {
 581                 /*
 582                  * If we had some success, report the number of irqs
 583                  * we succeeded in setting up.
 584                  */
 585                 struct msi_desc *entry;
 586                 int avail = 0;
 587
 588                 list_for_each_entry(entry, &dev->msi_list, list) {
 589                         if (entry->irq != 0)
 590                                 avail++;
 591                 }
 592                 if (avail != 0)
 593                         ret = avail;
 594         }
 595
 596         free_msi_irqs(dev);
 597
 598         return ret;
 599 }
 600
 601 /**
 602  * pci_msi_check_device - check whether MSI may be enabled on a device
 603  * @dev: pointer to the pci_dev data structure of MSI device function
 604  * @nvec: how many MSIs have been requested ?
 605  * @type: are we checking for MSI or MSI-X ?
 606  *
 607  * Look at global flags, the device itself, and its parent busses
 608  * to determine if MSI/-X are supported for the device. If MSI/-X is
 609  * supported return 0, else return an error code.
 610  **/
 611 static int pci_msi_check_device(struct pci_dev *dev, int nvec, int type)
 612 {
 613         struct pci_bus *bus;
 614         int ret;
 615
 616         /* MSI must be globally enabled and supported by the device */
 617         if (!pci_msi_enable || !dev || dev->no_msi)
 618                 return -EINVAL;
 619
 620         /*
 621          * You can't ask to have 0 or less MSIs configured.
 622          *  a) it's stupid ..
 623          *  b) the list manipulation code assumes nvec >= 1.
 624          */
 625         if (nvec < 1)
 626                 return -ERANGE;
 627
 628         /*
 629          * Any bridge which does NOT route MSI transactions from its
 630          * secondary bus to its primary bus must set NO_MSI flag on
 631          * the secondary pci_bus.
 632          * We expect only arch-specific PCI host bus controller driver
 633          * or quirks for specific PCI bridges to be setting NO_MSI.
 634          */
 635         for (bus = dev->bus; bus; bus = bus->parent)
 636                 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
 637                         return -EINVAL;
 638
 639         ret = arch_msi_check_device(dev, nvec, type);
 640         if (ret)
 641                 return ret;
 642
 643         if (!pci_find_capability(dev, type))
 644                 return -EINVAL;
 645
 646         return 0;
 647 }
 648
 649 /**
 650  * pci_enable_msi_block - configure device's MSI capability structure
 651  * @dev: device to configure
 652  * @nvec: number of interrupts to configure
 653  *
 654  * Allocate IRQs for a device with the MSI capability.
 655  * This function returns a negative errno if an error occurs.  If it
 656  * is unable to allocate the number of interrupts requested, it returns
 657  * the number of interrupts it might be able to allocate.  If it successfully
 658  * allocates at least the number of interrupts requested, it returns 0 and
 659  * updates the @dev's irq member to the lowest new interrupt number; the
 660  * other interrupt numbers allocated to this device are consecutive.
 661  */
 662 int pci_enable_msi_block(struct pci_dev *dev, unsigned int nvec)
 663 {
 664         int status, pos, maxvec;
 665         u16 msgctl;
 666
 667         pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
 668         if (!pos)
 669                 return -EINVAL;
 670         pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
 671         maxvec = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1);
 672         if (nvec > maxvec)
 673                 return maxvec;
 674
 675         status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSI);
 676         if (status)
 677                 return status;
 678
 679         WARN_ON(!!dev->msi_enabled);
 680
 681         /* Check whether driver already requested MSI-X irqs */
 682         if (dev->msix_enabled) {
 683                 dev_info(&dev->dev, "can't enable MSI "
 684                          "(MSI-X already enabled)\n");
 685                 return -EINVAL;
 686         }
 687
 688         status = msi_capability_init(dev, nvec);
 689         return status;
 690 }
 691 EXPORT_SYMBOL(pci_enable_msi_block);
 692
 693 void pci_msi_shutdown(struct pci_dev *dev)
 694 {
 695         struct msi_desc *desc;
 696         u32 mask;
 697         u16 ctrl;
 698         unsigned pos;
 699
 700         if (!pci_msi_enable || !dev || !dev->msi_enabled)
 701                 return;
 702
 703         BUG_ON(list_empty(&dev->msi_list));
 704         desc = list_first_entry(&dev->msi_list, struct msi_desc, list);
 705         pos = desc->msi_attrib.pos;
 706
 707         msi_set_enable(dev, pos, 0);
 708         pci_intx_for_msi(dev, 1);
 709         dev->msi_enabled = 0;
 710
 711         /* Return the device with MSI unmasked as initial states */
 712         pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &ctrl);
 713         mask = msi_capable_mask(ctrl);
 714         /* Keep cached state to be restored */
 715         __msi_mask_irq(desc, mask, ~mask);
 716
 717         /* Restore dev->irq to its default pin-assertion irq */
 718         dev->irq = desc->msi_attrib.default_irq;
 719 }
 720
 721 void pci_disable_msi(struct pci_dev *dev)
 722 {
 723         if (!pci_msi_enable || !dev || !dev->msi_enabled)
 724                 return;
 725
 726         pci_msi_shutdown(dev);
 727         free_msi_irqs(dev);
 728 }
 729 EXPORT_SYMBOL(pci_disable_msi);
 730
 731 /**
 732  * pci_msix_table_size - return the number of device's MSI-X table entries
 733  * @dev: pointer to the pci_dev data structure of MSI-X device function
 734  */
 735 int pci_msix_table_size(struct pci_dev *dev)
 736 {
 737         int pos;
 738         u16 control;
 739
 740         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
 741         if (!pos)
 742                 return 0;
 743
 744         pci_read_config_word(dev, msi_control_reg(pos), &control);
 745         return multi_msix_capable(control);
 746 }
 747
 748 /**
 749  * pci_enable_msix - configure device's MSI-X capability structure
 750  * @dev: pointer to the pci_dev data structure of MSI-X device function
 751  * @entries: pointer to an array of MSI-X entries
 752  * @nvec: number of MSI-X irqs requested for allocation by device driver
 753  *
 754  * Setup the MSI-X capability structure of device function with the number
 755  * of requested irqs upon its software driver call to request for
 756  * MSI-X mode enabled on its hardware device function. A return of zero
 757  * indicates the successful configuration of MSI-X capability structure
 758  * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
 759  * Or a return of > 0 indicates that driver request is exceeding the number
 760  * of irqs or MSI-X vectors available. Driver should use the returned value to
 761  * re-send its request.
 762  **/
 763 int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
 764 {
 765         int status, nr_entries;
 766         int i, j;
 767
 768         if (!entries)
 769                 return -EINVAL;
 770
 771         status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX);
 772         if (status)
 773                 return status;
 774
 775         nr_entries = pci_msix_table_size(dev);
 776         if (nvec > nr_entries)
 777                 return nr_entries;
 778
 779         /* Check for any invalid entries */
 780         for (i = 0; i < nvec; i++) {
 781                 if (entries[i].entry >= nr_entries)
 782                         return -EINVAL;         /* invalid entry */
 783                 for (j = i + 1; j < nvec; j++) {
 784                         if (entries[i].entry == entries[j].entry)
 785                                 return -EINVAL; /* duplicate entry */
 786                 }
 787         }
 788         WARN_ON(!!dev->msix_enabled);
 789
 790         /* Check whether driver already requested for MSI irq */
 791         if (dev->msi_enabled) {
 792                 dev_info(&dev->dev, "can't enable MSI-X "
 793                        "(MSI IRQ already assigned)\n");
 794                 return -EINVAL;
 795         }
 796         status = msix_capability_init(dev, entries, nvec);
 797         return status;
 798 }
 799 EXPORT_SYMBOL(pci_enable_msix);
 800
 801 void pci_msix_shutdown(struct pci_dev *dev)
 802 {
 803         struct msi_desc *entry;
 804
 805         if (!pci_msi_enable || !dev || !dev->msix_enabled)
 806                 return;
 807
 808         /* Return the device with MSI-X masked as initial states */
 809         list_for_each_entry(entry, &dev->msi_list, list) {
 810                 /* Keep cached states to be restored */
 811                 __msix_mask_irq(entry, 1);
 812         }
 813
 814         msix_set_enable(dev, 0);
 815         pci_intx_for_msi(dev, 1);
 816         dev->msix_enabled = 0;
 817 }
 818
 819 void pci_disable_msix(struct pci_dev *dev)
 820 {
 821         if (!pci_msi_enable || !dev || !dev->msix_enabled)
 822                 return;
 823
 824         pci_msix_shutdown(dev);
 825         free_msi_irqs(dev);
 826 }
 827 EXPORT_SYMBOL(pci_disable_msix);
 828
 829 /**
 830  * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state
 831  * @dev: pointer to the pci_dev data structure of MSI(X) device function
 832  *
 833  * Being called during hotplug remove, from which the device function
 834  * is hot-removed. All previous assigned MSI/MSI-X irqs, if
 835  * allocated for this device function, are reclaimed to unused state,
 836  * which may be used later on.
 837  **/
 838 void msi_remove_pci_irq_vectors(struct pci_dev *dev)
 839 {
 840         if (!pci_msi_enable || !dev)
 841                 return;
 842
 843         if (dev->msi_enabled || dev->msix_enabled)
 844                 free_msi_irqs(dev);
 845 }
 846
 847 void pci_no_msi(void)
 848 {
 849         pci_msi_enable = 0;
 850 }
 851
 852 /**
 853  * pci_msi_enabled - is MSI enabled?
 854  *
 855  * Returns true if MSI has not been disabled by the command-line option
 856  * pci=nomsi.
 857  **/
 858 int pci_msi_enabled(void)
 859 {
 860         return pci_msi_enable;
 861 }
 862 EXPORT_SYMBOL(pci_msi_enabled);
 863
 864 void pci_msi_init_pci_dev(struct pci_dev *dev)
 865 {
 866         INIT_LIST_HEAD(&dev->msi_list);
 867 }