kernel/2.6.29.6-aldebaran-rt/drivers/pci/msi.c

   1 /*
   2  * File:        msi.c
   3  * Purpose:     PCI Message Signaled Interrupt (MSI)
   4  *
   5  * Copyright (C) 2003-2004 Intel
   6  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
   7  */
   8
   9 #include <linux/err.h>
  10 #include <linux/mm.h>
  11 #include <linux/irq.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/init.h>
  14 #include <linux/ioport.h>
  15 #include <linux/pci.h>
  16 #include <linux/proc_fs.h>
  17 #include <linux/msi.h>
  18 #include <linux/smp.h>
  19
  20 #include <asm/errno.h>
  21 #include <asm/io.h>
  22
  23 #include "pci.h"
  24 #include "msi.h"
  25
  26 static int pci_msi_enable = 1;
  27
  28 /* Arch hooks */
  29
  30 int __attribute__ ((weak))
  31 arch_msi_check_device(struct pci_dev *dev, int nvec, int type)
  32 {
  33         return 0;
  34 }
  35
  36 int __attribute__ ((weak))
  37 arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *entry)
  38 {
  39         return 0;
  40 }
  41
  42 int __attribute__ ((weak))
  43 arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
  44 {
  45         struct msi_desc *entry;
  46         int ret;
  47
  48         list_for_each_entry(entry, &dev->msi_list, list) {
  49                 ret = arch_setup_msi_irq(dev, entry);
  50                 if (ret)
  51                         return ret;
  52         }
  53
  54         return 0;
  55 }
  56
  57 void __attribute__ ((weak)) arch_teardown_msi_irq(unsigned int irq)
  58 {
  59         return;
  60 }
  61
  62 void __attribute__ ((weak))
  63 arch_teardown_msi_irqs(struct pci_dev *dev)
  64 {
  65         struct msi_desc *entry;
  66
  67         list_for_each_entry(entry, &dev->msi_list, list) {
  68                 if (entry->irq != 0)
  69                         arch_teardown_msi_irq(entry->irq);
  70         }
  71 }
  72
  73 static void __msi_set_enable(struct pci_dev *dev, int pos, int enable)
  74 {
  75         u16 control;
  76
  77         if (pos) {
  78                 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
  79                 control &= ~PCI_MSI_FLAGS_ENABLE;
  80                 if (enable)
  81                         control |= PCI_MSI_FLAGS_ENABLE;
  82                 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
  83         }
  84 }
  85
  86 static void msi_set_enable(struct pci_dev *dev, int enable)
  87 {
  88         __msi_set_enable(dev, pci_find_capability(dev, PCI_CAP_ID_MSI), enable);
  89 }
  90
  91 static void msix_set_enable(struct pci_dev *dev, int enable)
  92 {
  93         int pos;
  94         u16 control;
  95
  96         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
  97         if (pos) {
  98                 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
  99                 control &= ~PCI_MSIX_FLAGS_ENABLE;
 100                 if (enable)
 101                         control |= PCI_MSIX_FLAGS_ENABLE;
 102                 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 103         }
 104 }
 105
 106 static inline __attribute_const__ u32 msi_mask(unsigned x)
 107 {
 108         /* Don't shift by >= width of type */
 109         if (x >= 5)
 110                 return 0xffffffff;
 111         return (1 << (1 << x)) - 1;
 112 }
 113
 114 static void msix_flush_writes(struct irq_desc *desc)
 115 {
 116         struct msi_desc *entry;
 117
 118         entry = get_irq_desc_msi(desc);
 119         BUG_ON(!entry || !entry->dev);
 120         switch (entry->msi_attrib.type) {
 121         case PCI_CAP_ID_MSI:
 122                 /* nothing to do */
 123                 break;
 124         case PCI_CAP_ID_MSIX:
 125         {
 126                 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
 127                         PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
 128                 readl(entry->mask_base + offset);
 129                 break;
 130         }
 131         default:
 132                 BUG();
 133                 break;
 134         }
 135 }
 136
 137 /*
 138  * PCI 2.3 does not specify mask bits for each MSI interrupt.  Attempting to
 139  * mask all MSI interrupts by clearing the MSI enable bit does not work
 140  * reliably as devices without an INTx disable bit will then generate a
 141  * level IRQ which will never be cleared.
 142  *
 143  * Returns 1 if it succeeded in masking the interrupt and 0 if the device
 144  * doesn't support MSI masking.
 145  */
 146 static int msi_set_mask_bits(struct irq_desc *desc, u32 mask, u32 flag)
 147 {
 148         struct msi_desc *entry;
 149
 150         entry = get_irq_desc_msi(desc);
 151         BUG_ON(!entry || !entry->dev);
 152         switch (entry->msi_attrib.type) {
 153         case PCI_CAP_ID_MSI:
 154                 if (entry->msi_attrib.maskbit) {
 155                         int pos;
 156                         u32 mask_bits;
 157
 158                         pos = (long)entry->mask_base;
 159                         pci_read_config_dword(entry->dev, pos, &mask_bits);
 160                         mask_bits &= ~(mask);
 161                         mask_bits |= flag & mask;
 162                         pci_write_config_dword(entry->dev, pos, mask_bits);
 163                 } else {
 164                         return 0;
 165                 }
 166                 break;
 167         case PCI_CAP_ID_MSIX:
 168         {
 169                 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
 170                         PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
 171                 writel(flag, entry->mask_base + offset);
 172                 readl(entry->mask_base + offset);
 173                 break;
 174         }
 175         default:
 176                 BUG();
 177                 break;
 178         }
 179         entry->msi_attrib.masked = !!flag;
 180         return 1;
 181 }
 182
 183 void read_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg)
 184 {
 185         struct msi_desc *entry = get_irq_desc_msi(desc);
 186         switch(entry->msi_attrib.type) {
 187         case PCI_CAP_ID_MSI:
 188         {
 189                 struct pci_dev *dev = entry->dev;
 190                 int pos = entry->msi_attrib.pos;
 191                 u16 data;
 192
 193                 pci_read_config_dword(dev, msi_lower_address_reg(pos),
 194                                         &msg->address_lo);
 195                 if (entry->msi_attrib.is_64) {
 196                         pci_read_config_dword(dev, msi_upper_address_reg(pos),
 197                                                 &msg->address_hi);
 198                         pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
 199                 } else {
 200                         msg->address_hi = 0;
 201                         pci_read_config_word(dev, msi_data_reg(pos, 0), &data);
 202                 }
 203                 msg->data = data;
 204                 break;
 205         }
 206         case PCI_CAP_ID_MSIX:
 207         {
 208                 void __iomem *base;
 209                 base = entry->mask_base +
 210                         entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
 211
 212                 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
 213                 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
 214                 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
 215                 break;
 216         }
 217         default:
 218                 BUG();
 219         }
 220 }
 221
 222 void read_msi_msg(unsigned int irq, struct msi_msg *msg)
 223 {
 224         struct irq_desc *desc = irq_to_desc(irq);
 225
 226         read_msi_msg_desc(desc, msg);
 227 }
 228
 229 void write_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg)
 230 {
 231         struct msi_desc *entry = get_irq_desc_msi(desc);
 232         switch (entry->msi_attrib.type) {
 233         case PCI_CAP_ID_MSI:
 234         {
 235                 struct pci_dev *dev = entry->dev;
 236                 int pos = entry->msi_attrib.pos;
 237
 238                 pci_write_config_dword(dev, msi_lower_address_reg(pos),
 239                                         msg->address_lo);
 240                 if (entry->msi_attrib.is_64) {
 241                         pci_write_config_dword(dev, msi_upper_address_reg(pos),
 242                                                 msg->address_hi);
 243                         pci_write_config_word(dev, msi_data_reg(pos, 1),
 244                                                 msg->data);
 245                 } else {
 246                         pci_write_config_word(dev, msi_data_reg(pos, 0),
 247                                                 msg->data);
 248                 }
 249                 break;
 250         }
 251         case PCI_CAP_ID_MSIX:
 252         {
 253                 void __iomem *base;
 254                 base = entry->mask_base +
 255                         entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
 256
 257                 writel(msg->address_lo,
 258                         base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
 259                 writel(msg->address_hi,
 260                         base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
 261                 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET);
 262                 break;
 263         }
 264         default:
 265                 BUG();
 266         }
 267         entry->msg = *msg;
 268 }
 269
 270 void write_msi_msg(unsigned int irq, struct msi_msg *msg)
 271 {
 272         struct irq_desc *desc = irq_to_desc(irq);
 273
 274         write_msi_msg_desc(desc, msg);
 275 }
 276
 277 void mask_msi_irq(unsigned int irq)
 278 {
 279         struct irq_desc *desc = irq_to_desc(irq);
 280
 281         msi_set_mask_bits(desc, 1, 1);
 282         msix_flush_writes(desc);
 283 }
 284
 285 void unmask_msi_irq(unsigned int irq)
 286 {
 287         struct irq_desc *desc = irq_to_desc(irq);
 288
 289         msi_set_mask_bits(desc, 1, 0);
 290         msix_flush_writes(desc);
 291 }
 292
 293 static int msi_free_irqs(struct pci_dev* dev);
 294
 295 static struct msi_desc* alloc_msi_entry(void)
 296 {
 297         struct msi_desc *entry;
 298
 299         entry = kzalloc(sizeof(struct msi_desc), GFP_KERNEL);
 300         if (!entry)
 301                 return NULL;
 302
 303         INIT_LIST_HEAD(&entry->list);
 304         entry->irq = 0;
 305         entry->dev = NULL;
 306
 307         return entry;
 308 }
 309
 310 static void pci_intx_for_msi(struct pci_dev *dev, int enable)
 311 {
 312         if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
 313                 pci_intx(dev, enable);
 314 }
 315
 316 static void __pci_restore_msi_state(struct pci_dev *dev)
 317 {
 318         int pos;
 319         u16 control;
 320         struct msi_desc *entry;
 321
 322         if (!dev->msi_enabled)
 323                 return;
 324
 325         entry = get_irq_msi(dev->irq);
 326         if (!entry) {
 327                 WARN_ON(1);
 328                 return;
 329         }
 330         pos = entry->msi_attrib.pos;
 331
 332         pci_intx_for_msi(dev, 0);
 333         msi_set_enable(dev, 0);
 334         write_msi_msg(dev->irq, &entry->msg);
 335         if (entry->msi_attrib.maskbit) {
 336                 struct irq_desc *desc = irq_to_desc(dev->irq);
 337                 msi_set_mask_bits(desc, entry->msi_attrib.maskbits_mask,
 338                                   entry->msi_attrib.masked);
 339         }
 340
 341         pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
 342         control &= ~PCI_MSI_FLAGS_QSIZE;
 343         control |= PCI_MSI_FLAGS_ENABLE;
 344         pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
 345 }
 346
 347 static void __pci_restore_msix_state(struct pci_dev *dev)
 348 {
 349         int pos;
 350         struct msi_desc *entry;
 351         u16 control;
 352
 353         if (!dev->msix_enabled)
 354                 return;
 355
 356         /* route the table */
 357         pci_intx_for_msi(dev, 0);
 358         msix_set_enable(dev, 0);
 359
 360         list_for_each_entry(entry, &dev->msi_list, list) {
 361                 struct irq_desc *desc = irq_to_desc(entry->irq);
 362                 write_msi_msg(entry->irq, &entry->msg);
 363                 msi_set_mask_bits(desc, 1, entry->msi_attrib.masked);
 364         }
 365
 366         BUG_ON(list_empty(&dev->msi_list));
 367         entry = list_entry(dev->msi_list.next, struct msi_desc, list);
 368         pos = entry->msi_attrib.pos;
 369         pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
 370         control &= ~PCI_MSIX_FLAGS_MASKALL;
 371         control |= PCI_MSIX_FLAGS_ENABLE;
 372         pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
 373 }
 374
 375 void pci_restore_msi_state(struct pci_dev *dev)
 376 {
 377         __pci_restore_msi_state(dev);
 378         __pci_restore_msix_state(dev);
 379 }
 380 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 381
 382 /**
 383  * msi_capability_init - configure device's MSI capability structure
 384  * @dev: pointer to the pci_dev data structure of MSI device function
 385  *
 386  * Setup the MSI capability structure of device function with a single
 387  * MSI irq, regardless of device function is capable of handling
 388  * multiple messages. A return of zero indicates the successful setup
 389  * of an entry zero with the new MSI irq or non-zero for otherwise.
 390  **/
 391 static int msi_capability_init(struct pci_dev *dev)
 392 {
 393         struct msi_desc *entry;
 394         int pos, ret;
 395         u16 control;
 396
 397         msi_set_enable(dev, 0); /* Ensure msi is disabled as I set it up */
 398
 399         pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
 400         pci_read_config_word(dev, msi_control_reg(pos), &control);
 401         /* MSI Entry Initialization */
 402         entry = alloc_msi_entry();
 403         if (!entry)
 404                 return -ENOMEM;
 405
 406         entry->msi_attrib.type = PCI_CAP_ID_MSI;
 407         entry->msi_attrib.is_64 = is_64bit_address(control);
 408         entry->msi_attrib.entry_nr = 0;
 409         entry->msi_attrib.maskbit = is_mask_bit_support(control);
 410         entry->msi_attrib.masked = 1;
 411         entry->msi_attrib.default_irq = dev->irq;       /* Save IOAPIC IRQ */
 412         entry->msi_attrib.pos = pos;
 413         entry->dev = dev;
 414         if (entry->msi_attrib.maskbit) {
 415                 unsigned int base, maskbits, temp;
 416
 417                 base = msi_mask_bits_reg(pos, entry->msi_attrib.is_64);
 418                 entry->mask_base = (void __iomem *)(long)base;
 419
 420                 /* All MSIs are unmasked by default, Mask them all */
 421                 pci_read_config_dword(dev, base, &maskbits);
 422                 temp = msi_mask((control & PCI_MSI_FLAGS_QMASK) >> 1);
 423                 maskbits |= temp;
 424                 pci_write_config_dword(dev, base, maskbits);
 425                 entry->msi_attrib.maskbits_mask = temp;
 426         }
 427         list_add_tail(&entry->list, &dev->msi_list);
 428
 429         /* Configure MSI capability structure */
 430         ret = arch_setup_msi_irqs(dev, 1, PCI_CAP_ID_MSI);
 431         if (ret) {
 432                 msi_free_irqs(dev);
 433                 return ret;
 434         }
 435
 436         /* Set MSI enabled bits  */
 437         pci_intx_for_msi(dev, 0);
 438         msi_set_enable(dev, 1);
 439         dev->msi_enabled = 1;
 440
 441         dev->irq = entry->irq;
 442         return 0;
 443 }
 444
 445 /**
 446  * msix_capability_init - configure device's MSI-X capability
 447  * @dev: pointer to the pci_dev data structure of MSI-X device function
 448  * @entries: pointer to an array of struct msix_entry entries
 449  * @nvec: number of @entries
 450  *
 451  * Setup the MSI-X capability structure of device function with a
 452  * single MSI-X irq. A return of zero indicates the successful setup of
 453  * requested MSI-X entries with allocated irqs or non-zero for otherwise.
 454  **/
 455 static int msix_capability_init(struct pci_dev *dev,
 456                                 struct msix_entry *entries, int nvec)
 457 {
 458         struct msi_desc *entry;
 459         int pos, i, j, nr_entries, ret;
 460         unsigned long phys_addr;
 461         u32 table_offset;
 462         u16 control;
 463         u8 bir;
 464         void __iomem *base;
 465
 466         msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
 467
 468         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
 469         /* Request & Map MSI-X table region */
 470         pci_read_config_word(dev, msi_control_reg(pos), &control);
 471         nr_entries = multi_msix_capable(control);
 472
 473         pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
 474         bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
 475         table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
 476         phys_addr = pci_resource_start (dev, bir) + table_offset;
 477         base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
 478         if (base == NULL)
 479                 return -ENOMEM;
 480
 481         /* MSI-X Table Initialization */
 482         for (i = 0; i < nvec; i++) {
 483                 entry = alloc_msi_entry();
 484                 if (!entry)
 485                         break;
 486
 487                 j = entries[i].entry;
 488                 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
 489                 entry->msi_attrib.is_64 = 1;
 490                 entry->msi_attrib.entry_nr = j;
 491                 entry->msi_attrib.maskbit = 1;
 492                 entry->msi_attrib.masked = 1;
 493                 entry->msi_attrib.default_irq = dev->irq;
 494                 entry->msi_attrib.pos = pos;
 495                 entry->dev = dev;
 496                 entry->mask_base = base;
 497
 498                 list_add_tail(&entry->list, &dev->msi_list);
 499         }
 500
 501         ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
 502         if (ret) {
 503                 int avail = 0;
 504                 list_for_each_entry(entry, &dev->msi_list, list) {
 505                         if (entry->irq != 0) {
 506                                 avail++;
 507                         }
 508                 }
 509
 510                 msi_free_irqs(dev);
 511
 512                 /* If we had some success report the number of irqs
 513                  * we succeeded in setting up.
 514                  */
 515                 if (avail == 0)
 516                         avail = ret;
 517                 return avail;
 518         }
 519
 520         i = 0;
 521         list_for_each_entry(entry, &dev->msi_list, list) {
 522                 entries[i].vector = entry->irq;
 523                 set_irq_msi(entry->irq, entry);
 524                 i++;
 525         }
 526         /* Set MSI-X enabled bits */
 527         pci_intx_for_msi(dev, 0);
 528         msix_set_enable(dev, 1);
 529         dev->msix_enabled = 1;
 530
 531         return 0;
 532 }
 533
 534 /**
 535  * pci_msi_check_device - check whether MSI may be enabled on a device
 536  * @dev: pointer to the pci_dev data structure of MSI device function
 537  * @nvec: how many MSIs have been requested ?
 538  * @type: are we checking for MSI or MSI-X ?
 539  *
 540  * Look at global flags, the device itself, and its parent busses
 541  * to determine if MSI/-X are supported for the device. If MSI/-X is
 542  * supported return 0, else return an error code.
 543  **/
 544 static int pci_msi_check_device(struct pci_dev* dev, int nvec, int type)
 545 {
 546         struct pci_bus *bus;
 547         int ret;
 548
 549         /* MSI must be globally enabled and supported by the device */
 550         if (!pci_msi_enable || !dev || dev->no_msi)
 551                 return -EINVAL;
 552
 553         /*
 554          * You can't ask to have 0 or less MSIs configured.
 555          *  a) it's stupid ..
 556          *  b) the list manipulation code assumes nvec >= 1.
 557          */
 558         if (nvec < 1)
 559                 return -ERANGE;
 560
 561         /* Any bridge which does NOT route MSI transactions from it's
 562          * secondary bus to it's primary bus must set NO_MSI flag on
 563          * the secondary pci_bus.
 564          * We expect only arch-specific PCI host bus controller driver
 565          * or quirks for specific PCI bridges to be setting NO_MSI.
 566          */
 567         for (bus = dev->bus; bus; bus = bus->parent)
 568                 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
 569                         return -EINVAL;
 570
 571         ret = arch_msi_check_device(dev, nvec, type);
 572         if (ret)
 573                 return ret;
 574
 575         if (!pci_find_capability(dev, type))
 576                 return -EINVAL;
 577
 578         return 0;
 579 }
 580
 581 /**
 582  * pci_enable_msi - configure device's MSI capability structure
 583  * @dev: pointer to the pci_dev data structure of MSI device function
 584  *
 585  * Setup the MSI capability structure of device function with
 586  * a single MSI irq upon its software driver call to request for
 587  * MSI mode enabled on its hardware device function. A return of zero
 588  * indicates the successful setup of an entry zero with the new MSI
 589  * irq or non-zero for otherwise.
 590  **/
 591 int pci_enable_msi(struct pci_dev* dev)
 592 {
 593         int status;
 594
 595         status = pci_msi_check_device(dev, 1, PCI_CAP_ID_MSI);
 596         if (status)
 597                 return status;
 598
 599         WARN_ON(!!dev->msi_enabled);
 600
 601         /* Check whether driver already requested for MSI-X irqs */
 602         if (dev->msix_enabled) {
 603                 dev_info(&dev->dev, "can't enable MSI "
 604                          "(MSI-X already enabled)\n");
 605                 return -EINVAL;
 606         }
 607         status = msi_capability_init(dev);
 608         return status;
 609 }
 610 EXPORT_SYMBOL(pci_enable_msi);
 611
 612 void pci_msi_shutdown(struct pci_dev* dev)
 613 {
 614         struct msi_desc *entry;
 615
 616         if (!pci_msi_enable || !dev || !dev->msi_enabled)
 617                 return;
 618
 619         msi_set_enable(dev, 0);
 620         pci_intx_for_msi(dev, 1);
 621         dev->msi_enabled = 0;
 622
 623         BUG_ON(list_empty(&dev->msi_list));
 624         entry = list_entry(dev->msi_list.next, struct msi_desc, list);
 625         /* Return the the pci reset with msi irqs unmasked */
 626         if (entry->msi_attrib.maskbit) {
 627                 u32 mask = entry->msi_attrib.maskbits_mask;
 628                 struct irq_desc *desc = irq_to_desc(dev->irq);
 629                 msi_set_mask_bits(desc, mask, ~mask);
 630         }
 631         if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI)
 632                 return;
 633
 634         /* Restore dev->irq to its default pin-assertion irq */
 635         dev->irq = entry->msi_attrib.default_irq;
 636 }
 637 void pci_disable_msi(struct pci_dev* dev)
 638 {
 639         struct msi_desc *entry;
 640
 641         if (!pci_msi_enable || !dev || !dev->msi_enabled)
 642                 return;
 643
 644         pci_msi_shutdown(dev);
 645
 646         entry = list_entry(dev->msi_list.next, struct msi_desc, list);
 647         if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI)
 648                 return;
 649
 650         msi_free_irqs(dev);
 651 }
 652 EXPORT_SYMBOL(pci_disable_msi);
 653
 654 static int msi_free_irqs(struct pci_dev* dev)
 655 {
 656         struct msi_desc *entry, *tmp;
 657
 658         list_for_each_entry(entry, &dev->msi_list, list) {
 659                 if (entry->irq)
 660                         BUG_ON(irq_has_action(entry->irq));
 661         }
 662
 663         arch_teardown_msi_irqs(dev);
 664
 665         list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) {
 666                 if (entry->msi_attrib.type == PCI_CAP_ID_MSIX) {
 667                         writel(1, entry->mask_base + entry->msi_attrib.entry_nr
 668                                   * PCI_MSIX_ENTRY_SIZE
 669                                   + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
 670
 671                         if (list_is_last(&entry->list, &dev->msi_list))
 672                                 iounmap(entry->mask_base);
 673                 }
 674                 list_del(&entry->list);
 675                 kfree(entry);
 676         }
 677
 678         return 0;
 679 }
 680
 681 /**
 682  * pci_enable_msix - configure device's MSI-X capability structure
 683  * @dev: pointer to the pci_dev data structure of MSI-X device function
 684  * @entries: pointer to an array of MSI-X entries
 685  * @nvec: number of MSI-X irqs requested for allocation by device driver
 686  *
 687  * Setup the MSI-X capability structure of device function with the number
 688  * of requested irqs upon its software driver call to request for
 689  * MSI-X mode enabled on its hardware device function. A return of zero
 690  * indicates the successful configuration of MSI-X capability structure
 691  * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
 692  * Or a return of > 0 indicates that driver request is exceeding the number
 693  * of irqs available. Driver should use the returned value to re-send
 694  * its request.
 695  **/
 696 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
 697 {
 698         int status, pos, nr_entries;
 699         int i, j;
 700         u16 control;
 701
 702         if (!entries)
 703                 return -EINVAL;
 704
 705         status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX);
 706         if (status)
 707                 return status;
 708
 709         pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
 710         pci_read_config_word(dev, msi_control_reg(pos), &control);
 711         nr_entries = multi_msix_capable(control);
 712         if (nvec > nr_entries)
 713                 return -EINVAL;
 714
 715         /* Check for any invalid entries */
 716         for (i = 0; i < nvec; i++) {
 717                 if (entries[i].entry >= nr_entries)
 718                         return -EINVAL;         /* invalid entry */
 719                 for (j = i + 1; j < nvec; j++) {
 720                         if (entries[i].entry == entries[j].entry)
 721                                 return -EINVAL; /* duplicate entry */
 722                 }
 723         }
 724         WARN_ON(!!dev->msix_enabled);
 725
 726         /* Check whether driver already requested for MSI irq */
 727         if (dev->msi_enabled) {
 728                 dev_info(&dev->dev, "can't enable MSI-X "
 729                        "(MSI IRQ already assigned)\n");
 730                 return -EINVAL;
 731         }
 732         status = msix_capability_init(dev, entries, nvec);
 733         return status;
 734 }
 735 EXPORT_SYMBOL(pci_enable_msix);
 736
 737 static void msix_free_all_irqs(struct pci_dev *dev)
 738 {
 739         msi_free_irqs(dev);
 740 }
 741
 742 void pci_msix_shutdown(struct pci_dev* dev)
 743 {
 744         if (!pci_msi_enable || !dev || !dev->msix_enabled)
 745                 return;
 746
 747         msix_set_enable(dev, 0);
 748         pci_intx_for_msi(dev, 1);
 749         dev->msix_enabled = 0;
 750 }
 751 void pci_disable_msix(struct pci_dev* dev)
 752 {
 753         if (!pci_msi_enable || !dev || !dev->msix_enabled)
 754                 return;
 755
 756         pci_msix_shutdown(dev);
 757
 758         msix_free_all_irqs(dev);
 759 }
 760 EXPORT_SYMBOL(pci_disable_msix);
 761
 762 /**
 763  * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state
 764  * @dev: pointer to the pci_dev data structure of MSI(X) device function
 765  *
 766  * Being called during hotplug remove, from which the device function
 767  * is hot-removed. All previous assigned MSI/MSI-X irqs, if
 768  * allocated for this device function, are reclaimed to unused state,
 769  * which may be used later on.
 770  **/
 771 void msi_remove_pci_irq_vectors(struct pci_dev* dev)
 772 {
 773         if (!pci_msi_enable || !dev)
 774                 return;
 775
 776         if (dev->msi_enabled)
 777                 msi_free_irqs(dev);
 778
 779         if (dev->msix_enabled)
 780                 msix_free_all_irqs(dev);
 781 }
 782
 783 void pci_no_msi(void)
 784 {
 785         pci_msi_enable = 0;
 786 }
 787
 788 /**
 789  * pci_msi_enabled - is MSI enabled?
 790  *
 791  * Returns true if MSI has not been disabled by the command-line option
 792  * pci=nomsi.
 793  **/
 794 int pci_msi_enabled(void)
 795 {
 796         return pci_msi_enable;
 797 }
 798 EXPORT_SYMBOL(pci_msi_enabled);
 799
 800 void pci_msi_init_pci_dev(struct pci_dev *dev)
 801 {
 802         INIT_LIST_HEAD(&dev->msi_list);
 803 }