?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[MIPS] ioc3.h: Uses u8, so include <linux/types.h>.
[linux-2.6.22.y-op.git] / drivers / pci / msi.c
blob7f8429284fabe0663e4c6dc30fc6deefeaa178e3
1 /*
2 * File: msi.c
3 * Purpose: PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
8
9 #include <linux/mm.h>
10 #include <linux/irq.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/config.h>
14 #include <linux/ioport.h>
15 #include <linux/smp_lock.h>
16 #include <linux/pci.h>
17 #include <linux/proc_fs.h>
18
19 #include <asm/errno.h>
20 #include <asm/io.h>
21 #include <asm/smp.h>
22
23 #include "pci.h"
24 #include "msi.h"
25
26 static DEFINE_SPINLOCK(msi_lock);
27 static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL };
28 static kmem_cache_t* msi_cachep;
29
30 static int pci_msi_enable = 1;
31 static int last_alloc_vector;
32 static int nr_released_vectors;
33 static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS;
34 static int nr_msix_devices;
35
36 #ifndef CONFIG_X86_IO_APIC
37 int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
38 #endif
39
40 static struct msi_ops *msi_ops;
41
42 int
43 msi_register(struct msi_ops *ops)
44 {
45 msi_ops = ops;
46 return 0;
47 }
48
49 static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags)
50 {
51 memset(p, 0, NR_IRQS * sizeof(struct msi_desc));
52 }
53
54 static int msi_cache_init(void)
55 {
56 msi_cachep = kmem_cache_create("msi_cache",
57 NR_IRQS * sizeof(struct msi_desc),
58 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL);
59 if (!msi_cachep)
60 return -ENOMEM;
61
62 return 0;
63 }
64
65 static void msi_set_mask_bit(unsigned int vector, int flag)
66 {
67 struct msi_desc *entry;
68
69 entry = (struct msi_desc *)msi_desc[vector];
70 if (!entry || !entry->dev || !entry->mask_base)
71 return;
72 switch (entry->msi_attrib.type) {
73 case PCI_CAP_ID_MSI:
74 {
75 int pos;
76 u32 mask_bits;
77
78 pos = (long)entry->mask_base;
79 pci_read_config_dword(entry->dev, pos, &mask_bits);
80 mask_bits &= ~(1);
81 mask_bits |= flag;
82 pci_write_config_dword(entry->dev, pos, mask_bits);
83 break;
84 }
85 case PCI_CAP_ID_MSIX:
86 {
87 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
88 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
89 writel(flag, entry->mask_base + offset);
90 break;
91 }
92 default:
93 break;
94 }
95 }
96
97 #ifdef CONFIG_SMP
98 static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
99 {
100 struct msi_desc *entry;
101 u32 address_hi, address_lo;
102 unsigned int irq = vector;
103 unsigned int dest_cpu = first_cpu(cpu_mask);
104
105 entry = (struct msi_desc *)msi_desc[vector];
106 if (!entry || !entry->dev)
107 return;
108
109 switch (entry->msi_attrib.type) {
110 case PCI_CAP_ID_MSI:
111 {
112 int pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI);
113
114 if (!pos)
115 return;
116
117 pci_read_config_dword(entry->dev, msi_upper_address_reg(pos),
118 &address_hi);
119 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos),
120 &address_lo);
121
122 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
123
124 pci_write_config_dword(entry->dev, msi_upper_address_reg(pos),
125 address_hi);
126 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos),
127 address_lo);
128 set_native_irq_info(irq, cpu_mask);
129 break;
130 }
131 case PCI_CAP_ID_MSIX:
132 {
133 int offset_hi =
134 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
135 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET;
136 int offset_lo =
137 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
138 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET;
139
140 address_hi = readl(entry->mask_base + offset_hi);
141 address_lo = readl(entry->mask_base + offset_lo);
142
143 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
144
145 writel(address_hi, entry->mask_base + offset_hi);
146 writel(address_lo, entry->mask_base + offset_lo);
147 set_native_irq_info(irq, cpu_mask);
148 break;
149 }
150 default:
151 break;
152 }
153 }
154 #else
155 #define set_msi_affinity NULL
156 #endif /* CONFIG_SMP */
157
158 static void mask_MSI_irq(unsigned int vector)
159 {
160 msi_set_mask_bit(vector, 1);
161 }
162
163 static void unmask_MSI_irq(unsigned int vector)
164 {
165 msi_set_mask_bit(vector, 0);
166 }
167
168 static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
169 {
170 struct msi_desc *entry;
171 unsigned long flags;
172
173 spin_lock_irqsave(&msi_lock, flags);
174 entry = msi_desc[vector];
175 if (!entry || !entry->dev) {
176 spin_unlock_irqrestore(&msi_lock, flags);
177 return 0;
178 }
179 entry->msi_attrib.state = 1; /* Mark it active */
180 spin_unlock_irqrestore(&msi_lock, flags);
181
182 return 0; /* never anything pending */
183 }
184
185 static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
186 {
187 startup_msi_irq_wo_maskbit(vector);
188 unmask_MSI_irq(vector);
189 return 0; /* never anything pending */
190 }
191
192 static void shutdown_msi_irq(unsigned int vector)
193 {
194 struct msi_desc *entry;
195 unsigned long flags;
196
197 spin_lock_irqsave(&msi_lock, flags);
198 entry = msi_desc[vector];
199 if (entry && entry->dev)
200 entry->msi_attrib.state = 0; /* Mark it not active */
201 spin_unlock_irqrestore(&msi_lock, flags);
202 }
203
204 static void end_msi_irq_wo_maskbit(unsigned int vector)
205 {
206 move_native_irq(vector);
207 ack_APIC_irq();
208 }
209
210 static void end_msi_irq_w_maskbit(unsigned int vector)
211 {
212 move_native_irq(vector);
213 unmask_MSI_irq(vector);
214 ack_APIC_irq();
215 }
216
217 static void do_nothing(unsigned int vector)
218 {
219 }
220
221 /*
222 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices,
223 * which implement the MSI-X Capability Structure.
224 */
225 static struct hw_interrupt_type msix_irq_type = {
226 .typename = "PCI-MSI-X",
227 .startup = startup_msi_irq_w_maskbit,
228 .shutdown = shutdown_msi_irq,
229 .enable = unmask_MSI_irq,
230 .disable = mask_MSI_irq,
231 .ack = mask_MSI_irq,
232 .end = end_msi_irq_w_maskbit,
233 .set_affinity = set_msi_affinity
234 };
235
236 /*
237 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
238 * which implement the MSI Capability Structure with
239 * Mask-and-Pending Bits.
240 */
241 static struct hw_interrupt_type msi_irq_w_maskbit_type = {
242 .typename = "PCI-MSI",
243 .startup = startup_msi_irq_w_maskbit,
244 .shutdown = shutdown_msi_irq,
245 .enable = unmask_MSI_irq,
246 .disable = mask_MSI_irq,
247 .ack = mask_MSI_irq,
248 .end = end_msi_irq_w_maskbit,
249 .set_affinity = set_msi_affinity
250 };
251
252 /*
253 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
254 * which implement the MSI Capability Structure without
255 * Mask-and-Pending Bits.
256 */
257 static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
258 .typename = "PCI-MSI",
259 .startup = startup_msi_irq_wo_maskbit,
260 .shutdown = shutdown_msi_irq,
261 .enable = do_nothing,
262 .disable = do_nothing,
263 .ack = do_nothing,
264 .end = end_msi_irq_wo_maskbit,
265 .set_affinity = set_msi_affinity
266 };
267
268 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
269 static int assign_msi_vector(void)
270 {
271 static int new_vector_avail = 1;
272 int vector;
273 unsigned long flags;
274
275 /*
276 * msi_lock is provided to ensure that successful allocation of MSI
277 * vector is assigned unique among drivers.
278 */
279 spin_lock_irqsave(&msi_lock, flags);
280
281 if (!new_vector_avail) {
282 int free_vector = 0;
283
284 /*
285 * vector_irq[] = -1 indicates that this specific vector is:
286 * - assigned for MSI (since MSI have no associated IRQ) or
287 * - assigned for legacy if less than 16, or
288 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
289 * vector_irq[] = 0 indicates that this vector, previously
290 * assigned for MSI, is freed by hotplug removed operations.
291 * This vector will be reused for any subsequent hotplug added
292 * operations.
293 * vector_irq[] > 0 indicates that this vector is assigned for
294 * IOxAPIC IRQs. This vector and its value provides a 1-to-1
295 * vector-to-IOxAPIC IRQ mapping.
296 */
297 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
298 if (vector_irq[vector] != 0)
299 continue;
300 free_vector = vector;
301 if (!msi_desc[vector])
302 break;
303 else
304 continue;
305 }
306 if (!free_vector) {
307 spin_unlock_irqrestore(&msi_lock, flags);
308 return -EBUSY;
309 }
310 vector_irq[free_vector] = -1;
311 nr_released_vectors--;
312 spin_unlock_irqrestore(&msi_lock, flags);
313 if (msi_desc[free_vector] != NULL) {
314 struct pci_dev *dev;
315 int tail;
316
317 /* free all linked vectors before re-assign */
318 do {
319 spin_lock_irqsave(&msi_lock, flags);
320 dev = msi_desc[free_vector]->dev;
321 tail = msi_desc[free_vector]->link.tail;
322 spin_unlock_irqrestore(&msi_lock, flags);
323 msi_free_vector(dev, tail, 1);
324 } while (free_vector != tail);
325 }
326
327 return free_vector;
328 }
329 vector = assign_irq_vector(AUTO_ASSIGN);
330 last_alloc_vector = vector;
331 if (vector == LAST_DEVICE_VECTOR)
332 new_vector_avail = 0;
333
334 spin_unlock_irqrestore(&msi_lock, flags);
335 return vector;
336 }
337
338 static int get_new_vector(void)
339 {
340 int vector = assign_msi_vector();
341
342 if (vector > 0)
343 set_intr_gate(vector, interrupt[vector]);
344
345 return vector;
346 }
347
348 static int msi_init(void)
349 {
350 static int status = -ENOMEM;
351
352 if (!status)
353 return status;
354
355 if (pci_msi_quirk) {
356 pci_msi_enable = 0;
357 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
358 status = -EINVAL;
359 return status;
360 }
361
362 status = msi_arch_init();
363 if (status < 0) {
364 pci_msi_enable = 0;
365 printk(KERN_WARNING
366 "PCI: MSI arch init failed. MSI disabled.\n");
367 return status;
368 }
369
370 if (! msi_ops) {
371 printk(KERN_WARNING
372 "PCI: MSI ops not registered. MSI disabled.\n");
373 status = -EINVAL;
374 return status;
375 }
376
377 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
378 status = msi_cache_init();
379 if (status < 0) {
380 pci_msi_enable = 0;
381 printk(KERN_WARNING "PCI: MSI cache init failed\n");
382 return status;
383 }
384
385 if (last_alloc_vector < 0) {
386 pci_msi_enable = 0;
387 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
388 status = -EBUSY;
389 return status;
390 }
391 vector_irq[last_alloc_vector] = 0;
392 nr_released_vectors++;
393
394 return status;
395 }
396
397 static int get_msi_vector(struct pci_dev *dev)
398 {
399 return get_new_vector();
400 }
401
402 static struct msi_desc* alloc_msi_entry(void)
403 {
404 struct msi_desc *entry;
405
406 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL);
407 if (!entry)
408 return NULL;
409
410 memset(entry, 0, sizeof(struct msi_desc));
411 entry->link.tail = entry->link.head = 0; /* single message */
412 entry->dev = NULL;
413
414 return entry;
415 }
416
417 static void attach_msi_entry(struct msi_desc *entry, int vector)
418 {
419 unsigned long flags;
420
421 spin_lock_irqsave(&msi_lock, flags);
422 msi_desc[vector] = entry;
423 spin_unlock_irqrestore(&msi_lock, flags);
424 }
425
426 static void irq_handler_init(int cap_id, int pos, int mask)
427 {
428 unsigned long flags;
429
430 spin_lock_irqsave(&irq_desc[pos].lock, flags);
431 if (cap_id == PCI_CAP_ID_MSIX)
432 irq_desc[pos].handler = &msix_irq_type;
433 else {
434 if (!mask)
435 irq_desc[pos].handler = &msi_irq_wo_maskbit_type;
436 else
437 irq_desc[pos].handler = &msi_irq_w_maskbit_type;
438 }
439 spin_unlock_irqrestore(&irq_desc[pos].lock, flags);
440 }
441
442 static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
443 {
444 u16 control;
445
446 pci_read_config_word(dev, msi_control_reg(pos), &control);
447 if (type == PCI_CAP_ID_MSI) {
448 /* Set enabled bits to single MSI & enable MSI_enable bit */
449 msi_enable(control, 1);
450 pci_write_config_word(dev, msi_control_reg(pos), control);
451 dev->msi_enabled = 1;
452 } else {
453 msix_enable(control);
454 pci_write_config_word(dev, msi_control_reg(pos), control);
455 dev->msix_enabled = 1;
456 }
457 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
458 /* PCI Express Endpoint device detected */
459 pci_intx(dev, 0); /* disable intx */
460 }
461 }
462
463 void disable_msi_mode(struct pci_dev *dev, int pos, int type)
464 {
465 u16 control;
466
467 pci_read_config_word(dev, msi_control_reg(pos), &control);
468 if (type == PCI_CAP_ID_MSI) {
469 /* Set enabled bits to single MSI & enable MSI_enable bit */
470 msi_disable(control);
471 pci_write_config_word(dev, msi_control_reg(pos), control);
472 dev->msi_enabled = 0;
473 } else {
474 msix_disable(control);
475 pci_write_config_word(dev, msi_control_reg(pos), control);
476 dev->msix_enabled = 0;
477 }
478 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
479 /* PCI Express Endpoint device detected */
480 pci_intx(dev, 1); /* enable intx */
481 }
482 }
483
484 static int msi_lookup_vector(struct pci_dev *dev, int type)
485 {
486 int vector;
487 unsigned long flags;
488
489 spin_lock_irqsave(&msi_lock, flags);
490 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
491 if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
492 msi_desc[vector]->msi_attrib.type != type ||
493 msi_desc[vector]->msi_attrib.default_vector != dev->irq)
494 continue;
495 spin_unlock_irqrestore(&msi_lock, flags);
496 /* This pre-assigned MSI vector for this device
497 already exits. Override dev->irq with this vector */
498 dev->irq = vector;
499 return 0;
500 }
501 spin_unlock_irqrestore(&msi_lock, flags);
502
503 return -EACCES;
504 }
505
506 void pci_scan_msi_device(struct pci_dev *dev)
507 {
508 if (!dev)
509 return;
510
511 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
512 nr_msix_devices++;
513 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
514 nr_reserved_vectors++;
515 }
516
517 #ifdef CONFIG_PM
518 int pci_save_msi_state(struct pci_dev *dev)
519 {
520 int pos, i = 0;
521 u16 control;
522 struct pci_cap_saved_state *save_state;
523 u32 *cap;
524
525 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
526 if (pos <= 0 || dev->no_msi)
527 return 0;
528
529 pci_read_config_word(dev, msi_control_reg(pos), &control);
530 if (!(control & PCI_MSI_FLAGS_ENABLE))
531 return 0;
532
533 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5,
534 GFP_KERNEL);
535 if (!save_state) {
536 printk(KERN_ERR "Out of memory in pci_save_msi_state\n");
537 return -ENOMEM;
538 }
539 cap = &save_state->data[0];
540
541 pci_read_config_dword(dev, pos, &cap[i++]);
542 control = cap[0] >> 16;
543 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]);
544 if (control & PCI_MSI_FLAGS_64BIT) {
545 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]);
546 pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]);
547 } else
548 pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]);
549 if (control & PCI_MSI_FLAGS_MASKBIT)
550 pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]);
551 save_state->cap_nr = PCI_CAP_ID_MSI;
552 pci_add_saved_cap(dev, save_state);
553 return 0;
554 }
555
556 void pci_restore_msi_state(struct pci_dev *dev)
557 {
558 int i = 0, pos;
559 u16 control;
560 struct pci_cap_saved_state *save_state;
561 u32 *cap;
562
563 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI);
564 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
565 if (!save_state || pos <= 0)
566 return;
567 cap = &save_state->data[0];
568
569 control = cap[i++] >> 16;
570 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]);
571 if (control & PCI_MSI_FLAGS_64BIT) {
572 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]);
573 pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]);
574 } else
575 pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]);
576 if (control & PCI_MSI_FLAGS_MASKBIT)
577 pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]);
578 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
579 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
580 pci_remove_saved_cap(save_state);
581 kfree(save_state);
582 }
583
584 int pci_save_msix_state(struct pci_dev *dev)
585 {
586 int pos;
587 int temp;
588 int vector, head, tail = 0;
589 u16 control;
590 struct pci_cap_saved_state *save_state;
591
592 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
593 if (pos <= 0 || dev->no_msi)
594 return 0;
595
596 /* save the capability */
597 pci_read_config_word(dev, msi_control_reg(pos), &control);
598 if (!(control & PCI_MSIX_FLAGS_ENABLE))
599 return 0;
600 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16),
601 GFP_KERNEL);
602 if (!save_state) {
603 printk(KERN_ERR "Out of memory in pci_save_msix_state\n");
604 return -ENOMEM;
605 }
606 *((u16 *)&save_state->data[0]) = control;
607
608 /* save the table */
609 temp = dev->irq;
610 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
611 kfree(save_state);
612 return -EINVAL;
613 }
614
615 vector = head = dev->irq;
616 while (head != tail) {
617 int j;
618 void __iomem *base;
619 struct msi_desc *entry;
620
621 entry = msi_desc[vector];
622 base = entry->mask_base;
623 j = entry->msi_attrib.entry_nr;
624
625 entry->address_lo_save =
626 readl(base + j * PCI_MSIX_ENTRY_SIZE +
627 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
628 entry->address_hi_save =
629 readl(base + j * PCI_MSIX_ENTRY_SIZE +
630 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
631 entry->data_save =
632 readl(base + j * PCI_MSIX_ENTRY_SIZE +
633 PCI_MSIX_ENTRY_DATA_OFFSET);
634
635 tail = msi_desc[vector]->link.tail;
636 vector = tail;
637 }
638 dev->irq = temp;
639
640 save_state->cap_nr = PCI_CAP_ID_MSIX;
641 pci_add_saved_cap(dev, save_state);
642 return 0;
643 }
644
645 void pci_restore_msix_state(struct pci_dev *dev)
646 {
647 u16 save;
648 int pos;
649 int vector, head, tail = 0;
650 void __iomem *base;
651 int j;
652 struct msi_desc *entry;
653 int temp;
654 struct pci_cap_saved_state *save_state;
655
656 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX);
657 if (!save_state)
658 return;
659 save = *((u16 *)&save_state->data[0]);
660 pci_remove_saved_cap(save_state);
661 kfree(save_state);
662
663 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
664 if (pos <= 0)
665 return;
666
667 /* route the table */
668 temp = dev->irq;
669 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX))
670 return;
671 vector = head = dev->irq;
672 while (head != tail) {
673 entry = msi_desc[vector];
674 base = entry->mask_base;
675 j = entry->msi_attrib.entry_nr;
676
677 writel(entry->address_lo_save,
678 base + j * PCI_MSIX_ENTRY_SIZE +
679 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
680 writel(entry->address_hi_save,
681 base + j * PCI_MSIX_ENTRY_SIZE +
682 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
683 writel(entry->data_save,
684 base + j * PCI_MSIX_ENTRY_SIZE +
685 PCI_MSIX_ENTRY_DATA_OFFSET);
686
687 tail = msi_desc[vector]->link.tail;
688 vector = tail;
689 }
690 dev->irq = temp;
691
692 pci_write_config_word(dev, msi_control_reg(pos), save);
693 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
694 }
695 #endif
696
697 static int msi_register_init(struct pci_dev *dev, struct msi_desc *entry)
698 {
699 int status;
700 u32 address_hi;
701 u32 address_lo;
702 u32 data;
703 int pos, vector = dev->irq;
704 u16 control;
705
706 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
707 pci_read_config_word(dev, msi_control_reg(pos), &control);
708
709 /* Configure MSI capability structure */
710 status = msi_ops->setup(dev, vector, &address_hi, &address_lo, &data);
711 if (status < 0)
712 return status;
713
714 pci_write_config_dword(dev, msi_lower_address_reg(pos), address_lo);
715 if (is_64bit_address(control)) {
716 pci_write_config_dword(dev,
717 msi_upper_address_reg(pos), address_hi);
718 pci_write_config_word(dev,
719 msi_data_reg(pos, 1), data);
720 } else
721 pci_write_config_word(dev,
722 msi_data_reg(pos, 0), data);
723 if (entry->msi_attrib.maskbit) {
724 unsigned int maskbits, temp;
725 /* All MSIs are unmasked by default, Mask them all */
726 pci_read_config_dword(dev,
727 msi_mask_bits_reg(pos, is_64bit_address(control)),
728 &maskbits);
729 temp = (1 << multi_msi_capable(control));
730 temp = ((temp - 1) & ~temp);
731 maskbits |= temp;
732 pci_write_config_dword(dev,
733 msi_mask_bits_reg(pos, is_64bit_address(control)),
734 maskbits);
735 }
736
737 return 0;
738 }
739
740 /**
741 * msi_capability_init - configure device's MSI capability structure
742 * @dev: pointer to the pci_dev data structure of MSI device function
743 *
744 * Setup the MSI capability structure of device function with a single
745 * MSI vector, regardless of device function is capable of handling
746 * multiple messages. A return of zero indicates the successful setup
747 * of an entry zero with the new MSI vector or non-zero for otherwise.
748 **/
749 static int msi_capability_init(struct pci_dev *dev)
750 {
751 int status;
752 struct msi_desc *entry;
753 int pos, vector;
754 u16 control;
755
756 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
757 pci_read_config_word(dev, msi_control_reg(pos), &control);
758 /* MSI Entry Initialization */
759 entry = alloc_msi_entry();
760 if (!entry)
761 return -ENOMEM;
762
763 vector = get_msi_vector(dev);
764 if (vector < 0) {
765 kmem_cache_free(msi_cachep, entry);
766 return -EBUSY;
767 }
768 entry->link.head = vector;
769 entry->link.tail = vector;
770 entry->msi_attrib.type = PCI_CAP_ID_MSI;
771 entry->msi_attrib.state = 0; /* Mark it not active */
772 entry->msi_attrib.entry_nr = 0;
773 entry->msi_attrib.maskbit = is_mask_bit_support(control);
774 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */
775 dev->irq = vector;
776 entry->dev = dev;
777 if (is_mask_bit_support(control)) {
778 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
779 is_64bit_address(control));
780 }
781 /* Replace with MSI handler */
782 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
783 /* Configure MSI capability structure */
784 status = msi_register_init(dev, entry);
785 if (status != 0) {
786 dev->irq = entry->msi_attrib.default_vector;
787 kmem_cache_free(msi_cachep, entry);
788 return status;
789 }
790
791 attach_msi_entry(entry, vector);
792 /* Set MSI enabled bits */
793 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
794
795 return 0;
796 }
797
798 /**
799 * msix_capability_init - configure device's MSI-X capability
800 * @dev: pointer to the pci_dev data structure of MSI-X device function
801 * @entries: pointer to an array of struct msix_entry entries
802 * @nvec: number of @entries
803 *
804 * Setup the MSI-X capability structure of device function with a
805 * single MSI-X vector. A return of zero indicates the successful setup of
806 * requested MSI-X entries with allocated vectors or non-zero for otherwise.
807 **/
808 static int msix_capability_init(struct pci_dev *dev,
809 struct msix_entry *entries, int nvec)
810 {
811 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
812 u32 address_hi;
813 u32 address_lo;
814 u32 data;
815 int status;
816 int vector, pos, i, j, nr_entries, temp = 0;
817 unsigned long phys_addr;
818 u32 table_offset;
819 u16 control;
820 u8 bir;
821 void __iomem *base;
822
823 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
824 /* Request & Map MSI-X table region */
825 pci_read_config_word(dev, msi_control_reg(pos), &control);
826 nr_entries = multi_msix_capable(control);
827
828 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
829 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
830 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
831 phys_addr = pci_resource_start (dev, bir) + table_offset;
832 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
833 if (base == NULL)
834 return -ENOMEM;
835
836 /* MSI-X Table Initialization */
837 for (i = 0; i < nvec; i++) {
838 entry = alloc_msi_entry();
839 if (!entry)
840 break;
841 vector = get_msi_vector(dev);
842 if (vector < 0) {
843 kmem_cache_free(msi_cachep, entry);
844 break;
845 }
846
847 j = entries[i].entry;
848 entries[i].vector = vector;
849 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
850 entry->msi_attrib.state = 0; /* Mark it not active */
851 entry->msi_attrib.entry_nr = j;
852 entry->msi_attrib.maskbit = 1;
853 entry->msi_attrib.default_vector = dev->irq;
854 entry->dev = dev;
855 entry->mask_base = base;
856 if (!head) {
857 entry->link.head = vector;
858 entry->link.tail = vector;
859 head = entry;
860 } else {
861 entry->link.head = temp;
862 entry->link.tail = tail->link.tail;
863 tail->link.tail = vector;
864 head->link.head = vector;
865 }
866 temp = vector;
867 tail = entry;
868 /* Replace with MSI-X handler */
869 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
870 /* Configure MSI-X capability structure */
871 status = msi_ops->setup(dev, vector,
872 &address_hi,
873 &address_lo,
874 &data);
875 if (status < 0)
876 break;
877
878 writel(address_lo,
879 base + j * PCI_MSIX_ENTRY_SIZE +
880 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
881 writel(address_hi,
882 base + j * PCI_MSIX_ENTRY_SIZE +
883 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
884 writel(data,
885 base + j * PCI_MSIX_ENTRY_SIZE +
886 PCI_MSIX_ENTRY_DATA_OFFSET);
887 attach_msi_entry(entry, vector);
888 }
889 if (i != nvec) {
890 i--;
891 for (; i >= 0; i--) {
892 vector = (entries + i)->vector;
893 msi_free_vector(dev, vector, 0);
894 (entries + i)->vector = 0;
895 }
896 return -EBUSY;
897 }
898 /* Set MSI-X enabled bits */
899 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
900
901 return 0;
902 }
903
904 /**
905 * pci_enable_msi - configure device's MSI capability structure
906 * @dev: pointer to the pci_dev data structure of MSI device function
907 *
908 * Setup the MSI capability structure of device function with
909 * a single MSI vector upon its software driver call to request for
910 * MSI mode enabled on its hardware device function. A return of zero
911 * indicates the successful setup of an entry zero with the new MSI
912 * vector or non-zero for otherwise.
913 **/
914 int pci_enable_msi(struct pci_dev* dev)
915 {
916 struct pci_bus *bus;
917 int pos, temp, status = -EINVAL;
918 u16 control;
919
920 if (!pci_msi_enable || !dev)
921 return status;
922
923 if (dev->no_msi)
924 return status;
925
926 for (bus = dev->bus; bus; bus = bus->parent)
927 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
928 return -EINVAL;
929
930 temp = dev->irq;
931
932 status = msi_init();
933 if (status < 0)
934 return status;
935
936 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
937 if (!pos)
938 return -EINVAL;
939
940 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
941 /* Lookup Sucess */
942 unsigned long flags;
943
944 pci_read_config_word(dev, msi_control_reg(pos), &control);
945 if (control & PCI_MSI_FLAGS_ENABLE)
946 return 0; /* Already in MSI mode */
947 spin_lock_irqsave(&msi_lock, flags);
948 if (!vector_irq[dev->irq]) {
949 msi_desc[dev->irq]->msi_attrib.state = 0;
950 vector_irq[dev->irq] = -1;
951 nr_released_vectors--;
952 spin_unlock_irqrestore(&msi_lock, flags);
953 status = msi_register_init(dev, msi_desc[dev->irq]);
954 if (status == 0)
955 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
956 return status;
957 }
958 spin_unlock_irqrestore(&msi_lock, flags);
959 dev->irq = temp;
960 }
961 /* Check whether driver already requested for MSI-X vectors */
962 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
963 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
964 printk(KERN_INFO "PCI: %s: Can't enable MSI. "
965 "Device already has MSI-X vectors assigned\n",
966 pci_name(dev));
967 dev->irq = temp;
968 return -EINVAL;
969 }
970 status = msi_capability_init(dev);
971 if (!status) {
972 if (!pos)
973 nr_reserved_vectors--; /* Only MSI capable */
974 else if (nr_msix_devices > 0)
975 nr_msix_devices--; /* Both MSI and MSI-X capable,
976 but choose enabling MSI */
977 }
978
979 return status;
980 }
981
982 void pci_disable_msi(struct pci_dev* dev)
983 {
984 struct msi_desc *entry;
985 int pos, default_vector;
986 u16 control;
987 unsigned long flags;
988
989 if (!pci_msi_enable)
990 return;
991 if (!dev)
992 return;
993
994 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
995 if (!pos)
996 return;
997
998 pci_read_config_word(dev, msi_control_reg(pos), &control);
999 if (!(control & PCI_MSI_FLAGS_ENABLE))
1000 return;
1001
1002 spin_lock_irqsave(&msi_lock, flags);
1003 entry = msi_desc[dev->irq];
1004 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
1005 spin_unlock_irqrestore(&msi_lock, flags);
1006 return;
1007 }
1008 if (entry->msi_attrib.state) {
1009 spin_unlock_irqrestore(&msi_lock, flags);
1010 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
1011 "free_irq() on MSI vector %d\n",
1012 pci_name(dev), dev->irq);
1013 BUG_ON(entry->msi_attrib.state > 0);
1014 } else {
1015 vector_irq[dev->irq] = 0; /* free it */
1016 nr_released_vectors++;
1017 default_vector = entry->msi_attrib.default_vector;
1018 spin_unlock_irqrestore(&msi_lock, flags);
1019 /* Restore dev->irq to its default pin-assertion vector */
1020 dev->irq = default_vector;
1021 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
1022 PCI_CAP_ID_MSI);
1023 }
1024 }
1025
1026 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
1027 {
1028 struct msi_desc *entry;
1029 int head, entry_nr, type;
1030 void __iomem *base;
1031 unsigned long flags;
1032
1033 msi_ops->teardown(vector);
1034
1035 spin_lock_irqsave(&msi_lock, flags);
1036 entry = msi_desc[vector];
1037 if (!entry || entry->dev != dev) {
1038 spin_unlock_irqrestore(&msi_lock, flags);
1039 return -EINVAL;
1040 }
1041 type = entry->msi_attrib.type;
1042 entry_nr = entry->msi_attrib.entry_nr;
1043 head = entry->link.head;
1044 base = entry->mask_base;
1045 msi_desc[entry->link.head]->link.tail = entry->link.tail;
1046 msi_desc[entry->link.tail]->link.head = entry->link.head;
1047 entry->dev = NULL;
1048 if (!reassign) {
1049 vector_irq[vector] = 0;
1050 nr_released_vectors++;
1051 }
1052 msi_desc[vector] = NULL;
1053 spin_unlock_irqrestore(&msi_lock, flags);
1054
1055 kmem_cache_free(msi_cachep, entry);
1056
1057 if (type == PCI_CAP_ID_MSIX) {
1058 if (!reassign)
1059 writel(1, base +
1060 entry_nr * PCI_MSIX_ENTRY_SIZE +
1061 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
1062
1063 if (head == vector)
1064 iounmap(base);
1065 }
1066
1067 return 0;
1068 }
1069
1070 static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
1071 {
1072 int vector = head, tail = 0;
1073 int i, j = 0, nr_entries = 0;
1074 void __iomem *base;
1075 unsigned long flags;
1076
1077 spin_lock_irqsave(&msi_lock, flags);
1078 while (head != tail) {
1079 nr_entries++;
1080 tail = msi_desc[vector]->link.tail;
1081 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
1082 j = vector;
1083 vector = tail;
1084 }
1085 if (*nvec > nr_entries) {
1086 spin_unlock_irqrestore(&msi_lock, flags);
1087 *nvec = nr_entries;
1088 return -EINVAL;
1089 }
1090 vector = ((j > 0) ? j : head);
1091 for (i = 0; i < *nvec; i++) {
1092 j = msi_desc[vector]->msi_attrib.entry_nr;
1093 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */
1094 vector_irq[vector] = -1; /* Mark it busy */
1095 nr_released_vectors--;
1096 entries[i].vector = vector;
1097 if (j != (entries + i)->entry) {
1098 base = msi_desc[vector]->mask_base;
1099 msi_desc[vector]->msi_attrib.entry_nr =
1100 (entries + i)->entry;
1101 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
1102 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
1103 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
1104 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
1105 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
1106 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
1107 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
1108 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
1109 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
1110 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
1111 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
1112 PCI_MSIX_ENTRY_DATA_OFFSET);
1113 }
1114 vector = msi_desc[vector]->link.tail;
1115 }
1116 spin_unlock_irqrestore(&msi_lock, flags);
1117
1118 return 0;
1119 }
1120
1121 /**
1122 * pci_enable_msix - configure device's MSI-X capability structure
1123 * @dev: pointer to the pci_dev data structure of MSI-X device function
1124 * @entries: pointer to an array of MSI-X entries
1125 * @nvec: number of MSI-X vectors requested for allocation by device driver
1126 *
1127 * Setup the MSI-X capability structure of device function with the number
1128 * of requested vectors upon its software driver call to request for
1129 * MSI-X mode enabled on its hardware device function. A return of zero
1130 * indicates the successful configuration of MSI-X capability structure
1131 * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
1132 * Or a return of > 0 indicates that driver request is exceeding the number
1133 * of vectors available. Driver should use the returned value to re-send
1134 * its request.
1135 **/
1136 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
1137 {
1138 struct pci_bus *bus;
1139 int status, pos, nr_entries, free_vectors;
1140 int i, j, temp;
1141 u16 control;
1142 unsigned long flags;
1143
1144 if (!pci_msi_enable || !dev || !entries)
1145 return -EINVAL;
1146
1147 if (dev->no_msi)
1148 return -EINVAL;
1149
1150 for (bus = dev->bus; bus; bus = bus->parent)
1151 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
1152 return -EINVAL;
1153
1154 status = msi_init();
1155 if (status < 0)
1156 return status;
1157
1158 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1159 if (!pos)
1160 return -EINVAL;
1161
1162 pci_read_config_word(dev, msi_control_reg(pos), &control);
1163 if (control & PCI_MSIX_FLAGS_ENABLE)
1164 return -EINVAL; /* Already in MSI-X mode */
1165
1166 nr_entries = multi_msix_capable(control);
1167 if (nvec > nr_entries)
1168 return -EINVAL;
1169
1170 /* Check for any invalid entries */
1171 for (i = 0; i < nvec; i++) {
1172 if (entries[i].entry >= nr_entries)
1173 return -EINVAL; /* invalid entry */
1174 for (j = i + 1; j < nvec; j++) {
1175 if (entries[i].entry == entries[j].entry)
1176 return -EINVAL; /* duplicate entry */
1177 }
1178 }
1179 temp = dev->irq;
1180 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1181 /* Lookup Sucess */
1182 nr_entries = nvec;
1183 /* Reroute MSI-X table */
1184 if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
1185 /* #requested > #previous-assigned */
1186 dev->irq = temp;
1187 return nr_entries;
1188 }
1189 dev->irq = temp;
1190 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
1191 return 0;
1192 }
1193 /* Check whether driver already requested for MSI vector */
1194 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
1195 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1196 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
1197 "Device already has an MSI vector assigned\n",
1198 pci_name(dev));
1199 dev->irq = temp;
1200 return -EINVAL;
1201 }
1202
1203 spin_lock_irqsave(&msi_lock, flags);
1204 /*
1205 * msi_lock is provided to ensure that enough vectors resources are
1206 * available before granting.
1207 */
1208 free_vectors = pci_vector_resources(last_alloc_vector,
1209 nr_released_vectors);
1210 /* Ensure that each MSI/MSI-X device has one vector reserved by
1211 default to avoid any MSI-X driver to take all available
1212 resources */
1213 free_vectors -= nr_reserved_vectors;
1214 /* Find the average of free vectors among MSI-X devices */
1215 if (nr_msix_devices > 0)
1216 free_vectors /= nr_msix_devices;
1217 spin_unlock_irqrestore(&msi_lock, flags);
1218
1219 if (nvec > free_vectors) {
1220 if (free_vectors > 0)
1221 return free_vectors;
1222 else
1223 return -EBUSY;
1224 }
1225
1226 status = msix_capability_init(dev, entries, nvec);
1227 if (!status && nr_msix_devices > 0)
1228 nr_msix_devices--;
1229
1230 return status;
1231 }
1232
1233 void pci_disable_msix(struct pci_dev* dev)
1234 {
1235 int pos, temp;
1236 u16 control;
1237
1238 if (!pci_msi_enable)
1239 return;
1240 if (!dev)
1241 return;
1242
1243 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1244 if (!pos)
1245 return;
1246
1247 pci_read_config_word(dev, msi_control_reg(pos), &control);
1248 if (!(control & PCI_MSIX_FLAGS_ENABLE))
1249 return;
1250
1251 temp = dev->irq;
1252 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1253 int state, vector, head, tail = 0, warning = 0;
1254 unsigned long flags;
1255
1256 vector = head = dev->irq;
1257 spin_lock_irqsave(&msi_lock, flags);
1258 while (head != tail) {
1259 state = msi_desc[vector]->msi_attrib.state;
1260 if (state)
1261 warning = 1;
1262 else {
1263 vector_irq[vector] = 0; /* free it */
1264 nr_released_vectors++;
1265 }
1266 tail = msi_desc[vector]->link.tail;
1267 vector = tail;
1268 }
1269 spin_unlock_irqrestore(&msi_lock, flags);
1270 if (warning) {
1271 dev->irq = temp;
1272 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
1273 "free_irq() on all MSI-X vectors\n",
1274 pci_name(dev));
1275 BUG_ON(warning > 0);
1276 } else {
1277 dev->irq = temp;
1278 disable_msi_mode(dev,
1279 pci_find_capability(dev, PCI_CAP_ID_MSIX),
1280 PCI_CAP_ID_MSIX);
1281
1282 }
1283 }
1284 }
1285
1286 /**
1287 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
1288 * @dev: pointer to the pci_dev data structure of MSI(X) device function
1289 *
1290 * Being called during hotplug remove, from which the device function
1291 * is hot-removed. All previous assigned MSI/MSI-X vectors, if
1292 * allocated for this device function, are reclaimed to unused state,
1293 * which may be used later on.
1294 **/
1295 void msi_remove_pci_irq_vectors(struct pci_dev* dev)
1296 {
1297 int state, pos, temp;
1298 unsigned long flags;
1299
1300 if (!pci_msi_enable || !dev)
1301 return;
1302
1303 temp = dev->irq; /* Save IOAPIC IRQ */
1304 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
1305 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1306 spin_lock_irqsave(&msi_lock, flags);
1307 state = msi_desc[dev->irq]->msi_attrib.state;
1308 spin_unlock_irqrestore(&msi_lock, flags);
1309 if (state) {
1310 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1311 "called without free_irq() on MSI vector %d\n",
1312 pci_name(dev), dev->irq);
1313 BUG_ON(state > 0);
1314 } else /* Release MSI vector assigned to this device */
1315 msi_free_vector(dev, dev->irq, 0);
1316 dev->irq = temp; /* Restore IOAPIC IRQ */
1317 }
1318 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1319 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1320 int vector, head, tail = 0, warning = 0;
1321 void __iomem *base = NULL;
1322
1323 vector = head = dev->irq;
1324 while (head != tail) {
1325 spin_lock_irqsave(&msi_lock, flags);
1326 state = msi_desc[vector]->msi_attrib.state;
1327 tail = msi_desc[vector]->link.tail;
1328 base = msi_desc[vector]->mask_base;
1329 spin_unlock_irqrestore(&msi_lock, flags);
1330 if (state)
1331 warning = 1;
1332 else if (vector != head) /* Release MSI-X vector */
1333 msi_free_vector(dev, vector, 0);
1334 vector = tail;
1335 }
1336 msi_free_vector(dev, vector, 0);
1337 if (warning) {
1338 iounmap(base);
1339 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1340 "called without free_irq() on all MSI-X vectors\n",
1341 pci_name(dev));
1342 BUG_ON(warning > 0);
1343 }
1344 dev->irq = temp; /* Restore IOAPIC IRQ */
1345 }
1346 }
1347
1348 void pci_no_msi(void)
1349 {
1350 pci_msi_enable = 0;
1351 }
1352
1353 EXPORT_SYMBOL(pci_enable_msi);
1354 EXPORT_SYMBOL(pci_disable_msi);
1355 EXPORT_SYMBOL(pci_enable_msix);
1356 EXPORT_SYMBOL(pci_disable_msix);
linux v2.6.22.y-op