1 #include <linux/seq_file.h>
2 #include <linux/cpumask.h>
3 #include <linux/kernel.h>
4 #include <linux/string.h>
5 #include <linux/errno.h>
10 #include <asm/hw_irq.h>
11 #include <asm/irq_remapping.h>
12 #include <asm/processor.h>
13 #include <asm/x86_init.h>
17 #include "irq_remapping.h"
19 int irq_remapping_enabled
;
21 int disable_irq_remap
;
23 int disable_sourceid_checking
;
26 static struct irq_remap_ops
*remap_ops
;
28 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
);
29 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
30 int index
, int sub_handle
);
31 static int set_remapped_irq_affinity(struct irq_data
*data
,
32 const struct cpumask
*mask
,
35 static bool irq_remapped(struct irq_cfg
*cfg
)
37 return (cfg
->remapped
== 1);
40 static void irq_remapping_disable_io_apic(void)
43 * With interrupt-remapping, for now we will use virtual wire A
44 * mode, as virtual wire B is little complex (need to configure
45 * both IOAPIC RTE as well as interrupt-remapping table entry).
46 * As this gets called during crash dump, keep this simple for
49 if (cpu_has_apic
|| apic_from_smp_config())
50 disconnect_bsp_APIC(0);
53 static int do_setup_msi_irqs(struct pci_dev
*dev
, int nvec
)
55 int ret
, sub_handle
, nvec_pow2
, index
= 0;
57 struct msi_desc
*msidesc
;
59 WARN_ON(!list_is_singular(&dev
->msi_list
));
60 msidesc
= list_entry(dev
->msi_list
.next
, struct msi_desc
, list
);
61 WARN_ON(msidesc
->irq
);
62 WARN_ON(msidesc
->msi_attrib
.multiple
);
63 WARN_ON(msidesc
->nvec_used
);
65 irq
= irq_alloc_hwirqs(nvec
, dev_to_node(&dev
->dev
));
69 nvec_pow2
= __roundup_pow_of_two(nvec
);
70 msidesc
->nvec_used
= nvec
;
71 msidesc
->msi_attrib
.multiple
= ilog2(nvec_pow2
);
72 for (sub_handle
= 0; sub_handle
< nvec
; sub_handle
++) {
74 index
= msi_alloc_remapped_irq(dev
, irq
, nvec_pow2
);
80 ret
= msi_setup_remapped_irq(dev
, irq
+ sub_handle
,
85 ret
= setup_msi_irq(dev
, msidesc
, irq
, sub_handle
);
92 irq_free_hwirqs(irq
, nvec
);
95 * Restore altered MSI descriptor fields and prevent just destroyed
96 * IRQs from tearing down again in default_teardown_msi_irqs()
99 msidesc
->nvec_used
= 0;
100 msidesc
->msi_attrib
.multiple
= 0;
105 static int do_setup_msix_irqs(struct pci_dev
*dev
, int nvec
)
107 int node
, ret
, sub_handle
, index
= 0;
108 struct msi_desc
*msidesc
;
111 node
= dev_to_node(&dev
->dev
);
114 list_for_each_entry(msidesc
, &dev
->msi_list
, list
) {
116 irq
= irq_alloc_hwirq(node
);
121 ret
= index
= msi_alloc_remapped_irq(dev
, irq
, nvec
);
123 ret
= msi_setup_remapped_irq(dev
, irq
, index
, sub_handle
);
128 ret
= setup_msi_irq(dev
, msidesc
, irq
, 0);
143 static int irq_remapping_setup_msi_irqs(struct pci_dev
*dev
,
146 if (type
== PCI_CAP_ID_MSI
)
147 return do_setup_msi_irqs(dev
, nvec
);
149 return do_setup_msix_irqs(dev
, nvec
);
152 static void eoi_ioapic_pin_remapped(int apic
, int pin
, int vector
)
155 * Intr-remapping uses pin number as the virtual vector
156 * in the RTE. Actual vector is programmed in
157 * intr-remapping table entry. Hence for the io-apic
158 * EOI we use the pin number.
160 io_apic_eoi(apic
, pin
);
163 static void __init
irq_remapping_modify_x86_ops(void)
165 x86_io_apic_ops
.disable
= irq_remapping_disable_io_apic
;
166 x86_io_apic_ops
.set_affinity
= set_remapped_irq_affinity
;
167 x86_io_apic_ops
.setup_entry
= setup_ioapic_remapped_entry
;
168 x86_io_apic_ops
.eoi_ioapic_pin
= eoi_ioapic_pin_remapped
;
169 x86_msi
.setup_msi_irqs
= irq_remapping_setup_msi_irqs
;
170 x86_msi
.setup_hpet_msi
= setup_hpet_msi_remapped
;
171 x86_msi
.compose_msi_msg
= compose_remapped_msi_msg
;
174 static __init
int setup_nointremap(char *str
)
176 disable_irq_remap
= 1;
179 early_param("nointremap", setup_nointremap
);
181 static __init
int setup_irqremap(char *str
)
187 if (!strncmp(str
, "on", 2))
188 disable_irq_remap
= 0;
189 else if (!strncmp(str
, "off", 3))
190 disable_irq_remap
= 1;
191 else if (!strncmp(str
, "nosid", 5))
192 disable_sourceid_checking
= 1;
193 else if (!strncmp(str
, "no_x2apic_optout", 16))
194 no_x2apic_optout
= 1;
196 str
+= strcspn(str
, ",");
203 early_param("intremap", setup_irqremap
);
205 void __init
setup_irq_remapping_ops(void)
207 remap_ops
= &intel_irq_remap_ops
;
209 #ifdef CONFIG_AMD_IOMMU
210 if (amd_iommu_irq_ops
.prepare() == 0)
211 remap_ops
= &amd_iommu_irq_ops
;
215 void set_irq_remapping_broken(void)
217 irq_remap_broken
= 1;
220 int irq_remapping_supported(void)
222 if (disable_irq_remap
)
225 if (!remap_ops
|| !remap_ops
->supported
)
228 return remap_ops
->supported();
231 int __init
irq_remapping_prepare(void)
233 if (!remap_ops
|| !remap_ops
->prepare
)
236 return remap_ops
->prepare();
239 int __init
irq_remapping_enable(void)
243 if (!remap_ops
|| !remap_ops
->enable
)
246 ret
= remap_ops
->enable();
248 if (irq_remapping_enabled
)
249 irq_remapping_modify_x86_ops();
254 void irq_remapping_disable(void)
256 if (!irq_remapping_enabled
||
261 remap_ops
->disable();
264 int irq_remapping_reenable(int mode
)
266 if (!irq_remapping_enabled
||
268 !remap_ops
->reenable
)
271 return remap_ops
->reenable(mode
);
274 int __init
irq_remap_enable_fault_handling(void)
276 if (!irq_remapping_enabled
)
279 if (!remap_ops
|| !remap_ops
->enable_faulting
)
282 return remap_ops
->enable_faulting();
285 int setup_ioapic_remapped_entry(int irq
,
286 struct IO_APIC_route_entry
*entry
,
287 unsigned int destination
, int vector
,
288 struct io_apic_irq_attr
*attr
)
290 if (!remap_ops
|| !remap_ops
->setup_ioapic_entry
)
293 return remap_ops
->setup_ioapic_entry(irq
, entry
, destination
,
297 static int set_remapped_irq_affinity(struct irq_data
*data
,
298 const struct cpumask
*mask
, bool force
)
300 if (!config_enabled(CONFIG_SMP
) || !remap_ops
||
301 !remap_ops
->set_affinity
)
304 return remap_ops
->set_affinity(data
, mask
, force
);
307 void free_remapped_irq(int irq
)
309 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
311 if (!remap_ops
|| !remap_ops
->free_irq
)
314 if (irq_remapped(cfg
))
315 remap_ops
->free_irq(irq
);
318 void compose_remapped_msi_msg(struct pci_dev
*pdev
,
319 unsigned int irq
, unsigned int dest
,
320 struct msi_msg
*msg
, u8 hpet_id
)
322 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
324 if (!irq_remapped(cfg
))
325 native_compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
326 else if (remap_ops
&& remap_ops
->compose_msi_msg
)
327 remap_ops
->compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
330 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
)
332 if (!remap_ops
|| !remap_ops
->msi_alloc_irq
)
335 return remap_ops
->msi_alloc_irq(pdev
, irq
, nvec
);
338 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
339 int index
, int sub_handle
)
341 if (!remap_ops
|| !remap_ops
->msi_setup_irq
)
344 return remap_ops
->msi_setup_irq(pdev
, irq
, index
, sub_handle
);
347 int setup_hpet_msi_remapped(unsigned int irq
, unsigned int id
)
351 if (!remap_ops
|| !remap_ops
->alloc_hpet_msi
)
354 ret
= remap_ops
->alloc_hpet_msi(irq
, id
);
358 return default_setup_hpet_msi(irq
, id
);
361 void panic_if_irq_remap(const char *msg
)
363 if (irq_remapping_enabled
)
367 static void ir_ack_apic_edge(struct irq_data
*data
)
372 static void ir_ack_apic_level(struct irq_data
*data
)
375 eoi_ioapic_irq(data
->irq
, data
->chip_data
);
378 static void ir_print_prefix(struct irq_data
*data
, struct seq_file
*p
)
380 seq_printf(p
, " IR-%s", data
->chip
->name
);
383 void irq_remap_modify_chip_defaults(struct irq_chip
*chip
)
385 chip
->irq_print_chip
= ir_print_prefix
;
386 chip
->irq_ack
= ir_ack_apic_edge
;
387 chip
->irq_eoi
= ir_ack_apic_level
;
388 chip
->irq_set_affinity
= x86_io_apic_ops
.set_affinity
;
391 bool setup_remapped_irq(int irq
, struct irq_cfg
*cfg
, struct irq_chip
*chip
)
393 if (!irq_remapped(cfg
))
395 irq_set_status_flags(irq
, IRQ_MOVE_PCNTXT
);
396 irq_remap_modify_chip_defaults(chip
);