2 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
3 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5 * This file contains the interrupt descriptor management code
7 * Detailed information is available in Documentation/DocBook/genericirq
10 #include <linux/irq.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/radix-tree.h>
16 #include <linux/bitmap.h>
18 #include "internals.h"
21 * lockdep: we want to handle all irq_desc locks as a single lock-class:
23 static struct lock_class_key irq_desc_lock_class
;
25 #if defined(CONFIG_SMP)
26 static void __init
init_irq_default_affinity(void)
28 alloc_cpumask_var(&irq_default_affinity
, GFP_NOWAIT
);
29 cpumask_setall(irq_default_affinity
);
32 static void __init
init_irq_default_affinity(void)
38 static int alloc_masks(struct irq_desc
*desc
, gfp_t gfp
, int node
)
40 if (!zalloc_cpumask_var_node(&desc
->irq_data
.affinity
, gfp
, node
))
43 #ifdef CONFIG_GENERIC_PENDING_IRQ
44 if (!zalloc_cpumask_var_node(&desc
->pending_mask
, gfp
, node
)) {
45 free_cpumask_var(desc
->irq_data
.affinity
);
52 static void desc_smp_init(struct irq_desc
*desc
, int node
)
54 desc
->irq_data
.node
= node
;
55 cpumask_copy(desc
->irq_data
.affinity
, irq_default_affinity
);
56 #ifdef CONFIG_GENERIC_PENDING_IRQ
57 cpumask_clear(desc
->pending_mask
);
61 static inline int desc_node(struct irq_desc
*desc
)
63 return desc
->irq_data
.node
;
68 alloc_masks(struct irq_desc
*desc
, gfp_t gfp
, int node
) { return 0; }
69 static inline void desc_smp_init(struct irq_desc
*desc
, int node
) { }
70 static inline int desc_node(struct irq_desc
*desc
) { return 0; }
73 static void desc_set_defaults(unsigned int irq
, struct irq_desc
*desc
, int node
,
78 desc
->irq_data
.irq
= irq
;
79 desc
->irq_data
.chip
= &no_irq_chip
;
80 desc
->irq_data
.chip_data
= NULL
;
81 desc
->irq_data
.handler_data
= NULL
;
82 desc
->irq_data
.msi_desc
= NULL
;
83 irq_settings_clr_and_set(desc
, ~0, _IRQ_DEFAULT_INIT_FLAGS
);
84 irqd_set(&desc
->irq_data
, IRQD_IRQ_DISABLED
);
85 desc
->handle_irq
= handle_bad_irq
;
88 desc
->irqs_unhandled
= 0;
91 for_each_possible_cpu(cpu
)
92 *per_cpu_ptr(desc
->kstat_irqs
, cpu
) = 0;
93 desc_smp_init(desc
, node
);
96 int nr_irqs
= NR_IRQS
;
97 EXPORT_SYMBOL_GPL(nr_irqs
);
99 static DEFINE_MUTEX(sparse_irq_lock
);
100 static DECLARE_BITMAP(allocated_irqs
, IRQ_BITMAP_BITS
);
102 #ifdef CONFIG_SPARSE_IRQ
104 static RADIX_TREE(irq_desc_tree
, GFP_KERNEL
);
106 static void irq_insert_desc(unsigned int irq
, struct irq_desc
*desc
)
108 radix_tree_insert(&irq_desc_tree
, irq
, desc
);
111 struct irq_desc
*irq_to_desc(unsigned int irq
)
113 return radix_tree_lookup(&irq_desc_tree
, irq
);
115 EXPORT_SYMBOL(irq_to_desc
);
117 static void delete_irq_desc(unsigned int irq
)
119 radix_tree_delete(&irq_desc_tree
, irq
);
123 static void free_masks(struct irq_desc
*desc
)
125 #ifdef CONFIG_GENERIC_PENDING_IRQ
126 free_cpumask_var(desc
->pending_mask
);
128 free_cpumask_var(desc
->irq_data
.affinity
);
131 static inline void free_masks(struct irq_desc
*desc
) { }
134 static struct irq_desc
*alloc_desc(int irq
, int node
, struct module
*owner
)
136 struct irq_desc
*desc
;
137 gfp_t gfp
= GFP_KERNEL
;
139 desc
= kzalloc_node(sizeof(*desc
), gfp
, node
);
142 /* allocate based on nr_cpu_ids */
143 desc
->kstat_irqs
= alloc_percpu(unsigned int);
144 if (!desc
->kstat_irqs
)
147 if (alloc_masks(desc
, gfp
, node
))
150 raw_spin_lock_init(&desc
->lock
);
151 lockdep_set_class(&desc
->lock
, &irq_desc_lock_class
);
153 desc_set_defaults(irq
, desc
, node
, owner
);
158 free_percpu(desc
->kstat_irqs
);
164 static void free_desc(unsigned int irq
)
166 struct irq_desc
*desc
= irq_to_desc(irq
);
168 unregister_irq_proc(irq
, desc
);
170 mutex_lock(&sparse_irq_lock
);
171 delete_irq_desc(irq
);
172 mutex_unlock(&sparse_irq_lock
);
175 free_percpu(desc
->kstat_irqs
);
179 static int alloc_descs(unsigned int start
, unsigned int cnt
, int node
,
180 struct module
*owner
)
182 struct irq_desc
*desc
;
185 for (i
= 0; i
< cnt
; i
++) {
186 desc
= alloc_desc(start
+ i
, node
, owner
);
189 mutex_lock(&sparse_irq_lock
);
190 irq_insert_desc(start
+ i
, desc
);
191 mutex_unlock(&sparse_irq_lock
);
196 for (i
--; i
>= 0; i
--)
197 free_desc(start
+ i
);
199 mutex_lock(&sparse_irq_lock
);
200 bitmap_clear(allocated_irqs
, start
, cnt
);
201 mutex_unlock(&sparse_irq_lock
);
205 static int irq_expand_nr_irqs(unsigned int nr
)
207 if (nr
> IRQ_BITMAP_BITS
)
213 int __init
early_irq_init(void)
215 int i
, initcnt
, node
= first_online_node
;
216 struct irq_desc
*desc
;
218 init_irq_default_affinity();
220 /* Let arch update nr_irqs and return the nr of preallocated irqs */
221 initcnt
= arch_probe_nr_irqs();
222 printk(KERN_INFO
"NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS
, nr_irqs
, initcnt
);
224 if (WARN_ON(nr_irqs
> IRQ_BITMAP_BITS
))
225 nr_irqs
= IRQ_BITMAP_BITS
;
227 if (WARN_ON(initcnt
> IRQ_BITMAP_BITS
))
228 initcnt
= IRQ_BITMAP_BITS
;
230 if (initcnt
> nr_irqs
)
233 for (i
= 0; i
< initcnt
; i
++) {
234 desc
= alloc_desc(i
, node
, NULL
);
235 set_bit(i
, allocated_irqs
);
236 irq_insert_desc(i
, desc
);
238 return arch_early_irq_init();
241 #else /* !CONFIG_SPARSE_IRQ */
243 struct irq_desc irq_desc
[NR_IRQS
] __cacheline_aligned_in_smp
= {
244 [0 ... NR_IRQS
-1] = {
245 .handle_irq
= handle_bad_irq
,
247 .lock
= __RAW_SPIN_LOCK_UNLOCKED(irq_desc
->lock
),
251 int __init
early_irq_init(void)
253 int count
, i
, node
= first_online_node
;
254 struct irq_desc
*desc
;
256 init_irq_default_affinity();
258 printk(KERN_INFO
"NR_IRQS:%d\n", NR_IRQS
);
261 count
= ARRAY_SIZE(irq_desc
);
263 for (i
= 0; i
< count
; i
++) {
264 desc
[i
].kstat_irqs
= alloc_percpu(unsigned int);
265 alloc_masks(&desc
[i
], GFP_KERNEL
, node
);
266 raw_spin_lock_init(&desc
[i
].lock
);
267 lockdep_set_class(&desc
[i
].lock
, &irq_desc_lock_class
);
268 desc_set_defaults(i
, &desc
[i
], node
, NULL
);
270 return arch_early_irq_init();
273 struct irq_desc
*irq_to_desc(unsigned int irq
)
275 return (irq
< NR_IRQS
) ? irq_desc
+ irq
: NULL
;
277 EXPORT_SYMBOL(irq_to_desc
);
279 static void free_desc(unsigned int irq
)
281 struct irq_desc
*desc
= irq_to_desc(irq
);
284 raw_spin_lock_irqsave(&desc
->lock
, flags
);
285 desc_set_defaults(irq
, desc
, desc_node(desc
), NULL
);
286 raw_spin_unlock_irqrestore(&desc
->lock
, flags
);
289 static inline int alloc_descs(unsigned int start
, unsigned int cnt
, int node
,
290 struct module
*owner
)
294 for (i
= 0; i
< cnt
; i
++) {
295 struct irq_desc
*desc
= irq_to_desc(start
+ i
);
302 static int irq_expand_nr_irqs(unsigned int nr
)
307 void irq_mark_irq(unsigned int irq
)
309 mutex_lock(&sparse_irq_lock
);
310 bitmap_set(allocated_irqs
, irq
, 1);
311 mutex_unlock(&sparse_irq_lock
);
314 #ifdef CONFIG_GENERIC_IRQ_LEGACY
315 void irq_init_desc(unsigned int irq
)
321 #endif /* !CONFIG_SPARSE_IRQ */
324 * generic_handle_irq - Invoke the handler for a particular irq
325 * @irq: The irq number to handle
328 int generic_handle_irq(unsigned int irq
)
330 struct irq_desc
*desc
= irq_to_desc(irq
);
334 generic_handle_irq_desc(irq
, desc
);
337 EXPORT_SYMBOL_GPL(generic_handle_irq
);
339 /* Dynamic interrupt handling */
342 * irq_free_descs - free irq descriptors
343 * @from: Start of descriptor range
344 * @cnt: Number of consecutive irqs to free
346 void irq_free_descs(unsigned int from
, unsigned int cnt
)
350 if (from
>= nr_irqs
|| (from
+ cnt
) > nr_irqs
)
353 for (i
= 0; i
< cnt
; i
++)
356 mutex_lock(&sparse_irq_lock
);
357 bitmap_clear(allocated_irqs
, from
, cnt
);
358 mutex_unlock(&sparse_irq_lock
);
360 EXPORT_SYMBOL_GPL(irq_free_descs
);
363 * irq_alloc_descs - allocate and initialize a range of irq descriptors
364 * @irq: Allocate for specific irq number if irq >= 0
365 * @from: Start the search from this irq number
366 * @cnt: Number of consecutive irqs to allocate.
367 * @node: Preferred node on which the irq descriptor should be allocated
368 * @owner: Owning module (can be NULL)
370 * Returns the first irq number or error code
373 __irq_alloc_descs(int irq
, unsigned int from
, unsigned int cnt
, int node
,
374 struct module
*owner
)
387 * For interrupts which are freely allocated the
388 * architecture can force a lower bound to the @from
389 * argument. x86 uses this to exclude the GSI space.
391 from
= arch_dynirq_lower_bound(from
);
394 mutex_lock(&sparse_irq_lock
);
396 start
= bitmap_find_next_zero_area(allocated_irqs
, IRQ_BITMAP_BITS
,
399 if (irq
>=0 && start
!= irq
)
402 if (start
+ cnt
> nr_irqs
) {
403 ret
= irq_expand_nr_irqs(start
+ cnt
);
408 bitmap_set(allocated_irqs
, start
, cnt
);
409 mutex_unlock(&sparse_irq_lock
);
410 return alloc_descs(start
, cnt
, node
, owner
);
413 mutex_unlock(&sparse_irq_lock
);
416 EXPORT_SYMBOL_GPL(__irq_alloc_descs
);
418 #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
420 * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
421 * @cnt: number of interrupts to allocate
422 * @node: node on which to allocate
424 * Returns an interrupt number > 0 or 0, if the allocation fails.
426 unsigned int irq_alloc_hwirqs(int cnt
, int node
)
428 int i
, irq
= __irq_alloc_descs(-1, 0, cnt
, node
, NULL
);
433 for (i
= irq
; cnt
> 0; i
++, cnt
--) {
434 if (arch_setup_hwirq(i
, node
))
436 irq_clear_status_flags(i
, _IRQ_NOREQUEST
);
441 for (i
--; i
>= irq
; i
--) {
442 irq_set_status_flags(i
, _IRQ_NOREQUEST
| _IRQ_NOPROBE
);
443 arch_teardown_hwirq(i
);
445 irq_free_descs(irq
, cnt
);
448 EXPORT_SYMBOL_GPL(irq_alloc_hwirqs
);
451 * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
452 * @from: Free from irq number
453 * @cnt: number of interrupts to free
456 void irq_free_hwirqs(unsigned int from
, int cnt
)
460 for (i
= from
, j
= cnt
; j
> 0; i
++, j
--) {
461 irq_set_status_flags(i
, _IRQ_NOREQUEST
| _IRQ_NOPROBE
);
462 arch_teardown_hwirq(i
);
464 irq_free_descs(from
, cnt
);
466 EXPORT_SYMBOL_GPL(irq_free_hwirqs
);
470 * irq_get_next_irq - get next allocated irq number
471 * @offset: where to start the search
473 * Returns next irq number after offset or nr_irqs if none is found.
475 unsigned int irq_get_next_irq(unsigned int offset
)
477 return find_next_bit(allocated_irqs
, nr_irqs
, offset
);
481 __irq_get_desc_lock(unsigned int irq
, unsigned long *flags
, bool bus
,
484 struct irq_desc
*desc
= irq_to_desc(irq
);
487 if (check
& _IRQ_DESC_CHECK
) {
488 if ((check
& _IRQ_DESC_PERCPU
) &&
489 !irq_settings_is_per_cpu_devid(desc
))
492 if (!(check
& _IRQ_DESC_PERCPU
) &&
493 irq_settings_is_per_cpu_devid(desc
))
499 raw_spin_lock_irqsave(&desc
->lock
, *flags
);
504 void __irq_put_desc_unlock(struct irq_desc
*desc
, unsigned long flags
, bool bus
)
506 raw_spin_unlock_irqrestore(&desc
->lock
, flags
);
508 chip_bus_sync_unlock(desc
);
511 int irq_set_percpu_devid(unsigned int irq
)
513 struct irq_desc
*desc
= irq_to_desc(irq
);
518 if (desc
->percpu_enabled
)
521 desc
->percpu_enabled
= kzalloc(sizeof(*desc
->percpu_enabled
), GFP_KERNEL
);
523 if (!desc
->percpu_enabled
)
526 irq_set_percpu_devid_flags(irq
);
530 void kstat_incr_irq_this_cpu(unsigned int irq
)
532 kstat_incr_irqs_this_cpu(irq
, irq_to_desc(irq
));
535 unsigned int kstat_irqs_cpu(unsigned int irq
, int cpu
)
537 struct irq_desc
*desc
= irq_to_desc(irq
);
539 return desc
&& desc
->kstat_irqs
?
540 *per_cpu_ptr(desc
->kstat_irqs
, cpu
) : 0;
543 unsigned int kstat_irqs(unsigned int irq
)
545 struct irq_desc
*desc
= irq_to_desc(irq
);
549 if (!desc
|| !desc
->kstat_irqs
)
551 for_each_possible_cpu(cpu
)
552 sum
+= *per_cpu_ptr(desc
->kstat_irqs
, cpu
);