2 * linux/arch/i386/nmi.c
4 * NMI watchdog support on APIC systems
6 * Started by Ingo Molnar <mingo@redhat.com>
9 * Mikael Pettersson : AMD K7 support for local APIC NMI watchdog.
10 * Mikael Pettersson : Power Management for local APIC NMI watchdog.
11 * Mikael Pettersson : Pentium 4 support for local APIC NMI watchdog.
13 * Mikael Pettersson : PM converted to driver model. Disable/enable API.
16 #include <linux/delay.h>
17 #include <linux/interrupt.h>
18 #include <linux/module.h>
19 #include <linux/nmi.h>
20 #include <linux/sysdev.h>
21 #include <linux/sysctl.h>
22 #include <linux/percpu.h>
23 #include <linux/dmi.h>
24 #include <linux/kprobes.h>
25 #include <linux/cpumask.h>
26 #include <linux/kernel_stat.h>
30 #include <asm/kdebug.h>
31 #include <asm/intel_arch_perfmon.h>
33 #include "mach_traps.h"
35 int unknown_nmi_panic
;
36 int nmi_watchdog_enabled
;
38 /* perfctr_nmi_owner tracks the ownership of the perfctr registers:
39 * evtsel_nmi_owner tracks the ownership of the event selection
40 * - different performance counters/ event selection may be reserved for
41 * different subsystems this reservation system just tries to coordinate
44 static DEFINE_PER_CPU(unsigned long, perfctr_nmi_owner
);
45 static DEFINE_PER_CPU(unsigned long, evntsel_nmi_owner
[3]);
47 static cpumask_t backtrace_mask
= CPU_MASK_NONE
;
49 /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
50 * offset from MSR_P4_BSU_ESCR0. It will be the max for all platforms (for now)
52 #define NMI_MAX_COUNTER_BITS 66
55 * >0: the lapic NMI watchdog is active, but can be disabled
56 * <0: the lapic NMI watchdog has not been set up, and cannot
58 * 0: the lapic NMI watchdog is disabled, but can be enabled
60 atomic_t nmi_active
= ATOMIC_INIT(0); /* oprofile uses this */
62 unsigned int nmi_watchdog
= NMI_DEFAULT
;
63 static unsigned int nmi_hz
= HZ
;
65 struct nmi_watchdog_ctlblk
{
68 unsigned int cccr_msr
;
69 unsigned int perfctr_msr
; /* the MSR to reset in NMI handler */
70 unsigned int evntsel_msr
; /* the MSR to select the events to handle */
72 static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk
, nmi_watchdog_ctlblk
);
74 /* local prototypes */
75 static int unknown_nmi_panic_callback(struct pt_regs
*regs
, int cpu
);
77 extern void show_registers(struct pt_regs
*regs
);
78 extern int unknown_nmi_panic
;
80 /* converts an msr to an appropriate reservation bit */
81 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr
)
83 /* returns the bit offset of the performance counter register */
84 switch (boot_cpu_data
.x86_vendor
) {
86 return (msr
- MSR_K7_PERFCTR0
);
87 case X86_VENDOR_INTEL
:
88 if (cpu_has(&boot_cpu_data
, X86_FEATURE_ARCH_PERFMON
))
89 return (msr
- MSR_ARCH_PERFMON_PERFCTR0
);
91 switch (boot_cpu_data
.x86
) {
93 return (msr
- MSR_P6_PERFCTR0
);
95 return (msr
- MSR_P4_BPU_PERFCTR0
);
101 /* converts an msr to an appropriate reservation bit */
102 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr
)
104 /* returns the bit offset of the event selection register */
105 switch (boot_cpu_data
.x86_vendor
) {
107 return (msr
- MSR_K7_EVNTSEL0
);
108 case X86_VENDOR_INTEL
:
109 if (cpu_has(&boot_cpu_data
, X86_FEATURE_ARCH_PERFMON
))
110 return (msr
- MSR_ARCH_PERFMON_EVENTSEL0
);
112 switch (boot_cpu_data
.x86
) {
114 return (msr
- MSR_P6_EVNTSEL0
);
116 return (msr
- MSR_P4_BSU_ESCR0
);
122 /* checks for a bit availability (hack for oprofile) */
123 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter
)
125 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
127 return (!test_bit(counter
, &__get_cpu_var(perfctr_nmi_owner
)));
130 /* checks the an msr for availability */
131 int avail_to_resrv_perfctr_nmi(unsigned int msr
)
133 unsigned int counter
;
135 counter
= nmi_perfctr_msr_to_bit(msr
);
136 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
138 return (!test_bit(counter
, &__get_cpu_var(perfctr_nmi_owner
)));
141 int reserve_perfctr_nmi(unsigned int msr
)
143 unsigned int counter
;
145 counter
= nmi_perfctr_msr_to_bit(msr
);
146 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
148 if (!test_and_set_bit(counter
, &__get_cpu_var(perfctr_nmi_owner
)))
153 void release_perfctr_nmi(unsigned int msr
)
155 unsigned int counter
;
157 counter
= nmi_perfctr_msr_to_bit(msr
);
158 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
160 clear_bit(counter
, &__get_cpu_var(perfctr_nmi_owner
));
163 int reserve_evntsel_nmi(unsigned int msr
)
165 unsigned int counter
;
167 counter
= nmi_evntsel_msr_to_bit(msr
);
168 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
170 if (!test_and_set_bit(counter
, &__get_cpu_var(evntsel_nmi_owner
)[0]))
175 void release_evntsel_nmi(unsigned int msr
)
177 unsigned int counter
;
179 counter
= nmi_evntsel_msr_to_bit(msr
);
180 BUG_ON(counter
> NMI_MAX_COUNTER_BITS
);
182 clear_bit(counter
, &__get_cpu_var(evntsel_nmi_owner
)[0]);
185 static __cpuinit
inline int nmi_known_cpu(void)
187 switch (boot_cpu_data
.x86_vendor
) {
189 return ((boot_cpu_data
.x86
== 15) || (boot_cpu_data
.x86
== 6)
190 || (boot_cpu_data
.x86
== 16));
191 case X86_VENDOR_INTEL
:
192 if (cpu_has(&boot_cpu_data
, X86_FEATURE_ARCH_PERFMON
))
195 return ((boot_cpu_data
.x86
== 15) || (boot_cpu_data
.x86
== 6));
200 static int endflag __initdata
= 0;
203 /* The performance counters used by NMI_LOCAL_APIC don't trigger when
204 * the CPU is idle. To make sure the NMI watchdog really ticks on all
205 * CPUs during the test make them busy.
207 static __init
void nmi_cpu_busy(void *data
)
209 local_irq_enable_in_hardirq();
210 /* Intentionally don't use cpu_relax here. This is
211 to make sure that the performance counter really ticks,
212 even if there is a simulator or similar that catches the
213 pause instruction. On a real HT machine this is fine because
214 all other CPUs are busy with "useless" delay loops and don't
215 care if they get somewhat less cycles. */
221 static unsigned int adjust_for_32bit_ctr(unsigned int hz
)
224 unsigned int retval
= hz
;
227 * On Intel CPUs with P6/ARCH_PERFMON only 32 bits in the counter
228 * are writable, with higher bits sign extending from bit 31.
229 * So, we can only program the counter with 31 bit values and
230 * 32nd bit should be 1, for 33.. to be 1.
231 * Find the appropriate nmi_hz
233 counter_val
= (u64
)cpu_khz
* 1000;
234 do_div(counter_val
, retval
);
235 if (counter_val
> 0x7fffffffULL
) {
236 u64 count
= (u64
)cpu_khz
* 1000;
237 do_div(count
, 0x7fffffffUL
);
243 static int __init
check_nmi_watchdog(void)
245 unsigned int *prev_nmi_count
;
248 if ((nmi_watchdog
== NMI_NONE
) || (nmi_watchdog
== NMI_DEFAULT
))
251 if (!atomic_read(&nmi_active
))
254 prev_nmi_count
= kmalloc(NR_CPUS
* sizeof(int), GFP_KERNEL
);
258 printk(KERN_INFO
"Testing NMI watchdog ... ");
260 if (nmi_watchdog
== NMI_LOCAL_APIC
)
261 smp_call_function(nmi_cpu_busy
, (void *)&endflag
, 0, 0);
263 for_each_possible_cpu(cpu
)
264 prev_nmi_count
[cpu
] = per_cpu(irq_stat
, cpu
).__nmi_count
;
266 mdelay((10*1000)/nmi_hz
); // wait 10 ticks
268 for_each_possible_cpu(cpu
) {
270 /* Check cpu_callin_map here because that is set
271 after the timer is started. */
272 if (!cpu_isset(cpu
, cpu_callin_map
))
275 if (!per_cpu(nmi_watchdog_ctlblk
, cpu
).enabled
)
277 if (nmi_count(cpu
) - prev_nmi_count
[cpu
] <= 5) {
278 printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n",
282 per_cpu(nmi_watchdog_ctlblk
, cpu
).enabled
= 0;
283 atomic_dec(&nmi_active
);
286 if (!atomic_read(&nmi_active
)) {
287 kfree(prev_nmi_count
);
288 atomic_set(&nmi_active
, -1);
294 /* now that we know it works we can reduce NMI frequency to
295 something more reasonable; makes a difference in some configs */
296 if (nmi_watchdog
== NMI_LOCAL_APIC
) {
297 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
301 if (wd
->perfctr_msr
== MSR_P6_PERFCTR0
||
302 wd
->perfctr_msr
== MSR_ARCH_PERFMON_PERFCTR0
) {
303 nmi_hz
= adjust_for_32bit_ctr(nmi_hz
);
307 kfree(prev_nmi_count
);
310 /* This needs to happen later in boot so counters are working */
311 late_initcall(check_nmi_watchdog
);
313 static int __init
setup_nmi_watchdog(char *str
)
317 get_option(&str
, &nmi
);
319 if ((nmi
>= NMI_INVALID
) || (nmi
< NMI_NONE
))
326 __setup("nmi_watchdog=", setup_nmi_watchdog
);
328 static void disable_lapic_nmi_watchdog(void)
330 BUG_ON(nmi_watchdog
!= NMI_LOCAL_APIC
);
332 if (atomic_read(&nmi_active
) <= 0)
335 on_each_cpu(stop_apic_nmi_watchdog
, NULL
, 0, 1);
337 BUG_ON(atomic_read(&nmi_active
) != 0);
340 static void enable_lapic_nmi_watchdog(void)
342 BUG_ON(nmi_watchdog
!= NMI_LOCAL_APIC
);
344 /* are we already enabled */
345 if (atomic_read(&nmi_active
) != 0)
348 /* are we lapic aware */
349 if (nmi_known_cpu() <= 0)
352 on_each_cpu(setup_apic_nmi_watchdog
, NULL
, 0, 1);
353 touch_nmi_watchdog();
356 void disable_timer_nmi_watchdog(void)
358 BUG_ON(nmi_watchdog
!= NMI_IO_APIC
);
360 if (atomic_read(&nmi_active
) <= 0)
364 on_each_cpu(stop_apic_nmi_watchdog
, NULL
, 0, 1);
366 BUG_ON(atomic_read(&nmi_active
) != 0);
369 void enable_timer_nmi_watchdog(void)
371 BUG_ON(nmi_watchdog
!= NMI_IO_APIC
);
373 if (atomic_read(&nmi_active
) == 0) {
374 touch_nmi_watchdog();
375 on_each_cpu(setup_apic_nmi_watchdog
, NULL
, 0, 1);
380 static void __acpi_nmi_disable(void *__unused
)
382 apic_write_around(APIC_LVT0
, APIC_DM_NMI
| APIC_LVT_MASKED
);
386 * Disable timer based NMIs on all CPUs:
388 void acpi_nmi_disable(void)
390 if (atomic_read(&nmi_active
) && nmi_watchdog
== NMI_IO_APIC
)
391 on_each_cpu(__acpi_nmi_disable
, NULL
, 0, 1);
394 static void __acpi_nmi_enable(void *__unused
)
396 apic_write_around(APIC_LVT0
, APIC_DM_NMI
);
400 * Enable timer based NMIs on all CPUs:
402 void acpi_nmi_enable(void)
404 if (atomic_read(&nmi_active
) && nmi_watchdog
== NMI_IO_APIC
)
405 on_each_cpu(__acpi_nmi_enable
, NULL
, 0, 1);
410 static int nmi_pm_active
; /* nmi_active before suspend */
412 static int lapic_nmi_suspend(struct sys_device
*dev
, pm_message_t state
)
414 /* only CPU0 goes here, other CPUs should be offline */
415 nmi_pm_active
= atomic_read(&nmi_active
);
416 stop_apic_nmi_watchdog(NULL
);
417 BUG_ON(atomic_read(&nmi_active
) != 0);
421 static int lapic_nmi_resume(struct sys_device
*dev
)
423 /* only CPU0 goes here, other CPUs should be offline */
424 if (nmi_pm_active
> 0) {
425 setup_apic_nmi_watchdog(NULL
);
426 touch_nmi_watchdog();
432 static struct sysdev_class nmi_sysclass
= {
433 set_kset_name("lapic_nmi"),
434 .resume
= lapic_nmi_resume
,
435 .suspend
= lapic_nmi_suspend
,
438 static struct sys_device device_lapic_nmi
= {
440 .cls
= &nmi_sysclass
,
443 static int __init
init_lapic_nmi_sysfs(void)
447 /* should really be a BUG_ON but b/c this is an
448 * init call, it just doesn't work. -dcz
450 if (nmi_watchdog
!= NMI_LOCAL_APIC
)
453 if ( atomic_read(&nmi_active
) < 0 )
456 error
= sysdev_class_register(&nmi_sysclass
);
458 error
= sysdev_register(&device_lapic_nmi
);
461 /* must come after the local APIC's device_initcall() */
462 late_initcall(init_lapic_nmi_sysfs
);
464 #endif /* CONFIG_PM */
467 * Activate the NMI watchdog via the local APIC.
468 * Original code written by Keith Owens.
471 static void write_watchdog_counter(unsigned int perfctr_msr
, const char *descr
)
473 u64 count
= (u64
)cpu_khz
* 1000;
475 do_div(count
, nmi_hz
);
477 Dprintk("setting %s to -0x%08Lx\n", descr
, count
);
478 wrmsrl(perfctr_msr
, 0 - count
);
481 static void write_watchdog_counter32(unsigned int perfctr_msr
,
484 u64 count
= (u64
)cpu_khz
* 1000;
486 do_div(count
, nmi_hz
);
488 Dprintk("setting %s to -0x%08Lx\n", descr
, count
);
489 wrmsr(perfctr_msr
, (u32
)(-count
), 0);
492 /* Note that these events don't tick when the CPU idles. This means
493 the frequency varies with CPU load. */
495 #define K7_EVNTSEL_ENABLE (1 << 22)
496 #define K7_EVNTSEL_INT (1 << 20)
497 #define K7_EVNTSEL_OS (1 << 17)
498 #define K7_EVNTSEL_USR (1 << 16)
499 #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING 0x76
500 #define K7_NMI_EVENT K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING
502 static int setup_k7_watchdog(void)
504 unsigned int perfctr_msr
, evntsel_msr
;
505 unsigned int evntsel
;
506 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
508 perfctr_msr
= MSR_K7_PERFCTR0
;
509 evntsel_msr
= MSR_K7_EVNTSEL0
;
510 if (!reserve_perfctr_nmi(perfctr_msr
))
513 if (!reserve_evntsel_nmi(evntsel_msr
))
516 wrmsrl(perfctr_msr
, 0UL);
518 evntsel
= K7_EVNTSEL_INT
523 /* setup the timer */
524 wrmsr(evntsel_msr
, evntsel
, 0);
525 write_watchdog_counter(perfctr_msr
, "K7_PERFCTR0");
526 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
527 evntsel
|= K7_EVNTSEL_ENABLE
;
528 wrmsr(evntsel_msr
, evntsel
, 0);
530 wd
->perfctr_msr
= perfctr_msr
;
531 wd
->evntsel_msr
= evntsel_msr
;
532 wd
->cccr_msr
= 0; //unused
533 wd
->check_bit
= 1ULL<<63;
536 release_perfctr_nmi(perfctr_msr
);
541 static void stop_k7_watchdog(void)
543 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
545 wrmsr(wd
->evntsel_msr
, 0, 0);
547 release_evntsel_nmi(wd
->evntsel_msr
);
548 release_perfctr_nmi(wd
->perfctr_msr
);
551 #define P6_EVNTSEL0_ENABLE (1 << 22)
552 #define P6_EVNTSEL_INT (1 << 20)
553 #define P6_EVNTSEL_OS (1 << 17)
554 #define P6_EVNTSEL_USR (1 << 16)
555 #define P6_EVENT_CPU_CLOCKS_NOT_HALTED 0x79
556 #define P6_NMI_EVENT P6_EVENT_CPU_CLOCKS_NOT_HALTED
558 static int setup_p6_watchdog(void)
560 unsigned int perfctr_msr
, evntsel_msr
;
561 unsigned int evntsel
;
562 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
564 perfctr_msr
= MSR_P6_PERFCTR0
;
565 evntsel_msr
= MSR_P6_EVNTSEL0
;
566 if (!reserve_perfctr_nmi(perfctr_msr
))
569 if (!reserve_evntsel_nmi(evntsel_msr
))
572 wrmsrl(perfctr_msr
, 0UL);
574 evntsel
= P6_EVNTSEL_INT
579 /* setup the timer */
580 wrmsr(evntsel_msr
, evntsel
, 0);
581 nmi_hz
= adjust_for_32bit_ctr(nmi_hz
);
582 write_watchdog_counter32(perfctr_msr
, "P6_PERFCTR0");
583 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
584 evntsel
|= P6_EVNTSEL0_ENABLE
;
585 wrmsr(evntsel_msr
, evntsel
, 0);
587 wd
->perfctr_msr
= perfctr_msr
;
588 wd
->evntsel_msr
= evntsel_msr
;
589 wd
->cccr_msr
= 0; //unused
590 wd
->check_bit
= 1ULL<<39;
593 release_perfctr_nmi(perfctr_msr
);
598 static void stop_p6_watchdog(void)
600 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
602 wrmsr(wd
->evntsel_msr
, 0, 0);
604 release_evntsel_nmi(wd
->evntsel_msr
);
605 release_perfctr_nmi(wd
->perfctr_msr
);
608 /* Note that these events don't tick when the CPU idles. This means
609 the frequency varies with CPU load. */
611 #define MSR_P4_MISC_ENABLE_PERF_AVAIL (1<<7)
612 #define P4_ESCR_EVENT_SELECT(N) ((N)<<25)
613 #define P4_ESCR_OS (1<<3)
614 #define P4_ESCR_USR (1<<2)
615 #define P4_CCCR_OVF_PMI0 (1<<26)
616 #define P4_CCCR_OVF_PMI1 (1<<27)
617 #define P4_CCCR_THRESHOLD(N) ((N)<<20)
618 #define P4_CCCR_COMPLEMENT (1<<19)
619 #define P4_CCCR_COMPARE (1<<18)
620 #define P4_CCCR_REQUIRED (3<<16)
621 #define P4_CCCR_ESCR_SELECT(N) ((N)<<13)
622 #define P4_CCCR_ENABLE (1<<12)
623 #define P4_CCCR_OVF (1<<31)
624 /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
625 CRU_ESCR0 (with any non-null event selector) through a complemented
626 max threshold. [IA32-Vol3, Section 14.9.9] */
628 static int setup_p4_watchdog(void)
630 unsigned int perfctr_msr
, evntsel_msr
, cccr_msr
;
631 unsigned int evntsel
, cccr_val
;
632 unsigned int misc_enable
, dummy
;
634 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
636 rdmsr(MSR_IA32_MISC_ENABLE
, misc_enable
, dummy
);
637 if (!(misc_enable
& MSR_P4_MISC_ENABLE_PERF_AVAIL
))
641 /* detect which hyperthread we are on */
642 if (smp_num_siblings
== 2) {
643 unsigned int ebx
, apicid
;
646 apicid
= (ebx
>> 24) & 0xff;
652 /* performance counters are shared resources
653 * assign each hyperthread its own set
654 * (re-use the ESCR0 register, seems safe
655 * and keeps the cccr_val the same)
659 perfctr_msr
= MSR_P4_IQ_PERFCTR0
;
660 evntsel_msr
= MSR_P4_CRU_ESCR0
;
661 cccr_msr
= MSR_P4_IQ_CCCR0
;
662 cccr_val
= P4_CCCR_OVF_PMI0
| P4_CCCR_ESCR_SELECT(4);
665 perfctr_msr
= MSR_P4_IQ_PERFCTR1
;
666 evntsel_msr
= MSR_P4_CRU_ESCR0
;
667 cccr_msr
= MSR_P4_IQ_CCCR1
;
668 cccr_val
= P4_CCCR_OVF_PMI1
| P4_CCCR_ESCR_SELECT(4);
671 if (!reserve_perfctr_nmi(perfctr_msr
))
674 if (!reserve_evntsel_nmi(evntsel_msr
))
677 evntsel
= P4_ESCR_EVENT_SELECT(0x3F)
681 cccr_val
|= P4_CCCR_THRESHOLD(15)
686 wrmsr(evntsel_msr
, evntsel
, 0);
687 wrmsr(cccr_msr
, cccr_val
, 0);
688 write_watchdog_counter(perfctr_msr
, "P4_IQ_COUNTER0");
689 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
690 cccr_val
|= P4_CCCR_ENABLE
;
691 wrmsr(cccr_msr
, cccr_val
, 0);
692 wd
->perfctr_msr
= perfctr_msr
;
693 wd
->evntsel_msr
= evntsel_msr
;
694 wd
->cccr_msr
= cccr_msr
;
695 wd
->check_bit
= 1ULL<<39;
698 release_perfctr_nmi(perfctr_msr
);
703 static void stop_p4_watchdog(void)
705 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
707 wrmsr(wd
->cccr_msr
, 0, 0);
708 wrmsr(wd
->evntsel_msr
, 0, 0);
710 release_evntsel_nmi(wd
->evntsel_msr
);
711 release_perfctr_nmi(wd
->perfctr_msr
);
714 #define ARCH_PERFMON_NMI_EVENT_SEL ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
715 #define ARCH_PERFMON_NMI_EVENT_UMASK ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK
717 static int setup_intel_arch_watchdog(void)
720 union cpuid10_eax eax
;
722 unsigned int perfctr_msr
, evntsel_msr
;
723 unsigned int evntsel
;
724 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
727 * Check whether the Architectural PerfMon supports
728 * Unhalted Core Cycles Event or not.
729 * NOTE: Corresponding bit = 0 in ebx indicates event present.
731 cpuid(10, &(eax
.full
), &ebx
, &unused
, &unused
);
732 if ((eax
.split
.mask_length
< (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX
+1)) ||
733 (ebx
& ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT
))
736 perfctr_msr
= MSR_ARCH_PERFMON_PERFCTR0
;
737 evntsel_msr
= MSR_ARCH_PERFMON_EVENTSEL0
;
739 if (!reserve_perfctr_nmi(perfctr_msr
))
742 if (!reserve_evntsel_nmi(evntsel_msr
))
745 wrmsrl(perfctr_msr
, 0UL);
747 evntsel
= ARCH_PERFMON_EVENTSEL_INT
748 | ARCH_PERFMON_EVENTSEL_OS
749 | ARCH_PERFMON_EVENTSEL_USR
750 | ARCH_PERFMON_NMI_EVENT_SEL
751 | ARCH_PERFMON_NMI_EVENT_UMASK
;
753 /* setup the timer */
754 wrmsr(evntsel_msr
, evntsel
, 0);
755 nmi_hz
= adjust_for_32bit_ctr(nmi_hz
);
756 write_watchdog_counter32(perfctr_msr
, "INTEL_ARCH_PERFCTR0");
757 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
758 evntsel
|= ARCH_PERFMON_EVENTSEL0_ENABLE
;
759 wrmsr(evntsel_msr
, evntsel
, 0);
761 wd
->perfctr_msr
= perfctr_msr
;
762 wd
->evntsel_msr
= evntsel_msr
;
763 wd
->cccr_msr
= 0; //unused
764 wd
->check_bit
= 1ULL << (eax
.split
.bit_width
- 1);
767 release_perfctr_nmi(perfctr_msr
);
772 static void stop_intel_arch_watchdog(void)
775 union cpuid10_eax eax
;
777 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
780 * Check whether the Architectural PerfMon supports
781 * Unhalted Core Cycles Event or not.
782 * NOTE: Corresponding bit = 0 in ebx indicates event present.
784 cpuid(10, &(eax
.full
), &ebx
, &unused
, &unused
);
785 if ((eax
.split
.mask_length
< (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX
+1)) ||
786 (ebx
& ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT
))
789 wrmsr(wd
->evntsel_msr
, 0, 0);
790 release_evntsel_nmi(wd
->evntsel_msr
);
791 release_perfctr_nmi(wd
->perfctr_msr
);
794 void setup_apic_nmi_watchdog (void *unused
)
796 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
798 /* only support LOCAL and IO APICs for now */
799 if ((nmi_watchdog
!= NMI_LOCAL_APIC
) &&
800 (nmi_watchdog
!= NMI_IO_APIC
))
803 if (wd
->enabled
== 1)
806 /* cheap hack to support suspend/resume */
807 /* if cpu0 is not active neither should the other cpus */
808 if ((smp_processor_id() != 0) && (atomic_read(&nmi_active
) <= 0))
811 if (nmi_watchdog
== NMI_LOCAL_APIC
) {
812 switch (boot_cpu_data
.x86_vendor
) {
814 if (boot_cpu_data
.x86
!= 6 && boot_cpu_data
.x86
!= 15 &&
815 boot_cpu_data
.x86
!= 16)
817 if (!setup_k7_watchdog())
820 case X86_VENDOR_INTEL
:
821 if (cpu_has(&boot_cpu_data
, X86_FEATURE_ARCH_PERFMON
)) {
822 if (!setup_intel_arch_watchdog())
826 switch (boot_cpu_data
.x86
) {
828 if (boot_cpu_data
.x86_model
> 0xd)
831 if (!setup_p6_watchdog())
835 if (boot_cpu_data
.x86_model
> 0x4)
838 if (!setup_p4_watchdog())
850 atomic_inc(&nmi_active
);
853 void stop_apic_nmi_watchdog(void *unused
)
855 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
857 /* only support LOCAL and IO APICs for now */
858 if ((nmi_watchdog
!= NMI_LOCAL_APIC
) &&
859 (nmi_watchdog
!= NMI_IO_APIC
))
862 if (wd
->enabled
== 0)
865 if (nmi_watchdog
== NMI_LOCAL_APIC
) {
866 switch (boot_cpu_data
.x86_vendor
) {
870 case X86_VENDOR_INTEL
:
871 if (cpu_has(&boot_cpu_data
, X86_FEATURE_ARCH_PERFMON
)) {
872 stop_intel_arch_watchdog();
875 switch (boot_cpu_data
.x86
) {
877 if (boot_cpu_data
.x86_model
> 0xd)
882 if (boot_cpu_data
.x86_model
> 0x4)
893 atomic_dec(&nmi_active
);
897 * the best way to detect whether a CPU has a 'hard lockup' problem
898 * is to check it's local APIC timer IRQ counts. If they are not
899 * changing then that CPU has some problem.
901 * as these watchdog NMI IRQs are generated on every CPU, we only
902 * have to check the current processor.
904 * since NMIs don't listen to _any_ locks, we have to be extremely
905 * careful not to rely on unsafe variables. The printk might lock
906 * up though, so we have to break up any console locks first ...
907 * [when there will be more tty-related locks, break them up
912 last_irq_sums
[NR_CPUS
],
913 alert_counter
[NR_CPUS
];
915 void touch_nmi_watchdog (void)
917 if (nmi_watchdog
> 0) {
921 * Just reset the alert counters, (other CPUs might be
922 * spinning on locks we hold):
924 for_each_present_cpu (cpu
)
925 alert_counter
[cpu
] = 0;
929 * Tickle the softlockup detector too:
931 touch_softlockup_watchdog();
933 EXPORT_SYMBOL(touch_nmi_watchdog
);
935 extern void die_nmi(struct pt_regs
*, const char *msg
);
937 __kprobes
int nmi_watchdog_tick(struct pt_regs
* regs
, unsigned reason
)
941 * Since current_thread_info()-> is always on the stack, and we
942 * always switch the stack NMI-atomically, it's safe to use
943 * smp_processor_id().
947 int cpu
= smp_processor_id();
948 struct nmi_watchdog_ctlblk
*wd
= &__get_cpu_var(nmi_watchdog_ctlblk
);
952 /* check for other users first */
953 if (notify_die(DIE_NMI
, "nmi", regs
, reason
, 2, SIGINT
)
959 if (cpu_isset(cpu
, backtrace_mask
)) {
960 static DEFINE_SPINLOCK(lock
); /* Serialise the printks */
963 printk("NMI backtrace for cpu %d\n", cpu
);
966 cpu_clear(cpu
, backtrace_mask
);
970 * Take the local apic timer and PIT/HPET into account. We don't
971 * know which one is active, when we have highres/dyntick on
973 sum
= per_cpu(irq_stat
, cpu
).apic_timer_irqs
+ kstat_irqs(0);
975 /* if the none of the timers isn't firing, this cpu isn't doing much */
976 if (!touched
&& last_irq_sums
[cpu
] == sum
) {
978 * Ayiee, looks like this CPU is stuck ...
979 * wait a few IRQs (5 seconds) before doing the oops ...
981 alert_counter
[cpu
]++;
982 if (alert_counter
[cpu
] == 5*nmi_hz
)
984 * die_nmi will return ONLY if NOTIFY_STOP happens..
986 die_nmi(regs
, "BUG: NMI Watchdog detected LOCKUP");
988 last_irq_sums
[cpu
] = sum
;
989 alert_counter
[cpu
] = 0;
991 /* see if the nmi watchdog went off */
993 if (nmi_watchdog
== NMI_LOCAL_APIC
) {
994 rdmsrl(wd
->perfctr_msr
, dummy
);
995 if (dummy
& wd
->check_bit
){
996 /* this wasn't a watchdog timer interrupt */
1000 /* only Intel P4 uses the cccr msr */
1001 if (wd
->cccr_msr
!= 0) {
1004 * - An overflown perfctr will assert its interrupt
1005 * until the OVF flag in its CCCR is cleared.
1006 * - LVTPC is masked on interrupt and must be
1007 * unmasked by the LVTPC handler.
1009 rdmsrl(wd
->cccr_msr
, dummy
);
1010 dummy
&= ~P4_CCCR_OVF
;
1011 wrmsrl(wd
->cccr_msr
, dummy
);
1012 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
1013 /* start the cycle over again */
1014 write_watchdog_counter(wd
->perfctr_msr
, NULL
);
1016 else if (wd
->perfctr_msr
== MSR_P6_PERFCTR0
||
1017 wd
->perfctr_msr
== MSR_ARCH_PERFMON_PERFCTR0
) {
1018 /* P6 based Pentium M need to re-unmask
1019 * the apic vector but it doesn't hurt
1021 * ArchPerfom/Core Duo also needs this */
1022 apic_write(APIC_LVTPC
, APIC_DM_NMI
);
1023 /* P6/ARCH_PERFMON has 32 bit counter write */
1024 write_watchdog_counter32(wd
->perfctr_msr
, NULL
);
1026 /* start the cycle over again */
1027 write_watchdog_counter(wd
->perfctr_msr
, NULL
);
1030 } else if (nmi_watchdog
== NMI_IO_APIC
) {
1031 /* don't know how to accurately check for this.
1032 * just assume it was a watchdog timer interrupt
1033 * This matches the old behaviour.
1042 int do_nmi_callback(struct pt_regs
* regs
, int cpu
)
1044 #ifdef CONFIG_SYSCTL
1045 if (unknown_nmi_panic
)
1046 return unknown_nmi_panic_callback(regs
, cpu
);
1051 #ifdef CONFIG_SYSCTL
1053 static int unknown_nmi_panic_callback(struct pt_regs
*regs
, int cpu
)
1055 unsigned char reason
= get_nmi_reason();
1058 sprintf(buf
, "NMI received for unknown reason %02x\n", reason
);
1064 * proc handler for /proc/sys/kernel/nmi
1066 int proc_nmi_enabled(struct ctl_table
*table
, int write
, struct file
*file
,
1067 void __user
*buffer
, size_t *length
, loff_t
*ppos
)
1071 nmi_watchdog_enabled
= (atomic_read(&nmi_active
) > 0) ? 1 : 0;
1072 old_state
= nmi_watchdog_enabled
;
1073 proc_dointvec(table
, write
, file
, buffer
, length
, ppos
);
1074 if (!!old_state
== !!nmi_watchdog_enabled
)
1077 if (atomic_read(&nmi_active
) < 0) {
1078 printk( KERN_WARNING
"NMI watchdog is permanently disabled\n");
1082 if (nmi_watchdog
== NMI_DEFAULT
) {
1083 if (nmi_known_cpu() > 0)
1084 nmi_watchdog
= NMI_LOCAL_APIC
;
1086 nmi_watchdog
= NMI_IO_APIC
;
1089 if (nmi_watchdog
== NMI_LOCAL_APIC
) {
1090 if (nmi_watchdog_enabled
)
1091 enable_lapic_nmi_watchdog();
1093 disable_lapic_nmi_watchdog();
1095 printk( KERN_WARNING
1096 "NMI watchdog doesn't know what hardware to touch\n");
1104 void __trigger_all_cpu_backtrace(void)
1108 backtrace_mask
= cpu_online_map
;
1109 /* Wait for up to 10 seconds for all CPUs to do the backtrace */
1110 for (i
= 0; i
< 10 * 1000; i
++) {
1111 if (cpus_empty(backtrace_mask
))
1117 EXPORT_SYMBOL(nmi_active
);
1118 EXPORT_SYMBOL(nmi_watchdog
);
1119 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi
);
1120 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit
);
1121 EXPORT_SYMBOL(reserve_perfctr_nmi
);
1122 EXPORT_SYMBOL(release_perfctr_nmi
);
1123 EXPORT_SYMBOL(reserve_evntsel_nmi
);
1124 EXPORT_SYMBOL(release_evntsel_nmi
);
1125 EXPORT_SYMBOL(disable_timer_nmi_watchdog
);
1126 EXPORT_SYMBOL(enable_timer_nmi_watchdog
);