1 /* Performance event support for sparc64.
3 * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net>
5 * This code is based almost entirely upon the x86 perf event
8 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
9 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
10 * Copyright (C) 2009 Jaswinder Singh Rajput
11 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
12 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
15 #include <linux/perf_event.h>
16 #include <linux/kprobes.h>
17 #include <linux/ftrace.h>
18 #include <linux/kernel.h>
19 #include <linux/kdebug.h>
20 #include <linux/mutex.h>
22 #include <asm/stacktrace.h>
23 #include <asm/cpudata.h>
24 #include <asm/uaccess.h>
25 #include <asm/atomic.h>
32 /* Sparc64 chips have two performance counters, 32-bits each, with
33 * overflow interrupts generated on transition from 0xffffffff to 0.
34 * The counters are accessed in one go using a 64-bit register.
36 * Both counters are controlled using a single control register. The
37 * only way to stop all sampling is to clear all of the context (user,
38 * supervisor, hypervisor) sampling enable bits. But these bits apply
39 * to both counters, thus the two counters can't be enabled/disabled
42 * The control register has two event fields, one for each of the two
43 * counters. It's thus nearly impossible to have one counter going
44 * while keeping the other one stopped. Therefore it is possible to
45 * get overflow interrupts for counters not currently "in use" and
46 * that condition must be checked in the overflow interrupt handler.
48 * So we use a hack, in that we program inactive counters with the
49 * "sw_count0" and "sw_count1" events. These count how many times
50 * the instruction "sethi %hi(0xfc000), %g0" is executed. It's an
51 * unusual way to encode a NOP and therefore will not trigger in
55 #define MAX_HWEVENTS 2
56 #define MAX_PERIOD ((1UL << 32) - 1)
58 #define PIC_UPPER_INDEX 0
59 #define PIC_LOWER_INDEX 1
60 #define PIC_NO_INDEX -1
62 struct cpu_hw_events
{
63 /* Number of events currently scheduled onto this cpu.
64 * This tells how many entries in the arrays below
69 /* Number of new events added since the last hw_perf_disable().
70 * This works because the perf event layer always adds new
71 * events inside of a perf_{disable,enable}() sequence.
75 /* Array of events current scheduled on this cpu. */
76 struct perf_event
*event
[MAX_HWEVENTS
];
78 /* Array of encoded longs, specifying the %pcr register
79 * encoding and the mask of PIC counters this even can
80 * be scheduled on. See perf_event_encode() et al.
82 unsigned long events
[MAX_HWEVENTS
];
84 /* The current counter index assigned to an event. When the
85 * event hasn't been programmed into the cpu yet, this will
86 * hold PIC_NO_INDEX. The event->hw.idx value tells us where
87 * we ought to schedule the event.
89 int current_idx
[MAX_HWEVENTS
];
91 /* Software copy of %pcr register on this cpu. */
94 /* Enabled/disable state. */
97 unsigned int group_flag
;
99 DEFINE_PER_CPU(struct cpu_hw_events
, cpu_hw_events
) = { .enabled
= 1, };
101 /* An event map describes the characteristics of a performance
102 * counter event. In particular it gives the encoding as well as
103 * a mask telling which counters the event can be measured on.
105 struct perf_event_map
{
108 #define PIC_NONE 0x00
109 #define PIC_UPPER 0x01
110 #define PIC_LOWER 0x02
113 /* Encode a perf_event_map entry into a long. */
114 static unsigned long perf_event_encode(const struct perf_event_map
*pmap
)
116 return ((unsigned long) pmap
->encoding
<< 16) | pmap
->pic_mask
;
119 static u8
perf_event_get_msk(unsigned long val
)
124 static u64
perf_event_get_enc(unsigned long val
)
129 #define C(x) PERF_COUNT_HW_CACHE_##x
131 #define CACHE_OP_UNSUPPORTED 0xfffe
132 #define CACHE_OP_NONSENSE 0xffff
134 typedef struct perf_event_map cache_map_t
135 [PERF_COUNT_HW_CACHE_MAX
]
136 [PERF_COUNT_HW_CACHE_OP_MAX
]
137 [PERF_COUNT_HW_CACHE_RESULT_MAX
];
140 const struct perf_event_map
*(*event_map
)(int);
141 const cache_map_t
*cache_map
;
152 static const struct perf_event_map ultra3_perfmon_event_map
[] = {
153 [PERF_COUNT_HW_CPU_CYCLES
] = { 0x0000, PIC_UPPER
| PIC_LOWER
},
154 [PERF_COUNT_HW_INSTRUCTIONS
] = { 0x0001, PIC_UPPER
| PIC_LOWER
},
155 [PERF_COUNT_HW_CACHE_REFERENCES
] = { 0x0009, PIC_LOWER
},
156 [PERF_COUNT_HW_CACHE_MISSES
] = { 0x0009, PIC_UPPER
},
159 static const struct perf_event_map
*ultra3_event_map(int event_id
)
161 return &ultra3_perfmon_event_map
[event_id
];
164 static const cache_map_t ultra3_cache_map
= {
167 [C(RESULT_ACCESS
)] = { 0x09, PIC_LOWER
, },
168 [C(RESULT_MISS
)] = { 0x09, PIC_UPPER
, },
171 [C(RESULT_ACCESS
)] = { 0x0a, PIC_LOWER
},
172 [C(RESULT_MISS
)] = { 0x0a, PIC_UPPER
},
175 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
176 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
181 [C(RESULT_ACCESS
)] = { 0x09, PIC_LOWER
, },
182 [C(RESULT_MISS
)] = { 0x09, PIC_UPPER
, },
185 [ C(RESULT_ACCESS
) ] = { CACHE_OP_NONSENSE
},
186 [ C(RESULT_MISS
) ] = { CACHE_OP_NONSENSE
},
188 [ C(OP_PREFETCH
) ] = {
189 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
190 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
195 [C(RESULT_ACCESS
)] = { 0x0c, PIC_LOWER
, },
196 [C(RESULT_MISS
)] = { 0x0c, PIC_UPPER
, },
199 [C(RESULT_ACCESS
)] = { 0x0c, PIC_LOWER
},
200 [C(RESULT_MISS
)] = { 0x0c, PIC_UPPER
},
203 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
204 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
209 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
210 [C(RESULT_MISS
)] = { 0x12, PIC_UPPER
, },
213 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
214 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
216 [ C(OP_PREFETCH
) ] = {
217 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
218 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
223 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
224 [C(RESULT_MISS
)] = { 0x11, PIC_UPPER
, },
227 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
228 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
230 [ C(OP_PREFETCH
) ] = {
231 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
232 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
237 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
238 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
241 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
242 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
244 [ C(OP_PREFETCH
) ] = {
245 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
246 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
251 static const struct sparc_pmu ultra3_pmu
= {
252 .event_map
= ultra3_event_map
,
253 .cache_map
= &ultra3_cache_map
,
254 .max_events
= ARRAY_SIZE(ultra3_perfmon_event_map
),
262 /* Niagara1 is very limited. The upper PIC is hard-locked to count
263 * only instructions, so it is free running which creates all kinds of
264 * problems. Some hardware designs make one wonder if the creator
265 * even looked at how this stuff gets used by software.
267 static const struct perf_event_map niagara1_perfmon_event_map
[] = {
268 [PERF_COUNT_HW_CPU_CYCLES
] = { 0x00, PIC_UPPER
},
269 [PERF_COUNT_HW_INSTRUCTIONS
] = { 0x00, PIC_UPPER
},
270 [PERF_COUNT_HW_CACHE_REFERENCES
] = { 0, PIC_NONE
},
271 [PERF_COUNT_HW_CACHE_MISSES
] = { 0x03, PIC_LOWER
},
274 static const struct perf_event_map
*niagara1_event_map(int event_id
)
276 return &niagara1_perfmon_event_map
[event_id
];
279 static const cache_map_t niagara1_cache_map
= {
282 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
283 [C(RESULT_MISS
)] = { 0x03, PIC_LOWER
, },
286 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
287 [C(RESULT_MISS
)] = { 0x03, PIC_LOWER
, },
290 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
291 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
296 [C(RESULT_ACCESS
)] = { 0x00, PIC_UPPER
},
297 [C(RESULT_MISS
)] = { 0x02, PIC_LOWER
, },
300 [ C(RESULT_ACCESS
) ] = { CACHE_OP_NONSENSE
},
301 [ C(RESULT_MISS
) ] = { CACHE_OP_NONSENSE
},
303 [ C(OP_PREFETCH
) ] = {
304 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
305 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
310 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
311 [C(RESULT_MISS
)] = { 0x07, PIC_LOWER
, },
314 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
315 [C(RESULT_MISS
)] = { 0x07, PIC_LOWER
, },
318 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
319 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
324 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
325 [C(RESULT_MISS
)] = { 0x05, PIC_LOWER
, },
328 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
329 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
331 [ C(OP_PREFETCH
) ] = {
332 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
333 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
338 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
339 [C(RESULT_MISS
)] = { 0x04, PIC_LOWER
, },
342 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
343 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
345 [ C(OP_PREFETCH
) ] = {
346 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
347 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
352 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
353 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
356 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
357 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
359 [ C(OP_PREFETCH
) ] = {
360 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
361 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
366 static const struct sparc_pmu niagara1_pmu
= {
367 .event_map
= niagara1_event_map
,
368 .cache_map
= &niagara1_cache_map
,
369 .max_events
= ARRAY_SIZE(niagara1_perfmon_event_map
),
377 static const struct perf_event_map niagara2_perfmon_event_map
[] = {
378 [PERF_COUNT_HW_CPU_CYCLES
] = { 0x02ff, PIC_UPPER
| PIC_LOWER
},
379 [PERF_COUNT_HW_INSTRUCTIONS
] = { 0x02ff, PIC_UPPER
| PIC_LOWER
},
380 [PERF_COUNT_HW_CACHE_REFERENCES
] = { 0x0208, PIC_UPPER
| PIC_LOWER
},
381 [PERF_COUNT_HW_CACHE_MISSES
] = { 0x0302, PIC_UPPER
| PIC_LOWER
},
382 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS
] = { 0x0201, PIC_UPPER
| PIC_LOWER
},
383 [PERF_COUNT_HW_BRANCH_MISSES
] = { 0x0202, PIC_UPPER
| PIC_LOWER
},
386 static const struct perf_event_map
*niagara2_event_map(int event_id
)
388 return &niagara2_perfmon_event_map
[event_id
];
391 static const cache_map_t niagara2_cache_map
= {
394 [C(RESULT_ACCESS
)] = { 0x0208, PIC_UPPER
| PIC_LOWER
, },
395 [C(RESULT_MISS
)] = { 0x0302, PIC_UPPER
| PIC_LOWER
, },
398 [C(RESULT_ACCESS
)] = { 0x0210, PIC_UPPER
| PIC_LOWER
, },
399 [C(RESULT_MISS
)] = { 0x0302, PIC_UPPER
| PIC_LOWER
, },
402 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
403 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
408 [C(RESULT_ACCESS
)] = { 0x02ff, PIC_UPPER
| PIC_LOWER
, },
409 [C(RESULT_MISS
)] = { 0x0301, PIC_UPPER
| PIC_LOWER
, },
412 [ C(RESULT_ACCESS
) ] = { CACHE_OP_NONSENSE
},
413 [ C(RESULT_MISS
) ] = { CACHE_OP_NONSENSE
},
415 [ C(OP_PREFETCH
) ] = {
416 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
417 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
422 [C(RESULT_ACCESS
)] = { 0x0208, PIC_UPPER
| PIC_LOWER
, },
423 [C(RESULT_MISS
)] = { 0x0330, PIC_UPPER
| PIC_LOWER
, },
426 [C(RESULT_ACCESS
)] = { 0x0210, PIC_UPPER
| PIC_LOWER
, },
427 [C(RESULT_MISS
)] = { 0x0320, PIC_UPPER
| PIC_LOWER
, },
430 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
431 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
436 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
437 [C(RESULT_MISS
)] = { 0x0b08, PIC_UPPER
| PIC_LOWER
, },
440 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
441 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
443 [ C(OP_PREFETCH
) ] = {
444 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
445 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
450 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
451 [C(RESULT_MISS
)] = { 0xb04, PIC_UPPER
| PIC_LOWER
, },
454 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
455 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
457 [ C(OP_PREFETCH
) ] = {
458 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
459 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
464 [C(RESULT_ACCESS
)] = { CACHE_OP_UNSUPPORTED
},
465 [C(RESULT_MISS
)] = { CACHE_OP_UNSUPPORTED
},
468 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
469 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
471 [ C(OP_PREFETCH
) ] = {
472 [ C(RESULT_ACCESS
) ] = { CACHE_OP_UNSUPPORTED
},
473 [ C(RESULT_MISS
) ] = { CACHE_OP_UNSUPPORTED
},
478 static const struct sparc_pmu niagara2_pmu
= {
479 .event_map
= niagara2_event_map
,
480 .cache_map
= &niagara2_cache_map
,
481 .max_events
= ARRAY_SIZE(niagara2_perfmon_event_map
),
491 static const struct sparc_pmu
*sparc_pmu __read_mostly
;
493 static u64
event_encoding(u64 event_id
, int idx
)
495 if (idx
== PIC_UPPER_INDEX
)
496 event_id
<<= sparc_pmu
->upper_shift
;
498 event_id
<<= sparc_pmu
->lower_shift
;
502 static u64
mask_for_index(int idx
)
504 return event_encoding(sparc_pmu
->event_mask
, idx
);
507 static u64
nop_for_index(int idx
)
509 return event_encoding(idx
== PIC_UPPER_INDEX
?
510 sparc_pmu
->upper_nop
:
511 sparc_pmu
->lower_nop
, idx
);
514 static inline void sparc_pmu_enable_event(struct cpu_hw_events
*cpuc
, struct hw_perf_event
*hwc
, int idx
)
516 u64 val
, mask
= mask_for_index(idx
);
523 pcr_ops
->write(cpuc
->pcr
);
526 static inline void sparc_pmu_disable_event(struct cpu_hw_events
*cpuc
, struct hw_perf_event
*hwc
, int idx
)
528 u64 mask
= mask_for_index(idx
);
529 u64 nop
= nop_for_index(idx
);
537 pcr_ops
->write(cpuc
->pcr
);
540 static u32
read_pmc(int idx
)
545 if (idx
== PIC_UPPER_INDEX
)
548 return val
& 0xffffffff;
551 static void write_pmc(int idx
, u64 val
)
553 u64 shift
, mask
, pic
;
556 if (idx
== PIC_UPPER_INDEX
)
559 mask
= ((u64
) 0xffffffff) << shift
;
568 static u64
sparc_perf_event_update(struct perf_event
*event
,
569 struct hw_perf_event
*hwc
, int idx
)
572 u64 prev_raw_count
, new_raw_count
;
576 prev_raw_count
= local64_read(&hwc
->prev_count
);
577 new_raw_count
= read_pmc(idx
);
579 if (local64_cmpxchg(&hwc
->prev_count
, prev_raw_count
,
580 new_raw_count
) != prev_raw_count
)
583 delta
= (new_raw_count
<< shift
) - (prev_raw_count
<< shift
);
586 local64_add(delta
, &event
->count
);
587 local64_sub(delta
, &hwc
->period_left
);
589 return new_raw_count
;
592 static int sparc_perf_event_set_period(struct perf_event
*event
,
593 struct hw_perf_event
*hwc
, int idx
)
595 s64 left
= local64_read(&hwc
->period_left
);
596 s64 period
= hwc
->sample_period
;
599 if (unlikely(left
<= -period
)) {
601 local64_set(&hwc
->period_left
, left
);
602 hwc
->last_period
= period
;
606 if (unlikely(left
<= 0)) {
608 local64_set(&hwc
->period_left
, left
);
609 hwc
->last_period
= period
;
612 if (left
> MAX_PERIOD
)
615 local64_set(&hwc
->prev_count
, (u64
)-left
);
617 write_pmc(idx
, (u64
)(-left
) & 0xffffffff);
619 perf_event_update_userpage(event
);
624 /* If performance event entries have been added, move existing
625 * events around (if necessary) and then assign new entries to
628 static u64
maybe_change_configuration(struct cpu_hw_events
*cpuc
, u64 pcr
)
635 /* Read in the counters which are moving. */
636 for (i
= 0; i
< cpuc
->n_events
; i
++) {
637 struct perf_event
*cp
= cpuc
->event
[i
];
639 if (cpuc
->current_idx
[i
] != PIC_NO_INDEX
&&
640 cpuc
->current_idx
[i
] != cp
->hw
.idx
) {
641 sparc_perf_event_update(cp
, &cp
->hw
,
642 cpuc
->current_idx
[i
]);
643 cpuc
->current_idx
[i
] = PIC_NO_INDEX
;
647 /* Assign to counters all unassigned events. */
648 for (i
= 0; i
< cpuc
->n_events
; i
++) {
649 struct perf_event
*cp
= cpuc
->event
[i
];
650 struct hw_perf_event
*hwc
= &cp
->hw
;
654 if (cpuc
->current_idx
[i
] != PIC_NO_INDEX
)
657 sparc_perf_event_set_period(cp
, hwc
, idx
);
658 cpuc
->current_idx
[i
] = idx
;
660 enc
= perf_event_get_enc(cpuc
->events
[i
]);
661 pcr
&= ~mask_for_index(idx
);
662 if (hwc
->state
& PERF_HES_STOPPED
)
663 pcr
|= nop_for_index(idx
);
665 pcr
|= event_encoding(enc
, idx
);
671 static void sparc_pmu_enable(struct pmu
*pmu
)
673 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
683 if (!cpuc
->n_events
) {
686 pcr
= maybe_change_configuration(cpuc
, pcr
);
688 /* We require that all of the events have the same
689 * configuration, so just fetch the settings from the
692 cpuc
->pcr
= pcr
| cpuc
->event
[0]->hw
.config_base
;
695 pcr_ops
->write(cpuc
->pcr
);
698 static void sparc_pmu_disable(struct pmu
*pmu
)
700 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
710 val
&= ~(PCR_UTRACE
| PCR_STRACE
|
711 sparc_pmu
->hv_bit
| sparc_pmu
->irq_bit
);
714 pcr_ops
->write(cpuc
->pcr
);
717 static int active_event_index(struct cpu_hw_events
*cpuc
,
718 struct perf_event
*event
)
722 for (i
= 0; i
< cpuc
->n_events
; i
++) {
723 if (cpuc
->event
[i
] == event
)
726 BUG_ON(i
== cpuc
->n_events
);
727 return cpuc
->current_idx
[i
];
730 static void sparc_pmu_start(struct perf_event
*event
, int flags
)
732 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
733 int idx
= active_event_index(cpuc
, event
);
735 if (flags
& PERF_EF_RELOAD
) {
736 WARN_ON_ONCE(!(event
->hw
.state
& PERF_HES_UPTODATE
));
737 sparc_perf_event_set_period(event
, &event
->hw
, idx
);
742 sparc_pmu_enable_event(cpuc
, &event
->hw
, idx
);
745 static void sparc_pmu_stop(struct perf_event
*event
, int flags
)
747 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
748 int idx
= active_event_index(cpuc
, event
);
750 if (!(event
->hw
.state
& PERF_HES_STOPPED
)) {
751 sparc_pmu_disable_event(cpuc
, &event
->hw
, idx
);
752 event
->hw
.state
|= PERF_HES_STOPPED
;
755 if (!(event
->hw
.state
& PERF_HES_UPTODATE
) && (flags
& PERF_EF_UPDATE
)) {
756 sparc_perf_event_update(event
, &event
->hw
, idx
);
757 event
->hw
.state
|= PERF_HES_UPTODATE
;
761 static void sparc_pmu_del(struct perf_event
*event
, int _flags
)
763 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
767 local_irq_save(flags
);
768 perf_pmu_disable(event
->pmu
);
770 for (i
= 0; i
< cpuc
->n_events
; i
++) {
771 if (event
== cpuc
->event
[i
]) {
772 /* Absorb the final count and turn off the
775 sparc_pmu_stop(event
, PERF_EF_UPDATE
);
777 /* Shift remaining entries down into
780 while (++i
< cpuc
->n_events
) {
781 cpuc
->event
[i
- 1] = cpuc
->event
[i
];
782 cpuc
->events
[i
- 1] = cpuc
->events
[i
];
783 cpuc
->current_idx
[i
- 1] =
784 cpuc
->current_idx
[i
];
787 perf_event_update_userpage(event
);
794 perf_pmu_enable(event
->pmu
);
795 local_irq_restore(flags
);
798 static void sparc_pmu_read(struct perf_event
*event
)
800 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
801 int idx
= active_event_index(cpuc
, event
);
802 struct hw_perf_event
*hwc
= &event
->hw
;
804 sparc_perf_event_update(event
, hwc
, idx
);
807 static atomic_t active_events
= ATOMIC_INIT(0);
808 static DEFINE_MUTEX(pmc_grab_mutex
);
810 static void perf_stop_nmi_watchdog(void *unused
)
812 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
814 stop_nmi_watchdog(NULL
);
815 cpuc
->pcr
= pcr_ops
->read();
818 void perf_event_grab_pmc(void)
820 if (atomic_inc_not_zero(&active_events
))
823 mutex_lock(&pmc_grab_mutex
);
824 if (atomic_read(&active_events
) == 0) {
825 if (atomic_read(&nmi_active
) > 0) {
826 on_each_cpu(perf_stop_nmi_watchdog
, NULL
, 1);
827 BUG_ON(atomic_read(&nmi_active
) != 0);
829 atomic_inc(&active_events
);
831 mutex_unlock(&pmc_grab_mutex
);
834 void perf_event_release_pmc(void)
836 if (atomic_dec_and_mutex_lock(&active_events
, &pmc_grab_mutex
)) {
837 if (atomic_read(&nmi_active
) == 0)
838 on_each_cpu(start_nmi_watchdog
, NULL
, 1);
839 mutex_unlock(&pmc_grab_mutex
);
843 static const struct perf_event_map
*sparc_map_cache_event(u64 config
)
845 unsigned int cache_type
, cache_op
, cache_result
;
846 const struct perf_event_map
*pmap
;
848 if (!sparc_pmu
->cache_map
)
849 return ERR_PTR(-ENOENT
);
851 cache_type
= (config
>> 0) & 0xff;
852 if (cache_type
>= PERF_COUNT_HW_CACHE_MAX
)
853 return ERR_PTR(-EINVAL
);
855 cache_op
= (config
>> 8) & 0xff;
856 if (cache_op
>= PERF_COUNT_HW_CACHE_OP_MAX
)
857 return ERR_PTR(-EINVAL
);
859 cache_result
= (config
>> 16) & 0xff;
860 if (cache_result
>= PERF_COUNT_HW_CACHE_RESULT_MAX
)
861 return ERR_PTR(-EINVAL
);
863 pmap
= &((*sparc_pmu
->cache_map
)[cache_type
][cache_op
][cache_result
]);
865 if (pmap
->encoding
== CACHE_OP_UNSUPPORTED
)
866 return ERR_PTR(-ENOENT
);
868 if (pmap
->encoding
== CACHE_OP_NONSENSE
)
869 return ERR_PTR(-EINVAL
);
874 static void hw_perf_event_destroy(struct perf_event
*event
)
876 perf_event_release_pmc();
879 /* Make sure all events can be scheduled into the hardware at
880 * the same time. This is simplified by the fact that we only
881 * need to support 2 simultaneous HW events.
883 * As a side effect, the evts[]->hw.idx values will be assigned
884 * on success. These are pending indexes. When the events are
885 * actually programmed into the chip, these values will propagate
886 * to the per-cpu cpuc->current_idx[] slots, see the code in
887 * maybe_change_configuration() for details.
889 static int sparc_check_constraints(struct perf_event
**evts
,
890 unsigned long *events
, int n_ev
)
892 u8 msk0
= 0, msk1
= 0;
895 /* This case is possible when we are invoked from
896 * hw_perf_group_sched_in().
901 if (n_ev
> MAX_HWEVENTS
)
904 msk0
= perf_event_get_msk(events
[0]);
906 if (msk0
& PIC_LOWER
)
911 msk1
= perf_event_get_msk(events
[1]);
913 /* If both events can go on any counter, OK. */
914 if (msk0
== (PIC_UPPER
| PIC_LOWER
) &&
915 msk1
== (PIC_UPPER
| PIC_LOWER
))
918 /* If one event is limited to a specific counter,
919 * and the other can go on both, OK.
921 if ((msk0
== PIC_UPPER
|| msk0
== PIC_LOWER
) &&
922 msk1
== (PIC_UPPER
| PIC_LOWER
)) {
923 if (msk0
& PIC_LOWER
)
928 if ((msk1
== PIC_UPPER
|| msk1
== PIC_LOWER
) &&
929 msk0
== (PIC_UPPER
| PIC_LOWER
)) {
930 if (msk1
& PIC_UPPER
)
935 /* If the events are fixed to different counters, OK. */
936 if ((msk0
== PIC_UPPER
&& msk1
== PIC_LOWER
) ||
937 (msk0
== PIC_LOWER
&& msk1
== PIC_UPPER
)) {
938 if (msk0
& PIC_LOWER
)
943 /* Otherwise, there is a conflict. */
947 evts
[0]->hw
.idx
= idx0
;
949 evts
[1]->hw
.idx
= idx0
^ 1;
953 static int check_excludes(struct perf_event
**evts
, int n_prev
, int n_new
)
955 int eu
= 0, ek
= 0, eh
= 0;
956 struct perf_event
*event
;
964 for (i
= 0; i
< n
; i
++) {
967 eu
= event
->attr
.exclude_user
;
968 ek
= event
->attr
.exclude_kernel
;
969 eh
= event
->attr
.exclude_hv
;
971 } else if (event
->attr
.exclude_user
!= eu
||
972 event
->attr
.exclude_kernel
!= ek
||
973 event
->attr
.exclude_hv
!= eh
) {
981 static int collect_events(struct perf_event
*group
, int max_count
,
982 struct perf_event
*evts
[], unsigned long *events
,
985 struct perf_event
*event
;
988 if (!is_software_event(group
)) {
992 events
[n
] = group
->hw
.event_base
;
993 current_idx
[n
++] = PIC_NO_INDEX
;
995 list_for_each_entry(event
, &group
->sibling_list
, group_entry
) {
996 if (!is_software_event(event
) &&
997 event
->state
!= PERF_EVENT_STATE_OFF
) {
1001 events
[n
] = event
->hw
.event_base
;
1002 current_idx
[n
++] = PIC_NO_INDEX
;
1008 static int sparc_pmu_add(struct perf_event
*event
, int ef_flags
)
1010 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
1011 int n0
, ret
= -EAGAIN
;
1012 unsigned long flags
;
1014 local_irq_save(flags
);
1015 perf_pmu_disable(event
->pmu
);
1017 n0
= cpuc
->n_events
;
1018 if (n0
>= MAX_HWEVENTS
)
1021 cpuc
->event
[n0
] = event
;
1022 cpuc
->events
[n0
] = event
->hw
.event_base
;
1023 cpuc
->current_idx
[n0
] = PIC_NO_INDEX
;
1025 event
->hw
.state
= PERF_HES_UPTODATE
;
1026 if (!(ef_flags
& PERF_EF_START
))
1027 event
->hw
.state
|= PERF_HES_STOPPED
;
1030 * If group events scheduling transaction was started,
1031 * skip the schedulability test here, it will be performed
1032 * at commit time(->commit_txn) as a whole
1034 if (cpuc
->group_flag
& PERF_EVENT_TXN
)
1037 if (check_excludes(cpuc
->event
, n0
, 1))
1039 if (sparc_check_constraints(cpuc
->event
, cpuc
->events
, n0
+ 1))
1048 perf_pmu_enable(event
->pmu
);
1049 local_irq_restore(flags
);
1053 static int sparc_pmu_event_init(struct perf_event
*event
)
1055 struct perf_event_attr
*attr
= &event
->attr
;
1056 struct perf_event
*evts
[MAX_HWEVENTS
];
1057 struct hw_perf_event
*hwc
= &event
->hw
;
1058 unsigned long events
[MAX_HWEVENTS
];
1059 int current_idx_dmy
[MAX_HWEVENTS
];
1060 const struct perf_event_map
*pmap
;
1063 if (atomic_read(&nmi_active
) < 0)
1066 switch (attr
->type
) {
1067 case PERF_TYPE_HARDWARE
:
1068 if (attr
->config
>= sparc_pmu
->max_events
)
1070 pmap
= sparc_pmu
->event_map(attr
->config
);
1073 case PERF_TYPE_HW_CACHE
:
1074 pmap
= sparc_map_cache_event(attr
->config
);
1076 return PTR_ERR(pmap
);
1089 hwc
->event_base
= perf_event_encode(pmap
);
1092 * User gives us "(encoding << 16) | pic_mask" for
1093 * PERF_TYPE_RAW events.
1095 hwc
->event_base
= attr
->config
;
1098 /* We save the enable bits in the config_base. */
1099 hwc
->config_base
= sparc_pmu
->irq_bit
;
1100 if (!attr
->exclude_user
)
1101 hwc
->config_base
|= PCR_UTRACE
;
1102 if (!attr
->exclude_kernel
)
1103 hwc
->config_base
|= PCR_STRACE
;
1104 if (!attr
->exclude_hv
)
1105 hwc
->config_base
|= sparc_pmu
->hv_bit
;
1108 if (event
->group_leader
!= event
) {
1109 n
= collect_events(event
->group_leader
,
1111 evts
, events
, current_idx_dmy
);
1115 events
[n
] = hwc
->event_base
;
1118 if (check_excludes(evts
, n
, 1))
1121 if (sparc_check_constraints(evts
, events
, n
+ 1))
1124 hwc
->idx
= PIC_NO_INDEX
;
1126 /* Try to do all error checking before this point, as unwinding
1127 * state after grabbing the PMC is difficult.
1129 perf_event_grab_pmc();
1130 event
->destroy
= hw_perf_event_destroy
;
1132 if (!hwc
->sample_period
) {
1133 hwc
->sample_period
= MAX_PERIOD
;
1134 hwc
->last_period
= hwc
->sample_period
;
1135 local64_set(&hwc
->period_left
, hwc
->sample_period
);
1142 * Start group events scheduling transaction
1143 * Set the flag to make pmu::enable() not perform the
1144 * schedulability test, it will be performed at commit time
1146 static void sparc_pmu_start_txn(struct pmu
*pmu
)
1148 struct cpu_hw_events
*cpuhw
= &__get_cpu_var(cpu_hw_events
);
1150 perf_pmu_disable(pmu
);
1151 cpuhw
->group_flag
|= PERF_EVENT_TXN
;
1155 * Stop group events scheduling transaction
1156 * Clear the flag and pmu::enable() will perform the
1157 * schedulability test.
1159 static void sparc_pmu_cancel_txn(struct pmu
*pmu
)
1161 struct cpu_hw_events
*cpuhw
= &__get_cpu_var(cpu_hw_events
);
1163 cpuhw
->group_flag
&= ~PERF_EVENT_TXN
;
1164 perf_pmu_enable(pmu
);
1168 * Commit group events scheduling transaction
1169 * Perform the group schedulability test as a whole
1170 * Return 0 if success
1172 static int sparc_pmu_commit_txn(struct pmu
*pmu
)
1174 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
1180 cpuc
= &__get_cpu_var(cpu_hw_events
);
1182 if (check_excludes(cpuc
->event
, 0, n
))
1184 if (sparc_check_constraints(cpuc
->event
, cpuc
->events
, n
))
1187 cpuc
->group_flag
&= ~PERF_EVENT_TXN
;
1188 perf_pmu_enable(pmu
);
1192 static struct pmu pmu
= {
1193 .pmu_enable
= sparc_pmu_enable
,
1194 .pmu_disable
= sparc_pmu_disable
,
1195 .event_init
= sparc_pmu_event_init
,
1196 .add
= sparc_pmu_add
,
1197 .del
= sparc_pmu_del
,
1198 .start
= sparc_pmu_start
,
1199 .stop
= sparc_pmu_stop
,
1200 .read
= sparc_pmu_read
,
1201 .start_txn
= sparc_pmu_start_txn
,
1202 .cancel_txn
= sparc_pmu_cancel_txn
,
1203 .commit_txn
= sparc_pmu_commit_txn
,
1206 void perf_event_print_debug(void)
1208 unsigned long flags
;
1215 local_irq_save(flags
);
1217 cpu
= smp_processor_id();
1219 pcr
= pcr_ops
->read();
1223 pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
1226 local_irq_restore(flags
);
1229 static int __kprobes
perf_event_nmi_handler(struct notifier_block
*self
,
1230 unsigned long cmd
, void *__args
)
1232 struct die_args
*args
= __args
;
1233 struct perf_sample_data data
;
1234 struct cpu_hw_events
*cpuc
;
1235 struct pt_regs
*regs
;
1238 if (!atomic_read(&active_events
))
1251 perf_sample_data_init(&data
, 0);
1253 cpuc
= &__get_cpu_var(cpu_hw_events
);
1255 /* If the PMU has the TOE IRQ enable bits, we need to do a
1256 * dummy write to the %pcr to clear the overflow bits and thus
1259 * Do this before we peek at the counters to determine
1260 * overflow so we don't lose any events.
1262 if (sparc_pmu
->irq_bit
)
1263 pcr_ops
->write(cpuc
->pcr
);
1265 for (i
= 0; i
< cpuc
->n_events
; i
++) {
1266 struct perf_event
*event
= cpuc
->event
[i
];
1267 int idx
= cpuc
->current_idx
[i
];
1268 struct hw_perf_event
*hwc
;
1272 val
= sparc_perf_event_update(event
, hwc
, idx
);
1273 if (val
& (1ULL << 31))
1276 data
.period
= event
->hw
.last_period
;
1277 if (!sparc_perf_event_set_period(event
, hwc
, idx
))
1280 if (perf_event_overflow(event
, 1, &data
, regs
))
1281 sparc_pmu_stop(event
, 0);
1287 static __read_mostly
struct notifier_block perf_event_nmi_notifier
= {
1288 .notifier_call
= perf_event_nmi_handler
,
1291 static bool __init
supported_pmu(void)
1293 if (!strcmp(sparc_pmu_type
, "ultra3") ||
1294 !strcmp(sparc_pmu_type
, "ultra3+") ||
1295 !strcmp(sparc_pmu_type
, "ultra3i") ||
1296 !strcmp(sparc_pmu_type
, "ultra4+")) {
1297 sparc_pmu
= &ultra3_pmu
;
1300 if (!strcmp(sparc_pmu_type
, "niagara")) {
1301 sparc_pmu
= &niagara1_pmu
;
1304 if (!strcmp(sparc_pmu_type
, "niagara2") ||
1305 !strcmp(sparc_pmu_type
, "niagara3")) {
1306 sparc_pmu
= &niagara2_pmu
;
1312 int __init
init_hw_perf_events(void)
1314 pr_info("Performance events: ");
1316 if (!supported_pmu()) {
1317 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type
);
1321 pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type
);
1323 perf_pmu_register(&pmu
, "cpu", PERF_TYPE_RAW
);
1324 register_die_notifier(&perf_event_nmi_notifier
);
1328 early_initcall(init_hw_perf_events
);
1330 void perf_callchain_kernel(struct perf_callchain_entry
*entry
,
1331 struct pt_regs
*regs
)
1333 unsigned long ksp
, fp
;
1334 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1338 stack_trace_flush();
1340 perf_callchain_store(entry
, regs
->tpc
);
1342 ksp
= regs
->u_regs
[UREG_I6
];
1343 fp
= ksp
+ STACK_BIAS
;
1345 struct sparc_stackf
*sf
;
1346 struct pt_regs
*regs
;
1349 if (!kstack_valid(current_thread_info(), fp
))
1352 sf
= (struct sparc_stackf
*) fp
;
1353 regs
= (struct pt_regs
*) (sf
+ 1);
1355 if (kstack_is_trap_frame(current_thread_info(), regs
)) {
1356 if (user_mode(regs
))
1359 fp
= regs
->u_regs
[UREG_I6
] + STACK_BIAS
;
1361 pc
= sf
->callers_pc
;
1362 fp
= (unsigned long)sf
->fp
+ STACK_BIAS
;
1364 perf_callchain_store(entry
, pc
);
1365 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1366 if ((pc
+ 8UL) == (unsigned long) &return_to_handler
) {
1367 int index
= current
->curr_ret_stack
;
1368 if (current
->ret_stack
&& index
>= graph
) {
1369 pc
= current
->ret_stack
[index
- graph
].ret
;
1370 perf_callchain_store(entry
, pc
);
1375 } while (entry
->nr
< PERF_MAX_STACK_DEPTH
);
1378 static void perf_callchain_user_64(struct perf_callchain_entry
*entry
,
1379 struct pt_regs
*regs
)
1383 perf_callchain_store(entry
, regs
->tpc
);
1385 ufp
= regs
->u_regs
[UREG_I6
] + STACK_BIAS
;
1387 struct sparc_stackf
*usf
, sf
;
1390 usf
= (struct sparc_stackf
*) ufp
;
1391 if (__copy_from_user_inatomic(&sf
, usf
, sizeof(sf
)))
1395 ufp
= (unsigned long)sf
.fp
+ STACK_BIAS
;
1396 perf_callchain_store(entry
, pc
);
1397 } while (entry
->nr
< PERF_MAX_STACK_DEPTH
);
1400 static void perf_callchain_user_32(struct perf_callchain_entry
*entry
,
1401 struct pt_regs
*regs
)
1405 perf_callchain_store(entry
, regs
->tpc
);
1407 ufp
= regs
->u_regs
[UREG_I6
] & 0xffffffffUL
;
1409 struct sparc_stackf32
*usf
, sf
;
1412 usf
= (struct sparc_stackf32
*) ufp
;
1413 if (__copy_from_user_inatomic(&sf
, usf
, sizeof(sf
)))
1417 ufp
= (unsigned long)sf
.fp
;
1418 perf_callchain_store(entry
, pc
);
1419 } while (entry
->nr
< PERF_MAX_STACK_DEPTH
);
1423 perf_callchain_user(struct perf_callchain_entry
*entry
, struct pt_regs
*regs
)
1426 if (test_thread_flag(TIF_32BIT
))
1427 perf_callchain_user_32(entry
, regs
);
1429 perf_callchain_user_64(entry
, regs
);