?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf, x86: Add a key to simplify template lookup in Pentium-4 PMU
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / arch / x86 / kernel / cpu / perf_event_p4.c
blob3e97ed3904ccc9bb96d24d28ff25fa80b9ce3512
1 /*
2 * Netburst Perfomance Events (P4, old Xeon)
3 *
4 * Copyright (C) 2010 Parallels, Inc., Cyrill Gorcunov <gorcunov@openvz.org>
5 * Copyright (C) 2010 Intel Corporation, Lin Ming <ming.m.lin@intel.com>
6 *
7 * For licencing details see kernel-base/COPYING
8 */
9
10 #ifdef CONFIG_CPU_SUP_INTEL
11
12 #include <asm/perf_event_p4.h>
13
14 /*
15 * array indices: 0,1 - HT threads, used with HT enabled cpu
16 */
17 struct p4_event_template {
18 u32 opcode; /* ESCR event + CCCR selector */
19 u64 config; /* packed predefined bits */
20 int dep; /* upstream dependency event index */
21 int key; /* index into p4_templates */
22 unsigned int emask; /* ESCR EventMask */
23 unsigned int escr_msr[2]; /* ESCR MSR for this event */
24 unsigned int cntr[2]; /* counter index (offset) */
25 };
26
27 struct p4_pmu_res {
28 /* maps hw_conf::idx into template for ESCR sake */
29 struct p4_event_template *tpl[ARCH_P4_MAX_CCCR];
30 };
31
32 static DEFINE_PER_CPU(struct p4_pmu_res, p4_pmu_config);
33
34 /*
35 * WARN: CCCR1 doesn't have a working enable bit so try to not
36 * use it if possible
37 *
38 * Also as only we start to support raw events we will need to
39 * append _all_ P4_EVENT_PACK'ed events here
40 */
41 struct p4_event_template p4_templates[] = {
42 [0] = {
43 .opcode = P4_GLOBAL_POWER_EVENTS,
44 .config = 0,
45 .dep = -1,
46 .key = 0,
47 .emask =
48 P4_EVENT_ATTR(P4_GLOBAL_POWER_EVENTS, RUNNING),
49 .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
50 .cntr = { 0, 2 },
51 },
52 [1] = {
53 .opcode = P4_INSTR_RETIRED,
54 .config = 0,
55 .dep = -1, /* needs front-end tagging */
56 .key = 1,
57 .emask =
58 P4_EVENT_ATTR(P4_INSTR_RETIRED, NBOGUSNTAG) |
59 P4_EVENT_ATTR(P4_INSTR_RETIRED, BOGUSNTAG),
60 .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
61 .cntr = { 12, 14 },
62 },
63 [2] = {
64 .opcode = P4_BSQ_CACHE_REFERENCE,
65 .config = 0,
66 .dep = -1,
67 .key = 2,
68 .emask =
69 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_2ndL_HITS) |
70 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_2ndL_HITE) |
71 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_2ndL_HITM) |
72 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_3rdL_HITS) |
73 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_3rdL_HITE) |
74 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_3rdL_HITM),
75 .escr_msr = { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR1 },
76 .cntr = { 0, 2 },
77 },
78 [3] = {
79 .opcode = P4_BSQ_CACHE_REFERENCE,
80 .config = 0,
81 .dep = -1,
82 .key = 3,
83 .emask =
84 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_2ndL_MISS) |
85 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, RD_3rdL_MISS) |
86 P4_EVENT_ATTR(P4_BSQ_CACHE_REFERENCE, WR_2ndL_MISS),
87 .escr_msr = { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR1 },
88 .cntr = { 0, 3 },
89 },
90 [4] = {
91 .opcode = P4_RETIRED_BRANCH_TYPE,
92 .config = 0,
93 .dep = -1,
94 .key = 4,
95 .emask =
96 P4_EVENT_ATTR(P4_RETIRED_BRANCH_TYPE, CONDITIONAL) |
97 P4_EVENT_ATTR(P4_RETIRED_BRANCH_TYPE, CALL) |
98 P4_EVENT_ATTR(P4_RETIRED_BRANCH_TYPE, RETURN) |
99 P4_EVENT_ATTR(P4_RETIRED_BRANCH_TYPE, INDIRECT),
100 .escr_msr = { MSR_P4_TBPU_ESCR0, MSR_P4_TBPU_ESCR1 },
101 .cntr = { 4, 6 },
102 },
103 [5] = {
104 .opcode = P4_MISPRED_BRANCH_RETIRED,
105 .config = 0,
106 .dep = -1,
107 .key = 5,
108 .emask =
109 P4_EVENT_ATTR(P4_MISPRED_BRANCH_RETIRED, NBOGUS),
110 .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
111 .cntr = { 12, 14 },
112 },
113 [6] = {
114 .opcode = P4_FSB_DATA_ACTIVITY,
115 .config = p4_config_pack_cccr(P4_CCCR_EDGE | P4_CCCR_COMPARE),
116 .dep = -1,
117 .key = 6,
118 .emask =
119 P4_EVENT_ATTR(P4_FSB_DATA_ACTIVITY, DRDY_DRV) |
120 P4_EVENT_ATTR(P4_FSB_DATA_ACTIVITY, DRDY_OWN),
121 .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
122 .cntr = { 0, 2 },
123 },
124 [7] = {
125 .opcode = P4_UOP_TYPE,
126 .config = 0,
127 .dep = -1,
128 .key = 7,
129 .emask =
130 P4_EVENT_ATTR(P4_UOP_TYPE, TAGLOADS) |
131 P4_EVENT_ATTR(P4_UOP_TYPE, TAGSTORES),
132 .escr_msr = { MSR_P4_RAT_ESCR0, MSR_P4_RAT_ESCR1 },
133 .cntr = { 16, 17 },
134 },
135 };
136
137 static u64 p4_pmu_event_map(int hw_event)
138 {
139 struct p4_event_template *tpl;
140 u64 config;
141
142 if (hw_event > ARRAY_SIZE(p4_templates)) {
143 printk_once(KERN_ERR "PMU: Incorrect event index\n");
144 return 0;
145 }
146 tpl = &p4_templates[hw_event];
147
148 /*
149 * fill config up according to
150 * a predefined event template
151 */
152 config = tpl->config;
153 config |= p4_config_pack_escr(P4_EVENT_UNPACK_EVENT(tpl->opcode) << P4_EVNTSEL_EVENT_SHIFT);
154 config |= p4_config_pack_escr(tpl->emask << P4_EVNTSEL_EVENTMASK_SHIFT);
155 config |= p4_config_pack_cccr(P4_EVENT_UNPACK_SELECTOR(tpl->opcode) << P4_CCCR_ESCR_SELECT_SHIFT);
156 config |= p4_config_pack_cccr(hw_event & P4_CCCR_RESERVED);
157
158 /* on HT machine we need a special bit */
159 if (p4_ht_active() && p4_ht_thread(raw_smp_processor_id()))
160 config = p4_set_ht_bit(config);
161
162 return config;
163 }
164
165 /*
166 * Note that we still have 5 events (from global events SDM list)
167 * intersected in opcode+emask bits so we will need another
168 * scheme there do distinguish templates.
169 */
170 static inline int p4_pmu_emask_match(unsigned int dst, unsigned int src)
171 {
172 return dst & src;
173 }
174
175 static struct p4_event_template *p4_pmu_template_lookup(u64 config)
176 {
177 int key = p4_config_unpack_key(config);
178
179 if (key < ARRAY_SIZE(p4_templates))
180 return &p4_templates[key];
181 else
182 return NULL;
183 }
184
185 /*
186 * We don't control raw events so it's up to the caller
187 * to pass sane values (and we don't count the thread number
188 * on HT machine but allow HT-compatible specifics to be
189 * passed on)
190 */
191 static u64 p4_pmu_raw_event(u64 hw_event)
192 {
193 return hw_event &
194 (p4_config_pack_escr(P4_EVNTSEL_MASK_HT) |
195 p4_config_pack_cccr(P4_CCCR_MASK_HT));
196 }
197
198 static int p4_hw_config(struct perf_event_attr *attr, struct hw_perf_event *hwc)
199 {
200 int cpu = raw_smp_processor_id();
201
202 /*
203 * the reason we use cpu that early is that: if we get scheduled
204 * first time on the same cpu -- we will not need swap thread
205 * specific flags in config (and will save some cpu cycles)
206 */
207
208 /* CCCR by default */
209 hwc->config = p4_config_pack_cccr(p4_default_cccr_conf(cpu));
210
211 /* Count user and OS events unless not requested to */
212 hwc->config |= p4_config_pack_escr(p4_default_escr_conf(cpu, attr->exclude_kernel,
213 attr->exclude_user));
214 return 0;
215 }
216
217 static inline void p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
218 {
219 unsigned long dummy;
220
221 rdmsrl(hwc->config_base + hwc->idx, dummy);
222 if (dummy & P4_CCCR_OVF) {
223 (void)checking_wrmsrl(hwc->config_base + hwc->idx,
224 ((u64)dummy) & ~P4_CCCR_OVF);
225 }
226 }
227
228 static inline void p4_pmu_disable_event(struct perf_event *event)
229 {
230 struct hw_perf_event *hwc = &event->hw;
231
232 /*
233 * If event gets disabled while counter is in overflowed
234 * state we need to clear P4_CCCR_OVF, otherwise interrupt get
235 * asserted again and again
236 */
237 (void)checking_wrmsrl(hwc->config_base + hwc->idx,
238 (u64)(p4_config_unpack_cccr(hwc->config)) &
239 ~P4_CCCR_ENABLE & ~P4_CCCR_OVF);
240 }
241
242 static void p4_pmu_disable_all(void)
243 {
244 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
245 int idx;
246
247 for (idx = 0; idx < x86_pmu.num_events; idx++) {
248 struct perf_event *event = cpuc->events[idx];
249 if (!test_bit(idx, cpuc->active_mask))
250 continue;
251 p4_pmu_disable_event(event);
252 }
253 }
254
255 static void p4_pmu_enable_event(struct perf_event *event)
256 {
257 struct hw_perf_event *hwc = &event->hw;
258 int thread = p4_ht_config_thread(hwc->config);
259 u64 escr_conf = p4_config_unpack_escr(p4_clear_ht_bit(hwc->config));
260 u64 escr_base;
261 struct p4_event_template *tpl;
262 struct p4_pmu_res *c;
263
264 /*
265 * some preparation work from per-cpu private fields
266 * since we need to find out which ESCR to use
267 */
268 c = &__get_cpu_var(p4_pmu_config);
269 tpl = c->tpl[hwc->idx];
270 if (!tpl) {
271 pr_crit("%s: Wrong index: %d\n", __func__, hwc->idx);
272 return;
273 }
274 escr_base = (u64)tpl->escr_msr[thread];
275
276 /*
277 * - we dont support cascaded counters yet
278 * - and counter 1 is broken (erratum)
279 */
280 WARN_ON_ONCE(p4_is_event_cascaded(hwc->config));
281 WARN_ON_ONCE(hwc->idx == 1);
282
283 (void)checking_wrmsrl(escr_base, escr_conf);
284 (void)checking_wrmsrl(hwc->config_base + hwc->idx,
285 (u64)(p4_config_unpack_cccr(hwc->config)) | P4_CCCR_ENABLE);
286 }
287
288 static void p4_pmu_enable_all(void)
289 {
290 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
291 int idx;
292
293 for (idx = 0; idx < x86_pmu.num_events; idx++) {
294 struct perf_event *event = cpuc->events[idx];
295 if (!test_bit(idx, cpuc->active_mask))
296 continue;
297 p4_pmu_enable_event(event);
298 }
299 }
300
301 static int p4_pmu_handle_irq(struct pt_regs *regs)
302 {
303 struct perf_sample_data data;
304 struct cpu_hw_events *cpuc;
305 struct perf_event *event;
306 struct hw_perf_event *hwc;
307 int idx, handled = 0;
308 u64 val;
309
310 data.addr = 0;
311 data.raw = NULL;
312
313 cpuc = &__get_cpu_var(cpu_hw_events);
314
315 for (idx = 0; idx < x86_pmu.num_events; idx++) {
316
317 if (!test_bit(idx, cpuc->active_mask))
318 continue;
319
320 event = cpuc->events[idx];
321 hwc = &event->hw;
322
323 WARN_ON_ONCE(hwc->idx != idx);
324
325 /*
326 * FIXME: Redundant call, actually not needed
327 * but just to check if we're screwed
328 */
329 p4_pmu_clear_cccr_ovf(hwc);
330
331 val = x86_perf_event_update(event);
332 if (val & (1ULL << (x86_pmu.event_bits - 1)))
333 continue;
334
335 /*
336 * event overflow
337 */
338 handled = 1;
339 data.period = event->hw.last_period;
340
341 if (!x86_perf_event_set_period(event))
342 continue;
343 if (perf_event_overflow(event, 1, &data, regs))
344 p4_pmu_disable_event(event);
345 }
346
347 if (handled) {
348 /* p4 quirk: unmask it again */
349 apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
350 inc_irq_stat(apic_perf_irqs);
351 }
352
353 return handled;
354 }
355
356 /*
357 * swap thread specific fields according to a thread
358 * we are going to run on
359 */
360 static void p4_pmu_swap_config_ts(struct hw_perf_event *hwc, int cpu)
361 {
362 u32 escr, cccr;
363
364 /*
365 * we either lucky and continue on same cpu or no HT support
366 */
367 if (!p4_should_swap_ts(hwc->config, cpu))
368 return;
369
370 /*
371 * the event is migrated from an another logical
372 * cpu, so we need to swap thread specific flags
373 */
374
375 escr = p4_config_unpack_escr(hwc->config);
376 cccr = p4_config_unpack_cccr(hwc->config);
377
378 if (p4_ht_thread(cpu)) {
379 cccr &= ~P4_CCCR_OVF_PMI_T0;
380 cccr |= P4_CCCR_OVF_PMI_T1;
381 if (escr & P4_EVNTSEL_T0_OS) {
382 escr &= ~P4_EVNTSEL_T0_OS;
383 escr |= P4_EVNTSEL_T1_OS;
384 }
385 if (escr & P4_EVNTSEL_T0_USR) {
386 escr &= ~P4_EVNTSEL_T0_USR;
387 escr |= P4_EVNTSEL_T1_USR;
388 }
389 hwc->config = p4_config_pack_escr(escr);
390 hwc->config |= p4_config_pack_cccr(cccr);
391 hwc->config |= P4_CONFIG_HT;
392 } else {
393 cccr &= ~P4_CCCR_OVF_PMI_T1;
394 cccr |= P4_CCCR_OVF_PMI_T0;
395 if (escr & P4_EVNTSEL_T1_OS) {
396 escr &= ~P4_EVNTSEL_T1_OS;
397 escr |= P4_EVNTSEL_T0_OS;
398 }
399 if (escr & P4_EVNTSEL_T1_USR) {
400 escr &= ~P4_EVNTSEL_T1_USR;
401 escr |= P4_EVNTSEL_T0_USR;
402 }
403 hwc->config = p4_config_pack_escr(escr);
404 hwc->config |= p4_config_pack_cccr(cccr);
405 hwc->config &= ~P4_CONFIG_HT;
406 }
407 }
408
409 /* ESCRs are not sequential in memory so we need a map */
410 static unsigned int p4_escr_map[ARCH_P4_TOTAL_ESCR] = {
411 MSR_P4_ALF_ESCR0, /* 0 */
412 MSR_P4_ALF_ESCR1, /* 1 */
413 MSR_P4_BPU_ESCR0, /* 2 */
414 MSR_P4_BPU_ESCR1, /* 3 */
415 MSR_P4_BSU_ESCR0, /* 4 */
416 MSR_P4_BSU_ESCR1, /* 5 */
417 MSR_P4_CRU_ESCR0, /* 6 */
418 MSR_P4_CRU_ESCR1, /* 7 */
419 MSR_P4_CRU_ESCR2, /* 8 */
420 MSR_P4_CRU_ESCR3, /* 9 */
421 MSR_P4_CRU_ESCR4, /* 10 */
422 MSR_P4_CRU_ESCR5, /* 11 */
423 MSR_P4_DAC_ESCR0, /* 12 */
424 MSR_P4_DAC_ESCR1, /* 13 */
425 MSR_P4_FIRM_ESCR0, /* 14 */
426 MSR_P4_FIRM_ESCR1, /* 15 */
427 MSR_P4_FLAME_ESCR0, /* 16 */
428 MSR_P4_FLAME_ESCR1, /* 17 */
429 MSR_P4_FSB_ESCR0, /* 18 */
430 MSR_P4_FSB_ESCR1, /* 19 */
431 MSR_P4_IQ_ESCR0, /* 20 */
432 MSR_P4_IQ_ESCR1, /* 21 */
433 MSR_P4_IS_ESCR0, /* 22 */
434 MSR_P4_IS_ESCR1, /* 23 */
435 MSR_P4_ITLB_ESCR0, /* 24 */
436 MSR_P4_ITLB_ESCR1, /* 25 */
437 MSR_P4_IX_ESCR0, /* 26 */
438 MSR_P4_IX_ESCR1, /* 27 */
439 MSR_P4_MOB_ESCR0, /* 28 */
440 MSR_P4_MOB_ESCR1, /* 29 */
441 MSR_P4_MS_ESCR0, /* 30 */
442 MSR_P4_MS_ESCR1, /* 31 */
443 MSR_P4_PMH_ESCR0, /* 32 */
444 MSR_P4_PMH_ESCR1, /* 33 */
445 MSR_P4_RAT_ESCR0, /* 34 */
446 MSR_P4_RAT_ESCR1, /* 35 */
447 MSR_P4_SAAT_ESCR0, /* 36 */
448 MSR_P4_SAAT_ESCR1, /* 37 */
449 MSR_P4_SSU_ESCR0, /* 38 */
450 MSR_P4_SSU_ESCR1, /* 39 */
451 MSR_P4_TBPU_ESCR0, /* 40 */
452 MSR_P4_TBPU_ESCR1, /* 41 */
453 MSR_P4_TC_ESCR0, /* 42 */
454 MSR_P4_TC_ESCR1, /* 43 */
455 MSR_P4_U2L_ESCR0, /* 44 */
456 MSR_P4_U2L_ESCR1, /* 45 */
457 };
458
459 static int p4_get_escr_idx(unsigned int addr)
460 {
461 unsigned int i;
462
463 for (i = 0; i < ARRAY_SIZE(p4_escr_map); i++) {
464 if (addr == p4_escr_map[i])
465 return i;
466 }
467
468 return -1;
469 }
470
471 static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
472 {
473 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
474 unsigned long escr_mask[BITS_TO_LONGS(ARCH_P4_TOTAL_ESCR)];
475
476 struct hw_perf_event *hwc;
477 struct p4_event_template *tpl;
478 struct p4_pmu_res *c;
479 int cpu = raw_smp_processor_id();
480 int escr_idx, thread, i, num;
481
482 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
483 bitmap_zero(escr_mask, ARCH_P4_TOTAL_ESCR);
484
485 c = &__get_cpu_var(p4_pmu_config);
486 /*
487 * Firstly find out which resource events are going
488 * to use, if ESCR+CCCR tuple is already borrowed
489 * then get out of here
490 */
491 for (i = 0, num = n; i < n; i++, num--) {
492 hwc = &cpuc->event_list[i]->hw;
493 tpl = p4_pmu_template_lookup(hwc->config);
494 if (!tpl)
495 goto done;
496 thread = p4_ht_thread(cpu);
497 escr_idx = p4_get_escr_idx(tpl->escr_msr[thread]);
498 if (escr_idx == -1)
499 goto done;
500
501 /* already allocated and remains on the same cpu */
502 if (hwc->idx != -1 && !p4_should_swap_ts(hwc->config, cpu)) {
503 if (assign)
504 assign[i] = hwc->idx;
505 /* upstream dependent event */
506 if (unlikely(tpl->dep != -1))
507 printk_once(KERN_WARNING "PMU: Dep events are "
508 "not implemented yet\n");
509 goto reserve;
510 }
511
512 /* it may be already borrowed */
513 if (test_bit(tpl->cntr[thread], used_mask) ||
514 test_bit(escr_idx, escr_mask))
515 goto done;
516
517 /*
518 * ESCR+CCCR+COUNTERs are available to use lets swap
519 * thread specific bits, push assigned bits
520 * back and save template into per-cpu
521 * area (which will allow us to find out the ESCR
522 * to be used at moment of "enable event via real MSR")
523 */
524 p4_pmu_swap_config_ts(hwc, cpu);
525 if (assign) {
526 assign[i] = tpl->cntr[thread];
527 c->tpl[assign[i]] = tpl;
528 }
529 reserve:
530 set_bit(tpl->cntr[thread], used_mask);
531 set_bit(escr_idx, escr_mask);
532 }
533
534 done:
535 return num ? -ENOSPC : 0;
536 }
537
538 static __initconst struct x86_pmu p4_pmu = {
539 .name = "Netburst P4/Xeon",
540 .handle_irq = p4_pmu_handle_irq,
541 .disable_all = p4_pmu_disable_all,
542 .enable_all = p4_pmu_enable_all,
543 .enable = p4_pmu_enable_event,
544 .disable = p4_pmu_disable_event,
545 .eventsel = MSR_P4_BPU_CCCR0,
546 .perfctr = MSR_P4_BPU_PERFCTR0,
547 .event_map = p4_pmu_event_map,
548 .raw_event = p4_pmu_raw_event,
549 .max_events = ARRAY_SIZE(p4_templates),
550 .get_event_constraints = x86_get_event_constraints,
551 /*
552 * IF HT disabled we may need to use all
553 * ARCH_P4_MAX_CCCR counters simulaneously
554 * though leave it restricted at moment assuming
555 * HT is on
556 */
557 .num_events = ARCH_P4_MAX_CCCR,
558 .apic = 1,
559 .event_bits = 40,
560 .event_mask = (1ULL << 40) - 1,
561 .max_period = (1ULL << 39) - 1,
562 .hw_config = p4_hw_config,
563 .schedule_events = p4_pmu_schedule_events,
564 };
565
566 static __init int p4_pmu_init(void)
567 {
568 unsigned int low, high;
569
570 /* If we get stripped -- indexig fails */
571 BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC);
572
573 rdmsr(MSR_IA32_MISC_ENABLE, low, high);
574 if (!(low & (1 << 7))) {
575 pr_cont("unsupported Netburst CPU model %d ",
576 boot_cpu_data.x86_model);
577 return -ENODEV;
578 }
579
580 pr_cont("Netburst events, ");
581
582 x86_pmu = p4_pmu;
583
584 return 0;
585 }
586
587 #endif /* CONFIG_CPU_SUP_INTEL */
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree