arch/arm/kernel/perf_event_xscale.c

   1 /*
   2  * ARMv5 [xscale] Performance counter handling code.
   3  *
   4  * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
   5  *
   6  * Based on the previous xscale OProfile code.
   7  *
   8  * There are two variants of the xscale PMU that we support:
   9  *      - xscale1pmu: 2 event counters and a cycle counter
  10  *      - xscale2pmu: 4 event counters and a cycle counter
  11  * The two variants share event definitions, but have different
  12  * PMU structures.
  13  */
  14
  15 #ifdef CONFIG_CPU_XSCALE
  16 enum xscale_perf_types {
  17         XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
  18         XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
  19         XSCALE_PERFCTR_DATA_STALL               = 0x02,
  20         XSCALE_PERFCTR_ITLB_MISS                = 0x03,
  21         XSCALE_PERFCTR_DTLB_MISS                = 0x04,
  22         XSCALE_PERFCTR_BRANCH                   = 0x05,
  23         XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
  24         XSCALE_PERFCTR_INSTRUCTION              = 0x07,
  25         XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
  26         XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
  27         XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
  28         XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
  29         XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
  30         XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
  31         XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
  32         XSCALE_PERFCTR_BCU_FULL                 = 0x11,
  33         XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
  34         XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
  35         XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
  36         XSCALE_PERFCTR_RMW                      = 0x16,
  37         /* XSCALE_PERFCTR_CCNT is not hardware defined */
  38         XSCALE_PERFCTR_CCNT                     = 0xFE,
  39         XSCALE_PERFCTR_UNUSED                   = 0xFF,
  40 };
  41
  42 enum xscale_counters {
  43         XSCALE_CYCLE_COUNTER    = 0,
  44         XSCALE_COUNTER0,
  45         XSCALE_COUNTER1,
  46         XSCALE_COUNTER2,
  47         XSCALE_COUNTER3,
  48 };
  49
  50 static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
  51         [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
  52         [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
  53         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
  54         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
  55         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
  56         [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
  57         [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
  58 };
  59
  60 static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  61                                            [PERF_COUNT_HW_CACHE_OP_MAX]
  62                                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  63         [C(L1D)] = {
  64                 [C(OP_READ)] = {
  65                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  66                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  67                 },
  68                 [C(OP_WRITE)] = {
  69                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  70                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  71                 },
  72                 [C(OP_PREFETCH)] = {
  73                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  74                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  75                 },
  76         },
  77         [C(L1I)] = {
  78                 [C(OP_READ)] = {
  79                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  80                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  81                 },
  82                 [C(OP_WRITE)] = {
  83                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  84                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  85                 },
  86                 [C(OP_PREFETCH)] = {
  87                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  88                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  89                 },
  90         },
  91         [C(LL)] = {
  92                 [C(OP_READ)] = {
  93                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  94                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  95                 },
  96                 [C(OP_WRITE)] = {
  97                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  98                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  99                 },
 100                 [C(OP_PREFETCH)] = {
 101                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 102                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 103                 },
 104         },
 105         [C(DTLB)] = {
 106                 [C(OP_READ)] = {
 107                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 108                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 109                 },
 110                 [C(OP_WRITE)] = {
 111                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 112                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 113                 },
 114                 [C(OP_PREFETCH)] = {
 115                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 116                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 117                 },
 118         },
 119         [C(ITLB)] = {
 120                 [C(OP_READ)] = {
 121                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 122                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 123                 },
 124                 [C(OP_WRITE)] = {
 125                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 126                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 127                 },
 128                 [C(OP_PREFETCH)] = {
 129                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 130                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 131                 },
 132         },
 133         [C(BPU)] = {
 134                 [C(OP_READ)] = {
 135                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 136                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 137                 },
 138                 [C(OP_WRITE)] = {
 139                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 140                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 141                 },
 142                 [C(OP_PREFETCH)] = {
 143                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 144                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 145                 },
 146         },
 147         [C(NODE)] = {
 148                 [C(OP_READ)] = {
 149                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 150                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 151                 },
 152                 [C(OP_WRITE)] = {
 153                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 154                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 155                 },
 156                 [C(OP_PREFETCH)] = {
 157                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 158                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 159                 },
 160         },
 161 };
 162
 163 #define XSCALE_PMU_ENABLE       0x001
 164 #define XSCALE_PMN_RESET        0x002
 165 #define XSCALE_CCNT_RESET       0x004
 166 #define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
 167 #define XSCALE_PMU_CNT64        0x008
 168
 169 #define XSCALE1_OVERFLOWED_MASK 0x700
 170 #define XSCALE1_CCOUNT_OVERFLOW 0x400
 171 #define XSCALE1_COUNT0_OVERFLOW 0x100
 172 #define XSCALE1_COUNT1_OVERFLOW 0x200
 173 #define XSCALE1_CCOUNT_INT_EN   0x040
 174 #define XSCALE1_COUNT0_INT_EN   0x010
 175 #define XSCALE1_COUNT1_INT_EN   0x020
 176 #define XSCALE1_COUNT0_EVT_SHFT 12
 177 #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
 178 #define XSCALE1_COUNT1_EVT_SHFT 20
 179 #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
 180
 181 static inline u32
 182 xscale1pmu_read_pmnc(void)
 183 {
 184         u32 val;
 185         asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
 186         return val;
 187 }
 188
 189 static inline void
 190 xscale1pmu_write_pmnc(u32 val)
 191 {
 192         /* upper 4bits and 7, 11 are write-as-0 */
 193         val &= 0xffff77f;
 194         asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
 195 }
 196
 197 static inline int
 198 xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
 199                                         enum xscale_counters counter)
 200 {
 201         int ret = 0;
 202
 203         switch (counter) {
 204         case XSCALE_CYCLE_COUNTER:
 205                 ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
 206                 break;
 207         case XSCALE_COUNTER0:
 208                 ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
 209                 break;
 210         case XSCALE_COUNTER1:
 211                 ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
 212                 break;
 213         default:
 214                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 215         }
 216
 217         return ret;
 218 }
 219
 220 static irqreturn_t
 221 xscale1pmu_handle_irq(int irq_num, void *dev)
 222 {
 223         unsigned long pmnc;
 224         struct perf_sample_data data;
 225         struct cpu_hw_events *cpuc;
 226         struct pt_regs *regs;
 227         int idx;
 228
 229         /*
 230          * NOTE: there's an A stepping erratum that states if an overflow
 231          *       bit already exists and another occurs, the previous
 232          *       Overflow bit gets cleared. There's no workaround.
 233          *       Fixed in B stepping or later.
 234          */
 235         pmnc = xscale1pmu_read_pmnc();
 236
 237         /*
 238          * Write the value back to clear the overflow flags. Overflow
 239          * flags remain in pmnc for use below. We also disable the PMU
 240          * while we process the interrupt.
 241          */
 242         xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 243
 244         if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
 245                 return IRQ_NONE;
 246
 247         regs = get_irq_regs();
 248
 249         perf_sample_data_init(&data, 0);
 250
 251         cpuc = &__get_cpu_var(cpu_hw_events);
 252         for (idx = 0; idx < armpmu->num_events; ++idx) {
 253                 struct perf_event *event = cpuc->events[idx];
 254                 struct hw_perf_event *hwc;
 255
 256                 if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
 257                         continue;
 258
 259                 hwc = &event->hw;
 260                 armpmu_event_update(event, hwc, idx, 1);
 261                 data.period = event->hw.last_period;
 262                 if (!armpmu_event_set_period(event, hwc, idx))
 263                         continue;
 264
 265                 if (perf_event_overflow(event, &data, regs))
 266                         armpmu->disable(hwc, idx);
 267         }
 268
 269         irq_work_run();
 270
 271         /*
 272          * Re-enable the PMU.
 273          */
 274         pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 275         xscale1pmu_write_pmnc(pmnc);
 276
 277         return IRQ_HANDLED;
 278 }
 279
 280 static void
 281 xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
 282 {
 283         unsigned long val, mask, evt, flags;
 284         struct cpu_hw_events *events = armpmu->get_hw_events();
 285
 286         switch (idx) {
 287         case XSCALE_CYCLE_COUNTER:
 288                 mask = 0;
 289                 evt = XSCALE1_CCOUNT_INT_EN;
 290                 break;
 291         case XSCALE_COUNTER0:
 292                 mask = XSCALE1_COUNT0_EVT_MASK;
 293                 evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
 294                         XSCALE1_COUNT0_INT_EN;
 295                 break;
 296         case XSCALE_COUNTER1:
 297                 mask = XSCALE1_COUNT1_EVT_MASK;
 298                 evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
 299                         XSCALE1_COUNT1_INT_EN;
 300                 break;
 301         default:
 302                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 303                 return;
 304         }
 305
 306         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 307         val = xscale1pmu_read_pmnc();
 308         val &= ~mask;
 309         val |= evt;
 310         xscale1pmu_write_pmnc(val);
 311         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 312 }
 313
 314 static void
 315 xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
 316 {
 317         unsigned long val, mask, evt, flags;
 318         struct cpu_hw_events *events = armpmu->get_hw_events();
 319
 320         switch (idx) {
 321         case XSCALE_CYCLE_COUNTER:
 322                 mask = XSCALE1_CCOUNT_INT_EN;
 323                 evt = 0;
 324                 break;
 325         case XSCALE_COUNTER0:
 326                 mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
 327                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
 328                 break;
 329         case XSCALE_COUNTER1:
 330                 mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
 331                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
 332                 break;
 333         default:
 334                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 335                 return;
 336         }
 337
 338         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 339         val = xscale1pmu_read_pmnc();
 340         val &= ~mask;
 341         val |= evt;
 342         xscale1pmu_write_pmnc(val);
 343         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 344 }
 345
 346 static int
 347 xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc,
 348                         struct hw_perf_event *event)
 349 {
 350         if (XSCALE_PERFCTR_CCNT == event->config_base) {
 351                 if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
 352                         return -EAGAIN;
 353
 354                 return XSCALE_CYCLE_COUNTER;
 355         } else {
 356                 if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
 357                         return XSCALE_COUNTER1;
 358
 359                 if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
 360                         return XSCALE_COUNTER0;
 361
 362                 return -EAGAIN;
 363         }
 364 }
 365
 366 static void
 367 xscale1pmu_start(void)
 368 {
 369         unsigned long flags, val;
 370         struct cpu_hw_events *events = armpmu->get_hw_events();
 371
 372         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 373         val = xscale1pmu_read_pmnc();
 374         val |= XSCALE_PMU_ENABLE;
 375         xscale1pmu_write_pmnc(val);
 376         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 377 }
 378
 379 static void
 380 xscale1pmu_stop(void)
 381 {
 382         unsigned long flags, val;
 383         struct cpu_hw_events *events = armpmu->get_hw_events();
 384
 385         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 386         val = xscale1pmu_read_pmnc();
 387         val &= ~XSCALE_PMU_ENABLE;
 388         xscale1pmu_write_pmnc(val);
 389         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 390 }
 391
 392 static inline u32
 393 xscale1pmu_read_counter(int counter)
 394 {
 395         u32 val = 0;
 396
 397         switch (counter) {
 398         case XSCALE_CYCLE_COUNTER:
 399                 asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
 400                 break;
 401         case XSCALE_COUNTER0:
 402                 asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
 403                 break;
 404         case XSCALE_COUNTER1:
 405                 asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
 406                 break;
 407         }
 408
 409         return val;
 410 }
 411
 412 static inline void
 413 xscale1pmu_write_counter(int counter, u32 val)
 414 {
 415         switch (counter) {
 416         case XSCALE_CYCLE_COUNTER:
 417                 asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
 418                 break;
 419         case XSCALE_COUNTER0:
 420                 asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
 421                 break;
 422         case XSCALE_COUNTER1:
 423                 asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
 424                 break;
 425         }
 426 }
 427
 428 static int xscale_map_event(struct perf_event *event)
 429 {
 430         return map_cpu_event(event, &xscale_perf_map,
 431                                 &xscale_perf_cache_map, 0xFF);
 432 }
 433
 434 static struct arm_pmu xscale1pmu = {
 435         .id             = ARM_PERF_PMU_ID_XSCALE1,
 436         .name           = "xscale1",
 437         .handle_irq     = xscale1pmu_handle_irq,
 438         .enable         = xscale1pmu_enable_event,
 439         .disable        = xscale1pmu_disable_event,
 440         .read_counter   = xscale1pmu_read_counter,
 441         .write_counter  = xscale1pmu_write_counter,
 442         .get_event_idx  = xscale1pmu_get_event_idx,
 443         .start          = xscale1pmu_start,
 444         .stop           = xscale1pmu_stop,
 445         .map_event      = xscale_map_event,
 446         .num_events     = 3,
 447         .max_period     = (1LLU << 32) - 1,
 448 };
 449
 450 static struct arm_pmu *__init xscale1pmu_init(void)
 451 {
 452         return &xscale1pmu;
 453 }
 454
 455 #define XSCALE2_OVERFLOWED_MASK 0x01f
 456 #define XSCALE2_CCOUNT_OVERFLOW 0x001
 457 #define XSCALE2_COUNT0_OVERFLOW 0x002
 458 #define XSCALE2_COUNT1_OVERFLOW 0x004
 459 #define XSCALE2_COUNT2_OVERFLOW 0x008
 460 #define XSCALE2_COUNT3_OVERFLOW 0x010
 461 #define XSCALE2_CCOUNT_INT_EN   0x001
 462 #define XSCALE2_COUNT0_INT_EN   0x002
 463 #define XSCALE2_COUNT1_INT_EN   0x004
 464 #define XSCALE2_COUNT2_INT_EN   0x008
 465 #define XSCALE2_COUNT3_INT_EN   0x010
 466 #define XSCALE2_COUNT0_EVT_SHFT 0
 467 #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
 468 #define XSCALE2_COUNT1_EVT_SHFT 8
 469 #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
 470 #define XSCALE2_COUNT2_EVT_SHFT 16
 471 #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
 472 #define XSCALE2_COUNT3_EVT_SHFT 24
 473 #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
 474
 475 static inline u32
 476 xscale2pmu_read_pmnc(void)
 477 {
 478         u32 val;
 479         asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
 480         /* bits 1-2 and 4-23 are read-unpredictable */
 481         return val & 0xff000009;
 482 }
 483
 484 static inline void
 485 xscale2pmu_write_pmnc(u32 val)
 486 {
 487         /* bits 4-23 are write-as-0, 24-31 are write ignored */
 488         val &= 0xf;
 489         asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
 490 }
 491
 492 static inline u32
 493 xscale2pmu_read_overflow_flags(void)
 494 {
 495         u32 val;
 496         asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
 497         return val;
 498 }
 499
 500 static inline void
 501 xscale2pmu_write_overflow_flags(u32 val)
 502 {
 503         asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
 504 }
 505
 506 static inline u32
 507 xscale2pmu_read_event_select(void)
 508 {
 509         u32 val;
 510         asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
 511         return val;
 512 }
 513
 514 static inline void
 515 xscale2pmu_write_event_select(u32 val)
 516 {
 517         asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
 518 }
 519
 520 static inline u32
 521 xscale2pmu_read_int_enable(void)
 522 {
 523         u32 val;
 524         asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
 525         return val;
 526 }
 527
 528 static void
 529 xscale2pmu_write_int_enable(u32 val)
 530 {
 531         asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
 532 }
 533
 534 static inline int
 535 xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
 536                                         enum xscale_counters counter)
 537 {
 538         int ret = 0;
 539
 540         switch (counter) {
 541         case XSCALE_CYCLE_COUNTER:
 542                 ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
 543                 break;
 544         case XSCALE_COUNTER0:
 545                 ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
 546                 break;
 547         case XSCALE_COUNTER1:
 548                 ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
 549                 break;
 550         case XSCALE_COUNTER2:
 551                 ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
 552                 break;
 553         case XSCALE_COUNTER3:
 554                 ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
 555                 break;
 556         default:
 557                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 558         }
 559
 560         return ret;
 561 }
 562
 563 static irqreturn_t
 564 xscale2pmu_handle_irq(int irq_num, void *dev)
 565 {
 566         unsigned long pmnc, of_flags;
 567         struct perf_sample_data data;
 568         struct cpu_hw_events *cpuc;
 569         struct pt_regs *regs;
 570         int idx;
 571
 572         /* Disable the PMU. */
 573         pmnc = xscale2pmu_read_pmnc();
 574         xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 575
 576         /* Check the overflow flag register. */
 577         of_flags = xscale2pmu_read_overflow_flags();
 578         if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
 579                 return IRQ_NONE;
 580
 581         /* Clear the overflow bits. */
 582         xscale2pmu_write_overflow_flags(of_flags);
 583
 584         regs = get_irq_regs();
 585
 586         perf_sample_data_init(&data, 0);
 587
 588         cpuc = &__get_cpu_var(cpu_hw_events);
 589         for (idx = 0; idx < armpmu->num_events; ++idx) {
 590                 struct perf_event *event = cpuc->events[idx];
 591                 struct hw_perf_event *hwc;
 592
 593                 if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx))
 594                         continue;
 595
 596                 hwc = &event->hw;
 597                 armpmu_event_update(event, hwc, idx, 1);
 598                 data.period = event->hw.last_period;
 599                 if (!armpmu_event_set_period(event, hwc, idx))
 600                         continue;
 601
 602                 if (perf_event_overflow(event, &data, regs))
 603                         armpmu->disable(hwc, idx);
 604         }
 605
 606         irq_work_run();
 607
 608         /*
 609          * Re-enable the PMU.
 610          */
 611         pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 612         xscale2pmu_write_pmnc(pmnc);
 613
 614         return IRQ_HANDLED;
 615 }
 616
 617 static void
 618 xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
 619 {
 620         unsigned long flags, ien, evtsel;
 621         struct cpu_hw_events *events = armpmu->get_hw_events();
 622
 623         ien = xscale2pmu_read_int_enable();
 624         evtsel = xscale2pmu_read_event_select();
 625
 626         switch (idx) {
 627         case XSCALE_CYCLE_COUNTER:
 628                 ien |= XSCALE2_CCOUNT_INT_EN;
 629                 break;
 630         case XSCALE_COUNTER0:
 631                 ien |= XSCALE2_COUNT0_INT_EN;
 632                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 633                 evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
 634                 break;
 635         case XSCALE_COUNTER1:
 636                 ien |= XSCALE2_COUNT1_INT_EN;
 637                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 638                 evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
 639                 break;
 640         case XSCALE_COUNTER2:
 641                 ien |= XSCALE2_COUNT2_INT_EN;
 642                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 643                 evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
 644                 break;
 645         case XSCALE_COUNTER3:
 646                 ien |= XSCALE2_COUNT3_INT_EN;
 647                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 648                 evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
 649                 break;
 650         default:
 651                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 652                 return;
 653         }
 654
 655         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 656         xscale2pmu_write_event_select(evtsel);
 657         xscale2pmu_write_int_enable(ien);
 658         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 659 }
 660
 661 static void
 662 xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
 663 {
 664         unsigned long flags, ien, evtsel;
 665         struct cpu_hw_events *events = armpmu->get_hw_events();
 666
 667         ien = xscale2pmu_read_int_enable();
 668         evtsel = xscale2pmu_read_event_select();
 669
 670         switch (idx) {
 671         case XSCALE_CYCLE_COUNTER:
 672                 ien &= ~XSCALE2_CCOUNT_INT_EN;
 673                 break;
 674         case XSCALE_COUNTER0:
 675                 ien &= ~XSCALE2_COUNT0_INT_EN;
 676                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 677                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
 678                 break;
 679         case XSCALE_COUNTER1:
 680                 ien &= ~XSCALE2_COUNT1_INT_EN;
 681                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 682                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
 683                 break;
 684         case XSCALE_COUNTER2:
 685                 ien &= ~XSCALE2_COUNT2_INT_EN;
 686                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 687                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
 688                 break;
 689         case XSCALE_COUNTER3:
 690                 ien &= ~XSCALE2_COUNT3_INT_EN;
 691                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 692                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
 693                 break;
 694         default:
 695                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 696                 return;
 697         }
 698
 699         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 700         xscale2pmu_write_event_select(evtsel);
 701         xscale2pmu_write_int_enable(ien);
 702         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 703 }
 704
 705 static int
 706 xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc,
 707                         struct hw_perf_event *event)
 708 {
 709         int idx = xscale1pmu_get_event_idx(cpuc, event);
 710         if (idx >= 0)
 711                 goto out;
 712
 713         if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
 714                 idx = XSCALE_COUNTER3;
 715         else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
 716                 idx = XSCALE_COUNTER2;
 717 out:
 718         return idx;
 719 }
 720
 721 static void
 722 xscale2pmu_start(void)
 723 {
 724         unsigned long flags, val;
 725         struct cpu_hw_events *events = armpmu->get_hw_events();
 726
 727         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 728         val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
 729         val |= XSCALE_PMU_ENABLE;
 730         xscale2pmu_write_pmnc(val);
 731         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 732 }
 733
 734 static void
 735 xscale2pmu_stop(void)
 736 {
 737         unsigned long flags, val;
 738         struct cpu_hw_events *events = armpmu->get_hw_events();
 739
 740         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 741         val = xscale2pmu_read_pmnc();
 742         val &= ~XSCALE_PMU_ENABLE;
 743         xscale2pmu_write_pmnc(val);
 744         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 745 }
 746
 747 static inline u32
 748 xscale2pmu_read_counter(int counter)
 749 {
 750         u32 val = 0;
 751
 752         switch (counter) {
 753         case XSCALE_CYCLE_COUNTER:
 754                 asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
 755                 break;
 756         case XSCALE_COUNTER0:
 757                 asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
 758                 break;
 759         case XSCALE_COUNTER1:
 760                 asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
 761                 break;
 762         case XSCALE_COUNTER2:
 763                 asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
 764                 break;
 765         case XSCALE_COUNTER3:
 766                 asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
 767                 break;
 768         }
 769
 770         return val;
 771 }
 772
 773 static inline void
 774 xscale2pmu_write_counter(int counter, u32 val)
 775 {
 776         switch (counter) {
 777         case XSCALE_CYCLE_COUNTER:
 778                 asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
 779                 break;
 780         case XSCALE_COUNTER0:
 781                 asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
 782                 break;
 783         case XSCALE_COUNTER1:
 784                 asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
 785                 break;
 786         case XSCALE_COUNTER2:
 787                 asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
 788                 break;
 789         case XSCALE_COUNTER3:
 790                 asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
 791                 break;
 792         }
 793 }
 794
 795 static struct arm_pmu xscale2pmu = {
 796         .id             = ARM_PERF_PMU_ID_XSCALE2,
 797         .name           = "xscale2",
 798         .handle_irq     = xscale2pmu_handle_irq,
 799         .enable         = xscale2pmu_enable_event,
 800         .disable        = xscale2pmu_disable_event,
 801         .read_counter   = xscale2pmu_read_counter,
 802         .write_counter  = xscale2pmu_write_counter,
 803         .get_event_idx  = xscale2pmu_get_event_idx,
 804         .start          = xscale2pmu_start,
 805         .stop           = xscale2pmu_stop,
 806         .map_event      = xscale_map_event,
 807         .num_events     = 5,
 808         .max_period     = (1LLU << 32) - 1,
 809 };
 810
 811 static struct arm_pmu *__init xscale2pmu_init(void)
 812 {
 813         return &xscale2pmu;
 814 }
 815 #else
 816 static struct arm_pmu *__init xscale1pmu_init(void)
 817 {
 818         return NULL;
 819 }
 820
 821 static struct arm_pmu *__init xscale2pmu_init(void)
 822 {
 823         return NULL;
 824 }
 825 #endif  /* CONFIG_CPU_XSCALE */