arch/x86/oprofile/nmi_int.c

   1 /**
   2  * @file nmi_int.c
   3  *
   4  * @remark Copyright 2002-2009 OProfile authors
   5  * @remark Read the file COPYING
   6  *
   7  * @author John Levon <levon@movementarian.org>
   8  * @author Robert Richter <robert.richter@amd.com>
   9  * @author Barry Kasindorf <barry.kasindorf@amd.com>
  10  * @author Jason Yeh <jason.yeh@amd.com>
  11  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  12  */
  13
  14 #include <linux/init.h>
  15 #include <linux/notifier.h>
  16 #include <linux/smp.h>
  17 #include <linux/oprofile.h>
  18 #include <linux/sysdev.h>
  19 #include <linux/slab.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kdebug.h>
  22 #include <linux/cpu.h>
  23 #include <asm/nmi.h>
  24 #include <asm/msr.h>
  25 #include <asm/apic.h>
  26
  27 #include "op_counter.h"
  28 #include "op_x86_model.h"
  29
  30 static struct op_x86_model_spec const *model;
  31 static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
  32 static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
  33
  34 /* 0 == registered but off, 1 == registered and on */
  35 static int nmi_enabled = 0;
  36
  37
  38 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  39 extern atomic_t multiplex_counter;
  40 #endif
  41
  42 struct op_counter_config counter_config[OP_MAX_COUNTER];
  43
  44 /* common functions */
  45
  46 u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
  47                     struct op_counter_config *counter_config)
  48 {
  49         u64 val = 0;
  50         u16 event = (u16)counter_config->event;
  51
  52         val |= ARCH_PERFMON_EVENTSEL_INT;
  53         val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0;
  54         val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0;
  55         val |= (counter_config->unit_mask & 0xFF) << 8;
  56         event &= model->event_mask ? model->event_mask : 0xFF;
  57         val |= event & 0xFF;
  58         val |= (event & 0x0F00) << 24;
  59
  60         return val;
  61 }
  62
  63
  64 static int profile_exceptions_notify(struct notifier_block *self,
  65                                      unsigned long val, void *data)
  66 {
  67         struct die_args *args = (struct die_args *)data;
  68         int ret = NOTIFY_DONE;
  69         int cpu = smp_processor_id();
  70
  71         switch (val) {
  72         case DIE_NMI:
  73         case DIE_NMI_IPI:
  74                 model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu));
  75                 ret = NOTIFY_STOP;
  76                 break;
  77         default:
  78                 break;
  79         }
  80         return ret;
  81 }
  82
  83 static void nmi_cpu_save_registers(struct op_msrs *msrs)
  84 {
  85         struct op_msr *counters = msrs->counters;
  86         struct op_msr *controls = msrs->controls;
  87         unsigned int i;
  88
  89         for (i = 0; i < model->num_counters; ++i) {
  90                 if (counters[i].addr)
  91                         rdmsrl(counters[i].addr, counters[i].saved);
  92         }
  93
  94         for (i = 0; i < model->num_controls; ++i) {
  95                 if (controls[i].addr)
  96                         rdmsrl(controls[i].addr, controls[i].saved);
  97         }
  98 }
  99
 100 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 101
 102 static DEFINE_PER_CPU(int, switch_index);
 103
 104 inline int op_x86_phys_to_virt(int phys)
 105 {
 106         return __get_cpu_var(switch_index) + phys;
 107 }
 108
 109 static void nmi_shutdown_mux(void)
 110 {
 111         int i;
 112         for_each_possible_cpu(i) {
 113                 kfree(per_cpu(cpu_msrs, i).multiplex);
 114                 per_cpu(cpu_msrs, i).multiplex = NULL;
 115                 per_cpu(switch_index, i) = 0;
 116         }
 117 }
 118
 119 static int nmi_setup_mux(void)
 120 {
 121         size_t multiplex_size =
 122                 sizeof(struct op_msr) * model->num_virt_counters;
 123         int i;
 124         for_each_possible_cpu(i) {
 125                 per_cpu(cpu_msrs, i).multiplex =
 126                         kmalloc(multiplex_size, GFP_KERNEL);
 127                 if (!per_cpu(cpu_msrs, i).multiplex)
 128                         return 0;
 129         }
 130         return 1;
 131 }
 132
 133 static void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs)
 134 {
 135         int i;
 136         struct op_msr *multiplex = msrs->multiplex;
 137
 138         for (i = 0; i < model->num_virt_counters; ++i) {
 139                 if (counter_config[i].enabled) {
 140                         multiplex[i].saved = -(u64)counter_config[i].count;
 141                 } else {
 142                         multiplex[i].addr  = 0;
 143                         multiplex[i].saved = 0;
 144                 }
 145         }
 146
 147         per_cpu(switch_index, cpu) = 0;
 148 }
 149
 150 static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
 151 {
 152         struct op_msr *multiplex = msrs->multiplex;
 153         int i;
 154
 155         for (i = 0; i < model->num_counters; ++i) {
 156                 int virt = op_x86_phys_to_virt(i);
 157                 if (multiplex[virt].addr)
 158                         rdmsrl(multiplex[virt].addr, multiplex[virt].saved);
 159         }
 160 }
 161
 162 static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
 163 {
 164         struct op_msr *multiplex = msrs->multiplex;
 165         int i;
 166
 167         for (i = 0; i < model->num_counters; ++i) {
 168                 int virt = op_x86_phys_to_virt(i);
 169                 if (multiplex[virt].addr)
 170                         wrmsrl(multiplex[virt].addr, multiplex[virt].saved);
 171         }
 172 }
 173
 174 #else
 175
 176 inline int op_x86_phys_to_virt(int phys) { return phys; }
 177 static inline void nmi_shutdown_mux(void) { }
 178 static inline int nmi_setup_mux(void) { return 1; }
 179 static inline void
 180 nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { }
 181
 182 #endif
 183
 184 static void free_msrs(void)
 185 {
 186         int i;
 187         for_each_possible_cpu(i) {
 188                 kfree(per_cpu(cpu_msrs, i).counters);
 189                 per_cpu(cpu_msrs, i).counters = NULL;
 190                 kfree(per_cpu(cpu_msrs, i).controls);
 191                 per_cpu(cpu_msrs, i).controls = NULL;
 192         }
 193 }
 194
 195 static int allocate_msrs(void)
 196 {
 197         size_t controls_size = sizeof(struct op_msr) * model->num_controls;
 198         size_t counters_size = sizeof(struct op_msr) * model->num_counters;
 199
 200         int i;
 201         for_each_possible_cpu(i) {
 202                 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size,
 203                                                         GFP_KERNEL);
 204                 if (!per_cpu(cpu_msrs, i).counters)
 205                         return 0;
 206                 per_cpu(cpu_msrs, i).controls = kmalloc(controls_size,
 207                                                         GFP_KERNEL);
 208                 if (!per_cpu(cpu_msrs, i).controls)
 209                         return 0;
 210         }
 211
 212         return 1;
 213 }
 214
 215 static void nmi_cpu_setup(void *dummy)
 216 {
 217         int cpu = smp_processor_id();
 218         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 219         nmi_cpu_save_registers(msrs);
 220         spin_lock(&oprofilefs_lock);
 221         model->setup_ctrs(model, msrs);
 222         nmi_cpu_setup_mux(cpu, msrs);
 223         spin_unlock(&oprofilefs_lock);
 224         per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
 225         apic_write(APIC_LVTPC, APIC_DM_NMI);
 226 }
 227
 228 static struct notifier_block profile_exceptions_nb = {
 229         .notifier_call = profile_exceptions_notify,
 230         .next = NULL,
 231         .priority = 2
 232 };
 233
 234 static int nmi_setup(void)
 235 {
 236         int err = 0;
 237         int cpu;
 238
 239         if (!allocate_msrs())
 240                 err = -ENOMEM;
 241         else if (!nmi_setup_mux())
 242                 err = -ENOMEM;
 243         else
 244                 err = register_die_notifier(&profile_exceptions_nb);
 245
 246         if (err) {
 247                 free_msrs();
 248                 nmi_shutdown_mux();
 249                 return err;
 250         }
 251
 252         /* We need to serialize save and setup for HT because the subset
 253          * of msrs are distinct for save and setup operations
 254          */
 255
 256         /* Assume saved/restored counters are the same on all CPUs */
 257         model->fill_in_addresses(&per_cpu(cpu_msrs, 0));
 258         for_each_possible_cpu(cpu) {
 259                 if (cpu != 0) {
 260                         memcpy(per_cpu(cpu_msrs, cpu).counters,
 261                                 per_cpu(cpu_msrs, 0).counters,
 262                                 sizeof(struct op_msr) * model->num_counters);
 263
 264                         memcpy(per_cpu(cpu_msrs, cpu).controls,
 265                                 per_cpu(cpu_msrs, 0).controls,
 266                                 sizeof(struct op_msr) * model->num_controls);
 267 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 268                         memcpy(per_cpu(cpu_msrs, cpu).multiplex,
 269                                 per_cpu(cpu_msrs, 0).multiplex,
 270                                 sizeof(struct op_msr) * model->num_virt_counters);
 271 #endif
 272                 }
 273         }
 274         on_each_cpu(nmi_cpu_setup, NULL, 1);
 275         nmi_enabled = 1;
 276         return 0;
 277 }
 278
 279 static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 280 {
 281         struct op_msr *counters = msrs->counters;
 282         struct op_msr *controls = msrs->controls;
 283         unsigned int i;
 284
 285         for (i = 0; i < model->num_controls; ++i) {
 286                 if (controls[i].addr)
 287                         wrmsrl(controls[i].addr, controls[i].saved);
 288         }
 289
 290         for (i = 0; i < model->num_counters; ++i) {
 291                 if (counters[i].addr)
 292                         wrmsrl(counters[i].addr, counters[i].saved);
 293         }
 294 }
 295
 296 static void nmi_cpu_shutdown(void *dummy)
 297 {
 298         unsigned int v;
 299         int cpu = smp_processor_id();
 300         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 301
 302         /* restoring APIC_LVTPC can trigger an apic error because the delivery
 303          * mode and vector nr combination can be illegal. That's by design: on
 304          * power on apic lvt contain a zero vector nr which are legal only for
 305          * NMI delivery mode. So inhibit apic err before restoring lvtpc
 306          */
 307         v = apic_read(APIC_LVTERR);
 308         apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
 309         apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
 310         apic_write(APIC_LVTERR, v);
 311         nmi_cpu_restore_registers(msrs);
 312 }
 313
 314 static void nmi_shutdown(void)
 315 {
 316         struct op_msrs *msrs;
 317
 318         nmi_enabled = 0;
 319         on_each_cpu(nmi_cpu_shutdown, NULL, 1);
 320         unregister_die_notifier(&profile_exceptions_nb);
 321         nmi_shutdown_mux();
 322         msrs = &get_cpu_var(cpu_msrs);
 323         model->shutdown(msrs);
 324         free_msrs();
 325         put_cpu_var(cpu_msrs);
 326 }
 327
 328 static void nmi_cpu_start(void *dummy)
 329 {
 330         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 331         model->start(msrs);
 332 }
 333
 334 static int nmi_start(void)
 335 {
 336         on_each_cpu(nmi_cpu_start, NULL, 1);
 337         return 0;
 338 }
 339
 340 static void nmi_cpu_stop(void *dummy)
 341 {
 342         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 343         model->stop(msrs);
 344 }
 345
 346 static void nmi_stop(void)
 347 {
 348         on_each_cpu(nmi_cpu_stop, NULL, 1);
 349 }
 350
 351 static int nmi_create_files(struct super_block *sb, struct dentry *root)
 352 {
 353         unsigned int i;
 354
 355         for (i = 0; i < model->num_virt_counters; ++i) {
 356                 struct dentry *dir;
 357                 char buf[4];
 358
 359 #ifndef CONFIG_OPROFILE_EVENT_MULTIPLEX
 360                 /* quick little hack to _not_ expose a counter if it is not
 361                  * available for use.  This should protect userspace app.
 362                  * NOTE:  assumes 1:1 mapping here (that counters are organized
 363                  *        sequentially in their struct assignment).
 364                  */
 365                 if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
 366                         continue;
 367 #endif /* CONFIG_OPROFILE_EVENT_MULTIPLEX */
 368
 369                 snprintf(buf,  sizeof(buf), "%d", i);
 370                 dir = oprofilefs_mkdir(sb, root, buf);
 371                 oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
 372                 oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event);
 373                 oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count);
 374                 oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
 375                 oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
 376                 oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
 377         }
 378
 379         return 0;
 380 }
 381
 382 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 383
 384 static void nmi_cpu_switch(void *dummy)
 385 {
 386         int cpu = smp_processor_id();
 387         int si = per_cpu(switch_index, cpu);
 388         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 389
 390         nmi_cpu_stop(NULL);
 391         nmi_cpu_save_mpx_registers(msrs);
 392
 393         /* move to next set */
 394         si += model->num_counters;
 395         if ((si > model->num_virt_counters) || (counter_config[si].count == 0))
 396                 per_cpu(switch_index, cpu) = 0;
 397         else
 398                 per_cpu(switch_index, cpu) = si;
 399
 400         model->switch_ctrl(model, msrs);
 401         nmi_cpu_restore_mpx_registers(msrs);
 402
 403         nmi_cpu_start(NULL);
 404 }
 405
 406
 407 /*
 408  * Quick check to see if multiplexing is necessary.
 409  * The check should be sufficient since counters are used
 410  * in ordre.
 411  */
 412 static int nmi_multiplex_on(void)
 413 {
 414         return counter_config[model->num_counters].count ? 0 : -EINVAL;
 415 }
 416
 417 static int nmi_switch_event(void)
 418 {
 419         if (!model->switch_ctrl)
 420                 return -ENOSYS;         /* not implemented */
 421         if (nmi_multiplex_on() < 0)
 422                 return -EINVAL;         /* not necessary */
 423
 424         on_each_cpu(nmi_cpu_switch, NULL, 1);
 425
 426         atomic_inc(&multiplex_counter);
 427
 428         return 0;
 429 }
 430
 431 #endif
 432
 433 #ifdef CONFIG_SMP
 434 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 435                                  void *data)
 436 {
 437         int cpu = (unsigned long)data;
 438         switch (action) {
 439         case CPU_DOWN_FAILED:
 440         case CPU_ONLINE:
 441                 smp_call_function_single(cpu, nmi_cpu_start, NULL, 0);
 442                 break;
 443         case CPU_DOWN_PREPARE:
 444                 smp_call_function_single(cpu, nmi_cpu_stop, NULL, 1);
 445                 break;
 446         }
 447         return NOTIFY_DONE;
 448 }
 449
 450 static struct notifier_block oprofile_cpu_nb = {
 451         .notifier_call = oprofile_cpu_notifier
 452 };
 453 #endif
 454
 455 #ifdef CONFIG_PM
 456
 457 static int nmi_suspend(struct sys_device *dev, pm_message_t state)
 458 {
 459         /* Only one CPU left, just stop that one */
 460         if (nmi_enabled == 1)
 461                 nmi_cpu_stop(NULL);
 462         return 0;
 463 }
 464
 465 static int nmi_resume(struct sys_device *dev)
 466 {
 467         if (nmi_enabled == 1)
 468                 nmi_cpu_start(NULL);
 469         return 0;
 470 }
 471
 472 static struct sysdev_class oprofile_sysclass = {
 473         .name           = "oprofile",
 474         .resume         = nmi_resume,
 475         .suspend        = nmi_suspend,
 476 };
 477
 478 static struct sys_device device_oprofile = {
 479         .id     = 0,
 480         .cls    = &oprofile_sysclass,
 481 };
 482
 483 static int __init init_sysfs(void)
 484 {
 485         int error;
 486
 487         error = sysdev_class_register(&oprofile_sysclass);
 488         if (!error)
 489                 error = sysdev_register(&device_oprofile);
 490         return error;
 491 }
 492
 493 static void exit_sysfs(void)
 494 {
 495         sysdev_unregister(&device_oprofile);
 496         sysdev_class_unregister(&oprofile_sysclass);
 497 }
 498
 499 #else
 500 #define init_sysfs() do { } while (0)
 501 #define exit_sysfs() do { } while (0)
 502 #endif /* CONFIG_PM */
 503
 504 static int __init p4_init(char **cpu_type)
 505 {
 506         __u8 cpu_model = boot_cpu_data.x86_model;
 507
 508         if (cpu_model > 6 || cpu_model == 5)
 509                 return 0;
 510
 511 #ifndef CONFIG_SMP
 512         *cpu_type = "i386/p4";
 513         model = &op_p4_spec;
 514         return 1;
 515 #else
 516         switch (smp_num_siblings) {
 517         case 1:
 518                 *cpu_type = "i386/p4";
 519                 model = &op_p4_spec;
 520                 return 1;
 521
 522         case 2:
 523                 *cpu_type = "i386/p4-ht";
 524                 model = &op_p4_ht2_spec;
 525                 return 1;
 526         }
 527 #endif
 528
 529         printk(KERN_INFO "oprofile: P4 HyperThreading detected with > 2 threads\n");
 530         printk(KERN_INFO "oprofile: Reverting to timer mode.\n");
 531         return 0;
 532 }
 533
 534 static int force_arch_perfmon;
 535 static int force_cpu_type(const char *str, struct kernel_param *kp)
 536 {
 537         if (!strcmp(str, "arch_perfmon")) {
 538                 force_arch_perfmon = 1;
 539                 printk(KERN_INFO "oprofile: forcing architectural perfmon\n");
 540         }
 541
 542         return 0;
 543 }
 544 module_param_call(cpu_type, force_cpu_type, NULL, NULL, 0);
 545
 546 static int __init ppro_init(char **cpu_type)
 547 {
 548         __u8 cpu_model = boot_cpu_data.x86_model;
 549         struct op_x86_model_spec const *spec = &op_ppro_spec;   /* default */
 550
 551         if (force_arch_perfmon && cpu_has_arch_perfmon)
 552                 return 0;
 553
 554         switch (cpu_model) {
 555         case 0 ... 2:
 556                 *cpu_type = "i386/ppro";
 557                 break;
 558         case 3 ... 5:
 559                 *cpu_type = "i386/pii";
 560                 break;
 561         case 6 ... 8:
 562         case 10 ... 11:
 563                 *cpu_type = "i386/piii";
 564                 break;
 565         case 9:
 566         case 13:
 567                 *cpu_type = "i386/p6_mobile";
 568                 break;
 569         case 14:
 570                 *cpu_type = "i386/core";
 571                 break;
 572         case 15: case 23:
 573                 *cpu_type = "i386/core_2";
 574                 break;
 575         case 26:
 576                 spec = &op_arch_perfmon_spec;
 577                 *cpu_type = "i386/core_i7";
 578                 break;
 579         case 28:
 580                 *cpu_type = "i386/atom";
 581                 break;
 582         default:
 583                 /* Unknown */
 584                 return 0;
 585         }
 586
 587         model = spec;
 588         return 1;
 589 }
 590
 591 /* in order to get sysfs right */
 592 static int using_nmi;
 593
 594 int __init op_nmi_init(struct oprofile_operations *ops)
 595 {
 596         __u8 vendor = boot_cpu_data.x86_vendor;
 597         __u8 family = boot_cpu_data.x86;
 598         char *cpu_type = NULL;
 599         int ret = 0;
 600
 601         if (!cpu_has_apic)
 602                 return -ENODEV;
 603
 604         switch (vendor) {
 605         case X86_VENDOR_AMD:
 606                 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
 607
 608                 switch (family) {
 609                 case 6:
 610                         cpu_type = "i386/athlon";
 611                         break;
 612                 case 0xf:
 613                         /*
 614                          * Actually it could be i386/hammer too, but
 615                          * give user space an consistent name.
 616                          */
 617                         cpu_type = "x86-64/hammer";
 618                         break;
 619                 case 0x10:
 620                         cpu_type = "x86-64/family10";
 621                         break;
 622                 case 0x11:
 623                         cpu_type = "x86-64/family11h";
 624                         break;
 625                 default:
 626                         return -ENODEV;
 627                 }
 628                 model = &op_amd_spec;
 629                 break;
 630
 631         case X86_VENDOR_INTEL:
 632                 switch (family) {
 633                         /* Pentium IV */
 634                 case 0xf:
 635                         p4_init(&cpu_type);
 636                         break;
 637
 638                         /* A P6-class processor */
 639                 case 6:
 640                         ppro_init(&cpu_type);
 641                         break;
 642
 643                 default:
 644                         break;
 645                 }
 646
 647                 if (cpu_type)
 648                         break;
 649
 650                 if (!cpu_has_arch_perfmon)
 651                         return -ENODEV;
 652
 653                 /* use arch perfmon as fallback */
 654                 cpu_type = "i386/arch_perfmon";
 655                 model = &op_arch_perfmon_spec;
 656                 break;
 657
 658         default:
 659                 return -ENODEV;
 660         }
 661
 662 #ifdef CONFIG_SMP
 663         register_cpu_notifier(&oprofile_cpu_nb);
 664 #endif
 665         /* default values, can be overwritten by model */
 666         ops->create_files       = nmi_create_files;
 667         ops->setup              = nmi_setup;
 668         ops->shutdown           = nmi_shutdown;
 669         ops->start              = nmi_start;
 670         ops->stop               = nmi_stop;
 671         ops->cpu_type           = cpu_type;
 672 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 673         ops->switch_events      = nmi_switch_event;
 674 #endif
 675
 676         if (model->init)
 677                 ret = model->init(ops);
 678         if (ret)
 679                 return ret;
 680
 681         init_sysfs();
 682         using_nmi = 1;
 683         printk(KERN_INFO "oprofile: using NMI interrupt.\n");
 684         return 0;
 685 }
 686
 687 void op_nmi_exit(void)
 688 {
 689         if (using_nmi) {
 690                 exit_sysfs();
 691 #ifdef CONFIG_SMP
 692                 unregister_cpu_notifier(&oprofile_cpu_nb);
 693 #endif
 694         }
 695         if (model->exit)
 696                 model->exit();
 697 }