arch/arm/kernel/hw_breakpoint.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License version 2 as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  14  *
  15  * Copyright (C) 2009, 2010 ARM Limited
  16  *
  17  * Author: Will Deacon <will.deacon@arm.com>
  18  */
  19
  20 /*
  21  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
  22  * using the CPU's debug registers.
  23  */
  24 #define pr_fmt(fmt) "hw-breakpoint: " fmt
  25
  26 #include <linux/errno.h>
  27 #include <linux/hardirq.h>
  28 #include <linux/perf_event.h>
  29 #include <linux/hw_breakpoint.h>
  30 #include <linux/smp.h>
  31 #include <linux/cpu_pm.h>
  32 #include <linux/coresight.h>
  33
  34 #include <asm/cacheflush.h>
  35 #include <asm/cputype.h>
  36 #include <asm/current.h>
  37 #include <asm/hw_breakpoint.h>
  38 #include <asm/traps.h>
  39
  40 /* Breakpoint currently in use for each BRP. */
  41 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
  42
  43 /* Watchpoint currently in use for each WRP. */
  44 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
  45
  46 /* Number of BRP/WRP registers on this CPU. */
  47 static int core_num_brps __ro_after_init;
  48 static int core_num_wrps __ro_after_init;
  49
  50 /* Debug architecture version. */
  51 static u8 debug_arch __ro_after_init;
  52
  53 /* Does debug architecture support OS Save and Restore? */
  54 static bool has_ossr __ro_after_init;
  55
  56 /* Maximum supported watchpoint length. */
  57 static u8 max_watchpoint_len __ro_after_init;
  58
  59 #define READ_WB_REG_CASE(OP2, M, VAL)                   \
  60         case ((OP2 << 4) + M):                          \
  61                 ARM_DBG_READ(c0, c ## M, OP2, VAL);     \
  62                 break
  63
  64 #define WRITE_WB_REG_CASE(OP2, M, VAL)                  \
  65         case ((OP2 << 4) + M):                          \
  66                 ARM_DBG_WRITE(c0, c ## M, OP2, VAL);    \
  67                 break
  68
  69 #define GEN_READ_WB_REG_CASES(OP2, VAL)         \
  70         READ_WB_REG_CASE(OP2, 0, VAL);          \
  71         READ_WB_REG_CASE(OP2, 1, VAL);          \
  72         READ_WB_REG_CASE(OP2, 2, VAL);          \
  73         READ_WB_REG_CASE(OP2, 3, VAL);          \
  74         READ_WB_REG_CASE(OP2, 4, VAL);          \
  75         READ_WB_REG_CASE(OP2, 5, VAL);          \
  76         READ_WB_REG_CASE(OP2, 6, VAL);          \
  77         READ_WB_REG_CASE(OP2, 7, VAL);          \
  78         READ_WB_REG_CASE(OP2, 8, VAL);          \
  79         READ_WB_REG_CASE(OP2, 9, VAL);          \
  80         READ_WB_REG_CASE(OP2, 10, VAL);         \
  81         READ_WB_REG_CASE(OP2, 11, VAL);         \
  82         READ_WB_REG_CASE(OP2, 12, VAL);         \
  83         READ_WB_REG_CASE(OP2, 13, VAL);         \
  84         READ_WB_REG_CASE(OP2, 14, VAL);         \
  85         READ_WB_REG_CASE(OP2, 15, VAL)
  86
  87 #define GEN_WRITE_WB_REG_CASES(OP2, VAL)        \
  88         WRITE_WB_REG_CASE(OP2, 0, VAL);         \
  89         WRITE_WB_REG_CASE(OP2, 1, VAL);         \
  90         WRITE_WB_REG_CASE(OP2, 2, VAL);         \
  91         WRITE_WB_REG_CASE(OP2, 3, VAL);         \
  92         WRITE_WB_REG_CASE(OP2, 4, VAL);         \
  93         WRITE_WB_REG_CASE(OP2, 5, VAL);         \
  94         WRITE_WB_REG_CASE(OP2, 6, VAL);         \
  95         WRITE_WB_REG_CASE(OP2, 7, VAL);         \
  96         WRITE_WB_REG_CASE(OP2, 8, VAL);         \
  97         WRITE_WB_REG_CASE(OP2, 9, VAL);         \
  98         WRITE_WB_REG_CASE(OP2, 10, VAL);        \
  99         WRITE_WB_REG_CASE(OP2, 11, VAL);        \
 100         WRITE_WB_REG_CASE(OP2, 12, VAL);        \
 101         WRITE_WB_REG_CASE(OP2, 13, VAL);        \
 102         WRITE_WB_REG_CASE(OP2, 14, VAL);        \
 103         WRITE_WB_REG_CASE(OP2, 15, VAL)
 104
 105 static u32 read_wb_reg(int n)
 106 {
 107         u32 val = 0;
 108
 109         switch (n) {
 110         GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
 111         GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
 112         GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
 113         GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
 114         default:
 115                 pr_warn("attempt to read from unknown breakpoint register %d\n",
 116                         n);
 117         }
 118
 119         return val;
 120 }
 121
 122 static void write_wb_reg(int n, u32 val)
 123 {
 124         switch (n) {
 125         GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
 126         GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
 127         GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
 128         GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
 129         default:
 130                 pr_warn("attempt to write to unknown breakpoint register %d\n",
 131                         n);
 132         }
 133         isb();
 134 }
 135
 136 /* Determine debug architecture. */
 137 static u8 get_debug_arch(void)
 138 {
 139         u32 didr;
 140
 141         /* Do we implement the extended CPUID interface? */
 142         if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
 143                 pr_warn_once("CPUID feature registers not supported. "
 144                              "Assuming v6 debug is present.\n");
 145                 return ARM_DEBUG_ARCH_V6;
 146         }
 147
 148         ARM_DBG_READ(c0, c0, 0, didr);
 149         return (didr >> 16) & 0xf;
 150 }
 151
 152 u8 arch_get_debug_arch(void)
 153 {
 154         return debug_arch;
 155 }
 156
 157 static int debug_arch_supported(void)
 158 {
 159         u8 arch = get_debug_arch();
 160
 161         /* We don't support the memory-mapped interface. */
 162         return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
 163                 arch >= ARM_DEBUG_ARCH_V7_1;
 164 }
 165
 166 /* Can we determine the watchpoint access type from the fsr? */
 167 static int debug_exception_updates_fsr(void)
 168 {
 169         return get_debug_arch() >= ARM_DEBUG_ARCH_V8;
 170 }
 171
 172 /* Determine number of WRP registers available. */
 173 static int get_num_wrp_resources(void)
 174 {
 175         u32 didr;
 176         ARM_DBG_READ(c0, c0, 0, didr);
 177         return ((didr >> 28) & 0xf) + 1;
 178 }
 179
 180 /* Determine number of BRP registers available. */
 181 static int get_num_brp_resources(void)
 182 {
 183         u32 didr;
 184         ARM_DBG_READ(c0, c0, 0, didr);
 185         return ((didr >> 24) & 0xf) + 1;
 186 }
 187
 188 /* Does this core support mismatch breakpoints? */
 189 static int core_has_mismatch_brps(void)
 190 {
 191         return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
 192                 get_num_brp_resources() > 1);
 193 }
 194
 195 /* Determine number of usable WRPs available. */
 196 static int get_num_wrps(void)
 197 {
 198         /*
 199          * On debug architectures prior to 7.1, when a watchpoint fires, the
 200          * only way to work out which watchpoint it was is by disassembling
 201          * the faulting instruction and working out the address of the memory
 202          * access.
 203          *
 204          * Furthermore, we can only do this if the watchpoint was precise
 205          * since imprecise watchpoints prevent us from calculating register
 206          * based addresses.
 207          *
 208          * Providing we have more than 1 breakpoint register, we only report
 209          * a single watchpoint register for the time being. This way, we always
 210          * know which watchpoint fired. In the future we can either add a
 211          * disassembler and address generation emulator, or we can insert a
 212          * check to see if the DFAR is set on watchpoint exception entry
 213          * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
 214          * that it is set on some implementations].
 215          */
 216         if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
 217                 return 1;
 218
 219         return get_num_wrp_resources();
 220 }
 221
 222 /* Determine number of usable BRPs available. */
 223 static int get_num_brps(void)
 224 {
 225         int brps = get_num_brp_resources();
 226         return core_has_mismatch_brps() ? brps - 1 : brps;
 227 }
 228
 229 /*
 230  * In order to access the breakpoint/watchpoint control registers,
 231  * we must be running in debug monitor mode. Unfortunately, we can
 232  * be put into halting debug mode at any time by an external debugger
 233  * but there is nothing we can do to prevent that.
 234  */
 235 static int monitor_mode_enabled(void)
 236 {
 237         u32 dscr;
 238         ARM_DBG_READ(c0, c1, 0, dscr);
 239         return !!(dscr & ARM_DSCR_MDBGEN);
 240 }
 241
 242 static int enable_monitor_mode(void)
 243 {
 244         u32 dscr;
 245         ARM_DBG_READ(c0, c1, 0, dscr);
 246
 247         /* If monitor mode is already enabled, just return. */
 248         if (dscr & ARM_DSCR_MDBGEN)
 249                 goto out;
 250
 251         /* Write to the corresponding DSCR. */
 252         switch (get_debug_arch()) {
 253         case ARM_DEBUG_ARCH_V6:
 254         case ARM_DEBUG_ARCH_V6_1:
 255                 ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
 256                 break;
 257         case ARM_DEBUG_ARCH_V7_ECP14:
 258         case ARM_DEBUG_ARCH_V7_1:
 259         case ARM_DEBUG_ARCH_V8:
 260                 ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
 261                 isb();
 262                 break;
 263         default:
 264                 return -ENODEV;
 265         }
 266
 267         /* Check that the write made it through. */
 268         ARM_DBG_READ(c0, c1, 0, dscr);
 269         if (!(dscr & ARM_DSCR_MDBGEN)) {
 270                 pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
 271                                 smp_processor_id());
 272                 return -EPERM;
 273         }
 274
 275 out:
 276         return 0;
 277 }
 278
 279 int hw_breakpoint_slots(int type)
 280 {
 281         if (!debug_arch_supported())
 282                 return 0;
 283
 284         /*
 285          * We can be called early, so don't rely on
 286          * our static variables being initialised.
 287          */
 288         switch (type) {
 289         case TYPE_INST:
 290                 return get_num_brps();
 291         case TYPE_DATA:
 292                 return get_num_wrps();
 293         default:
 294                 pr_warn("unknown slot type: %d\n", type);
 295                 return 0;
 296         }
 297 }
 298
 299 /*
 300  * Check if 8-bit byte-address select is available.
 301  * This clobbers WRP 0.
 302  */
 303 static u8 get_max_wp_len(void)
 304 {
 305         u32 ctrl_reg;
 306         struct arch_hw_breakpoint_ctrl ctrl;
 307         u8 size = 4;
 308
 309         if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
 310                 goto out;
 311
 312         memset(&ctrl, 0, sizeof(ctrl));
 313         ctrl.len = ARM_BREAKPOINT_LEN_8;
 314         ctrl_reg = encode_ctrl_reg(ctrl);
 315
 316         write_wb_reg(ARM_BASE_WVR, 0);
 317         write_wb_reg(ARM_BASE_WCR, ctrl_reg);
 318         if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
 319                 size = 8;
 320
 321 out:
 322         return size;
 323 }
 324
 325 u8 arch_get_max_wp_len(void)
 326 {
 327         return max_watchpoint_len;
 328 }
 329
 330 /*
 331  * Install a perf counter breakpoint.
 332  */
 333 int arch_install_hw_breakpoint(struct perf_event *bp)
 334 {
 335         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 336         struct perf_event **slot, **slots;
 337         int i, max_slots, ctrl_base, val_base;
 338         u32 addr, ctrl;
 339
 340         addr = info->address;
 341         ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
 342
 343         if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 344                 /* Breakpoint */
 345                 ctrl_base = ARM_BASE_BCR;
 346                 val_base = ARM_BASE_BVR;
 347                 slots = this_cpu_ptr(bp_on_reg);
 348                 max_slots = core_num_brps;
 349         } else {
 350                 /* Watchpoint */
 351                 ctrl_base = ARM_BASE_WCR;
 352                 val_base = ARM_BASE_WVR;
 353                 slots = this_cpu_ptr(wp_on_reg);
 354                 max_slots = core_num_wrps;
 355         }
 356
 357         for (i = 0; i < max_slots; ++i) {
 358                 slot = &slots[i];
 359
 360                 if (!*slot) {
 361                         *slot = bp;
 362                         break;
 363                 }
 364         }
 365
 366         if (i == max_slots) {
 367                 pr_warn("Can't find any breakpoint slot\n");
 368                 return -EBUSY;
 369         }
 370
 371         /* Override the breakpoint data with the step data. */
 372         if (info->step_ctrl.enabled) {
 373                 addr = info->trigger & ~0x3;
 374                 ctrl = encode_ctrl_reg(info->step_ctrl);
 375                 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
 376                         i = 0;
 377                         ctrl_base = ARM_BASE_BCR + core_num_brps;
 378                         val_base = ARM_BASE_BVR + core_num_brps;
 379                 }
 380         }
 381
 382         /* Setup the address register. */
 383         write_wb_reg(val_base + i, addr);
 384
 385         /* Setup the control register. */
 386         write_wb_reg(ctrl_base + i, ctrl);
 387         return 0;
 388 }
 389
 390 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 391 {
 392         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 393         struct perf_event **slot, **slots;
 394         int i, max_slots, base;
 395
 396         if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 397                 /* Breakpoint */
 398                 base = ARM_BASE_BCR;
 399                 slots = this_cpu_ptr(bp_on_reg);
 400                 max_slots = core_num_brps;
 401         } else {
 402                 /* Watchpoint */
 403                 base = ARM_BASE_WCR;
 404                 slots = this_cpu_ptr(wp_on_reg);
 405                 max_slots = core_num_wrps;
 406         }
 407
 408         /* Remove the breakpoint. */
 409         for (i = 0; i < max_slots; ++i) {
 410                 slot = &slots[i];
 411
 412                 if (*slot == bp) {
 413                         *slot = NULL;
 414                         break;
 415                 }
 416         }
 417
 418         if (i == max_slots) {
 419                 pr_warn("Can't find any breakpoint slot\n");
 420                 return;
 421         }
 422
 423         /* Ensure that we disable the mismatch breakpoint. */
 424         if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
 425             info->step_ctrl.enabled) {
 426                 i = 0;
 427                 base = ARM_BASE_BCR + core_num_brps;
 428         }
 429
 430         /* Reset the control register. */
 431         write_wb_reg(base + i, 0);
 432 }
 433
 434 static int get_hbp_len(u8 hbp_len)
 435 {
 436         unsigned int len_in_bytes = 0;
 437
 438         switch (hbp_len) {
 439         case ARM_BREAKPOINT_LEN_1:
 440                 len_in_bytes = 1;
 441                 break;
 442         case ARM_BREAKPOINT_LEN_2:
 443                 len_in_bytes = 2;
 444                 break;
 445         case ARM_BREAKPOINT_LEN_4:
 446                 len_in_bytes = 4;
 447                 break;
 448         case ARM_BREAKPOINT_LEN_8:
 449                 len_in_bytes = 8;
 450                 break;
 451         }
 452
 453         return len_in_bytes;
 454 }
 455
 456 /*
 457  * Check whether bp virtual address is in kernel space.
 458  */
 459 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
 460 {
 461         unsigned int len;
 462         unsigned long va;
 463
 464         va = hw->address;
 465         len = get_hbp_len(hw->ctrl.len);
 466
 467         return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
 468 }
 469
 470 /*
 471  * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
 472  * Hopefully this will disappear when ptrace can bypass the conversion
 473  * to generic breakpoint descriptions.
 474  */
 475 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
 476                            int *gen_len, int *gen_type)
 477 {
 478         /* Type */
 479         switch (ctrl.type) {
 480         case ARM_BREAKPOINT_EXECUTE:
 481                 *gen_type = HW_BREAKPOINT_X;
 482                 break;
 483         case ARM_BREAKPOINT_LOAD:
 484                 *gen_type = HW_BREAKPOINT_R;
 485                 break;
 486         case ARM_BREAKPOINT_STORE:
 487                 *gen_type = HW_BREAKPOINT_W;
 488                 break;
 489         case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
 490                 *gen_type = HW_BREAKPOINT_RW;
 491                 break;
 492         default:
 493                 return -EINVAL;
 494         }
 495
 496         /* Len */
 497         switch (ctrl.len) {
 498         case ARM_BREAKPOINT_LEN_1:
 499                 *gen_len = HW_BREAKPOINT_LEN_1;
 500                 break;
 501         case ARM_BREAKPOINT_LEN_2:
 502                 *gen_len = HW_BREAKPOINT_LEN_2;
 503                 break;
 504         case ARM_BREAKPOINT_LEN_4:
 505                 *gen_len = HW_BREAKPOINT_LEN_4;
 506                 break;
 507         case ARM_BREAKPOINT_LEN_8:
 508                 *gen_len = HW_BREAKPOINT_LEN_8;
 509                 break;
 510         default:
 511                 return -EINVAL;
 512         }
 513
 514         return 0;
 515 }
 516
 517 /*
 518  * Construct an arch_hw_breakpoint from a perf_event.
 519  */
 520 static int arch_build_bp_info(struct perf_event *bp,
 521                               const struct perf_event_attr *attr,
 522                               struct arch_hw_breakpoint *hw)
 523 {
 524         /* Type */
 525         switch (attr->bp_type) {
 526         case HW_BREAKPOINT_X:
 527                 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
 528                 break;
 529         case HW_BREAKPOINT_R:
 530                 hw->ctrl.type = ARM_BREAKPOINT_LOAD;
 531                 break;
 532         case HW_BREAKPOINT_W:
 533                 hw->ctrl.type = ARM_BREAKPOINT_STORE;
 534                 break;
 535         case HW_BREAKPOINT_RW:
 536                 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
 537                 break;
 538         default:
 539                 return -EINVAL;
 540         }
 541
 542         /* Len */
 543         switch (attr->bp_len) {
 544         case HW_BREAKPOINT_LEN_1:
 545                 hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
 546                 break;
 547         case HW_BREAKPOINT_LEN_2:
 548                 hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
 549                 break;
 550         case HW_BREAKPOINT_LEN_4:
 551                 hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
 552                 break;
 553         case HW_BREAKPOINT_LEN_8:
 554                 hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
 555                 if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE)
 556                         && max_watchpoint_len >= 8)
 557                         break;
 558         default:
 559                 return -EINVAL;
 560         }
 561
 562         /*
 563          * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
 564          * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
 565          * by the hardware and must be aligned to the appropriate number of
 566          * bytes.
 567          */
 568         if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
 569             hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
 570             hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
 571                 return -EINVAL;
 572
 573         /* Address */
 574         hw->address = attr->bp_addr;
 575
 576         /* Privilege */
 577         hw->ctrl.privilege = ARM_BREAKPOINT_USER;
 578         if (arch_check_bp_in_kernelspace(hw))
 579                 hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
 580
 581         /* Enabled? */
 582         hw->ctrl.enabled = !attr->disabled;
 583
 584         /* Mismatch */
 585         hw->ctrl.mismatch = 0;
 586
 587         return 0;
 588 }
 589
 590 /*
 591  * Validate the arch-specific HW Breakpoint register settings.
 592  */
 593 int hw_breakpoint_arch_parse(struct perf_event *bp,
 594                              const struct perf_event_attr *attr,
 595                              struct arch_hw_breakpoint *hw)
 596 {
 597         int ret = 0;
 598         u32 offset, alignment_mask = 0x3;
 599
 600         /* Ensure that we are in monitor debug mode. */
 601         if (!monitor_mode_enabled())
 602                 return -ENODEV;
 603
 604         /* Build the arch_hw_breakpoint. */
 605         ret = arch_build_bp_info(bp, attr, hw);
 606         if (ret)
 607                 goto out;
 608
 609         /* Check address alignment. */
 610         if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
 611                 alignment_mask = 0x7;
 612         offset = hw->address & alignment_mask;
 613         switch (offset) {
 614         case 0:
 615                 /* Aligned */
 616                 break;
 617         case 1:
 618         case 2:
 619                 /* Allow halfword watchpoints and breakpoints. */
 620                 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
 621                         break;
 622         case 3:
 623                 /* Allow single byte watchpoint. */
 624                 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
 625                         break;
 626         default:
 627                 ret = -EINVAL;
 628                 goto out;
 629         }
 630
 631         hw->address &= ~alignment_mask;
 632         hw->ctrl.len <<= offset;
 633
 634         if (is_default_overflow_handler(bp)) {
 635                 /*
 636                  * Mismatch breakpoints are required for single-stepping
 637                  * breakpoints.
 638                  */
 639                 if (!core_has_mismatch_brps())
 640                         return -EINVAL;
 641
 642                 /* We don't allow mismatch breakpoints in kernel space. */
 643                 if (arch_check_bp_in_kernelspace(hw))
 644                         return -EPERM;
 645
 646                 /*
 647                  * Per-cpu breakpoints are not supported by our stepping
 648                  * mechanism.
 649                  */
 650                 if (!bp->hw.target)
 651                         return -EINVAL;
 652
 653                 /*
 654                  * We only support specific access types if the fsr
 655                  * reports them.
 656                  */
 657                 if (!debug_exception_updates_fsr() &&
 658                     (hw->ctrl.type == ARM_BREAKPOINT_LOAD ||
 659                      hw->ctrl.type == ARM_BREAKPOINT_STORE))
 660                         return -EINVAL;
 661         }
 662
 663 out:
 664         return ret;
 665 }
 666
 667 /*
 668  * Enable/disable single-stepping over the breakpoint bp at address addr.
 669  */
 670 static void enable_single_step(struct perf_event *bp, u32 addr)
 671 {
 672         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 673
 674         arch_uninstall_hw_breakpoint(bp);
 675         info->step_ctrl.mismatch  = 1;
 676         info->step_ctrl.len       = ARM_BREAKPOINT_LEN_4;
 677         info->step_ctrl.type      = ARM_BREAKPOINT_EXECUTE;
 678         info->step_ctrl.privilege = info->ctrl.privilege;
 679         info->step_ctrl.enabled   = 1;
 680         info->trigger             = addr;
 681         arch_install_hw_breakpoint(bp);
 682 }
 683
 684 static void disable_single_step(struct perf_event *bp)
 685 {
 686         arch_uninstall_hw_breakpoint(bp);
 687         counter_arch_bp(bp)->step_ctrl.enabled = 0;
 688         arch_install_hw_breakpoint(bp);
 689 }
 690
 691 static void watchpoint_handler(unsigned long addr, unsigned int fsr,
 692                                struct pt_regs *regs)
 693 {
 694         int i, access;
 695         u32 val, ctrl_reg, alignment_mask;
 696         struct perf_event *wp, **slots;
 697         struct arch_hw_breakpoint *info;
 698         struct arch_hw_breakpoint_ctrl ctrl;
 699
 700         slots = this_cpu_ptr(wp_on_reg);
 701
 702         for (i = 0; i < core_num_wrps; ++i) {
 703                 rcu_read_lock();
 704
 705                 wp = slots[i];
 706
 707                 if (wp == NULL)
 708                         goto unlock;
 709
 710                 info = counter_arch_bp(wp);
 711                 /*
 712                  * The DFAR is an unknown value on debug architectures prior
 713                  * to 7.1. Since we only allow a single watchpoint on these
 714                  * older CPUs, we can set the trigger to the lowest possible
 715                  * faulting address.
 716                  */
 717                 if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
 718                         BUG_ON(i > 0);
 719                         info->trigger = wp->attr.bp_addr;
 720                 } else {
 721                         if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 722                                 alignment_mask = 0x7;
 723                         else
 724                                 alignment_mask = 0x3;
 725
 726                         /* Check if the watchpoint value matches. */
 727                         val = read_wb_reg(ARM_BASE_WVR + i);
 728                         if (val != (addr & ~alignment_mask))
 729                                 goto unlock;
 730
 731                         /* Possible match, check the byte address select. */
 732                         ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
 733                         decode_ctrl_reg(ctrl_reg, &ctrl);
 734                         if (!((1 << (addr & alignment_mask)) & ctrl.len))
 735                                 goto unlock;
 736
 737                         /* Check that the access type matches. */
 738                         if (debug_exception_updates_fsr()) {
 739                                 access = (fsr & ARM_FSR_ACCESS_MASK) ?
 740                                           HW_BREAKPOINT_W : HW_BREAKPOINT_R;
 741                                 if (!(access & hw_breakpoint_type(wp)))
 742                                         goto unlock;
 743                         }
 744
 745                         /* We have a winner. */
 746                         info->trigger = addr;
 747                 }
 748
 749                 pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
 750                 perf_bp_event(wp, regs);
 751
 752                 /*
 753                  * If no overflow handler is present, insert a temporary
 754                  * mismatch breakpoint so we can single-step over the
 755                  * watchpoint trigger.
 756                  */
 757                 if (is_default_overflow_handler(wp))
 758                         enable_single_step(wp, instruction_pointer(regs));
 759
 760 unlock:
 761                 rcu_read_unlock();
 762         }
 763 }
 764
 765 static void watchpoint_single_step_handler(unsigned long pc)
 766 {
 767         int i;
 768         struct perf_event *wp, **slots;
 769         struct arch_hw_breakpoint *info;
 770
 771         slots = this_cpu_ptr(wp_on_reg);
 772
 773         for (i = 0; i < core_num_wrps; ++i) {
 774                 rcu_read_lock();
 775
 776                 wp = slots[i];
 777
 778                 if (wp == NULL)
 779                         goto unlock;
 780
 781                 info = counter_arch_bp(wp);
 782                 if (!info->step_ctrl.enabled)
 783                         goto unlock;
 784
 785                 /*
 786                  * Restore the original watchpoint if we've completed the
 787                  * single-step.
 788                  */
 789                 if (info->trigger != pc)
 790                         disable_single_step(wp);
 791
 792 unlock:
 793                 rcu_read_unlock();
 794         }
 795 }
 796
 797 static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
 798 {
 799         int i;
 800         u32 ctrl_reg, val, addr;
 801         struct perf_event *bp, **slots;
 802         struct arch_hw_breakpoint *info;
 803         struct arch_hw_breakpoint_ctrl ctrl;
 804
 805         slots = this_cpu_ptr(bp_on_reg);
 806
 807         /* The exception entry code places the amended lr in the PC. */
 808         addr = regs->ARM_pc;
 809
 810         /* Check the currently installed breakpoints first. */
 811         for (i = 0; i < core_num_brps; ++i) {
 812                 rcu_read_lock();
 813
 814                 bp = slots[i];
 815
 816                 if (bp == NULL)
 817                         goto unlock;
 818
 819                 info = counter_arch_bp(bp);
 820
 821                 /* Check if the breakpoint value matches. */
 822                 val = read_wb_reg(ARM_BASE_BVR + i);
 823                 if (val != (addr & ~0x3))
 824                         goto mismatch;
 825
 826                 /* Possible match, check the byte address select to confirm. */
 827                 ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
 828                 decode_ctrl_reg(ctrl_reg, &ctrl);
 829                 if ((1 << (addr & 0x3)) & ctrl.len) {
 830                         info->trigger = addr;
 831                         pr_debug("breakpoint fired: address = 0x%x\n", addr);
 832                         perf_bp_event(bp, regs);
 833                         if (!bp->overflow_handler)
 834                                 enable_single_step(bp, addr);
 835                         goto unlock;
 836                 }
 837
 838 mismatch:
 839                 /* If we're stepping a breakpoint, it can now be restored. */
 840                 if (info->step_ctrl.enabled)
 841                         disable_single_step(bp);
 842 unlock:
 843                 rcu_read_unlock();
 844         }
 845
 846         /* Handle any pending watchpoint single-step breakpoints. */
 847         watchpoint_single_step_handler(addr);
 848 }
 849
 850 /*
 851  * Called from either the Data Abort Handler [watchpoint] or the
 852  * Prefetch Abort Handler [breakpoint] with interrupts disabled.
 853  */
 854 static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
 855                                  struct pt_regs *regs)
 856 {
 857         int ret = 0;
 858         u32 dscr;
 859
 860         preempt_disable();
 861
 862         if (interrupts_enabled(regs))
 863                 local_irq_enable();
 864
 865         /* We only handle watchpoints and hardware breakpoints. */
 866         ARM_DBG_READ(c0, c1, 0, dscr);
 867
 868         /* Perform perf callbacks. */
 869         switch (ARM_DSCR_MOE(dscr)) {
 870         case ARM_ENTRY_BREAKPOINT:
 871                 breakpoint_handler(addr, regs);
 872                 break;
 873         case ARM_ENTRY_ASYNC_WATCHPOINT:
 874                 WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
 875         case ARM_ENTRY_SYNC_WATCHPOINT:
 876                 watchpoint_handler(addr, fsr, regs);
 877                 break;
 878         default:
 879                 ret = 1; /* Unhandled fault. */
 880         }
 881
 882         preempt_enable();
 883
 884         return ret;
 885 }
 886
 887 /*
 888  * One-time initialisation.
 889  */
 890 static cpumask_t debug_err_mask;
 891
 892 static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
 893 {
 894         int cpu = smp_processor_id();
 895
 896         pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
 897                 instr, cpu);
 898
 899         /* Set the error flag for this CPU and skip the faulting instruction. */
 900         cpumask_set_cpu(cpu, &debug_err_mask);
 901         instruction_pointer(regs) += 4;
 902         return 0;
 903 }
 904
 905 static struct undef_hook debug_reg_hook = {
 906         .instr_mask     = 0x0fe80f10,
 907         .instr_val      = 0x0e000e10,
 908         .fn             = debug_reg_trap,
 909 };
 910
 911 /* Does this core support OS Save and Restore? */
 912 static bool core_has_os_save_restore(void)
 913 {
 914         u32 oslsr;
 915
 916         switch (get_debug_arch()) {
 917         case ARM_DEBUG_ARCH_V7_1:
 918                 return true;
 919         case ARM_DEBUG_ARCH_V7_ECP14:
 920                 ARM_DBG_READ(c1, c1, 4, oslsr);
 921                 if (oslsr & ARM_OSLSR_OSLM0)
 922                         return true;
 923         default:
 924                 return false;
 925         }
 926 }
 927
 928 static void reset_ctrl_regs(unsigned int cpu)
 929 {
 930         int i, raw_num_brps, err = 0;
 931         u32 val;
 932
 933         /*
 934          * v7 debug contains save and restore registers so that debug state
 935          * can be maintained across low-power modes without leaving the debug
 936          * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
 937          * the debug registers out of reset, so we must unlock the OS Lock
 938          * Access Register to avoid taking undefined instruction exceptions
 939          * later on.
 940          */
 941         switch (debug_arch) {
 942         case ARM_DEBUG_ARCH_V6:
 943         case ARM_DEBUG_ARCH_V6_1:
 944                 /* ARMv6 cores clear the registers out of reset. */
 945                 goto out_mdbgen;
 946         case ARM_DEBUG_ARCH_V7_ECP14:
 947                 /*
 948                  * Ensure sticky power-down is clear (i.e. debug logic is
 949                  * powered up).
 950                  */
 951                 ARM_DBG_READ(c1, c5, 4, val);
 952                 if ((val & 0x1) == 0)
 953                         err = -EPERM;
 954
 955                 if (!has_ossr)
 956                         goto clear_vcr;
 957                 break;
 958         case ARM_DEBUG_ARCH_V7_1:
 959                 /*
 960                  * Ensure the OS double lock is clear.
 961                  */
 962                 ARM_DBG_READ(c1, c3, 4, val);
 963                 if ((val & 0x1) == 1)
 964                         err = -EPERM;
 965                 break;
 966         }
 967
 968         if (err) {
 969                 pr_warn_once("CPU %d debug is powered down!\n", cpu);
 970                 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 971                 return;
 972         }
 973
 974         /*
 975          * Unconditionally clear the OS lock by writing a value
 976          * other than CS_LAR_KEY to the access register.
 977          */
 978         ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK);
 979         isb();
 980
 981         /*
 982          * Clear any configured vector-catch events before
 983          * enabling monitor mode.
 984          */
 985 clear_vcr:
 986         ARM_DBG_WRITE(c0, c7, 0, 0);
 987         isb();
 988
 989         if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
 990                 pr_warn_once("CPU %d failed to disable vector catch\n", cpu);
 991                 return;
 992         }
 993
 994         /*
 995          * The control/value register pairs are UNKNOWN out of reset so
 996          * clear them to avoid spurious debug events.
 997          */
 998         raw_num_brps = get_num_brp_resources();
 999         for (i = 0; i < raw_num_brps; ++i) {
1000                 write_wb_reg(ARM_BASE_BCR + i, 0UL);
1001                 write_wb_reg(ARM_BASE_BVR + i, 0UL);
1002         }
1003
1004         for (i = 0; i < core_num_wrps; ++i) {
1005                 write_wb_reg(ARM_BASE_WCR + i, 0UL);
1006                 write_wb_reg(ARM_BASE_WVR + i, 0UL);
1007         }
1008
1009         if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
1010                 pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu);
1011                 return;
1012         }
1013
1014         /*
1015          * Have a crack at enabling monitor mode. We don't actually need
1016          * it yet, but reporting an error early is useful if it fails.
1017          */
1018 out_mdbgen:
1019         if (enable_monitor_mode())
1020                 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1021 }
1022
1023 static int dbg_reset_online(unsigned int cpu)
1024 {
1025         local_irq_disable();
1026         reset_ctrl_regs(cpu);
1027         local_irq_enable();
1028         return 0;
1029 }
1030
1031 #ifdef CONFIG_CPU_PM
1032 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
1033                              void *v)
1034 {
1035         if (action == CPU_PM_EXIT)
1036                 reset_ctrl_regs(smp_processor_id());
1037
1038         return NOTIFY_OK;
1039 }
1040
1041 static struct notifier_block dbg_cpu_pm_nb = {
1042         .notifier_call = dbg_cpu_pm_notify,
1043 };
1044
1045 static void __init pm_init(void)
1046 {
1047         cpu_pm_register_notifier(&dbg_cpu_pm_nb);
1048 }
1049 #else
1050 static inline void pm_init(void)
1051 {
1052 }
1053 #endif
1054
1055 static int __init arch_hw_breakpoint_init(void)
1056 {
1057         int ret;
1058
1059         debug_arch = get_debug_arch();
1060
1061         if (!debug_arch_supported()) {
1062                 pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
1063                 return 0;
1064         }
1065
1066         /*
1067          * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD
1068          * whenever a WFI is issued, even if the core is not powered down, in
1069          * violation of the architecture.  When DBGPRSR.SPD is set, accesses to
1070          * breakpoint and watchpoint registers are treated as undefined, so
1071          * this results in boot time and runtime failures when these are
1072          * accessed and we unexpectedly take a trap.
1073          *
1074          * It's not clear if/how this can be worked around, so we blacklist
1075          * Scorpion CPUs to avoid these issues.
1076         */
1077         if (read_cpuid_part() == ARM_CPU_PART_SCORPION) {
1078                 pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n");
1079                 return 0;
1080         }
1081
1082         has_ossr = core_has_os_save_restore();
1083
1084         /* Determine how many BRPs/WRPs are available. */
1085         core_num_brps = get_num_brps();
1086         core_num_wrps = get_num_wrps();
1087
1088         /*
1089          * We need to tread carefully here because DBGSWENABLE may be
1090          * driven low on this core and there isn't an architected way to
1091          * determine that.
1092          */
1093         cpus_read_lock();
1094         register_undef_hook(&debug_reg_hook);
1095
1096         /*
1097          * Register CPU notifier which resets the breakpoint resources. We
1098          * assume that a halting debugger will leave the world in a nice state
1099          * for us.
1100          */
1101         ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
1102                                            "arm/hw_breakpoint:online",
1103                                            dbg_reset_online, NULL);
1104         unregister_undef_hook(&debug_reg_hook);
1105         if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
1106                 core_num_brps = 0;
1107                 core_num_wrps = 0;
1108                 if (ret > 0)
1109                         cpuhp_remove_state_nocalls_cpuslocked(ret);
1110                 cpus_read_unlock();
1111                 return 0;
1112         }
1113
1114         pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
1115                 core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
1116                 "", core_num_wrps);
1117
1118         /* Work out the maximum supported watchpoint length. */
1119         max_watchpoint_len = get_max_wp_len();
1120         pr_info("maximum watchpoint size is %u bytes.\n",
1121                         max_watchpoint_len);
1122
1123         /* Register debug fault handler. */
1124         hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1125                         TRAP_HWBKPT, "watchpoint debug exception");
1126         hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1127                         TRAP_HWBKPT, "breakpoint debug exception");
1128         cpus_read_unlock();
1129
1130         /* Register PM notifiers. */
1131         pm_init();
1132         return 0;
1133 }
1134 arch_initcall(arch_hw_breakpoint_init);
1135
1136 void hw_breakpoint_pmu_read(struct perf_event *bp)
1137 {
1138 }
1139
1140 /*
1141  * Dummy function to register with die_notifier.
1142  */
1143 int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1144                                         unsigned long val, void *data)
1145 {
1146         return NOTIFY_DONE;
1147 }