arch/x86/kernel/hw_breakpoint.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License as published by
   4  * the Free Software Foundation; either version 2 of the License, or
   5  * (at your option) any later version.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * along with this program; if not, write to the Free Software
  14  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15  *
  16  * Copyright (C) 2007 Alan Stern
  17  * Copyright (C) 2009 IBM Corporation
  18  * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
  19  */
  20
  21 /*
  22  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
  23  * using the CPU's debug registers.
  24  */
  25
  26 #include <linux/perf_event.h>
  27 #include <linux/hw_breakpoint.h>
  28 #include <linux/irqflags.h>
  29 #include <linux/notifier.h>
  30 #include <linux/kallsyms.h>
  31 #include <linux/kprobes.h>
  32 #include <linux/percpu.h>
  33 #include <linux/kdebug.h>
  34 #include <linux/kernel.h>
  35 #include <linux/module.h>
  36 #include <linux/sched.h>
  37 #include <linux/init.h>
  38 #include <linux/smp.h>
  39
  40 #include <asm/hw_breakpoint.h>
  41 #include <asm/processor.h>
  42 #include <asm/debugreg.h>
  43
  44 /* Per cpu debug control register value */
  45 DEFINE_PER_CPU(unsigned long, dr7);
  46
  47 /* Per cpu debug address registers values */
  48 static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
  49
  50 /*
  51  * Stores the breakpoints currently in use on each breakpoint address
  52  * register for each cpus
  53  */
  54 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
  55
  56
  57 /*
  58  * Encode the length, type, Exact, and Enable bits for a particular breakpoint
  59  * as stored in debug register 7.
  60  */
  61 unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
  62 {
  63         unsigned long bp_info;
  64
  65         bp_info = (len | type) & 0xf;
  66         bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
  67         bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)) |
  68                                 DR_GLOBAL_SLOWDOWN;
  69         return bp_info;
  70 }
  71
  72 /*
  73  * Decode the length and type bits for a particular breakpoint as
  74  * stored in debug register 7.  Return the "enabled" status.
  75  */
  76 int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
  77 {
  78         int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
  79
  80         *len = (bp_info & 0xc) | 0x40;
  81         *type = (bp_info & 0x3) | 0x80;
  82
  83         return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
  84 }
  85
  86 /*
  87  * Install a perf counter breakpoint.
  88  *
  89  * We seek a free debug address register and use it for this
  90  * breakpoint. Eventually we enable it in the debug control register.
  91  *
  92  * Atomic: we hold the counter->ctx->lock and we only handle variables
  93  * and registers local to this cpu.
  94  */
  95 int arch_install_hw_breakpoint(struct perf_event *bp)
  96 {
  97         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
  98         unsigned long *dr7;
  99         int i;
 100
 101         for (i = 0; i < HBP_NUM; i++) {
 102                 struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
 103
 104                 if (!*slot) {
 105                         *slot = bp;
 106                         break;
 107                 }
 108         }
 109
 110         if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
 111                 return -EBUSY;
 112
 113         set_debugreg(info->address, i);
 114         __get_cpu_var(cpu_debugreg[i]) = info->address;
 115
 116         dr7 = &__get_cpu_var(dr7);
 117         *dr7 |= encode_dr7(i, info->len, info->type);
 118
 119         set_debugreg(*dr7, 7);
 120
 121         return 0;
 122 }
 123
 124 /*
 125  * Uninstall the breakpoint contained in the given counter.
 126  *
 127  * First we search the debug address register it uses and then we disable
 128  * it.
 129  *
 130  * Atomic: we hold the counter->ctx->lock and we only handle variables
 131  * and registers local to this cpu.
 132  */
 133 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 134 {
 135         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 136         unsigned long *dr7;
 137         int i;
 138
 139         for (i = 0; i < HBP_NUM; i++) {
 140                 struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
 141
 142                 if (*slot == bp) {
 143                         *slot = NULL;
 144                         break;
 145                 }
 146         }
 147
 148         if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
 149                 return;
 150
 151         dr7 = &__get_cpu_var(dr7);
 152         *dr7 &= ~encode_dr7(i, info->len, info->type);
 153
 154         set_debugreg(*dr7, 7);
 155 }
 156
 157 static int get_hbp_len(u8 hbp_len)
 158 {
 159         unsigned int len_in_bytes = 0;
 160
 161         switch (hbp_len) {
 162         case X86_BREAKPOINT_LEN_1:
 163                 len_in_bytes = 1;
 164                 break;
 165         case X86_BREAKPOINT_LEN_2:
 166                 len_in_bytes = 2;
 167                 break;
 168         case X86_BREAKPOINT_LEN_4:
 169                 len_in_bytes = 4;
 170                 break;
 171 #ifdef CONFIG_X86_64
 172         case X86_BREAKPOINT_LEN_8:
 173                 len_in_bytes = 8;
 174                 break;
 175 #endif
 176         }
 177         return len_in_bytes;
 178 }
 179
 180 /*
 181  * Check for virtual address in user space.
 182  */
 183 int arch_check_va_in_userspace(unsigned long va, u8 hbp_len)
 184 {
 185         unsigned int len;
 186
 187         len = get_hbp_len(hbp_len);
 188
 189         return (va <= TASK_SIZE - len);
 190 }
 191
 192 /*
 193  * Check for virtual address in kernel space.
 194  */
 195 static int arch_check_va_in_kernelspace(unsigned long va, u8 hbp_len)
 196 {
 197         unsigned int len;
 198
 199         len = get_hbp_len(hbp_len);
 200
 201         return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
 202 }
 203
 204 /*
 205  * Store a breakpoint's encoded address, length, and type.
 206  */
 207 static int arch_store_info(struct perf_event *bp)
 208 {
 209         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 210         /*
 211          * For kernel-addresses, either the address or symbol name can be
 212          * specified.
 213          */
 214         if (info->name)
 215                 info->address = (unsigned long)
 216                                 kallsyms_lookup_name(info->name);
 217         if (info->address)
 218                 return 0;
 219
 220         return -EINVAL;
 221 }
 222
 223 int arch_bp_generic_fields(int x86_len, int x86_type,
 224                            int *gen_len, int *gen_type)
 225 {
 226         /* Len */
 227         switch (x86_len) {
 228         case X86_BREAKPOINT_LEN_1:
 229                 *gen_len = HW_BREAKPOINT_LEN_1;
 230                 break;
 231         case X86_BREAKPOINT_LEN_2:
 232                 *gen_len = HW_BREAKPOINT_LEN_2;
 233                 break;
 234         case X86_BREAKPOINT_LEN_4:
 235                 *gen_len = HW_BREAKPOINT_LEN_4;
 236                 break;
 237 #ifdef CONFIG_X86_64
 238         case X86_BREAKPOINT_LEN_8:
 239                 *gen_len = HW_BREAKPOINT_LEN_8;
 240                 break;
 241 #endif
 242         default:
 243                 return -EINVAL;
 244         }
 245
 246         /* Type */
 247         switch (x86_type) {
 248         case X86_BREAKPOINT_EXECUTE:
 249                 *gen_type = HW_BREAKPOINT_X;
 250                 break;
 251         case X86_BREAKPOINT_WRITE:
 252                 *gen_type = HW_BREAKPOINT_W;
 253                 break;
 254         case X86_BREAKPOINT_RW:
 255                 *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
 256                 break;
 257         default:
 258                 return -EINVAL;
 259         }
 260
 261         return 0;
 262 }
 263
 264
 265 static int arch_build_bp_info(struct perf_event *bp)
 266 {
 267         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 268
 269         info->address = bp->attr.bp_addr;
 270
 271         /* Len */
 272         switch (bp->attr.bp_len) {
 273         case HW_BREAKPOINT_LEN_1:
 274                 info->len = X86_BREAKPOINT_LEN_1;
 275                 break;
 276         case HW_BREAKPOINT_LEN_2:
 277                 info->len = X86_BREAKPOINT_LEN_2;
 278                 break;
 279         case HW_BREAKPOINT_LEN_4:
 280                 info->len = X86_BREAKPOINT_LEN_4;
 281                 break;
 282 #ifdef CONFIG_X86_64
 283         case HW_BREAKPOINT_LEN_8:
 284                 info->len = X86_BREAKPOINT_LEN_8;
 285                 break;
 286 #endif
 287         default:
 288                 return -EINVAL;
 289         }
 290
 291         /* Type */
 292         switch (bp->attr.bp_type) {
 293         case HW_BREAKPOINT_W:
 294                 info->type = X86_BREAKPOINT_WRITE;
 295                 break;
 296         case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
 297                 info->type = X86_BREAKPOINT_RW;
 298                 break;
 299         case HW_BREAKPOINT_X:
 300                 info->type = X86_BREAKPOINT_EXECUTE;
 301                 break;
 302         default:
 303                 return -EINVAL;
 304         }
 305
 306         return 0;
 307 }
 308 /*
 309  * Validate the arch-specific HW Breakpoint register settings
 310  */
 311 int arch_validate_hwbkpt_settings(struct perf_event *bp,
 312                                   struct task_struct *tsk)
 313 {
 314         struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 315         unsigned int align;
 316         int ret;
 317
 318
 319         ret = arch_build_bp_info(bp);
 320         if (ret)
 321                 return ret;
 322
 323         ret = -EINVAL;
 324
 325         if (info->type == X86_BREAKPOINT_EXECUTE)
 326                 /*
 327                  * Ptrace-refactoring code
 328                  * For now, we'll allow instruction breakpoint only for user-space
 329                  * addresses
 330                  */
 331                 if ((!arch_check_va_in_userspace(info->address, info->len)) &&
 332                         info->len != X86_BREAKPOINT_EXECUTE)
 333                         return ret;
 334
 335         switch (info->len) {
 336         case X86_BREAKPOINT_LEN_1:
 337                 align = 0;
 338                 break;
 339         case X86_BREAKPOINT_LEN_2:
 340                 align = 1;
 341                 break;
 342         case X86_BREAKPOINT_LEN_4:
 343                 align = 3;
 344                 break;
 345 #ifdef CONFIG_X86_64
 346         case X86_BREAKPOINT_LEN_8:
 347                 align = 7;
 348                 break;
 349 #endif
 350         default:
 351                 return ret;
 352         }
 353
 354         if (bp->callback)
 355                 ret = arch_store_info(bp);
 356
 357         if (ret < 0)
 358                 return ret;
 359         /*
 360          * Check that the low-order bits of the address are appropriate
 361          * for the alignment implied by len.
 362          */
 363         if (info->address & align)
 364                 return -EINVAL;
 365
 366         /* Check that the virtual address is in the proper range */
 367         if (tsk) {
 368                 if (!arch_check_va_in_userspace(info->address, info->len))
 369                         return -EFAULT;
 370         } else {
 371                 if (!arch_check_va_in_kernelspace(info->address, info->len))
 372                         return -EFAULT;
 373         }
 374
 375         return 0;
 376 }
 377
 378 /*
 379  * Release the user breakpoints used by ptrace
 380  */
 381 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 382 {
 383         int i;
 384         struct thread_struct *t = &tsk->thread;
 385
 386         for (i = 0; i < HBP_NUM; i++) {
 387                 unregister_hw_breakpoint(t->ptrace_bps[i]);
 388                 t->ptrace_bps[i] = NULL;
 389         }
 390 }
 391
 392 #ifdef CONFIG_KVM
 393 void hw_breakpoint_restore(void)
 394 {
 395         set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
 396         set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
 397         set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
 398         set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
 399         set_debugreg(current->thread.debugreg6, 6);
 400         set_debugreg(__get_cpu_var(dr7), 7);
 401 }
 402 EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
 403 #endif
 404
 405 /*
 406  * Handle debug exception notifications.
 407  *
 408  * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
 409  *
 410  * NOTIFY_DONE returned if one of the following conditions is true.
 411  * i) When the causative address is from user-space and the exception
 412  * is a valid one, i.e. not triggered as a result of lazy debug register
 413  * switching
 414  * ii) When there are more bits than trap<n> set in DR6 register (such
 415  * as BD, BS or BT) indicating that more than one debug condition is
 416  * met and requires some more action in do_debug().
 417  *
 418  * NOTIFY_STOP returned for all other cases
 419  *
 420  */
 421 static int __kprobes hw_breakpoint_handler(struct die_args *args)
 422 {
 423         int i, cpu, rc = NOTIFY_STOP;
 424         struct perf_event *bp;
 425         unsigned long dr7, dr6;
 426         unsigned long *dr6_p;
 427
 428         /* The DR6 value is pointed by args->err */
 429         dr6_p = (unsigned long *)ERR_PTR(args->err);
 430         dr6 = *dr6_p;
 431
 432         /* Do an early return if no trap bits are set in DR6 */
 433         if ((dr6 & DR_TRAP_BITS) == 0)
 434                 return NOTIFY_DONE;
 435
 436         get_debugreg(dr7, 7);
 437         /* Disable breakpoints during exception handling */
 438         set_debugreg(0UL, 7);
 439         /*
 440          * Assert that local interrupts are disabled
 441          * Reset the DRn bits in the virtualized register value.
 442          * The ptrace trigger routine will add in whatever is needed.
 443          */
 444         current->thread.debugreg6 &= ~DR_TRAP_BITS;
 445         cpu = get_cpu();
 446
 447         /* Handle all the breakpoints that were triggered */
 448         for (i = 0; i < HBP_NUM; ++i) {
 449                 if (likely(!(dr6 & (DR_TRAP0 << i))))
 450                         continue;
 451
 452                 /*
 453                  * The counter may be concurrently released but that can only
 454                  * occur from a call_rcu() path. We can then safely fetch
 455                  * the breakpoint, use its callback, touch its counter
 456                  * while we are in an rcu_read_lock() path.
 457                  */
 458                 rcu_read_lock();
 459
 460                 bp = per_cpu(bp_per_reg[i], cpu);
 461                 if (bp)
 462                         rc = NOTIFY_DONE;
 463                 /*
 464                  * Reset the 'i'th TRAP bit in dr6 to denote completion of
 465                  * exception handling
 466                  */
 467                 (*dr6_p) &= ~(DR_TRAP0 << i);
 468                 /*
 469                  * bp can be NULL due to lazy debug register switching
 470                  * or due to concurrent perf counter removing.
 471                  */
 472                 if (!bp) {
 473                         rcu_read_unlock();
 474                         break;
 475                 }
 476
 477                 (bp->callback)(bp, args->regs);
 478
 479                 rcu_read_unlock();
 480         }
 481         if (dr6 & (~DR_TRAP_BITS))
 482                 rc = NOTIFY_DONE;
 483
 484         set_debugreg(dr7, 7);
 485         put_cpu();
 486
 487         return rc;
 488 }
 489
 490 /*
 491  * Handle debug exception notifications.
 492  */
 493 int __kprobes hw_breakpoint_exceptions_notify(
 494                 struct notifier_block *unused, unsigned long val, void *data)
 495 {
 496         if (val != DIE_DEBUG)
 497                 return NOTIFY_DONE;
 498
 499         return hw_breakpoint_handler(data);
 500 }
 501
 502 void hw_breakpoint_pmu_read(struct perf_event *bp)
 503 {
 504         /* TODO */
 505 }
 506
 507 void hw_breakpoint_pmu_unthrottle(struct perf_event *bp)
 508 {
 509         /* TODO */
 510 }