arch/ia64/kernel/kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  arch/ia64/kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  * Copyright (C) Intel Corporation, 2005
  21  *
  22  * 2005-Apr     Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy
  23  *              <anil.s.keshavamurthy@intel.com> adapted from i386
  24  */
  25
  26 #include <linux/config.h>
  27 #include <linux/kprobes.h>
  28 #include <linux/ptrace.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/string.h>
  31 #include <linux/slab.h>
  32 #include <linux/preempt.h>
  33 #include <linux/moduleloader.h>
  34
  35 #include <asm/pgtable.h>
  36 #include <asm/kdebug.h>
  37 #include <asm/sections.h>
  38
  39 extern void jprobe_inst_return(void);
  40
  41 /* kprobe_status settings */
  42 #define KPROBE_HIT_ACTIVE       0x00000001
  43 #define KPROBE_HIT_SS           0x00000002
  44
  45 static struct kprobe *current_kprobe, *kprobe_prev;
  46 static unsigned long kprobe_status, kprobe_status_prev;
  47 static struct pt_regs jprobe_saved_regs;
  48
  49 enum instruction_type {A, I, M, F, B, L, X, u};
  50 static enum instruction_type bundle_encoding[32][3] = {
  51   { M, I, I },                          /* 00 */
  52   { M, I, I },                          /* 01 */
  53   { M, I, I },                          /* 02 */
  54   { M, I, I },                          /* 03 */
  55   { M, L, X },                          /* 04 */
  56   { M, L, X },                          /* 05 */
  57   { u, u, u },                          /* 06 */
  58   { u, u, u },                          /* 07 */
  59   { M, M, I },                          /* 08 */
  60   { M, M, I },                          /* 09 */
  61   { M, M, I },                          /* 0A */
  62   { M, M, I },                          /* 0B */
  63   { M, F, I },                          /* 0C */
  64   { M, F, I },                          /* 0D */
  65   { M, M, F },                          /* 0E */
  66   { M, M, F },                          /* 0F */
  67   { M, I, B },                          /* 10 */
  68   { M, I, B },                          /* 11 */
  69   { M, B, B },                          /* 12 */
  70   { M, B, B },                          /* 13 */
  71   { u, u, u },                          /* 14 */
  72   { u, u, u },                          /* 15 */
  73   { B, B, B },                          /* 16 */
  74   { B, B, B },                          /* 17 */
  75   { M, M, B },                          /* 18 */
  76   { M, M, B },                          /* 19 */
  77   { u, u, u },                          /* 1A */
  78   { u, u, u },                          /* 1B */
  79   { M, F, B },                          /* 1C */
  80   { M, F, B },                          /* 1D */
  81   { u, u, u },                          /* 1E */
  82   { u, u, u },                          /* 1F */
  83 };
  84
  85 /*
  86  * In this function we check to see if the instruction
  87  * is IP relative instruction and update the kprobe
  88  * inst flag accordingly
  89  */
  90 static void __kprobes update_kprobe_inst_flag(uint template, uint  slot,
  91                                               uint major_opcode,
  92                                               unsigned long kprobe_inst,
  93                                               struct kprobe *p)
  94 {
  95         p->ainsn.inst_flag = 0;
  96         p->ainsn.target_br_reg = 0;
  97
  98         if (bundle_encoding[template][slot] == B) {
  99                 switch (major_opcode) {
 100                   case INDIRECT_CALL_OPCODE:
 101                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
 102                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
 103                         break;
 104                   case IP_RELATIVE_PREDICT_OPCODE:
 105                   case IP_RELATIVE_BRANCH_OPCODE:
 106                         p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR;
 107                         break;
 108                   case IP_RELATIVE_CALL_OPCODE:
 109                         p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR;
 110                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
 111                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
 112                         break;
 113                 }
 114         } else if (bundle_encoding[template][slot] == X) {
 115                 switch (major_opcode) {
 116                   case LONG_CALL_OPCODE:
 117                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
 118                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
 119                   break;
 120                 }
 121         }
 122         return;
 123 }
 124
 125 /*
 126  * In this function we check to see if the instruction
 127  * on which we are inserting kprobe is supported.
 128  * Returns 0 if supported
 129  * Returns -EINVAL if unsupported
 130  */
 131 static int __kprobes unsupported_inst(uint template, uint  slot,
 132                                       uint major_opcode,
 133                                       unsigned long kprobe_inst,
 134                                       struct kprobe *p)
 135 {
 136         unsigned long addr = (unsigned long)p->addr;
 137
 138         if (bundle_encoding[template][slot] == I) {
 139                 switch (major_opcode) {
 140                         case 0x0: //I_UNIT_MISC_OPCODE:
 141                         /*
 142                          * Check for Integer speculation instruction
 143                          * - Bit 33-35 to be equal to 0x1
 144                          */
 145                         if (((kprobe_inst >> 33) & 0x7) == 1) {
 146                                 printk(KERN_WARNING
 147                                         "Kprobes on speculation inst at <0x%lx> not supported\n",
 148                                         addr);
 149                                 return -EINVAL;
 150                         }
 151
 152                         /*
 153                          * IP relative mov instruction
 154                          *  - Bit 27-35 to be equal to 0x30
 155                          */
 156                         if (((kprobe_inst >> 27) & 0x1FF) == 0x30) {
 157                                 printk(KERN_WARNING
 158                                         "Kprobes on \"mov r1=ip\" at <0x%lx> not supported\n",
 159                                         addr);
 160                                 return -EINVAL;
 161
 162                         }
 163                 }
 164         }
 165         return 0;
 166 }
 167
 168
 169 /*
 170  * In this function we check to see if the instruction
 171  * (qp) cmpx.crel.ctype p1,p2=r2,r3
 172  * on which we are inserting kprobe is cmp instruction
 173  * with ctype as unc.
 174  */
 175 static uint __kprobes is_cmp_ctype_unc_inst(uint template, uint slot,
 176                                             uint major_opcode,
 177                                             unsigned long kprobe_inst)
 178 {
 179         cmp_inst_t cmp_inst;
 180         uint ctype_unc = 0;
 181
 182         if (!((bundle_encoding[template][slot] == I) ||
 183                 (bundle_encoding[template][slot] == M)))
 184                 goto out;
 185
 186         if (!((major_opcode == 0xC) || (major_opcode == 0xD) ||
 187                 (major_opcode == 0xE)))
 188                 goto out;
 189
 190         cmp_inst.l = kprobe_inst;
 191         if ((cmp_inst.f.x2 == 0) || (cmp_inst.f.x2 == 1)) {
 192                 /* Integere compare - Register Register (A6 type)*/
 193                 if ((cmp_inst.f.tb == 0) && (cmp_inst.f.ta == 0)
 194                                 &&(cmp_inst.f.c == 1))
 195                         ctype_unc = 1;
 196         } else if ((cmp_inst.f.x2 == 2)||(cmp_inst.f.x2 == 3)) {
 197                 /* Integere compare - Immediate Register (A8 type)*/
 198                 if ((cmp_inst.f.ta == 0) &&(cmp_inst.f.c == 1))
 199                         ctype_unc = 1;
 200         }
 201 out:
 202         return ctype_unc;
 203 }
 204
 205 /*
 206  * In this function we override the bundle with
 207  * the break instruction at the given slot.
 208  */
 209 static void __kprobes prepare_break_inst(uint template, uint  slot,
 210                                          uint major_opcode,
 211                                          unsigned long kprobe_inst,
 212                                          struct kprobe *p)
 213 {
 214         unsigned long break_inst = BREAK_INST;
 215         bundle_t *bundle = &p->ainsn.insn.bundle;
 216
 217         /*
 218          * Copy the original kprobe_inst qualifying predicate(qp)
 219          * to the break instruction iff !is_cmp_ctype_unc_inst
 220          * because for cmp instruction with ctype equal to unc,
 221          * which is a special instruction always needs to be
 222          * executed regradless of qp
 223          */
 224         if (!is_cmp_ctype_unc_inst(template, slot, major_opcode, kprobe_inst))
 225                 break_inst |= (0x3f & kprobe_inst);
 226
 227         switch (slot) {
 228           case 0:
 229                 bundle->quad0.slot0 = break_inst;
 230                 break;
 231           case 1:
 232                 bundle->quad0.slot1_p0 = break_inst;
 233                 bundle->quad1.slot1_p1 = break_inst >> (64-46);
 234                 break;
 235           case 2:
 236                 bundle->quad1.slot2 = break_inst;
 237                 break;
 238         }
 239
 240         /*
 241          * Update the instruction flag, so that we can
 242          * emulate the instruction properly after we
 243          * single step on original instruction
 244          */
 245         update_kprobe_inst_flag(template, slot, major_opcode, kprobe_inst, p);
 246 }
 247
 248 static inline void get_kprobe_inst(bundle_t *bundle, uint slot,
 249                 unsigned long *kprobe_inst, uint *major_opcode)
 250 {
 251         unsigned long kprobe_inst_p0, kprobe_inst_p1;
 252         unsigned int template;
 253
 254         template = bundle->quad0.template;
 255
 256         switch (slot) {
 257           case 0:
 258                 *major_opcode = (bundle->quad0.slot0 >> SLOT0_OPCODE_SHIFT);
 259                 *kprobe_inst = bundle->quad0.slot0;
 260                 break;
 261           case 1:
 262                 *major_opcode = (bundle->quad1.slot1_p1 >> SLOT1_p1_OPCODE_SHIFT);
 263                 kprobe_inst_p0 = bundle->quad0.slot1_p0;
 264                 kprobe_inst_p1 = bundle->quad1.slot1_p1;
 265                 *kprobe_inst = kprobe_inst_p0 | (kprobe_inst_p1 << (64-46));
 266                 break;
 267           case 2:
 268                 *major_opcode = (bundle->quad1.slot2 >> SLOT2_OPCODE_SHIFT);
 269                 *kprobe_inst = bundle->quad1.slot2;
 270                 break;
 271         }
 272 }
 273
 274 /* Returns non-zero if the addr is in the Interrupt Vector Table */
 275 static inline int in_ivt_functions(unsigned long addr)
 276 {
 277         return (addr >= (unsigned long)__start_ivt_text
 278                 && addr < (unsigned long)__end_ivt_text);
 279 }
 280
 281 static int __kprobes valid_kprobe_addr(int template, int slot,
 282                                        unsigned long addr)
 283 {
 284         if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) {
 285                 printk(KERN_WARNING "Attempting to insert unaligned kprobe "
 286                                 "at 0x%lx\n", addr);
 287                 return -EINVAL;
 288         }
 289
 290         if (in_ivt_functions(addr)) {
 291                 printk(KERN_WARNING "Kprobes can't be inserted inside "
 292                                 "IVT functions at 0x%lx\n", addr);
 293                 return -EINVAL;
 294         }
 295
 296         if (slot == 1 && bundle_encoding[template][1] != L) {
 297                 printk(KERN_WARNING "Inserting kprobes on slot #1 "
 298                        "is not supported\n");
 299                 return -EINVAL;
 300         }
 301
 302         return 0;
 303 }
 304
 305 static inline void save_previous_kprobe(void)
 306 {
 307         kprobe_prev = current_kprobe;
 308         kprobe_status_prev = kprobe_status;
 309 }
 310
 311 static inline void restore_previous_kprobe(void)
 312 {
 313         current_kprobe = kprobe_prev;
 314         kprobe_status = kprobe_status_prev;
 315 }
 316
 317 static inline void set_current_kprobe(struct kprobe *p)
 318 {
 319         current_kprobe = p;
 320 }
 321
 322 static void kretprobe_trampoline(void)
 323 {
 324 }
 325
 326 /*
 327  * At this point the target function has been tricked into
 328  * returning into our trampoline.  Lookup the associated instance
 329  * and then:
 330  *    - call the handler function
 331  *    - cleanup by marking the instance as unused
 332  *    - long jump back to the original return address
 333  */
 334 int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 335 {
 336         struct kretprobe_instance *ri = NULL;
 337         struct hlist_head *head;
 338         struct hlist_node *node, *tmp;
 339         unsigned long orig_ret_address = 0;
 340         unsigned long trampoline_address =
 341                 ((struct fnptr *)kretprobe_trampoline)->ip;
 342
 343         head = kretprobe_inst_table_head(current);
 344
 345         /*
 346          * It is possible to have multiple instances associated with a given
 347          * task either because an multiple functions in the call path
 348          * have a return probe installed on them, and/or more then one return
 349          * return probe was registered for a target function.
 350          *
 351          * We can handle this because:
 352          *     - instances are always inserted at the head of the list
 353          *     - when multiple return probes are registered for the same
 354          *       function, the first instance's ret_addr will point to the
 355          *       real return address, and all the rest will point to
 356          *       kretprobe_trampoline
 357          */
 358         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 359                 if (ri->task != current)
 360                         /* another task is sharing our hash bucket */
 361                         continue;
 362
 363                 if (ri->rp && ri->rp->handler)
 364                         ri->rp->handler(ri, regs);
 365
 366                 orig_ret_address = (unsigned long)ri->ret_addr;
 367                 recycle_rp_inst(ri);
 368
 369                 if (orig_ret_address != trampoline_address)
 370                         /*
 371                          * This is the real return address. Any other
 372                          * instances associated with this task are for
 373                          * other calls deeper on the call stack
 374                          */
 375                         break;
 376         }
 377
 378         BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
 379         regs->cr_iip = orig_ret_address;
 380
 381         unlock_kprobes();
 382         preempt_enable_no_resched();
 383
 384         /*
 385          * By returning a non-zero value, we are telling
 386          * kprobe_handler() that we have handled unlocking
 387          * and re-enabling preemption.
 388          */
 389         return 1;
 390 }
 391
 392 void __kprobes arch_prepare_kretprobe(struct kretprobe *rp,
 393                                       struct pt_regs *regs)
 394 {
 395         struct kretprobe_instance *ri;
 396
 397         if ((ri = get_free_rp_inst(rp)) != NULL) {
 398                 ri->rp = rp;
 399                 ri->task = current;
 400                 ri->ret_addr = (kprobe_opcode_t *)regs->b0;
 401
 402                 /* Replace the return addr with trampoline addr */
 403                 regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;
 404
 405                 add_rp_inst(ri);
 406         } else {
 407                 rp->nmissed++;
 408         }
 409 }
 410
 411 int __kprobes arch_prepare_kprobe(struct kprobe *p)
 412 {
 413         unsigned long addr = (unsigned long) p->addr;
 414         unsigned long *kprobe_addr = (unsigned long *)(addr & ~0xFULL);
 415         unsigned long kprobe_inst=0;
 416         unsigned int slot = addr & 0xf, template, major_opcode = 0;
 417         bundle_t *bundle = &p->ainsn.insn.bundle;
 418
 419         memcpy(&p->opcode.bundle, kprobe_addr, sizeof(bundle_t));
 420         memcpy(&p->ainsn.insn.bundle, kprobe_addr, sizeof(bundle_t));
 421
 422         template = bundle->quad0.template;
 423
 424         if(valid_kprobe_addr(template, slot, addr))
 425                 return -EINVAL;
 426
 427         /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */
 428         if (slot == 1 && bundle_encoding[template][1] == L)
 429                 slot++;
 430
 431         /* Get kprobe_inst and major_opcode from the bundle */
 432         get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode);
 433
 434         if (unsupported_inst(template, slot, major_opcode, kprobe_inst, p))
 435                         return -EINVAL;
 436
 437         prepare_break_inst(template, slot, major_opcode, kprobe_inst, p);
 438
 439         return 0;
 440 }
 441
 442 void __kprobes arch_arm_kprobe(struct kprobe *p)
 443 {
 444         unsigned long addr = (unsigned long)p->addr;
 445         unsigned long arm_addr = addr & ~0xFULL;
 446
 447         memcpy((char *)arm_addr, &p->ainsn.insn.bundle, sizeof(bundle_t));
 448         flush_icache_range(arm_addr, arm_addr + sizeof(bundle_t));
 449 }
 450
 451 void __kprobes arch_disarm_kprobe(struct kprobe *p)
 452 {
 453         unsigned long addr = (unsigned long)p->addr;
 454         unsigned long arm_addr = addr & ~0xFULL;
 455
 456         /* p->opcode contains the original unaltered bundle */
 457         memcpy((char *) arm_addr, (char *) &p->opcode.bundle, sizeof(bundle_t));
 458         flush_icache_range(arm_addr, arm_addr + sizeof(bundle_t));
 459 }
 460
 461 void __kprobes arch_remove_kprobe(struct kprobe *p)
 462 {
 463 }
 464
 465 /*
 466  * We are resuming execution after a single step fault, so the pt_regs
 467  * structure reflects the register state after we executed the instruction
 468  * located in the kprobe (p->ainsn.insn.bundle).  We still need to adjust
 469  * the ip to point back to the original stack address. To set the IP address
 470  * to original stack address, handle the case where we need to fixup the
 471  * relative IP address and/or fixup branch register.
 472  */
 473 static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
 474 {
 475         unsigned long bundle_addr = ((unsigned long) (&p->opcode.bundle)) & ~0xFULL;
 476         unsigned long resume_addr = (unsigned long)p->addr & ~0xFULL;
 477         unsigned long template;
 478         int slot = ((unsigned long)p->addr & 0xf);
 479
 480         template = p->opcode.bundle.quad0.template;
 481
 482         if (slot == 1 && bundle_encoding[template][1] == L)
 483                 slot = 2;
 484
 485         if (p->ainsn.inst_flag) {
 486
 487                 if (p->ainsn.inst_flag & INST_FLAG_FIX_RELATIVE_IP_ADDR) {
 488                         /* Fix relative IP address */
 489                         regs->cr_iip = (regs->cr_iip - bundle_addr) + resume_addr;
 490                 }
 491
 492                 if (p->ainsn.inst_flag & INST_FLAG_FIX_BRANCH_REG) {
 493                 /*
 494                  * Fix target branch register, software convention is
 495                  * to use either b0 or b6 or b7, so just checking
 496                  * only those registers
 497                  */
 498                         switch (p->ainsn.target_br_reg) {
 499                         case 0:
 500                                 if ((regs->b0 == bundle_addr) ||
 501                                         (regs->b0 == bundle_addr + 0x10)) {
 502                                         regs->b0 = (regs->b0 - bundle_addr) +
 503                                                 resume_addr;
 504                                 }
 505                                 break;
 506                         case 6:
 507                                 if ((regs->b6 == bundle_addr) ||
 508                                         (regs->b6 == bundle_addr + 0x10)) {
 509                                         regs->b6 = (regs->b6 - bundle_addr) +
 510                                                 resume_addr;
 511                                 }
 512                                 break;
 513                         case 7:
 514                                 if ((regs->b7 == bundle_addr) ||
 515                                         (regs->b7 == bundle_addr + 0x10)) {
 516                                         regs->b7 = (regs->b7 - bundle_addr) +
 517                                                 resume_addr;
 518                                 }
 519                                 break;
 520                         } /* end switch */
 521                 }
 522                 goto turn_ss_off;
 523         }
 524
 525         if (slot == 2) {
 526                 if (regs->cr_iip == bundle_addr + 0x10) {
 527                         regs->cr_iip = resume_addr + 0x10;
 528                 }
 529         } else {
 530                 if (regs->cr_iip == bundle_addr) {
 531                         regs->cr_iip = resume_addr;
 532                 }
 533         }
 534
 535 turn_ss_off:
 536         /* Turn off Single Step bit */
 537         ia64_psr(regs)->ss = 0;
 538 }
 539
 540 static void __kprobes prepare_ss(struct kprobe *p, struct pt_regs *regs)
 541 {
 542         unsigned long bundle_addr = (unsigned long) &p->opcode.bundle;
 543         unsigned long slot = (unsigned long)p->addr & 0xf;
 544
 545         /* Update instruction pointer (IIP) and slot number (IPSR.ri) */
 546         regs->cr_iip = bundle_addr & ~0xFULL;
 547
 548         if (slot > 2)
 549                 slot = 0;
 550
 551         ia64_psr(regs)->ri = slot;
 552
 553         /* turn on single stepping */
 554         ia64_psr(regs)->ss = 1;
 555 }
 556
 557 static int __kprobes is_ia64_break_inst(struct pt_regs *regs)
 558 {
 559         unsigned int slot = ia64_psr(regs)->ri;
 560         unsigned int template, major_opcode;
 561         unsigned long kprobe_inst;
 562         unsigned long *kprobe_addr = (unsigned long *)regs->cr_iip;
 563         bundle_t bundle;
 564
 565         memcpy(&bundle, kprobe_addr, sizeof(bundle_t));
 566         template = bundle.quad0.template;
 567
 568         /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */
 569         if (slot == 1 && bundle_encoding[template][1] == L)
 570                 slot++;
 571
 572         /* Get Kprobe probe instruction at given slot*/
 573         get_kprobe_inst(&bundle, slot, &kprobe_inst, &major_opcode);
 574
 575         /* For break instruction,
 576          * Bits 37:40 Major opcode to be zero
 577          * Bits 27:32 X6 to be zero
 578          * Bits 32:35 X3 to be zero
 579          */
 580         if (major_opcode || ((kprobe_inst >> 27) & 0x1FF) ) {
 581                 /* Not a break instruction */
 582                 return 0;
 583         }
 584
 585         /* Is a break instruction */
 586         return 1;
 587 }
 588
 589 static int __kprobes pre_kprobes_handler(struct die_args *args)
 590 {
 591         struct kprobe *p;
 592         int ret = 0;
 593         struct pt_regs *regs = args->regs;
 594         kprobe_opcode_t *addr = (kprobe_opcode_t *)instruction_pointer(regs);
 595
 596         preempt_disable();
 597
 598         /* Handle recursion cases */
 599         if (kprobe_running()) {
 600                 p = get_kprobe(addr);
 601                 if (p) {
 602                         if (kprobe_status == KPROBE_HIT_SS) {
 603                                 unlock_kprobes();
 604                                 goto no_kprobe;
 605                         }
 606                         /* We have reentered the pre_kprobe_handler(), since
 607                          * another probe was hit while within the handler.
 608                          * We here save the original kprobes variables and
 609                          * just single step on the instruction of the new probe
 610                          * without calling any user handlers.
 611                          */
 612                         save_previous_kprobe();
 613                         set_current_kprobe(p);
 614                         p->nmissed++;
 615                         prepare_ss(p, regs);
 616                         kprobe_status = KPROBE_REENTER;
 617                         return 1;
 618                 } else if (args->err == __IA64_BREAK_JPROBE) {
 619                         /*
 620                          * jprobe instrumented function just completed
 621                          */
 622                         p = current_kprobe;
 623                         if (p->break_handler && p->break_handler(p, regs)) {
 624                                 goto ss_probe;
 625                         }
 626                 } else {
 627                         /* Not our break */
 628                         goto no_kprobe;
 629                 }
 630         }
 631
 632         lock_kprobes();
 633         p = get_kprobe(addr);
 634         if (!p) {
 635                 unlock_kprobes();
 636                 if (!is_ia64_break_inst(regs)) {
 637                         /*
 638                          * The breakpoint instruction was removed right
 639                          * after we hit it.  Another cpu has removed
 640                          * either a probepoint or a debugger breakpoint
 641                          * at this address.  In either case, no further
 642                          * handling of this interrupt is appropriate.
 643                          */
 644                         ret = 1;
 645
 646                 }
 647
 648                 /* Not one of our break, let kernel handle it */
 649                 goto no_kprobe;
 650         }
 651
 652         kprobe_status = KPROBE_HIT_ACTIVE;
 653         set_current_kprobe(p);
 654
 655         if (p->pre_handler && p->pre_handler(p, regs))
 656                 /*
 657                  * Our pre-handler is specifically requesting that we just
 658                  * do a return.  This is used for both the jprobe pre-handler
 659                  * and the kretprobe trampoline
 660                  */
 661                 return 1;
 662
 663 ss_probe:
 664         prepare_ss(p, regs);
 665         kprobe_status = KPROBE_HIT_SS;
 666         return 1;
 667
 668 no_kprobe:
 669         preempt_enable_no_resched();
 670         return ret;
 671 }
 672
 673 static int __kprobes post_kprobes_handler(struct pt_regs *regs)
 674 {
 675         if (!kprobe_running())
 676                 return 0;
 677
 678         if ((kprobe_status != KPROBE_REENTER) && current_kprobe->post_handler) {
 679                 kprobe_status = KPROBE_HIT_SSDONE;
 680                 current_kprobe->post_handler(current_kprobe, regs, 0);
 681         }
 682
 683         resume_execution(current_kprobe, regs);
 684
 685         /*Restore back the original saved kprobes variables and continue. */
 686         if (kprobe_status == KPROBE_REENTER) {
 687                 restore_previous_kprobe();
 688                 goto out;
 689         }
 690
 691         unlock_kprobes();
 692
 693 out:
 694         preempt_enable_no_resched();
 695         return 1;
 696 }
 697
 698 static int __kprobes kprobes_fault_handler(struct pt_regs *regs, int trapnr)
 699 {
 700         if (!kprobe_running())
 701                 return 0;
 702
 703         if (current_kprobe->fault_handler &&
 704             current_kprobe->fault_handler(current_kprobe, regs, trapnr))
 705                 return 1;
 706
 707         if (kprobe_status & KPROBE_HIT_SS) {
 708                 resume_execution(current_kprobe, regs);
 709                 unlock_kprobes();
 710                 preempt_enable_no_resched();
 711         }
 712
 713         return 0;
 714 }
 715
 716 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
 717                                        unsigned long val, void *data)
 718 {
 719         struct die_args *args = (struct die_args *)data;
 720         switch(val) {
 721         case DIE_BREAK:
 722                 if (pre_kprobes_handler(args))
 723                         return NOTIFY_STOP;
 724                 break;
 725         case DIE_SS:
 726                 if (post_kprobes_handler(args->regs))
 727                         return NOTIFY_STOP;
 728                 break;
 729         case DIE_PAGE_FAULT:
 730                 if (kprobes_fault_handler(args->regs, args->trapnr))
 731                         return NOTIFY_STOP;
 732         default:
 733                 break;
 734         }
 735         return NOTIFY_DONE;
 736 }
 737
 738 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
 739 {
 740         struct jprobe *jp = container_of(p, struct jprobe, kp);
 741         unsigned long addr = ((struct fnptr *)(jp->entry))->ip;
 742
 743         /* save architectural state */
 744         jprobe_saved_regs = *regs;
 745
 746         /* after rfi, execute the jprobe instrumented function */
 747         regs->cr_iip = addr & ~0xFULL;
 748         ia64_psr(regs)->ri = addr & 0xf;
 749         regs->r1 = ((struct fnptr *)(jp->entry))->gp;
 750
 751         /*
 752          * fix the return address to our jprobe_inst_return() function
 753          * in the jprobes.S file
 754          */
 755         regs->b0 = ((struct fnptr *)(jprobe_inst_return))->ip;
 756
 757         return 1;
 758 }
 759
 760 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
 761 {
 762         *regs = jprobe_saved_regs;
 763         return 1;
 764 }
 765
 766 static struct kprobe trampoline_p = {
 767         .pre_handler = trampoline_probe_handler
 768 };
 769
 770 int __init arch_init_kprobes(void)
 771 {
 772         trampoline_p.addr =
 773                 (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip;
 774         return register_kprobe(&trampoline_p);
 775 }