17568cb4eSPaul Mackerras /* 27568cb4eSPaul Mackerras * Kernel Probes (KProbes) 37568cb4eSPaul Mackerras * 47568cb4eSPaul Mackerras * This program is free software; you can redistribute it and/or modify 57568cb4eSPaul Mackerras * it under the terms of the GNU General Public License as published by 67568cb4eSPaul Mackerras * the Free Software Foundation; either version 2 of the License, or 77568cb4eSPaul Mackerras * (at your option) any later version. 87568cb4eSPaul Mackerras * 97568cb4eSPaul Mackerras * This program is distributed in the hope that it will be useful, 107568cb4eSPaul Mackerras * but WITHOUT ANY WARRANTY; without even the implied warranty of 117568cb4eSPaul Mackerras * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 127568cb4eSPaul Mackerras * GNU General Public License for more details. 137568cb4eSPaul Mackerras * 147568cb4eSPaul Mackerras * You should have received a copy of the GNU General Public License 157568cb4eSPaul Mackerras * along with this program; if not, write to the Free Software 167568cb4eSPaul Mackerras * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 177568cb4eSPaul Mackerras * 187568cb4eSPaul Mackerras * Copyright (C) IBM Corporation, 2002, 2004 197568cb4eSPaul Mackerras * 207568cb4eSPaul Mackerras * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 217568cb4eSPaul Mackerras * Probes initial implementation ( includes contributions from 227568cb4eSPaul Mackerras * Rusty Russell). 237568cb4eSPaul Mackerras * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 247568cb4eSPaul Mackerras * interface to access function arguments. 257568cb4eSPaul Mackerras * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port 267568cb4eSPaul Mackerras * for PPC64 277568cb4eSPaul Mackerras */ 287568cb4eSPaul Mackerras 297568cb4eSPaul Mackerras #include <linux/kprobes.h> 307568cb4eSPaul Mackerras #include <linux/ptrace.h> 317568cb4eSPaul Mackerras #include <linux/preempt.h> 328a39b05fSPaul Gortmaker #include <linux/extable.h> 331eeb66a1SChristoph Hellwig #include <linux/kdebug.h> 345a0e3ad6STejun Heo #include <linux/slab.h> 352f0143c9SMichael Ellerman #include <asm/code-patching.h> 367568cb4eSPaul Mackerras #include <asm/cacheflush.h> 377568cb4eSPaul Mackerras #include <asm/sstep.h> 3850e21f2bSPrasanna S Panchamukhi #include <asm/uaccess.h> 39f8279621SKumar Gala 407568cb4eSPaul Mackerras DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 417568cb4eSPaul Mackerras DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 427568cb4eSPaul Mackerras 43f438d914SMasami Hiramatsu struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 44f438d914SMasami Hiramatsu 457568cb4eSPaul Mackerras int __kprobes arch_prepare_kprobe(struct kprobe *p) 467568cb4eSPaul Mackerras { 477568cb4eSPaul Mackerras int ret = 0; 487568cb4eSPaul Mackerras kprobe_opcode_t insn = *p->addr; 497568cb4eSPaul Mackerras 507568cb4eSPaul Mackerras if ((unsigned long)p->addr & 0x03) { 517568cb4eSPaul Mackerras printk("Attempt to register kprobe at an unaligned address\n"); 527568cb4eSPaul Mackerras ret = -EINVAL; 5382090035SKumar Gala } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) { 5482090035SKumar Gala printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n"); 557568cb4eSPaul Mackerras ret = -EINVAL; 567568cb4eSPaul Mackerras } 577568cb4eSPaul Mackerras 58f8279621SKumar Gala /* insn must be on a special executable page on ppc64. This is 59f8279621SKumar Gala * not explicitly required on ppc32 (right now), but it doesn't hurt */ 607568cb4eSPaul Mackerras if (!ret) { 617568cb4eSPaul Mackerras p->ainsn.insn = get_insn_slot(); 627568cb4eSPaul Mackerras if (!p->ainsn.insn) 637568cb4eSPaul Mackerras ret = -ENOMEM; 647568cb4eSPaul Mackerras } 657568cb4eSPaul Mackerras 6649a2a1b8SAnil S Keshavamurthy if (!ret) { 67e6349a95SAnanth N Mavinakayanahalli memcpy(p->ainsn.insn, p->addr, 68e6349a95SAnanth N Mavinakayanahalli MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 697568cb4eSPaul Mackerras p->opcode = *p->addr; 7083db3ddeSAnanth N Mavinakayanahalli flush_icache_range((unsigned long)p->ainsn.insn, 7183db3ddeSAnanth N Mavinakayanahalli (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); 727568cb4eSPaul Mackerras } 737568cb4eSPaul Mackerras 74e6349a95SAnanth N Mavinakayanahalli p->ainsn.boostable = 0; 7549a2a1b8SAnil S Keshavamurthy return ret; 7649a2a1b8SAnil S Keshavamurthy } 7749a2a1b8SAnil S Keshavamurthy 787568cb4eSPaul Mackerras void __kprobes arch_arm_kprobe(struct kprobe *p) 797568cb4eSPaul Mackerras { 807568cb4eSPaul Mackerras *p->addr = BREAKPOINT_INSTRUCTION; 817568cb4eSPaul Mackerras flush_icache_range((unsigned long) p->addr, 827568cb4eSPaul Mackerras (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 837568cb4eSPaul Mackerras } 847568cb4eSPaul Mackerras 857568cb4eSPaul Mackerras void __kprobes arch_disarm_kprobe(struct kprobe *p) 867568cb4eSPaul Mackerras { 877568cb4eSPaul Mackerras *p->addr = p->opcode; 887568cb4eSPaul Mackerras flush_icache_range((unsigned long) p->addr, 897568cb4eSPaul Mackerras (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 907568cb4eSPaul Mackerras } 917568cb4eSPaul Mackerras 920498b635SAnanth N Mavinakayanahalli void __kprobes arch_remove_kprobe(struct kprobe *p) 937568cb4eSPaul Mackerras { 9412941560SMasami Hiramatsu if (p->ainsn.insn) { 95b4c6c34aSMasami Hiramatsu free_insn_slot(p->ainsn.insn, 0); 9612941560SMasami Hiramatsu p->ainsn.insn = NULL; 9712941560SMasami Hiramatsu } 987568cb4eSPaul Mackerras } 997568cb4eSPaul Mackerras 10046dbe2f4SPrasanna S Panchamukhi static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) 1017568cb4eSPaul Mackerras { 10235fd219aSSuzuki K. Poulose enable_single_step(regs); 1037568cb4eSPaul Mackerras 1040ccde0a2SAnanth N Mavinakayanahalli /* 1050ccde0a2SAnanth N Mavinakayanahalli * On powerpc we should single step on the original 1060ccde0a2SAnanth N Mavinakayanahalli * instruction even if the probed insn is a trap 1070ccde0a2SAnanth N Mavinakayanahalli * variant as values in regs could play a part in 1080ccde0a2SAnanth N Mavinakayanahalli * if the trap is taken or not 1090ccde0a2SAnanth N Mavinakayanahalli */ 1107568cb4eSPaul Mackerras regs->nip = (unsigned long)p->ainsn.insn; 1117568cb4eSPaul Mackerras } 1127568cb4eSPaul Mackerras 11346dbe2f4SPrasanna S Panchamukhi static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 1147568cb4eSPaul Mackerras { 1157568cb4eSPaul Mackerras kcb->prev_kprobe.kp = kprobe_running(); 1167568cb4eSPaul Mackerras kcb->prev_kprobe.status = kcb->kprobe_status; 1177568cb4eSPaul Mackerras kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; 1187568cb4eSPaul Mackerras } 1197568cb4eSPaul Mackerras 12046dbe2f4SPrasanna S Panchamukhi static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 1217568cb4eSPaul Mackerras { 12269111bacSChristoph Lameter __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 1237568cb4eSPaul Mackerras kcb->kprobe_status = kcb->prev_kprobe.status; 1247568cb4eSPaul Mackerras kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; 1257568cb4eSPaul Mackerras } 1267568cb4eSPaul Mackerras 12746dbe2f4SPrasanna S Panchamukhi static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 1287568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb) 1297568cb4eSPaul Mackerras { 13069111bacSChristoph Lameter __this_cpu_write(current_kprobe, p); 1317568cb4eSPaul Mackerras kcb->kprobe_saved_msr = regs->msr; 1327568cb4eSPaul Mackerras } 1337568cb4eSPaul Mackerras 1344c4308cbSChristoph Hellwig void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 1357568cb4eSPaul Mackerras struct pt_regs *regs) 1367568cb4eSPaul Mackerras { 1377568cb4eSPaul Mackerras ri->ret_addr = (kprobe_opcode_t *)regs->link; 1387568cb4eSPaul Mackerras 1397568cb4eSPaul Mackerras /* Replace the return addr with trampoline addr */ 1407568cb4eSPaul Mackerras regs->link = (unsigned long)kretprobe_trampoline; 1417568cb4eSPaul Mackerras } 1427568cb4eSPaul Mackerras 143*6cc89badSNaveen N. Rao int __kprobes kprobe_handler(struct pt_regs *regs) 1447568cb4eSPaul Mackerras { 1457568cb4eSPaul Mackerras struct kprobe *p; 1467568cb4eSPaul Mackerras int ret = 0; 1477568cb4eSPaul Mackerras unsigned int *addr = (unsigned int *)regs->nip; 1487568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb; 1497568cb4eSPaul Mackerras 150*6cc89badSNaveen N. Rao if (user_mode(regs)) 151*6cc89badSNaveen N. Rao return 0; 152*6cc89badSNaveen N. Rao 1537568cb4eSPaul Mackerras /* 1547568cb4eSPaul Mackerras * We don't want to be preempted for the entire 1557568cb4eSPaul Mackerras * duration of kprobe processing 1567568cb4eSPaul Mackerras */ 1577568cb4eSPaul Mackerras preempt_disable(); 1587568cb4eSPaul Mackerras kcb = get_kprobe_ctlblk(); 1597568cb4eSPaul Mackerras 1607568cb4eSPaul Mackerras /* Check we're not actually recursing */ 1617568cb4eSPaul Mackerras if (kprobe_running()) { 1627568cb4eSPaul Mackerras p = get_kprobe(addr); 1637568cb4eSPaul Mackerras if (p) { 1647568cb4eSPaul Mackerras kprobe_opcode_t insn = *p->ainsn.insn; 1657568cb4eSPaul Mackerras if (kcb->kprobe_status == KPROBE_HIT_SS && 1667568cb4eSPaul Mackerras is_trap(insn)) { 167f8279621SKumar Gala /* Turn off 'trace' bits */ 168f8279621SKumar Gala regs->msr &= ~MSR_SINGLESTEP; 1697568cb4eSPaul Mackerras regs->msr |= kcb->kprobe_saved_msr; 1707568cb4eSPaul Mackerras goto no_kprobe; 1717568cb4eSPaul Mackerras } 1727568cb4eSPaul Mackerras /* We have reentered the kprobe_handler(), since 1737568cb4eSPaul Mackerras * another probe was hit while within the handler. 1747568cb4eSPaul Mackerras * We here save the original kprobes variables and 1757568cb4eSPaul Mackerras * just single step on the instruction of the new probe 1767568cb4eSPaul Mackerras * without calling any user handlers. 1777568cb4eSPaul Mackerras */ 1787568cb4eSPaul Mackerras save_previous_kprobe(kcb); 1797568cb4eSPaul Mackerras set_current_kprobe(p, regs, kcb); 1807568cb4eSPaul Mackerras kcb->kprobe_saved_msr = regs->msr; 181bf8d5c52SKeshavamurthy Anil S kprobes_inc_nmissed_count(p); 1827568cb4eSPaul Mackerras prepare_singlestep(p, regs); 1837568cb4eSPaul Mackerras kcb->kprobe_status = KPROBE_REENTER; 1847568cb4eSPaul Mackerras return 1; 1857568cb4eSPaul Mackerras } else { 186eb3a7292SKeshavamurthy Anil S if (*addr != BREAKPOINT_INSTRUCTION) { 187eb3a7292SKeshavamurthy Anil S /* If trap variant, then it belongs not to us */ 188eb3a7292SKeshavamurthy Anil S kprobe_opcode_t cur_insn = *addr; 189eb3a7292SKeshavamurthy Anil S if (is_trap(cur_insn)) 190eb3a7292SKeshavamurthy Anil S goto no_kprobe; 191eb3a7292SKeshavamurthy Anil S /* The breakpoint instruction was removed by 192eb3a7292SKeshavamurthy Anil S * another cpu right after we hit, no further 193eb3a7292SKeshavamurthy Anil S * handling of this interrupt is appropriate 194eb3a7292SKeshavamurthy Anil S */ 195eb3a7292SKeshavamurthy Anil S ret = 1; 196eb3a7292SKeshavamurthy Anil S goto no_kprobe; 197eb3a7292SKeshavamurthy Anil S } 19869111bacSChristoph Lameter p = __this_cpu_read(current_kprobe); 1997568cb4eSPaul Mackerras if (p->break_handler && p->break_handler(p, regs)) { 2007568cb4eSPaul Mackerras goto ss_probe; 2017568cb4eSPaul Mackerras } 2027568cb4eSPaul Mackerras } 2037568cb4eSPaul Mackerras goto no_kprobe; 2047568cb4eSPaul Mackerras } 2057568cb4eSPaul Mackerras 2067568cb4eSPaul Mackerras p = get_kprobe(addr); 2077568cb4eSPaul Mackerras if (!p) { 2087568cb4eSPaul Mackerras if (*addr != BREAKPOINT_INSTRUCTION) { 2097568cb4eSPaul Mackerras /* 2107568cb4eSPaul Mackerras * PowerPC has multiple variants of the "trap" 2117568cb4eSPaul Mackerras * instruction. If the current instruction is a 2127568cb4eSPaul Mackerras * trap variant, it could belong to someone else 2137568cb4eSPaul Mackerras */ 2147568cb4eSPaul Mackerras kprobe_opcode_t cur_insn = *addr; 2157568cb4eSPaul Mackerras if (is_trap(cur_insn)) 2167568cb4eSPaul Mackerras goto no_kprobe; 2177568cb4eSPaul Mackerras /* 2187568cb4eSPaul Mackerras * The breakpoint instruction was removed right 2197568cb4eSPaul Mackerras * after we hit it. Another cpu has removed 2207568cb4eSPaul Mackerras * either a probepoint or a debugger breakpoint 2217568cb4eSPaul Mackerras * at this address. In either case, no further 2227568cb4eSPaul Mackerras * handling of this interrupt is appropriate. 2237568cb4eSPaul Mackerras */ 2247568cb4eSPaul Mackerras ret = 1; 2257568cb4eSPaul Mackerras } 2267568cb4eSPaul Mackerras /* Not one of ours: let kernel handle it */ 2277568cb4eSPaul Mackerras goto no_kprobe; 2287568cb4eSPaul Mackerras } 2297568cb4eSPaul Mackerras 2307568cb4eSPaul Mackerras kcb->kprobe_status = KPROBE_HIT_ACTIVE; 2317568cb4eSPaul Mackerras set_current_kprobe(p, regs, kcb); 2327568cb4eSPaul Mackerras if (p->pre_handler && p->pre_handler(p, regs)) 2337568cb4eSPaul Mackerras /* handler has already set things up, so skip ss setup */ 2347568cb4eSPaul Mackerras return 1; 2357568cb4eSPaul Mackerras 2367568cb4eSPaul Mackerras ss_probe: 237e6349a95SAnanth N Mavinakayanahalli if (p->ainsn.boostable >= 0) { 238e6349a95SAnanth N Mavinakayanahalli unsigned int insn = *p->ainsn.insn; 239e6349a95SAnanth N Mavinakayanahalli 240e6349a95SAnanth N Mavinakayanahalli /* regs->nip is also adjusted if emulate_step returns 1 */ 241e6349a95SAnanth N Mavinakayanahalli ret = emulate_step(regs, insn); 242e6349a95SAnanth N Mavinakayanahalli if (ret > 0) { 243e6349a95SAnanth N Mavinakayanahalli /* 244e6349a95SAnanth N Mavinakayanahalli * Once this instruction has been boosted 245e6349a95SAnanth N Mavinakayanahalli * successfully, set the boostable flag 246e6349a95SAnanth N Mavinakayanahalli */ 247e6349a95SAnanth N Mavinakayanahalli if (unlikely(p->ainsn.boostable == 0)) 248e6349a95SAnanth N Mavinakayanahalli p->ainsn.boostable = 1; 249e6349a95SAnanth N Mavinakayanahalli 250e6349a95SAnanth N Mavinakayanahalli if (p->post_handler) 251e6349a95SAnanth N Mavinakayanahalli p->post_handler(p, regs, 0); 252e6349a95SAnanth N Mavinakayanahalli 253e6349a95SAnanth N Mavinakayanahalli kcb->kprobe_status = KPROBE_HIT_SSDONE; 254e6349a95SAnanth N Mavinakayanahalli reset_current_kprobe(); 255e6349a95SAnanth N Mavinakayanahalli preempt_enable_no_resched(); 256e6349a95SAnanth N Mavinakayanahalli return 1; 257e6349a95SAnanth N Mavinakayanahalli } else if (ret < 0) { 258e6349a95SAnanth N Mavinakayanahalli /* 259e6349a95SAnanth N Mavinakayanahalli * We don't allow kprobes on mtmsr(d)/rfi(d), etc. 260e6349a95SAnanth N Mavinakayanahalli * So, we should never get here... but, its still 261e6349a95SAnanth N Mavinakayanahalli * good to catch them, just in case... 262e6349a95SAnanth N Mavinakayanahalli */ 263e6349a95SAnanth N Mavinakayanahalli printk("Can't step on instruction %x\n", insn); 264e6349a95SAnanth N Mavinakayanahalli BUG(); 265e6349a95SAnanth N Mavinakayanahalli } else if (ret == 0) 266e6349a95SAnanth N Mavinakayanahalli /* This instruction can't be boosted */ 267e6349a95SAnanth N Mavinakayanahalli p->ainsn.boostable = -1; 268e6349a95SAnanth N Mavinakayanahalli } 2697568cb4eSPaul Mackerras prepare_singlestep(p, regs); 2707568cb4eSPaul Mackerras kcb->kprobe_status = KPROBE_HIT_SS; 2717568cb4eSPaul Mackerras return 1; 2727568cb4eSPaul Mackerras 2737568cb4eSPaul Mackerras no_kprobe: 2747568cb4eSPaul Mackerras preempt_enable_no_resched(); 2757568cb4eSPaul Mackerras return ret; 2767568cb4eSPaul Mackerras } 2777568cb4eSPaul Mackerras 2787568cb4eSPaul Mackerras /* 2797568cb4eSPaul Mackerras * Function return probe trampoline: 2807568cb4eSPaul Mackerras * - init_kprobes() establishes a probepoint here 2817568cb4eSPaul Mackerras * - When the probed function returns, this probe 2827568cb4eSPaul Mackerras * causes the handlers to fire 2837568cb4eSPaul Mackerras */ 28461ed9cfbSThiago Jung Bauermann asm(".global kretprobe_trampoline\n" 28561ed9cfbSThiago Jung Bauermann ".type kretprobe_trampoline, @function\n" 2867568cb4eSPaul Mackerras "kretprobe_trampoline:\n" 28761ed9cfbSThiago Jung Bauermann "nop\n" 28861ed9cfbSThiago Jung Bauermann ".size kretprobe_trampoline, .-kretprobe_trampoline\n"); 2897568cb4eSPaul Mackerras 2907568cb4eSPaul Mackerras /* 2917568cb4eSPaul Mackerras * Called when the probe at kretprobe trampoline is hit 2927568cb4eSPaul Mackerras */ 29366200ea2SAnanth N Mavinakayanahalli static int __kprobes trampoline_probe_handler(struct kprobe *p, 29466200ea2SAnanth N Mavinakayanahalli struct pt_regs *regs) 2957568cb4eSPaul Mackerras { 2967568cb4eSPaul Mackerras struct kretprobe_instance *ri = NULL; 29799219a3fSbibo,mao struct hlist_head *head, empty_rp; 298b67bfe0dSSasha Levin struct hlist_node *tmp; 2997568cb4eSPaul Mackerras unsigned long flags, orig_ret_address = 0; 3007568cb4eSPaul Mackerras unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; 3017568cb4eSPaul Mackerras 30299219a3fSbibo,mao INIT_HLIST_HEAD(&empty_rp); 303ef53d9c5SSrinivasa D S kretprobe_hash_lock(current, &head, &flags); 3047568cb4eSPaul Mackerras 3057568cb4eSPaul Mackerras /* 3067568cb4eSPaul Mackerras * It is possible to have multiple instances associated with a given 3077568cb4eSPaul Mackerras * task either because an multiple functions in the call path 308025dfdafSFrederik Schwarzer * have a return probe installed on them, and/or more than one return 3097568cb4eSPaul Mackerras * return probe was registered for a target function. 3107568cb4eSPaul Mackerras * 3117568cb4eSPaul Mackerras * We can handle this because: 3127568cb4eSPaul Mackerras * - instances are always inserted at the head of the list 3137568cb4eSPaul Mackerras * - when multiple return probes are registered for the same 3147568cb4eSPaul Mackerras * function, the first instance's ret_addr will point to the 3157568cb4eSPaul Mackerras * real return address, and all the rest will point to 3167568cb4eSPaul Mackerras * kretprobe_trampoline 3177568cb4eSPaul Mackerras */ 318b67bfe0dSSasha Levin hlist_for_each_entry_safe(ri, tmp, head, hlist) { 3197568cb4eSPaul Mackerras if (ri->task != current) 3207568cb4eSPaul Mackerras /* another task is sharing our hash bucket */ 3217568cb4eSPaul Mackerras continue; 3227568cb4eSPaul Mackerras 3237568cb4eSPaul Mackerras if (ri->rp && ri->rp->handler) 3247568cb4eSPaul Mackerras ri->rp->handler(ri, regs); 3257568cb4eSPaul Mackerras 3267568cb4eSPaul Mackerras orig_ret_address = (unsigned long)ri->ret_addr; 32799219a3fSbibo,mao recycle_rp_inst(ri, &empty_rp); 3287568cb4eSPaul Mackerras 3297568cb4eSPaul Mackerras if (orig_ret_address != trampoline_address) 3307568cb4eSPaul Mackerras /* 3317568cb4eSPaul Mackerras * This is the real return address. Any other 3327568cb4eSPaul Mackerras * instances associated with this task are for 3337568cb4eSPaul Mackerras * other calls deeper on the call stack 3347568cb4eSPaul Mackerras */ 3357568cb4eSPaul Mackerras break; 3367568cb4eSPaul Mackerras } 3377568cb4eSPaul Mackerras 3380f95b7fcSAnanth N Mavinakayanahalli kretprobe_assert(ri, orig_ret_address, trampoline_address); 3397568cb4eSPaul Mackerras regs->nip = orig_ret_address; 3407568cb4eSPaul Mackerras 3417568cb4eSPaul Mackerras reset_current_kprobe(); 342ef53d9c5SSrinivasa D S kretprobe_hash_unlock(current, &flags); 3437568cb4eSPaul Mackerras preempt_enable_no_resched(); 3447568cb4eSPaul Mackerras 345b67bfe0dSSasha Levin hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 34699219a3fSbibo,mao hlist_del(&ri->hlist); 34799219a3fSbibo,mao kfree(ri); 34899219a3fSbibo,mao } 3497568cb4eSPaul Mackerras /* 3507568cb4eSPaul Mackerras * By returning a non-zero value, we are telling 3517568cb4eSPaul Mackerras * kprobe_handler() that we don't want the post_handler 3527568cb4eSPaul Mackerras * to run (and have re-enabled preemption) 3537568cb4eSPaul Mackerras */ 3547568cb4eSPaul Mackerras return 1; 3557568cb4eSPaul Mackerras } 3567568cb4eSPaul Mackerras 3577568cb4eSPaul Mackerras /* 3587568cb4eSPaul Mackerras * Called after single-stepping. p->addr is the address of the 3597568cb4eSPaul Mackerras * instruction whose first byte has been replaced by the "breakpoint" 3607568cb4eSPaul Mackerras * instruction. To avoid the SMP problems that can occur when we 3617568cb4eSPaul Mackerras * temporarily put back the original opcode to single-step, we 3627568cb4eSPaul Mackerras * single-stepped a copy of the instruction. The address of this 3637568cb4eSPaul Mackerras * copy is p->ainsn.insn. 3647568cb4eSPaul Mackerras */ 365*6cc89badSNaveen N. Rao int __kprobes kprobe_post_handler(struct pt_regs *regs) 3667568cb4eSPaul Mackerras { 3677568cb4eSPaul Mackerras struct kprobe *cur = kprobe_running(); 3687568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 3697568cb4eSPaul Mackerras 370*6cc89badSNaveen N. Rao if (!cur || user_mode(regs)) 3717568cb4eSPaul Mackerras return 0; 3727568cb4eSPaul Mackerras 373b76e59d1SKumar Gala /* make sure we got here for instruction we have a kprobe on */ 374b76e59d1SKumar Gala if (((unsigned long)cur->ainsn.insn + 4) != regs->nip) 375b76e59d1SKumar Gala return 0; 376b76e59d1SKumar Gala 3777568cb4eSPaul Mackerras if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 3787568cb4eSPaul Mackerras kcb->kprobe_status = KPROBE_HIT_SSDONE; 3797568cb4eSPaul Mackerras cur->post_handler(cur, regs, 0); 3807568cb4eSPaul Mackerras } 3817568cb4eSPaul Mackerras 382db97bc7fSAnanth N Mavinakayanahalli /* Adjust nip to after the single-stepped instruction */ 383db97bc7fSAnanth N Mavinakayanahalli regs->nip = (unsigned long)cur->addr + 4; 3847568cb4eSPaul Mackerras regs->msr |= kcb->kprobe_saved_msr; 3857568cb4eSPaul Mackerras 3867568cb4eSPaul Mackerras /*Restore back the original saved kprobes variables and continue. */ 3877568cb4eSPaul Mackerras if (kcb->kprobe_status == KPROBE_REENTER) { 3887568cb4eSPaul Mackerras restore_previous_kprobe(kcb); 3897568cb4eSPaul Mackerras goto out; 3907568cb4eSPaul Mackerras } 3917568cb4eSPaul Mackerras reset_current_kprobe(); 3927568cb4eSPaul Mackerras out: 3937568cb4eSPaul Mackerras preempt_enable_no_resched(); 3947568cb4eSPaul Mackerras 3957568cb4eSPaul Mackerras /* 3967568cb4eSPaul Mackerras * if somebody else is singlestepping across a probe point, msr 397f8279621SKumar Gala * will have DE/SE set, in which case, continue the remaining processing 3987568cb4eSPaul Mackerras * of do_debug, as if this is not a probe hit. 3997568cb4eSPaul Mackerras */ 400f8279621SKumar Gala if (regs->msr & MSR_SINGLESTEP) 4017568cb4eSPaul Mackerras return 0; 4027568cb4eSPaul Mackerras 4037568cb4eSPaul Mackerras return 1; 4047568cb4eSPaul Mackerras } 4057568cb4eSPaul Mackerras 4069f90b997SChristoph Hellwig int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 4077568cb4eSPaul Mackerras { 4087568cb4eSPaul Mackerras struct kprobe *cur = kprobe_running(); 4097568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 41050e21f2bSPrasanna S Panchamukhi const struct exception_table_entry *entry; 4117568cb4eSPaul Mackerras 41250e21f2bSPrasanna S Panchamukhi switch(kcb->kprobe_status) { 41350e21f2bSPrasanna S Panchamukhi case KPROBE_HIT_SS: 41450e21f2bSPrasanna S Panchamukhi case KPROBE_REENTER: 41550e21f2bSPrasanna S Panchamukhi /* 41650e21f2bSPrasanna S Panchamukhi * We are here because the instruction being single 41750e21f2bSPrasanna S Panchamukhi * stepped caused a page fault. We reset the current 41850e21f2bSPrasanna S Panchamukhi * kprobe and the nip points back to the probe address 41950e21f2bSPrasanna S Panchamukhi * and allow the page fault handler to continue as a 42050e21f2bSPrasanna S Panchamukhi * normal page fault. 42150e21f2bSPrasanna S Panchamukhi */ 42250e21f2bSPrasanna S Panchamukhi regs->nip = (unsigned long)cur->addr; 423f8279621SKumar Gala regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */ 42450e21f2bSPrasanna S Panchamukhi regs->msr |= kcb->kprobe_saved_msr; 42550e21f2bSPrasanna S Panchamukhi if (kcb->kprobe_status == KPROBE_REENTER) 42650e21f2bSPrasanna S Panchamukhi restore_previous_kprobe(kcb); 42750e21f2bSPrasanna S Panchamukhi else 42850e21f2bSPrasanna S Panchamukhi reset_current_kprobe(); 42950e21f2bSPrasanna S Panchamukhi preempt_enable_no_resched(); 43050e21f2bSPrasanna S Panchamukhi break; 43150e21f2bSPrasanna S Panchamukhi case KPROBE_HIT_ACTIVE: 43250e21f2bSPrasanna S Panchamukhi case KPROBE_HIT_SSDONE: 43350e21f2bSPrasanna S Panchamukhi /* 43450e21f2bSPrasanna S Panchamukhi * We increment the nmissed count for accounting, 43523d6d3dbSAnoop Thomas Mathew * we can also use npre/npostfault count for accounting 43650e21f2bSPrasanna S Panchamukhi * these specific fault cases. 43750e21f2bSPrasanna S Panchamukhi */ 43850e21f2bSPrasanna S Panchamukhi kprobes_inc_nmissed_count(cur); 43950e21f2bSPrasanna S Panchamukhi 44050e21f2bSPrasanna S Panchamukhi /* 44150e21f2bSPrasanna S Panchamukhi * We come here because instructions in the pre/post 44250e21f2bSPrasanna S Panchamukhi * handler caused the page_fault, this could happen 44350e21f2bSPrasanna S Panchamukhi * if handler tries to access user space by 44450e21f2bSPrasanna S Panchamukhi * copy_from_user(), get_user() etc. Let the 44550e21f2bSPrasanna S Panchamukhi * user-specified handler try to fix it first. 44650e21f2bSPrasanna S Panchamukhi */ 4477568cb4eSPaul Mackerras if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 4487568cb4eSPaul Mackerras return 1; 4497568cb4eSPaul Mackerras 45050e21f2bSPrasanna S Panchamukhi /* 45150e21f2bSPrasanna S Panchamukhi * In case the user-specified fault handler returned 45250e21f2bSPrasanna S Panchamukhi * zero, try to fix up. 45350e21f2bSPrasanna S Panchamukhi */ 45450e21f2bSPrasanna S Panchamukhi if ((entry = search_exception_tables(regs->nip)) != NULL) { 45561a92f70SNicholas Piggin regs->nip = extable_fixup(entry); 45650e21f2bSPrasanna S Panchamukhi return 1; 45750e21f2bSPrasanna S Panchamukhi } 4587568cb4eSPaul Mackerras 45950e21f2bSPrasanna S Panchamukhi /* 46050e21f2bSPrasanna S Panchamukhi * fixup_exception() could not handle it, 46150e21f2bSPrasanna S Panchamukhi * Let do_page_fault() fix it. 46250e21f2bSPrasanna S Panchamukhi */ 46350e21f2bSPrasanna S Panchamukhi break; 46450e21f2bSPrasanna S Panchamukhi default: 46550e21f2bSPrasanna S Panchamukhi break; 4667568cb4eSPaul Mackerras } 4677568cb4eSPaul Mackerras return 0; 4687568cb4eSPaul Mackerras } 4697568cb4eSPaul Mackerras 4707568cb4eSPaul Mackerras /* 4717568cb4eSPaul Mackerras * Wrapper routine to for handling exceptions. 4727568cb4eSPaul Mackerras */ 4737568cb4eSPaul Mackerras int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 4747568cb4eSPaul Mackerras unsigned long val, void *data) 4757568cb4eSPaul Mackerras { 476*6cc89badSNaveen N. Rao return NOTIFY_DONE; 4777568cb4eSPaul Mackerras } 4787568cb4eSPaul Mackerras 4793d7e3382SMichael Ellerman unsigned long arch_deref_entry_point(void *entry) 4803d7e3382SMichael Ellerman { 4812f0143c9SMichael Ellerman return ppc_global_function_entry(entry); 4823d7e3382SMichael Ellerman } 4833d7e3382SMichael Ellerman 4847568cb4eSPaul Mackerras int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 4857568cb4eSPaul Mackerras { 4867568cb4eSPaul Mackerras struct jprobe *jp = container_of(p, struct jprobe, kp); 4877568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 4887568cb4eSPaul Mackerras 4897568cb4eSPaul Mackerras memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); 4907568cb4eSPaul Mackerras 4917568cb4eSPaul Mackerras /* setup return addr to the jprobe handler routine */ 4923d7e3382SMichael Ellerman regs->nip = arch_deref_entry_point(jp->entry); 493f55d9665SMichael Ellerman #ifdef PPC64_ELF_ABI_v2 4942f0143c9SMichael Ellerman regs->gpr[12] = (unsigned long)jp->entry; 495f55d9665SMichael Ellerman #elif defined(PPC64_ELF_ABI_v1) 4967568cb4eSPaul Mackerras regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc); 49782090035SKumar Gala #endif 4987568cb4eSPaul Mackerras 4997568cb4eSPaul Mackerras return 1; 5007568cb4eSPaul Mackerras } 5017568cb4eSPaul Mackerras 50266200ea2SAnanth N Mavinakayanahalli void __used __kprobes jprobe_return(void) 5037568cb4eSPaul Mackerras { 5047568cb4eSPaul Mackerras asm volatile("trap" ::: "memory"); 5057568cb4eSPaul Mackerras } 5067568cb4eSPaul Mackerras 50766200ea2SAnanth N Mavinakayanahalli static void __used __kprobes jprobe_return_end(void) 5087568cb4eSPaul Mackerras { 5097568cb4eSPaul Mackerras }; 5107568cb4eSPaul Mackerras 5117568cb4eSPaul Mackerras int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 5127568cb4eSPaul Mackerras { 5137568cb4eSPaul Mackerras struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 5147568cb4eSPaul Mackerras 5157568cb4eSPaul Mackerras /* 5167568cb4eSPaul Mackerras * FIXME - we should ideally be validating that we got here 'cos 5177568cb4eSPaul Mackerras * of the "trap" in jprobe_return() above, before restoring the 5187568cb4eSPaul Mackerras * saved regs... 5197568cb4eSPaul Mackerras */ 5207568cb4eSPaul Mackerras memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); 5217568cb4eSPaul Mackerras preempt_enable_no_resched(); 5227568cb4eSPaul Mackerras return 1; 5237568cb4eSPaul Mackerras } 5247568cb4eSPaul Mackerras 5257568cb4eSPaul Mackerras static struct kprobe trampoline_p = { 5267568cb4eSPaul Mackerras .addr = (kprobe_opcode_t *) &kretprobe_trampoline, 5277568cb4eSPaul Mackerras .pre_handler = trampoline_probe_handler 5287568cb4eSPaul Mackerras }; 5297568cb4eSPaul Mackerras 5307568cb4eSPaul Mackerras int __init arch_init_kprobes(void) 5317568cb4eSPaul Mackerras { 5327568cb4eSPaul Mackerras return register_kprobe(&trampoline_p); 5337568cb4eSPaul Mackerras } 534bf8f6e5bSAnanth N Mavinakayanahalli 535bf8f6e5bSAnanth N Mavinakayanahalli int __kprobes arch_trampoline_kprobe(struct kprobe *p) 536bf8f6e5bSAnanth N Mavinakayanahalli { 537bf8f6e5bSAnanth N Mavinakayanahalli if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) 538bf8f6e5bSAnanth N Mavinakayanahalli return 1; 539bf8f6e5bSAnanth N Mavinakayanahalli 540bf8f6e5bSAnanth N Mavinakayanahalli return 0; 541bf8f6e5bSAnanth N Mavinakayanahalli } 542