xref: /linux/arch/powerpc/kernel/kprobes.c (revision 6cc89bad60a673a24386f1ada83de8a068a78909)
17568cb4eSPaul Mackerras /*
27568cb4eSPaul Mackerras  *  Kernel Probes (KProbes)
37568cb4eSPaul Mackerras  *
47568cb4eSPaul Mackerras  * This program is free software; you can redistribute it and/or modify
57568cb4eSPaul Mackerras  * it under the terms of the GNU General Public License as published by
67568cb4eSPaul Mackerras  * the Free Software Foundation; either version 2 of the License, or
77568cb4eSPaul Mackerras  * (at your option) any later version.
87568cb4eSPaul Mackerras  *
97568cb4eSPaul Mackerras  * This program is distributed in the hope that it will be useful,
107568cb4eSPaul Mackerras  * but WITHOUT ANY WARRANTY; without even the implied warranty of
117568cb4eSPaul Mackerras  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
127568cb4eSPaul Mackerras  * GNU General Public License for more details.
137568cb4eSPaul Mackerras  *
147568cb4eSPaul Mackerras  * You should have received a copy of the GNU General Public License
157568cb4eSPaul Mackerras  * along with this program; if not, write to the Free Software
167568cb4eSPaul Mackerras  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
177568cb4eSPaul Mackerras  *
187568cb4eSPaul Mackerras  * Copyright (C) IBM Corporation, 2002, 2004
197568cb4eSPaul Mackerras  *
207568cb4eSPaul Mackerras  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
217568cb4eSPaul Mackerras  *		Probes initial implementation ( includes contributions from
227568cb4eSPaul Mackerras  *		Rusty Russell).
237568cb4eSPaul Mackerras  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
247568cb4eSPaul Mackerras  *		interface to access function arguments.
257568cb4eSPaul Mackerras  * 2004-Nov	Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
267568cb4eSPaul Mackerras  *		for PPC64
277568cb4eSPaul Mackerras  */
287568cb4eSPaul Mackerras 
297568cb4eSPaul Mackerras #include <linux/kprobes.h>
307568cb4eSPaul Mackerras #include <linux/ptrace.h>
317568cb4eSPaul Mackerras #include <linux/preempt.h>
328a39b05fSPaul Gortmaker #include <linux/extable.h>
331eeb66a1SChristoph Hellwig #include <linux/kdebug.h>
345a0e3ad6STejun Heo #include <linux/slab.h>
352f0143c9SMichael Ellerman #include <asm/code-patching.h>
367568cb4eSPaul Mackerras #include <asm/cacheflush.h>
377568cb4eSPaul Mackerras #include <asm/sstep.h>
3850e21f2bSPrasanna S Panchamukhi #include <asm/uaccess.h>
39f8279621SKumar Gala 
407568cb4eSPaul Mackerras DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
417568cb4eSPaul Mackerras DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
427568cb4eSPaul Mackerras 
43f438d914SMasami Hiramatsu struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
44f438d914SMasami Hiramatsu 
457568cb4eSPaul Mackerras int __kprobes arch_prepare_kprobe(struct kprobe *p)
467568cb4eSPaul Mackerras {
477568cb4eSPaul Mackerras 	int ret = 0;
487568cb4eSPaul Mackerras 	kprobe_opcode_t insn = *p->addr;
497568cb4eSPaul Mackerras 
507568cb4eSPaul Mackerras 	if ((unsigned long)p->addr & 0x03) {
517568cb4eSPaul Mackerras 		printk("Attempt to register kprobe at an unaligned address\n");
527568cb4eSPaul Mackerras 		ret = -EINVAL;
5382090035SKumar Gala 	} else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
5482090035SKumar Gala 		printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
557568cb4eSPaul Mackerras 		ret = -EINVAL;
567568cb4eSPaul Mackerras 	}
577568cb4eSPaul Mackerras 
58f8279621SKumar Gala 	/* insn must be on a special executable page on ppc64.  This is
59f8279621SKumar Gala 	 * not explicitly required on ppc32 (right now), but it doesn't hurt */
607568cb4eSPaul Mackerras 	if (!ret) {
617568cb4eSPaul Mackerras 		p->ainsn.insn = get_insn_slot();
627568cb4eSPaul Mackerras 		if (!p->ainsn.insn)
637568cb4eSPaul Mackerras 			ret = -ENOMEM;
647568cb4eSPaul Mackerras 	}
657568cb4eSPaul Mackerras 
6649a2a1b8SAnil S Keshavamurthy 	if (!ret) {
67e6349a95SAnanth N Mavinakayanahalli 		memcpy(p->ainsn.insn, p->addr,
68e6349a95SAnanth N Mavinakayanahalli 				MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
697568cb4eSPaul Mackerras 		p->opcode = *p->addr;
7083db3ddeSAnanth N Mavinakayanahalli 		flush_icache_range((unsigned long)p->ainsn.insn,
7183db3ddeSAnanth N Mavinakayanahalli 			(unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
727568cb4eSPaul Mackerras 	}
737568cb4eSPaul Mackerras 
74e6349a95SAnanth N Mavinakayanahalli 	p->ainsn.boostable = 0;
7549a2a1b8SAnil S Keshavamurthy 	return ret;
7649a2a1b8SAnil S Keshavamurthy }
7749a2a1b8SAnil S Keshavamurthy 
787568cb4eSPaul Mackerras void __kprobes arch_arm_kprobe(struct kprobe *p)
797568cb4eSPaul Mackerras {
807568cb4eSPaul Mackerras 	*p->addr = BREAKPOINT_INSTRUCTION;
817568cb4eSPaul Mackerras 	flush_icache_range((unsigned long) p->addr,
827568cb4eSPaul Mackerras 			   (unsigned long) p->addr + sizeof(kprobe_opcode_t));
837568cb4eSPaul Mackerras }
847568cb4eSPaul Mackerras 
857568cb4eSPaul Mackerras void __kprobes arch_disarm_kprobe(struct kprobe *p)
867568cb4eSPaul Mackerras {
877568cb4eSPaul Mackerras 	*p->addr = p->opcode;
887568cb4eSPaul Mackerras 	flush_icache_range((unsigned long) p->addr,
897568cb4eSPaul Mackerras 			   (unsigned long) p->addr + sizeof(kprobe_opcode_t));
907568cb4eSPaul Mackerras }
917568cb4eSPaul Mackerras 
920498b635SAnanth N Mavinakayanahalli void __kprobes arch_remove_kprobe(struct kprobe *p)
937568cb4eSPaul Mackerras {
9412941560SMasami Hiramatsu 	if (p->ainsn.insn) {
95b4c6c34aSMasami Hiramatsu 		free_insn_slot(p->ainsn.insn, 0);
9612941560SMasami Hiramatsu 		p->ainsn.insn = NULL;
9712941560SMasami Hiramatsu 	}
987568cb4eSPaul Mackerras }
997568cb4eSPaul Mackerras 
10046dbe2f4SPrasanna S Panchamukhi static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
1017568cb4eSPaul Mackerras {
10235fd219aSSuzuki K. Poulose 	enable_single_step(regs);
1037568cb4eSPaul Mackerras 
1040ccde0a2SAnanth N Mavinakayanahalli 	/*
1050ccde0a2SAnanth N Mavinakayanahalli 	 * On powerpc we should single step on the original
1060ccde0a2SAnanth N Mavinakayanahalli 	 * instruction even if the probed insn is a trap
1070ccde0a2SAnanth N Mavinakayanahalli 	 * variant as values in regs could play a part in
1080ccde0a2SAnanth N Mavinakayanahalli 	 * if the trap is taken or not
1090ccde0a2SAnanth N Mavinakayanahalli 	 */
1107568cb4eSPaul Mackerras 	regs->nip = (unsigned long)p->ainsn.insn;
1117568cb4eSPaul Mackerras }
1127568cb4eSPaul Mackerras 
11346dbe2f4SPrasanna S Panchamukhi static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
1147568cb4eSPaul Mackerras {
1157568cb4eSPaul Mackerras 	kcb->prev_kprobe.kp = kprobe_running();
1167568cb4eSPaul Mackerras 	kcb->prev_kprobe.status = kcb->kprobe_status;
1177568cb4eSPaul Mackerras 	kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
1187568cb4eSPaul Mackerras }
1197568cb4eSPaul Mackerras 
12046dbe2f4SPrasanna S Panchamukhi static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
1217568cb4eSPaul Mackerras {
12269111bacSChristoph Lameter 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
1237568cb4eSPaul Mackerras 	kcb->kprobe_status = kcb->prev_kprobe.status;
1247568cb4eSPaul Mackerras 	kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
1257568cb4eSPaul Mackerras }
1267568cb4eSPaul Mackerras 
12746dbe2f4SPrasanna S Panchamukhi static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
1287568cb4eSPaul Mackerras 				struct kprobe_ctlblk *kcb)
1297568cb4eSPaul Mackerras {
13069111bacSChristoph Lameter 	__this_cpu_write(current_kprobe, p);
1317568cb4eSPaul Mackerras 	kcb->kprobe_saved_msr = regs->msr;
1327568cb4eSPaul Mackerras }
1337568cb4eSPaul Mackerras 
1344c4308cbSChristoph Hellwig void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
1357568cb4eSPaul Mackerras 				      struct pt_regs *regs)
1367568cb4eSPaul Mackerras {
1377568cb4eSPaul Mackerras 	ri->ret_addr = (kprobe_opcode_t *)regs->link;
1387568cb4eSPaul Mackerras 
1397568cb4eSPaul Mackerras 	/* Replace the return addr with trampoline addr */
1407568cb4eSPaul Mackerras 	regs->link = (unsigned long)kretprobe_trampoline;
1417568cb4eSPaul Mackerras }
1427568cb4eSPaul Mackerras 
143*6cc89badSNaveen N. Rao int __kprobes kprobe_handler(struct pt_regs *regs)
1447568cb4eSPaul Mackerras {
1457568cb4eSPaul Mackerras 	struct kprobe *p;
1467568cb4eSPaul Mackerras 	int ret = 0;
1477568cb4eSPaul Mackerras 	unsigned int *addr = (unsigned int *)regs->nip;
1487568cb4eSPaul Mackerras 	struct kprobe_ctlblk *kcb;
1497568cb4eSPaul Mackerras 
150*6cc89badSNaveen N. Rao 	if (user_mode(regs))
151*6cc89badSNaveen N. Rao 		return 0;
152*6cc89badSNaveen N. Rao 
1537568cb4eSPaul Mackerras 	/*
1547568cb4eSPaul Mackerras 	 * We don't want to be preempted for the entire
1557568cb4eSPaul Mackerras 	 * duration of kprobe processing
1567568cb4eSPaul Mackerras 	 */
1577568cb4eSPaul Mackerras 	preempt_disable();
1587568cb4eSPaul Mackerras 	kcb = get_kprobe_ctlblk();
1597568cb4eSPaul Mackerras 
1607568cb4eSPaul Mackerras 	/* Check we're not actually recursing */
1617568cb4eSPaul Mackerras 	if (kprobe_running()) {
1627568cb4eSPaul Mackerras 		p = get_kprobe(addr);
1637568cb4eSPaul Mackerras 		if (p) {
1647568cb4eSPaul Mackerras 			kprobe_opcode_t insn = *p->ainsn.insn;
1657568cb4eSPaul Mackerras 			if (kcb->kprobe_status == KPROBE_HIT_SS &&
1667568cb4eSPaul Mackerras 					is_trap(insn)) {
167f8279621SKumar Gala 				/* Turn off 'trace' bits */
168f8279621SKumar Gala 				regs->msr &= ~MSR_SINGLESTEP;
1697568cb4eSPaul Mackerras 				regs->msr |= kcb->kprobe_saved_msr;
1707568cb4eSPaul Mackerras 				goto no_kprobe;
1717568cb4eSPaul Mackerras 			}
1727568cb4eSPaul Mackerras 			/* We have reentered the kprobe_handler(), since
1737568cb4eSPaul Mackerras 			 * another probe was hit while within the handler.
1747568cb4eSPaul Mackerras 			 * We here save the original kprobes variables and
1757568cb4eSPaul Mackerras 			 * just single step on the instruction of the new probe
1767568cb4eSPaul Mackerras 			 * without calling any user handlers.
1777568cb4eSPaul Mackerras 			 */
1787568cb4eSPaul Mackerras 			save_previous_kprobe(kcb);
1797568cb4eSPaul Mackerras 			set_current_kprobe(p, regs, kcb);
1807568cb4eSPaul Mackerras 			kcb->kprobe_saved_msr = regs->msr;
181bf8d5c52SKeshavamurthy Anil S 			kprobes_inc_nmissed_count(p);
1827568cb4eSPaul Mackerras 			prepare_singlestep(p, regs);
1837568cb4eSPaul Mackerras 			kcb->kprobe_status = KPROBE_REENTER;
1847568cb4eSPaul Mackerras 			return 1;
1857568cb4eSPaul Mackerras 		} else {
186eb3a7292SKeshavamurthy Anil S 			if (*addr != BREAKPOINT_INSTRUCTION) {
187eb3a7292SKeshavamurthy Anil S 				/* If trap variant, then it belongs not to us */
188eb3a7292SKeshavamurthy Anil S 				kprobe_opcode_t cur_insn = *addr;
189eb3a7292SKeshavamurthy Anil S 				if (is_trap(cur_insn))
190eb3a7292SKeshavamurthy Anil S 		       			goto no_kprobe;
191eb3a7292SKeshavamurthy Anil S 				/* The breakpoint instruction was removed by
192eb3a7292SKeshavamurthy Anil S 				 * another cpu right after we hit, no further
193eb3a7292SKeshavamurthy Anil S 				 * handling of this interrupt is appropriate
194eb3a7292SKeshavamurthy Anil S 				 */
195eb3a7292SKeshavamurthy Anil S 				ret = 1;
196eb3a7292SKeshavamurthy Anil S 				goto no_kprobe;
197eb3a7292SKeshavamurthy Anil S 			}
19869111bacSChristoph Lameter 			p = __this_cpu_read(current_kprobe);
1997568cb4eSPaul Mackerras 			if (p->break_handler && p->break_handler(p, regs)) {
2007568cb4eSPaul Mackerras 				goto ss_probe;
2017568cb4eSPaul Mackerras 			}
2027568cb4eSPaul Mackerras 		}
2037568cb4eSPaul Mackerras 		goto no_kprobe;
2047568cb4eSPaul Mackerras 	}
2057568cb4eSPaul Mackerras 
2067568cb4eSPaul Mackerras 	p = get_kprobe(addr);
2077568cb4eSPaul Mackerras 	if (!p) {
2087568cb4eSPaul Mackerras 		if (*addr != BREAKPOINT_INSTRUCTION) {
2097568cb4eSPaul Mackerras 			/*
2107568cb4eSPaul Mackerras 			 * PowerPC has multiple variants of the "trap"
2117568cb4eSPaul Mackerras 			 * instruction. If the current instruction is a
2127568cb4eSPaul Mackerras 			 * trap variant, it could belong to someone else
2137568cb4eSPaul Mackerras 			 */
2147568cb4eSPaul Mackerras 			kprobe_opcode_t cur_insn = *addr;
2157568cb4eSPaul Mackerras 			if (is_trap(cur_insn))
2167568cb4eSPaul Mackerras 		       		goto no_kprobe;
2177568cb4eSPaul Mackerras 			/*
2187568cb4eSPaul Mackerras 			 * The breakpoint instruction was removed right
2197568cb4eSPaul Mackerras 			 * after we hit it.  Another cpu has removed
2207568cb4eSPaul Mackerras 			 * either a probepoint or a debugger breakpoint
2217568cb4eSPaul Mackerras 			 * at this address.  In either case, no further
2227568cb4eSPaul Mackerras 			 * handling of this interrupt is appropriate.
2237568cb4eSPaul Mackerras 			 */
2247568cb4eSPaul Mackerras 			ret = 1;
2257568cb4eSPaul Mackerras 		}
2267568cb4eSPaul Mackerras 		/* Not one of ours: let kernel handle it */
2277568cb4eSPaul Mackerras 		goto no_kprobe;
2287568cb4eSPaul Mackerras 	}
2297568cb4eSPaul Mackerras 
2307568cb4eSPaul Mackerras 	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2317568cb4eSPaul Mackerras 	set_current_kprobe(p, regs, kcb);
2327568cb4eSPaul Mackerras 	if (p->pre_handler && p->pre_handler(p, regs))
2337568cb4eSPaul Mackerras 		/* handler has already set things up, so skip ss setup */
2347568cb4eSPaul Mackerras 		return 1;
2357568cb4eSPaul Mackerras 
2367568cb4eSPaul Mackerras ss_probe:
237e6349a95SAnanth N Mavinakayanahalli 	if (p->ainsn.boostable >= 0) {
238e6349a95SAnanth N Mavinakayanahalli 		unsigned int insn = *p->ainsn.insn;
239e6349a95SAnanth N Mavinakayanahalli 
240e6349a95SAnanth N Mavinakayanahalli 		/* regs->nip is also adjusted if emulate_step returns 1 */
241e6349a95SAnanth N Mavinakayanahalli 		ret = emulate_step(regs, insn);
242e6349a95SAnanth N Mavinakayanahalli 		if (ret > 0) {
243e6349a95SAnanth N Mavinakayanahalli 			/*
244e6349a95SAnanth N Mavinakayanahalli 			 * Once this instruction has been boosted
245e6349a95SAnanth N Mavinakayanahalli 			 * successfully, set the boostable flag
246e6349a95SAnanth N Mavinakayanahalli 			 */
247e6349a95SAnanth N Mavinakayanahalli 			if (unlikely(p->ainsn.boostable == 0))
248e6349a95SAnanth N Mavinakayanahalli 				p->ainsn.boostable = 1;
249e6349a95SAnanth N Mavinakayanahalli 
250e6349a95SAnanth N Mavinakayanahalli 			if (p->post_handler)
251e6349a95SAnanth N Mavinakayanahalli 				p->post_handler(p, regs, 0);
252e6349a95SAnanth N Mavinakayanahalli 
253e6349a95SAnanth N Mavinakayanahalli 			kcb->kprobe_status = KPROBE_HIT_SSDONE;
254e6349a95SAnanth N Mavinakayanahalli 			reset_current_kprobe();
255e6349a95SAnanth N Mavinakayanahalli 			preempt_enable_no_resched();
256e6349a95SAnanth N Mavinakayanahalli 			return 1;
257e6349a95SAnanth N Mavinakayanahalli 		} else if (ret < 0) {
258e6349a95SAnanth N Mavinakayanahalli 			/*
259e6349a95SAnanth N Mavinakayanahalli 			 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
260e6349a95SAnanth N Mavinakayanahalli 			 * So, we should never get here... but, its still
261e6349a95SAnanth N Mavinakayanahalli 			 * good to catch them, just in case...
262e6349a95SAnanth N Mavinakayanahalli 			 */
263e6349a95SAnanth N Mavinakayanahalli 			printk("Can't step on instruction %x\n", insn);
264e6349a95SAnanth N Mavinakayanahalli 			BUG();
265e6349a95SAnanth N Mavinakayanahalli 		} else if (ret == 0)
266e6349a95SAnanth N Mavinakayanahalli 			/* This instruction can't be boosted */
267e6349a95SAnanth N Mavinakayanahalli 			p->ainsn.boostable = -1;
268e6349a95SAnanth N Mavinakayanahalli 	}
2697568cb4eSPaul Mackerras 	prepare_singlestep(p, regs);
2707568cb4eSPaul Mackerras 	kcb->kprobe_status = KPROBE_HIT_SS;
2717568cb4eSPaul Mackerras 	return 1;
2727568cb4eSPaul Mackerras 
2737568cb4eSPaul Mackerras no_kprobe:
2747568cb4eSPaul Mackerras 	preempt_enable_no_resched();
2757568cb4eSPaul Mackerras 	return ret;
2767568cb4eSPaul Mackerras }
2777568cb4eSPaul Mackerras 
2787568cb4eSPaul Mackerras /*
2797568cb4eSPaul Mackerras  * Function return probe trampoline:
2807568cb4eSPaul Mackerras  * 	- init_kprobes() establishes a probepoint here
2817568cb4eSPaul Mackerras  * 	- When the probed function returns, this probe
2827568cb4eSPaul Mackerras  * 		causes the handlers to fire
2837568cb4eSPaul Mackerras  */
28461ed9cfbSThiago Jung Bauermann asm(".global kretprobe_trampoline\n"
28561ed9cfbSThiago Jung Bauermann 	".type kretprobe_trampoline, @function\n"
2867568cb4eSPaul Mackerras 	"kretprobe_trampoline:\n"
28761ed9cfbSThiago Jung Bauermann 	"nop\n"
28861ed9cfbSThiago Jung Bauermann 	".size kretprobe_trampoline, .-kretprobe_trampoline\n");
2897568cb4eSPaul Mackerras 
2907568cb4eSPaul Mackerras /*
2917568cb4eSPaul Mackerras  * Called when the probe at kretprobe trampoline is hit
2927568cb4eSPaul Mackerras  */
29366200ea2SAnanth N Mavinakayanahalli static int __kprobes trampoline_probe_handler(struct kprobe *p,
29466200ea2SAnanth N Mavinakayanahalli 						struct pt_regs *regs)
2957568cb4eSPaul Mackerras {
2967568cb4eSPaul Mackerras 	struct kretprobe_instance *ri = NULL;
29799219a3fSbibo,mao 	struct hlist_head *head, empty_rp;
298b67bfe0dSSasha Levin 	struct hlist_node *tmp;
2997568cb4eSPaul Mackerras 	unsigned long flags, orig_ret_address = 0;
3007568cb4eSPaul Mackerras 	unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
3017568cb4eSPaul Mackerras 
30299219a3fSbibo,mao 	INIT_HLIST_HEAD(&empty_rp);
303ef53d9c5SSrinivasa D S 	kretprobe_hash_lock(current, &head, &flags);
3047568cb4eSPaul Mackerras 
3057568cb4eSPaul Mackerras 	/*
3067568cb4eSPaul Mackerras 	 * It is possible to have multiple instances associated with a given
3077568cb4eSPaul Mackerras 	 * task either because an multiple functions in the call path
308025dfdafSFrederik Schwarzer 	 * have a return probe installed on them, and/or more than one return
3097568cb4eSPaul Mackerras 	 * return probe was registered for a target function.
3107568cb4eSPaul Mackerras 	 *
3117568cb4eSPaul Mackerras 	 * We can handle this because:
3127568cb4eSPaul Mackerras 	 *     - instances are always inserted at the head of the list
3137568cb4eSPaul Mackerras 	 *     - when multiple return probes are registered for the same
3147568cb4eSPaul Mackerras 	 *       function, the first instance's ret_addr will point to the
3157568cb4eSPaul Mackerras 	 *       real return address, and all the rest will point to
3167568cb4eSPaul Mackerras 	 *       kretprobe_trampoline
3177568cb4eSPaul Mackerras 	 */
318b67bfe0dSSasha Levin 	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
3197568cb4eSPaul Mackerras 		if (ri->task != current)
3207568cb4eSPaul Mackerras 			/* another task is sharing our hash bucket */
3217568cb4eSPaul Mackerras 			continue;
3227568cb4eSPaul Mackerras 
3237568cb4eSPaul Mackerras 		if (ri->rp && ri->rp->handler)
3247568cb4eSPaul Mackerras 			ri->rp->handler(ri, regs);
3257568cb4eSPaul Mackerras 
3267568cb4eSPaul Mackerras 		orig_ret_address = (unsigned long)ri->ret_addr;
32799219a3fSbibo,mao 		recycle_rp_inst(ri, &empty_rp);
3287568cb4eSPaul Mackerras 
3297568cb4eSPaul Mackerras 		if (orig_ret_address != trampoline_address)
3307568cb4eSPaul Mackerras 			/*
3317568cb4eSPaul Mackerras 			 * This is the real return address. Any other
3327568cb4eSPaul Mackerras 			 * instances associated with this task are for
3337568cb4eSPaul Mackerras 			 * other calls deeper on the call stack
3347568cb4eSPaul Mackerras 			 */
3357568cb4eSPaul Mackerras 			break;
3367568cb4eSPaul Mackerras 	}
3377568cb4eSPaul Mackerras 
3380f95b7fcSAnanth N Mavinakayanahalli 	kretprobe_assert(ri, orig_ret_address, trampoline_address);
3397568cb4eSPaul Mackerras 	regs->nip = orig_ret_address;
3407568cb4eSPaul Mackerras 
3417568cb4eSPaul Mackerras 	reset_current_kprobe();
342ef53d9c5SSrinivasa D S 	kretprobe_hash_unlock(current, &flags);
3437568cb4eSPaul Mackerras 	preempt_enable_no_resched();
3447568cb4eSPaul Mackerras 
345b67bfe0dSSasha Levin 	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
34699219a3fSbibo,mao 		hlist_del(&ri->hlist);
34799219a3fSbibo,mao 		kfree(ri);
34899219a3fSbibo,mao 	}
3497568cb4eSPaul Mackerras 	/*
3507568cb4eSPaul Mackerras 	 * By returning a non-zero value, we are telling
3517568cb4eSPaul Mackerras 	 * kprobe_handler() that we don't want the post_handler
3527568cb4eSPaul Mackerras 	 * to run (and have re-enabled preemption)
3537568cb4eSPaul Mackerras 	 */
3547568cb4eSPaul Mackerras 	return 1;
3557568cb4eSPaul Mackerras }
3567568cb4eSPaul Mackerras 
3577568cb4eSPaul Mackerras /*
3587568cb4eSPaul Mackerras  * Called after single-stepping.  p->addr is the address of the
3597568cb4eSPaul Mackerras  * instruction whose first byte has been replaced by the "breakpoint"
3607568cb4eSPaul Mackerras  * instruction.  To avoid the SMP problems that can occur when we
3617568cb4eSPaul Mackerras  * temporarily put back the original opcode to single-step, we
3627568cb4eSPaul Mackerras  * single-stepped a copy of the instruction.  The address of this
3637568cb4eSPaul Mackerras  * copy is p->ainsn.insn.
3647568cb4eSPaul Mackerras  */
365*6cc89badSNaveen N. Rao int __kprobes kprobe_post_handler(struct pt_regs *regs)
3667568cb4eSPaul Mackerras {
3677568cb4eSPaul Mackerras 	struct kprobe *cur = kprobe_running();
3687568cb4eSPaul Mackerras 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
3697568cb4eSPaul Mackerras 
370*6cc89badSNaveen N. Rao 	if (!cur || user_mode(regs))
3717568cb4eSPaul Mackerras 		return 0;
3727568cb4eSPaul Mackerras 
373b76e59d1SKumar Gala 	/* make sure we got here for instruction we have a kprobe on */
374b76e59d1SKumar Gala 	if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
375b76e59d1SKumar Gala 		return 0;
376b76e59d1SKumar Gala 
3777568cb4eSPaul Mackerras 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
3787568cb4eSPaul Mackerras 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
3797568cb4eSPaul Mackerras 		cur->post_handler(cur, regs, 0);
3807568cb4eSPaul Mackerras 	}
3817568cb4eSPaul Mackerras 
382db97bc7fSAnanth N Mavinakayanahalli 	/* Adjust nip to after the single-stepped instruction */
383db97bc7fSAnanth N Mavinakayanahalli 	regs->nip = (unsigned long)cur->addr + 4;
3847568cb4eSPaul Mackerras 	regs->msr |= kcb->kprobe_saved_msr;
3857568cb4eSPaul Mackerras 
3867568cb4eSPaul Mackerras 	/*Restore back the original saved kprobes variables and continue. */
3877568cb4eSPaul Mackerras 	if (kcb->kprobe_status == KPROBE_REENTER) {
3887568cb4eSPaul Mackerras 		restore_previous_kprobe(kcb);
3897568cb4eSPaul Mackerras 		goto out;
3907568cb4eSPaul Mackerras 	}
3917568cb4eSPaul Mackerras 	reset_current_kprobe();
3927568cb4eSPaul Mackerras out:
3937568cb4eSPaul Mackerras 	preempt_enable_no_resched();
3947568cb4eSPaul Mackerras 
3957568cb4eSPaul Mackerras 	/*
3967568cb4eSPaul Mackerras 	 * if somebody else is singlestepping across a probe point, msr
397f8279621SKumar Gala 	 * will have DE/SE set, in which case, continue the remaining processing
3987568cb4eSPaul Mackerras 	 * of do_debug, as if this is not a probe hit.
3997568cb4eSPaul Mackerras 	 */
400f8279621SKumar Gala 	if (regs->msr & MSR_SINGLESTEP)
4017568cb4eSPaul Mackerras 		return 0;
4027568cb4eSPaul Mackerras 
4037568cb4eSPaul Mackerras 	return 1;
4047568cb4eSPaul Mackerras }
4057568cb4eSPaul Mackerras 
4069f90b997SChristoph Hellwig int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
4077568cb4eSPaul Mackerras {
4087568cb4eSPaul Mackerras 	struct kprobe *cur = kprobe_running();
4097568cb4eSPaul Mackerras 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
41050e21f2bSPrasanna S Panchamukhi 	const struct exception_table_entry *entry;
4117568cb4eSPaul Mackerras 
41250e21f2bSPrasanna S Panchamukhi 	switch(kcb->kprobe_status) {
41350e21f2bSPrasanna S Panchamukhi 	case KPROBE_HIT_SS:
41450e21f2bSPrasanna S Panchamukhi 	case KPROBE_REENTER:
41550e21f2bSPrasanna S Panchamukhi 		/*
41650e21f2bSPrasanna S Panchamukhi 		 * We are here because the instruction being single
41750e21f2bSPrasanna S Panchamukhi 		 * stepped caused a page fault. We reset the current
41850e21f2bSPrasanna S Panchamukhi 		 * kprobe and the nip points back to the probe address
41950e21f2bSPrasanna S Panchamukhi 		 * and allow the page fault handler to continue as a
42050e21f2bSPrasanna S Panchamukhi 		 * normal page fault.
42150e21f2bSPrasanna S Panchamukhi 		 */
42250e21f2bSPrasanna S Panchamukhi 		regs->nip = (unsigned long)cur->addr;
423f8279621SKumar Gala 		regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */
42450e21f2bSPrasanna S Panchamukhi 		regs->msr |= kcb->kprobe_saved_msr;
42550e21f2bSPrasanna S Panchamukhi 		if (kcb->kprobe_status == KPROBE_REENTER)
42650e21f2bSPrasanna S Panchamukhi 			restore_previous_kprobe(kcb);
42750e21f2bSPrasanna S Panchamukhi 		else
42850e21f2bSPrasanna S Panchamukhi 			reset_current_kprobe();
42950e21f2bSPrasanna S Panchamukhi 		preempt_enable_no_resched();
43050e21f2bSPrasanna S Panchamukhi 		break;
43150e21f2bSPrasanna S Panchamukhi 	case KPROBE_HIT_ACTIVE:
43250e21f2bSPrasanna S Panchamukhi 	case KPROBE_HIT_SSDONE:
43350e21f2bSPrasanna S Panchamukhi 		/*
43450e21f2bSPrasanna S Panchamukhi 		 * We increment the nmissed count for accounting,
43523d6d3dbSAnoop Thomas Mathew 		 * we can also use npre/npostfault count for accounting
43650e21f2bSPrasanna S Panchamukhi 		 * these specific fault cases.
43750e21f2bSPrasanna S Panchamukhi 		 */
43850e21f2bSPrasanna S Panchamukhi 		kprobes_inc_nmissed_count(cur);
43950e21f2bSPrasanna S Panchamukhi 
44050e21f2bSPrasanna S Panchamukhi 		/*
44150e21f2bSPrasanna S Panchamukhi 		 * We come here because instructions in the pre/post
44250e21f2bSPrasanna S Panchamukhi 		 * handler caused the page_fault, this could happen
44350e21f2bSPrasanna S Panchamukhi 		 * if handler tries to access user space by
44450e21f2bSPrasanna S Panchamukhi 		 * copy_from_user(), get_user() etc. Let the
44550e21f2bSPrasanna S Panchamukhi 		 * user-specified handler try to fix it first.
44650e21f2bSPrasanna S Panchamukhi 		 */
4477568cb4eSPaul Mackerras 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
4487568cb4eSPaul Mackerras 			return 1;
4497568cb4eSPaul Mackerras 
45050e21f2bSPrasanna S Panchamukhi 		/*
45150e21f2bSPrasanna S Panchamukhi 		 * In case the user-specified fault handler returned
45250e21f2bSPrasanna S Panchamukhi 		 * zero, try to fix up.
45350e21f2bSPrasanna S Panchamukhi 		 */
45450e21f2bSPrasanna S Panchamukhi 		if ((entry = search_exception_tables(regs->nip)) != NULL) {
45561a92f70SNicholas Piggin 			regs->nip = extable_fixup(entry);
45650e21f2bSPrasanna S Panchamukhi 			return 1;
45750e21f2bSPrasanna S Panchamukhi 		}
4587568cb4eSPaul Mackerras 
45950e21f2bSPrasanna S Panchamukhi 		/*
46050e21f2bSPrasanna S Panchamukhi 		 * fixup_exception() could not handle it,
46150e21f2bSPrasanna S Panchamukhi 		 * Let do_page_fault() fix it.
46250e21f2bSPrasanna S Panchamukhi 		 */
46350e21f2bSPrasanna S Panchamukhi 		break;
46450e21f2bSPrasanna S Panchamukhi 	default:
46550e21f2bSPrasanna S Panchamukhi 		break;
4667568cb4eSPaul Mackerras 	}
4677568cb4eSPaul Mackerras 	return 0;
4687568cb4eSPaul Mackerras }
4697568cb4eSPaul Mackerras 
4707568cb4eSPaul Mackerras /*
4717568cb4eSPaul Mackerras  * Wrapper routine to for handling exceptions.
4727568cb4eSPaul Mackerras  */
4737568cb4eSPaul Mackerras int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
4747568cb4eSPaul Mackerras 				       unsigned long val, void *data)
4757568cb4eSPaul Mackerras {
476*6cc89badSNaveen N. Rao 	return NOTIFY_DONE;
4777568cb4eSPaul Mackerras }
4787568cb4eSPaul Mackerras 
4793d7e3382SMichael Ellerman unsigned long arch_deref_entry_point(void *entry)
4803d7e3382SMichael Ellerman {
4812f0143c9SMichael Ellerman 	return ppc_global_function_entry(entry);
4823d7e3382SMichael Ellerman }
4833d7e3382SMichael Ellerman 
4847568cb4eSPaul Mackerras int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
4857568cb4eSPaul Mackerras {
4867568cb4eSPaul Mackerras 	struct jprobe *jp = container_of(p, struct jprobe, kp);
4877568cb4eSPaul Mackerras 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
4887568cb4eSPaul Mackerras 
4897568cb4eSPaul Mackerras 	memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
4907568cb4eSPaul Mackerras 
4917568cb4eSPaul Mackerras 	/* setup return addr to the jprobe handler routine */
4923d7e3382SMichael Ellerman 	regs->nip = arch_deref_entry_point(jp->entry);
493f55d9665SMichael Ellerman #ifdef PPC64_ELF_ABI_v2
4942f0143c9SMichael Ellerman 	regs->gpr[12] = (unsigned long)jp->entry;
495f55d9665SMichael Ellerman #elif defined(PPC64_ELF_ABI_v1)
4967568cb4eSPaul Mackerras 	regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
49782090035SKumar Gala #endif
4987568cb4eSPaul Mackerras 
4997568cb4eSPaul Mackerras 	return 1;
5007568cb4eSPaul Mackerras }
5017568cb4eSPaul Mackerras 
50266200ea2SAnanth N Mavinakayanahalli void __used __kprobes jprobe_return(void)
5037568cb4eSPaul Mackerras {
5047568cb4eSPaul Mackerras 	asm volatile("trap" ::: "memory");
5057568cb4eSPaul Mackerras }
5067568cb4eSPaul Mackerras 
50766200ea2SAnanth N Mavinakayanahalli static void __used __kprobes jprobe_return_end(void)
5087568cb4eSPaul Mackerras {
5097568cb4eSPaul Mackerras };
5107568cb4eSPaul Mackerras 
5117568cb4eSPaul Mackerras int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
5127568cb4eSPaul Mackerras {
5137568cb4eSPaul Mackerras 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
5147568cb4eSPaul Mackerras 
5157568cb4eSPaul Mackerras 	/*
5167568cb4eSPaul Mackerras 	 * FIXME - we should ideally be validating that we got here 'cos
5177568cb4eSPaul Mackerras 	 * of the "trap" in jprobe_return() above, before restoring the
5187568cb4eSPaul Mackerras 	 * saved regs...
5197568cb4eSPaul Mackerras 	 */
5207568cb4eSPaul Mackerras 	memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
5217568cb4eSPaul Mackerras 	preempt_enable_no_resched();
5227568cb4eSPaul Mackerras 	return 1;
5237568cb4eSPaul Mackerras }
5247568cb4eSPaul Mackerras 
5257568cb4eSPaul Mackerras static struct kprobe trampoline_p = {
5267568cb4eSPaul Mackerras 	.addr = (kprobe_opcode_t *) &kretprobe_trampoline,
5277568cb4eSPaul Mackerras 	.pre_handler = trampoline_probe_handler
5287568cb4eSPaul Mackerras };
5297568cb4eSPaul Mackerras 
5307568cb4eSPaul Mackerras int __init arch_init_kprobes(void)
5317568cb4eSPaul Mackerras {
5327568cb4eSPaul Mackerras 	return register_kprobe(&trampoline_p);
5337568cb4eSPaul Mackerras }
534bf8f6e5bSAnanth N Mavinakayanahalli 
535bf8f6e5bSAnanth N Mavinakayanahalli int __kprobes arch_trampoline_kprobe(struct kprobe *p)
536bf8f6e5bSAnanth N Mavinakayanahalli {
537bf8f6e5bSAnanth N Mavinakayanahalli 	if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
538bf8f6e5bSAnanth N Mavinakayanahalli 		return 1;
539bf8f6e5bSAnanth N Mavinakayanahalli 
540bf8f6e5bSAnanth N Mavinakayanahalli 	return 0;
541bf8f6e5bSAnanth N Mavinakayanahalli }
542