1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/preempt.h> 5 #include <asm/break.h> 6 7 #define KPROBE_BP_INSN __emit_break(BRK_KPROBE_BP) 8 #define KPROBE_SSTEPBP_INSN __emit_break(BRK_KPROBE_SSTEPBP) 9 10 DEFINE_PER_CPU(struct kprobe *, current_kprobe); 11 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 12 13 static void arch_prepare_ss_slot(struct kprobe *p) 14 { 15 p->ainsn.insn[0] = *p->addr; 16 p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN; 17 p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; 18 } 19 NOKPROBE_SYMBOL(arch_prepare_ss_slot); 20 21 static void arch_prepare_simulate(struct kprobe *p) 22 { 23 p->ainsn.restore = 0; 24 } 25 NOKPROBE_SYMBOL(arch_prepare_simulate); 26 27 int arch_prepare_kprobe(struct kprobe *p) 28 { 29 union loongarch_instruction insn; 30 31 if ((unsigned long)p->addr & 0x3) 32 return -EILSEQ; 33 34 /* copy instruction */ 35 p->opcode = *p->addr; 36 insn.word = p->opcode; 37 38 /* decode instruction */ 39 if (insns_not_supported(insn)) 40 return -EINVAL; 41 42 if (insns_need_simulation(insn)) { 43 p->ainsn.insn = NULL; 44 } else { 45 p->ainsn.insn = get_insn_slot(); 46 if (!p->ainsn.insn) 47 return -ENOMEM; 48 } 49 50 /* prepare the instruction */ 51 if (p->ainsn.insn) 52 arch_prepare_ss_slot(p); 53 else 54 arch_prepare_simulate(p); 55 56 return 0; 57 } 58 NOKPROBE_SYMBOL(arch_prepare_kprobe); 59 60 /* Install breakpoint in text */ 61 void arch_arm_kprobe(struct kprobe *p) 62 { 63 u32 insn = KPROBE_BP_INSN; 64 65 larch_insn_text_copy(p->addr, &insn, LOONGARCH_INSN_SIZE); 66 } 67 NOKPROBE_SYMBOL(arch_arm_kprobe); 68 69 /* Remove breakpoint from text */ 70 void arch_disarm_kprobe(struct kprobe *p) 71 { 72 u32 insn = p->opcode; 73 74 larch_insn_text_copy(p->addr, &insn, LOONGARCH_INSN_SIZE); 75 } 76 NOKPROBE_SYMBOL(arch_disarm_kprobe); 77 78 void arch_remove_kprobe(struct kprobe *p) 79 { 80 if (p->ainsn.insn) { 81 free_insn_slot(p->ainsn.insn, 0); 82 p->ainsn.insn = NULL; 83 } 84 } 85 NOKPROBE_SYMBOL(arch_remove_kprobe); 86 87 static void save_previous_kprobe(struct kprobe_ctlblk *kcb) 88 { 89 kcb->prev_kprobe.kp = kprobe_running(); 90 kcb->prev_kprobe.status = kcb->kprobe_status; 91 } 92 NOKPROBE_SYMBOL(save_previous_kprobe); 93 94 static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) 95 { 96 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 97 kcb->kprobe_status = kcb->prev_kprobe.status; 98 } 99 NOKPROBE_SYMBOL(restore_previous_kprobe); 100 101 static void set_current_kprobe(struct kprobe *p) 102 { 103 __this_cpu_write(current_kprobe, p); 104 } 105 NOKPROBE_SYMBOL(set_current_kprobe); 106 107 /* 108 * Interrupts need to be disabled before single-step mode is set, 109 * and not reenabled until after single-step mode ends. 110 * Without disabling interrupt on local CPU, there is a chance of 111 * interrupt occurrence in the period of exception return and start 112 * of out-of-line single-step, that result in wrongly single stepping 113 * into the interrupt handler. 114 */ 115 static void save_local_irqflag(struct kprobe_ctlblk *kcb, 116 struct pt_regs *regs) 117 { 118 kcb->saved_status = regs->csr_prmd; 119 regs->csr_prmd &= ~CSR_PRMD_PIE; 120 } 121 NOKPROBE_SYMBOL(save_local_irqflag); 122 123 static void restore_local_irqflag(struct kprobe_ctlblk *kcb, 124 struct pt_regs *regs) 125 { 126 regs->csr_prmd = kcb->saved_status; 127 } 128 NOKPROBE_SYMBOL(restore_local_irqflag); 129 130 static void post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, 131 struct pt_regs *regs) 132 { 133 /* return addr restore if non-branching insn */ 134 if (cur->ainsn.restore != 0) 135 instruction_pointer_set(regs, cur->ainsn.restore); 136 137 /* restore back original saved kprobe variables and continue */ 138 if (kcb->kprobe_status == KPROBE_REENTER) { 139 restore_previous_kprobe(kcb); 140 preempt_enable_no_resched(); 141 return; 142 } 143 144 /* 145 * update the kcb status even if the cur->post_handler is 146 * not set because reset_curent_kprobe() doesn't update kcb. 147 */ 148 kcb->kprobe_status = KPROBE_HIT_SSDONE; 149 if (cur->post_handler) 150 cur->post_handler(cur, regs, 0); 151 152 reset_current_kprobe(); 153 preempt_enable_no_resched(); 154 } 155 NOKPROBE_SYMBOL(post_kprobe_handler); 156 157 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 158 struct kprobe_ctlblk *kcb, int reenter) 159 { 160 union loongarch_instruction insn; 161 162 if (reenter) { 163 save_previous_kprobe(kcb); 164 set_current_kprobe(p); 165 kcb->kprobe_status = KPROBE_REENTER; 166 } else { 167 kcb->kprobe_status = KPROBE_HIT_SS; 168 } 169 170 if (p->ainsn.insn) { 171 /* IRQs and single stepping do not mix well */ 172 save_local_irqflag(kcb, regs); 173 /* set ip register to prepare for single stepping */ 174 regs->csr_era = (unsigned long)p->ainsn.insn; 175 } else { 176 /* simulate single steping */ 177 insn.word = p->opcode; 178 arch_simulate_insn(insn, regs); 179 /* now go for post processing */ 180 post_kprobe_handler(p, kcb, regs); 181 } 182 } 183 NOKPROBE_SYMBOL(setup_singlestep); 184 185 static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 186 struct kprobe_ctlblk *kcb) 187 { 188 switch (kcb->kprobe_status) { 189 case KPROBE_HIT_SSDONE: 190 case KPROBE_HIT_ACTIVE: 191 kprobes_inc_nmissed_count(p); 192 setup_singlestep(p, regs, kcb, 1); 193 break; 194 case KPROBE_HIT_SS: 195 case KPROBE_REENTER: 196 pr_warn("Failed to recover from reentered kprobes.\n"); 197 dump_kprobe(p); 198 BUG(); 199 break; 200 default: 201 WARN_ON(1); 202 return false; 203 } 204 205 return true; 206 } 207 NOKPROBE_SYMBOL(reenter_kprobe); 208 209 bool kprobe_breakpoint_handler(struct pt_regs *regs) 210 { 211 struct kprobe_ctlblk *kcb; 212 struct kprobe *p, *cur_kprobe; 213 kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->csr_era; 214 215 /* 216 * We don't want to be preempted for the entire 217 * duration of kprobe processing. 218 */ 219 preempt_disable(); 220 kcb = get_kprobe_ctlblk(); 221 cur_kprobe = kprobe_running(); 222 223 p = get_kprobe(addr); 224 if (p) { 225 if (cur_kprobe) { 226 if (reenter_kprobe(p, regs, kcb)) 227 return true; 228 } else { 229 /* Probe hit */ 230 set_current_kprobe(p); 231 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 232 233 /* 234 * If we have no pre-handler or it returned 0, we 235 * continue with normal processing. If we have a 236 * pre-handler and it returned non-zero, it will 237 * modify the execution path and no need to single 238 * stepping. Let's just reset current kprobe and exit. 239 * 240 * pre_handler can hit a breakpoint and can step thru 241 * before return. 242 */ 243 if (!p->pre_handler || !p->pre_handler(p, regs)) { 244 setup_singlestep(p, regs, kcb, 0); 245 } else { 246 reset_current_kprobe(); 247 preempt_enable_no_resched(); 248 } 249 return true; 250 } 251 } 252 253 if (*addr != KPROBE_BP_INSN) { 254 /* 255 * The breakpoint instruction was removed right 256 * after we hit it. Another cpu has removed 257 * either a probepoint or a debugger breakpoint 258 * at this address. In either case, no further 259 * handling of this interrupt is appropriate. 260 * Return back to original instruction, and continue. 261 */ 262 regs->csr_era = (unsigned long)addr; 263 preempt_enable_no_resched(); 264 return true; 265 } 266 267 preempt_enable_no_resched(); 268 return false; 269 } 270 NOKPROBE_SYMBOL(kprobe_breakpoint_handler); 271 272 bool kprobe_singlestep_handler(struct pt_regs *regs) 273 { 274 struct kprobe *cur = kprobe_running(); 275 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 276 unsigned long addr = instruction_pointer(regs); 277 278 if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && 279 ((unsigned long)&cur->ainsn.insn[1] == addr)) { 280 restore_local_irqflag(kcb, regs); 281 post_kprobe_handler(cur, kcb, regs); 282 return true; 283 } 284 285 preempt_enable_no_resched(); 286 return false; 287 } 288 NOKPROBE_SYMBOL(kprobe_singlestep_handler); 289 290 bool kprobe_fault_handler(struct pt_regs *regs, int trapnr) 291 { 292 struct kprobe *cur = kprobe_running(); 293 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 294 295 switch (kcb->kprobe_status) { 296 case KPROBE_HIT_SS: 297 case KPROBE_REENTER: 298 /* 299 * We are here because the instruction being single 300 * stepped caused a page fault. We reset the current 301 * kprobe and the ip points back to the probe address 302 * and allow the page fault handler to continue as a 303 * normal page fault. 304 */ 305 regs->csr_era = (unsigned long)cur->addr; 306 WARN_ON_ONCE(!instruction_pointer(regs)); 307 308 if (kcb->kprobe_status == KPROBE_REENTER) { 309 restore_previous_kprobe(kcb); 310 } else { 311 restore_local_irqflag(kcb, regs); 312 reset_current_kprobe(); 313 } 314 preempt_enable_no_resched(); 315 break; 316 } 317 return false; 318 } 319 NOKPROBE_SYMBOL(kprobe_fault_handler); 320 321 /* 322 * Provide a blacklist of symbols identifying ranges which cannot be kprobed. 323 * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). 324 */ 325 int __init arch_populate_kprobe_blacklist(void) 326 { 327 return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, 328 (unsigned long)__irqentry_text_end); 329 } 330 331 int __init arch_init_kprobes(void) 332 { 333 return 0; 334 } 335 336 int arch_trampoline_kprobe(struct kprobe *p) 337 { 338 return 0; 339 } 340 NOKPROBE_SYMBOL(arch_trampoline_kprobe); 341