1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2023 Loongson Technology Corporation Limited 4 */ 5 #define pr_fmt(fmt) "hw-breakpoint: " fmt 6 7 #include <linux/hw_breakpoint.h> 8 #include <linux/kprobes.h> 9 #include <linux/perf_event.h> 10 11 #include <asm/hw_breakpoint.h> 12 13 /* Breakpoint currently in use for each BRP. */ 14 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]); 15 16 /* Watchpoint currently in use for each WRP. */ 17 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]); 18 19 int hw_breakpoint_slots(int type) 20 { 21 /* 22 * We can be called early, so don't rely on 23 * our static variables being initialised. 24 */ 25 switch (type) { 26 case TYPE_INST: 27 return get_num_brps(); 28 case TYPE_DATA: 29 return get_num_wrps(); 30 default: 31 pr_warn("unknown slot type: %d\n", type); 32 return 0; 33 } 34 } 35 36 #define READ_WB_REG_CASE(OFF, N, REG, T, VAL) \ 37 case (OFF + N): \ 38 LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL); \ 39 break 40 41 #define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL) \ 42 case (OFF + N): \ 43 LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL); \ 44 break 45 46 #define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL) \ 47 READ_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 48 READ_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 49 READ_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 50 READ_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 51 READ_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 52 READ_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 53 READ_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 54 READ_WB_REG_CASE(OFF, 7, REG, T, VAL); 55 56 #define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL) \ 57 WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL); \ 58 WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL); \ 59 WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL); \ 60 WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL); \ 61 WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL); \ 62 WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL); \ 63 WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL); \ 64 WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL); 65 66 static u64 read_wb_reg(int reg, int n, int t) 67 { 68 u64 val = 0; 69 70 switch (reg + n) { 71 GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 72 GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 73 GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 74 GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 75 default: 76 pr_warn("Attempt to read from unknown breakpoint register %d\n", n); 77 } 78 79 return val; 80 } 81 NOKPROBE_SYMBOL(read_wb_reg); 82 83 static void write_wb_reg(int reg, int n, int t, u64 val) 84 { 85 switch (reg + n) { 86 GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); 87 GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); 88 GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); 89 GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); 90 default: 91 pr_warn("Attempt to write to unknown breakpoint register %d\n", n); 92 } 93 } 94 NOKPROBE_SYMBOL(write_wb_reg); 95 96 enum hw_breakpoint_ops { 97 HW_BREAKPOINT_INSTALL, 98 HW_BREAKPOINT_UNINSTALL, 99 }; 100 101 /* 102 * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations 103 * 104 * @slots: pointer to array of slots 105 * @max_slots: max number of slots 106 * @bp: perf_event to setup 107 * @ops: operation to be carried out on the slot 108 * 109 * Return: 110 * slot index on success 111 * -ENOSPC if no slot is available/matches 112 * -EINVAL on wrong operations parameter 113 */ 114 115 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, 116 struct perf_event *bp, enum hw_breakpoint_ops ops) 117 { 118 int i; 119 struct perf_event **slot; 120 121 for (i = 0; i < max_slots; ++i) { 122 slot = &slots[i]; 123 switch (ops) { 124 case HW_BREAKPOINT_INSTALL: 125 if (!*slot) { 126 *slot = bp; 127 return i; 128 } 129 break; 130 case HW_BREAKPOINT_UNINSTALL: 131 if (*slot == bp) { 132 *slot = NULL; 133 return i; 134 } 135 break; 136 default: 137 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); 138 return -EINVAL; 139 } 140 } 141 142 return -ENOSPC; 143 } 144 145 void ptrace_hw_copy_thread(struct task_struct *tsk) 146 { 147 memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break)); 148 memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch)); 149 } 150 151 /* 152 * Unregister breakpoints from this task and reset the pointers in the thread_struct. 153 */ 154 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 155 { 156 int i; 157 struct thread_struct *t = &tsk->thread; 158 159 for (i = 0; i < LOONGARCH_MAX_BRP; i++) { 160 if (t->hbp_break[i]) { 161 unregister_hw_breakpoint(t->hbp_break[i]); 162 t->hbp_break[i] = NULL; 163 } 164 } 165 166 for (i = 0; i < LOONGARCH_MAX_WRP; i++) { 167 if (t->hbp_watch[i]) { 168 unregister_hw_breakpoint(t->hbp_watch[i]); 169 t->hbp_watch[i] = NULL; 170 } 171 } 172 } 173 174 static int hw_breakpoint_control(struct perf_event *bp, 175 enum hw_breakpoint_ops ops) 176 { 177 u32 ctrl; 178 int i, max_slots, enable; 179 struct perf_event **slots; 180 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 181 182 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 183 /* Breakpoint */ 184 slots = this_cpu_ptr(bp_on_reg); 185 max_slots = boot_cpu_data.watch_ireg_count; 186 } else { 187 /* Watchpoint */ 188 slots = this_cpu_ptr(wp_on_reg); 189 max_slots = boot_cpu_data.watch_dreg_count; 190 } 191 192 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); 193 194 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) 195 return i; 196 197 switch (ops) { 198 case HW_BREAKPOINT_INSTALL: 199 /* Set the FWPnCFG/MWPnCFG 1~4 register. */ 200 write_wb_reg(CSR_CFG_ADDR, i, 0, info->address); 201 write_wb_reg(CSR_CFG_ADDR, i, 1, info->address); 202 write_wb_reg(CSR_CFG_MASK, i, 0, info->mask); 203 write_wb_reg(CSR_CFG_MASK, i, 1, info->mask); 204 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 205 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 206 if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { 207 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 208 } else { 209 ctrl = encode_ctrl_reg(info->ctrl); 210 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | CTRL_PLV_ENABLE | 211 1 << MWPnCFG3_LoadEn | 1 << MWPnCFG3_StoreEn); 212 } 213 enable = csr_read64(LOONGARCH_CSR_CRMD); 214 csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD); 215 break; 216 case HW_BREAKPOINT_UNINSTALL: 217 /* Reset the FWPnCFG/MWPnCFG 1~4 register. */ 218 write_wb_reg(CSR_CFG_ADDR, i, 0, 0); 219 write_wb_reg(CSR_CFG_ADDR, i, 1, 0); 220 write_wb_reg(CSR_CFG_MASK, i, 0, 0); 221 write_wb_reg(CSR_CFG_MASK, i, 1, 0); 222 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 223 write_wb_reg(CSR_CFG_CTRL, i, 1, 0); 224 write_wb_reg(CSR_CFG_ASID, i, 0, 0); 225 write_wb_reg(CSR_CFG_ASID, i, 1, 0); 226 break; 227 } 228 229 return 0; 230 } 231 232 /* 233 * Install a perf counter breakpoint. 234 */ 235 int arch_install_hw_breakpoint(struct perf_event *bp) 236 { 237 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); 238 } 239 240 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 241 { 242 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); 243 } 244 245 static int get_hbp_len(u8 hbp_len) 246 { 247 unsigned int len_in_bytes = 0; 248 249 switch (hbp_len) { 250 case LOONGARCH_BREAKPOINT_LEN_1: 251 len_in_bytes = 1; 252 break; 253 case LOONGARCH_BREAKPOINT_LEN_2: 254 len_in_bytes = 2; 255 break; 256 case LOONGARCH_BREAKPOINT_LEN_4: 257 len_in_bytes = 4; 258 break; 259 case LOONGARCH_BREAKPOINT_LEN_8: 260 len_in_bytes = 8; 261 break; 262 } 263 264 return len_in_bytes; 265 } 266 267 /* 268 * Check whether bp virtual address is in kernel space. 269 */ 270 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 271 { 272 unsigned int len; 273 unsigned long va; 274 275 va = hw->address; 276 len = get_hbp_len(hw->ctrl.len); 277 278 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 279 } 280 281 /* 282 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 283 * Hopefully this will disappear when ptrace can bypass the conversion 284 * to generic breakpoint descriptions. 285 */ 286 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 287 int *gen_len, int *gen_type, int *offset) 288 { 289 /* Type */ 290 switch (ctrl.type) { 291 case LOONGARCH_BREAKPOINT_EXECUTE: 292 *gen_type = HW_BREAKPOINT_X; 293 break; 294 case LOONGARCH_BREAKPOINT_LOAD: 295 *gen_type = HW_BREAKPOINT_R; 296 break; 297 case LOONGARCH_BREAKPOINT_STORE: 298 *gen_type = HW_BREAKPOINT_W; 299 break; 300 case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE: 301 *gen_type = HW_BREAKPOINT_RW; 302 break; 303 default: 304 return -EINVAL; 305 } 306 307 if (!ctrl.len) 308 return -EINVAL; 309 310 *offset = __ffs(ctrl.len); 311 312 /* Len */ 313 switch (ctrl.len) { 314 case LOONGARCH_BREAKPOINT_LEN_1: 315 *gen_len = HW_BREAKPOINT_LEN_1; 316 break; 317 case LOONGARCH_BREAKPOINT_LEN_2: 318 *gen_len = HW_BREAKPOINT_LEN_2; 319 break; 320 case LOONGARCH_BREAKPOINT_LEN_4: 321 *gen_len = HW_BREAKPOINT_LEN_4; 322 break; 323 case LOONGARCH_BREAKPOINT_LEN_8: 324 *gen_len = HW_BREAKPOINT_LEN_8; 325 break; 326 default: 327 return -EINVAL; 328 } 329 330 return 0; 331 } 332 333 /* 334 * Construct an arch_hw_breakpoint from a perf_event. 335 */ 336 static int arch_build_bp_info(struct perf_event *bp, 337 const struct perf_event_attr *attr, 338 struct arch_hw_breakpoint *hw) 339 { 340 /* Type */ 341 switch (attr->bp_type) { 342 case HW_BREAKPOINT_X: 343 hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE; 344 break; 345 case HW_BREAKPOINT_R: 346 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD; 347 break; 348 case HW_BREAKPOINT_W: 349 hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE; 350 break; 351 case HW_BREAKPOINT_RW: 352 hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE; 353 break; 354 default: 355 return -EINVAL; 356 } 357 358 /* Len */ 359 switch (attr->bp_len) { 360 case HW_BREAKPOINT_LEN_1: 361 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1; 362 break; 363 case HW_BREAKPOINT_LEN_2: 364 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2; 365 break; 366 case HW_BREAKPOINT_LEN_4: 367 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4; 368 break; 369 case HW_BREAKPOINT_LEN_8: 370 hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8; 371 break; 372 default: 373 return -EINVAL; 374 } 375 376 /* Address */ 377 hw->address = attr->bp_addr; 378 379 return 0; 380 } 381 382 /* 383 * Validate the arch-specific HW Breakpoint register settings. 384 */ 385 int hw_breakpoint_arch_parse(struct perf_event *bp, 386 const struct perf_event_attr *attr, 387 struct arch_hw_breakpoint *hw) 388 { 389 int ret; 390 u64 alignment_mask, offset; 391 392 /* Build the arch_hw_breakpoint. */ 393 ret = arch_build_bp_info(bp, attr, hw); 394 if (ret) 395 return ret; 396 397 if (hw->ctrl.type != LOONGARCH_BREAKPOINT_EXECUTE) 398 alignment_mask = 0x7; 399 offset = hw->address & alignment_mask; 400 401 hw->address &= ~alignment_mask; 402 hw->ctrl.len <<= offset; 403 404 return 0; 405 } 406 407 static void update_bp_registers(struct pt_regs *regs, int enable, int type) 408 { 409 u32 ctrl; 410 int i, max_slots; 411 struct perf_event **slots; 412 struct arch_hw_breakpoint *info; 413 414 switch (type) { 415 case 0: 416 slots = this_cpu_ptr(bp_on_reg); 417 max_slots = boot_cpu_data.watch_ireg_count; 418 break; 419 case 1: 420 slots = this_cpu_ptr(wp_on_reg); 421 max_slots = boot_cpu_data.watch_dreg_count; 422 break; 423 default: 424 return; 425 } 426 427 for (i = 0; i < max_slots; ++i) { 428 if (!slots[i]) 429 continue; 430 431 info = counter_arch_bp(slots[i]); 432 if (enable) { 433 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 434 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 435 write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); 436 } else { 437 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 438 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 439 ctrl |= 0x1 << MWPnCFG3_LoadEn; 440 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 441 ctrl |= 0x1 << MWPnCFG3_StoreEn; 442 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 443 } 444 regs->csr_prmd |= CSR_PRMD_PWE; 445 } else { 446 if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { 447 write_wb_reg(CSR_CFG_CTRL, i, 0, 0); 448 } else { 449 ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); 450 if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) 451 ctrl &= ~0x1 << MWPnCFG3_LoadEn; 452 if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) 453 ctrl &= ~0x1 << MWPnCFG3_StoreEn; 454 write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); 455 } 456 regs->csr_prmd &= ~CSR_PRMD_PWE; 457 } 458 } 459 } 460 NOKPROBE_SYMBOL(update_bp_registers); 461 462 /* 463 * Debug exception handlers. 464 */ 465 void breakpoint_handler(struct pt_regs *regs) 466 { 467 int i; 468 struct perf_event *bp, **slots; 469 470 slots = this_cpu_ptr(bp_on_reg); 471 472 for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) { 473 bp = slots[i]; 474 if (bp == NULL) 475 continue; 476 perf_bp_event(bp, regs); 477 } 478 update_bp_registers(regs, 0, 0); 479 } 480 NOKPROBE_SYMBOL(breakpoint_handler); 481 482 void watchpoint_handler(struct pt_regs *regs) 483 { 484 int i; 485 struct perf_event *wp, **slots; 486 487 slots = this_cpu_ptr(wp_on_reg); 488 489 for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) { 490 wp = slots[i]; 491 if (wp == NULL) 492 continue; 493 perf_bp_event(wp, regs); 494 } 495 update_bp_registers(regs, 0, 1); 496 } 497 NOKPROBE_SYMBOL(watchpoint_handler); 498 499 static int __init arch_hw_breakpoint_init(void) 500 { 501 int cpu; 502 503 boot_cpu_data.watch_ireg_count = get_num_brps(); 504 boot_cpu_data.watch_dreg_count = get_num_wrps(); 505 506 pr_info("Found %d breakpoint and %d watchpoint registers.\n", 507 boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count); 508 509 for (cpu = 1; cpu < NR_CPUS; cpu++) { 510 cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count; 511 cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count; 512 } 513 514 return 0; 515 } 516 arch_initcall(arch_hw_breakpoint_init); 517 518 void hw_breakpoint_thread_switch(struct task_struct *next) 519 { 520 u64 addr, mask; 521 struct pt_regs *regs = task_pt_regs(next); 522 523 if (test_tsk_thread_flag(next, TIF_SINGLESTEP)) { 524 addr = read_wb_reg(CSR_CFG_ADDR, 0, 0); 525 mask = read_wb_reg(CSR_CFG_MASK, 0, 0); 526 if (!((regs->csr_era ^ addr) & ~mask)) 527 csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); 528 regs->csr_prmd |= CSR_PRMD_PWE; 529 } else { 530 /* Update breakpoints */ 531 update_bp_registers(regs, 1, 0); 532 /* Update watchpoints */ 533 update_bp_registers(regs, 1, 1); 534 } 535 } 536 537 void hw_breakpoint_pmu_read(struct perf_event *bp) 538 { 539 } 540 541 /* 542 * Dummy function to register with die_notifier. 543 */ 544 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 545 unsigned long val, void *data) 546 { 547 return NOTIFY_DONE; 548 } 549