1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 4 * using the CPU's debug registers. 5 * 6 * Copyright (C) 2012 ARM Limited 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #define pr_fmt(fmt) "hw-breakpoint: " fmt 11 12 #include <linux/compat.h> 13 #include <linux/cpu_pm.h> 14 #include <linux/errno.h> 15 #include <linux/hw_breakpoint.h> 16 #include <linux/kprobes.h> 17 #include <linux/perf_event.h> 18 #include <linux/ptrace.h> 19 #include <linux/smp.h> 20 #include <linux/uaccess.h> 21 22 #include <asm/current.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/esr.h> 25 #include <asm/hw_breakpoint.h> 26 #include <asm/traps.h> 27 #include <asm/cputype.h> 28 #include <asm/system_misc.h> 29 30 /* Breakpoint currently in use for each BRP. */ 31 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); 32 33 /* Watchpoint currently in use for each WRP. */ 34 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); 35 36 /* Currently stepping a per-CPU kernel breakpoint. */ 37 static DEFINE_PER_CPU(int, stepping_kernel_bp); 38 39 /* Number of BRP/WRP registers on this CPU. */ 40 static int core_num_brps; 41 static int core_num_wrps; 42 43 int hw_breakpoint_slots(int type) 44 { 45 /* 46 * We can be called early, so don't rely on 47 * our static variables being initialised. 48 */ 49 switch (type) { 50 case TYPE_INST: 51 return get_num_brps(); 52 case TYPE_DATA: 53 return get_num_wrps(); 54 default: 55 pr_warn("unknown slot type: %d\n", type); 56 return 0; 57 } 58 } 59 60 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \ 61 case (OFF + N): \ 62 AARCH64_DBG_READ(N, REG, VAL); \ 63 break 64 65 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ 66 case (OFF + N): \ 67 AARCH64_DBG_WRITE(N, REG, VAL); \ 68 break 69 70 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \ 71 READ_WB_REG_CASE(OFF, 0, REG, VAL); \ 72 READ_WB_REG_CASE(OFF, 1, REG, VAL); \ 73 READ_WB_REG_CASE(OFF, 2, REG, VAL); \ 74 READ_WB_REG_CASE(OFF, 3, REG, VAL); \ 75 READ_WB_REG_CASE(OFF, 4, REG, VAL); \ 76 READ_WB_REG_CASE(OFF, 5, REG, VAL); \ 77 READ_WB_REG_CASE(OFF, 6, REG, VAL); \ 78 READ_WB_REG_CASE(OFF, 7, REG, VAL); \ 79 READ_WB_REG_CASE(OFF, 8, REG, VAL); \ 80 READ_WB_REG_CASE(OFF, 9, REG, VAL); \ 81 READ_WB_REG_CASE(OFF, 10, REG, VAL); \ 82 READ_WB_REG_CASE(OFF, 11, REG, VAL); \ 83 READ_WB_REG_CASE(OFF, 12, REG, VAL); \ 84 READ_WB_REG_CASE(OFF, 13, REG, VAL); \ 85 READ_WB_REG_CASE(OFF, 14, REG, VAL); \ 86 READ_WB_REG_CASE(OFF, 15, REG, VAL) 87 88 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \ 89 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ 90 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ 91 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ 92 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ 93 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ 94 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ 95 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ 96 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ 97 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ 98 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ 99 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ 100 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ 101 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ 102 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ 103 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ 104 WRITE_WB_REG_CASE(OFF, 15, REG, VAL) 105 106 static u64 read_wb_reg(int reg, int n) 107 { 108 u64 val = 0; 109 110 switch (reg + n) { 111 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); 112 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); 113 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); 114 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); 115 default: 116 pr_warn("attempt to read from unknown breakpoint register %d\n", n); 117 } 118 119 return val; 120 } 121 NOKPROBE_SYMBOL(read_wb_reg); 122 123 static void write_wb_reg(int reg, int n, u64 val) 124 { 125 switch (reg + n) { 126 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); 127 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); 128 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); 129 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); 130 default: 131 pr_warn("attempt to write to unknown breakpoint register %d\n", n); 132 } 133 isb(); 134 } 135 NOKPROBE_SYMBOL(write_wb_reg); 136 137 /* 138 * Convert a breakpoint privilege level to the corresponding exception 139 * level. 140 */ 141 static enum dbg_active_el debug_exception_level(int privilege) 142 { 143 switch (privilege) { 144 case AARCH64_BREAKPOINT_EL0: 145 return DBG_ACTIVE_EL0; 146 case AARCH64_BREAKPOINT_EL1: 147 return DBG_ACTIVE_EL1; 148 default: 149 pr_warn("invalid breakpoint privilege level %d\n", privilege); 150 return -EINVAL; 151 } 152 } 153 NOKPROBE_SYMBOL(debug_exception_level); 154 155 enum hw_breakpoint_ops { 156 HW_BREAKPOINT_INSTALL, 157 HW_BREAKPOINT_UNINSTALL, 158 HW_BREAKPOINT_RESTORE 159 }; 160 161 static int is_compat_bp(struct perf_event *bp) 162 { 163 struct task_struct *tsk = bp->hw.target; 164 165 /* 166 * tsk can be NULL for per-cpu (non-ptrace) breakpoints. 167 * In this case, use the native interface, since we don't have 168 * the notion of a "compat CPU" and could end up relying on 169 * deprecated behaviour if we use unaligned watchpoints in 170 * AArch64 state. 171 */ 172 return tsk && is_compat_thread(task_thread_info(tsk)); 173 } 174 175 /** 176 * hw_breakpoint_slot_setup - Find and setup a perf slot according to 177 * operations 178 * 179 * @slots: pointer to array of slots 180 * @max_slots: max number of slots 181 * @bp: perf_event to setup 182 * @ops: operation to be carried out on the slot 183 * 184 * Return: 185 * slot index on success 186 * -ENOSPC if no slot is available/matches 187 * -EINVAL on wrong operations parameter 188 */ 189 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, 190 struct perf_event *bp, 191 enum hw_breakpoint_ops ops) 192 { 193 int i; 194 struct perf_event **slot; 195 196 for (i = 0; i < max_slots; ++i) { 197 slot = &slots[i]; 198 switch (ops) { 199 case HW_BREAKPOINT_INSTALL: 200 if (!*slot) { 201 *slot = bp; 202 return i; 203 } 204 break; 205 case HW_BREAKPOINT_UNINSTALL: 206 if (*slot == bp) { 207 *slot = NULL; 208 return i; 209 } 210 break; 211 case HW_BREAKPOINT_RESTORE: 212 if (*slot == bp) 213 return i; 214 break; 215 default: 216 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); 217 return -EINVAL; 218 } 219 } 220 return -ENOSPC; 221 } 222 223 static int hw_breakpoint_control(struct perf_event *bp, 224 enum hw_breakpoint_ops ops) 225 { 226 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 227 struct perf_event **slots; 228 struct debug_info *debug_info = ¤t->thread.debug; 229 int i, max_slots, ctrl_reg, val_reg, reg_enable; 230 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege); 231 u32 ctrl; 232 233 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 234 /* Breakpoint */ 235 ctrl_reg = AARCH64_DBG_REG_BCR; 236 val_reg = AARCH64_DBG_REG_BVR; 237 slots = this_cpu_ptr(bp_on_reg); 238 max_slots = core_num_brps; 239 reg_enable = !debug_info->bps_disabled; 240 } else { 241 /* Watchpoint */ 242 ctrl_reg = AARCH64_DBG_REG_WCR; 243 val_reg = AARCH64_DBG_REG_WVR; 244 slots = this_cpu_ptr(wp_on_reg); 245 max_slots = core_num_wrps; 246 reg_enable = !debug_info->wps_disabled; 247 } 248 249 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); 250 251 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) 252 return i; 253 254 switch (ops) { 255 case HW_BREAKPOINT_INSTALL: 256 /* 257 * Ensure debug monitors are enabled at the correct exception 258 * level. 259 */ 260 enable_debug_monitors(dbg_el); 261 fallthrough; 262 case HW_BREAKPOINT_RESTORE: 263 /* Setup the address register. */ 264 write_wb_reg(val_reg, i, info->address); 265 266 /* Setup the control register. */ 267 ctrl = encode_ctrl_reg(info->ctrl); 268 write_wb_reg(ctrl_reg, i, 269 reg_enable ? ctrl | 0x1 : ctrl & ~0x1); 270 break; 271 case HW_BREAKPOINT_UNINSTALL: 272 /* Reset the control register. */ 273 write_wb_reg(ctrl_reg, i, 0); 274 275 /* 276 * Release the debug monitors for the correct exception 277 * level. 278 */ 279 disable_debug_monitors(dbg_el); 280 break; 281 } 282 283 return 0; 284 } 285 286 /* 287 * Install a perf counter breakpoint. 288 */ 289 int arch_install_hw_breakpoint(struct perf_event *bp) 290 { 291 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); 292 } 293 294 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 295 { 296 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); 297 } 298 299 static int get_hbp_len(u8 hbp_len) 300 { 301 unsigned int len_in_bytes = 0; 302 303 switch (hbp_len) { 304 case ARM_BREAKPOINT_LEN_1: 305 len_in_bytes = 1; 306 break; 307 case ARM_BREAKPOINT_LEN_2: 308 len_in_bytes = 2; 309 break; 310 case ARM_BREAKPOINT_LEN_3: 311 len_in_bytes = 3; 312 break; 313 case ARM_BREAKPOINT_LEN_4: 314 len_in_bytes = 4; 315 break; 316 case ARM_BREAKPOINT_LEN_5: 317 len_in_bytes = 5; 318 break; 319 case ARM_BREAKPOINT_LEN_6: 320 len_in_bytes = 6; 321 break; 322 case ARM_BREAKPOINT_LEN_7: 323 len_in_bytes = 7; 324 break; 325 case ARM_BREAKPOINT_LEN_8: 326 len_in_bytes = 8; 327 break; 328 } 329 330 return len_in_bytes; 331 } 332 333 /* 334 * Check whether bp virtual address is in kernel space. 335 */ 336 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 337 { 338 unsigned int len; 339 unsigned long va; 340 341 va = hw->address; 342 len = get_hbp_len(hw->ctrl.len); 343 344 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 345 } 346 347 /* 348 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 349 * Hopefully this will disappear when ptrace can bypass the conversion 350 * to generic breakpoint descriptions. 351 */ 352 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 353 int *gen_len, int *gen_type, int *offset) 354 { 355 /* Type */ 356 switch (ctrl.type) { 357 case ARM_BREAKPOINT_EXECUTE: 358 *gen_type = HW_BREAKPOINT_X; 359 break; 360 case ARM_BREAKPOINT_LOAD: 361 *gen_type = HW_BREAKPOINT_R; 362 break; 363 case ARM_BREAKPOINT_STORE: 364 *gen_type = HW_BREAKPOINT_W; 365 break; 366 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: 367 *gen_type = HW_BREAKPOINT_RW; 368 break; 369 default: 370 return -EINVAL; 371 } 372 373 if (!ctrl.len) 374 return -EINVAL; 375 *offset = __ffs(ctrl.len); 376 377 /* Len */ 378 switch (ctrl.len >> *offset) { 379 case ARM_BREAKPOINT_LEN_1: 380 *gen_len = HW_BREAKPOINT_LEN_1; 381 break; 382 case ARM_BREAKPOINT_LEN_2: 383 *gen_len = HW_BREAKPOINT_LEN_2; 384 break; 385 case ARM_BREAKPOINT_LEN_3: 386 *gen_len = HW_BREAKPOINT_LEN_3; 387 break; 388 case ARM_BREAKPOINT_LEN_4: 389 *gen_len = HW_BREAKPOINT_LEN_4; 390 break; 391 case ARM_BREAKPOINT_LEN_5: 392 *gen_len = HW_BREAKPOINT_LEN_5; 393 break; 394 case ARM_BREAKPOINT_LEN_6: 395 *gen_len = HW_BREAKPOINT_LEN_6; 396 break; 397 case ARM_BREAKPOINT_LEN_7: 398 *gen_len = HW_BREAKPOINT_LEN_7; 399 break; 400 case ARM_BREAKPOINT_LEN_8: 401 *gen_len = HW_BREAKPOINT_LEN_8; 402 break; 403 default: 404 return -EINVAL; 405 } 406 407 return 0; 408 } 409 410 /* 411 * Construct an arch_hw_breakpoint from a perf_event. 412 */ 413 static int arch_build_bp_info(struct perf_event *bp, 414 const struct perf_event_attr *attr, 415 struct arch_hw_breakpoint *hw) 416 { 417 /* Type */ 418 switch (attr->bp_type) { 419 case HW_BREAKPOINT_X: 420 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; 421 break; 422 case HW_BREAKPOINT_R: 423 hw->ctrl.type = ARM_BREAKPOINT_LOAD; 424 break; 425 case HW_BREAKPOINT_W: 426 hw->ctrl.type = ARM_BREAKPOINT_STORE; 427 break; 428 case HW_BREAKPOINT_RW: 429 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; 430 break; 431 default: 432 return -EINVAL; 433 } 434 435 /* Len */ 436 switch (attr->bp_len) { 437 case HW_BREAKPOINT_LEN_1: 438 hw->ctrl.len = ARM_BREAKPOINT_LEN_1; 439 break; 440 case HW_BREAKPOINT_LEN_2: 441 hw->ctrl.len = ARM_BREAKPOINT_LEN_2; 442 break; 443 case HW_BREAKPOINT_LEN_3: 444 hw->ctrl.len = ARM_BREAKPOINT_LEN_3; 445 break; 446 case HW_BREAKPOINT_LEN_4: 447 hw->ctrl.len = ARM_BREAKPOINT_LEN_4; 448 break; 449 case HW_BREAKPOINT_LEN_5: 450 hw->ctrl.len = ARM_BREAKPOINT_LEN_5; 451 break; 452 case HW_BREAKPOINT_LEN_6: 453 hw->ctrl.len = ARM_BREAKPOINT_LEN_6; 454 break; 455 case HW_BREAKPOINT_LEN_7: 456 hw->ctrl.len = ARM_BREAKPOINT_LEN_7; 457 break; 458 case HW_BREAKPOINT_LEN_8: 459 hw->ctrl.len = ARM_BREAKPOINT_LEN_8; 460 break; 461 default: 462 return -EINVAL; 463 } 464 465 /* 466 * On AArch64, we only permit breakpoints of length 4, whereas 467 * AArch32 also requires breakpoints of length 2 for Thumb. 468 * Watchpoints can be of length 1, 2, 4 or 8 bytes. 469 */ 470 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 471 if (is_compat_bp(bp)) { 472 if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && 473 hw->ctrl.len != ARM_BREAKPOINT_LEN_4) 474 return -EINVAL; 475 } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) { 476 /* 477 * FIXME: Some tools (I'm looking at you perf) assume 478 * that breakpoints should be sizeof(long). This 479 * is nonsense. For now, we fix up the parameter 480 * but we should probably return -EINVAL instead. 481 */ 482 hw->ctrl.len = ARM_BREAKPOINT_LEN_4; 483 } 484 } 485 486 /* Address */ 487 hw->address = attr->bp_addr; 488 489 /* 490 * Privilege 491 * Note that we disallow combined EL0/EL1 breakpoints because 492 * that would complicate the stepping code. 493 */ 494 if (arch_check_bp_in_kernelspace(hw)) 495 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1; 496 else 497 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0; 498 499 /* Enabled? */ 500 hw->ctrl.enabled = !attr->disabled; 501 502 return 0; 503 } 504 505 /* 506 * Validate the arch-specific HW Breakpoint register settings. 507 */ 508 int hw_breakpoint_arch_parse(struct perf_event *bp, 509 const struct perf_event_attr *attr, 510 struct arch_hw_breakpoint *hw) 511 { 512 int ret; 513 u64 alignment_mask, offset; 514 515 /* Build the arch_hw_breakpoint. */ 516 ret = arch_build_bp_info(bp, attr, hw); 517 if (ret) 518 return ret; 519 520 /* 521 * Check address alignment. 522 * We don't do any clever alignment correction for watchpoints 523 * because using 64-bit unaligned addresses is deprecated for 524 * AArch64. 525 * 526 * AArch32 tasks expect some simple alignment fixups, so emulate 527 * that here. 528 */ 529 if (is_compat_bp(bp)) { 530 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) 531 alignment_mask = 0x7; 532 else 533 alignment_mask = 0x3; 534 offset = hw->address & alignment_mask; 535 switch (offset) { 536 case 0: 537 /* Aligned */ 538 break; 539 case 1: 540 case 2: 541 /* Allow halfword watchpoints and breakpoints. */ 542 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) 543 break; 544 545 fallthrough; 546 case 3: 547 /* Allow single byte watchpoint. */ 548 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) 549 break; 550 551 fallthrough; 552 default: 553 return -EINVAL; 554 } 555 } else { 556 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) 557 alignment_mask = 0x3; 558 else 559 alignment_mask = 0x7; 560 offset = hw->address & alignment_mask; 561 } 562 563 hw->address &= ~alignment_mask; 564 hw->ctrl.len <<= offset; 565 566 /* 567 * Disallow per-task kernel breakpoints since these would 568 * complicate the stepping code. 569 */ 570 if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) 571 return -EINVAL; 572 573 return 0; 574 } 575 576 /* 577 * Enable/disable all of the breakpoints active at the specified 578 * exception level at the register level. 579 * This is used when single-stepping after a breakpoint exception. 580 */ 581 static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable) 582 { 583 int i, max_slots, privilege; 584 u32 ctrl; 585 struct perf_event **slots; 586 587 switch (reg) { 588 case AARCH64_DBG_REG_BCR: 589 slots = this_cpu_ptr(bp_on_reg); 590 max_slots = core_num_brps; 591 break; 592 case AARCH64_DBG_REG_WCR: 593 slots = this_cpu_ptr(wp_on_reg); 594 max_slots = core_num_wrps; 595 break; 596 default: 597 return; 598 } 599 600 for (i = 0; i < max_slots; ++i) { 601 if (!slots[i]) 602 continue; 603 604 privilege = counter_arch_bp(slots[i])->ctrl.privilege; 605 if (debug_exception_level(privilege) != el) 606 continue; 607 608 ctrl = read_wb_reg(reg, i); 609 if (enable) 610 ctrl |= 0x1; 611 else 612 ctrl &= ~0x1; 613 write_wb_reg(reg, i, ctrl); 614 } 615 } 616 NOKPROBE_SYMBOL(toggle_bp_registers); 617 618 /* 619 * Debug exception handlers. 620 */ 621 static int breakpoint_handler(unsigned long unused, unsigned long esr, 622 struct pt_regs *regs) 623 { 624 int i, step = 0, *kernel_step; 625 u32 ctrl_reg; 626 u64 addr, val; 627 struct perf_event *bp, **slots; 628 struct debug_info *debug_info; 629 struct arch_hw_breakpoint_ctrl ctrl; 630 631 slots = this_cpu_ptr(bp_on_reg); 632 addr = instruction_pointer(regs); 633 debug_info = ¤t->thread.debug; 634 635 for (i = 0; i < core_num_brps; ++i) { 636 rcu_read_lock(); 637 638 bp = slots[i]; 639 640 if (bp == NULL) 641 goto unlock; 642 643 /* Check if the breakpoint value matches. */ 644 val = read_wb_reg(AARCH64_DBG_REG_BVR, i); 645 if (val != (addr & ~0x3)) 646 goto unlock; 647 648 /* Possible match, check the byte address select to confirm. */ 649 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i); 650 decode_ctrl_reg(ctrl_reg, &ctrl); 651 if (!((1 << (addr & 0x3)) & ctrl.len)) 652 goto unlock; 653 654 counter_arch_bp(bp)->trigger = addr; 655 perf_bp_event(bp, regs); 656 657 /* Do we need to handle the stepping? */ 658 if (is_default_overflow_handler(bp)) 659 step = 1; 660 unlock: 661 rcu_read_unlock(); 662 } 663 664 if (!step) 665 return 0; 666 667 if (user_mode(regs)) { 668 debug_info->bps_disabled = 1; 669 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0); 670 671 /* If we're already stepping a watchpoint, just return. */ 672 if (debug_info->wps_disabled) 673 return 0; 674 675 if (test_thread_flag(TIF_SINGLESTEP)) 676 debug_info->suspended_step = 1; 677 else 678 user_enable_single_step(current); 679 } else { 680 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); 681 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 682 683 if (*kernel_step != ARM_KERNEL_STEP_NONE) 684 return 0; 685 686 if (kernel_active_single_step()) { 687 *kernel_step = ARM_KERNEL_STEP_SUSPEND; 688 } else { 689 *kernel_step = ARM_KERNEL_STEP_ACTIVE; 690 kernel_enable_single_step(regs); 691 } 692 } 693 694 return 0; 695 } 696 NOKPROBE_SYMBOL(breakpoint_handler); 697 698 /* 699 * Arm64 hardware does not always report a watchpoint hit address that matches 700 * one of the watchpoints set. It can also report an address "near" the 701 * watchpoint if a single instruction access both watched and unwatched 702 * addresses. There is no straight-forward way, short of disassembling the 703 * offending instruction, to map that address back to the watchpoint. This 704 * function computes the distance of the memory access from the watchpoint as a 705 * heuristic for the likelihood that a given access triggered the watchpoint. 706 * 707 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint 708 * exception" of ARMv8 Architecture Reference Manual for details. 709 * 710 * The function returns the distance of the address from the bytes watched by 711 * the watchpoint. In case of an exact match, it returns 0. 712 */ 713 static u64 get_distance_from_watchpoint(unsigned long addr, u64 val, 714 struct arch_hw_breakpoint_ctrl *ctrl) 715 { 716 u64 wp_low, wp_high; 717 u32 lens, lene; 718 719 addr = untagged_addr(addr); 720 721 lens = __ffs(ctrl->len); 722 lene = __fls(ctrl->len); 723 724 wp_low = val + lens; 725 wp_high = val + lene; 726 if (addr < wp_low) 727 return wp_low - addr; 728 else if (addr > wp_high) 729 return addr - wp_high; 730 else 731 return 0; 732 } 733 734 static int watchpoint_report(struct perf_event *wp, unsigned long addr, 735 struct pt_regs *regs) 736 { 737 int step = is_default_overflow_handler(wp); 738 struct arch_hw_breakpoint *info = counter_arch_bp(wp); 739 740 info->trigger = addr; 741 742 /* 743 * If we triggered a user watchpoint from a uaccess routine, then 744 * handle the stepping ourselves since userspace really can't help 745 * us with this. 746 */ 747 if (!user_mode(regs) && info->ctrl.privilege == AARCH64_BREAKPOINT_EL0) 748 step = 1; 749 else 750 perf_bp_event(wp, regs); 751 752 return step; 753 } 754 755 static int watchpoint_handler(unsigned long addr, unsigned long esr, 756 struct pt_regs *regs) 757 { 758 int i, step = 0, *kernel_step, access, closest_match = 0; 759 u64 min_dist = -1, dist; 760 u32 ctrl_reg; 761 u64 val; 762 struct perf_event *wp, **slots; 763 struct debug_info *debug_info; 764 struct arch_hw_breakpoint_ctrl ctrl; 765 766 slots = this_cpu_ptr(wp_on_reg); 767 debug_info = ¤t->thread.debug; 768 769 /* 770 * Find all watchpoints that match the reported address. If no exact 771 * match is found. Attribute the hit to the closest watchpoint. 772 */ 773 rcu_read_lock(); 774 for (i = 0; i < core_num_wrps; ++i) { 775 wp = slots[i]; 776 if (wp == NULL) 777 continue; 778 779 /* 780 * Check that the access type matches. 781 * 0 => load, otherwise => store 782 */ 783 access = (esr & ESR_ELx_WNR) ? HW_BREAKPOINT_W : 784 HW_BREAKPOINT_R; 785 if (!(access & hw_breakpoint_type(wp))) 786 continue; 787 788 /* Check if the watchpoint value and byte select match. */ 789 val = read_wb_reg(AARCH64_DBG_REG_WVR, i); 790 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); 791 decode_ctrl_reg(ctrl_reg, &ctrl); 792 dist = get_distance_from_watchpoint(addr, val, &ctrl); 793 if (dist < min_dist) { 794 min_dist = dist; 795 closest_match = i; 796 } 797 /* Is this an exact match? */ 798 if (dist != 0) 799 continue; 800 801 step = watchpoint_report(wp, addr, regs); 802 } 803 804 /* No exact match found? */ 805 if (min_dist > 0 && min_dist != -1) 806 step = watchpoint_report(slots[closest_match], addr, regs); 807 808 rcu_read_unlock(); 809 810 if (!step) 811 return 0; 812 813 /* 814 * We always disable EL0 watchpoints because the kernel can 815 * cause these to fire via an unprivileged access. 816 */ 817 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0); 818 819 if (user_mode(regs)) { 820 debug_info->wps_disabled = 1; 821 822 /* If we're already stepping a breakpoint, just return. */ 823 if (debug_info->bps_disabled) 824 return 0; 825 826 if (test_thread_flag(TIF_SINGLESTEP)) 827 debug_info->suspended_step = 1; 828 else 829 user_enable_single_step(current); 830 } else { 831 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); 832 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 833 834 if (*kernel_step != ARM_KERNEL_STEP_NONE) 835 return 0; 836 837 if (kernel_active_single_step()) { 838 *kernel_step = ARM_KERNEL_STEP_SUSPEND; 839 } else { 840 *kernel_step = ARM_KERNEL_STEP_ACTIVE; 841 kernel_enable_single_step(regs); 842 } 843 } 844 845 return 0; 846 } 847 NOKPROBE_SYMBOL(watchpoint_handler); 848 849 /* 850 * Handle single-step exception. 851 */ 852 int reinstall_suspended_bps(struct pt_regs *regs) 853 { 854 struct debug_info *debug_info = ¤t->thread.debug; 855 int handled_exception = 0, *kernel_step; 856 857 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 858 859 /* 860 * Called from single-step exception handler. 861 * Return 0 if execution can resume, 1 if a SIGTRAP should be 862 * reported. 863 */ 864 if (user_mode(regs)) { 865 if (debug_info->bps_disabled) { 866 debug_info->bps_disabled = 0; 867 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1); 868 handled_exception = 1; 869 } 870 871 if (debug_info->wps_disabled) { 872 debug_info->wps_disabled = 0; 873 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); 874 handled_exception = 1; 875 } 876 877 if (handled_exception) { 878 if (debug_info->suspended_step) { 879 debug_info->suspended_step = 0; 880 /* Allow exception handling to fall-through. */ 881 handled_exception = 0; 882 } else { 883 user_disable_single_step(current); 884 } 885 } 886 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) { 887 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1); 888 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1); 889 890 if (!debug_info->wps_disabled) 891 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); 892 893 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) { 894 kernel_disable_single_step(); 895 handled_exception = 1; 896 } else { 897 handled_exception = 0; 898 } 899 900 *kernel_step = ARM_KERNEL_STEP_NONE; 901 } 902 903 return !handled_exception; 904 } 905 NOKPROBE_SYMBOL(reinstall_suspended_bps); 906 907 /* 908 * Context-switcher for restoring suspended breakpoints. 909 */ 910 void hw_breakpoint_thread_switch(struct task_struct *next) 911 { 912 /* 913 * current next 914 * disabled: 0 0 => The usual case, NOTIFY_DONE 915 * 0 1 => Disable the registers 916 * 1 0 => Enable the registers 917 * 1 1 => NOTIFY_DONE. per-task bps will 918 * get taken care of by perf. 919 */ 920 921 struct debug_info *current_debug_info, *next_debug_info; 922 923 current_debug_info = ¤t->thread.debug; 924 next_debug_info = &next->thread.debug; 925 926 /* Update breakpoints. */ 927 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled) 928 toggle_bp_registers(AARCH64_DBG_REG_BCR, 929 DBG_ACTIVE_EL0, 930 !next_debug_info->bps_disabled); 931 932 /* Update watchpoints. */ 933 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled) 934 toggle_bp_registers(AARCH64_DBG_REG_WCR, 935 DBG_ACTIVE_EL0, 936 !next_debug_info->wps_disabled); 937 } 938 939 /* 940 * CPU initialisation. 941 */ 942 static int hw_breakpoint_reset(unsigned int cpu) 943 { 944 int i; 945 struct perf_event **slots; 946 /* 947 * When a CPU goes through cold-boot, it does not have any installed 948 * slot, so it is safe to share the same function for restoring and 949 * resetting breakpoints; when a CPU is hotplugged in, it goes 950 * through the slots, which are all empty, hence it just resets control 951 * and value for debug registers. 952 * When this function is triggered on warm-boot through a CPU PM 953 * notifier some slots might be initialized; if so they are 954 * reprogrammed according to the debug slots content. 955 */ 956 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { 957 if (slots[i]) { 958 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); 959 } else { 960 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); 961 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); 962 } 963 } 964 965 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) { 966 if (slots[i]) { 967 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); 968 } else { 969 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); 970 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); 971 } 972 } 973 974 return 0; 975 } 976 977 /* 978 * One-time initialisation. 979 */ 980 static int __init arch_hw_breakpoint_init(void) 981 { 982 int ret; 983 984 core_num_brps = get_num_brps(); 985 core_num_wrps = get_num_wrps(); 986 987 pr_info("found %d breakpoint and %d watchpoint registers.\n", 988 core_num_brps, core_num_wrps); 989 990 /* Register debug fault handlers. */ 991 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, 992 TRAP_HWBKPT, "hw-breakpoint handler"); 993 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP, 994 TRAP_HWBKPT, "hw-watchpoint handler"); 995 996 /* 997 * Reset the breakpoint resources. We assume that a halting 998 * debugger will leave the world in a nice state for us. 999 */ 1000 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING, 1001 "perf/arm64/hw_breakpoint:starting", 1002 hw_breakpoint_reset, NULL); 1003 if (ret) 1004 pr_err("failed to register CPU hotplug notifier: %d\n", ret); 1005 1006 /* Register cpu_suspend hw breakpoint restore hook */ 1007 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset); 1008 1009 return ret; 1010 } 1011 arch_initcall(arch_hw_breakpoint_init); 1012 1013 void hw_breakpoint_pmu_read(struct perf_event *bp) 1014 { 1015 } 1016 1017 /* 1018 * Dummy function to register with die_notifier. 1019 */ 1020 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 1021 unsigned long val, void *data) 1022 { 1023 return NOTIFY_DONE; 1024 } 1025