1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2009, 2010 ARM Limited 5 * 6 * Author: Will Deacon <will.deacon@arm.com> 7 */ 8 9 /* 10 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, 11 * using the CPU's debug registers. 12 */ 13 #define pr_fmt(fmt) "hw-breakpoint: " fmt 14 15 #include <linux/errno.h> 16 #include <linux/hardirq.h> 17 #include <linux/perf_event.h> 18 #include <linux/hw_breakpoint.h> 19 #include <linux/smp.h> 20 #include <linux/cpu_pm.h> 21 #include <linux/coresight.h> 22 23 #include <asm/cacheflush.h> 24 #include <asm/cputype.h> 25 #include <asm/current.h> 26 #include <asm/hw_breakpoint.h> 27 #include <asm/traps.h> 28 29 /* Breakpoint currently in use for each BRP. */ 30 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); 31 32 /* Watchpoint currently in use for each WRP. */ 33 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); 34 35 /* Number of BRP/WRP registers on this CPU. */ 36 static int core_num_brps __ro_after_init; 37 static int core_num_wrps __ro_after_init; 38 39 /* Debug architecture version. */ 40 static u8 debug_arch __ro_after_init; 41 42 /* Does debug architecture support OS Save and Restore? */ 43 static bool has_ossr __ro_after_init; 44 45 /* Maximum supported watchpoint length. */ 46 static u8 max_watchpoint_len __ro_after_init; 47 48 #define READ_WB_REG_CASE(OP2, M, VAL) \ 49 case ((OP2 << 4) + M): \ 50 ARM_DBG_READ(c0, c ## M, OP2, VAL); \ 51 break 52 53 #define WRITE_WB_REG_CASE(OP2, M, VAL) \ 54 case ((OP2 << 4) + M): \ 55 ARM_DBG_WRITE(c0, c ## M, OP2, VAL); \ 56 break 57 58 #define GEN_READ_WB_REG_CASES(OP2, VAL) \ 59 READ_WB_REG_CASE(OP2, 0, VAL); \ 60 READ_WB_REG_CASE(OP2, 1, VAL); \ 61 READ_WB_REG_CASE(OP2, 2, VAL); \ 62 READ_WB_REG_CASE(OP2, 3, VAL); \ 63 READ_WB_REG_CASE(OP2, 4, VAL); \ 64 READ_WB_REG_CASE(OP2, 5, VAL); \ 65 READ_WB_REG_CASE(OP2, 6, VAL); \ 66 READ_WB_REG_CASE(OP2, 7, VAL); \ 67 READ_WB_REG_CASE(OP2, 8, VAL); \ 68 READ_WB_REG_CASE(OP2, 9, VAL); \ 69 READ_WB_REG_CASE(OP2, 10, VAL); \ 70 READ_WB_REG_CASE(OP2, 11, VAL); \ 71 READ_WB_REG_CASE(OP2, 12, VAL); \ 72 READ_WB_REG_CASE(OP2, 13, VAL); \ 73 READ_WB_REG_CASE(OP2, 14, VAL); \ 74 READ_WB_REG_CASE(OP2, 15, VAL) 75 76 #define GEN_WRITE_WB_REG_CASES(OP2, VAL) \ 77 WRITE_WB_REG_CASE(OP2, 0, VAL); \ 78 WRITE_WB_REG_CASE(OP2, 1, VAL); \ 79 WRITE_WB_REG_CASE(OP2, 2, VAL); \ 80 WRITE_WB_REG_CASE(OP2, 3, VAL); \ 81 WRITE_WB_REG_CASE(OP2, 4, VAL); \ 82 WRITE_WB_REG_CASE(OP2, 5, VAL); \ 83 WRITE_WB_REG_CASE(OP2, 6, VAL); \ 84 WRITE_WB_REG_CASE(OP2, 7, VAL); \ 85 WRITE_WB_REG_CASE(OP2, 8, VAL); \ 86 WRITE_WB_REG_CASE(OP2, 9, VAL); \ 87 WRITE_WB_REG_CASE(OP2, 10, VAL); \ 88 WRITE_WB_REG_CASE(OP2, 11, VAL); \ 89 WRITE_WB_REG_CASE(OP2, 12, VAL); \ 90 WRITE_WB_REG_CASE(OP2, 13, VAL); \ 91 WRITE_WB_REG_CASE(OP2, 14, VAL); \ 92 WRITE_WB_REG_CASE(OP2, 15, VAL) 93 94 static u32 read_wb_reg(int n) 95 { 96 u32 val = 0; 97 98 switch (n) { 99 GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val); 100 GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val); 101 GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val); 102 GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val); 103 default: 104 pr_warn("attempt to read from unknown breakpoint register %d\n", 105 n); 106 } 107 108 return val; 109 } 110 111 static void write_wb_reg(int n, u32 val) 112 { 113 switch (n) { 114 GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val); 115 GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val); 116 GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val); 117 GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val); 118 default: 119 pr_warn("attempt to write to unknown breakpoint register %d\n", 120 n); 121 } 122 isb(); 123 } 124 125 /* Determine debug architecture. */ 126 static u8 get_debug_arch(void) 127 { 128 u32 didr; 129 130 /* Do we implement the extended CPUID interface? */ 131 if (((read_cpuid_id() >> 16) & 0xf) != 0xf) { 132 pr_warn_once("CPUID feature registers not supported. " 133 "Assuming v6 debug is present.\n"); 134 return ARM_DEBUG_ARCH_V6; 135 } 136 137 ARM_DBG_READ(c0, c0, 0, didr); 138 return (didr >> 16) & 0xf; 139 } 140 141 u8 arch_get_debug_arch(void) 142 { 143 return debug_arch; 144 } 145 146 static int debug_arch_supported(void) 147 { 148 u8 arch = get_debug_arch(); 149 150 /* We don't support the memory-mapped interface. */ 151 return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) || 152 arch >= ARM_DEBUG_ARCH_V7_1; 153 } 154 155 /* Can we determine the watchpoint access type from the fsr? */ 156 static int debug_exception_updates_fsr(void) 157 { 158 return get_debug_arch() >= ARM_DEBUG_ARCH_V8; 159 } 160 161 /* Determine number of WRP registers available. */ 162 static int get_num_wrp_resources(void) 163 { 164 u32 didr; 165 ARM_DBG_READ(c0, c0, 0, didr); 166 return ((didr >> 28) & 0xf) + 1; 167 } 168 169 /* Determine number of BRP registers available. */ 170 static int get_num_brp_resources(void) 171 { 172 u32 didr; 173 ARM_DBG_READ(c0, c0, 0, didr); 174 return ((didr >> 24) & 0xf) + 1; 175 } 176 177 /* Does this core support mismatch breakpoints? */ 178 static int core_has_mismatch_brps(void) 179 { 180 return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 && 181 get_num_brp_resources() > 1); 182 } 183 184 /* Determine number of usable WRPs available. */ 185 static int get_num_wrps(void) 186 { 187 /* 188 * On debug architectures prior to 7.1, when a watchpoint fires, the 189 * only way to work out which watchpoint it was is by disassembling 190 * the faulting instruction and working out the address of the memory 191 * access. 192 * 193 * Furthermore, we can only do this if the watchpoint was precise 194 * since imprecise watchpoints prevent us from calculating register 195 * based addresses. 196 * 197 * Providing we have more than 1 breakpoint register, we only report 198 * a single watchpoint register for the time being. This way, we always 199 * know which watchpoint fired. In the future we can either add a 200 * disassembler and address generation emulator, or we can insert a 201 * check to see if the DFAR is set on watchpoint exception entry 202 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows 203 * that it is set on some implementations]. 204 */ 205 if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1) 206 return 1; 207 208 return get_num_wrp_resources(); 209 } 210 211 /* Determine number of usable BRPs available. */ 212 static int get_num_brps(void) 213 { 214 int brps = get_num_brp_resources(); 215 return core_has_mismatch_brps() ? brps - 1 : brps; 216 } 217 218 /* 219 * In order to access the breakpoint/watchpoint control registers, 220 * we must be running in debug monitor mode. Unfortunately, we can 221 * be put into halting debug mode at any time by an external debugger 222 * but there is nothing we can do to prevent that. 223 */ 224 static int monitor_mode_enabled(void) 225 { 226 u32 dscr; 227 ARM_DBG_READ(c0, c1, 0, dscr); 228 return !!(dscr & ARM_DSCR_MDBGEN); 229 } 230 231 static int enable_monitor_mode(void) 232 { 233 u32 dscr; 234 ARM_DBG_READ(c0, c1, 0, dscr); 235 236 /* If monitor mode is already enabled, just return. */ 237 if (dscr & ARM_DSCR_MDBGEN) 238 goto out; 239 240 /* Write to the corresponding DSCR. */ 241 switch (get_debug_arch()) { 242 case ARM_DEBUG_ARCH_V6: 243 case ARM_DEBUG_ARCH_V6_1: 244 ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN)); 245 break; 246 case ARM_DEBUG_ARCH_V7_ECP14: 247 case ARM_DEBUG_ARCH_V7_1: 248 case ARM_DEBUG_ARCH_V8: 249 case ARM_DEBUG_ARCH_V8_1: 250 case ARM_DEBUG_ARCH_V8_2: 251 case ARM_DEBUG_ARCH_V8_4: 252 ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN)); 253 isb(); 254 break; 255 default: 256 return -ENODEV; 257 } 258 259 /* Check that the write made it through. */ 260 ARM_DBG_READ(c0, c1, 0, dscr); 261 if (!(dscr & ARM_DSCR_MDBGEN)) { 262 pr_warn_once("Failed to enable monitor mode on CPU %d.\n", 263 smp_processor_id()); 264 return -EPERM; 265 } 266 267 out: 268 return 0; 269 } 270 271 int hw_breakpoint_slots(int type) 272 { 273 if (!debug_arch_supported()) 274 return 0; 275 276 /* 277 * We can be called early, so don't rely on 278 * our static variables being initialised. 279 */ 280 switch (type) { 281 case TYPE_INST: 282 return get_num_brps(); 283 case TYPE_DATA: 284 return get_num_wrps(); 285 default: 286 pr_warn("unknown slot type: %d\n", type); 287 return 0; 288 } 289 } 290 291 /* 292 * Check if 8-bit byte-address select is available. 293 * This clobbers WRP 0. 294 */ 295 static u8 get_max_wp_len(void) 296 { 297 u32 ctrl_reg; 298 struct arch_hw_breakpoint_ctrl ctrl; 299 u8 size = 4; 300 301 if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14) 302 goto out; 303 304 memset(&ctrl, 0, sizeof(ctrl)); 305 ctrl.len = ARM_BREAKPOINT_LEN_8; 306 ctrl_reg = encode_ctrl_reg(ctrl); 307 308 write_wb_reg(ARM_BASE_WVR, 0); 309 write_wb_reg(ARM_BASE_WCR, ctrl_reg); 310 if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg) 311 size = 8; 312 313 out: 314 return size; 315 } 316 317 u8 arch_get_max_wp_len(void) 318 { 319 return max_watchpoint_len; 320 } 321 322 /* 323 * Install a perf counter breakpoint. 324 */ 325 int arch_install_hw_breakpoint(struct perf_event *bp) 326 { 327 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 328 struct perf_event **slot, **slots; 329 int i, max_slots, ctrl_base, val_base; 330 u32 addr, ctrl; 331 332 addr = info->address; 333 ctrl = encode_ctrl_reg(info->ctrl) | 0x1; 334 335 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 336 /* Breakpoint */ 337 ctrl_base = ARM_BASE_BCR; 338 val_base = ARM_BASE_BVR; 339 slots = this_cpu_ptr(bp_on_reg); 340 max_slots = core_num_brps; 341 } else { 342 /* Watchpoint */ 343 ctrl_base = ARM_BASE_WCR; 344 val_base = ARM_BASE_WVR; 345 slots = this_cpu_ptr(wp_on_reg); 346 max_slots = core_num_wrps; 347 } 348 349 for (i = 0; i < max_slots; ++i) { 350 slot = &slots[i]; 351 352 if (!*slot) { 353 *slot = bp; 354 break; 355 } 356 } 357 358 if (i == max_slots) { 359 pr_warn("Can't find any breakpoint slot\n"); 360 return -EBUSY; 361 } 362 363 /* Override the breakpoint data with the step data. */ 364 if (info->step_ctrl.enabled) { 365 addr = info->trigger & ~0x3; 366 ctrl = encode_ctrl_reg(info->step_ctrl); 367 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) { 368 i = 0; 369 ctrl_base = ARM_BASE_BCR + core_num_brps; 370 val_base = ARM_BASE_BVR + core_num_brps; 371 } 372 } 373 374 /* Setup the address register. */ 375 write_wb_reg(val_base + i, addr); 376 377 /* Setup the control register. */ 378 write_wb_reg(ctrl_base + i, ctrl); 379 return 0; 380 } 381 382 void arch_uninstall_hw_breakpoint(struct perf_event *bp) 383 { 384 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 385 struct perf_event **slot, **slots; 386 int i, max_slots, base; 387 388 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 389 /* Breakpoint */ 390 base = ARM_BASE_BCR; 391 slots = this_cpu_ptr(bp_on_reg); 392 max_slots = core_num_brps; 393 } else { 394 /* Watchpoint */ 395 base = ARM_BASE_WCR; 396 slots = this_cpu_ptr(wp_on_reg); 397 max_slots = core_num_wrps; 398 } 399 400 /* Remove the breakpoint. */ 401 for (i = 0; i < max_slots; ++i) { 402 slot = &slots[i]; 403 404 if (*slot == bp) { 405 *slot = NULL; 406 break; 407 } 408 } 409 410 if (i == max_slots) { 411 pr_warn("Can't find any breakpoint slot\n"); 412 return; 413 } 414 415 /* Ensure that we disable the mismatch breakpoint. */ 416 if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE && 417 info->step_ctrl.enabled) { 418 i = 0; 419 base = ARM_BASE_BCR + core_num_brps; 420 } 421 422 /* Reset the control register. */ 423 write_wb_reg(base + i, 0); 424 } 425 426 static int get_hbp_len(u8 hbp_len) 427 { 428 unsigned int len_in_bytes = 0; 429 430 switch (hbp_len) { 431 case ARM_BREAKPOINT_LEN_1: 432 len_in_bytes = 1; 433 break; 434 case ARM_BREAKPOINT_LEN_2: 435 len_in_bytes = 2; 436 break; 437 case ARM_BREAKPOINT_LEN_4: 438 len_in_bytes = 4; 439 break; 440 case ARM_BREAKPOINT_LEN_8: 441 len_in_bytes = 8; 442 break; 443 } 444 445 return len_in_bytes; 446 } 447 448 /* 449 * Check whether bp virtual address is in kernel space. 450 */ 451 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) 452 { 453 unsigned int len; 454 unsigned long va; 455 456 va = hw->address; 457 len = get_hbp_len(hw->ctrl.len); 458 459 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); 460 } 461 462 /* 463 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. 464 * Hopefully this will disappear when ptrace can bypass the conversion 465 * to generic breakpoint descriptions. 466 */ 467 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 468 int *gen_len, int *gen_type) 469 { 470 /* Type */ 471 switch (ctrl.type) { 472 case ARM_BREAKPOINT_EXECUTE: 473 *gen_type = HW_BREAKPOINT_X; 474 break; 475 case ARM_BREAKPOINT_LOAD: 476 *gen_type = HW_BREAKPOINT_R; 477 break; 478 case ARM_BREAKPOINT_STORE: 479 *gen_type = HW_BREAKPOINT_W; 480 break; 481 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: 482 *gen_type = HW_BREAKPOINT_RW; 483 break; 484 default: 485 return -EINVAL; 486 } 487 488 /* Len */ 489 switch (ctrl.len) { 490 case ARM_BREAKPOINT_LEN_1: 491 *gen_len = HW_BREAKPOINT_LEN_1; 492 break; 493 case ARM_BREAKPOINT_LEN_2: 494 *gen_len = HW_BREAKPOINT_LEN_2; 495 break; 496 case ARM_BREAKPOINT_LEN_4: 497 *gen_len = HW_BREAKPOINT_LEN_4; 498 break; 499 case ARM_BREAKPOINT_LEN_8: 500 *gen_len = HW_BREAKPOINT_LEN_8; 501 break; 502 default: 503 return -EINVAL; 504 } 505 506 return 0; 507 } 508 509 /* 510 * Construct an arch_hw_breakpoint from a perf_event. 511 */ 512 static int arch_build_bp_info(struct perf_event *bp, 513 const struct perf_event_attr *attr, 514 struct arch_hw_breakpoint *hw) 515 { 516 /* Type */ 517 switch (attr->bp_type) { 518 case HW_BREAKPOINT_X: 519 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; 520 break; 521 case HW_BREAKPOINT_R: 522 hw->ctrl.type = ARM_BREAKPOINT_LOAD; 523 break; 524 case HW_BREAKPOINT_W: 525 hw->ctrl.type = ARM_BREAKPOINT_STORE; 526 break; 527 case HW_BREAKPOINT_RW: 528 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; 529 break; 530 default: 531 return -EINVAL; 532 } 533 534 /* Len */ 535 switch (attr->bp_len) { 536 case HW_BREAKPOINT_LEN_1: 537 hw->ctrl.len = ARM_BREAKPOINT_LEN_1; 538 break; 539 case HW_BREAKPOINT_LEN_2: 540 hw->ctrl.len = ARM_BREAKPOINT_LEN_2; 541 break; 542 case HW_BREAKPOINT_LEN_4: 543 hw->ctrl.len = ARM_BREAKPOINT_LEN_4; 544 break; 545 case HW_BREAKPOINT_LEN_8: 546 hw->ctrl.len = ARM_BREAKPOINT_LEN_8; 547 if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE) 548 && max_watchpoint_len >= 8) 549 break; 550 /* Else, fall through */ 551 default: 552 return -EINVAL; 553 } 554 555 /* 556 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes. 557 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported 558 * by the hardware and must be aligned to the appropriate number of 559 * bytes. 560 */ 561 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE && 562 hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && 563 hw->ctrl.len != ARM_BREAKPOINT_LEN_4) 564 return -EINVAL; 565 566 /* Address */ 567 hw->address = attr->bp_addr; 568 569 /* Privilege */ 570 hw->ctrl.privilege = ARM_BREAKPOINT_USER; 571 if (arch_check_bp_in_kernelspace(hw)) 572 hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV; 573 574 /* Enabled? */ 575 hw->ctrl.enabled = !attr->disabled; 576 577 /* Mismatch */ 578 hw->ctrl.mismatch = 0; 579 580 return 0; 581 } 582 583 /* 584 * Validate the arch-specific HW Breakpoint register settings. 585 */ 586 int hw_breakpoint_arch_parse(struct perf_event *bp, 587 const struct perf_event_attr *attr, 588 struct arch_hw_breakpoint *hw) 589 { 590 int ret = 0; 591 u32 offset, alignment_mask = 0x3; 592 593 /* Ensure that we are in monitor debug mode. */ 594 if (!monitor_mode_enabled()) 595 return -ENODEV; 596 597 /* Build the arch_hw_breakpoint. */ 598 ret = arch_build_bp_info(bp, attr, hw); 599 if (ret) 600 goto out; 601 602 /* Check address alignment. */ 603 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) 604 alignment_mask = 0x7; 605 offset = hw->address & alignment_mask; 606 switch (offset) { 607 case 0: 608 /* Aligned */ 609 break; 610 case 1: 611 case 2: 612 /* Allow halfword watchpoints and breakpoints. */ 613 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) 614 break; 615 /* Else, fall through */ 616 case 3: 617 /* Allow single byte watchpoint. */ 618 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) 619 break; 620 /* Else, fall through */ 621 default: 622 ret = -EINVAL; 623 goto out; 624 } 625 626 hw->address &= ~alignment_mask; 627 hw->ctrl.len <<= offset; 628 629 if (is_default_overflow_handler(bp)) { 630 /* 631 * Mismatch breakpoints are required for single-stepping 632 * breakpoints. 633 */ 634 if (!core_has_mismatch_brps()) 635 return -EINVAL; 636 637 /* We don't allow mismatch breakpoints in kernel space. */ 638 if (arch_check_bp_in_kernelspace(hw)) 639 return -EPERM; 640 641 /* 642 * Per-cpu breakpoints are not supported by our stepping 643 * mechanism. 644 */ 645 if (!bp->hw.target) 646 return -EINVAL; 647 648 /* 649 * We only support specific access types if the fsr 650 * reports them. 651 */ 652 if (!debug_exception_updates_fsr() && 653 (hw->ctrl.type == ARM_BREAKPOINT_LOAD || 654 hw->ctrl.type == ARM_BREAKPOINT_STORE)) 655 return -EINVAL; 656 } 657 658 out: 659 return ret; 660 } 661 662 /* 663 * Enable/disable single-stepping over the breakpoint bp at address addr. 664 */ 665 static void enable_single_step(struct perf_event *bp, u32 addr) 666 { 667 struct arch_hw_breakpoint *info = counter_arch_bp(bp); 668 669 arch_uninstall_hw_breakpoint(bp); 670 info->step_ctrl.mismatch = 1; 671 info->step_ctrl.len = ARM_BREAKPOINT_LEN_4; 672 info->step_ctrl.type = ARM_BREAKPOINT_EXECUTE; 673 info->step_ctrl.privilege = info->ctrl.privilege; 674 info->step_ctrl.enabled = 1; 675 info->trigger = addr; 676 arch_install_hw_breakpoint(bp); 677 } 678 679 static void disable_single_step(struct perf_event *bp) 680 { 681 arch_uninstall_hw_breakpoint(bp); 682 counter_arch_bp(bp)->step_ctrl.enabled = 0; 683 arch_install_hw_breakpoint(bp); 684 } 685 686 static void watchpoint_handler(unsigned long addr, unsigned int fsr, 687 struct pt_regs *regs) 688 { 689 int i, access; 690 u32 val, ctrl_reg, alignment_mask; 691 struct perf_event *wp, **slots; 692 struct arch_hw_breakpoint *info; 693 struct arch_hw_breakpoint_ctrl ctrl; 694 695 slots = this_cpu_ptr(wp_on_reg); 696 697 for (i = 0; i < core_num_wrps; ++i) { 698 rcu_read_lock(); 699 700 wp = slots[i]; 701 702 if (wp == NULL) 703 goto unlock; 704 705 info = counter_arch_bp(wp); 706 /* 707 * The DFAR is an unknown value on debug architectures prior 708 * to 7.1. Since we only allow a single watchpoint on these 709 * older CPUs, we can set the trigger to the lowest possible 710 * faulting address. 711 */ 712 if (debug_arch < ARM_DEBUG_ARCH_V7_1) { 713 BUG_ON(i > 0); 714 info->trigger = wp->attr.bp_addr; 715 } else { 716 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) 717 alignment_mask = 0x7; 718 else 719 alignment_mask = 0x3; 720 721 /* Check if the watchpoint value matches. */ 722 val = read_wb_reg(ARM_BASE_WVR + i); 723 if (val != (addr & ~alignment_mask)) 724 goto unlock; 725 726 /* Possible match, check the byte address select. */ 727 ctrl_reg = read_wb_reg(ARM_BASE_WCR + i); 728 decode_ctrl_reg(ctrl_reg, &ctrl); 729 if (!((1 << (addr & alignment_mask)) & ctrl.len)) 730 goto unlock; 731 732 /* Check that the access type matches. */ 733 if (debug_exception_updates_fsr()) { 734 access = (fsr & ARM_FSR_ACCESS_MASK) ? 735 HW_BREAKPOINT_W : HW_BREAKPOINT_R; 736 if (!(access & hw_breakpoint_type(wp))) 737 goto unlock; 738 } 739 740 /* We have a winner. */ 741 info->trigger = addr; 742 } 743 744 pr_debug("watchpoint fired: address = 0x%x\n", info->trigger); 745 perf_bp_event(wp, regs); 746 747 /* 748 * If no overflow handler is present, insert a temporary 749 * mismatch breakpoint so we can single-step over the 750 * watchpoint trigger. 751 */ 752 if (is_default_overflow_handler(wp)) 753 enable_single_step(wp, instruction_pointer(regs)); 754 755 unlock: 756 rcu_read_unlock(); 757 } 758 } 759 760 static void watchpoint_single_step_handler(unsigned long pc) 761 { 762 int i; 763 struct perf_event *wp, **slots; 764 struct arch_hw_breakpoint *info; 765 766 slots = this_cpu_ptr(wp_on_reg); 767 768 for (i = 0; i < core_num_wrps; ++i) { 769 rcu_read_lock(); 770 771 wp = slots[i]; 772 773 if (wp == NULL) 774 goto unlock; 775 776 info = counter_arch_bp(wp); 777 if (!info->step_ctrl.enabled) 778 goto unlock; 779 780 /* 781 * Restore the original watchpoint if we've completed the 782 * single-step. 783 */ 784 if (info->trigger != pc) 785 disable_single_step(wp); 786 787 unlock: 788 rcu_read_unlock(); 789 } 790 } 791 792 static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) 793 { 794 int i; 795 u32 ctrl_reg, val, addr; 796 struct perf_event *bp, **slots; 797 struct arch_hw_breakpoint *info; 798 struct arch_hw_breakpoint_ctrl ctrl; 799 800 slots = this_cpu_ptr(bp_on_reg); 801 802 /* The exception entry code places the amended lr in the PC. */ 803 addr = regs->ARM_pc; 804 805 /* Check the currently installed breakpoints first. */ 806 for (i = 0; i < core_num_brps; ++i) { 807 rcu_read_lock(); 808 809 bp = slots[i]; 810 811 if (bp == NULL) 812 goto unlock; 813 814 info = counter_arch_bp(bp); 815 816 /* Check if the breakpoint value matches. */ 817 val = read_wb_reg(ARM_BASE_BVR + i); 818 if (val != (addr & ~0x3)) 819 goto mismatch; 820 821 /* Possible match, check the byte address select to confirm. */ 822 ctrl_reg = read_wb_reg(ARM_BASE_BCR + i); 823 decode_ctrl_reg(ctrl_reg, &ctrl); 824 if ((1 << (addr & 0x3)) & ctrl.len) { 825 info->trigger = addr; 826 pr_debug("breakpoint fired: address = 0x%x\n", addr); 827 perf_bp_event(bp, regs); 828 if (!bp->overflow_handler) 829 enable_single_step(bp, addr); 830 goto unlock; 831 } 832 833 mismatch: 834 /* If we're stepping a breakpoint, it can now be restored. */ 835 if (info->step_ctrl.enabled) 836 disable_single_step(bp); 837 unlock: 838 rcu_read_unlock(); 839 } 840 841 /* Handle any pending watchpoint single-step breakpoints. */ 842 watchpoint_single_step_handler(addr); 843 } 844 845 /* 846 * Called from either the Data Abort Handler [watchpoint] or the 847 * Prefetch Abort Handler [breakpoint] with interrupts disabled. 848 */ 849 static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, 850 struct pt_regs *regs) 851 { 852 int ret = 0; 853 u32 dscr; 854 855 preempt_disable(); 856 857 if (interrupts_enabled(regs)) 858 local_irq_enable(); 859 860 /* We only handle watchpoints and hardware breakpoints. */ 861 ARM_DBG_READ(c0, c1, 0, dscr); 862 863 /* Perform perf callbacks. */ 864 switch (ARM_DSCR_MOE(dscr)) { 865 case ARM_ENTRY_BREAKPOINT: 866 breakpoint_handler(addr, regs); 867 break; 868 case ARM_ENTRY_ASYNC_WATCHPOINT: 869 WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n"); 870 /* Fall through */ 871 case ARM_ENTRY_SYNC_WATCHPOINT: 872 watchpoint_handler(addr, fsr, regs); 873 break; 874 default: 875 ret = 1; /* Unhandled fault. */ 876 } 877 878 preempt_enable(); 879 880 return ret; 881 } 882 883 /* 884 * One-time initialisation. 885 */ 886 static cpumask_t debug_err_mask; 887 888 static int debug_reg_trap(struct pt_regs *regs, unsigned int instr) 889 { 890 int cpu = smp_processor_id(); 891 892 pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n", 893 instr, cpu); 894 895 /* Set the error flag for this CPU and skip the faulting instruction. */ 896 cpumask_set_cpu(cpu, &debug_err_mask); 897 instruction_pointer(regs) += 4; 898 return 0; 899 } 900 901 static struct undef_hook debug_reg_hook = { 902 .instr_mask = 0x0fe80f10, 903 .instr_val = 0x0e000e10, 904 .fn = debug_reg_trap, 905 }; 906 907 /* Does this core support OS Save and Restore? */ 908 static bool core_has_os_save_restore(void) 909 { 910 u32 oslsr; 911 912 switch (get_debug_arch()) { 913 case ARM_DEBUG_ARCH_V7_1: 914 return true; 915 case ARM_DEBUG_ARCH_V7_ECP14: 916 ARM_DBG_READ(c1, c1, 4, oslsr); 917 if (oslsr & ARM_OSLSR_OSLM0) 918 return true; 919 /* Else, fall through */ 920 default: 921 return false; 922 } 923 } 924 925 static void reset_ctrl_regs(unsigned int cpu) 926 { 927 int i, raw_num_brps, err = 0; 928 u32 val; 929 930 /* 931 * v7 debug contains save and restore registers so that debug state 932 * can be maintained across low-power modes without leaving the debug 933 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access 934 * the debug registers out of reset, so we must unlock the OS Lock 935 * Access Register to avoid taking undefined instruction exceptions 936 * later on. 937 */ 938 switch (debug_arch) { 939 case ARM_DEBUG_ARCH_V6: 940 case ARM_DEBUG_ARCH_V6_1: 941 /* ARMv6 cores clear the registers out of reset. */ 942 goto out_mdbgen; 943 case ARM_DEBUG_ARCH_V7_ECP14: 944 /* 945 * Ensure sticky power-down is clear (i.e. debug logic is 946 * powered up). 947 */ 948 ARM_DBG_READ(c1, c5, 4, val); 949 if ((val & 0x1) == 0) 950 err = -EPERM; 951 952 if (!has_ossr) 953 goto clear_vcr; 954 break; 955 case ARM_DEBUG_ARCH_V7_1: 956 /* 957 * Ensure the OS double lock is clear. 958 */ 959 ARM_DBG_READ(c1, c3, 4, val); 960 if ((val & 0x1) == 1) 961 err = -EPERM; 962 break; 963 } 964 965 if (err) { 966 pr_warn_once("CPU %d debug is powered down!\n", cpu); 967 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 968 return; 969 } 970 971 /* 972 * Unconditionally clear the OS lock by writing a value 973 * other than CS_LAR_KEY to the access register. 974 */ 975 ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK); 976 isb(); 977 978 /* 979 * Clear any configured vector-catch events before 980 * enabling monitor mode. 981 */ 982 clear_vcr: 983 ARM_DBG_WRITE(c0, c7, 0, 0); 984 isb(); 985 986 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 987 pr_warn_once("CPU %d failed to disable vector catch\n", cpu); 988 return; 989 } 990 991 /* 992 * The control/value register pairs are UNKNOWN out of reset so 993 * clear them to avoid spurious debug events. 994 */ 995 raw_num_brps = get_num_brp_resources(); 996 for (i = 0; i < raw_num_brps; ++i) { 997 write_wb_reg(ARM_BASE_BCR + i, 0UL); 998 write_wb_reg(ARM_BASE_BVR + i, 0UL); 999 } 1000 1001 for (i = 0; i < core_num_wrps; ++i) { 1002 write_wb_reg(ARM_BASE_WCR + i, 0UL); 1003 write_wb_reg(ARM_BASE_WVR + i, 0UL); 1004 } 1005 1006 if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) { 1007 pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu); 1008 return; 1009 } 1010 1011 /* 1012 * Have a crack at enabling monitor mode. We don't actually need 1013 * it yet, but reporting an error early is useful if it fails. 1014 */ 1015 out_mdbgen: 1016 if (enable_monitor_mode()) 1017 cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); 1018 } 1019 1020 static int dbg_reset_online(unsigned int cpu) 1021 { 1022 local_irq_disable(); 1023 reset_ctrl_regs(cpu); 1024 local_irq_enable(); 1025 return 0; 1026 } 1027 1028 #ifdef CONFIG_CPU_PM 1029 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action, 1030 void *v) 1031 { 1032 if (action == CPU_PM_EXIT) 1033 reset_ctrl_regs(smp_processor_id()); 1034 1035 return NOTIFY_OK; 1036 } 1037 1038 static struct notifier_block dbg_cpu_pm_nb = { 1039 .notifier_call = dbg_cpu_pm_notify, 1040 }; 1041 1042 static void __init pm_init(void) 1043 { 1044 cpu_pm_register_notifier(&dbg_cpu_pm_nb); 1045 } 1046 #else 1047 static inline void pm_init(void) 1048 { 1049 } 1050 #endif 1051 1052 static int __init arch_hw_breakpoint_init(void) 1053 { 1054 int ret; 1055 1056 debug_arch = get_debug_arch(); 1057 1058 if (!debug_arch_supported()) { 1059 pr_info("debug architecture 0x%x unsupported.\n", debug_arch); 1060 return 0; 1061 } 1062 1063 /* 1064 * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD 1065 * whenever a WFI is issued, even if the core is not powered down, in 1066 * violation of the architecture. When DBGPRSR.SPD is set, accesses to 1067 * breakpoint and watchpoint registers are treated as undefined, so 1068 * this results in boot time and runtime failures when these are 1069 * accessed and we unexpectedly take a trap. 1070 * 1071 * It's not clear if/how this can be worked around, so we blacklist 1072 * Scorpion CPUs to avoid these issues. 1073 */ 1074 if (read_cpuid_part() == ARM_CPU_PART_SCORPION) { 1075 pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n"); 1076 return 0; 1077 } 1078 1079 has_ossr = core_has_os_save_restore(); 1080 1081 /* Determine how many BRPs/WRPs are available. */ 1082 core_num_brps = get_num_brps(); 1083 core_num_wrps = get_num_wrps(); 1084 1085 /* 1086 * We need to tread carefully here because DBGSWENABLE may be 1087 * driven low on this core and there isn't an architected way to 1088 * determine that. 1089 */ 1090 cpus_read_lock(); 1091 register_undef_hook(&debug_reg_hook); 1092 1093 /* 1094 * Register CPU notifier which resets the breakpoint resources. We 1095 * assume that a halting debugger will leave the world in a nice state 1096 * for us. 1097 */ 1098 ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN, 1099 "arm/hw_breakpoint:online", 1100 dbg_reset_online, NULL); 1101 unregister_undef_hook(&debug_reg_hook); 1102 if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) { 1103 core_num_brps = 0; 1104 core_num_wrps = 0; 1105 if (ret > 0) 1106 cpuhp_remove_state_nocalls_cpuslocked(ret); 1107 cpus_read_unlock(); 1108 return 0; 1109 } 1110 1111 pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n", 1112 core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " : 1113 "", core_num_wrps); 1114 1115 /* Work out the maximum supported watchpoint length. */ 1116 max_watchpoint_len = get_max_wp_len(); 1117 pr_info("maximum watchpoint size is %u bytes.\n", 1118 max_watchpoint_len); 1119 1120 /* Register debug fault handler. */ 1121 hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1122 TRAP_HWBKPT, "watchpoint debug exception"); 1123 hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, 1124 TRAP_HWBKPT, "breakpoint debug exception"); 1125 cpus_read_unlock(); 1126 1127 /* Register PM notifiers. */ 1128 pm_init(); 1129 return 0; 1130 } 1131 arch_initcall(arch_hw_breakpoint_init); 1132 1133 void hw_breakpoint_pmu_read(struct perf_event *bp) 1134 { 1135 } 1136 1137 /* 1138 * Dummy function to register with die_notifier. 1139 */ 1140 int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 1141 unsigned long val, void *data) 1142 { 1143 return NOTIFY_DONE; 1144 } 1145