1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Branch Record Buffer Extension Driver. 4 * 5 * Copyright (C) 2022-2025 ARM Limited 6 * 7 * Author: Anshuman Khandual <anshuman.khandual@arm.com> 8 */ 9 #include <linux/types.h> 10 #include <linux/bitmap.h> 11 #include <linux/perf/arm_pmu.h> 12 #include "arm_brbe.h" 13 14 #define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \ 15 BRBFCR_EL1_INDIRECT | \ 16 BRBFCR_EL1_RTN | \ 17 BRBFCR_EL1_INDCALL | \ 18 BRBFCR_EL1_DIRCALL | \ 19 BRBFCR_EL1_CONDDIR) 20 21 /* 22 * BRBTS_EL1 is currently not used for branch stack implementation 23 * purpose but BRBCR_ELx.TS needs to have a valid value from all 24 * available options. BRBCR_ELx_TS_VIRTUAL is selected for this. 25 */ 26 #define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL) 27 28 /* 29 * BRBE Buffer Organization 30 * 31 * BRBE buffer is arranged as multiple banks of 32 branch record 32 * entries each. An individual branch record in a given bank could 33 * be accessed, after selecting the bank in BRBFCR_EL1.BANK and 34 * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with 35 * indices [0..31]. 36 * 37 * Bank 0 38 * 39 * --------------------------------- ------ 40 * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 | 41 * --------------------------------- ------ 42 * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 | 43 * --------------------------------- ------ 44 * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 45 * --------------------------------- ------ 46 * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 | 47 * --------------------------------- ------ 48 * 49 * Bank 1 50 * 51 * --------------------------------- ------ 52 * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 | 53 * --------------------------------- ------ 54 * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 | 55 * --------------------------------- ------ 56 * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 57 * --------------------------------- ------ 58 * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 | 59 * --------------------------------- ------ 60 */ 61 #define BRBE_BANK_MAX_ENTRIES 32 62 63 struct brbe_regset { 64 u64 brbsrc; 65 u64 brbtgt; 66 u64 brbinf; 67 }; 68 69 #define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX) 70 71 struct brbe_hw_attr { 72 int brbe_version; 73 int brbe_cc; 74 int brbe_nr; 75 int brbe_format; 76 }; 77 78 #define BRBE_REGN_CASE(n, case_macro) \ 79 case n: case_macro(n); break 80 81 #define BRBE_REGN_SWITCH(x, case_macro) \ 82 do { \ 83 switch (x) { \ 84 BRBE_REGN_CASE(0, case_macro); \ 85 BRBE_REGN_CASE(1, case_macro); \ 86 BRBE_REGN_CASE(2, case_macro); \ 87 BRBE_REGN_CASE(3, case_macro); \ 88 BRBE_REGN_CASE(4, case_macro); \ 89 BRBE_REGN_CASE(5, case_macro); \ 90 BRBE_REGN_CASE(6, case_macro); \ 91 BRBE_REGN_CASE(7, case_macro); \ 92 BRBE_REGN_CASE(8, case_macro); \ 93 BRBE_REGN_CASE(9, case_macro); \ 94 BRBE_REGN_CASE(10, case_macro); \ 95 BRBE_REGN_CASE(11, case_macro); \ 96 BRBE_REGN_CASE(12, case_macro); \ 97 BRBE_REGN_CASE(13, case_macro); \ 98 BRBE_REGN_CASE(14, case_macro); \ 99 BRBE_REGN_CASE(15, case_macro); \ 100 BRBE_REGN_CASE(16, case_macro); \ 101 BRBE_REGN_CASE(17, case_macro); \ 102 BRBE_REGN_CASE(18, case_macro); \ 103 BRBE_REGN_CASE(19, case_macro); \ 104 BRBE_REGN_CASE(20, case_macro); \ 105 BRBE_REGN_CASE(21, case_macro); \ 106 BRBE_REGN_CASE(22, case_macro); \ 107 BRBE_REGN_CASE(23, case_macro); \ 108 BRBE_REGN_CASE(24, case_macro); \ 109 BRBE_REGN_CASE(25, case_macro); \ 110 BRBE_REGN_CASE(26, case_macro); \ 111 BRBE_REGN_CASE(27, case_macro); \ 112 BRBE_REGN_CASE(28, case_macro); \ 113 BRBE_REGN_CASE(29, case_macro); \ 114 BRBE_REGN_CASE(30, case_macro); \ 115 BRBE_REGN_CASE(31, case_macro); \ 116 default: WARN(1, "Invalid BRB* index %d\n", x); \ 117 } \ 118 } while (0) 119 120 #define RETURN_READ_BRBSRCN(n) \ 121 return read_sysreg_s(SYS_BRBSRC_EL1(n)) 122 static inline u64 get_brbsrc_reg(int idx) 123 { 124 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN); 125 return 0; 126 } 127 128 #define RETURN_READ_BRBTGTN(n) \ 129 return read_sysreg_s(SYS_BRBTGT_EL1(n)) 130 static u64 get_brbtgt_reg(int idx) 131 { 132 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN); 133 return 0; 134 } 135 136 #define RETURN_READ_BRBINFN(n) \ 137 return read_sysreg_s(SYS_BRBINF_EL1(n)) 138 static u64 get_brbinf_reg(int idx) 139 { 140 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN); 141 return 0; 142 } 143 144 static u64 brbe_record_valid(u64 brbinf) 145 { 146 return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf); 147 } 148 149 static bool brbe_invalid(u64 brbinf) 150 { 151 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE; 152 } 153 154 static bool brbe_record_is_complete(u64 brbinf) 155 { 156 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL; 157 } 158 159 static bool brbe_record_is_source_only(u64 brbinf) 160 { 161 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE; 162 } 163 164 static bool brbe_record_is_target_only(u64 brbinf) 165 { 166 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET; 167 } 168 169 static int brbinf_get_mispredict(u64 brbinf) 170 { 171 return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf); 172 } 173 174 static u16 brbinf_get_cycles(u64 brbinf) 175 { 176 u32 exp, mant, cycles; 177 /* 178 * Captured cycle count is unknown and hence 179 * should not be passed on to userspace. 180 */ 181 if (brbinf & BRBINFx_EL1_CCU) 182 return 0; 183 184 exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf); 185 mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf); 186 187 if (!exp) 188 return mant; 189 190 cycles = (mant | 0x100) << (exp - 1); 191 192 return min(cycles, U16_MAX); 193 } 194 195 static int brbinf_get_type(u64 brbinf) 196 { 197 return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf); 198 } 199 200 static int brbinf_get_el(u64 brbinf) 201 { 202 return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf); 203 } 204 205 void brbe_invalidate(void) 206 { 207 /* Ensure all branches before this point are recorded */ 208 isb(); 209 asm volatile(BRB_IALL_INSN); 210 /* Ensure all branch records are invalidated after this point */ 211 isb(); 212 } 213 214 static bool valid_brbe_nr(int brbe_nr) 215 { 216 return brbe_nr == BRBIDR0_EL1_NUMREC_8 || 217 brbe_nr == BRBIDR0_EL1_NUMREC_16 || 218 brbe_nr == BRBIDR0_EL1_NUMREC_32 || 219 brbe_nr == BRBIDR0_EL1_NUMREC_64; 220 } 221 222 static bool valid_brbe_cc(int brbe_cc) 223 { 224 return brbe_cc == BRBIDR0_EL1_CC_20_BIT; 225 } 226 227 static bool valid_brbe_format(int brbe_format) 228 { 229 return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0; 230 } 231 232 static bool valid_brbidr(u64 brbidr) 233 { 234 int brbe_format, brbe_cc, brbe_nr; 235 236 brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr); 237 brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr); 238 brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr); 239 240 return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr); 241 } 242 243 static bool valid_brbe_version(int brbe_version) 244 { 245 return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP || 246 brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1; 247 } 248 249 static void select_brbe_bank(int bank) 250 { 251 u64 brbfcr; 252 253 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1); 254 brbfcr &= ~BRBFCR_EL1_BANK_MASK; 255 brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank); 256 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 257 /* 258 * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync 259 * between setting BANK and accessing branch records. 260 */ 261 isb(); 262 } 263 264 static bool __read_brbe_regset(struct brbe_regset *entry, int idx) 265 { 266 entry->brbinf = get_brbinf_reg(idx); 267 268 if (brbe_invalid(entry->brbinf)) 269 return false; 270 271 entry->brbsrc = get_brbsrc_reg(idx); 272 entry->brbtgt = get_brbtgt_reg(idx); 273 return true; 274 } 275 276 /* 277 * Generic perf branch filters supported on BRBE 278 * 279 * New branch filters need to be evaluated whether they could be supported on 280 * BRBE. This ensures that such branch filters would not just be accepted, to 281 * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively 282 * supported only on platforms where kernel is in hyp mode. 283 */ 284 #define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \ 285 PERF_SAMPLE_BRANCH_IN_TX | \ 286 PERF_SAMPLE_BRANCH_NO_TX | \ 287 PERF_SAMPLE_BRANCH_CALL_STACK | \ 288 PERF_SAMPLE_BRANCH_COUNTERS) 289 290 #define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \ 291 PERF_SAMPLE_BRANCH_ANY_CALL | \ 292 PERF_SAMPLE_BRANCH_ANY_RETURN | \ 293 PERF_SAMPLE_BRANCH_IND_CALL | \ 294 PERF_SAMPLE_BRANCH_COND | \ 295 PERF_SAMPLE_BRANCH_IND_JUMP | \ 296 PERF_SAMPLE_BRANCH_CALL) 297 298 299 #define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \ 300 PERF_SAMPLE_BRANCH_KERNEL | \ 301 PERF_SAMPLE_BRANCH_HV | \ 302 BRBE_ALLOWED_BRANCH_TYPES | \ 303 PERF_SAMPLE_BRANCH_NO_FLAGS | \ 304 PERF_SAMPLE_BRANCH_NO_CYCLES | \ 305 PERF_SAMPLE_BRANCH_TYPE_SAVE | \ 306 PERF_SAMPLE_BRANCH_HW_INDEX | \ 307 PERF_SAMPLE_BRANCH_PRIV_SAVE) 308 309 #define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \ 310 BRBE_EXCLUDE_BRANCH_FILTERS) 311 312 /* 313 * BRBE supports the following functional branch type filters while 314 * generating branch records. These branch filters can be enabled, 315 * either individually or as a group i.e ORing multiple filters 316 * with each other. 317 * 318 * BRBFCR_EL1_CONDDIR - Conditional direct branch 319 * BRBFCR_EL1_DIRCALL - Direct call 320 * BRBFCR_EL1_INDCALL - Indirect call 321 * BRBFCR_EL1_INDIRECT - Indirect branch 322 * BRBFCR_EL1_DIRECT - Direct branch 323 * BRBFCR_EL1_RTN - Subroutine return 324 */ 325 static u64 branch_type_to_brbfcr(int branch_type) 326 { 327 u64 brbfcr = 0; 328 329 if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 330 brbfcr |= BRBFCR_EL1_BRANCH_FILTERS; 331 return brbfcr; 332 } 333 334 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) { 335 brbfcr |= BRBFCR_EL1_INDCALL; 336 brbfcr |= BRBFCR_EL1_DIRCALL; 337 } 338 339 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 340 brbfcr |= BRBFCR_EL1_RTN; 341 342 if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL) 343 brbfcr |= BRBFCR_EL1_INDCALL; 344 345 if (branch_type & PERF_SAMPLE_BRANCH_COND) 346 brbfcr |= BRBFCR_EL1_CONDDIR; 347 348 if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP) 349 brbfcr |= BRBFCR_EL1_INDIRECT; 350 351 if (branch_type & PERF_SAMPLE_BRANCH_CALL) 352 brbfcr |= BRBFCR_EL1_DIRCALL; 353 354 return brbfcr; 355 } 356 357 /* 358 * BRBE supports the following privilege mode filters while generating 359 * branch records. 360 * 361 * BRBCR_ELx_E0BRE - EL0 branch records 362 * BRBCR_ELx_ExBRE - EL1/EL2 branch records 363 * 364 * BRBE also supports the following additional functional branch type 365 * filters while generating branch records. 366 * 367 * BRBCR_ELx_EXCEPTION - Exception 368 * BRBCR_ELx_ERTN - Exception return 369 */ 370 static u64 branch_type_to_brbcr(int branch_type) 371 { 372 u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS; 373 374 if (branch_type & PERF_SAMPLE_BRANCH_USER) 375 brbcr |= BRBCR_ELx_E0BRE; 376 377 /* 378 * When running in the hyp mode, writing into BRBCR_EL1 379 * actually writes into BRBCR_EL2 instead. Field E2BRE 380 * is also at the same position as E1BRE. 381 */ 382 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) 383 brbcr |= BRBCR_ELx_ExBRE; 384 385 if (branch_type & PERF_SAMPLE_BRANCH_HV) { 386 if (is_kernel_in_hyp_mode()) 387 brbcr |= BRBCR_ELx_ExBRE; 388 } 389 390 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)) 391 brbcr |= BRBCR_ELx_CC; 392 393 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)) 394 brbcr |= BRBCR_ELx_MPRED; 395 396 /* 397 * The exception and exception return branches could be 398 * captured, irrespective of the perf event's privilege. 399 * If the perf event does not have enough privilege for 400 * a given exception level, then addresses which falls 401 * under that exception level will be reported as zero 402 * for the captured branch record, creating source only 403 * or target only records. 404 */ 405 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) { 406 if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 407 brbcr |= BRBCR_ELx_EXCEPTION; 408 brbcr |= BRBCR_ELx_ERTN; 409 } 410 411 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) 412 brbcr |= BRBCR_ELx_EXCEPTION; 413 414 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 415 brbcr |= BRBCR_ELx_ERTN; 416 } 417 return brbcr; 418 } 419 420 bool brbe_branch_attr_valid(struct perf_event *event) 421 { 422 u64 branch_type = event->attr.branch_sample_type; 423 424 /* 425 * Ensure both perf branch filter allowed and exclude 426 * masks are always in sync with the generic perf ABI. 427 */ 428 BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1)); 429 430 if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) { 431 pr_debug("requested branch filter not supported 0x%llx\n", branch_type); 432 return false; 433 } 434 435 /* Ensure at least 1 branch type is enabled */ 436 if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) { 437 pr_debug("no branch type enabled 0x%llx\n", branch_type); 438 return false; 439 } 440 441 /* 442 * No branches are recorded in guests nor nVHE hypervisors, so 443 * excluding the host or both kernel and user is invalid. 444 * 445 * Ideally we'd just require exclude_guest and exclude_hv, but setting 446 * event filters with perf for kernel or user don't set exclude_guest. 447 * So effectively, exclude_guest and exclude_hv are ignored. 448 */ 449 if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) { 450 pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type); 451 return false; 452 } 453 454 event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type); 455 event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type); 456 457 return true; 458 } 459 460 unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) 461 { 462 return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr); 463 } 464 465 void brbe_probe(struct arm_pmu *armpmu) 466 { 467 u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); 468 u32 brbe; 469 470 brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT); 471 if (!valid_brbe_version(brbe)) 472 return; 473 474 brbidr = read_sysreg_s(SYS_BRBIDR0_EL1); 475 if (!valid_brbidr(brbidr)) 476 return; 477 478 armpmu->reg_brbidr = brbidr; 479 } 480 481 /* 482 * BRBE is assumed to be disabled/paused on entry 483 */ 484 void brbe_enable(const struct arm_pmu *arm_pmu) 485 { 486 struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events); 487 u64 brbfcr = 0, brbcr = 0; 488 489 /* 490 * Discard existing records to avoid a discontinuity, e.g. records 491 * missed during handling an overflow. 492 */ 493 brbe_invalidate(); 494 495 /* 496 * Merge the permitted branch filters of all events. 497 */ 498 for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) { 499 struct perf_event *event = cpuc->events[i]; 500 501 if (event && has_branch_stack(event)) { 502 brbfcr |= event->hw.branch_reg.config; 503 brbcr |= event->hw.extra_reg.config; 504 } 505 } 506 507 /* 508 * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP 509 * controls freezing the branch records on counter overflow rather than 510 * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to). 511 * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1 512 * and EL0 recording disabled for guests. 513 * 514 * As BRBCR_EL1 CC and MPRED bits also need to match, use the same 515 * value for both registers just masking the exception levels. 516 */ 517 if (is_kernel_in_hyp_mode()) 518 write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12); 519 write_sysreg_s(brbcr, SYS_BRBCR_EL1); 520 /* Ensure BRBCR_ELx settings take effect before unpausing */ 521 isb(); 522 523 /* Finally write SYS_BRBFCR_EL to unpause BRBE */ 524 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 525 /* Synchronization in PMCR write ensures ordering WRT PMU enabling */ 526 } 527 528 void brbe_disable(void) 529 { 530 /* 531 * No need for synchronization here as synchronization in PMCR write 532 * ensures ordering and in the interrupt handler this is a NOP as 533 * we're already paused. 534 */ 535 write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1); 536 write_sysreg_s(0, SYS_BRBCR_EL1); 537 } 538 539 static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = { 540 [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 }, 541 [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 }, 542 [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 }, 543 [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 }, 544 [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 }, 545 [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 }, 546 [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 }, 547 [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 }, 548 [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 }, 549 [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 }, 550 [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 }, 551 [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN }, 552 [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST }, 553 [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA }, 554 }; 555 556 static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf) 557 { 558 int brbe_type = brbinf_get_type(brbinf); 559 560 if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) { 561 const int *br_type = brbe_type_to_perf_type_map[brbe_type]; 562 563 entry->type = br_type[0]; 564 entry->new_type = br_type[1]; 565 } 566 } 567 568 static int brbinf_get_perf_priv(u64 brbinf) 569 { 570 int brbe_el = brbinf_get_el(brbinf); 571 572 switch (brbe_el) { 573 case BRBINFx_EL1_EL_EL0: 574 return PERF_BR_PRIV_USER; 575 case BRBINFx_EL1_EL_EL1: 576 return PERF_BR_PRIV_KERNEL; 577 case BRBINFx_EL1_EL_EL2: 578 if (is_kernel_in_hyp_mode()) 579 return PERF_BR_PRIV_KERNEL; 580 return PERF_BR_PRIV_HV; 581 default: 582 pr_warn_once("%d - unknown branch privilege captured\n", brbe_el); 583 return PERF_BR_PRIV_UNKNOWN; 584 } 585 } 586 587 static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry, 588 const struct perf_event *event) 589 { 590 struct brbe_regset bregs; 591 u64 brbinf; 592 593 if (!__read_brbe_regset(&bregs, index)) 594 return false; 595 596 brbinf = bregs.brbinf; 597 *entry = (struct perf_branch_entry){ }; 598 if (brbe_record_is_complete(brbinf)) { 599 entry->from = bregs.brbsrc; 600 entry->to = bregs.brbtgt; 601 } else if (brbe_record_is_source_only(brbinf)) { 602 entry->from = bregs.brbsrc; 603 entry->to = 0; 604 } else if (brbe_record_is_target_only(brbinf)) { 605 entry->from = 0; 606 entry->to = bregs.brbtgt; 607 } 608 609 brbe_set_perf_entry_type(entry, brbinf); 610 611 if (!branch_sample_no_cycles(event)) 612 entry->cycles = brbinf_get_cycles(brbinf); 613 614 if (!branch_sample_no_flags(event)) { 615 /* Mispredict info is available for source only and complete branch records. */ 616 if (!brbe_record_is_target_only(brbinf)) { 617 entry->mispred = brbinf_get_mispredict(brbinf); 618 entry->predicted = !entry->mispred; 619 } 620 } 621 622 /* 623 * Branch privilege level is available for target only and complete 624 * branch records. 625 */ 626 if (!brbe_record_is_source_only(brbinf)) 627 entry->priv = brbinf_get_perf_priv(brbinf); 628 629 return true; 630 } 631 632 #define PERF_BR_ARM64_ALL ( \ 633 BIT(PERF_BR_COND) | \ 634 BIT(PERF_BR_UNCOND) | \ 635 BIT(PERF_BR_IND) | \ 636 BIT(PERF_BR_CALL) | \ 637 BIT(PERF_BR_IND_CALL) | \ 638 BIT(PERF_BR_RET)) 639 640 #define PERF_BR_ARM64_ALL_KERNEL ( \ 641 BIT(PERF_BR_SYSCALL) | \ 642 BIT(PERF_BR_IRQ) | \ 643 BIT(PERF_BR_SERROR) | \ 644 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \ 645 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \ 646 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST)) 647 648 static void prepare_event_branch_type_mask(u64 branch_sample, 649 unsigned long *event_type_mask) 650 { 651 if (branch_sample & PERF_SAMPLE_BRANCH_ANY) { 652 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 653 bitmap_from_u64(event_type_mask, 654 BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL | 655 PERF_BR_ARM64_ALL_KERNEL); 656 else 657 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL); 658 return; 659 } 660 661 bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX); 662 663 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) { 664 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 665 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL); 666 667 set_bit(PERF_BR_CALL, event_type_mask); 668 set_bit(PERF_BR_IND_CALL, event_type_mask); 669 } 670 671 if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP) 672 set_bit(PERF_BR_IND, event_type_mask); 673 674 if (branch_sample & PERF_SAMPLE_BRANCH_COND) 675 set_bit(PERF_BR_COND, event_type_mask); 676 677 if (branch_sample & PERF_SAMPLE_BRANCH_CALL) 678 set_bit(PERF_BR_CALL, event_type_mask); 679 680 if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL) 681 set_bit(PERF_BR_IND_CALL, event_type_mask); 682 683 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) { 684 set_bit(PERF_BR_RET, event_type_mask); 685 686 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 687 set_bit(PERF_BR_ERET, event_type_mask); 688 } 689 } 690 691 /* 692 * BRBE is configured with an OR of permissions from all events, so there may 693 * be events which have to be dropped or events where just the source or target 694 * address has to be zeroed. 695 */ 696 static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type) 697 { 698 bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4); 699 bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4); 700 bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) || 701 (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV))); 702 703 /* We can only have a half record if permissions have not been expanded */ 704 if (!entry->from || !entry->to) 705 return true; 706 707 /* 708 * If record is within a single exception level, just need to either 709 * drop or keep the entire record. 710 */ 711 if (from_user == to_user) 712 return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) || 713 ((entry->priv == PERF_BR_PRIV_USER) && 714 (branch_sample_type & PERF_SAMPLE_BRANCH_USER)); 715 716 /* 717 * Record is across exception levels, mask addresses for the exception 718 * level we're not capturing. 719 */ 720 if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) { 721 if (from_user) 722 entry->from = 0; 723 if (to_user) 724 entry->to = 0; 725 } 726 727 if (exclude_kernel) { 728 if (!from_user) 729 entry->from = 0; 730 if (!to_user) 731 entry->to = 0; 732 } 733 734 return true; 735 } 736 737 static bool filter_branch_type(struct perf_branch_entry *entry, 738 const unsigned long *event_type_mask) 739 { 740 if (entry->type == PERF_BR_EXTEND_ABI) 741 return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask); 742 else 743 return test_bit(entry->type, event_type_mask); 744 } 745 746 static bool filter_branch_record(struct perf_branch_entry *entry, 747 u64 branch_sample, 748 const unsigned long *event_type_mask) 749 { 750 return filter_branch_type(entry, event_type_mask) && 751 filter_branch_privilege(entry, branch_sample); 752 } 753 754 void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, 755 const struct perf_event *event) 756 { 757 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 758 int nr_hw = brbe_num_branch_records(cpu_pmu); 759 int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES); 760 int nr_filtered = 0; 761 u64 branch_sample_type = event->attr.branch_sample_type; 762 DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX); 763 764 prepare_event_branch_type_mask(branch_sample_type, event_type_mask); 765 766 for (int bank = 0; bank < nr_banks; bank++) { 767 int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES); 768 int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES); 769 770 select_brbe_bank(bank); 771 772 for (int i = 0; i < nr_this_bank; i++) { 773 struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered]; 774 775 if (!perf_entry_from_brbe_regset(i, pbe, event)) 776 goto done; 777 778 if (!filter_branch_record(pbe, branch_sample_type, event_type_mask)) 779 continue; 780 781 nr_filtered++; 782 } 783 } 784 785 done: 786 branch_stack->nr = nr_filtered; 787 } 788