1 /* 2 * Performance events x86 architecture header 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 17 #include <asm/fpu/xstate.h> 18 #include <asm/intel_ds.h> 19 #include <asm/cpu.h> 20 #include <asm/msr.h> 21 22 /* To enable MSR tracing please use the generic trace points. */ 23 24 /* 25 * | NHM/WSM | SNB | 26 * register ------------------------------- 27 * | HT | no HT | HT | no HT | 28 *----------------------------------------- 29 * offcore | core | core | cpu | core | 30 * lbr_sel | core | core | cpu | core | 31 * ld_lat | cpu | core | cpu | core | 32 *----------------------------------------- 33 * 34 * Given that there is a small number of shared regs, 35 * we can pre-allocate their slot in the per-cpu 36 * per-core reg tables. 37 */ 38 enum extra_reg_type { 39 EXTRA_REG_NONE = -1, /* not used */ 40 41 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */ 42 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */ 43 EXTRA_REG_LBR = 2, /* lbr_select */ 44 EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */ 45 EXTRA_REG_FE = 4, /* fe_* */ 46 EXTRA_REG_SNOOP_0 = 5, /* snoop response 0 */ 47 EXTRA_REG_SNOOP_1 = 6, /* snoop response 1 */ 48 49 EXTRA_REG_MAX /* number of entries needed */ 50 }; 51 52 struct event_constraint { 53 union { 54 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 55 u64 idxmsk64; 56 }; 57 u64 code; 58 u64 cmask; 59 int weight; 60 int overlap; 61 int flags; 62 unsigned int size; 63 }; 64 65 static inline bool constraint_match(struct event_constraint *c, u64 ecode) 66 { 67 return ((ecode & c->cmask) - c->code) <= (u64)c->size; 68 } 69 70 #define PERF_ARCH(name, val) \ 71 PERF_X86_EVENT_##name = val, 72 73 /* 74 * struct hw_perf_event.flags flags 75 */ 76 enum { 77 #include "perf_event_flags.h" 78 }; 79 80 #undef PERF_ARCH 81 82 #define PERF_ARCH(name, val) \ 83 static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) == \ 84 PERF_X86_EVENT_##name); 85 86 #include "perf_event_flags.h" 87 88 #undef PERF_ARCH 89 90 static inline bool is_topdown_count(struct perf_event *event) 91 { 92 return event->hw.flags & PERF_X86_EVENT_TOPDOWN; 93 } 94 95 static inline bool is_metric_event(struct perf_event *event) 96 { 97 u64 config = event->attr.config; 98 99 return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) && 100 ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING) && 101 ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX); 102 } 103 104 static inline bool is_slots_event(struct perf_event *event) 105 { 106 return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS; 107 } 108 109 static inline bool is_topdown_event(struct perf_event *event) 110 { 111 return is_metric_event(event) || is_slots_event(event); 112 } 113 114 int is_x86_event(struct perf_event *event); 115 116 static inline bool check_leader_group(struct perf_event *leader, int flags) 117 { 118 return is_x86_event(leader) ? !!(leader->hw.flags & flags) : false; 119 } 120 121 static inline bool is_branch_counters_group(struct perf_event *event) 122 { 123 return check_leader_group(event->group_leader, PERF_X86_EVENT_BRANCH_COUNTERS); 124 } 125 126 static inline bool is_pebs_counter_event_group(struct perf_event *event) 127 { 128 return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR); 129 } 130 131 struct amd_nb { 132 int nb_id; /* NorthBridge id */ 133 int refcnt; /* reference count */ 134 struct perf_event *owners[X86_PMC_IDX_MAX]; 135 struct event_constraint event_constraints[X86_PMC_IDX_MAX]; 136 }; 137 138 #define PEBS_COUNTER_MASK ((1ULL << MAX_PEBS_EVENTS) - 1) 139 #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60) 140 #define PEBS_OUTPUT_OFFSET 61 141 #define PEBS_OUTPUT_MASK (3ull << PEBS_OUTPUT_OFFSET) 142 #define PEBS_OUTPUT_PT (1ull << PEBS_OUTPUT_OFFSET) 143 #define PEBS_VIA_PT_MASK (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD) 144 145 /* 146 * Flags PEBS can handle without an PMI. 147 * 148 * TID can only be handled by flushing at context switch. 149 * REGS_USER can be handled for events limited to ring 3. 150 * 151 */ 152 #define LARGE_PEBS_FLAGS \ 153 (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \ 154 PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \ 155 PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \ 156 PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \ 157 PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \ 158 PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \ 159 PERF_SAMPLE_WEIGHT_TYPE) 160 161 #define PEBS_GP_REGS \ 162 ((1ULL << PERF_REG_X86_AX) | \ 163 (1ULL << PERF_REG_X86_BX) | \ 164 (1ULL << PERF_REG_X86_CX) | \ 165 (1ULL << PERF_REG_X86_DX) | \ 166 (1ULL << PERF_REG_X86_DI) | \ 167 (1ULL << PERF_REG_X86_SI) | \ 168 (1ULL << PERF_REG_X86_SP) | \ 169 (1ULL << PERF_REG_X86_BP) | \ 170 (1ULL << PERF_REG_X86_IP) | \ 171 (1ULL << PERF_REG_X86_FLAGS) | \ 172 (1ULL << PERF_REG_X86_R8) | \ 173 (1ULL << PERF_REG_X86_R9) | \ 174 (1ULL << PERF_REG_X86_R10) | \ 175 (1ULL << PERF_REG_X86_R11) | \ 176 (1ULL << PERF_REG_X86_R12) | \ 177 (1ULL << PERF_REG_X86_R13) | \ 178 (1ULL << PERF_REG_X86_R14) | \ 179 (1ULL << PERF_REG_X86_R15)) 180 181 /* 182 * Per register state. 183 */ 184 struct er_account { 185 raw_spinlock_t lock; /* per-core: protect structure */ 186 u64 config; /* extra MSR config */ 187 u64 reg; /* extra MSR number */ 188 atomic_t ref; /* reference count */ 189 }; 190 191 /* 192 * Per core/cpu state 193 * 194 * Used to coordinate shared registers between HT threads or 195 * among events on a single PMU. 196 */ 197 struct intel_shared_regs { 198 struct er_account regs[EXTRA_REG_MAX]; 199 int refcnt; /* per-core: #HT threads */ 200 unsigned core_id; /* per-core: core id */ 201 }; 202 203 enum intel_excl_state_type { 204 INTEL_EXCL_UNUSED = 0, /* counter is unused */ 205 INTEL_EXCL_SHARED = 1, /* counter can be used by both threads */ 206 INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */ 207 }; 208 209 struct intel_excl_states { 210 enum intel_excl_state_type state[X86_PMC_IDX_MAX]; 211 bool sched_started; /* true if scheduling has started */ 212 }; 213 214 struct intel_excl_cntrs { 215 raw_spinlock_t lock; 216 217 struct intel_excl_states states[2]; 218 219 union { 220 u16 has_exclusive[2]; 221 u32 exclusive_present; 222 }; 223 224 int refcnt; /* per-core: #HT threads */ 225 unsigned core_id; /* per-core: core id */ 226 }; 227 228 struct x86_perf_task_context; 229 #define MAX_LBR_ENTRIES 32 230 231 enum { 232 LBR_FORMAT_32 = 0x00, 233 LBR_FORMAT_LIP = 0x01, 234 LBR_FORMAT_EIP = 0x02, 235 LBR_FORMAT_EIP_FLAGS = 0x03, 236 LBR_FORMAT_EIP_FLAGS2 = 0x04, 237 LBR_FORMAT_INFO = 0x05, 238 LBR_FORMAT_TIME = 0x06, 239 LBR_FORMAT_INFO2 = 0x07, 240 LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_INFO2, 241 }; 242 243 enum { 244 X86_PERF_KFREE_SHARED = 0, 245 X86_PERF_KFREE_EXCL = 1, 246 X86_PERF_KFREE_MAX 247 }; 248 249 struct cpu_hw_events { 250 /* 251 * Generic x86 PMC bits 252 */ 253 struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */ 254 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 255 unsigned long dirty[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 256 int enabled; 257 258 int n_events; /* the # of events in the below arrays */ 259 int n_added; /* the # last events in the below arrays; 260 they've never been enabled yet */ 261 int n_txn; /* the # last events in the below arrays; 262 added in the current transaction */ 263 int n_txn_pair; 264 int n_txn_metric; 265 int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */ 266 u64 tags[X86_PMC_IDX_MAX]; 267 268 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */ 269 struct event_constraint *event_constraint[X86_PMC_IDX_MAX]; 270 271 int n_excl; /* the number of exclusive events */ 272 273 unsigned int txn_flags; 274 int is_fake; 275 276 /* 277 * Intel DebugStore bits 278 */ 279 struct debug_store *ds; 280 void *ds_pebs_vaddr; 281 void *ds_bts_vaddr; 282 u64 pebs_enabled; 283 int n_pebs; 284 int n_large_pebs; 285 int n_pebs_via_pt; 286 int pebs_output; 287 288 /* Current super set of events hardware configuration */ 289 u64 pebs_data_cfg; 290 u64 active_pebs_data_cfg; 291 int pebs_record_size; 292 293 /* Intel Fixed counter configuration */ 294 u64 fixed_ctrl_val; 295 u64 active_fixed_ctrl_val; 296 297 /* 298 * Intel LBR bits 299 */ 300 int lbr_users; 301 int lbr_pebs_users; 302 struct perf_branch_stack lbr_stack; 303 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES]; 304 u64 lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */ 305 union { 306 struct er_account *lbr_sel; 307 struct er_account *lbr_ctl; 308 }; 309 u64 br_sel; 310 void *last_task_ctx; 311 int last_log_id; 312 int lbr_select; 313 void *lbr_xsave; 314 315 /* 316 * Intel host/guest exclude bits 317 */ 318 u64 intel_ctrl_guest_mask; 319 u64 intel_ctrl_host_mask; 320 struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX]; 321 322 /* 323 * Intel checkpoint mask 324 */ 325 u64 intel_cp_status; 326 327 /* 328 * manage shared (per-core, per-cpu) registers 329 * used on Intel NHM/WSM/SNB 330 */ 331 struct intel_shared_regs *shared_regs; 332 /* 333 * manage exclusive counter access between hyperthread 334 */ 335 struct event_constraint *constraint_list; /* in enable order */ 336 struct intel_excl_cntrs *excl_cntrs; 337 int excl_thread_id; /* 0 or 1 */ 338 339 /* 340 * SKL TSX_FORCE_ABORT shadow 341 */ 342 u64 tfa_shadow; 343 344 /* 345 * Perf Metrics 346 */ 347 /* number of accepted metrics events */ 348 int n_metric; 349 350 /* 351 * AMD specific bits 352 */ 353 struct amd_nb *amd_nb; 354 int brs_active; /* BRS is enabled */ 355 356 /* Inverted mask of bits to clear in the perf_ctr ctrl registers */ 357 u64 perf_ctr_virt_mask; 358 int n_pair; /* Large increment events */ 359 360 void *kfree_on_online[X86_PERF_KFREE_MAX]; 361 362 struct pmu *pmu; 363 }; 364 365 #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \ 366 { .idxmsk64 = (n) }, \ 367 .code = (c), \ 368 .size = (e) - (c), \ 369 .cmask = (m), \ 370 .weight = (w), \ 371 .overlap = (o), \ 372 .flags = f, \ 373 } 374 375 #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \ 376 __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f) 377 378 #define EVENT_CONSTRAINT(c, n, m) \ 379 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0) 380 381 /* 382 * The constraint_match() function only works for 'simple' event codes 383 * and not for extended (AMD64_EVENTSEL_EVENT) events codes. 384 */ 385 #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \ 386 __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0) 387 388 #define INTEL_EXCLEVT_CONSTRAINT(c, n) \ 389 __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\ 390 0, PERF_X86_EVENT_EXCL) 391 392 /* 393 * The overlap flag marks event constraints with overlapping counter 394 * masks. This is the case if the counter mask of such an event is not 395 * a subset of any other counter mask of a constraint with an equal or 396 * higher weight, e.g.: 397 * 398 * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0); 399 * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0); 400 * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0); 401 * 402 * The event scheduler may not select the correct counter in the first 403 * cycle because it needs to know which subsequent events will be 404 * scheduled. It may fail to schedule the events then. So we set the 405 * overlap flag for such constraints to give the scheduler a hint which 406 * events to select for counter rescheduling. 407 * 408 * Care must be taken as the rescheduling algorithm is O(n!) which 409 * will increase scheduling cycles for an over-committed system 410 * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros 411 * and its counter masks must be kept at a minimum. 412 */ 413 #define EVENT_CONSTRAINT_OVERLAP(c, n, m) \ 414 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0) 415 416 /* 417 * Constraint on the Event code. 418 */ 419 #define INTEL_EVENT_CONSTRAINT(c, n) \ 420 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT) 421 422 /* 423 * Constraint on a range of Event codes 424 */ 425 #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n) \ 426 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT) 427 428 /* 429 * Constraint on the Event code + UMask + fixed-mask 430 * 431 * filter mask to validate fixed counter events. 432 * the following filters disqualify for fixed counters: 433 * - inv 434 * - edge 435 * - cnt-mask 436 * - in_tx 437 * - in_tx_checkpointed 438 * The other filters are supported by fixed counters. 439 * The any-thread option is supported starting with v3. 440 */ 441 #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED) 442 #define FIXED_EVENT_CONSTRAINT(c, n) \ 443 EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS) 444 445 /* 446 * The special metric counters do not actually exist. They are calculated from 447 * the combination of the FxCtr3 + MSR_PERF_METRICS. 448 * 449 * The special metric counters are mapped to a dummy offset for the scheduler. 450 * The sharing between multiple users of the same metric without multiplexing 451 * is not allowed, even though the hardware supports that in principle. 452 */ 453 454 #define METRIC_EVENT_CONSTRAINT(c, n) \ 455 EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)), \ 456 INTEL_ARCH_EVENT_MASK) 457 458 /* 459 * Constraint on the Event code + UMask 460 */ 461 #define INTEL_UEVENT_CONSTRAINT(c, n) \ 462 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK) 463 464 /* Constraint on specific umask bit only + event */ 465 #define INTEL_UBIT_EVENT_CONSTRAINT(c, n) \ 466 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c)) 467 468 /* Like UEVENT_CONSTRAINT, but match flags too */ 469 #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \ 470 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS) 471 472 #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \ 473 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \ 474 HWEIGHT(n), 0, PERF_X86_EVENT_EXCL) 475 476 #define INTEL_PLD_CONSTRAINT(c, n) \ 477 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 478 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT) 479 480 #define INTEL_PSD_CONSTRAINT(c, n) \ 481 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 482 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT) 483 484 #define INTEL_PST_CONSTRAINT(c, n) \ 485 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 486 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST) 487 488 #define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \ 489 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 490 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID) 491 492 #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \ 493 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 494 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW) 495 496 #define INTEL_HYBRID_STLAT_CONSTRAINT(c, n) \ 497 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 498 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_ST_HSW) 499 500 /* Event constraint, but match on all event flags too. */ 501 #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \ 502 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS) 503 504 #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n) \ 505 EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS) 506 507 /* Check only flags, but allow all event/umask */ 508 #define INTEL_ALL_EVENT_CONSTRAINT(code, n) \ 509 EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS) 510 511 /* Check flags and event code, and set the HSW store flag */ 512 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \ 513 __EVENT_CONSTRAINT(code, n, \ 514 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \ 515 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW) 516 517 /* Check flags and event code, and set the HSW load flag */ 518 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \ 519 __EVENT_CONSTRAINT(code, n, \ 520 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \ 521 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW) 522 523 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \ 524 __EVENT_CONSTRAINT_RANGE(code, end, n, \ 525 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \ 526 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW) 527 528 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \ 529 __EVENT_CONSTRAINT(code, n, \ 530 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \ 531 HWEIGHT(n), 0, \ 532 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL) 533 534 /* Check flags and event code/umask, and set the HSW store flag */ 535 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \ 536 __EVENT_CONSTRAINT(code, n, \ 537 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 538 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW) 539 540 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \ 541 __EVENT_CONSTRAINT(code, n, \ 542 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 543 HWEIGHT(n), 0, \ 544 PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL) 545 546 /* Check flags and event code/umask, and set the HSW load flag */ 547 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \ 548 __EVENT_CONSTRAINT(code, n, \ 549 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 550 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW) 551 552 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \ 553 __EVENT_CONSTRAINT(code, n, \ 554 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 555 HWEIGHT(n), 0, \ 556 PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL) 557 558 /* Check flags and event code/umask, and set the HSW N/A flag */ 559 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \ 560 __EVENT_CONSTRAINT(code, n, \ 561 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ 562 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW) 563 564 565 /* 566 * We define the end marker as having a weight of -1 567 * to enable blacklisting of events using a counter bitmask 568 * of zero and thus a weight of zero. 569 * The end marker has a weight that cannot possibly be 570 * obtained from counting the bits in the bitmask. 571 */ 572 #define EVENT_CONSTRAINT_END { .weight = -1 } 573 574 /* 575 * Check for end marker with weight == -1 576 */ 577 #define for_each_event_constraint(e, c) \ 578 for ((e) = (c); (e)->weight != -1; (e)++) 579 580 /* 581 * Extra registers for specific events. 582 * 583 * Some events need large masks and require external MSRs. 584 * Those extra MSRs end up being shared for all events on 585 * a PMU and sometimes between PMU of sibling HT threads. 586 * In either case, the kernel needs to handle conflicting 587 * accesses to those extra, shared, regs. The data structure 588 * to manage those registers is stored in cpu_hw_event. 589 */ 590 struct extra_reg { 591 unsigned int event; 592 unsigned int msr; 593 u64 config_mask; 594 u64 valid_mask; 595 int idx; /* per_xxx->regs[] reg index */ 596 bool extra_msr_access; 597 }; 598 599 #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \ 600 .event = (e), \ 601 .msr = (ms), \ 602 .config_mask = (m), \ 603 .valid_mask = (vm), \ 604 .idx = EXTRA_REG_##i, \ 605 .extra_msr_access = true, \ 606 } 607 608 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \ 609 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx) 610 611 #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \ 612 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \ 613 ARCH_PERFMON_EVENTSEL_UMASK, vm, idx) 614 615 #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \ 616 INTEL_UEVENT_EXTRA_REG(c, \ 617 MSR_PEBS_LD_LAT_THRESHOLD, \ 618 0xffff, \ 619 LDLAT) 620 621 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0) 622 623 union perf_capabilities { 624 struct { 625 u64 lbr_format:6; 626 u64 pebs_trap:1; 627 u64 pebs_arch_reg:1; 628 u64 pebs_format:4; 629 u64 smm_freeze:1; 630 /* 631 * PMU supports separate counter range for writing 632 * values > 32bit. 633 */ 634 u64 full_width_write:1; 635 u64 pebs_baseline:1; 636 u64 perf_metrics:1; 637 u64 pebs_output_pt_available:1; 638 u64 pebs_timing_info:1; 639 u64 anythread_deprecated:1; 640 u64 rdpmc_metrics_clear:1; 641 }; 642 u64 capabilities; 643 }; 644 645 struct x86_pmu_quirk { 646 struct x86_pmu_quirk *next; 647 void (*func)(void); 648 }; 649 650 union x86_pmu_config { 651 struct { 652 u64 event:8, 653 umask:8, 654 usr:1, 655 os:1, 656 edge:1, 657 pc:1, 658 interrupt:1, 659 __reserved1:1, 660 en:1, 661 inv:1, 662 cmask:8, 663 event2:4, 664 __reserved2:4, 665 go:1, 666 ho:1; 667 } bits; 668 u64 value; 669 }; 670 671 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value 672 673 enum { 674 x86_lbr_exclusive_lbr, 675 x86_lbr_exclusive_bts, 676 x86_lbr_exclusive_pt, 677 x86_lbr_exclusive_max, 678 }; 679 680 #define PERF_PEBS_DATA_SOURCE_MAX 0x100 681 #define PERF_PEBS_DATA_SOURCE_MASK (PERF_PEBS_DATA_SOURCE_MAX - 1) 682 #define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10 683 #define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1) 684 685 #define X86_HYBRID_PMU_ATOM_IDX 0 686 #define X86_HYBRID_PMU_CORE_IDX 1 687 #define X86_HYBRID_PMU_TINY_IDX 2 688 689 enum hybrid_pmu_type { 690 not_hybrid, 691 hybrid_small = BIT(X86_HYBRID_PMU_ATOM_IDX), 692 hybrid_big = BIT(X86_HYBRID_PMU_CORE_IDX), 693 hybrid_tiny = BIT(X86_HYBRID_PMU_TINY_IDX), 694 695 /* The belows are only used for matching */ 696 hybrid_big_small = hybrid_big | hybrid_small, 697 hybrid_small_tiny = hybrid_small | hybrid_tiny, 698 hybrid_big_small_tiny = hybrid_big | hybrid_small_tiny, 699 }; 700 701 struct x86_hybrid_pmu { 702 struct pmu pmu; 703 const char *name; 704 enum hybrid_pmu_type pmu_type; 705 cpumask_t supported_cpus; 706 union perf_capabilities intel_cap; 707 u64 intel_ctrl; 708 u64 pebs_events_mask; 709 u64 config_mask; 710 union { 711 u64 cntr_mask64; 712 unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 713 }; 714 union { 715 u64 fixed_cntr_mask64; 716 unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 717 }; 718 struct event_constraint unconstrained; 719 720 u64 hw_cache_event_ids 721 [PERF_COUNT_HW_CACHE_MAX] 722 [PERF_COUNT_HW_CACHE_OP_MAX] 723 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 724 u64 hw_cache_extra_regs 725 [PERF_COUNT_HW_CACHE_MAX] 726 [PERF_COUNT_HW_CACHE_OP_MAX] 727 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 728 struct event_constraint *event_constraints; 729 struct event_constraint *pebs_constraints; 730 struct extra_reg *extra_regs; 731 732 unsigned int late_ack :1, 733 mid_ack :1, 734 enabled_ack :1; 735 736 u64 pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX]; 737 }; 738 739 static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu) 740 { 741 return container_of(pmu, struct x86_hybrid_pmu, pmu); 742 } 743 744 extern struct static_key_false perf_is_hybrid; 745 #define is_hybrid() static_branch_unlikely(&perf_is_hybrid) 746 747 #define hybrid(_pmu, _field) \ 748 (*({ \ 749 typeof(&x86_pmu._field) __Fp = &x86_pmu._field; \ 750 \ 751 if (is_hybrid() && (_pmu)) \ 752 __Fp = &hybrid_pmu(_pmu)->_field; \ 753 \ 754 __Fp; \ 755 })) 756 757 #define hybrid_var(_pmu, _var) \ 758 (*({ \ 759 typeof(&_var) __Fp = &_var; \ 760 \ 761 if (is_hybrid() && (_pmu)) \ 762 __Fp = &hybrid_pmu(_pmu)->_var; \ 763 \ 764 __Fp; \ 765 })) 766 767 #define hybrid_bit(_pmu, _field) \ 768 ({ \ 769 bool __Fp = x86_pmu._field; \ 770 \ 771 if (is_hybrid() && (_pmu)) \ 772 __Fp = hybrid_pmu(_pmu)->_field; \ 773 \ 774 __Fp; \ 775 }) 776 777 /* 778 * struct x86_pmu - generic x86 pmu 779 */ 780 struct x86_pmu { 781 /* 782 * Generic x86 PMC bits 783 */ 784 const char *name; 785 int version; 786 int (*handle_irq)(struct pt_regs *); 787 void (*disable_all)(void); 788 void (*enable_all)(int added); 789 void (*enable)(struct perf_event *); 790 void (*disable)(struct perf_event *); 791 void (*assign)(struct perf_event *event, int idx); 792 void (*add)(struct perf_event *); 793 void (*del)(struct perf_event *); 794 void (*read)(struct perf_event *event); 795 int (*set_period)(struct perf_event *event); 796 u64 (*update)(struct perf_event *event); 797 int (*hw_config)(struct perf_event *event); 798 int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); 799 void (*late_setup)(void); 800 unsigned eventsel; 801 unsigned perfctr; 802 unsigned fixedctr; 803 int (*addr_offset)(int index, bool eventsel); 804 int (*rdpmc_index)(int index); 805 u64 (*event_map)(int); 806 int max_events; 807 u64 config_mask; 808 union { 809 u64 cntr_mask64; 810 unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 811 }; 812 union { 813 u64 fixed_cntr_mask64; 814 unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 815 }; 816 int cntval_bits; 817 u64 cntval_mask; 818 union { 819 unsigned long events_maskl; 820 unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)]; 821 }; 822 int events_mask_len; 823 int apic; 824 u64 max_period; 825 struct event_constraint * 826 (*get_event_constraints)(struct cpu_hw_events *cpuc, 827 int idx, 828 struct perf_event *event); 829 830 void (*put_event_constraints)(struct cpu_hw_events *cpuc, 831 struct perf_event *event); 832 833 void (*start_scheduling)(struct cpu_hw_events *cpuc); 834 835 void (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr); 836 837 void (*stop_scheduling)(struct cpu_hw_events *cpuc); 838 839 struct event_constraint *event_constraints; 840 struct x86_pmu_quirk *quirks; 841 void (*limit_period)(struct perf_event *event, s64 *l); 842 843 /* PMI handler bits */ 844 unsigned int late_ack :1, 845 mid_ack :1, 846 enabled_ack :1; 847 /* 848 * sysfs attrs 849 */ 850 int attr_rdpmc_broken; 851 int attr_rdpmc; 852 struct attribute **format_attrs; 853 854 ssize_t (*events_sysfs_show)(char *page, u64 config); 855 const struct attribute_group **attr_update; 856 857 unsigned long attr_freeze_on_smi; 858 859 /* 860 * CPU Hotplug hooks 861 */ 862 int (*cpu_prepare)(int cpu); 863 void (*cpu_starting)(int cpu); 864 void (*cpu_dying)(int cpu); 865 void (*cpu_dead)(int cpu); 866 867 void (*check_microcode)(void); 868 void (*sched_task)(struct perf_event_pmu_context *pmu_ctx, 869 struct task_struct *task, bool sched_in); 870 871 /* 872 * Intel Arch Perfmon v2+ 873 */ 874 u64 intel_ctrl; 875 union perf_capabilities intel_cap; 876 877 /* 878 * Intel DebugStore bits 879 */ 880 unsigned int bts :1, 881 bts_active :1, 882 pebs :1, 883 pebs_active :1, 884 pebs_broken :1, 885 pebs_prec_dist :1, 886 pebs_no_tlb :1, 887 pebs_no_isolation :1, 888 pebs_block :1, 889 pebs_ept :1; 890 int pebs_record_size; 891 int pebs_buffer_size; 892 u64 pebs_events_mask; 893 void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); 894 struct event_constraint *pebs_constraints; 895 void (*pebs_aliases)(struct perf_event *event); 896 u64 (*pebs_latency_data)(struct perf_event *event, u64 status); 897 unsigned long large_pebs_flags; 898 u64 rtm_abort_event; 899 u64 pebs_capable; 900 901 /* 902 * Intel LBR 903 */ 904 unsigned int lbr_tos, lbr_from, lbr_to, 905 lbr_info, lbr_nr; /* LBR base regs and size */ 906 union { 907 u64 lbr_sel_mask; /* LBR_SELECT valid bits */ 908 u64 lbr_ctl_mask; /* LBR_CTL valid bits */ 909 }; 910 union { 911 const int *lbr_sel_map; /* lbr_select mappings */ 912 int *lbr_ctl_map; /* LBR_CTL mappings */ 913 }; 914 u64 lbr_callstack_users; /* lbr callstack system wide users */ 915 bool lbr_double_abort; /* duplicated lbr aborts */ 916 bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */ 917 918 unsigned int lbr_has_info:1; 919 unsigned int lbr_has_tsx:1; 920 unsigned int lbr_from_flags:1; 921 unsigned int lbr_to_cycles:1; 922 923 /* 924 * Intel Architectural LBR CPUID Enumeration 925 */ 926 unsigned int lbr_depth_mask:8; 927 unsigned int lbr_deep_c_reset:1; 928 unsigned int lbr_lip:1; 929 unsigned int lbr_cpl:1; 930 unsigned int lbr_filter:1; 931 unsigned int lbr_call_stack:1; 932 unsigned int lbr_mispred:1; 933 unsigned int lbr_timed_lbr:1; 934 unsigned int lbr_br_type:1; 935 unsigned int lbr_counters:4; 936 937 void (*lbr_reset)(void); 938 void (*lbr_read)(struct cpu_hw_events *cpuc); 939 void (*lbr_save)(void *ctx); 940 void (*lbr_restore)(void *ctx); 941 942 /* 943 * Intel PT/LBR/BTS are exclusive 944 */ 945 atomic_t lbr_exclusive[x86_lbr_exclusive_max]; 946 947 /* 948 * Intel perf metrics 949 */ 950 int num_topdown_events; 951 952 /* 953 * AMD bits 954 */ 955 unsigned int amd_nb_constraints : 1; 956 u64 perf_ctr_pair_en; 957 958 /* 959 * Extra registers for events 960 */ 961 struct extra_reg *extra_regs; 962 unsigned int flags; 963 964 /* 965 * Intel host/guest support (KVM) 966 */ 967 struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data); 968 969 /* 970 * Check period value for PERF_EVENT_IOC_PERIOD ioctl. 971 */ 972 int (*check_period) (struct perf_event *event, u64 period); 973 974 int (*aux_output_match) (struct perf_event *event); 975 976 void (*filter)(struct pmu *pmu, int cpu, bool *ret); 977 /* 978 * Hybrid support 979 * 980 * Most PMU capabilities are the same among different hybrid PMUs. 981 * The global x86_pmu saves the architecture capabilities, which 982 * are available for all PMUs. The hybrid_pmu only includes the 983 * unique capabilities. 984 */ 985 int num_hybrid_pmus; 986 struct x86_hybrid_pmu *hybrid_pmu; 987 enum intel_cpu_type (*get_hybrid_cpu_type) (void); 988 }; 989 990 struct x86_perf_task_context_opt { 991 int lbr_callstack_users; 992 int lbr_stack_state; 993 int log_id; 994 }; 995 996 struct x86_perf_task_context { 997 u64 lbr_sel; 998 int tos; 999 int valid_lbrs; 1000 struct x86_perf_task_context_opt opt; 1001 struct lbr_entry lbr[MAX_LBR_ENTRIES]; 1002 }; 1003 1004 struct x86_perf_task_context_arch_lbr { 1005 struct x86_perf_task_context_opt opt; 1006 struct lbr_entry entries[]; 1007 }; 1008 1009 /* 1010 * Add padding to guarantee the 64-byte alignment of the state buffer. 1011 * 1012 * The structure is dynamically allocated. The size of the LBR state may vary 1013 * based on the number of LBR registers. 1014 * 1015 * Do not put anything after the LBR state. 1016 */ 1017 struct x86_perf_task_context_arch_lbr_xsave { 1018 struct x86_perf_task_context_opt opt; 1019 1020 union { 1021 struct xregs_state xsave; 1022 struct { 1023 struct fxregs_state i387; 1024 struct xstate_header header; 1025 struct arch_lbr_state lbr; 1026 } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT))); 1027 }; 1028 }; 1029 1030 #define x86_add_quirk(func_) \ 1031 do { \ 1032 static struct x86_pmu_quirk __quirk __initdata = { \ 1033 .func = func_, \ 1034 }; \ 1035 __quirk.next = x86_pmu.quirks; \ 1036 x86_pmu.quirks = &__quirk; \ 1037 } while (0) 1038 1039 /* 1040 * x86_pmu flags 1041 */ 1042 #define PMU_FL_NO_HT_SHARING 0x1 /* no hyper-threading resource sharing */ 1043 #define PMU_FL_HAS_RSP_1 0x2 /* has 2 equivalent offcore_rsp regs */ 1044 #define PMU_FL_EXCL_CNTRS 0x4 /* has exclusive counter requirements */ 1045 #define PMU_FL_EXCL_ENABLED 0x8 /* exclusive counter active */ 1046 #define PMU_FL_PEBS_ALL 0x10 /* all events are valid PEBS events */ 1047 #define PMU_FL_TFA 0x20 /* deal with TSX force abort */ 1048 #define PMU_FL_PAIR 0x40 /* merge counters for large incr. events */ 1049 #define PMU_FL_INSTR_LATENCY 0x80 /* Support Instruction Latency in PEBS Memory Info Record */ 1050 #define PMU_FL_MEM_LOADS_AUX 0x100 /* Require an auxiliary event for the complete memory info */ 1051 #define PMU_FL_RETIRE_LATENCY 0x200 /* Support Retire Latency in PEBS */ 1052 #define PMU_FL_BR_CNTR 0x400 /* Support branch counter logging */ 1053 1054 #define EVENT_VAR(_id) event_attr_##_id 1055 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr 1056 1057 #define EVENT_ATTR(_name, _id) \ 1058 static struct perf_pmu_events_attr EVENT_VAR(_id) = { \ 1059 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \ 1060 .id = PERF_COUNT_HW_##_id, \ 1061 .event_str = NULL, \ 1062 }; 1063 1064 #define EVENT_ATTR_STR(_name, v, str) \ 1065 static struct perf_pmu_events_attr event_attr_##v = { \ 1066 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \ 1067 .id = 0, \ 1068 .event_str = str, \ 1069 }; 1070 1071 #define EVENT_ATTR_STR_HT(_name, v, noht, ht) \ 1072 static struct perf_pmu_events_ht_attr event_attr_##v = { \ 1073 .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\ 1074 .id = 0, \ 1075 .event_str_noht = noht, \ 1076 .event_str_ht = ht, \ 1077 } 1078 1079 #define EVENT_ATTR_STR_HYBRID(_name, v, str, _pmu) \ 1080 static struct perf_pmu_events_hybrid_attr event_attr_##v = { \ 1081 .attr = __ATTR(_name, 0444, events_hybrid_sysfs_show, NULL),\ 1082 .id = 0, \ 1083 .event_str = str, \ 1084 .pmu_type = _pmu, \ 1085 } 1086 1087 #define FORMAT_HYBRID_PTR(_id) (&format_attr_hybrid_##_id.attr.attr) 1088 1089 #define FORMAT_ATTR_HYBRID(_name, _pmu) \ 1090 static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\ 1091 .attr = __ATTR_RO(_name), \ 1092 .pmu_type = _pmu, \ 1093 } 1094 1095 struct pmu *x86_get_pmu(unsigned int cpu); 1096 extern struct x86_pmu x86_pmu __read_mostly; 1097 1098 DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period); 1099 DECLARE_STATIC_CALL(x86_pmu_update, *x86_pmu.update); 1100 DECLARE_STATIC_CALL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); 1101 DECLARE_STATIC_CALL(x86_pmu_late_setup, *x86_pmu.late_setup); 1102 1103 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) 1104 { 1105 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 1106 return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt; 1107 1108 return &((struct x86_perf_task_context *)ctx)->opt; 1109 } 1110 1111 static inline bool x86_pmu_has_lbr_callstack(void) 1112 { 1113 return x86_pmu.lbr_sel_map && 1114 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0; 1115 } 1116 1117 DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 1118 DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1119 1120 int x86_perf_event_set_period(struct perf_event *event); 1121 1122 /* 1123 * Generalized hw caching related hw_event table, filled 1124 * in on a per model basis. A value of 0 means 1125 * 'not supported', -1 means 'hw_event makes no sense on 1126 * this CPU', any other value means the raw hw_event 1127 * ID. 1128 */ 1129 1130 #define C(x) PERF_COUNT_HW_CACHE_##x 1131 1132 extern u64 __read_mostly hw_cache_event_ids 1133 [PERF_COUNT_HW_CACHE_MAX] 1134 [PERF_COUNT_HW_CACHE_OP_MAX] 1135 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 1136 extern u64 __read_mostly hw_cache_extra_regs 1137 [PERF_COUNT_HW_CACHE_MAX] 1138 [PERF_COUNT_HW_CACHE_OP_MAX] 1139 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 1140 1141 u64 x86_perf_event_update(struct perf_event *event); 1142 1143 static inline u64 intel_pmu_topdown_event_update(struct perf_event *event, u64 *val) 1144 { 1145 return x86_perf_event_update(event); 1146 } 1147 DECLARE_STATIC_CALL(intel_pmu_update_topdown_event, intel_pmu_topdown_event_update); 1148 1149 static inline unsigned int x86_pmu_config_addr(int index) 1150 { 1151 return x86_pmu.eventsel + (x86_pmu.addr_offset ? 1152 x86_pmu.addr_offset(index, true) : index); 1153 } 1154 1155 static inline unsigned int x86_pmu_event_addr(int index) 1156 { 1157 return x86_pmu.perfctr + (x86_pmu.addr_offset ? 1158 x86_pmu.addr_offset(index, false) : index); 1159 } 1160 1161 static inline unsigned int x86_pmu_fixed_ctr_addr(int index) 1162 { 1163 return x86_pmu.fixedctr + (x86_pmu.addr_offset ? 1164 x86_pmu.addr_offset(index, false) : index); 1165 } 1166 1167 static inline int x86_pmu_rdpmc_index(int index) 1168 { 1169 return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index; 1170 } 1171 1172 bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, 1173 unsigned long *fixed_cntr_mask); 1174 1175 int x86_add_exclusive(unsigned int what); 1176 1177 void x86_del_exclusive(unsigned int what); 1178 1179 int x86_reserve_hardware(void); 1180 1181 void x86_release_hardware(void); 1182 1183 int x86_pmu_max_precise(void); 1184 1185 void hw_perf_lbr_event_destroy(struct perf_event *event); 1186 1187 int x86_setup_perfctr(struct perf_event *event); 1188 1189 int x86_pmu_hw_config(struct perf_event *event); 1190 1191 void x86_pmu_disable_all(void); 1192 1193 static inline bool has_amd_brs(struct hw_perf_event *hwc) 1194 { 1195 return hwc->flags & PERF_X86_EVENT_AMD_BRS; 1196 } 1197 1198 static inline bool is_counter_pair(struct hw_perf_event *hwc) 1199 { 1200 return hwc->flags & PERF_X86_EVENT_PAIR; 1201 } 1202 1203 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, 1204 u64 enable_mask) 1205 { 1206 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask); 1207 1208 if (hwc->extra_reg.reg) 1209 wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config); 1210 1211 /* 1212 * Add enabled Merge event on next counter 1213 * if large increment event being enabled on this counter 1214 */ 1215 if (is_counter_pair(hwc)) 1216 wrmsrq(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en); 1217 1218 wrmsrq(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask); 1219 } 1220 1221 void x86_pmu_enable_all(int added); 1222 1223 int perf_assign_events(struct event_constraint **constraints, int n, 1224 int wmin, int wmax, int gpmax, int *assign); 1225 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign); 1226 1227 void x86_pmu_stop(struct perf_event *event, int flags); 1228 1229 static inline void x86_pmu_disable_event(struct perf_event *event) 1230 { 1231 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask); 1232 struct hw_perf_event *hwc = &event->hw; 1233 1234 wrmsrq(hwc->config_base, hwc->config & ~disable_mask); 1235 1236 if (is_counter_pair(hwc)) 1237 wrmsrq(x86_pmu_config_addr(hwc->idx + 1), 0); 1238 } 1239 1240 void x86_pmu_enable_event(struct perf_event *event); 1241 1242 int x86_pmu_handle_irq(struct pt_regs *regs); 1243 1244 void x86_pmu_show_pmu_cap(struct pmu *pmu); 1245 1246 static inline int x86_pmu_num_counters(struct pmu *pmu) 1247 { 1248 return hweight64(hybrid(pmu, cntr_mask64)); 1249 } 1250 1251 static inline int x86_pmu_max_num_counters(struct pmu *pmu) 1252 { 1253 return fls64(hybrid(pmu, cntr_mask64)); 1254 } 1255 1256 static inline int x86_pmu_num_counters_fixed(struct pmu *pmu) 1257 { 1258 return hweight64(hybrid(pmu, fixed_cntr_mask64)); 1259 } 1260 1261 static inline int x86_pmu_max_num_counters_fixed(struct pmu *pmu) 1262 { 1263 return fls64(hybrid(pmu, fixed_cntr_mask64)); 1264 } 1265 1266 static inline u64 x86_pmu_get_event_config(struct perf_event *event) 1267 { 1268 return event->attr.config & hybrid(event->pmu, config_mask); 1269 } 1270 1271 extern struct event_constraint emptyconstraint; 1272 1273 extern struct event_constraint unconstrained; 1274 1275 static inline bool kernel_ip(unsigned long ip) 1276 { 1277 #ifdef CONFIG_X86_32 1278 return ip > PAGE_OFFSET; 1279 #else 1280 return (long)ip < 0; 1281 #endif 1282 } 1283 1284 /* 1285 * Not all PMUs provide the right context information to place the reported IP 1286 * into full context. Specifically segment registers are typically not 1287 * supplied. 1288 * 1289 * Assuming the address is a linear address (it is for IBS), we fake the CS and 1290 * vm86 mode using the known zero-based code segment and 'fix up' the registers 1291 * to reflect this. 1292 * 1293 * Intel PEBS/LBR appear to typically provide the effective address, nothing 1294 * much we can do about that but pray and treat it like a linear address. 1295 */ 1296 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip) 1297 { 1298 regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS; 1299 if (regs->flags & X86_VM_MASK) 1300 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK); 1301 regs->ip = ip; 1302 } 1303 1304 /* 1305 * x86control flow change classification 1306 * x86control flow changes include branches, interrupts, traps, faults 1307 */ 1308 enum { 1309 X86_BR_NONE = 0, /* unknown */ 1310 1311 X86_BR_USER = 1 << 0, /* branch target is user */ 1312 X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ 1313 1314 X86_BR_CALL = 1 << 2, /* call */ 1315 X86_BR_RET = 1 << 3, /* return */ 1316 X86_BR_SYSCALL = 1 << 4, /* syscall */ 1317 X86_BR_SYSRET = 1 << 5, /* syscall return */ 1318 X86_BR_INT = 1 << 6, /* sw interrupt */ 1319 X86_BR_IRET = 1 << 7, /* return from interrupt */ 1320 X86_BR_JCC = 1 << 8, /* conditional */ 1321 X86_BR_JMP = 1 << 9, /* jump */ 1322 X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ 1323 X86_BR_IND_CALL = 1 << 11,/* indirect calls */ 1324 X86_BR_ABORT = 1 << 12,/* transaction abort */ 1325 X86_BR_IN_TX = 1 << 13,/* in transaction */ 1326 X86_BR_NO_TX = 1 << 14,/* not in transaction */ 1327 X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ 1328 X86_BR_CALL_STACK = 1 << 16,/* call stack */ 1329 X86_BR_IND_JMP = 1 << 17,/* indirect jump */ 1330 1331 X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */ 1332 1333 }; 1334 1335 #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) 1336 #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) 1337 1338 #define X86_BR_ANY \ 1339 (X86_BR_CALL |\ 1340 X86_BR_RET |\ 1341 X86_BR_SYSCALL |\ 1342 X86_BR_SYSRET |\ 1343 X86_BR_INT |\ 1344 X86_BR_IRET |\ 1345 X86_BR_JCC |\ 1346 X86_BR_JMP |\ 1347 X86_BR_IRQ |\ 1348 X86_BR_ABORT |\ 1349 X86_BR_IND_CALL |\ 1350 X86_BR_IND_JMP |\ 1351 X86_BR_ZERO_CALL) 1352 1353 #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) 1354 1355 #define X86_BR_ANY_CALL \ 1356 (X86_BR_CALL |\ 1357 X86_BR_IND_CALL |\ 1358 X86_BR_ZERO_CALL |\ 1359 X86_BR_SYSCALL |\ 1360 X86_BR_IRQ |\ 1361 X86_BR_INT) 1362 1363 int common_branch_type(int type); 1364 int branch_type(unsigned long from, unsigned long to, int abort); 1365 int branch_type_fused(unsigned long from, unsigned long to, int abort, 1366 int *offset); 1367 1368 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event); 1369 ssize_t intel_event_sysfs_show(char *page, u64 config); 1370 1371 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, 1372 char *page); 1373 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1374 char *page); 1375 ssize_t events_hybrid_sysfs_show(struct device *dev, 1376 struct device_attribute *attr, 1377 char *page); 1378 1379 static inline bool fixed_counter_disabled(int i, struct pmu *pmu) 1380 { 1381 u64 intel_ctrl = hybrid(pmu, intel_ctrl); 1382 1383 return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED)); 1384 } 1385 1386 #ifdef CONFIG_CPU_SUP_AMD 1387 1388 int amd_pmu_init(void); 1389 1390 int amd_pmu_lbr_init(void); 1391 void amd_pmu_lbr_reset(void); 1392 void amd_pmu_lbr_read(void); 1393 void amd_pmu_lbr_add(struct perf_event *event); 1394 void amd_pmu_lbr_del(struct perf_event *event); 1395 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, 1396 struct task_struct *task, bool sched_in); 1397 void amd_pmu_lbr_enable_all(void); 1398 void amd_pmu_lbr_disable_all(void); 1399 int amd_pmu_lbr_hw_config(struct perf_event *event); 1400 1401 static __always_inline void __amd_pmu_lbr_disable(void) 1402 { 1403 u64 dbg_ctl, dbg_extn_cfg; 1404 1405 rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); 1406 wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); 1407 1408 if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { 1409 rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); 1410 wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 1411 } 1412 } 1413 1414 #ifdef CONFIG_PERF_EVENTS_AMD_BRS 1415 1416 #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */ 1417 1418 int amd_brs_init(void); 1419 void amd_brs_disable(void); 1420 void amd_brs_enable(void); 1421 void amd_brs_enable_all(void); 1422 void amd_brs_disable_all(void); 1423 void amd_brs_drain(void); 1424 void amd_brs_lopwr_init(void); 1425 int amd_brs_hw_config(struct perf_event *event); 1426 void amd_brs_reset(void); 1427 1428 static inline void amd_pmu_brs_add(struct perf_event *event) 1429 { 1430 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1431 1432 perf_sched_cb_inc(event->pmu); 1433 cpuc->lbr_users++; 1434 /* 1435 * No need to reset BRS because it is reset 1436 * on brs_enable() and it is saturating 1437 */ 1438 } 1439 1440 static inline void amd_pmu_brs_del(struct perf_event *event) 1441 { 1442 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1443 1444 cpuc->lbr_users--; 1445 WARN_ON_ONCE(cpuc->lbr_users < 0); 1446 1447 perf_sched_cb_dec(event->pmu); 1448 } 1449 1450 void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, 1451 struct task_struct *task, bool sched_in); 1452 #else 1453 static inline int amd_brs_init(void) 1454 { 1455 return 0; 1456 } 1457 static inline void amd_brs_disable(void) {} 1458 static inline void amd_brs_enable(void) {} 1459 static inline void amd_brs_drain(void) {} 1460 static inline void amd_brs_lopwr_init(void) {} 1461 static inline void amd_brs_disable_all(void) {} 1462 static inline int amd_brs_hw_config(struct perf_event *event) 1463 { 1464 return 0; 1465 } 1466 static inline void amd_brs_reset(void) {} 1467 1468 static inline void amd_pmu_brs_add(struct perf_event *event) 1469 { 1470 } 1471 1472 static inline void amd_pmu_brs_del(struct perf_event *event) 1473 { 1474 } 1475 1476 static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, 1477 struct task_struct *task, bool sched_in) 1478 { 1479 } 1480 1481 static inline void amd_brs_enable_all(void) 1482 { 1483 } 1484 1485 #endif 1486 1487 #else /* CONFIG_CPU_SUP_AMD */ 1488 1489 static inline int amd_pmu_init(void) 1490 { 1491 return 0; 1492 } 1493 1494 static inline int amd_brs_init(void) 1495 { 1496 return -EOPNOTSUPP; 1497 } 1498 1499 static inline void amd_brs_drain(void) 1500 { 1501 } 1502 1503 static inline void amd_brs_enable_all(void) 1504 { 1505 } 1506 1507 static inline void amd_brs_disable_all(void) 1508 { 1509 } 1510 #endif /* CONFIG_CPU_SUP_AMD */ 1511 1512 static inline int is_pebs_pt(struct perf_event *event) 1513 { 1514 return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT); 1515 } 1516 1517 #ifdef CONFIG_CPU_SUP_INTEL 1518 1519 static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period) 1520 { 1521 struct hw_perf_event *hwc = &event->hw; 1522 unsigned int hw_event, bts_event; 1523 1524 if (event->attr.freq) 1525 return false; 1526 1527 hw_event = hwc->config & INTEL_ARCH_EVENT_MASK; 1528 bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 1529 1530 return hw_event == bts_event && period == 1; 1531 } 1532 1533 static inline bool intel_pmu_has_bts(struct perf_event *event) 1534 { 1535 struct hw_perf_event *hwc = &event->hw; 1536 1537 return intel_pmu_has_bts_period(event, hwc->sample_period); 1538 } 1539 1540 static __always_inline void __intel_pmu_pebs_disable_all(void) 1541 { 1542 wrmsrq(MSR_IA32_PEBS_ENABLE, 0); 1543 } 1544 1545 static __always_inline void __intel_pmu_arch_lbr_disable(void) 1546 { 1547 wrmsrq(MSR_ARCH_LBR_CTL, 0); 1548 } 1549 1550 static __always_inline void __intel_pmu_lbr_disable(void) 1551 { 1552 u64 debugctl; 1553 1554 rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 1555 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 1556 wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 1557 } 1558 1559 int intel_pmu_save_and_restart(struct perf_event *event); 1560 1561 struct event_constraint * 1562 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx, 1563 struct perf_event *event); 1564 1565 extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu); 1566 extern void intel_cpuc_finish(struct cpu_hw_events *cpuc); 1567 1568 int intel_pmu_init(void); 1569 1570 void init_debug_store_on_cpu(int cpu); 1571 1572 void fini_debug_store_on_cpu(int cpu); 1573 1574 void release_ds_buffers(void); 1575 1576 void reserve_ds_buffers(void); 1577 1578 void release_lbr_buffers(void); 1579 1580 void reserve_lbr_buffers(void); 1581 1582 extern struct event_constraint bts_constraint; 1583 extern struct event_constraint vlbr_constraint; 1584 1585 void intel_pmu_enable_bts(u64 config); 1586 1587 void intel_pmu_disable_bts(void); 1588 1589 int intel_pmu_drain_bts_buffer(void); 1590 1591 u64 grt_latency_data(struct perf_event *event, u64 status); 1592 1593 u64 cmt_latency_data(struct perf_event *event, u64 status); 1594 1595 u64 lnl_latency_data(struct perf_event *event, u64 status); 1596 1597 u64 arl_h_latency_data(struct perf_event *event, u64 status); 1598 1599 extern struct event_constraint intel_core2_pebs_event_constraints[]; 1600 1601 extern struct event_constraint intel_atom_pebs_event_constraints[]; 1602 1603 extern struct event_constraint intel_slm_pebs_event_constraints[]; 1604 1605 extern struct event_constraint intel_glm_pebs_event_constraints[]; 1606 1607 extern struct event_constraint intel_glp_pebs_event_constraints[]; 1608 1609 extern struct event_constraint intel_grt_pebs_event_constraints[]; 1610 1611 extern struct event_constraint intel_nehalem_pebs_event_constraints[]; 1612 1613 extern struct event_constraint intel_westmere_pebs_event_constraints[]; 1614 1615 extern struct event_constraint intel_snb_pebs_event_constraints[]; 1616 1617 extern struct event_constraint intel_ivb_pebs_event_constraints[]; 1618 1619 extern struct event_constraint intel_hsw_pebs_event_constraints[]; 1620 1621 extern struct event_constraint intel_bdw_pebs_event_constraints[]; 1622 1623 extern struct event_constraint intel_skl_pebs_event_constraints[]; 1624 1625 extern struct event_constraint intel_icl_pebs_event_constraints[]; 1626 1627 extern struct event_constraint intel_glc_pebs_event_constraints[]; 1628 1629 extern struct event_constraint intel_lnc_pebs_event_constraints[]; 1630 1631 struct event_constraint *intel_pebs_constraints(struct perf_event *event); 1632 1633 void intel_pmu_pebs_add(struct perf_event *event); 1634 1635 void intel_pmu_pebs_del(struct perf_event *event); 1636 1637 void intel_pmu_pebs_enable(struct perf_event *event); 1638 1639 void intel_pmu_pebs_disable(struct perf_event *event); 1640 1641 void intel_pmu_pebs_enable_all(void); 1642 1643 void intel_pmu_pebs_disable_all(void); 1644 1645 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in); 1646 1647 void intel_pmu_drain_pebs_buffer(void); 1648 1649 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr); 1650 1651 void intel_ds_init(void); 1652 1653 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data, 1654 struct cpu_hw_events *cpuc, 1655 struct perf_event *event); 1656 1657 void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, 1658 struct task_struct *task, bool sched_in); 1659 1660 u64 lbr_from_signext_quirk_wr(u64 val); 1661 1662 void intel_pmu_lbr_reset(void); 1663 1664 void intel_pmu_lbr_reset_32(void); 1665 1666 void intel_pmu_lbr_reset_64(void); 1667 1668 void intel_pmu_lbr_add(struct perf_event *event); 1669 1670 void intel_pmu_lbr_del(struct perf_event *event); 1671 1672 void intel_pmu_lbr_enable_all(bool pmi); 1673 1674 void intel_pmu_lbr_disable_all(void); 1675 1676 void intel_pmu_lbr_read(void); 1677 1678 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc); 1679 1680 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc); 1681 1682 void intel_pmu_lbr_save(void *ctx); 1683 1684 void intel_pmu_lbr_restore(void *ctx); 1685 1686 void intel_pmu_lbr_init_core(void); 1687 1688 void intel_pmu_lbr_init_nhm(void); 1689 1690 void intel_pmu_lbr_init_atom(void); 1691 1692 void intel_pmu_lbr_init_slm(void); 1693 1694 void intel_pmu_lbr_init_snb(void); 1695 1696 void intel_pmu_lbr_init_hsw(void); 1697 1698 void intel_pmu_lbr_init_skl(void); 1699 1700 void intel_pmu_lbr_init_knl(void); 1701 1702 void intel_pmu_lbr_init(void); 1703 1704 void intel_pmu_arch_lbr_init(void); 1705 1706 void intel_pmu_pebs_data_source_nhm(void); 1707 1708 void intel_pmu_pebs_data_source_skl(bool pmem); 1709 1710 void intel_pmu_pebs_data_source_adl(void); 1711 1712 void intel_pmu_pebs_data_source_grt(void); 1713 1714 void intel_pmu_pebs_data_source_mtl(void); 1715 1716 void intel_pmu_pebs_data_source_arl_h(void); 1717 1718 void intel_pmu_pebs_data_source_cmt(void); 1719 1720 void intel_pmu_pebs_data_source_lnl(void); 1721 1722 int intel_pmu_setup_lbr_filter(struct perf_event *event); 1723 1724 void intel_pt_interrupt(void); 1725 1726 int intel_bts_interrupt(void); 1727 1728 void intel_bts_enable_local(void); 1729 1730 void intel_bts_disable_local(void); 1731 1732 int p4_pmu_init(void); 1733 1734 int p6_pmu_init(void); 1735 1736 int knc_pmu_init(void); 1737 1738 static inline int is_ht_workaround_enabled(void) 1739 { 1740 return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED); 1741 } 1742 1743 static inline u64 intel_pmu_pebs_mask(u64 cntr_mask) 1744 { 1745 return MAX_PEBS_EVENTS_MASK & cntr_mask; 1746 } 1747 1748 static inline int intel_pmu_max_num_pebs(struct pmu *pmu) 1749 { 1750 static_assert(MAX_PEBS_EVENTS == 32); 1751 return fls((u32)hybrid(pmu, pebs_events_mask)); 1752 } 1753 1754 #else /* CONFIG_CPU_SUP_INTEL */ 1755 1756 static inline void reserve_ds_buffers(void) 1757 { 1758 } 1759 1760 static inline void release_ds_buffers(void) 1761 { 1762 } 1763 1764 static inline void release_lbr_buffers(void) 1765 { 1766 } 1767 1768 static inline void reserve_lbr_buffers(void) 1769 { 1770 } 1771 1772 static inline int intel_pmu_init(void) 1773 { 1774 return 0; 1775 } 1776 1777 static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) 1778 { 1779 return 0; 1780 } 1781 1782 static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc) 1783 { 1784 } 1785 1786 static inline int is_ht_workaround_enabled(void) 1787 { 1788 return 0; 1789 } 1790 #endif /* CONFIG_CPU_SUP_INTEL */ 1791 1792 #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN)) 1793 int zhaoxin_pmu_init(void); 1794 #else 1795 static inline int zhaoxin_pmu_init(void) 1796 { 1797 return 0; 1798 } 1799 #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/ 1800