127f6d22bSBorislav Petkov /*
227f6d22bSBorislav Petkov * Performance events x86 architecture header
327f6d22bSBorislav Petkov *
427f6d22bSBorislav Petkov * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
527f6d22bSBorislav Petkov * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
627f6d22bSBorislav Petkov * Copyright (C) 2009 Jaswinder Singh Rajput
727f6d22bSBorislav Petkov * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
827f6d22bSBorislav Petkov * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
927f6d22bSBorislav Petkov * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
1027f6d22bSBorislav Petkov * Copyright (C) 2009 Google, Inc., Stephane Eranian
1127f6d22bSBorislav Petkov *
1227f6d22bSBorislav Petkov * For licencing details see kernel-base/COPYING
1327f6d22bSBorislav Petkov */
1427f6d22bSBorislav Petkov
1527f6d22bSBorislav Petkov #include <linux/perf_event.h>
1627f6d22bSBorislav Petkov
17b50854ecSThomas Gleixner #include <asm/fpu/xstate.h>
1810043e02SThomas Gleixner #include <asm/intel_ds.h>
19d9977c43SKan Liang #include <asm/cpu.h>
2010043e02SThomas Gleixner
2127f6d22bSBorislav Petkov /* To enable MSR tracing please use the generic trace points. */
2227f6d22bSBorislav Petkov
2327f6d22bSBorislav Petkov /*
2427f6d22bSBorislav Petkov * | NHM/WSM | SNB |
2527f6d22bSBorislav Petkov * register -------------------------------
2627f6d22bSBorislav Petkov * | HT | no HT | HT | no HT |
2727f6d22bSBorislav Petkov *-----------------------------------------
2827f6d22bSBorislav Petkov * offcore | core | core | cpu | core |
2927f6d22bSBorislav Petkov * lbr_sel | core | core | cpu | core |
3027f6d22bSBorislav Petkov * ld_lat | cpu | core | cpu | core |
3127f6d22bSBorislav Petkov *-----------------------------------------
3227f6d22bSBorislav Petkov *
3327f6d22bSBorislav Petkov * Given that there is a small number of shared regs,
3427f6d22bSBorislav Petkov * we can pre-allocate their slot in the per-cpu
3527f6d22bSBorislav Petkov * per-core reg tables.
3627f6d22bSBorislav Petkov */
3727f6d22bSBorislav Petkov enum extra_reg_type {
3827f6d22bSBorislav Petkov EXTRA_REG_NONE = -1, /* not used */
3927f6d22bSBorislav Petkov
4027f6d22bSBorislav Petkov EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
4127f6d22bSBorislav Petkov EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
4227f6d22bSBorislav Petkov EXTRA_REG_LBR = 2, /* lbr_select */
4327f6d22bSBorislav Petkov EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
4427f6d22bSBorislav Petkov EXTRA_REG_FE = 4, /* fe_* */
4538aaf921SKan Liang EXTRA_REG_SNOOP_0 = 5, /* snoop response 0 */
4638aaf921SKan Liang EXTRA_REG_SNOOP_1 = 6, /* snoop response 1 */
4727f6d22bSBorislav Petkov
4827f6d22bSBorislav Petkov EXTRA_REG_MAX /* number of entries needed */
4927f6d22bSBorislav Petkov };
5027f6d22bSBorislav Petkov
5127f6d22bSBorislav Petkov struct event_constraint {
5227f6d22bSBorislav Petkov union {
5327f6d22bSBorislav Petkov unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
5427f6d22bSBorislav Petkov u64 idxmsk64;
5527f6d22bSBorislav Petkov };
5627f6d22bSBorislav Petkov u64 code;
5727f6d22bSBorislav Petkov u64 cmask;
5827f6d22bSBorislav Petkov int weight;
5927f6d22bSBorislav Petkov int overlap;
6027f6d22bSBorislav Petkov int flags;
6163b79f6eSPeter Zijlstra unsigned int size;
6227f6d22bSBorislav Petkov };
631f6a1e2dSPeter Zijlstra
constraint_match(struct event_constraint * c,u64 ecode)6463b79f6eSPeter Zijlstra static inline bool constraint_match(struct event_constraint *c, u64 ecode)
6563b79f6eSPeter Zijlstra {
6663b79f6eSPeter Zijlstra return ((ecode & c->cmask) - c->code) <= (u64)c->size;
6763b79f6eSPeter Zijlstra }
6863b79f6eSPeter Zijlstra
6988081cfbSAnshuman Khandual #define PERF_ARCH(name, val) \
7088081cfbSAnshuman Khandual PERF_X86_EVENT_##name = val,
7188081cfbSAnshuman Khandual
7227f6d22bSBorislav Petkov /*
7327f6d22bSBorislav Petkov * struct hw_perf_event.flags flags
7427f6d22bSBorislav Petkov */
7588081cfbSAnshuman Khandual enum {
7688081cfbSAnshuman Khandual #include "perf_event_flags.h"
7788081cfbSAnshuman Khandual };
78369461ceSRob Herring
7988081cfbSAnshuman Khandual #undef PERF_ARCH
8088081cfbSAnshuman Khandual
8188081cfbSAnshuman Khandual #define PERF_ARCH(name, val) \
8288081cfbSAnshuman Khandual static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) == \
8388081cfbSAnshuman Khandual PERF_X86_EVENT_##name);
8488081cfbSAnshuman Khandual
8588081cfbSAnshuman Khandual #include "perf_event_flags.h"
8688081cfbSAnshuman Khandual
8788081cfbSAnshuman Khandual #undef PERF_ARCH
887b2c05a1SKan Liang
is_topdown_count(struct perf_event * event)897b2c05a1SKan Liang static inline bool is_topdown_count(struct perf_event *event)
907b2c05a1SKan Liang {
917b2c05a1SKan Liang return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
927b2c05a1SKan Liang }
937b2c05a1SKan Liang
is_metric_event(struct perf_event * event)947b2c05a1SKan Liang static inline bool is_metric_event(struct perf_event *event)
957b2c05a1SKan Liang {
967b2c05a1SKan Liang u64 config = event->attr.config;
977b2c05a1SKan Liang
987b2c05a1SKan Liang return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
997b2c05a1SKan Liang ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING) &&
1007b2c05a1SKan Liang ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
1017b2c05a1SKan Liang }
1027b2c05a1SKan Liang
is_slots_event(struct perf_event * event)1037b2c05a1SKan Liang static inline bool is_slots_event(struct perf_event *event)
1047b2c05a1SKan Liang {
1057b2c05a1SKan Liang return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
1067b2c05a1SKan Liang }
1077b2c05a1SKan Liang
is_topdown_event(struct perf_event * event)1087b2c05a1SKan Liang static inline bool is_topdown_event(struct perf_event *event)
1097b2c05a1SKan Liang {
1107b2c05a1SKan Liang return is_metric_event(event) || is_slots_event(event);
1117b2c05a1SKan Liang }
11227f6d22bSBorislav Petkov
is_branch_counters_group(struct perf_event * event)11333744916SKan Liang static inline bool is_branch_counters_group(struct perf_event *event)
11433744916SKan Liang {
11533744916SKan Liang return event->group_leader->hw.flags & PERF_X86_EVENT_BRANCH_COUNTERS;
11633744916SKan Liang }
11733744916SKan Liang
11827f6d22bSBorislav Petkov struct amd_nb {
11927f6d22bSBorislav Petkov int nb_id; /* NorthBridge id */
12027f6d22bSBorislav Petkov int refcnt; /* reference count */
12127f6d22bSBorislav Petkov struct perf_event *owners[X86_PMC_IDX_MAX];
12227f6d22bSBorislav Petkov struct event_constraint event_constraints[X86_PMC_IDX_MAX];
12327f6d22bSBorislav Petkov };
12427f6d22bSBorislav Petkov
125fd583ad1SKan Liang #define PEBS_COUNTER_MASK ((1ULL << MAX_PEBS_EVENTS) - 1)
12642880f72SAlexander Shishkin #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
12742880f72SAlexander Shishkin #define PEBS_OUTPUT_OFFSET 61
12842880f72SAlexander Shishkin #define PEBS_OUTPUT_MASK (3ull << PEBS_OUTPUT_OFFSET)
12942880f72SAlexander Shishkin #define PEBS_OUTPUT_PT (1ull << PEBS_OUTPUT_OFFSET)
13042880f72SAlexander Shishkin #define PEBS_VIA_PT_MASK (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
13127f6d22bSBorislav Petkov
13227f6d22bSBorislav Petkov /*
13327f6d22bSBorislav Petkov * Flags PEBS can handle without an PMI.
13427f6d22bSBorislav Petkov *
13527f6d22bSBorislav Petkov * TID can only be handled by flushing at context switch.
1362fe1bc1fSAndi Kleen * REGS_USER can be handled for events limited to ring 3.
13727f6d22bSBorislav Petkov *
13827f6d22bSBorislav Petkov */
139174afc3eSKan Liang #define LARGE_PEBS_FLAGS \
14027f6d22bSBorislav Petkov (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
14127f6d22bSBorislav Petkov PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
14227f6d22bSBorislav Petkov PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
1432fe1bc1fSAndi Kleen PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
14411974914SJiri Olsa PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
145e60b7cb0SLike Xu PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \
146e60b7cb0SLike Xu PERF_SAMPLE_WEIGHT_TYPE)
14727f6d22bSBorislav Petkov
1489d5dcc93SKan Liang #define PEBS_GP_REGS \
1499d5dcc93SKan Liang ((1ULL << PERF_REG_X86_AX) | \
1509d5dcc93SKan Liang (1ULL << PERF_REG_X86_BX) | \
1519d5dcc93SKan Liang (1ULL << PERF_REG_X86_CX) | \
1529d5dcc93SKan Liang (1ULL << PERF_REG_X86_DX) | \
1539d5dcc93SKan Liang (1ULL << PERF_REG_X86_DI) | \
1549d5dcc93SKan Liang (1ULL << PERF_REG_X86_SI) | \
1559d5dcc93SKan Liang (1ULL << PERF_REG_X86_SP) | \
1569d5dcc93SKan Liang (1ULL << PERF_REG_X86_BP) | \
1579d5dcc93SKan Liang (1ULL << PERF_REG_X86_IP) | \
1589d5dcc93SKan Liang (1ULL << PERF_REG_X86_FLAGS) | \
1599d5dcc93SKan Liang (1ULL << PERF_REG_X86_R8) | \
1609d5dcc93SKan Liang (1ULL << PERF_REG_X86_R9) | \
1619d5dcc93SKan Liang (1ULL << PERF_REG_X86_R10) | \
1629d5dcc93SKan Liang (1ULL << PERF_REG_X86_R11) | \
1639d5dcc93SKan Liang (1ULL << PERF_REG_X86_R12) | \
1649d5dcc93SKan Liang (1ULL << PERF_REG_X86_R13) | \
1659d5dcc93SKan Liang (1ULL << PERF_REG_X86_R14) | \
1669d5dcc93SKan Liang (1ULL << PERF_REG_X86_R15))
1672fe1bc1fSAndi Kleen
16827f6d22bSBorislav Petkov /*
16927f6d22bSBorislav Petkov * Per register state.
17027f6d22bSBorislav Petkov */
17127f6d22bSBorislav Petkov struct er_account {
17227f6d22bSBorislav Petkov raw_spinlock_t lock; /* per-core: protect structure */
17327f6d22bSBorislav Petkov u64 config; /* extra MSR config */
17427f6d22bSBorislav Petkov u64 reg; /* extra MSR number */
17527f6d22bSBorislav Petkov atomic_t ref; /* reference count */
17627f6d22bSBorislav Petkov };
17727f6d22bSBorislav Petkov
17827f6d22bSBorislav Petkov /*
17927f6d22bSBorislav Petkov * Per core/cpu state
18027f6d22bSBorislav Petkov *
18127f6d22bSBorislav Petkov * Used to coordinate shared registers between HT threads or
18227f6d22bSBorislav Petkov * among events on a single PMU.
18327f6d22bSBorislav Petkov */
18427f6d22bSBorislav Petkov struct intel_shared_regs {
18527f6d22bSBorislav Petkov struct er_account regs[EXTRA_REG_MAX];
18627f6d22bSBorislav Petkov int refcnt; /* per-core: #HT threads */
18727f6d22bSBorislav Petkov unsigned core_id; /* per-core: core id */
18827f6d22bSBorislav Petkov };
18927f6d22bSBorislav Petkov
19027f6d22bSBorislav Petkov enum intel_excl_state_type {
19127f6d22bSBorislav Petkov INTEL_EXCL_UNUSED = 0, /* counter is unused */
19227f6d22bSBorislav Petkov INTEL_EXCL_SHARED = 1, /* counter can be used by both threads */
19327f6d22bSBorislav Petkov INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
19427f6d22bSBorislav Petkov };
19527f6d22bSBorislav Petkov
19627f6d22bSBorislav Petkov struct intel_excl_states {
19727f6d22bSBorislav Petkov enum intel_excl_state_type state[X86_PMC_IDX_MAX];
19827f6d22bSBorislav Petkov bool sched_started; /* true if scheduling has started */
19927f6d22bSBorislav Petkov };
20027f6d22bSBorislav Petkov
20127f6d22bSBorislav Petkov struct intel_excl_cntrs {
20227f6d22bSBorislav Petkov raw_spinlock_t lock;
20327f6d22bSBorislav Petkov
20427f6d22bSBorislav Petkov struct intel_excl_states states[2];
20527f6d22bSBorislav Petkov
20627f6d22bSBorislav Petkov union {
20727f6d22bSBorislav Petkov u16 has_exclusive[2];
20827f6d22bSBorislav Petkov u32 exclusive_present;
20927f6d22bSBorislav Petkov };
21027f6d22bSBorislav Petkov
21127f6d22bSBorislav Petkov int refcnt; /* per-core: #HT threads */
21227f6d22bSBorislav Petkov unsigned core_id; /* per-core: core id */
21327f6d22bSBorislav Petkov };
21427f6d22bSBorislav Petkov
2158b077e4aSKan Liang struct x86_perf_task_context;
21627f6d22bSBorislav Petkov #define MAX_LBR_ENTRIES 32
21727f6d22bSBorislav Petkov
21827f6d22bSBorislav Petkov enum {
2199f354a72SKan Liang LBR_FORMAT_32 = 0x00,
2209f354a72SKan Liang LBR_FORMAT_LIP = 0x01,
2219f354a72SKan Liang LBR_FORMAT_EIP = 0x02,
2229f354a72SKan Liang LBR_FORMAT_EIP_FLAGS = 0x03,
2239f354a72SKan Liang LBR_FORMAT_EIP_FLAGS2 = 0x04,
2249f354a72SKan Liang LBR_FORMAT_INFO = 0x05,
2259f354a72SKan Liang LBR_FORMAT_TIME = 0x06,
2261ac7fd81SPeter Zijlstra (Intel) LBR_FORMAT_INFO2 = 0x07,
2271ac7fd81SPeter Zijlstra (Intel) LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_INFO2,
2289f354a72SKan Liang };
2299f354a72SKan Liang
2309f354a72SKan Liang enum {
23127f6d22bSBorislav Petkov X86_PERF_KFREE_SHARED = 0,
23227f6d22bSBorislav Petkov X86_PERF_KFREE_EXCL = 1,
23327f6d22bSBorislav Petkov X86_PERF_KFREE_MAX
23427f6d22bSBorislav Petkov };
23527f6d22bSBorislav Petkov
23627f6d22bSBorislav Petkov struct cpu_hw_events {
23727f6d22bSBorislav Petkov /*
23827f6d22bSBorislav Petkov * Generic x86 PMC bits
23927f6d22bSBorislav Petkov */
24027f6d22bSBorislav Petkov struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
24127f6d22bSBorislav Petkov unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
2425471eea5SKan Liang unsigned long dirty[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
24327f6d22bSBorislav Petkov int enabled;
24427f6d22bSBorislav Petkov
24527f6d22bSBorislav Petkov int n_events; /* the # of events in the below arrays */
24627f6d22bSBorislav Petkov int n_added; /* the # last events in the below arrays;
24727f6d22bSBorislav Petkov they've never been enabled yet */
24827f6d22bSBorislav Petkov int n_txn; /* the # last events in the below arrays;
24927f6d22bSBorislav Petkov added in the current transaction */
250871a93b0SPeter Zijlstra int n_txn_pair;
2513dbde695SPeter Zijlstra int n_txn_metric;
25227f6d22bSBorislav Petkov int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
25327f6d22bSBorislav Petkov u64 tags[X86_PMC_IDX_MAX];
25427f6d22bSBorislav Petkov
25527f6d22bSBorislav Petkov struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
25627f6d22bSBorislav Petkov struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
25727f6d22bSBorislav Petkov
25827f6d22bSBorislav Petkov int n_excl; /* the number of exclusive events */
25927f6d22bSBorislav Petkov
26027f6d22bSBorislav Petkov unsigned int txn_flags;
26127f6d22bSBorislav Petkov int is_fake;
26227f6d22bSBorislav Petkov
26327f6d22bSBorislav Petkov /*
26427f6d22bSBorislav Petkov * Intel DebugStore bits
26527f6d22bSBorislav Petkov */
26627f6d22bSBorislav Petkov struct debug_store *ds;
267c1961a46SHugh Dickins void *ds_pebs_vaddr;
268c1961a46SHugh Dickins void *ds_bts_vaddr;
26927f6d22bSBorislav Petkov u64 pebs_enabled;
27009e61b4fSPeter Zijlstra int n_pebs;
27109e61b4fSPeter Zijlstra int n_large_pebs;
27242880f72SAlexander Shishkin int n_pebs_via_pt;
27342880f72SAlexander Shishkin int pebs_output;
27427f6d22bSBorislav Petkov
275c22497f5SKan Liang /* Current super set of events hardware configuration */
276c22497f5SKan Liang u64 pebs_data_cfg;
277c22497f5SKan Liang u64 active_pebs_data_cfg;
278c22497f5SKan Liang int pebs_record_size;
279c22497f5SKan Liang
280fae9ebdeSKan Liang /* Intel Fixed counter configuration */
281fae9ebdeSKan Liang u64 fixed_ctrl_val;
282fae9ebdeSKan Liang u64 active_fixed_ctrl_val;
283fae9ebdeSKan Liang
28427f6d22bSBorislav Petkov /*
28527f6d22bSBorislav Petkov * Intel LBR bits
28627f6d22bSBorislav Petkov */
28727f6d22bSBorislav Petkov int lbr_users;
288d3617b98SAndi Kleen int lbr_pebs_users;
28927f6d22bSBorislav Petkov struct perf_branch_stack lbr_stack;
29027f6d22bSBorislav Petkov struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
29133744916SKan Liang u64 lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */
29249d8184fSKan Liang union {
29327f6d22bSBorislav Petkov struct er_account *lbr_sel;
29449d8184fSKan Liang struct er_account *lbr_ctl;
29549d8184fSKan Liang };
29627f6d22bSBorislav Petkov u64 br_sel;
297f42be865SKan Liang void *last_task_ctx;
2988b077e4aSKan Liang int last_log_id;
299e1ad1ac2SLike Xu int lbr_select;
300c085fb87SKan Liang void *lbr_xsave;
30127f6d22bSBorislav Petkov
30227f6d22bSBorislav Petkov /*
30327f6d22bSBorislav Petkov * Intel host/guest exclude bits
30427f6d22bSBorislav Petkov */
30527f6d22bSBorislav Petkov u64 intel_ctrl_guest_mask;
30627f6d22bSBorislav Petkov u64 intel_ctrl_host_mask;
30727f6d22bSBorislav Petkov struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
30827f6d22bSBorislav Petkov
30927f6d22bSBorislav Petkov /*
31027f6d22bSBorislav Petkov * Intel checkpoint mask
31127f6d22bSBorislav Petkov */
31227f6d22bSBorislav Petkov u64 intel_cp_status;
31327f6d22bSBorislav Petkov
31427f6d22bSBorislav Petkov /*
31527f6d22bSBorislav Petkov * manage shared (per-core, per-cpu) registers
31627f6d22bSBorislav Petkov * used on Intel NHM/WSM/SNB
31727f6d22bSBorislav Petkov */
31827f6d22bSBorislav Petkov struct intel_shared_regs *shared_regs;
31927f6d22bSBorislav Petkov /*
32027f6d22bSBorislav Petkov * manage exclusive counter access between hyperthread
32127f6d22bSBorislav Petkov */
32227f6d22bSBorislav Petkov struct event_constraint *constraint_list; /* in enable order */
32327f6d22bSBorislav Petkov struct intel_excl_cntrs *excl_cntrs;
32427f6d22bSBorislav Petkov int excl_thread_id; /* 0 or 1 */
32527f6d22bSBorislav Petkov
32627f6d22bSBorislav Petkov /*
327400816f6SPeter Zijlstra (Intel) * SKL TSX_FORCE_ABORT shadow
328400816f6SPeter Zijlstra (Intel) */
329400816f6SPeter Zijlstra (Intel) u64 tfa_shadow;
330400816f6SPeter Zijlstra (Intel)
331400816f6SPeter Zijlstra (Intel) /*
3327b2c05a1SKan Liang * Perf Metrics
3337b2c05a1SKan Liang */
3347b2c05a1SKan Liang /* number of accepted metrics events */
3357b2c05a1SKan Liang int n_metric;
3367b2c05a1SKan Liang
3377b2c05a1SKan Liang /*
33827f6d22bSBorislav Petkov * AMD specific bits
33927f6d22bSBorislav Petkov */
34027f6d22bSBorislav Petkov struct amd_nb *amd_nb;
341ada54345SStephane Eranian int brs_active; /* BRS is enabled */
342ada54345SStephane Eranian
34327f6d22bSBorislav Petkov /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
34427f6d22bSBorislav Petkov u64 perf_ctr_virt_mask;
34557388912SKim Phillips int n_pair; /* Large increment events */
34627f6d22bSBorislav Petkov
34727f6d22bSBorislav Petkov void *kfree_on_online[X86_PERF_KFREE_MAX];
34861e76d53SKan Liang
34961e76d53SKan Liang struct pmu *pmu;
35027f6d22bSBorislav Petkov };
35127f6d22bSBorislav Petkov
35263b79f6eSPeter Zijlstra #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
35327f6d22bSBorislav Petkov { .idxmsk64 = (n) }, \
35427f6d22bSBorislav Petkov .code = (c), \
35563b79f6eSPeter Zijlstra .size = (e) - (c), \
35627f6d22bSBorislav Petkov .cmask = (m), \
35727f6d22bSBorislav Petkov .weight = (w), \
35827f6d22bSBorislav Petkov .overlap = (o), \
35927f6d22bSBorislav Petkov .flags = f, \
36027f6d22bSBorislav Petkov }
36127f6d22bSBorislav Petkov
36263b79f6eSPeter Zijlstra #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
36363b79f6eSPeter Zijlstra __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
36463b79f6eSPeter Zijlstra
36527f6d22bSBorislav Petkov #define EVENT_CONSTRAINT(c, n, m) \
36627f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
36727f6d22bSBorislav Petkov
36863b79f6eSPeter Zijlstra /*
36963b79f6eSPeter Zijlstra * The constraint_match() function only works for 'simple' event codes
37063b79f6eSPeter Zijlstra * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
37163b79f6eSPeter Zijlstra */
37263b79f6eSPeter Zijlstra #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
37363b79f6eSPeter Zijlstra __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
37463b79f6eSPeter Zijlstra
37527f6d22bSBorislav Petkov #define INTEL_EXCLEVT_CONSTRAINT(c, n) \
37627f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
37727f6d22bSBorislav Petkov 0, PERF_X86_EVENT_EXCL)
37827f6d22bSBorislav Petkov
37927f6d22bSBorislav Petkov /*
38027f6d22bSBorislav Petkov * The overlap flag marks event constraints with overlapping counter
38127f6d22bSBorislav Petkov * masks. This is the case if the counter mask of such an event is not
38227f6d22bSBorislav Petkov * a subset of any other counter mask of a constraint with an equal or
38327f6d22bSBorislav Petkov * higher weight, e.g.:
38427f6d22bSBorislav Petkov *
38527f6d22bSBorislav Petkov * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
38627f6d22bSBorislav Petkov * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
38727f6d22bSBorislav Petkov * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
38827f6d22bSBorislav Petkov *
38927f6d22bSBorislav Petkov * The event scheduler may not select the correct counter in the first
39027f6d22bSBorislav Petkov * cycle because it needs to know which subsequent events will be
39127f6d22bSBorislav Petkov * scheduled. It may fail to schedule the events then. So we set the
39227f6d22bSBorislav Petkov * overlap flag for such constraints to give the scheduler a hint which
39327f6d22bSBorislav Petkov * events to select for counter rescheduling.
39427f6d22bSBorislav Petkov *
39527f6d22bSBorislav Petkov * Care must be taken as the rescheduling algorithm is O(n!) which
39600f52685SIngo Molnar * will increase scheduling cycles for an over-committed system
39727f6d22bSBorislav Petkov * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
39827f6d22bSBorislav Petkov * and its counter masks must be kept at a minimum.
39927f6d22bSBorislav Petkov */
40027f6d22bSBorislav Petkov #define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
40127f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
40227f6d22bSBorislav Petkov
40327f6d22bSBorislav Petkov /*
40427f6d22bSBorislav Petkov * Constraint on the Event code.
40527f6d22bSBorislav Petkov */
40627f6d22bSBorislav Petkov #define INTEL_EVENT_CONSTRAINT(c, n) \
40727f6d22bSBorislav Petkov EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
40827f6d22bSBorislav Petkov
40927f6d22bSBorislav Petkov /*
41063b79f6eSPeter Zijlstra * Constraint on a range of Event codes
41163b79f6eSPeter Zijlstra */
41263b79f6eSPeter Zijlstra #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n) \
41363b79f6eSPeter Zijlstra EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
41463b79f6eSPeter Zijlstra
41563b79f6eSPeter Zijlstra /*
41627f6d22bSBorislav Petkov * Constraint on the Event code + UMask + fixed-mask
41727f6d22bSBorislav Petkov *
41827f6d22bSBorislav Petkov * filter mask to validate fixed counter events.
41927f6d22bSBorislav Petkov * the following filters disqualify for fixed counters:
42027f6d22bSBorislav Petkov * - inv
42127f6d22bSBorislav Petkov * - edge
42227f6d22bSBorislav Petkov * - cnt-mask
42327f6d22bSBorislav Petkov * - in_tx
42427f6d22bSBorislav Petkov * - in_tx_checkpointed
42527f6d22bSBorislav Petkov * The other filters are supported by fixed counters.
42627f6d22bSBorislav Petkov * The any-thread option is supported starting with v3.
42727f6d22bSBorislav Petkov */
42827f6d22bSBorislav Petkov #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
42927f6d22bSBorislav Petkov #define FIXED_EVENT_CONSTRAINT(c, n) \
43027f6d22bSBorislav Petkov EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
43127f6d22bSBorislav Petkov
43227f6d22bSBorislav Petkov /*
43359a854e2SKan Liang * The special metric counters do not actually exist. They are calculated from
43459a854e2SKan Liang * the combination of the FxCtr3 + MSR_PERF_METRICS.
43559a854e2SKan Liang *
43659a854e2SKan Liang * The special metric counters are mapped to a dummy offset for the scheduler.
43759a854e2SKan Liang * The sharing between multiple users of the same metric without multiplexing
43859a854e2SKan Liang * is not allowed, even though the hardware supports that in principle.
43959a854e2SKan Liang */
44059a854e2SKan Liang
44159a854e2SKan Liang #define METRIC_EVENT_CONSTRAINT(c, n) \
44259a854e2SKan Liang EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)), \
44359a854e2SKan Liang INTEL_ARCH_EVENT_MASK)
44459a854e2SKan Liang
44559a854e2SKan Liang /*
44627f6d22bSBorislav Petkov * Constraint on the Event code + UMask
44727f6d22bSBorislav Petkov */
44827f6d22bSBorislav Petkov #define INTEL_UEVENT_CONSTRAINT(c, n) \
44927f6d22bSBorislav Petkov EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
45027f6d22bSBorislav Petkov
45127f6d22bSBorislav Petkov /* Constraint on specific umask bit only + event */
45227f6d22bSBorislav Petkov #define INTEL_UBIT_EVENT_CONSTRAINT(c, n) \
45327f6d22bSBorislav Petkov EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
45427f6d22bSBorislav Petkov
45527f6d22bSBorislav Petkov /* Like UEVENT_CONSTRAINT, but match flags too */
45627f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \
45727f6d22bSBorislav Petkov EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
45827f6d22bSBorislav Petkov
45927f6d22bSBorislav Petkov #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
46027f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
46127f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
46227f6d22bSBorislav Petkov
46327f6d22bSBorislav Petkov #define INTEL_PLD_CONSTRAINT(c, n) \
46427f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
46527f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
46627f6d22bSBorislav Petkov
46761b985e3SKan Liang #define INTEL_PSD_CONSTRAINT(c, n) \
46861b985e3SKan Liang __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
46961b985e3SKan Liang HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
47061b985e3SKan Liang
47127f6d22bSBorislav Petkov #define INTEL_PST_CONSTRAINT(c, n) \
47227f6d22bSBorislav Petkov __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
47327f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
47427f6d22bSBorislav Petkov
47539a41278SKan Liang #define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \
47639a41278SKan Liang __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
47739a41278SKan Liang HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID)
47839a41278SKan Liang
479608f6976SKan Liang #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \
480608f6976SKan Liang __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
481608f6976SKan Liang HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW)
482608f6976SKan Liang
483608f6976SKan Liang #define INTEL_HYBRID_STLAT_CONSTRAINT(c, n) \
484608f6976SKan Liang __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
485608f6976SKan Liang HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_ST_HSW)
486608f6976SKan Liang
48727f6d22bSBorislav Petkov /* Event constraint, but match on all event flags too. */
48827f6d22bSBorislav Petkov #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
4896b89d4c1SStephane Eranian EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
49027f6d22bSBorislav Petkov
49163b79f6eSPeter Zijlstra #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n) \
4926b89d4c1SStephane Eranian EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
49363b79f6eSPeter Zijlstra
49427f6d22bSBorislav Petkov /* Check only flags, but allow all event/umask */
49527f6d22bSBorislav Petkov #define INTEL_ALL_EVENT_CONSTRAINT(code, n) \
49627f6d22bSBorislav Petkov EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
49727f6d22bSBorislav Petkov
49827f6d22bSBorislav Petkov /* Check flags and event code, and set the HSW store flag */
49927f6d22bSBorislav Petkov #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
50027f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
50127f6d22bSBorislav Petkov ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
50227f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
50327f6d22bSBorislav Petkov
50427f6d22bSBorislav Petkov /* Check flags and event code, and set the HSW load flag */
50527f6d22bSBorislav Petkov #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
50627f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
50727f6d22bSBorislav Petkov ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
50827f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
50927f6d22bSBorislav Petkov
51063b79f6eSPeter Zijlstra #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
51163b79f6eSPeter Zijlstra __EVENT_CONSTRAINT_RANGE(code, end, n, \
51263b79f6eSPeter Zijlstra ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
51363b79f6eSPeter Zijlstra HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
51463b79f6eSPeter Zijlstra
51527f6d22bSBorislav Petkov #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
51627f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
51727f6d22bSBorislav Petkov ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
51827f6d22bSBorislav Petkov HWEIGHT(n), 0, \
51927f6d22bSBorislav Petkov PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
52027f6d22bSBorislav Petkov
52127f6d22bSBorislav Petkov /* Check flags and event code/umask, and set the HSW store flag */
52227f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
52327f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
52427f6d22bSBorislav Petkov INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
52527f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
52627f6d22bSBorislav Petkov
52727f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
52827f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
52927f6d22bSBorislav Petkov INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
53027f6d22bSBorislav Petkov HWEIGHT(n), 0, \
53127f6d22bSBorislav Petkov PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
53227f6d22bSBorislav Petkov
53327f6d22bSBorislav Petkov /* Check flags and event code/umask, and set the HSW load flag */
53427f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
53527f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
53627f6d22bSBorislav Petkov INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
53727f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
53827f6d22bSBorislav Petkov
53927f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
54027f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
54127f6d22bSBorislav Petkov INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
54227f6d22bSBorislav Petkov HWEIGHT(n), 0, \
54327f6d22bSBorislav Petkov PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
54427f6d22bSBorislav Petkov
54527f6d22bSBorislav Petkov /* Check flags and event code/umask, and set the HSW N/A flag */
54627f6d22bSBorislav Petkov #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
54727f6d22bSBorislav Petkov __EVENT_CONSTRAINT(code, n, \
54827f6d22bSBorislav Petkov INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
54927f6d22bSBorislav Petkov HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
55027f6d22bSBorislav Petkov
55127f6d22bSBorislav Petkov
55227f6d22bSBorislav Petkov /*
55327f6d22bSBorislav Petkov * We define the end marker as having a weight of -1
55427f6d22bSBorislav Petkov * to enable blacklisting of events using a counter bitmask
55527f6d22bSBorislav Petkov * of zero and thus a weight of zero.
55627f6d22bSBorislav Petkov * The end marker has a weight that cannot possibly be
55727f6d22bSBorislav Petkov * obtained from counting the bits in the bitmask.
55827f6d22bSBorislav Petkov */
55927f6d22bSBorislav Petkov #define EVENT_CONSTRAINT_END { .weight = -1 }
56027f6d22bSBorislav Petkov
56127f6d22bSBorislav Petkov /*
56227f6d22bSBorislav Petkov * Check for end marker with weight == -1
56327f6d22bSBorislav Petkov */
56427f6d22bSBorislav Petkov #define for_each_event_constraint(e, c) \
56527f6d22bSBorislav Petkov for ((e) = (c); (e)->weight != -1; (e)++)
56627f6d22bSBorislav Petkov
56727f6d22bSBorislav Petkov /*
56827f6d22bSBorislav Petkov * Extra registers for specific events.
56927f6d22bSBorislav Petkov *
57027f6d22bSBorislav Petkov * Some events need large masks and require external MSRs.
57127f6d22bSBorislav Petkov * Those extra MSRs end up being shared for all events on
57227f6d22bSBorislav Petkov * a PMU and sometimes between PMU of sibling HT threads.
57327f6d22bSBorislav Petkov * In either case, the kernel needs to handle conflicting
57427f6d22bSBorislav Petkov * accesses to those extra, shared, regs. The data structure
57527f6d22bSBorislav Petkov * to manage those registers is stored in cpu_hw_event.
57627f6d22bSBorislav Petkov */
57727f6d22bSBorislav Petkov struct extra_reg {
57827f6d22bSBorislav Petkov unsigned int event;
57927f6d22bSBorislav Petkov unsigned int msr;
58027f6d22bSBorislav Petkov u64 config_mask;
58127f6d22bSBorislav Petkov u64 valid_mask;
58227f6d22bSBorislav Petkov int idx; /* per_xxx->regs[] reg index */
58327f6d22bSBorislav Petkov bool extra_msr_access;
58427f6d22bSBorislav Petkov };
58527f6d22bSBorislav Petkov
58627f6d22bSBorislav Petkov #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
58727f6d22bSBorislav Petkov .event = (e), \
58827f6d22bSBorislav Petkov .msr = (ms), \
58927f6d22bSBorislav Petkov .config_mask = (m), \
59027f6d22bSBorislav Petkov .valid_mask = (vm), \
59127f6d22bSBorislav Petkov .idx = EXTRA_REG_##i, \
59227f6d22bSBorislav Petkov .extra_msr_access = true, \
59327f6d22bSBorislav Petkov }
59427f6d22bSBorislav Petkov
59527f6d22bSBorislav Petkov #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
59627f6d22bSBorislav Petkov EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
59727f6d22bSBorislav Petkov
59827f6d22bSBorislav Petkov #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
59927f6d22bSBorislav Petkov EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
60027f6d22bSBorislav Petkov ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
60127f6d22bSBorislav Petkov
60227f6d22bSBorislav Petkov #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
60327f6d22bSBorislav Petkov INTEL_UEVENT_EXTRA_REG(c, \
60427f6d22bSBorislav Petkov MSR_PEBS_LD_LAT_THRESHOLD, \
60527f6d22bSBorislav Petkov 0xffff, \
60627f6d22bSBorislav Petkov LDLAT)
60727f6d22bSBorislav Petkov
60827f6d22bSBorislav Petkov #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
60927f6d22bSBorislav Petkov
61027f6d22bSBorislav Petkov union perf_capabilities {
61127f6d22bSBorislav Petkov struct {
61227f6d22bSBorislav Petkov u64 lbr_format:6;
61327f6d22bSBorislav Petkov u64 pebs_trap:1;
61427f6d22bSBorislav Petkov u64 pebs_arch_reg:1;
61527f6d22bSBorislav Petkov u64 pebs_format:4;
61627f6d22bSBorislav Petkov u64 smm_freeze:1;
61727f6d22bSBorislav Petkov /*
61827f6d22bSBorislav Petkov * PMU supports separate counter range for writing
61927f6d22bSBorislav Petkov * values > 32bit.
62027f6d22bSBorislav Petkov */
62127f6d22bSBorislav Petkov u64 full_width_write:1;
622c22497f5SKan Liang u64 pebs_baseline:1;
623bbdbde2aSKan Liang u64 perf_metrics:1;
62442880f72SAlexander Shishkin u64 pebs_output_pt_available:1;
625c87a3109SKan Liang u64 pebs_timing_info:1;
626cadbaa03SStephane Eranian u64 anythread_deprecated:1;
62727f6d22bSBorislav Petkov };
62827f6d22bSBorislav Petkov u64 capabilities;
62927f6d22bSBorislav Petkov };
63027f6d22bSBorislav Petkov
63127f6d22bSBorislav Petkov struct x86_pmu_quirk {
63227f6d22bSBorislav Petkov struct x86_pmu_quirk *next;
63327f6d22bSBorislav Petkov void (*func)(void);
63427f6d22bSBorislav Petkov };
63527f6d22bSBorislav Petkov
63627f6d22bSBorislav Petkov union x86_pmu_config {
63727f6d22bSBorislav Petkov struct {
63827f6d22bSBorislav Petkov u64 event:8,
63927f6d22bSBorislav Petkov umask:8,
64027f6d22bSBorislav Petkov usr:1,
64127f6d22bSBorislav Petkov os:1,
64227f6d22bSBorislav Petkov edge:1,
64327f6d22bSBorislav Petkov pc:1,
64427f6d22bSBorislav Petkov interrupt:1,
64527f6d22bSBorislav Petkov __reserved1:1,
64627f6d22bSBorislav Petkov en:1,
64727f6d22bSBorislav Petkov inv:1,
64827f6d22bSBorislav Petkov cmask:8,
64927f6d22bSBorislav Petkov event2:4,
65027f6d22bSBorislav Petkov __reserved2:4,
65127f6d22bSBorislav Petkov go:1,
65227f6d22bSBorislav Petkov ho:1;
65327f6d22bSBorislav Petkov } bits;
65427f6d22bSBorislav Petkov u64 value;
65527f6d22bSBorislav Petkov };
65627f6d22bSBorislav Petkov
65727f6d22bSBorislav Petkov #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
65827f6d22bSBorislav Petkov
65927f6d22bSBorislav Petkov enum {
66027f6d22bSBorislav Petkov x86_lbr_exclusive_lbr,
66127f6d22bSBorislav Petkov x86_lbr_exclusive_bts,
66227f6d22bSBorislav Petkov x86_lbr_exclusive_pt,
66327f6d22bSBorislav Petkov x86_lbr_exclusive_max,
66427f6d22bSBorislav Petkov };
66527f6d22bSBorislav Petkov
666608f6976SKan Liang #define PERF_PEBS_DATA_SOURCE_MAX 0x100
66738aaf921SKan Liang #define PERF_PEBS_DATA_SOURCE_MASK (PERF_PEBS_DATA_SOURCE_MAX - 1)
668608f6976SKan Liang #define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10
669608f6976SKan Liang #define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
670ccf170e9SKan Liang
671b0560bfdSKan Liang enum hybrid_cpu_type {
672b0560bfdSKan Liang HYBRID_INTEL_NONE,
673b0560bfdSKan Liang HYBRID_INTEL_ATOM = 0x20,
674b0560bfdSKan Liang HYBRID_INTEL_CORE = 0x40,
675b0560bfdSKan Liang };
676b0560bfdSKan Liang
677b0560bfdSKan Liang enum hybrid_pmu_type {
678b0560bfdSKan Liang not_hybrid,
679b0560bfdSKan Liang hybrid_small = BIT(0),
680b0560bfdSKan Liang hybrid_big = BIT(1),
681b0560bfdSKan Liang
682b0560bfdSKan Liang hybrid_big_small = hybrid_big | hybrid_small, /* only used for matching */
683b0560bfdSKan Liang };
684b0560bfdSKan Liang
685b0560bfdSKan Liang #define X86_HYBRID_PMU_ATOM_IDX 0
686b0560bfdSKan Liang #define X86_HYBRID_PMU_CORE_IDX 1
687b0560bfdSKan Liang
688b0560bfdSKan Liang #define X86_HYBRID_NUM_PMUS 2
689b0560bfdSKan Liang
690d0946a88SKan Liang struct x86_hybrid_pmu {
691d0946a88SKan Liang struct pmu pmu;
692d9977c43SKan Liang const char *name;
693b0560bfdSKan Liang enum hybrid_pmu_type pmu_type;
694d9977c43SKan Liang cpumask_t supported_cpus;
695d0946a88SKan Liang union perf_capabilities intel_cap;
696fc4b8fcaSKan Liang u64 intel_ctrl;
697a23eb2fcSKan Liang u64 pebs_events_mask;
698e8fb5d6eSKan Liang u64 config_mask;
699722e42e4SKan Liang union {
700722e42e4SKan Liang u64 cntr_mask64;
701722e42e4SKan Liang unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
702722e42e4SKan Liang };
703722e42e4SKan Liang union {
704722e42e4SKan Liang u64 fixed_cntr_mask64;
705722e42e4SKan Liang unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
706722e42e4SKan Liang };
707eaacf07dSKan Liang struct event_constraint unconstrained;
7080d18f2dfSKan Liang
7090d18f2dfSKan Liang u64 hw_cache_event_ids
7100d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_MAX]
7110d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_OP_MAX]
7120d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_RESULT_MAX];
7130d18f2dfSKan Liang u64 hw_cache_extra_regs
7140d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_MAX]
7150d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_OP_MAX]
7160d18f2dfSKan Liang [PERF_COUNT_HW_CACHE_RESULT_MAX];
71724ee38ffSKan Liang struct event_constraint *event_constraints;
71824ee38ffSKan Liang struct event_constraint *pebs_constraints;
719183af736SKan Liang struct extra_reg *extra_regs;
720acade637SKan Liang
721acade637SKan Liang unsigned int late_ack :1,
722acade637SKan Liang mid_ack :1,
723acade637SKan Liang enabled_ack :1;
724ccf170e9SKan Liang
725ccf170e9SKan Liang u64 pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
726d0946a88SKan Liang };
727d0946a88SKan Liang
hybrid_pmu(struct pmu * pmu)728d0946a88SKan Liang static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu)
729d0946a88SKan Liang {
730d0946a88SKan Liang return container_of(pmu, struct x86_hybrid_pmu, pmu);
731d0946a88SKan Liang }
732d0946a88SKan Liang
733d0946a88SKan Liang extern struct static_key_false perf_is_hybrid;
734d0946a88SKan Liang #define is_hybrid() static_branch_unlikely(&perf_is_hybrid)
735d0946a88SKan Liang
736d0946a88SKan Liang #define hybrid(_pmu, _field) \
737d0946a88SKan Liang (*({ \
738d0946a88SKan Liang typeof(&x86_pmu._field) __Fp = &x86_pmu._field; \
739d0946a88SKan Liang \
740d0946a88SKan Liang if (is_hybrid() && (_pmu)) \
741d0946a88SKan Liang __Fp = &hybrid_pmu(_pmu)->_field; \
742d0946a88SKan Liang \
743d0946a88SKan Liang __Fp; \
744d0946a88SKan Liang }))
745d0946a88SKan Liang
746eaacf07dSKan Liang #define hybrid_var(_pmu, _var) \
747eaacf07dSKan Liang (*({ \
748eaacf07dSKan Liang typeof(&_var) __Fp = &_var; \
749eaacf07dSKan Liang \
750eaacf07dSKan Liang if (is_hybrid() && (_pmu)) \
751eaacf07dSKan Liang __Fp = &hybrid_pmu(_pmu)->_var; \
752eaacf07dSKan Liang \
753eaacf07dSKan Liang __Fp; \
754eaacf07dSKan Liang }))
755eaacf07dSKan Liang
756acade637SKan Liang #define hybrid_bit(_pmu, _field) \
757acade637SKan Liang ({ \
758acade637SKan Liang bool __Fp = x86_pmu._field; \
759acade637SKan Liang \
760acade637SKan Liang if (is_hybrid() && (_pmu)) \
761acade637SKan Liang __Fp = hybrid_pmu(_pmu)->_field; \
762acade637SKan Liang \
763acade637SKan Liang __Fp; \
764acade637SKan Liang })
765acade637SKan Liang
76627f6d22bSBorislav Petkov /*
76727f6d22bSBorislav Petkov * struct x86_pmu - generic x86 pmu
76827f6d22bSBorislav Petkov */
76927f6d22bSBorislav Petkov struct x86_pmu {
77027f6d22bSBorislav Petkov /*
77127f6d22bSBorislav Petkov * Generic x86 PMC bits
77227f6d22bSBorislav Petkov */
77327f6d22bSBorislav Petkov const char *name;
77427f6d22bSBorislav Petkov int version;
77527f6d22bSBorislav Petkov int (*handle_irq)(struct pt_regs *);
77627f6d22bSBorislav Petkov void (*disable_all)(void);
77727f6d22bSBorislav Petkov void (*enable_all)(int added);
77827f6d22bSBorislav Petkov void (*enable)(struct perf_event *);
77927f6d22bSBorislav Petkov void (*disable)(struct perf_event *);
7808b8ff8ccSAdrian Hunter void (*assign)(struct perf_event *event, int idx);
78168f7082fSPeter Zijlstra void (*add)(struct perf_event *);
78268f7082fSPeter Zijlstra void (*del)(struct perf_event *);
783bcfbe5c4SKan Liang void (*read)(struct perf_event *event);
78473759c34SPeter Zijlstra int (*set_period)(struct perf_event *event);
78573759c34SPeter Zijlstra u64 (*update)(struct perf_event *event);
78627f6d22bSBorislav Petkov int (*hw_config)(struct perf_event *event);
78727f6d22bSBorislav Petkov int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
78827f6d22bSBorislav Petkov unsigned eventsel;
78927f6d22bSBorislav Petkov unsigned perfctr;
790*149fd471SKan Liang unsigned fixedctr;
79127f6d22bSBorislav Petkov int (*addr_offset)(int index, bool eventsel);
79227f6d22bSBorislav Petkov int (*rdpmc_index)(int index);
79327f6d22bSBorislav Petkov u64 (*event_map)(int);
79427f6d22bSBorislav Petkov int max_events;
795e8fb5d6eSKan Liang u64 config_mask;
796722e42e4SKan Liang union {
797722e42e4SKan Liang u64 cntr_mask64;
798722e42e4SKan Liang unsigned long cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
799722e42e4SKan Liang };
800722e42e4SKan Liang union {
801722e42e4SKan Liang u64 fixed_cntr_mask64;
802722e42e4SKan Liang unsigned long fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
803722e42e4SKan Liang };
80427f6d22bSBorislav Petkov int cntval_bits;
80527f6d22bSBorislav Petkov u64 cntval_mask;
80627f6d22bSBorislav Petkov union {
80727f6d22bSBorislav Petkov unsigned long events_maskl;
80827f6d22bSBorislav Petkov unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
80927f6d22bSBorislav Petkov };
81027f6d22bSBorislav Petkov int events_mask_len;
81127f6d22bSBorislav Petkov int apic;
81227f6d22bSBorislav Petkov u64 max_period;
81327f6d22bSBorislav Petkov struct event_constraint *
81427f6d22bSBorislav Petkov (*get_event_constraints)(struct cpu_hw_events *cpuc,
81527f6d22bSBorislav Petkov int idx,
81627f6d22bSBorislav Petkov struct perf_event *event);
81727f6d22bSBorislav Petkov
81827f6d22bSBorislav Petkov void (*put_event_constraints)(struct cpu_hw_events *cpuc,
81927f6d22bSBorislav Petkov struct perf_event *event);
82027f6d22bSBorislav Petkov
82127f6d22bSBorislav Petkov void (*start_scheduling)(struct cpu_hw_events *cpuc);
82227f6d22bSBorislav Petkov
82327f6d22bSBorislav Petkov void (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
82427f6d22bSBorislav Petkov
82527f6d22bSBorislav Petkov void (*stop_scheduling)(struct cpu_hw_events *cpuc);
82627f6d22bSBorislav Petkov
82727f6d22bSBorislav Petkov struct event_constraint *event_constraints;
82827f6d22bSBorislav Petkov struct x86_pmu_quirk *quirks;
82928f0f3c4SPeter Zijlstra void (*limit_period)(struct perf_event *event, s64 *l);
83027f6d22bSBorislav Petkov
831af3bdb99SAndi Kleen /* PMI handler bits */
832af3bdb99SAndi Kleen unsigned int late_ack :1,
833acade637SKan Liang mid_ack :1,
8343daa96d6SPeter Zijlstra enabled_ack :1;
83527f6d22bSBorislav Petkov /*
83627f6d22bSBorislav Petkov * sysfs attrs
83727f6d22bSBorislav Petkov */
83827f6d22bSBorislav Petkov int attr_rdpmc_broken;
83927f6d22bSBorislav Petkov int attr_rdpmc;
84027f6d22bSBorislav Petkov struct attribute **format_attrs;
84127f6d22bSBorislav Petkov
84227f6d22bSBorislav Petkov ssize_t (*events_sysfs_show)(char *page, u64 config);
843baa0c833SJiri Olsa const struct attribute_group **attr_update;
84427f6d22bSBorislav Petkov
8456089327fSKan Liang unsigned long attr_freeze_on_smi;
8466089327fSKan Liang
84727f6d22bSBorislav Petkov /*
84827f6d22bSBorislav Petkov * CPU Hotplug hooks
84927f6d22bSBorislav Petkov */
85027f6d22bSBorislav Petkov int (*cpu_prepare)(int cpu);
85127f6d22bSBorislav Petkov void (*cpu_starting)(int cpu);
85227f6d22bSBorislav Petkov void (*cpu_dying)(int cpu);
85327f6d22bSBorislav Petkov void (*cpu_dead)(int cpu);
85427f6d22bSBorislav Petkov
85527f6d22bSBorislav Petkov void (*check_microcode)(void);
856bd275681SPeter Zijlstra void (*sched_task)(struct perf_event_pmu_context *pmu_ctx,
85727f6d22bSBorislav Petkov bool sched_in);
85827f6d22bSBorislav Petkov
85927f6d22bSBorislav Petkov /*
86027f6d22bSBorislav Petkov * Intel Arch Perfmon v2+
86127f6d22bSBorislav Petkov */
86227f6d22bSBorislav Petkov u64 intel_ctrl;
86327f6d22bSBorislav Petkov union perf_capabilities intel_cap;
86427f6d22bSBorislav Petkov
86527f6d22bSBorislav Petkov /*
86627f6d22bSBorislav Petkov * Intel DebugStore bits
86727f6d22bSBorislav Petkov */
86827f6d22bSBorislav Petkov unsigned int bts :1,
86927f6d22bSBorislav Petkov bts_active :1,
87027f6d22bSBorislav Petkov pebs :1,
87127f6d22bSBorislav Petkov pebs_active :1,
87227f6d22bSBorislav Petkov pebs_broken :1,
87395298355SAndi Kleen pebs_prec_dist :1,
8749b545c04SAndi Kleen pebs_no_tlb :1,
87561b985e3SKan Liang pebs_no_isolation :1,
876fb358e0bSLike Xu pebs_block :1,
877fb358e0bSLike Xu pebs_ept :1;
87827f6d22bSBorislav Petkov int pebs_record_size;
879e72daf3fSJiri Olsa int pebs_buffer_size;
880a23eb2fcSKan Liang u64 pebs_events_mask;
8819dfa9a5cSPeter Zijlstra void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
88227f6d22bSBorislav Petkov struct event_constraint *pebs_constraints;
88327f6d22bSBorislav Petkov void (*pebs_aliases)(struct perf_event *event);
88439a41278SKan Liang u64 (*pebs_latency_data)(struct perf_event *event, u64 status);
885174afc3eSKan Liang unsigned long large_pebs_flags;
886c22497f5SKan Liang u64 rtm_abort_event;
8870d23dc34SPeter Zijlstra (Intel) u64 pebs_capable;
88827f6d22bSBorislav Petkov
88927f6d22bSBorislav Petkov /*
89027f6d22bSBorislav Petkov * Intel LBR
89127f6d22bSBorislav Petkov */
8923cb9d546SWei Wang unsigned int lbr_tos, lbr_from, lbr_to,
893fda1f99fSKan Liang lbr_info, lbr_nr; /* LBR base regs and size */
89449d8184fSKan Liang union {
89527f6d22bSBorislav Petkov u64 lbr_sel_mask; /* LBR_SELECT valid bits */
89649d8184fSKan Liang u64 lbr_ctl_mask; /* LBR_CTL valid bits */
89749d8184fSKan Liang };
89849d8184fSKan Liang union {
89927f6d22bSBorislav Petkov const int *lbr_sel_map; /* lbr_select mappings */
90049d8184fSKan Liang int *lbr_ctl_map; /* LBR_CTL mappings */
90149d8184fSKan Liang };
90227f6d22bSBorislav Petkov bool lbr_double_abort; /* duplicated lbr aborts */
903b0c1ef52SAndi Kleen bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */
90427f6d22bSBorislav Petkov
9051ac7fd81SPeter Zijlstra (Intel) unsigned int lbr_has_info:1;
9061ac7fd81SPeter Zijlstra (Intel) unsigned int lbr_has_tsx:1;
9071ac7fd81SPeter Zijlstra (Intel) unsigned int lbr_from_flags:1;
9081ac7fd81SPeter Zijlstra (Intel) unsigned int lbr_to_cycles:1;
9091ac7fd81SPeter Zijlstra (Intel)
910af6cf129SKan Liang /*
911af6cf129SKan Liang * Intel Architectural LBR CPUID Enumeration
912af6cf129SKan Liang */
913af6cf129SKan Liang unsigned int lbr_depth_mask:8;
914af6cf129SKan Liang unsigned int lbr_deep_c_reset:1;
915af6cf129SKan Liang unsigned int lbr_lip:1;
916af6cf129SKan Liang unsigned int lbr_cpl:1;
917af6cf129SKan Liang unsigned int lbr_filter:1;
918af6cf129SKan Liang unsigned int lbr_call_stack:1;
919af6cf129SKan Liang unsigned int lbr_mispred:1;
920af6cf129SKan Liang unsigned int lbr_timed_lbr:1;
921af6cf129SKan Liang unsigned int lbr_br_type:1;
92233744916SKan Liang unsigned int lbr_counters:4;
923af6cf129SKan Liang
9249f354a72SKan Liang void (*lbr_reset)(void);
925c301b1d8SKan Liang void (*lbr_read)(struct cpu_hw_events *cpuc);
926799571bfSKan Liang void (*lbr_save)(void *ctx);
927799571bfSKan Liang void (*lbr_restore)(void *ctx);
9289f354a72SKan Liang
92927f6d22bSBorislav Petkov /*
93027f6d22bSBorislav Petkov * Intel PT/LBR/BTS are exclusive
93127f6d22bSBorislav Petkov */
93227f6d22bSBorislav Petkov atomic_t lbr_exclusive[x86_lbr_exclusive_max];
93327f6d22bSBorislav Petkov
93427f6d22bSBorislav Petkov /*
9357b2c05a1SKan Liang * Intel perf metrics
9367b2c05a1SKan Liang */
9371ab5f235SKan Liang int num_topdown_events;
9387b2c05a1SKan Liang
9397b2c05a1SKan Liang /*
940bd275681SPeter Zijlstra * perf task context (i.e. struct perf_event_pmu_context::task_ctx_data)
941fc1adfe3SAlexey Budankov * switch helper to bridge calls from perf/core to perf/x86.
942fc1adfe3SAlexey Budankov * See struct pmu::swap_task_ctx() usage for examples;
943fc1adfe3SAlexey Budankov */
944bd275681SPeter Zijlstra void (*swap_task_ctx)(struct perf_event_pmu_context *prev_epc,
945bd275681SPeter Zijlstra struct perf_event_pmu_context *next_epc);
946fc1adfe3SAlexey Budankov
947fc1adfe3SAlexey Budankov /*
94832b62f44SPeter Zijlstra * AMD bits
94932b62f44SPeter Zijlstra */
95032b62f44SPeter Zijlstra unsigned int amd_nb_constraints : 1;
95157388912SKim Phillips u64 perf_ctr_pair_en;
95232b62f44SPeter Zijlstra
95332b62f44SPeter Zijlstra /*
95427f6d22bSBorislav Petkov * Extra registers for events
95527f6d22bSBorislav Petkov */
95627f6d22bSBorislav Petkov struct extra_reg *extra_regs;
95727f6d22bSBorislav Petkov unsigned int flags;
95827f6d22bSBorislav Petkov
95927f6d22bSBorislav Petkov /*
96027f6d22bSBorislav Petkov * Intel host/guest support (KVM)
96127f6d22bSBorislav Petkov */
96239a4d779SLike Xu struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
96381ec3f3cSJiri Olsa
96481ec3f3cSJiri Olsa /*
96581ec3f3cSJiri Olsa * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
96681ec3f3cSJiri Olsa */
96781ec3f3cSJiri Olsa int (*check_period) (struct perf_event *event, u64 period);
96842880f72SAlexander Shishkin
96942880f72SAlexander Shishkin int (*aux_output_match) (struct perf_event *event);
970d0946a88SKan Liang
971bd275681SPeter Zijlstra void (*filter)(struct pmu *pmu, int cpu, bool *ret);
972d0946a88SKan Liang /*
973d0946a88SKan Liang * Hybrid support
974d0946a88SKan Liang *
975d0946a88SKan Liang * Most PMU capabilities are the same among different hybrid PMUs.
976d0946a88SKan Liang * The global x86_pmu saves the architecture capabilities, which
977d0946a88SKan Liang * are available for all PMUs. The hybrid_pmu only includes the
978d0946a88SKan Liang * unique capabilities.
979d0946a88SKan Liang */
980d4b294bfSKan Liang int num_hybrid_pmus;
981d0946a88SKan Liang struct x86_hybrid_pmu *hybrid_pmu;
982b0560bfdSKan Liang enum hybrid_cpu_type (*get_hybrid_cpu_type) (void);
98327f6d22bSBorislav Petkov };
98427f6d22bSBorislav Petkov
985530bfff6SKan Liang struct x86_perf_task_context_opt {
986530bfff6SKan Liang int lbr_callstack_users;
987530bfff6SKan Liang int lbr_stack_state;
988530bfff6SKan Liang int log_id;
989530bfff6SKan Liang };
990530bfff6SKan Liang
99127f6d22bSBorislav Petkov struct x86_perf_task_context {
992e1ad1ac2SLike Xu u64 lbr_sel;
99327f6d22bSBorislav Petkov int tos;
9940592e57bSKan Liang int valid_lbrs;
995530bfff6SKan Liang struct x86_perf_task_context_opt opt;
9965624986dSKan Liang struct lbr_entry lbr[MAX_LBR_ENTRIES];
99727f6d22bSBorislav Petkov };
99827f6d22bSBorislav Petkov
99947125db2SKan Liang struct x86_perf_task_context_arch_lbr {
100047125db2SKan Liang struct x86_perf_task_context_opt opt;
100147125db2SKan Liang struct lbr_entry entries[];
100247125db2SKan Liang };
100347125db2SKan Liang
1004ce711ea3SKan Liang /*
1005ce711ea3SKan Liang * Add padding to guarantee the 64-byte alignment of the state buffer.
1006ce711ea3SKan Liang *
1007ce711ea3SKan Liang * The structure is dynamically allocated. The size of the LBR state may vary
1008ce711ea3SKan Liang * based on the number of LBR registers.
1009ce711ea3SKan Liang *
1010ce711ea3SKan Liang * Do not put anything after the LBR state.
1011ce711ea3SKan Liang */
1012ce711ea3SKan Liang struct x86_perf_task_context_arch_lbr_xsave {
1013ce711ea3SKan Liang struct x86_perf_task_context_opt opt;
1014ce711ea3SKan Liang
1015ce711ea3SKan Liang union {
1016ce711ea3SKan Liang struct xregs_state xsave;
1017ce711ea3SKan Liang struct {
1018ce711ea3SKan Liang struct fxregs_state i387;
1019ce711ea3SKan Liang struct xstate_header header;
1020ce711ea3SKan Liang struct arch_lbr_state lbr;
1021ce711ea3SKan Liang } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
1022ce711ea3SKan Liang };
1023ce711ea3SKan Liang };
1024ce711ea3SKan Liang
102527f6d22bSBorislav Petkov #define x86_add_quirk(func_) \
102627f6d22bSBorislav Petkov do { \
102727f6d22bSBorislav Petkov static struct x86_pmu_quirk __quirk __initdata = { \
102827f6d22bSBorislav Petkov .func = func_, \
102927f6d22bSBorislav Petkov }; \
103027f6d22bSBorislav Petkov __quirk.next = x86_pmu.quirks; \
103127f6d22bSBorislav Petkov x86_pmu.quirks = &__quirk; \
103227f6d22bSBorislav Petkov } while (0)
103327f6d22bSBorislav Petkov
103427f6d22bSBorislav Petkov /*
103527f6d22bSBorislav Petkov * x86_pmu flags
103627f6d22bSBorislav Petkov */
103727f6d22bSBorislav Petkov #define PMU_FL_NO_HT_SHARING 0x1 /* no hyper-threading resource sharing */
103827f6d22bSBorislav Petkov #define PMU_FL_HAS_RSP_1 0x2 /* has 2 equivalent offcore_rsp regs */
103927f6d22bSBorislav Petkov #define PMU_FL_EXCL_CNTRS 0x4 /* has exclusive counter requirements */
104027f6d22bSBorislav Petkov #define PMU_FL_EXCL_ENABLED 0x8 /* exclusive counter active */
104131962340SKan Liang #define PMU_FL_PEBS_ALL 0x10 /* all events are valid PEBS events */
1042400816f6SPeter Zijlstra (Intel) #define PMU_FL_TFA 0x20 /* deal with TSX force abort */
1043471af006SKim Phillips #define PMU_FL_PAIR 0x40 /* merge counters for large incr. events */
104461b985e3SKan Liang #define PMU_FL_INSTR_LATENCY 0x80 /* Support Instruction Latency in PEBS Memory Info Record */
104561b985e3SKan Liang #define PMU_FL_MEM_LOADS_AUX 0x100 /* Require an auxiliary event for the complete memory info */
1046c87a3109SKan Liang #define PMU_FL_RETIRE_LATENCY 0x200 /* Support Retire Latency in PEBS */
104733744916SKan Liang #define PMU_FL_BR_CNTR 0x400 /* Support branch counter logging */
104827f6d22bSBorislav Petkov
104927f6d22bSBorislav Petkov #define EVENT_VAR(_id) event_attr_##_id
105027f6d22bSBorislav Petkov #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
105127f6d22bSBorislav Petkov
105227f6d22bSBorislav Petkov #define EVENT_ATTR(_name, _id) \
105327f6d22bSBorislav Petkov static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
105427f6d22bSBorislav Petkov .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
105527f6d22bSBorislav Petkov .id = PERF_COUNT_HW_##_id, \
105627f6d22bSBorislav Petkov .event_str = NULL, \
105727f6d22bSBorislav Petkov };
105827f6d22bSBorislav Petkov
105927f6d22bSBorislav Petkov #define EVENT_ATTR_STR(_name, v, str) \
106027f6d22bSBorislav Petkov static struct perf_pmu_events_attr event_attr_##v = { \
106127f6d22bSBorislav Petkov .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
106227f6d22bSBorislav Petkov .id = 0, \
106327f6d22bSBorislav Petkov .event_str = str, \
106427f6d22bSBorislav Petkov };
106527f6d22bSBorislav Petkov
1066fc07e9f9SAndi Kleen #define EVENT_ATTR_STR_HT(_name, v, noht, ht) \
1067fc07e9f9SAndi Kleen static struct perf_pmu_events_ht_attr event_attr_##v = { \
1068fc07e9f9SAndi Kleen .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
1069fc07e9f9SAndi Kleen .id = 0, \
1070fc07e9f9SAndi Kleen .event_str_noht = noht, \
1071fc07e9f9SAndi Kleen .event_str_ht = ht, \
1072fc07e9f9SAndi Kleen }
1073fc07e9f9SAndi Kleen
1074a9c81ccdSKan Liang #define EVENT_ATTR_STR_HYBRID(_name, v, str, _pmu) \
1075a9c81ccdSKan Liang static struct perf_pmu_events_hybrid_attr event_attr_##v = { \
1076a9c81ccdSKan Liang .attr = __ATTR(_name, 0444, events_hybrid_sysfs_show, NULL),\
1077a9c81ccdSKan Liang .id = 0, \
1078a9c81ccdSKan Liang .event_str = str, \
1079a9c81ccdSKan Liang .pmu_type = _pmu, \
1080a9c81ccdSKan Liang }
1081a9c81ccdSKan Liang
1082a9c81ccdSKan Liang #define FORMAT_HYBRID_PTR(_id) (&format_attr_hybrid_##_id.attr.attr)
1083a9c81ccdSKan Liang
1084a9c81ccdSKan Liang #define FORMAT_ATTR_HYBRID(_name, _pmu) \
1085a9c81ccdSKan Liang static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\
1086a9c81ccdSKan Liang .attr = __ATTR_RO(_name), \
1087a9c81ccdSKan Liang .pmu_type = _pmu, \
1088a9c81ccdSKan Liang }
1089a9c81ccdSKan Liang
109061e76d53SKan Liang struct pmu *x86_get_pmu(unsigned int cpu);
109127f6d22bSBorislav Petkov extern struct x86_pmu x86_pmu __read_mostly;
109227f6d22bSBorislav Petkov
109373759c34SPeter Zijlstra DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period);
109473759c34SPeter Zijlstra DECLARE_STATIC_CALL(x86_pmu_update, *x86_pmu.update);
109573759c34SPeter Zijlstra
task_context_opt(void * ctx)1096f42be865SKan Liang static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
1097f42be865SKan Liang {
109847125db2SKan Liang if (static_cpu_has(X86_FEATURE_ARCH_LBR))
109947125db2SKan Liang return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
110047125db2SKan Liang
1101f42be865SKan Liang return &((struct x86_perf_task_context *)ctx)->opt;
1102f42be865SKan Liang }
1103f42be865SKan Liang
x86_pmu_has_lbr_callstack(void)110427f6d22bSBorislav Petkov static inline bool x86_pmu_has_lbr_callstack(void)
110527f6d22bSBorislav Petkov {
110627f6d22bSBorislav Petkov return x86_pmu.lbr_sel_map &&
110727f6d22bSBorislav Petkov x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
110827f6d22bSBorislav Petkov }
110927f6d22bSBorislav Petkov
111027f6d22bSBorislav Petkov DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
1111dbf4e792SPeter Zijlstra DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
111227f6d22bSBorislav Petkov
111327f6d22bSBorislav Petkov int x86_perf_event_set_period(struct perf_event *event);
111427f6d22bSBorislav Petkov
111527f6d22bSBorislav Petkov /*
111627f6d22bSBorislav Petkov * Generalized hw caching related hw_event table, filled
111727f6d22bSBorislav Petkov * in on a per model basis. A value of 0 means
111827f6d22bSBorislav Petkov * 'not supported', -1 means 'hw_event makes no sense on
111927f6d22bSBorislav Petkov * this CPU', any other value means the raw hw_event
112027f6d22bSBorislav Petkov * ID.
112127f6d22bSBorislav Petkov */
112227f6d22bSBorislav Petkov
112327f6d22bSBorislav Petkov #define C(x) PERF_COUNT_HW_CACHE_##x
112427f6d22bSBorislav Petkov
112527f6d22bSBorislav Petkov extern u64 __read_mostly hw_cache_event_ids
112627f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_MAX]
112727f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_OP_MAX]
112827f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_RESULT_MAX];
112927f6d22bSBorislav Petkov extern u64 __read_mostly hw_cache_extra_regs
113027f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_MAX]
113127f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_OP_MAX]
113227f6d22bSBorislav Petkov [PERF_COUNT_HW_CACHE_RESULT_MAX];
113327f6d22bSBorislav Petkov
113427f6d22bSBorislav Petkov u64 x86_perf_event_update(struct perf_event *event);
113527f6d22bSBorislav Petkov
x86_pmu_config_addr(int index)113627f6d22bSBorislav Petkov static inline unsigned int x86_pmu_config_addr(int index)
113727f6d22bSBorislav Petkov {
113827f6d22bSBorislav Petkov return x86_pmu.eventsel + (x86_pmu.addr_offset ?
113927f6d22bSBorislav Petkov x86_pmu.addr_offset(index, true) : index);
114027f6d22bSBorislav Petkov }
114127f6d22bSBorislav Petkov
x86_pmu_event_addr(int index)114227f6d22bSBorislav Petkov static inline unsigned int x86_pmu_event_addr(int index)
114327f6d22bSBorislav Petkov {
114427f6d22bSBorislav Petkov return x86_pmu.perfctr + (x86_pmu.addr_offset ?
114527f6d22bSBorislav Petkov x86_pmu.addr_offset(index, false) : index);
114627f6d22bSBorislav Petkov }
114727f6d22bSBorislav Petkov
x86_pmu_fixed_ctr_addr(int index)1148*149fd471SKan Liang static inline unsigned int x86_pmu_fixed_ctr_addr(int index)
1149*149fd471SKan Liang {
1150*149fd471SKan Liang return x86_pmu.fixedctr + (x86_pmu.addr_offset ?
1151*149fd471SKan Liang x86_pmu.addr_offset(index, false) : index);
1152*149fd471SKan Liang }
1153*149fd471SKan Liang
x86_pmu_rdpmc_index(int index)115427f6d22bSBorislav Petkov static inline int x86_pmu_rdpmc_index(int index)
115527f6d22bSBorislav Petkov {
115627f6d22bSBorislav Petkov return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
115727f6d22bSBorislav Petkov }
115827f6d22bSBorislav Petkov
1159722e42e4SKan Liang bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
1160722e42e4SKan Liang unsigned long *fixed_cntr_mask);
1161fc4b8fcaSKan Liang
116227f6d22bSBorislav Petkov int x86_add_exclusive(unsigned int what);
116327f6d22bSBorislav Petkov
116427f6d22bSBorislav Petkov void x86_del_exclusive(unsigned int what);
116527f6d22bSBorislav Petkov
116627f6d22bSBorislav Petkov int x86_reserve_hardware(void);
116727f6d22bSBorislav Petkov
116827f6d22bSBorislav Petkov void x86_release_hardware(void);
116927f6d22bSBorislav Petkov
1170b00233b5SAndi Kleen int x86_pmu_max_precise(void);
1171b00233b5SAndi Kleen
117227f6d22bSBorislav Petkov void hw_perf_lbr_event_destroy(struct perf_event *event);
117327f6d22bSBorislav Petkov
117427f6d22bSBorislav Petkov int x86_setup_perfctr(struct perf_event *event);
117527f6d22bSBorislav Petkov
117627f6d22bSBorislav Petkov int x86_pmu_hw_config(struct perf_event *event);
117727f6d22bSBorislav Petkov
117827f6d22bSBorislav Petkov void x86_pmu_disable_all(void);
117927f6d22bSBorislav Petkov
has_amd_brs(struct hw_perf_event * hwc)1180ada54345SStephane Eranian static inline bool has_amd_brs(struct hw_perf_event *hwc)
1181ada54345SStephane Eranian {
1182ada54345SStephane Eranian return hwc->flags & PERF_X86_EVENT_AMD_BRS;
1183ada54345SStephane Eranian }
1184ada54345SStephane Eranian
is_counter_pair(struct hw_perf_event * hwc)118557388912SKim Phillips static inline bool is_counter_pair(struct hw_perf_event *hwc)
118657388912SKim Phillips {
118757388912SKim Phillips return hwc->flags & PERF_X86_EVENT_PAIR;
118857388912SKim Phillips }
118957388912SKim Phillips
__x86_pmu_enable_event(struct hw_perf_event * hwc,u64 enable_mask)119027f6d22bSBorislav Petkov static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
119127f6d22bSBorislav Petkov u64 enable_mask)
119227f6d22bSBorislav Petkov {
119327f6d22bSBorislav Petkov u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
119427f6d22bSBorislav Petkov
119527f6d22bSBorislav Petkov if (hwc->extra_reg.reg)
119627f6d22bSBorislav Petkov wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
119757388912SKim Phillips
119857388912SKim Phillips /*
119957388912SKim Phillips * Add enabled Merge event on next counter
120057388912SKim Phillips * if large increment event being enabled on this counter
120157388912SKim Phillips */
120257388912SKim Phillips if (is_counter_pair(hwc))
120357388912SKim Phillips wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
120457388912SKim Phillips
120527f6d22bSBorislav Petkov wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
120627f6d22bSBorislav Petkov }
120727f6d22bSBorislav Petkov
120827f6d22bSBorislav Petkov void x86_pmu_enable_all(int added);
120927f6d22bSBorislav Petkov
121027f6d22bSBorislav Petkov int perf_assign_events(struct event_constraint **constraints, int n,
121127f6d22bSBorislav Petkov int wmin, int wmax, int gpmax, int *assign);
121227f6d22bSBorislav Petkov int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
121327f6d22bSBorislav Petkov
121427f6d22bSBorislav Petkov void x86_pmu_stop(struct perf_event *event, int flags);
121527f6d22bSBorislav Petkov
x86_pmu_disable_event(struct perf_event * event)121627f6d22bSBorislav Petkov static inline void x86_pmu_disable_event(struct perf_event *event)
121727f6d22bSBorislav Petkov {
1218df51fe7eSLike Xu u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
121927f6d22bSBorislav Petkov struct hw_perf_event *hwc = &event->hw;
122027f6d22bSBorislav Petkov
1221df51fe7eSLike Xu wrmsrl(hwc->config_base, hwc->config & ~disable_mask);
122257388912SKim Phillips
122357388912SKim Phillips if (is_counter_pair(hwc))
122457388912SKim Phillips wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
122527f6d22bSBorislav Petkov }
122627f6d22bSBorislav Petkov
122727f6d22bSBorislav Petkov void x86_pmu_enable_event(struct perf_event *event);
122827f6d22bSBorislav Petkov
122927f6d22bSBorislav Petkov int x86_pmu_handle_irq(struct pt_regs *regs);
123027f6d22bSBorislav Petkov
1231722e42e4SKan Liang void x86_pmu_show_pmu_cap(struct pmu *pmu);
1232722e42e4SKan Liang
x86_pmu_num_counters(struct pmu * pmu)1233722e42e4SKan Liang static inline int x86_pmu_num_counters(struct pmu *pmu)
1234722e42e4SKan Liang {
1235722e42e4SKan Liang return hweight64(hybrid(pmu, cntr_mask64));
1236722e42e4SKan Liang }
1237722e42e4SKan Liang
x86_pmu_max_num_counters(struct pmu * pmu)1238722e42e4SKan Liang static inline int x86_pmu_max_num_counters(struct pmu *pmu)
1239722e42e4SKan Liang {
1240722e42e4SKan Liang return fls64(hybrid(pmu, cntr_mask64));
1241722e42e4SKan Liang }
1242722e42e4SKan Liang
x86_pmu_num_counters_fixed(struct pmu * pmu)1243722e42e4SKan Liang static inline int x86_pmu_num_counters_fixed(struct pmu *pmu)
1244722e42e4SKan Liang {
1245722e42e4SKan Liang return hweight64(hybrid(pmu, fixed_cntr_mask64));
1246722e42e4SKan Liang }
1247722e42e4SKan Liang
x86_pmu_max_num_counters_fixed(struct pmu * pmu)1248722e42e4SKan Liang static inline int x86_pmu_max_num_counters_fixed(struct pmu *pmu)
1249722e42e4SKan Liang {
1250722e42e4SKan Liang return fls64(hybrid(pmu, fixed_cntr_mask64));
1251722e42e4SKan Liang }
1252e11c1a7eSKan Liang
x86_pmu_get_event_config(struct perf_event * event)1253e8fb5d6eSKan Liang static inline u64 x86_pmu_get_event_config(struct perf_event *event)
1254e8fb5d6eSKan Liang {
1255e8fb5d6eSKan Liang return event->attr.config & hybrid(event->pmu, config_mask);
1256e8fb5d6eSKan Liang }
1257e8fb5d6eSKan Liang
125827f6d22bSBorislav Petkov extern struct event_constraint emptyconstraint;
125927f6d22bSBorislav Petkov
126027f6d22bSBorislav Petkov extern struct event_constraint unconstrained;
126127f6d22bSBorislav Petkov
kernel_ip(unsigned long ip)126227f6d22bSBorislav Petkov static inline bool kernel_ip(unsigned long ip)
126327f6d22bSBorislav Petkov {
126427f6d22bSBorislav Petkov #ifdef CONFIG_X86_32
126527f6d22bSBorislav Petkov return ip > PAGE_OFFSET;
126627f6d22bSBorislav Petkov #else
126727f6d22bSBorislav Petkov return (long)ip < 0;
126827f6d22bSBorislav Petkov #endif
126927f6d22bSBorislav Petkov }
127027f6d22bSBorislav Petkov
127127f6d22bSBorislav Petkov /*
127227f6d22bSBorislav Petkov * Not all PMUs provide the right context information to place the reported IP
127327f6d22bSBorislav Petkov * into full context. Specifically segment registers are typically not
127427f6d22bSBorislav Petkov * supplied.
127527f6d22bSBorislav Petkov *
127627f6d22bSBorislav Petkov * Assuming the address is a linear address (it is for IBS), we fake the CS and
127727f6d22bSBorislav Petkov * vm86 mode using the known zero-based code segment and 'fix up' the registers
127827f6d22bSBorislav Petkov * to reflect this.
127927f6d22bSBorislav Petkov *
128027f6d22bSBorislav Petkov * Intel PEBS/LBR appear to typically provide the effective address, nothing
128127f6d22bSBorislav Petkov * much we can do about that but pray and treat it like a linear address.
128227f6d22bSBorislav Petkov */
set_linear_ip(struct pt_regs * regs,unsigned long ip)128327f6d22bSBorislav Petkov static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
128427f6d22bSBorislav Petkov {
128527f6d22bSBorislav Petkov regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
128627f6d22bSBorislav Petkov if (regs->flags & X86_VM_MASK)
128727f6d22bSBorislav Petkov regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
128827f6d22bSBorislav Petkov regs->ip = ip;
128927f6d22bSBorislav Petkov }
129027f6d22bSBorislav Petkov
12914462fbfeSSandipan Das /*
12924462fbfeSSandipan Das * x86control flow change classification
12934462fbfeSSandipan Das * x86control flow changes include branches, interrupts, traps, faults
12944462fbfeSSandipan Das */
12954462fbfeSSandipan Das enum {
12964462fbfeSSandipan Das X86_BR_NONE = 0, /* unknown */
12974462fbfeSSandipan Das
12984462fbfeSSandipan Das X86_BR_USER = 1 << 0, /* branch target is user */
12994462fbfeSSandipan Das X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
13004462fbfeSSandipan Das
13014462fbfeSSandipan Das X86_BR_CALL = 1 << 2, /* call */
13024462fbfeSSandipan Das X86_BR_RET = 1 << 3, /* return */
13034462fbfeSSandipan Das X86_BR_SYSCALL = 1 << 4, /* syscall */
13044462fbfeSSandipan Das X86_BR_SYSRET = 1 << 5, /* syscall return */
13054462fbfeSSandipan Das X86_BR_INT = 1 << 6, /* sw interrupt */
13064462fbfeSSandipan Das X86_BR_IRET = 1 << 7, /* return from interrupt */
13074462fbfeSSandipan Das X86_BR_JCC = 1 << 8, /* conditional */
13084462fbfeSSandipan Das X86_BR_JMP = 1 << 9, /* jump */
13094462fbfeSSandipan Das X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
13104462fbfeSSandipan Das X86_BR_IND_CALL = 1 << 11,/* indirect calls */
13114462fbfeSSandipan Das X86_BR_ABORT = 1 << 12,/* transaction abort */
13124462fbfeSSandipan Das X86_BR_IN_TX = 1 << 13,/* in transaction */
13134462fbfeSSandipan Das X86_BR_NO_TX = 1 << 14,/* not in transaction */
13144462fbfeSSandipan Das X86_BR_ZERO_CALL = 1 << 15,/* zero length call */
13154462fbfeSSandipan Das X86_BR_CALL_STACK = 1 << 16,/* call stack */
13164462fbfeSSandipan Das X86_BR_IND_JMP = 1 << 17,/* indirect jump */
13174462fbfeSSandipan Das
13184462fbfeSSandipan Das X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */
13194462fbfeSSandipan Das
13204462fbfeSSandipan Das };
13214462fbfeSSandipan Das
13224462fbfeSSandipan Das #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
13234462fbfeSSandipan Das #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
13244462fbfeSSandipan Das
13254462fbfeSSandipan Das #define X86_BR_ANY \
13264462fbfeSSandipan Das (X86_BR_CALL |\
13274462fbfeSSandipan Das X86_BR_RET |\
13284462fbfeSSandipan Das X86_BR_SYSCALL |\
13294462fbfeSSandipan Das X86_BR_SYSRET |\
13304462fbfeSSandipan Das X86_BR_INT |\
13314462fbfeSSandipan Das X86_BR_IRET |\
13324462fbfeSSandipan Das X86_BR_JCC |\
13334462fbfeSSandipan Das X86_BR_JMP |\
13344462fbfeSSandipan Das X86_BR_IRQ |\
13354462fbfeSSandipan Das X86_BR_ABORT |\
13364462fbfeSSandipan Das X86_BR_IND_CALL |\
13374462fbfeSSandipan Das X86_BR_IND_JMP |\
13384462fbfeSSandipan Das X86_BR_ZERO_CALL)
13394462fbfeSSandipan Das
13404462fbfeSSandipan Das #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
13414462fbfeSSandipan Das
13424462fbfeSSandipan Das #define X86_BR_ANY_CALL \
13434462fbfeSSandipan Das (X86_BR_CALL |\
13444462fbfeSSandipan Das X86_BR_IND_CALL |\
13454462fbfeSSandipan Das X86_BR_ZERO_CALL |\
13464462fbfeSSandipan Das X86_BR_SYSCALL |\
13474462fbfeSSandipan Das X86_BR_IRQ |\
13484462fbfeSSandipan Das X86_BR_INT)
13494462fbfeSSandipan Das
13504462fbfeSSandipan Das int common_branch_type(int type);
13514462fbfeSSandipan Das int branch_type(unsigned long from, unsigned long to, int abort);
1352df3e9612SSandipan Das int branch_type_fused(unsigned long from, unsigned long to, int abort,
1353df3e9612SSandipan Das int *offset);
13544462fbfeSSandipan Das
135527f6d22bSBorislav Petkov ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
135627f6d22bSBorislav Petkov ssize_t intel_event_sysfs_show(char *page, u64 config);
135727f6d22bSBorislav Petkov
1358a49ac9f8SHuang Rui ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1359a49ac9f8SHuang Rui char *page);
1360fc07e9f9SAndi Kleen ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1361fc07e9f9SAndi Kleen char *page);
1362a9c81ccdSKan Liang ssize_t events_hybrid_sysfs_show(struct device *dev,
1363a9c81ccdSKan Liang struct device_attribute *attr,
1364a9c81ccdSKan Liang char *page);
1365a49ac9f8SHuang Rui
fixed_counter_disabled(int i,struct pmu * pmu)1366fc4b8fcaSKan Liang static inline bool fixed_counter_disabled(int i, struct pmu *pmu)
136732451614SKan Liang {
1368fc4b8fcaSKan Liang u64 intel_ctrl = hybrid(pmu, intel_ctrl);
1369fc4b8fcaSKan Liang
1370fc4b8fcaSKan Liang return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
137132451614SKan Liang }
137232451614SKan Liang
137327f6d22bSBorislav Petkov #ifdef CONFIG_CPU_SUP_AMD
137427f6d22bSBorislav Petkov
137527f6d22bSBorislav Petkov int amd_pmu_init(void);
1376cc37e520SStephane Eranian
1377703fb765SSandipan Das int amd_pmu_lbr_init(void);
1378ca5b7c0dSSandipan Das void amd_pmu_lbr_reset(void);
1379ca5b7c0dSSandipan Das void amd_pmu_lbr_read(void);
1380ca5b7c0dSSandipan Das void amd_pmu_lbr_add(struct perf_event *event);
1381ca5b7c0dSSandipan Das void amd_pmu_lbr_del(struct perf_event *event);
1382bd275681SPeter Zijlstra void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1383ca5b7c0dSSandipan Das void amd_pmu_lbr_enable_all(void);
1384ca5b7c0dSSandipan Das void amd_pmu_lbr_disable_all(void);
1385ca5b7c0dSSandipan Das int amd_pmu_lbr_hw_config(struct perf_event *event);
1386703fb765SSandipan Das
__amd_pmu_lbr_disable(void)13871eddf187SAndrii Nakryiko static __always_inline void __amd_pmu_lbr_disable(void)
13881eddf187SAndrii Nakryiko {
13891eddf187SAndrii Nakryiko u64 dbg_ctl, dbg_extn_cfg;
13901eddf187SAndrii Nakryiko
13911eddf187SAndrii Nakryiko rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
13921eddf187SAndrii Nakryiko wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
13931eddf187SAndrii Nakryiko
13941eddf187SAndrii Nakryiko if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
13951eddf187SAndrii Nakryiko rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
13961eddf187SAndrii Nakryiko wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
13971eddf187SAndrii Nakryiko }
13981eddf187SAndrii Nakryiko }
13991eddf187SAndrii Nakryiko
1400cc37e520SStephane Eranian #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1401b40d0156SSandipan Das
1402b40d0156SSandipan Das #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
1403b40d0156SSandipan Das
1404ada54345SStephane Eranian int amd_brs_init(void);
1405ada54345SStephane Eranian void amd_brs_disable(void);
1406ada54345SStephane Eranian void amd_brs_enable(void);
1407ada54345SStephane Eranian void amd_brs_enable_all(void);
1408ada54345SStephane Eranian void amd_brs_disable_all(void);
1409ada54345SStephane Eranian void amd_brs_drain(void);
1410d5616bacSStephane Eranian void amd_brs_lopwr_init(void);
1411b40d0156SSandipan Das int amd_brs_hw_config(struct perf_event *event);
1412ada54345SStephane Eranian void amd_brs_reset(void);
1413ada54345SStephane Eranian
amd_pmu_brs_add(struct perf_event * event)1414ada54345SStephane Eranian static inline void amd_pmu_brs_add(struct perf_event *event)
1415ada54345SStephane Eranian {
1416ada54345SStephane Eranian struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1417ada54345SStephane Eranian
1418bd275681SPeter Zijlstra perf_sched_cb_inc(event->pmu);
1419ada54345SStephane Eranian cpuc->lbr_users++;
1420ada54345SStephane Eranian /*
1421ada54345SStephane Eranian * No need to reset BRS because it is reset
1422ada54345SStephane Eranian * on brs_enable() and it is saturating
1423ada54345SStephane Eranian */
1424ada54345SStephane Eranian }
1425ada54345SStephane Eranian
amd_pmu_brs_del(struct perf_event * event)1426ada54345SStephane Eranian static inline void amd_pmu_brs_del(struct perf_event *event)
1427ada54345SStephane Eranian {
1428ada54345SStephane Eranian struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1429ada54345SStephane Eranian
1430ada54345SStephane Eranian cpuc->lbr_users--;
1431ada54345SStephane Eranian WARN_ON_ONCE(cpuc->lbr_users < 0);
1432ada54345SStephane Eranian
1433bd275681SPeter Zijlstra perf_sched_cb_dec(event->pmu);
1434ada54345SStephane Eranian }
1435ada54345SStephane Eranian
1436bd275681SPeter Zijlstra void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1437cc37e520SStephane Eranian #else
amd_brs_init(void)1438cc37e520SStephane Eranian static inline int amd_brs_init(void)
1439cc37e520SStephane Eranian {
1440cc37e520SStephane Eranian return 0;
1441cc37e520SStephane Eranian }
amd_brs_disable(void)1442cc37e520SStephane Eranian static inline void amd_brs_disable(void) {}
amd_brs_enable(void)1443cc37e520SStephane Eranian static inline void amd_brs_enable(void) {}
amd_brs_drain(void)1444cc37e520SStephane Eranian static inline void amd_brs_drain(void) {}
amd_brs_lopwr_init(void)1445cc37e520SStephane Eranian static inline void amd_brs_lopwr_init(void) {}
amd_brs_disable_all(void)1446cc37e520SStephane Eranian static inline void amd_brs_disable_all(void) {}
amd_brs_hw_config(struct perf_event * event)1447b40d0156SSandipan Das static inline int amd_brs_hw_config(struct perf_event *event)
1448cc37e520SStephane Eranian {
1449cc37e520SStephane Eranian return 0;
1450cc37e520SStephane Eranian }
amd_brs_reset(void)1451cc37e520SStephane Eranian static inline void amd_brs_reset(void) {}
1452cc37e520SStephane Eranian
amd_pmu_brs_add(struct perf_event * event)1453cc37e520SStephane Eranian static inline void amd_pmu_brs_add(struct perf_event *event)
1454cc37e520SStephane Eranian {
1455cc37e520SStephane Eranian }
1456cc37e520SStephane Eranian
amd_pmu_brs_del(struct perf_event * event)1457cc37e520SStephane Eranian static inline void amd_pmu_brs_del(struct perf_event *event)
1458cc37e520SStephane Eranian {
1459cc37e520SStephane Eranian }
1460cc37e520SStephane Eranian
amd_pmu_brs_sched_task(struct perf_event_pmu_context * pmu_ctx,bool sched_in)1461bd275681SPeter Zijlstra static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
1462cc37e520SStephane Eranian {
1463cc37e520SStephane Eranian }
1464cc37e520SStephane Eranian
amd_brs_enable_all(void)1465cc37e520SStephane Eranian static inline void amd_brs_enable_all(void)
1466cc37e520SStephane Eranian {
1467cc37e520SStephane Eranian }
1468cc37e520SStephane Eranian
1469cc37e520SStephane Eranian #endif
1470ba2fe750SStephane Eranian
147127f6d22bSBorislav Petkov #else /* CONFIG_CPU_SUP_AMD */
147227f6d22bSBorislav Petkov
amd_pmu_init(void)147327f6d22bSBorislav Petkov static inline int amd_pmu_init(void)
147427f6d22bSBorislav Petkov {
147527f6d22bSBorislav Petkov return 0;
147627f6d22bSBorislav Petkov }
147727f6d22bSBorislav Petkov
amd_brs_init(void)1478ada54345SStephane Eranian static inline int amd_brs_init(void)
1479ada54345SStephane Eranian {
1480ada54345SStephane Eranian return -EOPNOTSUPP;
1481ada54345SStephane Eranian }
1482ada54345SStephane Eranian
amd_brs_drain(void)1483ada54345SStephane Eranian static inline void amd_brs_drain(void)
1484ada54345SStephane Eranian {
1485ada54345SStephane Eranian }
1486ada54345SStephane Eranian
amd_brs_enable_all(void)1487ada54345SStephane Eranian static inline void amd_brs_enable_all(void)
1488ada54345SStephane Eranian {
1489ada54345SStephane Eranian }
1490ada54345SStephane Eranian
amd_brs_disable_all(void)1491ada54345SStephane Eranian static inline void amd_brs_disable_all(void)
1492ada54345SStephane Eranian {
1493ada54345SStephane Eranian }
149427f6d22bSBorislav Petkov #endif /* CONFIG_CPU_SUP_AMD */
149527f6d22bSBorislav Petkov
is_pebs_pt(struct perf_event * event)149642880f72SAlexander Shishkin static inline int is_pebs_pt(struct perf_event *event)
149742880f72SAlexander Shishkin {
149842880f72SAlexander Shishkin return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
149942880f72SAlexander Shishkin }
150042880f72SAlexander Shishkin
150127f6d22bSBorislav Petkov #ifdef CONFIG_CPU_SUP_INTEL
150227f6d22bSBorislav Petkov
intel_pmu_has_bts_period(struct perf_event * event,u64 period)150381ec3f3cSJiri Olsa static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
150427f6d22bSBorislav Petkov {
150567266c10SJiri Olsa struct hw_perf_event *hwc = &event->hw;
150667266c10SJiri Olsa unsigned int hw_event, bts_event;
150727f6d22bSBorislav Petkov
150867266c10SJiri Olsa if (event->attr.freq)
150927f6d22bSBorislav Petkov return false;
151067266c10SJiri Olsa
151167266c10SJiri Olsa hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
151267266c10SJiri Olsa bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
151367266c10SJiri Olsa
151481ec3f3cSJiri Olsa return hw_event == bts_event && period == 1;
151581ec3f3cSJiri Olsa }
151681ec3f3cSJiri Olsa
intel_pmu_has_bts(struct perf_event * event)151781ec3f3cSJiri Olsa static inline bool intel_pmu_has_bts(struct perf_event *event)
151881ec3f3cSJiri Olsa {
151981ec3f3cSJiri Olsa struct hw_perf_event *hwc = &event->hw;
152081ec3f3cSJiri Olsa
152181ec3f3cSJiri Olsa return intel_pmu_has_bts_period(event, hwc->sample_period);
152227f6d22bSBorislav Petkov }
152327f6d22bSBorislav Petkov
__intel_pmu_pebs_disable_all(void)1524c22ac2a3SSong Liu static __always_inline void __intel_pmu_pebs_disable_all(void)
1525c22ac2a3SSong Liu {
1526c22ac2a3SSong Liu wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1527c22ac2a3SSong Liu }
1528c22ac2a3SSong Liu
__intel_pmu_arch_lbr_disable(void)1529c22ac2a3SSong Liu static __always_inline void __intel_pmu_arch_lbr_disable(void)
1530c22ac2a3SSong Liu {
1531c22ac2a3SSong Liu wrmsrl(MSR_ARCH_LBR_CTL, 0);
1532c22ac2a3SSong Liu }
1533c22ac2a3SSong Liu
__intel_pmu_lbr_disable(void)1534c22ac2a3SSong Liu static __always_inline void __intel_pmu_lbr_disable(void)
1535c22ac2a3SSong Liu {
1536c22ac2a3SSong Liu u64 debugctl;
1537c22ac2a3SSong Liu
1538c22ac2a3SSong Liu rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1539c22ac2a3SSong Liu debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1540c22ac2a3SSong Liu wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1541c22ac2a3SSong Liu }
1542c22ac2a3SSong Liu
154327f6d22bSBorislav Petkov int intel_pmu_save_and_restart(struct perf_event *event);
154427f6d22bSBorislav Petkov
154527f6d22bSBorislav Petkov struct event_constraint *
154627f6d22bSBorislav Petkov x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
154727f6d22bSBorislav Petkov struct perf_event *event);
154827f6d22bSBorislav Petkov
1549d01b1f96SPeter Zijlstra (Intel) extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
1550d01b1f96SPeter Zijlstra (Intel) extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
155127f6d22bSBorislav Petkov
155227f6d22bSBorislav Petkov int intel_pmu_init(void);
155327f6d22bSBorislav Petkov
155427f6d22bSBorislav Petkov void init_debug_store_on_cpu(int cpu);
155527f6d22bSBorislav Petkov
155627f6d22bSBorislav Petkov void fini_debug_store_on_cpu(int cpu);
155727f6d22bSBorislav Petkov
155827f6d22bSBorislav Petkov void release_ds_buffers(void);
155927f6d22bSBorislav Petkov
156027f6d22bSBorislav Petkov void reserve_ds_buffers(void);
156127f6d22bSBorislav Petkov
1562c085fb87SKan Liang void release_lbr_buffers(void);
1563c085fb87SKan Liang
1564488e13a4SLike Xu void reserve_lbr_buffers(void);
1565488e13a4SLike Xu
156627f6d22bSBorislav Petkov extern struct event_constraint bts_constraint;
1567097e4311SLike Xu extern struct event_constraint vlbr_constraint;
156827f6d22bSBorislav Petkov
156927f6d22bSBorislav Petkov void intel_pmu_enable_bts(u64 config);
157027f6d22bSBorislav Petkov
157127f6d22bSBorislav Petkov void intel_pmu_disable_bts(void);
157227f6d22bSBorislav Petkov
157327f6d22bSBorislav Petkov int intel_pmu_drain_bts_buffer(void);
157427f6d22bSBorislav Petkov
157509026243SKan Liang u64 grt_latency_data(struct perf_event *event, u64 status);
157639a41278SKan Liang
157709026243SKan Liang u64 cmt_latency_data(struct perf_event *event, u64 status);
157838aaf921SKan Liang
1579608f6976SKan Liang u64 lnl_latency_data(struct perf_event *event, u64 status);
1580608f6976SKan Liang
158127f6d22bSBorislav Petkov extern struct event_constraint intel_core2_pebs_event_constraints[];
158227f6d22bSBorislav Petkov
158327f6d22bSBorislav Petkov extern struct event_constraint intel_atom_pebs_event_constraints[];
158427f6d22bSBorislav Petkov
158527f6d22bSBorislav Petkov extern struct event_constraint intel_slm_pebs_event_constraints[];
158627f6d22bSBorislav Petkov
15878b92c3a7SKan Liang extern struct event_constraint intel_glm_pebs_event_constraints[];
15888b92c3a7SKan Liang
1589dd0b06b5SKan Liang extern struct event_constraint intel_glp_pebs_event_constraints[];
1590dd0b06b5SKan Liang
1591f83d2f91SKan Liang extern struct event_constraint intel_grt_pebs_event_constraints[];
1592f83d2f91SKan Liang
159327f6d22bSBorislav Petkov extern struct event_constraint intel_nehalem_pebs_event_constraints[];
159427f6d22bSBorislav Petkov
159527f6d22bSBorislav Petkov extern struct event_constraint intel_westmere_pebs_event_constraints[];
159627f6d22bSBorislav Petkov
159727f6d22bSBorislav Petkov extern struct event_constraint intel_snb_pebs_event_constraints[];
159827f6d22bSBorislav Petkov
159927f6d22bSBorislav Petkov extern struct event_constraint intel_ivb_pebs_event_constraints[];
160027f6d22bSBorislav Petkov
160127f6d22bSBorislav Petkov extern struct event_constraint intel_hsw_pebs_event_constraints[];
160227f6d22bSBorislav Petkov
1603b3e62463SStephane Eranian extern struct event_constraint intel_bdw_pebs_event_constraints[];
1604b3e62463SStephane Eranian
160527f6d22bSBorislav Petkov extern struct event_constraint intel_skl_pebs_event_constraints[];
160627f6d22bSBorislav Petkov
160760176089SKan Liang extern struct event_constraint intel_icl_pebs_event_constraints[];
160860176089SKan Liang
1609d4b5694cSKan Liang extern struct event_constraint intel_glc_pebs_event_constraints[];
161061b985e3SKan Liang
1611a932aa0eSKan Liang extern struct event_constraint intel_lnc_pebs_event_constraints[];
1612a932aa0eSKan Liang
161327f6d22bSBorislav Petkov struct event_constraint *intel_pebs_constraints(struct perf_event *event);
161427f6d22bSBorislav Petkov
161568f7082fSPeter Zijlstra void intel_pmu_pebs_add(struct perf_event *event);
161668f7082fSPeter Zijlstra
161768f7082fSPeter Zijlstra void intel_pmu_pebs_del(struct perf_event *event);
161868f7082fSPeter Zijlstra
161927f6d22bSBorislav Petkov void intel_pmu_pebs_enable(struct perf_event *event);
162027f6d22bSBorislav Petkov
162127f6d22bSBorislav Petkov void intel_pmu_pebs_disable(struct perf_event *event);
162227f6d22bSBorislav Petkov
162327f6d22bSBorislav Petkov void intel_pmu_pebs_enable_all(void);
162427f6d22bSBorislav Petkov
162527f6d22bSBorislav Petkov void intel_pmu_pebs_disable_all(void);
162627f6d22bSBorislav Petkov
1627bd275681SPeter Zijlstra void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
162827f6d22bSBorislav Petkov
16295bee2cc6SKan Liang void intel_pmu_auto_reload_read(struct perf_event *event);
16305bee2cc6SKan Liang
16315624986dSKan Liang void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1632c22497f5SKan Liang
163327f6d22bSBorislav Petkov void intel_ds_init(void);
163427f6d22bSBorislav Petkov
163533744916SKan Liang void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
163633744916SKan Liang struct cpu_hw_events *cpuc,
163733744916SKan Liang struct perf_event *event);
163833744916SKan Liang
1639bd275681SPeter Zijlstra void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
1640bd275681SPeter Zijlstra struct perf_event_pmu_context *next_epc);
1641421ca868SAlexey Budankov
1642bd275681SPeter Zijlstra void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
164327f6d22bSBorislav Petkov
164419fc9dddSDavid Carrillo-Cisneros u64 lbr_from_signext_quirk_wr(u64 val);
164519fc9dddSDavid Carrillo-Cisneros
164627f6d22bSBorislav Petkov void intel_pmu_lbr_reset(void);
164727f6d22bSBorislav Petkov
16489f354a72SKan Liang void intel_pmu_lbr_reset_32(void);
16499f354a72SKan Liang
16509f354a72SKan Liang void intel_pmu_lbr_reset_64(void);
16519f354a72SKan Liang
165268f7082fSPeter Zijlstra void intel_pmu_lbr_add(struct perf_event *event);
165327f6d22bSBorislav Petkov
165468f7082fSPeter Zijlstra void intel_pmu_lbr_del(struct perf_event *event);
165527f6d22bSBorislav Petkov
165627f6d22bSBorislav Petkov void intel_pmu_lbr_enable_all(bool pmi);
165727f6d22bSBorislav Petkov
165827f6d22bSBorislav Petkov void intel_pmu_lbr_disable_all(void);
165927f6d22bSBorislav Petkov
166027f6d22bSBorislav Petkov void intel_pmu_lbr_read(void);
166127f6d22bSBorislav Petkov
1662c301b1d8SKan Liang void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
1663c301b1d8SKan Liang
1664c301b1d8SKan Liang void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
1665c301b1d8SKan Liang
1666799571bfSKan Liang void intel_pmu_lbr_save(void *ctx);
1667799571bfSKan Liang
1668799571bfSKan Liang void intel_pmu_lbr_restore(void *ctx);
1669799571bfSKan Liang
167027f6d22bSBorislav Petkov void intel_pmu_lbr_init_core(void);
167127f6d22bSBorislav Petkov
167227f6d22bSBorislav Petkov void intel_pmu_lbr_init_nhm(void);
167327f6d22bSBorislav Petkov
167427f6d22bSBorislav Petkov void intel_pmu_lbr_init_atom(void);
167527f6d22bSBorislav Petkov
1676f21d5adcSKan Liang void intel_pmu_lbr_init_slm(void);
1677f21d5adcSKan Liang
167827f6d22bSBorislav Petkov void intel_pmu_lbr_init_snb(void);
167927f6d22bSBorislav Petkov
168027f6d22bSBorislav Petkov void intel_pmu_lbr_init_hsw(void);
168127f6d22bSBorislav Petkov
168227f6d22bSBorislav Petkov void intel_pmu_lbr_init_skl(void);
168327f6d22bSBorislav Petkov
168427f6d22bSBorislav Petkov void intel_pmu_lbr_init_knl(void);
168527f6d22bSBorislav Petkov
16861ac7fd81SPeter Zijlstra (Intel) void intel_pmu_lbr_init(void);
16871ac7fd81SPeter Zijlstra (Intel)
168847125db2SKan Liang void intel_pmu_arch_lbr_init(void);
168947125db2SKan Liang
1690e17dc653SAndi Kleen void intel_pmu_pebs_data_source_nhm(void);
1691e17dc653SAndi Kleen
16926ae5fa61SAndi Kleen void intel_pmu_pebs_data_source_skl(bool pmem);
16936ae5fa61SAndi Kleen
1694ccf170e9SKan Liang void intel_pmu_pebs_data_source_adl(void);
1695ccf170e9SKan Liang
169624919fdeSKan Liang void intel_pmu_pebs_data_source_grt(void);
169724919fdeSKan Liang
169838aaf921SKan Liang void intel_pmu_pebs_data_source_mtl(void);
169938aaf921SKan Liang
1700a430021fSKan Liang void intel_pmu_pebs_data_source_cmt(void);
1701a430021fSKan Liang
1702608f6976SKan Liang void intel_pmu_pebs_data_source_lnl(void);
1703608f6976SKan Liang
170427f6d22bSBorislav Petkov int intel_pmu_setup_lbr_filter(struct perf_event *event);
170527f6d22bSBorislav Petkov
170627f6d22bSBorislav Petkov void intel_pt_interrupt(void);
170727f6d22bSBorislav Petkov
170827f6d22bSBorislav Petkov int intel_bts_interrupt(void);
170927f6d22bSBorislav Petkov
171027f6d22bSBorislav Petkov void intel_bts_enable_local(void);
171127f6d22bSBorislav Petkov
171227f6d22bSBorislav Petkov void intel_bts_disable_local(void);
171327f6d22bSBorislav Petkov
171427f6d22bSBorislav Petkov int p4_pmu_init(void);
171527f6d22bSBorislav Petkov
171627f6d22bSBorislav Petkov int p6_pmu_init(void);
171727f6d22bSBorislav Petkov
171827f6d22bSBorislav Petkov int knc_pmu_init(void);
171927f6d22bSBorislav Petkov
is_ht_workaround_enabled(void)172027f6d22bSBorislav Petkov static inline int is_ht_workaround_enabled(void)
172127f6d22bSBorislav Petkov {
172227f6d22bSBorislav Petkov return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
172327f6d22bSBorislav Petkov }
172427f6d22bSBorislav Petkov
intel_pmu_pebs_mask(u64 cntr_mask)1725a23eb2fcSKan Liang static inline u64 intel_pmu_pebs_mask(u64 cntr_mask)
1726a23eb2fcSKan Liang {
1727a23eb2fcSKan Liang return MAX_PEBS_EVENTS_MASK & cntr_mask;
1728a23eb2fcSKan Liang }
1729a23eb2fcSKan Liang
intel_pmu_max_num_pebs(struct pmu * pmu)1730a23eb2fcSKan Liang static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
1731a23eb2fcSKan Liang {
1732a23eb2fcSKan Liang static_assert(MAX_PEBS_EVENTS == 32);
1733a23eb2fcSKan Liang return fls((u32)hybrid(pmu, pebs_events_mask));
1734a23eb2fcSKan Liang }
1735a23eb2fcSKan Liang
173627f6d22bSBorislav Petkov #else /* CONFIG_CPU_SUP_INTEL */
173727f6d22bSBorislav Petkov
reserve_ds_buffers(void)173827f6d22bSBorislav Petkov static inline void reserve_ds_buffers(void)
173927f6d22bSBorislav Petkov {
174027f6d22bSBorislav Petkov }
174127f6d22bSBorislav Petkov
release_ds_buffers(void)174227f6d22bSBorislav Petkov static inline void release_ds_buffers(void)
174327f6d22bSBorislav Petkov {
174427f6d22bSBorislav Petkov }
174527f6d22bSBorislav Petkov
release_lbr_buffers(void)1746c085fb87SKan Liang static inline void release_lbr_buffers(void)
1747c085fb87SKan Liang {
1748c085fb87SKan Liang }
1749c085fb87SKan Liang
reserve_lbr_buffers(void)1750488e13a4SLike Xu static inline void reserve_lbr_buffers(void)
1751488e13a4SLike Xu {
1752488e13a4SLike Xu }
1753488e13a4SLike Xu
intel_pmu_init(void)175427f6d22bSBorislav Petkov static inline int intel_pmu_init(void)
175527f6d22bSBorislav Petkov {
175627f6d22bSBorislav Petkov return 0;
175727f6d22bSBorislav Petkov }
175827f6d22bSBorislav Petkov
intel_cpuc_prepare(struct cpu_hw_events * cpuc,int cpu)1759f764c58bSPeter Zijlstra static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
176027f6d22bSBorislav Petkov {
1761d01b1f96SPeter Zijlstra (Intel) return 0;
1762d01b1f96SPeter Zijlstra (Intel) }
1763d01b1f96SPeter Zijlstra (Intel)
intel_cpuc_finish(struct cpu_hw_events * cpuc)1764f764c58bSPeter Zijlstra static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1765d01b1f96SPeter Zijlstra (Intel) {
176627f6d22bSBorislav Petkov }
176727f6d22bSBorislav Petkov
is_ht_workaround_enabled(void)176827f6d22bSBorislav Petkov static inline int is_ht_workaround_enabled(void)
176927f6d22bSBorislav Petkov {
177027f6d22bSBorislav Petkov return 0;
177127f6d22bSBorislav Petkov }
177227f6d22bSBorislav Petkov #endif /* CONFIG_CPU_SUP_INTEL */
17733a4ac121SCodyYao-oc
17743a4ac121SCodyYao-oc #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
17753a4ac121SCodyYao-oc int zhaoxin_pmu_init(void);
17763a4ac121SCodyYao-oc #else
zhaoxin_pmu_init(void)17773a4ac121SCodyYao-oc static inline int zhaoxin_pmu_init(void)
17783a4ac121SCodyYao-oc {
17793a4ac121SCodyYao-oc return 0;
17803a4ac121SCodyYao-oc }
17813a4ac121SCodyYao-oc #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/
1782