1 // SPDX-License-Identifier: GPL-2.0-only
2
3 /*
4 * Copyright(c) 2023 Huawei
5 *
6 * The CXL 3.0 specification includes a standard Performance Monitoring Unit,
7 * called the CXL PMU, or CPMU. In order to allow a high degree of
8 * implementation flexibility the specification provides a wide range of
9 * options all of which are self describing.
10 *
11 * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface
12 */
13
14 #include <linux/io-64-nonatomic-lo-hi.h>
15 #include <linux/perf_event.h>
16 #include <linux/bitops.h>
17 #include <linux/device.h>
18 #include <linux/bits.h>
19 #include <linux/list.h>
20 #include <linux/bug.h>
21 #include <linux/pci.h>
22
23 #include "../cxl/cxlpci.h"
24 #include "../cxl/cxl.h"
25 #include "../cxl/pmu.h"
26
27 #define CXL_PMU_CAP_REG 0x0
28 #define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(5, 0)
29 #define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8)
30 #define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20)
31 #define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32)
32 #define CXL_PMU_FILTER_HDM BIT(0)
33 #define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1)
34 #define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44)
35 #define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48)
36 #define CXL_PMU_CAP_FREEZE BIT_ULL(49)
37 #define CXL_PMU_CAP_INT BIT_ULL(50)
38 #define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60)
39
40 #define CXL_PMU_OVERFLOW_REG 0x10
41 #define CXL_PMU_FREEZE_REG 0x18
42 #define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n))
43 #define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0)
44 #define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32)
45 #define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48)
46
47 #define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n))
48 #define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0)
49 #define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0
50 #define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1
51 #define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2
52 #define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8)
53 #define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9)
54 #define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10)
55 #define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11)
56 #define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12)
57 #define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16)
58 #define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24)
59 #define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59)
60
61 #define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8))
62 #define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(31, 0)
63
64 #define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n))
65
66 /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */
67 #define CXL_PMU_GID_CLOCK_TICKS 0x00
68 #define CXL_PMU_GID_D2H_REQ 0x0010
69 #define CXL_PMU_GID_D2H_RSP 0x0011
70 #define CXL_PMU_GID_H2D_REQ 0x0012
71 #define CXL_PMU_GID_H2D_RSP 0x0013
72 #define CXL_PMU_GID_CACHE_DATA 0x0014
73 #define CXL_PMU_GID_M2S_REQ 0x0020
74 #define CXL_PMU_GID_M2S_RWD 0x0021
75 #define CXL_PMU_GID_M2S_BIRSP 0x0022
76 #define CXL_PMU_GID_S2M_BISNP 0x0023
77 #define CXL_PMU_GID_S2M_NDR 0x0024
78 #define CXL_PMU_GID_S2M_DRS 0x0025
79 #define CXL_PMU_GID_DDR 0x8000
80 #define CXL_PMU_GID_QUEUE_OCC 0x8001
81 #define CXL_PMU_GID_QUEUE_RESID 0x8002
82 #define CXL_PMU_GID_RETRY_EVENTS 0x8003
83 #define CXL_PMU_GID_THROTTLE 0x8004
84
85 static int cxl_pmu_cpuhp_state_num;
86
87 struct cxl_pmu_ev_cap {
88 u16 vid;
89 u16 gid;
90 u32 msk;
91 union {
92 int counter_idx; /* fixed counters */
93 int event_idx; /* configurable counters */
94 };
95 struct list_head node;
96 };
97
98 #define CXL_PMU_MAX_COUNTERS 64
99 struct cxl_pmu_info {
100 struct pmu pmu;
101 void __iomem *base;
102 struct perf_event **hw_events;
103 struct list_head event_caps_configurable;
104 struct list_head event_caps_fixed;
105 DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS);
106 DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS);
107 u16 counter_width;
108 u8 num_counters;
109 u8 num_event_capabilities;
110 int on_cpu;
111 struct hlist_node node;
112 bool filter_hdm;
113 bool filter_crb;
114 int irq;
115 };
116
117 #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu)
118
119 /*
120 * All CPMU counters are discoverable via the Event Capabilities Registers.
121 * Each Event Capability register contains a VID / GroupID.
122 * A counter may then count any combination (by summing) of events in
123 * that group which are in the Supported Events Bitmask.
124 * However, there are some complexities to the scheme.
125 * - Fixed function counters refer to an Event Capabilities register.
126 * That event capability register is not then used for Configurable
127 * counters.
128 */
cxl_pmu_parse_caps(struct device * dev,struct cxl_pmu_info * info)129 static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info)
130 {
131 unsigned long fixed_counter_event_cap_bm = 0;
132 void __iomem *base = info->base;
133 bool freeze_for_enable;
134 u64 val, eval;
135 int i;
136
137 val = readq(base + CXL_PMU_CAP_REG);
138 freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) &&
139 FIELD_GET(CXL_PMU_CAP_FREEZE, val);
140 if (!freeze_for_enable) {
141 dev_err(dev, "Counters not writable while frozen\n");
142 return -ENODEV;
143 }
144
145 info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1;
146 info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val);
147 info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1;
148
149 info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM;
150 info->filter_crb = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) &
151 CXL_PMU_FILTER_CHAN_RANK_BANK;
152 if (FIELD_GET(CXL_PMU_CAP_INT, val))
153 info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val);
154 else
155 info->irq = -1;
156
157 /* First handle fixed function counters; note if configurable counters found */
158 for (i = 0; i < info->num_counters; i++) {
159 struct cxl_pmu_ev_cap *pmu_ev;
160 u32 events_msk;
161 u8 group_idx;
162
163 val = readq(base + CXL_PMU_COUNTER_CFG_REG(i));
164
165 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) ==
166 CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) {
167 set_bit(i, info->conf_counter_bm);
168 }
169
170 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) !=
171 CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN)
172 continue;
173
174 /* In this case we know which fields are const */
175 group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val);
176 events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val);
177 eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx));
178 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
179 if (!pmu_ev)
180 return -ENOMEM;
181
182 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
183 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
184 /* For a fixed purpose counter use the events mask from the counter CFG */
185 pmu_ev->msk = events_msk;
186 pmu_ev->counter_idx = i;
187 /* This list add is never unwound as all entries deleted on remove */
188 list_add(&pmu_ev->node, &info->event_caps_fixed);
189 /*
190 * Configurable counters must not use an Event Capability registers that
191 * is in use for a Fixed counter
192 */
193 set_bit(group_idx, &fixed_counter_event_cap_bm);
194 }
195
196 if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) {
197 struct cxl_pmu_ev_cap *pmu_ev;
198 int j;
199 /* Walk event capabilities unused by fixed counters */
200 for_each_clear_bit(j, &fixed_counter_event_cap_bm,
201 info->num_event_capabilities) {
202 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
203 if (!pmu_ev)
204 return -ENOMEM;
205
206 eval = readq(base + CXL_PMU_EVENT_CAP_REG(j));
207 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
208 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
209 pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval);
210 pmu_ev->event_idx = j;
211 list_add(&pmu_ev->node, &info->event_caps_configurable);
212 }
213 }
214
215 return 0;
216 }
217
218 #define CXL_PMU_FORMAT_ATTR(_name, _format)\
219 (&((struct dev_ext_attribute[]) { \
220 { \
221 .attr = __ATTR(_name, 0444, device_show_string, NULL), \
222 .var = (void *)_format \
223 } \
224 })[0].attr.attr)
225
226 enum {
227 cxl_pmu_mask_attr,
228 cxl_pmu_gid_attr,
229 cxl_pmu_vid_attr,
230 cxl_pmu_threshold_attr,
231 cxl_pmu_invert_attr,
232 cxl_pmu_edge_attr,
233 cxl_pmu_hdm_filter_en_attr,
234 cxl_pmu_hdm_attr,
235 cxl_pmu_crb_filter_en_attr,
236 cxl_pmu_crb_attr,
237 };
238
239 static struct attribute *cxl_pmu_format_attr[] = {
240 [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"),
241 [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"),
242 [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"),
243 [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"),
244 [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"),
245 [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"),
246 [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"),
247 [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"),
248 [cxl_pmu_crb_filter_en_attr] = CXL_PMU_FORMAT_ATTR(crb_filter_en, "config1:19"),
249 [cxl_pmu_crb_attr] = CXL_PMU_FORMAT_ATTR(crb, "config2:32-63"),
250 NULL
251 };
252
253 #define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0)
254 #define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32)
255 #define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48)
256 #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0)
257 #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16)
258 #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17)
259 #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18)
260 #define CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK BIT(19)
261 #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0)
262 #define CXL_PMU_ATTR_CONFIG2_CRB_MSK GENMASK_ULL(63, 32)
263
cxl_pmu_format_is_visible(struct kobject * kobj,struct attribute * attr,int a)264 static umode_t cxl_pmu_format_is_visible(struct kobject *kobj,
265 struct attribute *attr, int a)
266 {
267 struct device *dev = kobj_to_dev(kobj);
268 struct cxl_pmu_info *info = dev_get_drvdata(dev);
269
270 /*
271 * Filter capability at the CPMU level, so hide the attributes if the particular
272 * filter is not supported.
273 */
274 if (!info->filter_hdm &&
275 (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] ||
276 attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr]))
277 return 0;
278
279 if (!info->filter_crb &&
280 (attr == cxl_pmu_format_attr[cxl_pmu_crb_filter_en_attr] ||
281 attr == cxl_pmu_format_attr[cxl_pmu_crb_attr]))
282 return 0;
283
284 return attr->mode;
285 }
286
287 static const struct attribute_group cxl_pmu_format_group = {
288 .name = "format",
289 .attrs = cxl_pmu_format_attr,
290 .is_visible = cxl_pmu_format_is_visible,
291 };
292
cxl_pmu_config_get_mask(struct perf_event * event)293 static u32 cxl_pmu_config_get_mask(struct perf_event *event)
294 {
295 return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config);
296 }
297
cxl_pmu_config_get_gid(struct perf_event * event)298 static u16 cxl_pmu_config_get_gid(struct perf_event *event)
299 {
300 return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config);
301 }
302
cxl_pmu_config_get_vid(struct perf_event * event)303 static u16 cxl_pmu_config_get_vid(struct perf_event *event)
304 {
305 return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config);
306 }
307
cxl_pmu_config1_get_threshold(struct perf_event * event)308 static u8 cxl_pmu_config1_get_threshold(struct perf_event *event)
309 {
310 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1);
311 }
312
cxl_pmu_config1_get_invert(struct perf_event * event)313 static bool cxl_pmu_config1_get_invert(struct perf_event *event)
314 {
315 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1);
316 }
317
cxl_pmu_config1_get_edge(struct perf_event * event)318 static bool cxl_pmu_config1_get_edge(struct perf_event *event)
319 {
320 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1);
321 }
322
323 /*
324 * CPMU specification allows for 8 filters, each with a 32 bit value...
325 * So we need to find 8x32bits to store it in.
326 * As the value used for disable is 0xffff_ffff, a separate enable switch
327 * is needed.
328 */
329
cxl_pmu_config1_hdm_filter_en(struct perf_event * event)330 static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event)
331 {
332 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1);
333 }
334
cxl_pmu_config2_get_hdm_decoder(struct perf_event * event)335 static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event)
336 {
337 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2);
338 }
339
cxl_pmu_config1_crb_filter_en(struct perf_event * event)340 static u16 cxl_pmu_config1_crb_filter_en(struct perf_event *event)
341 {
342 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK,
343 event->attr.config1);
344 }
345
cxl_pmu_config2_get_crb(struct perf_event * event)346 static u32 cxl_pmu_config2_get_crb(struct perf_event *event)
347 {
348 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_CRB_MSK, event->attr.config2);
349 }
350
cxl_pmu_event_sysfs_show(struct device * dev,struct device_attribute * attr,char * buf)351 static ssize_t cxl_pmu_event_sysfs_show(struct device *dev,
352 struct device_attribute *attr, char *buf)
353 {
354 struct perf_pmu_events_attr *pmu_attr =
355 container_of(attr, struct perf_pmu_events_attr, attr);
356
357 return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id);
358 }
359
360 #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \
361 PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \
362 ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk))
363
364 /* For CXL spec defined events */
365 #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \
366 CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk)
367
368 static struct attribute *cxl_pmu_event_attrs[] = {
369 CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)),
370 /* CXL rev 3.0 Table 3-17 - Device to Host Requests */
371 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)),
372 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)),
373 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)),
374 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)),
375 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)),
376 CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)),
377 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)),
378 CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)),
379 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)),
380 CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)),
381 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)),
382 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)),
383 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)),
384 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)),
385 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)),
386 /* CXL rev 3.0 Table 3-20 - D2H Response Encodings */
387 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)),
388 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)),
389 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)),
390 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)),
391 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)),
392 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)),
393 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)),
394 /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */
395 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)),
396 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)),
397 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)),
398 /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */
399 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)),
400 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)),
401 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)),
402 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)),
403 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)),
404 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)),
405 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)),
406 /* CXL rev 3.0 Table 13-5 directly lists these */
407 CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)),
408 CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)),
409 /* CXL rev 3.1 Table 3-35 M2S Req Memory Opcodes */
410 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)),
411 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)),
412 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)),
413 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)),
414 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)),
415 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)),
416 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)),
417 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvtee, CXL_PMU_GID_M2S_REQ, BIT(7)),
418 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)),
419 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)),
420 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)),
421 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvptee, CXL_PMU_GID_M2S_REQ, BIT(11)),
422 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrdtee, CXL_PMU_GID_M2S_REQ, BIT(12)),
423 CXL_PMU_EVENT_CXL_ATTR(m2s_req_teupdate, CXL_PMU_GID_M2S_REQ, BIT(13)),
424 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevcttee, CXL_PMU_GID_M2S_REQ, BIT(14)),
425 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevctu, CXL_PMU_GID_M2S_REQ, BIT(15)),
426 /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */
427 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)),
428 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)),
429 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)),
430 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfill, CXL_PMU_GID_M2S_RWD, BIT(5)),
431 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrtee, CXL_PMU_GID_M2S_RWD, BIT(9)),
432 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptltee, CXL_PMU_GID_M2S_RWD, BIT(10)),
433 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfilltee, CXL_PMU_GID_M2S_RWD, BIT(13)),
434 /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */
435 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)),
436 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)),
437 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)),
438 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)),
439 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)),
440 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)),
441 /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */
442 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)),
443 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)),
444 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)),
445 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)),
446 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)),
447 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)),
448 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curtee, CXL_PMU_GID_S2M_BISNP, BIT(8)),
449 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datatee, CXL_PMU_GID_S2M_BISNP, BIT(9)),
450 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invtee, CXL_PMU_GID_S2M_BISNP, BIT(10)),
451 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblktee, CXL_PMU_GID_S2M_BISNP, BIT(12)),
452 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datablktee, CXL_PMU_GID_S2M_BISNP, BIT(13)),
453 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblktee, CXL_PMU_GID_S2M_BISNP, BIT(14)),
454 /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */
455 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)),
456 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)),
457 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)),
458 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)),
459 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)),
460 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee, CXL_PMU_GID_S2M_NDR, BIT(5)),
461 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_s, CXL_PMU_GID_S2M_NDR, BIT(6)),
462 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_e, CXL_PMU_GID_S2M_NDR, BIT(7)),
463 /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */
464 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)),
465 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)),
466 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatatee, CXL_PMU_GID_S2M_DRS, BIT(2)),
467 /* CXL rev 3.0 Table 13-5 directly lists these */
468 CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)),
469 CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)),
470 CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)),
471 CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)),
472 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)),
473 CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)),
474 CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)),
475 /* CXL 4.0 Table 13-5 DDR add-on events opcodes */
476 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_rd_ap, CXL_PMU_GID_DDR, BIT(7)),
477 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_wr_ap, CXL_PMU_GID_DDR, BIT(8)),
478 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_all_banks, CXL_PMU_GID_DDR, BIT(9)),
479 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_same_bank, CXL_PMU_GID_DDR, BIT(10)),
480 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_entry, CXL_PMU_GID_DDR, BIT(11)),
481 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_exit, CXL_PMU_GID_DDR, BIT(12)),
482 CXL_PMU_EVENT_CXL_ATTR(ddr_rd_wr_ddr_bus_switching, CXL_PMU_GID_DDR, BIT(13)),
483 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_rd_req, CXL_PMU_GID_DDR, BIT(14)),
484 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_wr_req, CXL_PMU_GID_DDR, BIT(15)),
485 /* CXL 4.0 Table 13-5 QUEUE OCCUPANCY events opcodes */
486 CXL_PMU_EVENT_CXL_ATTR(rd_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(0)),
487 CXL_PMU_EVENT_CXL_ATTR(wr_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(1)),
488 CXL_PMU_EVENT_CXL_ATTR(rd_wr_merged_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(2)),
489 CXL_PMU_EVENT_CXL_ATTR(pwrdn_event, CXL_PMU_GID_QUEUE_OCC, BIT(3)),
490 /* CXL 4.0 Table 13-5 QUEUE RESIDENCY events opcodes */
491 CXL_PMU_EVENT_CXL_ATTR(mc_rd_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(0)),
492 CXL_PMU_EVENT_CXL_ATTR(mc_wr_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(1)),
493 /* CXL 4.0 Table 13-5 RETRY events opcodes */
494 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_rd_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(0)),
495 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_wr_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(1)),
496 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ca_parity, CXL_PMU_GID_RETRY_EVENTS, BIT(2)),
497 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ecc, CXL_PMU_GID_RETRY_EVENTS, BIT(3)),
498 /* CXL 4.0 Table 13-5 THROTTLE events opcodes */
499 CXL_PMU_EVENT_CXL_ATTR(thermal_throttle_event, CXL_PMU_GID_THROTTLE, BIT(0)),
500 CXL_PMU_EVENT_CXL_ATTR(power_throttle_event, CXL_PMU_GID_THROTTLE, BIT(1)),
501 NULL
502 };
503
cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info * info,int vid,int gid,int msk)504 static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info,
505 int vid, int gid, int msk)
506 {
507 struct cxl_pmu_ev_cap *pmu_ev;
508
509 list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) {
510 if (vid != pmu_ev->vid || gid != pmu_ev->gid)
511 continue;
512
513 /* Precise match for fixed counter */
514 if (msk == pmu_ev->msk)
515 return pmu_ev;
516 }
517
518 return ERR_PTR(-EINVAL);
519 }
520
cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info * info,int vid,int gid,int msk)521 static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info,
522 int vid, int gid, int msk)
523 {
524 struct cxl_pmu_ev_cap *pmu_ev;
525
526 list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) {
527 if (vid != pmu_ev->vid || gid != pmu_ev->gid)
528 continue;
529
530 /* Request mask must be subset of supported */
531 if (msk & ~pmu_ev->msk)
532 continue;
533
534 return pmu_ev;
535 }
536
537 return ERR_PTR(-EINVAL);
538 }
539
cxl_pmu_event_is_visible(struct kobject * kobj,struct attribute * attr,int a)540 static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a)
541 {
542 struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr);
543 struct perf_pmu_events_attr *pmu_attr =
544 container_of(dev_attr, struct perf_pmu_events_attr, attr);
545 struct device *dev = kobj_to_dev(kobj);
546 struct cxl_pmu_info *info = dev_get_drvdata(dev);
547 int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id);
548 int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id);
549 int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id);
550
551 if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk)))
552 return attr->mode;
553
554 if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk)))
555 return attr->mode;
556
557 return 0;
558 }
559
560 static const struct attribute_group cxl_pmu_events = {
561 .name = "events",
562 .attrs = cxl_pmu_event_attrs,
563 .is_visible = cxl_pmu_event_is_visible,
564 };
565
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)566 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
567 char *buf)
568 {
569 struct cxl_pmu_info *info = dev_get_drvdata(dev);
570
571 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(info->on_cpu)));
572 }
573 static DEVICE_ATTR_RO(cpumask);
574
575 static struct attribute *cxl_pmu_cpumask_attrs[] = {
576 &dev_attr_cpumask.attr,
577 NULL
578 };
579
580 static const struct attribute_group cxl_pmu_cpumask_group = {
581 .attrs = cxl_pmu_cpumask_attrs,
582 };
583
584 static const struct attribute_group *cxl_pmu_attr_groups[] = {
585 &cxl_pmu_events,
586 &cxl_pmu_format_group,
587 &cxl_pmu_cpumask_group,
588 NULL
589 };
590
591 /* If counter_idx == NULL, don't try to allocate a counter. */
cxl_pmu_get_event_idx(struct perf_event * event,int * counter_idx,int * event_idx)592 static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx,
593 int *event_idx)
594 {
595 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
596 DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS);
597 struct cxl_pmu_ev_cap *pmu_ev;
598 u32 mask;
599 u16 gid, vid;
600 int i;
601
602 vid = cxl_pmu_config_get_vid(event);
603 gid = cxl_pmu_config_get_gid(event);
604 mask = cxl_pmu_config_get_mask(event);
605
606 pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask);
607 if (!IS_ERR(pmu_ev)) {
608 if (!counter_idx)
609 return 0;
610 if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) {
611 *counter_idx = pmu_ev->counter_idx;
612 return 0;
613 }
614 /* Fixed counter is in use, but maybe a configurable one? */
615 }
616
617 pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask);
618 if (!IS_ERR(pmu_ev)) {
619 if (!counter_idx)
620 return 0;
621
622 bitmap_andnot(configurable_and_free, info->conf_counter_bm,
623 info->used_counter_bm, CXL_PMU_MAX_COUNTERS);
624
625 i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS);
626 if (i == CXL_PMU_MAX_COUNTERS)
627 return -EINVAL;
628
629 *counter_idx = i;
630 return 0;
631 }
632
633 return -EINVAL;
634 }
635
cxl_pmu_event_init(struct perf_event * event)636 static int cxl_pmu_event_init(struct perf_event *event)
637 {
638 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
639 int rc;
640
641 /* Top level type sanity check - is this a Hardware Event being requested */
642 if (event->attr.type != event->pmu->type)
643 return -ENOENT;
644
645 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
646 return -EOPNOTSUPP;
647 /* TODO: Validation of any filter */
648
649 if (cxl_pmu_config1_crb_filter_en(event)) {
650 if (!info->filter_crb)
651 return -EINVAL;
652 /* event group IDs are scoped by the CXL vendor ID */
653 if (cxl_pmu_config_get_vid(event) != PCI_VENDOR_ID_CXL)
654 return -EINVAL;
655
656 /*
657 * CRB filtering (Filter ID 1) is only valid for the DDR
658 * Interface, Queue Occupancy, Queue Residency and Retry
659 * event groups (CXL 4.0 Table 13-5).
660 */
661 switch (cxl_pmu_config_get_gid(event)) {
662 case CXL_PMU_GID_DDR:
663 case CXL_PMU_GID_QUEUE_OCC:
664 case CXL_PMU_GID_QUEUE_RESID:
665 case CXL_PMU_GID_RETRY_EVENTS:
666 break;
667 default:
668 return -EINVAL;
669 }
670
671 /*
672 * Filtering while counting multiple events is
673 * undefined behavior.
674 */
675 if (hweight32(cxl_pmu_config_get_mask(event)) > 1)
676 return -EINVAL;
677 }
678
679 /*
680 * Verify that it is possible to count what was requested. Either must
681 * be a fixed counter that is a precise match or a configurable counter
682 * where this is a subset.
683 */
684 rc = cxl_pmu_get_event_idx(event, NULL, NULL);
685 if (rc < 0)
686 return rc;
687
688 event->cpu = info->on_cpu;
689
690 return 0;
691 }
692
cxl_pmu_enable(struct pmu * pmu)693 static void cxl_pmu_enable(struct pmu *pmu)
694 {
695 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
696 void __iomem *base = info->base;
697
698 /* Can assume frozen at this stage */
699 writeq(0, base + CXL_PMU_FREEZE_REG);
700 }
701
cxl_pmu_disable(struct pmu * pmu)702 static void cxl_pmu_disable(struct pmu *pmu)
703 {
704 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
705 void __iomem *base = info->base;
706
707 /*
708 * Whilst bits above number of counters are RsvdZ
709 * they are unlikely to be repurposed given
710 * number of counters is allowed to be 64 leaving
711 * no reserved bits. Hence this is only slightly
712 * naughty.
713 */
714 writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG);
715 }
716
cxl_pmu_event_start(struct perf_event * event,int flags)717 static void cxl_pmu_event_start(struct perf_event *event, int flags)
718 {
719 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
720 struct hw_perf_event *hwc = &event->hw;
721 void __iomem *base = info->base;
722 u64 cfg;
723
724 /*
725 * All paths to here should either set these flags directly or
726 * call cxl_pmu_event_stop() which will ensure the correct state.
727 */
728 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
729 return;
730
731 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
732 hwc->state = 0;
733
734 /*
735 * Filter ID=0: HDM decoder filter
736 * Filter ID=1: Channel/Rank/Bank (CRB) filter
737 */
738 if (info->filter_hdm) {
739 if (cxl_pmu_config1_hdm_filter_en(event))
740 cfg = cxl_pmu_config2_get_hdm_decoder(event);
741 else
742 cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */
743 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0));
744 }
745
746 if (info->filter_crb) {
747 if (cxl_pmu_config1_crb_filter_en(event))
748 cfg = cxl_pmu_config2_get_crb(event);
749 else
750 cfg = GENMASK(31, 0); /* no filtering if 0xFFFF_FFFF */
751 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 1));
752 }
753
754 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
755 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1);
756 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1);
757 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1);
758 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE,
759 cxl_pmu_config1_get_edge(event) ? 1 : 0);
760 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT,
761 cxl_pmu_config1_get_invert(event) ? 1 : 0);
762
763 /* Fixed purpose counters have next two fields RO */
764 if (test_bit(hwc->idx, info->conf_counter_bm)) {
765 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK,
766 hwc->event_base);
767 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK,
768 cxl_pmu_config_get_mask(event));
769 }
770 cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK;
771 /*
772 * For events that generate only 1 count per clock the CXL 3.0 spec
773 * states the threshold shall be set to 1 but if set to 0 it will
774 * count the raw value anwyay?
775 * There is no definition of what events will count multiple per cycle
776 * and hence to which non 1 values of threshold can apply.
777 * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition)
778 */
779 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK,
780 cxl_pmu_config1_get_threshold(event));
781 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
782
783 local64_set(&hwc->prev_count, 0);
784 writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx));
785
786 perf_event_update_userpage(event);
787 }
788
cxl_pmu_read_counter(struct perf_event * event)789 static u64 cxl_pmu_read_counter(struct perf_event *event)
790 {
791 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
792 void __iomem *base = info->base;
793
794 return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx));
795 }
796
__cxl_pmu_read(struct perf_event * event,bool overflow)797 static void __cxl_pmu_read(struct perf_event *event, bool overflow)
798 {
799 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
800 struct hw_perf_event *hwc = &event->hw;
801 u64 new_cnt, prev_cnt, delta;
802
803 do {
804 prev_cnt = local64_read(&hwc->prev_count);
805 new_cnt = cxl_pmu_read_counter(event);
806 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt);
807
808 /*
809 * If we know an overflow occur then take that into account.
810 * Note counter is not reset as that would lose events
811 */
812 delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0);
813 if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0))
814 delta += (1UL << info->counter_width);
815
816 local64_add(delta, &event->count);
817 }
818
cxl_pmu_read(struct perf_event * event)819 static void cxl_pmu_read(struct perf_event *event)
820 {
821 __cxl_pmu_read(event, false);
822 }
823
cxl_pmu_event_stop(struct perf_event * event,int flags)824 static void cxl_pmu_event_stop(struct perf_event *event, int flags)
825 {
826 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
827 void __iomem *base = info->base;
828 struct hw_perf_event *hwc = &event->hw;
829 u64 cfg;
830
831 cxl_pmu_read(event);
832 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
833 hwc->state |= PERF_HES_STOPPED;
834
835 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
836 cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) |
837 FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1));
838 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
839
840 hwc->state |= PERF_HES_UPTODATE;
841 }
842
cxl_pmu_event_add(struct perf_event * event,int flags)843 static int cxl_pmu_event_add(struct perf_event *event, int flags)
844 {
845 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
846 struct hw_perf_event *hwc = &event->hw;
847 int idx, rc;
848 int event_idx = 0;
849
850 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
851
852 rc = cxl_pmu_get_event_idx(event, &idx, &event_idx);
853 if (rc < 0)
854 return rc;
855
856 hwc->idx = idx;
857
858 /* Only set for configurable counters */
859 hwc->event_base = event_idx;
860 info->hw_events[idx] = event;
861 set_bit(idx, info->used_counter_bm);
862
863 if (flags & PERF_EF_START)
864 cxl_pmu_event_start(event, PERF_EF_RELOAD);
865
866 return 0;
867 }
868
cxl_pmu_event_del(struct perf_event * event,int flags)869 static void cxl_pmu_event_del(struct perf_event *event, int flags)
870 {
871 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
872 struct hw_perf_event *hwc = &event->hw;
873
874 cxl_pmu_event_stop(event, PERF_EF_UPDATE);
875 clear_bit(hwc->idx, info->used_counter_bm);
876 info->hw_events[hwc->idx] = NULL;
877 perf_event_update_userpage(event);
878 }
879
cxl_pmu_irq(int irq,void * data)880 static irqreturn_t cxl_pmu_irq(int irq, void *data)
881 {
882 struct cxl_pmu_info *info = data;
883 void __iomem *base = info->base;
884 u64 overflowed;
885 DECLARE_BITMAP(overflowedbm, 64);
886 int i;
887
888 overflowed = readq(base + CXL_PMU_OVERFLOW_REG);
889
890 /* Interrupt may be shared, so maybe it isn't ours */
891 if (!overflowed)
892 return IRQ_NONE;
893
894 bitmap_from_arr64(overflowedbm, &overflowed, 64);
895 for_each_set_bit(i, overflowedbm, info->num_counters) {
896 struct perf_event *event = info->hw_events[i];
897
898 if (!event) {
899 dev_dbg(info->pmu.dev,
900 "overflow but on non enabled counter %d\n", i);
901 continue;
902 }
903
904 __cxl_pmu_read(event, true);
905 }
906
907 writeq(overflowed, base + CXL_PMU_OVERFLOW_REG);
908
909 return IRQ_HANDLED;
910 }
911
cxl_pmu_perf_unregister(void * _info)912 static void cxl_pmu_perf_unregister(void *_info)
913 {
914 struct cxl_pmu_info *info = _info;
915
916 perf_pmu_unregister(&info->pmu);
917 }
918
cxl_pmu_cpuhp_remove(void * _info)919 static void cxl_pmu_cpuhp_remove(void *_info)
920 {
921 struct cxl_pmu_info *info = _info;
922
923 cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node);
924 }
925
cxl_pmu_probe(struct device * dev)926 static int cxl_pmu_probe(struct device *dev)
927 {
928 struct cxl_pmu *pmu = to_cxl_pmu(dev);
929 struct pci_dev *pdev = to_pci_dev(dev->parent);
930 struct cxl_pmu_info *info;
931 char *irq_name;
932 char *dev_name;
933 int rc, irq;
934
935 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
936 if (!info)
937 return -ENOMEM;
938
939 dev_set_drvdata(dev, info);
940 INIT_LIST_HEAD(&info->event_caps_fixed);
941 INIT_LIST_HEAD(&info->event_caps_configurable);
942
943 info->base = pmu->base;
944
945 info->on_cpu = -1;
946 rc = cxl_pmu_parse_caps(dev, info);
947 if (rc)
948 return rc;
949
950 info->hw_events = devm_kcalloc(dev, info->num_counters,
951 sizeof(*info->hw_events), GFP_KERNEL);
952 if (!info->hw_events)
953 return -ENOMEM;
954
955 switch (pmu->type) {
956 case CXL_PMU_MEMDEV:
957 dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d",
958 pmu->assoc_id, pmu->index);
959 break;
960 }
961 if (!dev_name)
962 return -ENOMEM;
963
964 info->pmu = (struct pmu) {
965 .name = dev_name,
966 .parent = dev,
967 .module = THIS_MODULE,
968 .event_init = cxl_pmu_event_init,
969 .pmu_enable = cxl_pmu_enable,
970 .pmu_disable = cxl_pmu_disable,
971 .add = cxl_pmu_event_add,
972 .del = cxl_pmu_event_del,
973 .start = cxl_pmu_event_start,
974 .stop = cxl_pmu_event_stop,
975 .read = cxl_pmu_read,
976 .task_ctx_nr = perf_invalid_context,
977 .attr_groups = cxl_pmu_attr_groups,
978 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
979 };
980
981 if (info->irq <= 0)
982 return -EINVAL;
983
984 rc = pci_irq_vector(pdev, info->irq);
985 if (rc < 0)
986 return rc;
987 irq = rc;
988
989 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name);
990 if (!irq_name)
991 return -ENOMEM;
992
993 rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_NO_THREAD,
994 irq_name, info);
995 if (rc)
996 return rc;
997 info->irq = irq;
998
999 rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node);
1000 if (rc)
1001 return rc;
1002
1003 rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info);
1004 if (rc)
1005 return rc;
1006
1007 rc = perf_pmu_register(&info->pmu, info->pmu.name, -1);
1008 if (rc)
1009 return rc;
1010
1011 rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info);
1012 if (rc)
1013 return rc;
1014
1015 return 0;
1016 }
1017
1018 static struct cxl_driver cxl_pmu_driver = {
1019 .name = "cxl_pmu",
1020 .probe = cxl_pmu_probe,
1021 .id = CXL_DEVICE_PMU,
1022 };
1023
cxl_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)1024 static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
1025 {
1026 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
1027
1028 if (info->on_cpu != -1)
1029 return 0;
1030
1031 info->on_cpu = cpu;
1032 /*
1033 * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet
1034 * gone away again.
1035 */
1036 WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu)));
1037
1038 return 0;
1039 }
1040
cxl_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)1041 static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1042 {
1043 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
1044 unsigned int target;
1045
1046 if (info->on_cpu != cpu)
1047 return 0;
1048
1049 info->on_cpu = -1;
1050 target = cpumask_any_but(cpu_online_mask, cpu);
1051 if (target >= nr_cpu_ids) {
1052 dev_err(info->pmu.dev, "Unable to find a suitable CPU\n");
1053 return 0;
1054 }
1055
1056 perf_pmu_migrate_context(&info->pmu, cpu, target);
1057 info->on_cpu = target;
1058 /*
1059 * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet
1060 * gone away.
1061 */
1062 WARN_ON(irq_set_affinity(info->irq, cpumask_of(target)));
1063
1064 return 0;
1065 }
1066
cxl_pmu_init(void)1067 static __init int cxl_pmu_init(void)
1068 {
1069 int rc;
1070
1071 rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1072 "AP_PERF_CXL_PMU_ONLINE",
1073 cxl_pmu_online_cpu, cxl_pmu_offline_cpu);
1074 if (rc < 0)
1075 return rc;
1076 cxl_pmu_cpuhp_state_num = rc;
1077
1078 rc = cxl_driver_register(&cxl_pmu_driver);
1079 if (rc)
1080 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
1081
1082 return rc;
1083 }
1084
cxl_pmu_exit(void)1085 static __exit void cxl_pmu_exit(void)
1086 {
1087 cxl_driver_unregister(&cxl_pmu_driver);
1088 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
1089 }
1090
1091 MODULE_DESCRIPTION("CXL Performance Monitor Driver");
1092 MODULE_LICENSE("GPL");
1093 MODULE_IMPORT_NS("CXL");
1094 module_init(cxl_pmu_init);
1095 module_exit(cxl_pmu_exit);
1096 MODULE_ALIAS_CXL(CXL_DEVICE_PMU);
1097