xref: /linux/drivers/perf/cxl_pmu.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /*
4  * Copyright(c) 2023 Huawei
5  *
6  * The CXL 3.0 specification includes a standard Performance Monitoring Unit,
7  * called the CXL PMU, or CPMU. In order to allow a high degree of
8  * implementation flexibility the specification provides a wide range of
9  * options all of which are self describing.
10  *
11  * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface
12  */
13 
14 #include <linux/io-64-nonatomic-lo-hi.h>
15 #include <linux/perf_event.h>
16 #include <linux/bitops.h>
17 #include <linux/device.h>
18 #include <linux/bits.h>
19 #include <linux/list.h>
20 #include <linux/bug.h>
21 #include <linux/pci.h>
22 
23 #include "../cxl/cxlpci.h"
24 #include "../cxl/cxl.h"
25 #include "../cxl/pmu.h"
26 
27 #define CXL_PMU_CAP_REG			0x0
28 #define   CXL_PMU_CAP_NUM_COUNTERS_MSK			GENMASK_ULL(5, 0)
29 #define   CXL_PMU_CAP_COUNTER_WIDTH_MSK			GENMASK_ULL(15, 8)
30 #define   CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK		GENMASK_ULL(24, 20)
31 #define   CXL_PMU_CAP_FILTERS_SUP_MSK			GENMASK_ULL(39, 32)
32 #define     CXL_PMU_FILTER_HDM				BIT(0)
33 #define     CXL_PMU_FILTER_CHAN_RANK_BANK		BIT(1)
34 #define   CXL_PMU_CAP_MSI_N_MSK				GENMASK_ULL(47, 44)
35 #define   CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN		BIT_ULL(48)
36 #define   CXL_PMU_CAP_FREEZE				BIT_ULL(49)
37 #define   CXL_PMU_CAP_INT				BIT_ULL(50)
38 #define   CXL_PMU_CAP_VERSION_MSK			GENMASK_ULL(63, 60)
39 
40 #define CXL_PMU_OVERFLOW_REG		0x10
41 #define CXL_PMU_FREEZE_REG		0x18
42 #define CXL_PMU_EVENT_CAP_REG(n)	(0x100 + 8 * (n))
43 #define   CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK	GENMASK_ULL(31, 0)
44 #define   CXL_PMU_EVENT_CAP_GROUP_ID_MSK		GENMASK_ULL(47, 32)
45 #define   CXL_PMU_EVENT_CAP_VENDOR_ID_MSK		GENMASK_ULL(63, 48)
46 
47 #define CXL_PMU_COUNTER_CFG_REG(n)	(0x200 + 8 * (n))
48 #define   CXL_PMU_COUNTER_CFG_TYPE_MSK			GENMASK_ULL(1, 0)
49 #define     CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN		0
50 #define     CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN		1
51 #define     CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE	2
52 #define   CXL_PMU_COUNTER_CFG_ENABLE			BIT_ULL(8)
53 #define   CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW		BIT_ULL(9)
54 #define   CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW		BIT_ULL(10)
55 #define   CXL_PMU_COUNTER_CFG_EDGE			BIT_ULL(11)
56 #define   CXL_PMU_COUNTER_CFG_INVERT			BIT_ULL(12)
57 #define   CXL_PMU_COUNTER_CFG_THRESHOLD_MSK		GENMASK_ULL(23, 16)
58 #define   CXL_PMU_COUNTER_CFG_EVENTS_MSK		GENMASK_ULL(55, 24)
59 #define   CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK	GENMASK_ULL(63, 59)
60 
61 #define CXL_PMU_FILTER_CFG_REG(n, f)	(0x400 + 4 * ((f) + (n) * 8))
62 #define   CXL_PMU_FILTER_CFG_VALUE_MSK			GENMASK(31, 0)
63 
64 #define CXL_PMU_COUNTER_REG(n)		(0xc00 + 8 * (n))
65 
66 /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */
67 #define CXL_PMU_GID_CLOCK_TICKS		0x00
68 #define CXL_PMU_GID_D2H_REQ		0x0010
69 #define CXL_PMU_GID_D2H_RSP		0x0011
70 #define CXL_PMU_GID_H2D_REQ		0x0012
71 #define CXL_PMU_GID_H2D_RSP		0x0013
72 #define CXL_PMU_GID_CACHE_DATA		0x0014
73 #define CXL_PMU_GID_M2S_REQ		0x0020
74 #define CXL_PMU_GID_M2S_RWD		0x0021
75 #define CXL_PMU_GID_M2S_BIRSP		0x0022
76 #define CXL_PMU_GID_S2M_BISNP		0x0023
77 #define CXL_PMU_GID_S2M_NDR		0x0024
78 #define CXL_PMU_GID_S2M_DRS		0x0025
79 #define CXL_PMU_GID_DDR			0x8000
80 
81 static int cxl_pmu_cpuhp_state_num;
82 
83 struct cxl_pmu_ev_cap {
84 	u16 vid;
85 	u16 gid;
86 	u32 msk;
87 	union {
88 		int counter_idx; /* fixed counters */
89 		int event_idx; /* configurable counters */
90 	};
91 	struct list_head node;
92 };
93 
94 #define CXL_PMU_MAX_COUNTERS 64
95 struct cxl_pmu_info {
96 	struct pmu pmu;
97 	void __iomem *base;
98 	struct perf_event **hw_events;
99 	struct list_head event_caps_configurable;
100 	struct list_head event_caps_fixed;
101 	DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS);
102 	DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS);
103 	u16 counter_width;
104 	u8 num_counters;
105 	u8 num_event_capabilities;
106 	int on_cpu;
107 	struct hlist_node node;
108 	bool filter_hdm;
109 	int irq;
110 };
111 
112 #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu)
113 
114 /*
115  * All CPMU counters are discoverable via the Event Capabilities Registers.
116  * Each Event Capability register contains a a VID / GroupID.
117  * A counter may then count any combination (by summing) of events in
118  * that group which are in the Supported Events Bitmask.
119  * However, there are some complexities to the scheme.
120  *  - Fixed function counters refer to an Event Capabilities register.
121  *    That event capability register is not then used for Configurable
122  *    counters.
123  */
cxl_pmu_parse_caps(struct device * dev,struct cxl_pmu_info * info)124 static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info)
125 {
126 	unsigned long fixed_counter_event_cap_bm = 0;
127 	void __iomem *base = info->base;
128 	bool freeze_for_enable;
129 	u64 val, eval;
130 	int i;
131 
132 	val = readq(base + CXL_PMU_CAP_REG);
133 	freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) &&
134 		FIELD_GET(CXL_PMU_CAP_FREEZE, val);
135 	if (!freeze_for_enable) {
136 		dev_err(dev, "Counters not writable while frozen\n");
137 		return -ENODEV;
138 	}
139 
140 	info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1;
141 	info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val);
142 	info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1;
143 
144 	info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM;
145 	if (FIELD_GET(CXL_PMU_CAP_INT, val))
146 		info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val);
147 	else
148 		info->irq = -1;
149 
150 	/* First handle fixed function counters; note if configurable counters found */
151 	for (i = 0; i < info->num_counters; i++) {
152 		struct cxl_pmu_ev_cap *pmu_ev;
153 		u32 events_msk;
154 		u8 group_idx;
155 
156 		val = readq(base + CXL_PMU_COUNTER_CFG_REG(i));
157 
158 		if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) ==
159 			CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) {
160 			set_bit(i, info->conf_counter_bm);
161 		}
162 
163 		if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) !=
164 		    CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN)
165 			continue;
166 
167 		/* In this case we know which fields are const */
168 		group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val);
169 		events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val);
170 		eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx));
171 		pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
172 		if (!pmu_ev)
173 			return -ENOMEM;
174 
175 		pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
176 		pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
177 		/* For a fixed purpose counter use the events mask from the counter CFG */
178 		pmu_ev->msk = events_msk;
179 		pmu_ev->counter_idx = i;
180 		/* This list add is never unwound as all entries deleted on remove */
181 		list_add(&pmu_ev->node, &info->event_caps_fixed);
182 		/*
183 		 * Configurable counters must not use an Event Capability registers that
184 		 * is in use for a Fixed counter
185 		 */
186 		set_bit(group_idx, &fixed_counter_event_cap_bm);
187 	}
188 
189 	if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) {
190 		struct cxl_pmu_ev_cap *pmu_ev;
191 		int j;
192 		/* Walk event capabilities unused by fixed counters */
193 		for_each_clear_bit(j, &fixed_counter_event_cap_bm,
194 				   info->num_event_capabilities) {
195 			pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
196 			if (!pmu_ev)
197 				return -ENOMEM;
198 
199 			eval = readq(base + CXL_PMU_EVENT_CAP_REG(j));
200 			pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
201 			pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
202 			pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval);
203 			pmu_ev->event_idx = j;
204 			list_add(&pmu_ev->node, &info->event_caps_configurable);
205 		}
206 	}
207 
208 	return 0;
209 }
210 
211 #define CXL_PMU_FORMAT_ATTR(_name, _format)\
212 	(&((struct dev_ext_attribute[]) {					\
213 		{								\
214 			.attr = __ATTR(_name, 0444, device_show_string, NULL),	\
215 			.var = (void *)_format					\
216 		}								\
217 		})[0].attr.attr)
218 
219 enum {
220 	cxl_pmu_mask_attr,
221 	cxl_pmu_gid_attr,
222 	cxl_pmu_vid_attr,
223 	cxl_pmu_threshold_attr,
224 	cxl_pmu_invert_attr,
225 	cxl_pmu_edge_attr,
226 	cxl_pmu_hdm_filter_en_attr,
227 	cxl_pmu_hdm_attr,
228 };
229 
230 static struct attribute *cxl_pmu_format_attr[] = {
231 	[cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"),
232 	[cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"),
233 	[cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"),
234 	[cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"),
235 	[cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"),
236 	[cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"),
237 	[cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"),
238 	[cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"),
239 	NULL
240 };
241 
242 #define CXL_PMU_ATTR_CONFIG_MASK_MSK		GENMASK_ULL(31, 0)
243 #define CXL_PMU_ATTR_CONFIG_GID_MSK		GENMASK_ULL(47, 32)
244 #define CXL_PMU_ATTR_CONFIG_VID_MSK		GENMASK_ULL(63, 48)
245 #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK	GENMASK_ULL(15, 0)
246 #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK		BIT(16)
247 #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK		BIT(17)
248 #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK	BIT(18)
249 #define CXL_PMU_ATTR_CONFIG2_HDM_MSK		GENMASK(15, 0)
250 
cxl_pmu_format_is_visible(struct kobject * kobj,struct attribute * attr,int a)251 static umode_t cxl_pmu_format_is_visible(struct kobject *kobj,
252 					 struct attribute *attr, int a)
253 {
254 	struct device *dev = kobj_to_dev(kobj);
255 	struct cxl_pmu_info *info = dev_get_drvdata(dev);
256 
257 	/*
258 	 * Filter capability at the CPMU level, so hide the attributes if the particular
259 	 * filter is not supported.
260 	 */
261 	if (!info->filter_hdm &&
262 	    (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] ||
263 	     attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr]))
264 		return 0;
265 
266 	return attr->mode;
267 }
268 
269 static const struct attribute_group cxl_pmu_format_group = {
270 	.name = "format",
271 	.attrs = cxl_pmu_format_attr,
272 	.is_visible = cxl_pmu_format_is_visible,
273 };
274 
cxl_pmu_config_get_mask(struct perf_event * event)275 static u32 cxl_pmu_config_get_mask(struct perf_event *event)
276 {
277 	return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config);
278 }
279 
cxl_pmu_config_get_gid(struct perf_event * event)280 static u16 cxl_pmu_config_get_gid(struct perf_event *event)
281 {
282 	return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config);
283 }
284 
cxl_pmu_config_get_vid(struct perf_event * event)285 static u16 cxl_pmu_config_get_vid(struct perf_event *event)
286 {
287 	return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config);
288 }
289 
cxl_pmu_config1_get_threshold(struct perf_event * event)290 static u8 cxl_pmu_config1_get_threshold(struct perf_event *event)
291 {
292 	return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1);
293 }
294 
cxl_pmu_config1_get_invert(struct perf_event * event)295 static bool cxl_pmu_config1_get_invert(struct perf_event *event)
296 {
297 	return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1);
298 }
299 
cxl_pmu_config1_get_edge(struct perf_event * event)300 static bool cxl_pmu_config1_get_edge(struct perf_event *event)
301 {
302 	return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1);
303 }
304 
305 /*
306  * CPMU specification allows for 8 filters, each with a 32 bit value...
307  * So we need to find 8x32bits to store it in.
308  * As the value used for disable is 0xffff_ffff, a separate enable switch
309  * is needed.
310  */
311 
cxl_pmu_config1_hdm_filter_en(struct perf_event * event)312 static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event)
313 {
314 	return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1);
315 }
316 
cxl_pmu_config2_get_hdm_decoder(struct perf_event * event)317 static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event)
318 {
319 	return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2);
320 }
321 
cxl_pmu_event_sysfs_show(struct device * dev,struct device_attribute * attr,char * buf)322 static ssize_t cxl_pmu_event_sysfs_show(struct device *dev,
323 					struct device_attribute *attr, char *buf)
324 {
325 	struct perf_pmu_events_attr *pmu_attr =
326 		container_of(attr, struct perf_pmu_events_attr, attr);
327 
328 	return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id);
329 }
330 
331 #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk)			\
332 	PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show,		\
333 			  ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk))
334 
335 /* For CXL spec defined events */
336 #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk)			\
337 	CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk)
338 
339 static struct attribute *cxl_pmu_event_attrs[] = {
340 	CXL_PMU_EVENT_CXL_ATTR(clock_ticks,			CXL_PMU_GID_CLOCK_TICKS, BIT(0)),
341 	/* CXL rev 3.0 Table 3-17 - Device to Host Requests */
342 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr,			CXL_PMU_GID_D2H_REQ, BIT(1)),
343 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown,			CXL_PMU_GID_D2H_REQ, BIT(2)),
344 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared,		CXL_PMU_GID_D2H_REQ, BIT(3)),
345 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany,			CXL_PMU_GID_D2H_REQ, BIT(4)),
346 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata,		CXL_PMU_GID_D2H_REQ, BIT(5)),
347 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr,			CXL_PMU_GID_D2H_REQ, BIT(6)),
348 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr,			CXL_PMU_GID_D2H_REQ, BIT(7)),
349 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush,			CXL_PMU_GID_D2H_REQ, BIT(8)),
350 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict,		CXL_PMU_GID_D2H_REQ, BIT(9)),
351 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict,		CXL_PMU_GID_D2H_REQ, BIT(10)),
352 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata,	CXL_PMU_GID_D2H_REQ, BIT(11)),
353 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv,			CXL_PMU_GID_D2H_REQ, BIT(12)),
354 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf,		CXL_PMU_GID_D2H_REQ, BIT(13)),
355 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv,			CXL_PMU_GID_D2H_REQ, BIT(14)),
356 	CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed,		CXL_PMU_GID_D2H_REQ, BIT(16)),
357 	/* CXL rev 3.0 Table 3-20 - D2H Repsonse Encodings */
358 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti,		CXL_PMU_GID_D2H_RSP, BIT(4)),
359 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv,		CXL_PMU_GID_D2H_RSP, BIT(6)),
360 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse,		CXL_PMU_GID_D2H_RSP, BIT(5)),
361 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse,		CXL_PMU_GID_D2H_RSP, BIT(1)),
362 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm,		CXL_PMU_GID_D2H_RSP, BIT(7)),
363 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm,		CXL_PMU_GID_D2H_RSP, BIT(15)),
364 	CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv,		CXL_PMU_GID_D2H_RSP, BIT(22)),
365 	/* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */
366 	CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata,			CXL_PMU_GID_H2D_REQ, BIT(1)),
367 	CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv,			CXL_PMU_GID_H2D_REQ, BIT(2)),
368 	CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur,			CXL_PMU_GID_H2D_REQ, BIT(3)),
369 	/* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */
370 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull,		CXL_PMU_GID_H2D_RSP, BIT(1)),
371 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go,			CXL_PMU_GID_H2D_RSP, BIT(4)),
372 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull,		CXL_PMU_GID_H2D_RSP, BIT(5)),
373 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp,			CXL_PMU_GID_H2D_RSP, BIT(6)),
374 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop,		CXL_PMU_GID_H2D_RSP, BIT(8)),
375 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull,		CXL_PMU_GID_H2D_RSP, BIT(13)),
376 	CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull,		CXL_PMU_GID_H2D_RSP, BIT(15)),
377 	/* CXL rev 3.0 Table 13-5 directly lists these */
378 	CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data,		CXL_PMU_GID_CACHE_DATA, BIT(0)),
379 	CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data,		CXL_PMU_GID_CACHE_DATA, BIT(1)),
380 	/* CXL rev 3.0 Table 3-29 M2S Req Memory Opcodes */
381 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv,			CXL_PMU_GID_M2S_REQ, BIT(0)),
382 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd,			CXL_PMU_GID_M2S_REQ, BIT(1)),
383 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata,		CXL_PMU_GID_M2S_REQ, BIT(2)),
384 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd,		CXL_PMU_GID_M2S_REQ, BIT(3)),
385 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd,		CXL_PMU_GID_M2S_REQ, BIT(4)),
386 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd,		CXL_PMU_GID_M2S_REQ, BIT(8)),
387 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt,		CXL_PMU_GID_M2S_REQ, BIT(9)),
388 	CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict,		CXL_PMU_GID_M2S_REQ, BIT(10)),
389 	/* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */
390 	CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr,			CXL_PMU_GID_M2S_RWD, BIT(1)),
391 	CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl,		CXL_PMU_GID_M2S_RWD, BIT(2)),
392 	CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict,		CXL_PMU_GID_M2S_RWD, BIT(4)),
393 	/* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */
394 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i,			CXL_PMU_GID_M2S_BIRSP, BIT(0)),
395 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s,			CXL_PMU_GID_M2S_BIRSP, BIT(1)),
396 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e,			CXL_PMU_GID_M2S_BIRSP, BIT(2)),
397 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk,			CXL_PMU_GID_M2S_BIRSP, BIT(4)),
398 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk,			CXL_PMU_GID_M2S_BIRSP, BIT(5)),
399 	CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk,			CXL_PMU_GID_M2S_BIRSP, BIT(6)),
400 	/* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */
401 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur,			CXL_PMU_GID_S2M_BISNP, BIT(0)),
402 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data,			CXL_PMU_GID_S2M_BISNP, BIT(1)),
403 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv,			CXL_PMU_GID_S2M_BISNP, BIT(2)),
404 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk,		CXL_PMU_GID_S2M_BISNP, BIT(4)),
405 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk,		CXL_PMU_GID_S2M_BISNP, BIT(5)),
406 	CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk,		CXL_PMU_GID_S2M_BISNP, BIT(6)),
407 	/* CXL rev 3.0 Table 3-43 S2M NDR Opcopdes */
408 	CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp,			CXL_PMU_GID_S2M_NDR, BIT(0)),
409 	CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps,			CXL_PMU_GID_S2M_NDR, BIT(1)),
410 	CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe,			CXL_PMU_GID_S2M_NDR, BIT(2)),
411 	CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack,		CXL_PMU_GID_S2M_NDR, BIT(4)),
412 	/* CXL rev 3.0 Table 3-46 S2M DRS opcodes */
413 	CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata,			CXL_PMU_GID_S2M_DRS, BIT(0)),
414 	CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm,		CXL_PMU_GID_S2M_DRS, BIT(1)),
415 	/* CXL rev 3.0 Table 13-5 directly lists these */
416 	CXL_PMU_EVENT_CXL_ATTR(ddr_act,				CXL_PMU_GID_DDR, BIT(0)),
417 	CXL_PMU_EVENT_CXL_ATTR(ddr_pre,				CXL_PMU_GID_DDR, BIT(1)),
418 	CXL_PMU_EVENT_CXL_ATTR(ddr_casrd,			CXL_PMU_GID_DDR, BIT(2)),
419 	CXL_PMU_EVENT_CXL_ATTR(ddr_caswr,			CXL_PMU_GID_DDR, BIT(3)),
420 	CXL_PMU_EVENT_CXL_ATTR(ddr_refresh,			CXL_PMU_GID_DDR, BIT(4)),
421 	CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent,		CXL_PMU_GID_DDR, BIT(5)),
422 	CXL_PMU_EVENT_CXL_ATTR(ddr_rfm,				CXL_PMU_GID_DDR, BIT(6)),
423 	NULL
424 };
425 
cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info * info,int vid,int gid,int msk)426 static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info,
427 								int vid, int gid, int msk)
428 {
429 	struct cxl_pmu_ev_cap *pmu_ev;
430 
431 	list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) {
432 		if (vid != pmu_ev->vid || gid != pmu_ev->gid)
433 			continue;
434 
435 		/* Precise match for fixed counter */
436 		if (msk == pmu_ev->msk)
437 			return pmu_ev;
438 	}
439 
440 	return ERR_PTR(-EINVAL);
441 }
442 
cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info * info,int vid,int gid,int msk)443 static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info,
444 								 int vid, int gid, int msk)
445 {
446 	struct cxl_pmu_ev_cap *pmu_ev;
447 
448 	list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) {
449 		if (vid != pmu_ev->vid || gid != pmu_ev->gid)
450 			continue;
451 
452 		/* Request mask must be subset of supported */
453 		if (msk & ~pmu_ev->msk)
454 			continue;
455 
456 		return pmu_ev;
457 	}
458 
459 	return ERR_PTR(-EINVAL);
460 }
461 
cxl_pmu_event_is_visible(struct kobject * kobj,struct attribute * attr,int a)462 static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a)
463 {
464 	struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr);
465 	struct perf_pmu_events_attr *pmu_attr =
466 		container_of(dev_attr, struct perf_pmu_events_attr, attr);
467 	struct device *dev = kobj_to_dev(kobj);
468 	struct cxl_pmu_info *info = dev_get_drvdata(dev);
469 	int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id);
470 	int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id);
471 	int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id);
472 
473 	if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk)))
474 		return attr->mode;
475 
476 	if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk)))
477 		return attr->mode;
478 
479 	return 0;
480 }
481 
482 static const struct attribute_group cxl_pmu_events = {
483 	.name = "events",
484 	.attrs = cxl_pmu_event_attrs,
485 	.is_visible = cxl_pmu_event_is_visible,
486 };
487 
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)488 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
489 			    char *buf)
490 {
491 	struct cxl_pmu_info *info = dev_get_drvdata(dev);
492 
493 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(info->on_cpu));
494 }
495 static DEVICE_ATTR_RO(cpumask);
496 
497 static struct attribute *cxl_pmu_cpumask_attrs[] = {
498 	&dev_attr_cpumask.attr,
499 	NULL
500 };
501 
502 static const struct attribute_group cxl_pmu_cpumask_group = {
503 	.attrs = cxl_pmu_cpumask_attrs,
504 };
505 
506 static const struct attribute_group *cxl_pmu_attr_groups[] = {
507 	&cxl_pmu_events,
508 	&cxl_pmu_format_group,
509 	&cxl_pmu_cpumask_group,
510 	NULL
511 };
512 
513 /* If counter_idx == NULL, don't try to allocate a counter. */
cxl_pmu_get_event_idx(struct perf_event * event,int * counter_idx,int * event_idx)514 static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx,
515 				 int *event_idx)
516 {
517 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
518 	DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS);
519 	struct cxl_pmu_ev_cap *pmu_ev;
520 	u32 mask;
521 	u16 gid, vid;
522 	int i;
523 
524 	vid = cxl_pmu_config_get_vid(event);
525 	gid = cxl_pmu_config_get_gid(event);
526 	mask = cxl_pmu_config_get_mask(event);
527 
528 	pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask);
529 	if (!IS_ERR(pmu_ev)) {
530 		if (!counter_idx)
531 			return 0;
532 		if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) {
533 			*counter_idx = pmu_ev->counter_idx;
534 			return 0;
535 		}
536 		/* Fixed counter is in use, but maybe a configurable one? */
537 	}
538 
539 	pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask);
540 	if (!IS_ERR(pmu_ev)) {
541 		if (!counter_idx)
542 			return 0;
543 
544 		bitmap_andnot(configurable_and_free, info->conf_counter_bm,
545 			info->used_counter_bm, CXL_PMU_MAX_COUNTERS);
546 
547 		i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS);
548 		if (i == CXL_PMU_MAX_COUNTERS)
549 			return -EINVAL;
550 
551 		*counter_idx = i;
552 		return 0;
553 	}
554 
555 	return -EINVAL;
556 }
557 
cxl_pmu_event_init(struct perf_event * event)558 static int cxl_pmu_event_init(struct perf_event *event)
559 {
560 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
561 	int rc;
562 
563 	/* Top level type sanity check - is this a Hardware Event being requested */
564 	if (event->attr.type != event->pmu->type)
565 		return -ENOENT;
566 
567 	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
568 		return -EOPNOTSUPP;
569 	/* TODO: Validation of any filter */
570 
571 	/*
572 	 * Verify that it is possible to count what was requested. Either must
573 	 * be a fixed counter that is a precise match or a configurable counter
574 	 * where this is a subset.
575 	 */
576 	rc = cxl_pmu_get_event_idx(event, NULL, NULL);
577 	if (rc < 0)
578 		return rc;
579 
580 	event->cpu = info->on_cpu;
581 
582 	return 0;
583 }
584 
cxl_pmu_enable(struct pmu * pmu)585 static void cxl_pmu_enable(struct pmu *pmu)
586 {
587 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
588 	void __iomem *base = info->base;
589 
590 	/* Can assume frozen at this stage */
591 	writeq(0, base + CXL_PMU_FREEZE_REG);
592 }
593 
cxl_pmu_disable(struct pmu * pmu)594 static void cxl_pmu_disable(struct pmu *pmu)
595 {
596 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
597 	void __iomem *base = info->base;
598 
599 	/*
600 	 * Whilst bits above number of counters are RsvdZ
601 	 * they are unlikely to be repurposed given
602 	 * number of counters is allowed to be 64 leaving
603 	 * no reserved bits.  Hence this is only slightly
604 	 * naughty.
605 	 */
606 	writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG);
607 }
608 
cxl_pmu_event_start(struct perf_event * event,int flags)609 static void cxl_pmu_event_start(struct perf_event *event, int flags)
610 {
611 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
612 	struct hw_perf_event *hwc = &event->hw;
613 	void __iomem *base = info->base;
614 	u64 cfg;
615 
616 	/*
617 	 * All paths to here should either set these flags directly or
618 	 * call cxl_pmu_event_stop() which will ensure the correct state.
619 	 */
620 	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
621 		return;
622 
623 	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
624 	hwc->state = 0;
625 
626 	/*
627 	 * Currently only hdm filter control is implemnted, this code will
628 	 * want generalizing when more filters are added.
629 	 */
630 	if (info->filter_hdm) {
631 		if (cxl_pmu_config1_hdm_filter_en(event))
632 			cfg = cxl_pmu_config2_get_hdm_decoder(event);
633 		else
634 			cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */
635 		writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0));
636 	}
637 
638 	cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
639 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1);
640 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1);
641 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1);
642 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE,
643 			  cxl_pmu_config1_get_edge(event) ? 1 : 0);
644 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT,
645 			  cxl_pmu_config1_get_invert(event) ? 1 : 0);
646 
647 	/* Fixed purpose counters have next two fields RO */
648 	if (test_bit(hwc->idx, info->conf_counter_bm)) {
649 		cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK,
650 				  hwc->event_base);
651 		cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK,
652 				  cxl_pmu_config_get_mask(event));
653 	}
654 	cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK;
655 	/*
656 	 * For events that generate only 1 count per clock the CXL 3.0 spec
657 	 * states the threshold shall be set to 1 but if set to 0 it will
658 	 * count the raw value anwyay?
659 	 * There is no definition of what events will count multiple per cycle
660 	 * and hence to which non 1 values of threshold can apply.
661 	 * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition)
662 	 */
663 	cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK,
664 			  cxl_pmu_config1_get_threshold(event));
665 	writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
666 
667 	local64_set(&hwc->prev_count, 0);
668 	writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx));
669 
670 	perf_event_update_userpage(event);
671 }
672 
cxl_pmu_read_counter(struct perf_event * event)673 static u64 cxl_pmu_read_counter(struct perf_event *event)
674 {
675 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
676 	void __iomem *base = info->base;
677 
678 	return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx));
679 }
680 
__cxl_pmu_read(struct perf_event * event,bool overflow)681 static void __cxl_pmu_read(struct perf_event *event, bool overflow)
682 {
683 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
684 	struct hw_perf_event *hwc = &event->hw;
685 	u64 new_cnt, prev_cnt, delta;
686 
687 	do {
688 		prev_cnt = local64_read(&hwc->prev_count);
689 		new_cnt = cxl_pmu_read_counter(event);
690 	} while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt);
691 
692 	/*
693 	 * If we know an overflow occur then take that into account.
694 	 * Note counter is not reset as that would lose events
695 	 */
696 	delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0);
697 	if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0))
698 		delta += (1UL << info->counter_width);
699 
700 	local64_add(delta, &event->count);
701 }
702 
cxl_pmu_read(struct perf_event * event)703 static void cxl_pmu_read(struct perf_event *event)
704 {
705 	__cxl_pmu_read(event, false);
706 }
707 
cxl_pmu_event_stop(struct perf_event * event,int flags)708 static void cxl_pmu_event_stop(struct perf_event *event, int flags)
709 {
710 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
711 	void __iomem *base = info->base;
712 	struct hw_perf_event *hwc = &event->hw;
713 	u64 cfg;
714 
715 	cxl_pmu_read(event);
716 	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
717 	hwc->state |= PERF_HES_STOPPED;
718 
719 	cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
720 	cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) |
721 		 FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1));
722 	writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
723 
724 	hwc->state |= PERF_HES_UPTODATE;
725 }
726 
cxl_pmu_event_add(struct perf_event * event,int flags)727 static int cxl_pmu_event_add(struct perf_event *event, int flags)
728 {
729 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
730 	struct hw_perf_event *hwc = &event->hw;
731 	int idx, rc;
732 	int event_idx = 0;
733 
734 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
735 
736 	rc = cxl_pmu_get_event_idx(event, &idx, &event_idx);
737 	if (rc < 0)
738 		return rc;
739 
740 	hwc->idx = idx;
741 
742 	/* Only set for configurable counters */
743 	hwc->event_base = event_idx;
744 	info->hw_events[idx] = event;
745 	set_bit(idx, info->used_counter_bm);
746 
747 	if (flags & PERF_EF_START)
748 		cxl_pmu_event_start(event, PERF_EF_RELOAD);
749 
750 	return 0;
751 }
752 
cxl_pmu_event_del(struct perf_event * event,int flags)753 static void cxl_pmu_event_del(struct perf_event *event, int flags)
754 {
755 	struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
756 	struct hw_perf_event *hwc = &event->hw;
757 
758 	cxl_pmu_event_stop(event, PERF_EF_UPDATE);
759 	clear_bit(hwc->idx, info->used_counter_bm);
760 	info->hw_events[hwc->idx] = NULL;
761 	perf_event_update_userpage(event);
762 }
763 
cxl_pmu_irq(int irq,void * data)764 static irqreturn_t cxl_pmu_irq(int irq, void *data)
765 {
766 	struct cxl_pmu_info *info = data;
767 	void __iomem *base = info->base;
768 	u64 overflowed;
769 	DECLARE_BITMAP(overflowedbm, 64);
770 	int i;
771 
772 	overflowed = readq(base + CXL_PMU_OVERFLOW_REG);
773 
774 	/* Interrupt may be shared, so maybe it isn't ours */
775 	if (!overflowed)
776 		return IRQ_NONE;
777 
778 	bitmap_from_arr64(overflowedbm, &overflowed, 64);
779 	for_each_set_bit(i, overflowedbm, info->num_counters) {
780 		struct perf_event *event = info->hw_events[i];
781 
782 		if (!event) {
783 			dev_dbg(info->pmu.dev,
784 				"overflow but on non enabled counter %d\n", i);
785 			continue;
786 		}
787 
788 		__cxl_pmu_read(event, true);
789 	}
790 
791 	writeq(overflowed, base + CXL_PMU_OVERFLOW_REG);
792 
793 	return IRQ_HANDLED;
794 }
795 
cxl_pmu_perf_unregister(void * _info)796 static void cxl_pmu_perf_unregister(void *_info)
797 {
798 	struct cxl_pmu_info *info = _info;
799 
800 	perf_pmu_unregister(&info->pmu);
801 }
802 
cxl_pmu_cpuhp_remove(void * _info)803 static void cxl_pmu_cpuhp_remove(void *_info)
804 {
805 	struct cxl_pmu_info *info = _info;
806 
807 	cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node);
808 }
809 
cxl_pmu_probe(struct device * dev)810 static int cxl_pmu_probe(struct device *dev)
811 {
812 	struct cxl_pmu *pmu = to_cxl_pmu(dev);
813 	struct pci_dev *pdev = to_pci_dev(dev->parent);
814 	struct cxl_pmu_info *info;
815 	char *irq_name;
816 	char *dev_name;
817 	int rc, irq;
818 
819 	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
820 	if (!info)
821 		return -ENOMEM;
822 
823 	dev_set_drvdata(dev, info);
824 	INIT_LIST_HEAD(&info->event_caps_fixed);
825 	INIT_LIST_HEAD(&info->event_caps_configurable);
826 
827 	info->base = pmu->base;
828 
829 	info->on_cpu = -1;
830 	rc = cxl_pmu_parse_caps(dev, info);
831 	if (rc)
832 		return rc;
833 
834 	info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events),
835 				       info->num_counters, GFP_KERNEL);
836 	if (!info->hw_events)
837 		return -ENOMEM;
838 
839 	switch (pmu->type) {
840 	case CXL_PMU_MEMDEV:
841 		dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d",
842 					  pmu->assoc_id, pmu->index);
843 		break;
844 	}
845 	if (!dev_name)
846 		return -ENOMEM;
847 
848 	info->pmu = (struct pmu) {
849 		.name = dev_name,
850 		.parent = dev,
851 		.module = THIS_MODULE,
852 		.event_init = cxl_pmu_event_init,
853 		.pmu_enable = cxl_pmu_enable,
854 		.pmu_disable = cxl_pmu_disable,
855 		.add = cxl_pmu_event_add,
856 		.del = cxl_pmu_event_del,
857 		.start = cxl_pmu_event_start,
858 		.stop = cxl_pmu_event_stop,
859 		.read = cxl_pmu_read,
860 		.task_ctx_nr = perf_invalid_context,
861 		.attr_groups = cxl_pmu_attr_groups,
862 		.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
863 	};
864 
865 	if (info->irq <= 0)
866 		return -EINVAL;
867 
868 	rc = pci_irq_vector(pdev, info->irq);
869 	if (rc < 0)
870 		return rc;
871 	irq = rc;
872 
873 	irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name);
874 	if (!irq_name)
875 		return -ENOMEM;
876 
877 	rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT,
878 			      irq_name, info);
879 	if (rc)
880 		return rc;
881 	info->irq = irq;
882 
883 	rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node);
884 	if (rc)
885 		return rc;
886 
887 	rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info);
888 	if (rc)
889 		return rc;
890 
891 	rc = perf_pmu_register(&info->pmu, info->pmu.name, -1);
892 	if (rc)
893 		return rc;
894 
895 	rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info);
896 	if (rc)
897 		return rc;
898 
899 	return 0;
900 }
901 
902 static struct cxl_driver cxl_pmu_driver = {
903 	.name = "cxl_pmu",
904 	.probe = cxl_pmu_probe,
905 	.id = CXL_DEVICE_PMU,
906 };
907 
cxl_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)908 static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
909 {
910 	struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
911 
912 	if (info->on_cpu != -1)
913 		return 0;
914 
915 	info->on_cpu = cpu;
916 	/*
917 	 * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet
918 	 * gone away again.
919 	 */
920 	WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu)));
921 
922 	return 0;
923 }
924 
cxl_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)925 static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
926 {
927 	struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
928 	unsigned int target;
929 
930 	if (info->on_cpu != cpu)
931 		return 0;
932 
933 	info->on_cpu = -1;
934 	target = cpumask_any_but(cpu_online_mask, cpu);
935 	if (target >= nr_cpu_ids) {
936 		dev_err(info->pmu.dev, "Unable to find a suitable CPU\n");
937 		return 0;
938 	}
939 
940 	perf_pmu_migrate_context(&info->pmu, cpu, target);
941 	info->on_cpu = target;
942 	/*
943 	 * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet
944 	 * gone away.
945 	 */
946 	WARN_ON(irq_set_affinity(info->irq, cpumask_of(target)));
947 
948 	return 0;
949 }
950 
cxl_pmu_init(void)951 static __init int cxl_pmu_init(void)
952 {
953 	int rc;
954 
955 	rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
956 				     "AP_PERF_CXL_PMU_ONLINE",
957 				     cxl_pmu_online_cpu, cxl_pmu_offline_cpu);
958 	if (rc < 0)
959 		return rc;
960 	cxl_pmu_cpuhp_state_num = rc;
961 
962 	rc = cxl_driver_register(&cxl_pmu_driver);
963 	if (rc)
964 		cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
965 
966 	return rc;
967 }
968 
cxl_pmu_exit(void)969 static __exit void cxl_pmu_exit(void)
970 {
971 	cxl_driver_unregister(&cxl_pmu_driver);
972 	cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
973 }
974 
975 MODULE_DESCRIPTION("CXL Performance Monitor Driver");
976 MODULE_LICENSE("GPL");
977 MODULE_IMPORT_NS(CXL);
978 module_init(cxl_pmu_init);
979 module_exit(cxl_pmu_exit);
980 MODULE_ALIAS_CXL(CXL_DEVICE_PMU);
981