1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright(c) 2023 Huawei 5 * 6 * The CXL 3.0 specification includes a standard Performance Monitoring Unit, 7 * called the CXL PMU, or CPMU. In order to allow a high degree of 8 * implementation flexibility the specification provides a wide range of 9 * options all of which are self describing. 10 * 11 * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface 12 */ 13 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/perf_event.h> 16 #include <linux/bitops.h> 17 #include <linux/device.h> 18 #include <linux/bits.h> 19 #include <linux/list.h> 20 #include <linux/bug.h> 21 #include <linux/pci.h> 22 23 #include "../cxl/cxlpci.h" 24 #include "../cxl/cxl.h" 25 #include "../cxl/pmu.h" 26 27 #define CXL_PMU_CAP_REG 0x0 28 #define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(5, 0) 29 #define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8) 30 #define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20) 31 #define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32) 32 #define CXL_PMU_FILTER_HDM BIT(0) 33 #define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1) 34 #define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44) 35 #define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48) 36 #define CXL_PMU_CAP_FREEZE BIT_ULL(49) 37 #define CXL_PMU_CAP_INT BIT_ULL(50) 38 #define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60) 39 40 #define CXL_PMU_OVERFLOW_REG 0x10 41 #define CXL_PMU_FREEZE_REG 0x18 42 #define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n)) 43 #define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0) 44 #define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32) 45 #define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48) 46 47 #define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n)) 48 #define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0) 49 #define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0 50 #define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1 51 #define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2 52 #define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8) 53 #define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9) 54 #define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10) 55 #define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11) 56 #define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12) 57 #define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16) 58 #define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24) 59 #define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59) 60 61 #define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8)) 62 #define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(31, 0) 63 64 #define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n)) 65 66 /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */ 67 #define CXL_PMU_GID_CLOCK_TICKS 0x00 68 #define CXL_PMU_GID_D2H_REQ 0x0010 69 #define CXL_PMU_GID_D2H_RSP 0x0011 70 #define CXL_PMU_GID_H2D_REQ 0x0012 71 #define CXL_PMU_GID_H2D_RSP 0x0013 72 #define CXL_PMU_GID_CACHE_DATA 0x0014 73 #define CXL_PMU_GID_M2S_REQ 0x0020 74 #define CXL_PMU_GID_M2S_RWD 0x0021 75 #define CXL_PMU_GID_M2S_BIRSP 0x0022 76 #define CXL_PMU_GID_S2M_BISNP 0x0023 77 #define CXL_PMU_GID_S2M_NDR 0x0024 78 #define CXL_PMU_GID_S2M_DRS 0x0025 79 #define CXL_PMU_GID_DDR 0x8000 80 81 static int cxl_pmu_cpuhp_state_num; 82 83 struct cxl_pmu_ev_cap { 84 u16 vid; 85 u16 gid; 86 u32 msk; 87 union { 88 int counter_idx; /* fixed counters */ 89 int event_idx; /* configurable counters */ 90 }; 91 struct list_head node; 92 }; 93 94 #define CXL_PMU_MAX_COUNTERS 64 95 struct cxl_pmu_info { 96 struct pmu pmu; 97 void __iomem *base; 98 struct perf_event **hw_events; 99 struct list_head event_caps_configurable; 100 struct list_head event_caps_fixed; 101 DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS); 102 DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS); 103 u16 counter_width; 104 u8 num_counters; 105 u8 num_event_capabilities; 106 int on_cpu; 107 struct hlist_node node; 108 bool filter_hdm; 109 int irq; 110 }; 111 112 #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu) 113 114 /* 115 * All CPMU counters are discoverable via the Event Capabilities Registers. 116 * Each Event Capability register contains a a VID / GroupID. 117 * A counter may then count any combination (by summing) of events in 118 * that group which are in the Supported Events Bitmask. 119 * However, there are some complexities to the scheme. 120 * - Fixed function counters refer to an Event Capabilities register. 121 * That event capability register is not then used for Configurable 122 * counters. 123 */ 124 static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) 125 { 126 unsigned long fixed_counter_event_cap_bm = 0; 127 void __iomem *base = info->base; 128 bool freeze_for_enable; 129 u64 val, eval; 130 int i; 131 132 val = readq(base + CXL_PMU_CAP_REG); 133 freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) && 134 FIELD_GET(CXL_PMU_CAP_FREEZE, val); 135 if (!freeze_for_enable) { 136 dev_err(dev, "Counters not writable while frozen\n"); 137 return -ENODEV; 138 } 139 140 info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1; 141 info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val); 142 info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; 143 144 info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; 145 if (FIELD_GET(CXL_PMU_CAP_INT, val)) 146 info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); 147 else 148 info->irq = -1; 149 150 /* First handle fixed function counters; note if configurable counters found */ 151 for (i = 0; i < info->num_counters; i++) { 152 struct cxl_pmu_ev_cap *pmu_ev; 153 u32 events_msk; 154 u8 group_idx; 155 156 val = readq(base + CXL_PMU_COUNTER_CFG_REG(i)); 157 158 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) == 159 CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) { 160 set_bit(i, info->conf_counter_bm); 161 } 162 163 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) != 164 CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN) 165 continue; 166 167 /* In this case we know which fields are const */ 168 group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val); 169 events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val); 170 eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx)); 171 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 172 if (!pmu_ev) 173 return -ENOMEM; 174 175 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 176 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 177 /* For a fixed purpose counter use the events mask from the counter CFG */ 178 pmu_ev->msk = events_msk; 179 pmu_ev->counter_idx = i; 180 /* This list add is never unwound as all entries deleted on remove */ 181 list_add(&pmu_ev->node, &info->event_caps_fixed); 182 /* 183 * Configurable counters must not use an Event Capability registers that 184 * is in use for a Fixed counter 185 */ 186 set_bit(group_idx, &fixed_counter_event_cap_bm); 187 } 188 189 if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) { 190 struct cxl_pmu_ev_cap *pmu_ev; 191 int j; 192 /* Walk event capabilities unused by fixed counters */ 193 for_each_clear_bit(j, &fixed_counter_event_cap_bm, 194 info->num_event_capabilities) { 195 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 196 if (!pmu_ev) 197 return -ENOMEM; 198 199 eval = readq(base + CXL_PMU_EVENT_CAP_REG(j)); 200 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 201 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 202 pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval); 203 pmu_ev->event_idx = j; 204 list_add(&pmu_ev->node, &info->event_caps_configurable); 205 } 206 } 207 208 return 0; 209 } 210 211 #define CXL_PMU_FORMAT_ATTR(_name, _format)\ 212 (&((struct dev_ext_attribute[]) { \ 213 { \ 214 .attr = __ATTR(_name, 0444, device_show_string, NULL), \ 215 .var = (void *)_format \ 216 } \ 217 })[0].attr.attr) 218 219 enum { 220 cxl_pmu_mask_attr, 221 cxl_pmu_gid_attr, 222 cxl_pmu_vid_attr, 223 cxl_pmu_threshold_attr, 224 cxl_pmu_invert_attr, 225 cxl_pmu_edge_attr, 226 cxl_pmu_hdm_filter_en_attr, 227 cxl_pmu_hdm_attr, 228 }; 229 230 static struct attribute *cxl_pmu_format_attr[] = { 231 [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"), 232 [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"), 233 [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"), 234 [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"), 235 [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"), 236 [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), 237 [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), 238 [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), 239 NULL 240 }; 241 242 #define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0) 243 #define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32) 244 #define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48) 245 #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0) 246 #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) 247 #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) 248 #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) 249 #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) 250 251 static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, 252 struct attribute *attr, int a) 253 { 254 struct device *dev = kobj_to_dev(kobj); 255 struct cxl_pmu_info *info = dev_get_drvdata(dev); 256 257 /* 258 * Filter capability at the CPMU level, so hide the attributes if the particular 259 * filter is not supported. 260 */ 261 if (!info->filter_hdm && 262 (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] || 263 attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) 264 return 0; 265 266 return attr->mode; 267 } 268 269 static const struct attribute_group cxl_pmu_format_group = { 270 .name = "format", 271 .attrs = cxl_pmu_format_attr, 272 .is_visible = cxl_pmu_format_is_visible, 273 }; 274 275 static u32 cxl_pmu_config_get_mask(struct perf_event *event) 276 { 277 return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config); 278 } 279 280 static u16 cxl_pmu_config_get_gid(struct perf_event *event) 281 { 282 return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config); 283 } 284 285 static u16 cxl_pmu_config_get_vid(struct perf_event *event) 286 { 287 return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config); 288 } 289 290 static u8 cxl_pmu_config1_get_threshold(struct perf_event *event) 291 { 292 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1); 293 } 294 295 static bool cxl_pmu_config1_get_invert(struct perf_event *event) 296 { 297 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1); 298 } 299 300 static bool cxl_pmu_config1_get_edge(struct perf_event *event) 301 { 302 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1); 303 } 304 305 /* 306 * CPMU specification allows for 8 filters, each with a 32 bit value... 307 * So we need to find 8x32bits to store it in. 308 * As the value used for disable is 0xffff_ffff, a separate enable switch 309 * is needed. 310 */ 311 312 static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event) 313 { 314 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1); 315 } 316 317 static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) 318 { 319 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); 320 } 321 322 static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, 323 struct device_attribute *attr, char *buf) 324 { 325 struct perf_pmu_events_attr *pmu_attr = 326 container_of(attr, struct perf_pmu_events_attr, attr); 327 328 return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id); 329 } 330 331 #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \ 332 PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \ 333 ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk)) 334 335 /* For CXL spec defined events */ 336 #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \ 337 CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk) 338 339 static struct attribute *cxl_pmu_event_attrs[] = { 340 CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)), 341 /* CXL rev 3.0 Table 3-17 - Device to Host Requests */ 342 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)), 343 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)), 344 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)), 345 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)), 346 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)), 347 CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)), 348 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)), 349 CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)), 350 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)), 351 CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)), 352 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)), 353 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)), 354 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)), 355 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)), 356 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)), 357 /* CXL rev 3.0 Table 3-20 - D2H Response Encodings */ 358 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)), 359 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)), 360 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)), 361 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)), 362 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)), 363 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)), 364 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)), 365 /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */ 366 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)), 367 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)), 368 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)), 369 /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */ 370 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)), 371 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)), 372 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)), 373 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)), 374 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)), 375 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)), 376 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)), 377 /* CXL rev 3.0 Table 13-5 directly lists these */ 378 CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)), 379 CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)), 380 /* CXL rev 3.1 Table 3-35 M2S Req Memory Opcodes */ 381 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)), 382 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)), 383 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)), 384 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)), 385 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), 386 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)), 387 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)), 388 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), 389 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), 390 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), 391 /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ 392 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), 393 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), 394 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), 395 /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ 396 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), 397 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), 398 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)), 399 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)), 400 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)), 401 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)), 402 /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */ 403 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)), 404 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)), 405 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)), 406 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), 407 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), 408 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), 409 /* CXL rev 3.1 Table 3-50 S2M NDR Opcopdes */ 410 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), 411 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), 412 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), 413 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)), 414 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)), 415 /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ 416 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), 417 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), 418 /* CXL rev 3.0 Table 13-5 directly lists these */ 419 CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), 420 CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), 421 CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)), 422 CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)), 423 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), 424 CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), 425 CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), 426 NULL 427 }; 428 429 static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info, 430 int vid, int gid, int msk) 431 { 432 struct cxl_pmu_ev_cap *pmu_ev; 433 434 list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) { 435 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 436 continue; 437 438 /* Precise match for fixed counter */ 439 if (msk == pmu_ev->msk) 440 return pmu_ev; 441 } 442 443 return ERR_PTR(-EINVAL); 444 } 445 446 static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info, 447 int vid, int gid, int msk) 448 { 449 struct cxl_pmu_ev_cap *pmu_ev; 450 451 list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) { 452 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 453 continue; 454 455 /* Request mask must be subset of supported */ 456 if (msk & ~pmu_ev->msk) 457 continue; 458 459 return pmu_ev; 460 } 461 462 return ERR_PTR(-EINVAL); 463 } 464 465 static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a) 466 { 467 struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr); 468 struct perf_pmu_events_attr *pmu_attr = 469 container_of(dev_attr, struct perf_pmu_events_attr, attr); 470 struct device *dev = kobj_to_dev(kobj); 471 struct cxl_pmu_info *info = dev_get_drvdata(dev); 472 int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id); 473 int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id); 474 int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id); 475 476 if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk))) 477 return attr->mode; 478 479 if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk))) 480 return attr->mode; 481 482 return 0; 483 } 484 485 static const struct attribute_group cxl_pmu_events = { 486 .name = "events", 487 .attrs = cxl_pmu_event_attrs, 488 .is_visible = cxl_pmu_event_is_visible, 489 }; 490 491 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 492 char *buf) 493 { 494 struct cxl_pmu_info *info = dev_get_drvdata(dev); 495 496 return cpumap_print_to_pagebuf(true, buf, cpumask_of(info->on_cpu)); 497 } 498 static DEVICE_ATTR_RO(cpumask); 499 500 static struct attribute *cxl_pmu_cpumask_attrs[] = { 501 &dev_attr_cpumask.attr, 502 NULL 503 }; 504 505 static const struct attribute_group cxl_pmu_cpumask_group = { 506 .attrs = cxl_pmu_cpumask_attrs, 507 }; 508 509 static const struct attribute_group *cxl_pmu_attr_groups[] = { 510 &cxl_pmu_events, 511 &cxl_pmu_format_group, 512 &cxl_pmu_cpumask_group, 513 NULL 514 }; 515 516 /* If counter_idx == NULL, don't try to allocate a counter. */ 517 static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx, 518 int *event_idx) 519 { 520 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 521 DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS); 522 struct cxl_pmu_ev_cap *pmu_ev; 523 u32 mask; 524 u16 gid, vid; 525 int i; 526 527 vid = cxl_pmu_config_get_vid(event); 528 gid = cxl_pmu_config_get_gid(event); 529 mask = cxl_pmu_config_get_mask(event); 530 531 pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask); 532 if (!IS_ERR(pmu_ev)) { 533 if (!counter_idx) 534 return 0; 535 if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) { 536 *counter_idx = pmu_ev->counter_idx; 537 return 0; 538 } 539 /* Fixed counter is in use, but maybe a configurable one? */ 540 } 541 542 pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask); 543 if (!IS_ERR(pmu_ev)) { 544 if (!counter_idx) 545 return 0; 546 547 bitmap_andnot(configurable_and_free, info->conf_counter_bm, 548 info->used_counter_bm, CXL_PMU_MAX_COUNTERS); 549 550 i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS); 551 if (i == CXL_PMU_MAX_COUNTERS) 552 return -EINVAL; 553 554 *counter_idx = i; 555 return 0; 556 } 557 558 return -EINVAL; 559 } 560 561 static int cxl_pmu_event_init(struct perf_event *event) 562 { 563 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 564 int rc; 565 566 /* Top level type sanity check - is this a Hardware Event being requested */ 567 if (event->attr.type != event->pmu->type) 568 return -ENOENT; 569 570 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 571 return -EOPNOTSUPP; 572 /* TODO: Validation of any filter */ 573 574 /* 575 * Verify that it is possible to count what was requested. Either must 576 * be a fixed counter that is a precise match or a configurable counter 577 * where this is a subset. 578 */ 579 rc = cxl_pmu_get_event_idx(event, NULL, NULL); 580 if (rc < 0) 581 return rc; 582 583 event->cpu = info->on_cpu; 584 585 return 0; 586 } 587 588 static void cxl_pmu_enable(struct pmu *pmu) 589 { 590 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 591 void __iomem *base = info->base; 592 593 /* Can assume frozen at this stage */ 594 writeq(0, base + CXL_PMU_FREEZE_REG); 595 } 596 597 static void cxl_pmu_disable(struct pmu *pmu) 598 { 599 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 600 void __iomem *base = info->base; 601 602 /* 603 * Whilst bits above number of counters are RsvdZ 604 * they are unlikely to be repurposed given 605 * number of counters is allowed to be 64 leaving 606 * no reserved bits. Hence this is only slightly 607 * naughty. 608 */ 609 writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG); 610 } 611 612 static void cxl_pmu_event_start(struct perf_event *event, int flags) 613 { 614 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 615 struct hw_perf_event *hwc = &event->hw; 616 void __iomem *base = info->base; 617 u64 cfg; 618 619 /* 620 * All paths to here should either set these flags directly or 621 * call cxl_pmu_event_stop() which will ensure the correct state. 622 */ 623 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 624 return; 625 626 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 627 hwc->state = 0; 628 629 /* 630 * Currently only hdm filter control is implemnted, this code will 631 * want generalizing when more filters are added. 632 */ 633 if (info->filter_hdm) { 634 if (cxl_pmu_config1_hdm_filter_en(event)) 635 cfg = cxl_pmu_config2_get_hdm_decoder(event); 636 else 637 cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ 638 writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); 639 } 640 641 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 642 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1); 643 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1); 644 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1); 645 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE, 646 cxl_pmu_config1_get_edge(event) ? 1 : 0); 647 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT, 648 cxl_pmu_config1_get_invert(event) ? 1 : 0); 649 650 /* Fixed purpose counters have next two fields RO */ 651 if (test_bit(hwc->idx, info->conf_counter_bm)) { 652 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, 653 hwc->event_base); 654 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK, 655 cxl_pmu_config_get_mask(event)); 656 } 657 cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK; 658 /* 659 * For events that generate only 1 count per clock the CXL 3.0 spec 660 * states the threshold shall be set to 1 but if set to 0 it will 661 * count the raw value anwyay? 662 * There is no definition of what events will count multiple per cycle 663 * and hence to which non 1 values of threshold can apply. 664 * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition) 665 */ 666 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK, 667 cxl_pmu_config1_get_threshold(event)); 668 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 669 670 local64_set(&hwc->prev_count, 0); 671 writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx)); 672 673 perf_event_update_userpage(event); 674 } 675 676 static u64 cxl_pmu_read_counter(struct perf_event *event) 677 { 678 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 679 void __iomem *base = info->base; 680 681 return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx)); 682 } 683 684 static void __cxl_pmu_read(struct perf_event *event, bool overflow) 685 { 686 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 687 struct hw_perf_event *hwc = &event->hw; 688 u64 new_cnt, prev_cnt, delta; 689 690 do { 691 prev_cnt = local64_read(&hwc->prev_count); 692 new_cnt = cxl_pmu_read_counter(event); 693 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt); 694 695 /* 696 * If we know an overflow occur then take that into account. 697 * Note counter is not reset as that would lose events 698 */ 699 delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0); 700 if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0)) 701 delta += (1UL << info->counter_width); 702 703 local64_add(delta, &event->count); 704 } 705 706 static void cxl_pmu_read(struct perf_event *event) 707 { 708 __cxl_pmu_read(event, false); 709 } 710 711 static void cxl_pmu_event_stop(struct perf_event *event, int flags) 712 { 713 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 714 void __iomem *base = info->base; 715 struct hw_perf_event *hwc = &event->hw; 716 u64 cfg; 717 718 cxl_pmu_read(event); 719 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 720 hwc->state |= PERF_HES_STOPPED; 721 722 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 723 cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) | 724 FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1)); 725 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 726 727 hwc->state |= PERF_HES_UPTODATE; 728 } 729 730 static int cxl_pmu_event_add(struct perf_event *event, int flags) 731 { 732 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 733 struct hw_perf_event *hwc = &event->hw; 734 int idx, rc; 735 int event_idx = 0; 736 737 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 738 739 rc = cxl_pmu_get_event_idx(event, &idx, &event_idx); 740 if (rc < 0) 741 return rc; 742 743 hwc->idx = idx; 744 745 /* Only set for configurable counters */ 746 hwc->event_base = event_idx; 747 info->hw_events[idx] = event; 748 set_bit(idx, info->used_counter_bm); 749 750 if (flags & PERF_EF_START) 751 cxl_pmu_event_start(event, PERF_EF_RELOAD); 752 753 return 0; 754 } 755 756 static void cxl_pmu_event_del(struct perf_event *event, int flags) 757 { 758 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 759 struct hw_perf_event *hwc = &event->hw; 760 761 cxl_pmu_event_stop(event, PERF_EF_UPDATE); 762 clear_bit(hwc->idx, info->used_counter_bm); 763 info->hw_events[hwc->idx] = NULL; 764 perf_event_update_userpage(event); 765 } 766 767 static irqreturn_t cxl_pmu_irq(int irq, void *data) 768 { 769 struct cxl_pmu_info *info = data; 770 void __iomem *base = info->base; 771 u64 overflowed; 772 DECLARE_BITMAP(overflowedbm, 64); 773 int i; 774 775 overflowed = readq(base + CXL_PMU_OVERFLOW_REG); 776 777 /* Interrupt may be shared, so maybe it isn't ours */ 778 if (!overflowed) 779 return IRQ_NONE; 780 781 bitmap_from_arr64(overflowedbm, &overflowed, 64); 782 for_each_set_bit(i, overflowedbm, info->num_counters) { 783 struct perf_event *event = info->hw_events[i]; 784 785 if (!event) { 786 dev_dbg(info->pmu.dev, 787 "overflow but on non enabled counter %d\n", i); 788 continue; 789 } 790 791 __cxl_pmu_read(event, true); 792 } 793 794 writeq(overflowed, base + CXL_PMU_OVERFLOW_REG); 795 796 return IRQ_HANDLED; 797 } 798 799 static void cxl_pmu_perf_unregister(void *_info) 800 { 801 struct cxl_pmu_info *info = _info; 802 803 perf_pmu_unregister(&info->pmu); 804 } 805 806 static void cxl_pmu_cpuhp_remove(void *_info) 807 { 808 struct cxl_pmu_info *info = _info; 809 810 cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node); 811 } 812 813 static int cxl_pmu_probe(struct device *dev) 814 { 815 struct cxl_pmu *pmu = to_cxl_pmu(dev); 816 struct pci_dev *pdev = to_pci_dev(dev->parent); 817 struct cxl_pmu_info *info; 818 char *irq_name; 819 char *dev_name; 820 int rc, irq; 821 822 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); 823 if (!info) 824 return -ENOMEM; 825 826 dev_set_drvdata(dev, info); 827 INIT_LIST_HEAD(&info->event_caps_fixed); 828 INIT_LIST_HEAD(&info->event_caps_configurable); 829 830 info->base = pmu->base; 831 832 info->on_cpu = -1; 833 rc = cxl_pmu_parse_caps(dev, info); 834 if (rc) 835 return rc; 836 837 info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events), 838 info->num_counters, GFP_KERNEL); 839 if (!info->hw_events) 840 return -ENOMEM; 841 842 switch (pmu->type) { 843 case CXL_PMU_MEMDEV: 844 dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d", 845 pmu->assoc_id, pmu->index); 846 break; 847 } 848 if (!dev_name) 849 return -ENOMEM; 850 851 info->pmu = (struct pmu) { 852 .name = dev_name, 853 .parent = dev, 854 .module = THIS_MODULE, 855 .event_init = cxl_pmu_event_init, 856 .pmu_enable = cxl_pmu_enable, 857 .pmu_disable = cxl_pmu_disable, 858 .add = cxl_pmu_event_add, 859 .del = cxl_pmu_event_del, 860 .start = cxl_pmu_event_start, 861 .stop = cxl_pmu_event_stop, 862 .read = cxl_pmu_read, 863 .task_ctx_nr = perf_invalid_context, 864 .attr_groups = cxl_pmu_attr_groups, 865 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 866 }; 867 868 if (info->irq <= 0) 869 return -EINVAL; 870 871 rc = pci_irq_vector(pdev, info->irq); 872 if (rc < 0) 873 return rc; 874 irq = rc; 875 876 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name); 877 if (!irq_name) 878 return -ENOMEM; 879 880 rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT, 881 irq_name, info); 882 if (rc) 883 return rc; 884 info->irq = irq; 885 886 rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node); 887 if (rc) 888 return rc; 889 890 rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info); 891 if (rc) 892 return rc; 893 894 rc = perf_pmu_register(&info->pmu, info->pmu.name, -1); 895 if (rc) 896 return rc; 897 898 rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info); 899 if (rc) 900 return rc; 901 902 return 0; 903 } 904 905 static struct cxl_driver cxl_pmu_driver = { 906 .name = "cxl_pmu", 907 .probe = cxl_pmu_probe, 908 .id = CXL_DEVICE_PMU, 909 }; 910 911 static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 912 { 913 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 914 915 if (info->on_cpu != -1) 916 return 0; 917 918 info->on_cpu = cpu; 919 /* 920 * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet 921 * gone away again. 922 */ 923 WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu))); 924 925 return 0; 926 } 927 928 static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 929 { 930 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 931 unsigned int target; 932 933 if (info->on_cpu != cpu) 934 return 0; 935 936 info->on_cpu = -1; 937 target = cpumask_any_but(cpu_online_mask, cpu); 938 if (target >= nr_cpu_ids) { 939 dev_err(info->pmu.dev, "Unable to find a suitable CPU\n"); 940 return 0; 941 } 942 943 perf_pmu_migrate_context(&info->pmu, cpu, target); 944 info->on_cpu = target; 945 /* 946 * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet 947 * gone away. 948 */ 949 WARN_ON(irq_set_affinity(info->irq, cpumask_of(target))); 950 951 return 0; 952 } 953 954 static __init int cxl_pmu_init(void) 955 { 956 int rc; 957 958 rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 959 "AP_PERF_CXL_PMU_ONLINE", 960 cxl_pmu_online_cpu, cxl_pmu_offline_cpu); 961 if (rc < 0) 962 return rc; 963 cxl_pmu_cpuhp_state_num = rc; 964 965 rc = cxl_driver_register(&cxl_pmu_driver); 966 if (rc) 967 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 968 969 return rc; 970 } 971 972 static __exit void cxl_pmu_exit(void) 973 { 974 cxl_driver_unregister(&cxl_pmu_driver); 975 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 976 } 977 978 MODULE_DESCRIPTION("CXL Performance Monitor Driver"); 979 MODULE_LICENSE("GPL"); 980 MODULE_IMPORT_NS(CXL); 981 module_init(cxl_pmu_init); 982 module_exit(cxl_pmu_exit); 983 MODULE_ALIAS_CXL(CXL_DEVICE_PMU); 984