1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright(c) 2023 Huawei 5 * 6 * The CXL 3.0 specification includes a standard Performance Monitoring Unit, 7 * called the CXL PMU, or CPMU. In order to allow a high degree of 8 * implementation flexibility the specification provides a wide range of 9 * options all of which are self describing. 10 * 11 * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface 12 */ 13 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/perf_event.h> 16 #include <linux/bitops.h> 17 #include <linux/device.h> 18 #include <linux/bits.h> 19 #include <linux/list.h> 20 #include <linux/bug.h> 21 #include <linux/pci.h> 22 23 #include "../cxl/cxlpci.h" 24 #include "../cxl/cxl.h" 25 #include "../cxl/pmu.h" 26 27 #define CXL_PMU_CAP_REG 0x0 28 #define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(5, 0) 29 #define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8) 30 #define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20) 31 #define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32) 32 #define CXL_PMU_FILTER_HDM BIT(0) 33 #define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1) 34 #define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44) 35 #define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48) 36 #define CXL_PMU_CAP_FREEZE BIT_ULL(49) 37 #define CXL_PMU_CAP_INT BIT_ULL(50) 38 #define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60) 39 40 #define CXL_PMU_OVERFLOW_REG 0x10 41 #define CXL_PMU_FREEZE_REG 0x18 42 #define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n)) 43 #define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0) 44 #define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32) 45 #define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48) 46 47 #define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n)) 48 #define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0) 49 #define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0 50 #define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1 51 #define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2 52 #define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8) 53 #define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9) 54 #define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10) 55 #define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11) 56 #define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12) 57 #define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16) 58 #define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24) 59 #define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59) 60 61 #define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8)) 62 #define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(31, 0) 63 64 #define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n)) 65 66 /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */ 67 #define CXL_PMU_GID_CLOCK_TICKS 0x00 68 #define CXL_PMU_GID_D2H_REQ 0x0010 69 #define CXL_PMU_GID_D2H_RSP 0x0011 70 #define CXL_PMU_GID_H2D_REQ 0x0012 71 #define CXL_PMU_GID_H2D_RSP 0x0013 72 #define CXL_PMU_GID_CACHE_DATA 0x0014 73 #define CXL_PMU_GID_M2S_REQ 0x0020 74 #define CXL_PMU_GID_M2S_RWD 0x0021 75 #define CXL_PMU_GID_M2S_BIRSP 0x0022 76 #define CXL_PMU_GID_S2M_BISNP 0x0023 77 #define CXL_PMU_GID_S2M_NDR 0x0024 78 #define CXL_PMU_GID_S2M_DRS 0x0025 79 #define CXL_PMU_GID_DDR 0x8000 80 #define CXL_PMU_GID_QUEUE_OCC 0x8001 81 #define CXL_PMU_GID_QUEUE_RESID 0x8002 82 #define CXL_PMU_GID_RETRY_EVENTS 0x8003 83 #define CXL_PMU_GID_THROTTLE 0x8004 84 85 static int cxl_pmu_cpuhp_state_num; 86 87 struct cxl_pmu_ev_cap { 88 u16 vid; 89 u16 gid; 90 u32 msk; 91 union { 92 int counter_idx; /* fixed counters */ 93 int event_idx; /* configurable counters */ 94 }; 95 struct list_head node; 96 }; 97 98 #define CXL_PMU_MAX_COUNTERS 64 99 struct cxl_pmu_info { 100 struct pmu pmu; 101 void __iomem *base; 102 struct perf_event **hw_events; 103 struct list_head event_caps_configurable; 104 struct list_head event_caps_fixed; 105 DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS); 106 DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS); 107 u16 counter_width; 108 u8 num_counters; 109 u8 num_event_capabilities; 110 int on_cpu; 111 struct hlist_node node; 112 bool filter_hdm; 113 bool filter_crb; 114 int msi_vec; 115 int irq; 116 /* Set between pmu_enable() and pmu_disable(), read by the IRQ handler */ 117 bool enabled; 118 }; 119 120 #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu) 121 122 /* 123 * All CPMU counters are discoverable via the Event Capabilities Registers. 124 * Each Event Capability register contains a VID / GroupID. 125 * A counter may then count any combination (by summing) of events in 126 * that group which are in the Supported Events Bitmask. 127 * However, there are some complexities to the scheme. 128 * - Fixed function counters refer to an Event Capabilities register. 129 * That event capability register is not then used for Configurable 130 * counters. 131 */ 132 static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) 133 { 134 unsigned long fixed_counter_event_cap_bm = 0; 135 void __iomem *base = info->base; 136 bool freeze_for_enable; 137 u64 val, eval; 138 int i; 139 140 val = readq(base + CXL_PMU_CAP_REG); 141 freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) && 142 FIELD_GET(CXL_PMU_CAP_FREEZE, val); 143 if (!freeze_for_enable) { 144 dev_err(dev, "Counters not writable while frozen\n"); 145 return -ENODEV; 146 } 147 148 info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1; 149 info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val); 150 /* 151 * The Counter Data register is 64 bits wide, so a Counter Width of 0 or 152 * >64 is invalid. Reject it rather than let GENMASK_ULL(width - 1, 0) in 153 * the read path shift out of range. 154 */ 155 if (info->counter_width == 0 || info->counter_width > 64) { 156 dev_err(dev, "Invalid counter width %d\n", info->counter_width); 157 return -ENODEV; 158 } 159 info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; 160 161 info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; 162 info->filter_crb = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & 163 CXL_PMU_FILTER_CHAN_RANK_BANK; 164 if (FIELD_GET(CXL_PMU_CAP_INT, val)) 165 info->msi_vec = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); 166 else 167 info->msi_vec = -1; 168 169 /* First handle fixed function counters; note if configurable counters found */ 170 for (i = 0; i < info->num_counters; i++) { 171 struct cxl_pmu_ev_cap *pmu_ev; 172 u32 events_msk; 173 u8 group_idx; 174 175 val = readq(base + CXL_PMU_COUNTER_CFG_REG(i)); 176 177 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) == 178 CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) { 179 set_bit(i, info->conf_counter_bm); 180 } 181 182 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) != 183 CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN) 184 continue; 185 186 /* In this case we know which fields are const */ 187 group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val); 188 events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val); 189 eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx)); 190 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 191 if (!pmu_ev) 192 return -ENOMEM; 193 194 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 195 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 196 /* For a fixed purpose counter use the events mask from the counter CFG */ 197 pmu_ev->msk = events_msk; 198 pmu_ev->counter_idx = i; 199 /* This list add is never unwound as all entries deleted on remove */ 200 list_add(&pmu_ev->node, &info->event_caps_fixed); 201 /* 202 * Configurable counters must not use an Event Capability registers that 203 * is in use for a Fixed counter 204 */ 205 set_bit(group_idx, &fixed_counter_event_cap_bm); 206 } 207 208 if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) { 209 struct cxl_pmu_ev_cap *pmu_ev; 210 int j; 211 /* Walk event capabilities unused by fixed counters */ 212 for_each_clear_bit(j, &fixed_counter_event_cap_bm, 213 info->num_event_capabilities) { 214 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 215 if (!pmu_ev) 216 return -ENOMEM; 217 218 eval = readq(base + CXL_PMU_EVENT_CAP_REG(j)); 219 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 220 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 221 pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval); 222 pmu_ev->event_idx = j; 223 list_add(&pmu_ev->node, &info->event_caps_configurable); 224 } 225 } 226 227 return 0; 228 } 229 230 #define CXL_PMU_FORMAT_ATTR(_name, _format)\ 231 (&((struct dev_ext_attribute[]) { \ 232 { \ 233 .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \ 234 .var = (void *)_format \ 235 } \ 236 })[0].attr.attr) 237 238 enum { 239 cxl_pmu_mask_attr, 240 cxl_pmu_gid_attr, 241 cxl_pmu_vid_attr, 242 cxl_pmu_threshold_attr, 243 cxl_pmu_invert_attr, 244 cxl_pmu_edge_attr, 245 cxl_pmu_hdm_filter_en_attr, 246 cxl_pmu_hdm_attr, 247 cxl_pmu_crb_filter_en_attr, 248 cxl_pmu_crb_attr, 249 }; 250 251 static struct attribute *cxl_pmu_format_attr[] = { 252 [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"), 253 [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"), 254 [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"), 255 [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"), 256 [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"), 257 [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), 258 [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), 259 [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), 260 [cxl_pmu_crb_filter_en_attr] = CXL_PMU_FORMAT_ATTR(crb_filter_en, "config1:19"), 261 [cxl_pmu_crb_attr] = CXL_PMU_FORMAT_ATTR(crb, "config2:32-63"), 262 NULL 263 }; 264 265 #define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0) 266 #define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32) 267 #define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48) 268 #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0) 269 #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) 270 #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) 271 #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) 272 #define CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK BIT(19) 273 #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) 274 #define CXL_PMU_ATTR_CONFIG2_CRB_MSK GENMASK_ULL(63, 32) 275 276 static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, 277 struct attribute *attr, int a) 278 { 279 struct device *dev = kobj_to_dev(kobj); 280 struct cxl_pmu_info *info = dev_get_drvdata(dev); 281 282 /* 283 * Filter capability at the CPMU level, so hide the attributes if the particular 284 * filter is not supported. 285 */ 286 if (!info->filter_hdm && 287 (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] || 288 attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) 289 return 0; 290 291 if (!info->filter_crb && 292 (attr == cxl_pmu_format_attr[cxl_pmu_crb_filter_en_attr] || 293 attr == cxl_pmu_format_attr[cxl_pmu_crb_attr])) 294 return 0; 295 296 return attr->mode; 297 } 298 299 static const struct attribute_group cxl_pmu_format_group = { 300 .name = "format", 301 .attrs = cxl_pmu_format_attr, 302 .is_visible = cxl_pmu_format_is_visible, 303 }; 304 305 static u32 cxl_pmu_config_get_mask(struct perf_event *event) 306 { 307 return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config); 308 } 309 310 static u16 cxl_pmu_config_get_gid(struct perf_event *event) 311 { 312 return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config); 313 } 314 315 static u16 cxl_pmu_config_get_vid(struct perf_event *event) 316 { 317 return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config); 318 } 319 320 static u8 cxl_pmu_config1_get_threshold(struct perf_event *event) 321 { 322 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1); 323 } 324 325 static bool cxl_pmu_config1_get_invert(struct perf_event *event) 326 { 327 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1); 328 } 329 330 static bool cxl_pmu_config1_get_edge(struct perf_event *event) 331 { 332 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1); 333 } 334 335 /* 336 * CPMU specification allows for 8 filters, each with a 32 bit value... 337 * So we need to find 8x32bits to store it in. 338 * As the value used for disable is 0xffff_ffff, a separate enable switch 339 * is needed. 340 */ 341 342 static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event) 343 { 344 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1); 345 } 346 347 static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) 348 { 349 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); 350 } 351 352 static u16 cxl_pmu_config1_crb_filter_en(struct perf_event *event) 353 { 354 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK, 355 event->attr.config1); 356 } 357 358 static u32 cxl_pmu_config2_get_crb(struct perf_event *event) 359 { 360 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_CRB_MSK, event->attr.config2); 361 } 362 363 static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, 364 struct device_attribute *attr, char *buf) 365 { 366 struct perf_pmu_events_attr *pmu_attr = 367 container_of(attr, struct perf_pmu_events_attr, attr); 368 369 return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id); 370 } 371 372 #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \ 373 PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \ 374 ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk)) 375 376 /* For CXL spec defined events */ 377 #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \ 378 CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk) 379 380 static struct attribute *cxl_pmu_event_attrs[] = { 381 CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)), 382 /* CXL rev 3.0 Table 3-17 - Device to Host Requests */ 383 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)), 384 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)), 385 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)), 386 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)), 387 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)), 388 CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)), 389 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)), 390 CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)), 391 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)), 392 CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)), 393 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)), 394 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)), 395 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)), 396 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)), 397 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)), 398 /* CXL rev 3.0 Table 3-20 - D2H Response Encodings */ 399 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)), 400 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)), 401 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)), 402 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)), 403 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)), 404 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)), 405 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)), 406 /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */ 407 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)), 408 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)), 409 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)), 410 /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */ 411 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)), 412 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)), 413 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)), 414 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)), 415 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)), 416 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)), 417 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)), 418 /* CXL rev 3.0 Table 13-5 directly lists these */ 419 CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)), 420 CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)), 421 /* CXL rev 3.1 Table 3-35 M2S Req Memory Opcodes */ 422 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)), 423 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)), 424 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)), 425 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)), 426 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), 427 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)), 428 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)), 429 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvtee, CXL_PMU_GID_M2S_REQ, BIT(7)), 430 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), 431 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), 432 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), 433 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvptee, CXL_PMU_GID_M2S_REQ, BIT(11)), 434 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrdtee, CXL_PMU_GID_M2S_REQ, BIT(12)), 435 CXL_PMU_EVENT_CXL_ATTR(m2s_req_teupdate, CXL_PMU_GID_M2S_REQ, BIT(13)), 436 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevcttee, CXL_PMU_GID_M2S_REQ, BIT(14)), 437 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevctu, CXL_PMU_GID_M2S_REQ, BIT(15)), 438 /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ 439 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), 440 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), 441 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), 442 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfill, CXL_PMU_GID_M2S_RWD, BIT(5)), 443 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrtee, CXL_PMU_GID_M2S_RWD, BIT(9)), 444 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptltee, CXL_PMU_GID_M2S_RWD, BIT(10)), 445 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfilltee, CXL_PMU_GID_M2S_RWD, BIT(13)), 446 /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ 447 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), 448 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), 449 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)), 450 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)), 451 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)), 452 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)), 453 /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */ 454 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)), 455 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)), 456 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)), 457 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), 458 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), 459 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), 460 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curtee, CXL_PMU_GID_S2M_BISNP, BIT(8)), 461 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datatee, CXL_PMU_GID_S2M_BISNP, BIT(9)), 462 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invtee, CXL_PMU_GID_S2M_BISNP, BIT(10)), 463 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblktee, CXL_PMU_GID_S2M_BISNP, BIT(12)), 464 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datablktee, CXL_PMU_GID_S2M_BISNP, BIT(13)), 465 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblktee, CXL_PMU_GID_S2M_BISNP, BIT(14)), 466 /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */ 467 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), 468 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), 469 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), 470 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)), 471 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)), 472 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee, CXL_PMU_GID_S2M_NDR, BIT(5)), 473 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_s, CXL_PMU_GID_S2M_NDR, BIT(6)), 474 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_e, CXL_PMU_GID_S2M_NDR, BIT(7)), 475 /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ 476 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), 477 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), 478 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatatee, CXL_PMU_GID_S2M_DRS, BIT(2)), 479 /* CXL rev 3.0 Table 13-5 directly lists these */ 480 CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), 481 CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), 482 CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)), 483 CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)), 484 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), 485 CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), 486 CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), 487 /* CXL 4.0 Table 13-5 DDR add-on events opcodes */ 488 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_rd_ap, CXL_PMU_GID_DDR, BIT(7)), 489 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_wr_ap, CXL_PMU_GID_DDR, BIT(8)), 490 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_all_banks, CXL_PMU_GID_DDR, BIT(9)), 491 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_same_bank, CXL_PMU_GID_DDR, BIT(10)), 492 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_entry, CXL_PMU_GID_DDR, BIT(11)), 493 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_exit, CXL_PMU_GID_DDR, BIT(12)), 494 CXL_PMU_EVENT_CXL_ATTR(ddr_rd_wr_ddr_bus_switching, CXL_PMU_GID_DDR, BIT(13)), 495 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_rd_req, CXL_PMU_GID_DDR, BIT(14)), 496 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_wr_req, CXL_PMU_GID_DDR, BIT(15)), 497 /* CXL 4.0 Table 13-5 QUEUE OCCUPANCY events opcodes */ 498 CXL_PMU_EVENT_CXL_ATTR(rd_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(0)), 499 CXL_PMU_EVENT_CXL_ATTR(wr_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(1)), 500 CXL_PMU_EVENT_CXL_ATTR(rd_wr_merged_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(2)), 501 CXL_PMU_EVENT_CXL_ATTR(pwrdn_event, CXL_PMU_GID_QUEUE_OCC, BIT(3)), 502 /* CXL 4.0 Table 13-5 QUEUE RESIDENCY events opcodes */ 503 CXL_PMU_EVENT_CXL_ATTR(mc_rd_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(0)), 504 CXL_PMU_EVENT_CXL_ATTR(mc_wr_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(1)), 505 /* CXL 4.0 Table 13-5 RETRY events opcodes */ 506 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_rd_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(0)), 507 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_wr_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(1)), 508 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ca_parity, CXL_PMU_GID_RETRY_EVENTS, BIT(2)), 509 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ecc, CXL_PMU_GID_RETRY_EVENTS, BIT(3)), 510 /* CXL 4.0 Table 13-5 THROTTLE events opcodes */ 511 CXL_PMU_EVENT_CXL_ATTR(thermal_throttle_event, CXL_PMU_GID_THROTTLE, BIT(0)), 512 CXL_PMU_EVENT_CXL_ATTR(power_throttle_event, CXL_PMU_GID_THROTTLE, BIT(1)), 513 NULL 514 }; 515 516 static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info, 517 int vid, int gid, int msk) 518 { 519 struct cxl_pmu_ev_cap *pmu_ev; 520 521 list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) { 522 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 523 continue; 524 525 /* Precise match for fixed counter */ 526 if (msk == pmu_ev->msk) 527 return pmu_ev; 528 } 529 530 return ERR_PTR(-EINVAL); 531 } 532 533 static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info, 534 int vid, int gid, int msk) 535 { 536 struct cxl_pmu_ev_cap *pmu_ev; 537 538 list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) { 539 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 540 continue; 541 542 /* Request mask must be subset of supported */ 543 if (msk & ~pmu_ev->msk) 544 continue; 545 546 return pmu_ev; 547 } 548 549 return ERR_PTR(-EINVAL); 550 } 551 552 static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a) 553 { 554 struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr); 555 struct perf_pmu_events_attr *pmu_attr = 556 container_of(dev_attr, struct perf_pmu_events_attr, attr); 557 struct device *dev = kobj_to_dev(kobj); 558 struct cxl_pmu_info *info = dev_get_drvdata(dev); 559 int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id); 560 int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id); 561 int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id); 562 563 if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk))) 564 return attr->mode; 565 566 if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk))) 567 return attr->mode; 568 569 return 0; 570 } 571 572 static const struct attribute_group cxl_pmu_events = { 573 .name = "events", 574 .attrs = cxl_pmu_event_attrs, 575 .is_visible = cxl_pmu_event_is_visible, 576 }; 577 578 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 579 char *buf) 580 { 581 struct cxl_pmu_info *info = dev_get_drvdata(dev); 582 583 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(info->on_cpu))); 584 } 585 static DEVICE_ATTR_RO(cpumask); 586 587 static struct attribute *cxl_pmu_cpumask_attrs[] = { 588 &dev_attr_cpumask.attr, 589 NULL 590 }; 591 592 static const struct attribute_group cxl_pmu_cpumask_group = { 593 .attrs = cxl_pmu_cpumask_attrs, 594 }; 595 596 static const struct attribute_group *cxl_pmu_attr_groups[] = { 597 &cxl_pmu_events, 598 &cxl_pmu_format_group, 599 &cxl_pmu_cpumask_group, 600 NULL 601 }; 602 603 /* 604 * If counter_idx == NULL, don't try to allocate a counter. Callers that 605 * allocate pass both counter_idx and event_idx. 606 */ 607 static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx, 608 int *event_idx) 609 { 610 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 611 DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS); 612 struct cxl_pmu_ev_cap *pmu_ev; 613 u32 mask; 614 u16 gid, vid; 615 int i; 616 617 vid = cxl_pmu_config_get_vid(event); 618 gid = cxl_pmu_config_get_gid(event); 619 mask = cxl_pmu_config_get_mask(event); 620 621 pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask); 622 if (!IS_ERR(pmu_ev)) { 623 if (!counter_idx) 624 return 0; 625 if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) { 626 *counter_idx = pmu_ev->counter_idx; 627 return 0; 628 } 629 /* Fixed counter is in use, but maybe a configurable one? */ 630 } 631 632 pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask); 633 if (!IS_ERR(pmu_ev)) { 634 if (!counter_idx || !event_idx) 635 return 0; 636 637 bitmap_andnot(configurable_and_free, info->conf_counter_bm, 638 info->used_counter_bm, CXL_PMU_MAX_COUNTERS); 639 640 i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS); 641 if (i == CXL_PMU_MAX_COUNTERS) 642 return -EINVAL; 643 644 *counter_idx = i; 645 *event_idx = pmu_ev->event_idx; 646 return 0; 647 } 648 649 return -EINVAL; 650 } 651 652 static int cxl_pmu_event_init(struct perf_event *event) 653 { 654 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 655 int rc; 656 657 /* Top level type sanity check - is this a Hardware Event being requested */ 658 if (event->attr.type != event->pmu->type) 659 return -ENOENT; 660 661 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 662 return -EOPNOTSUPP; 663 /* TODO: Validation of any filter */ 664 665 if (cxl_pmu_config1_crb_filter_en(event)) { 666 if (!info->filter_crb) 667 return -EINVAL; 668 /* event group IDs are scoped by the CXL vendor ID */ 669 if (cxl_pmu_config_get_vid(event) != PCI_VENDOR_ID_CXL) 670 return -EINVAL; 671 672 /* 673 * CRB filtering (Filter ID 1) is only valid for the DDR 674 * Interface, Queue Occupancy, Queue Residency and Retry 675 * event groups (CXL 4.0 Table 13-5). 676 */ 677 switch (cxl_pmu_config_get_gid(event)) { 678 case CXL_PMU_GID_DDR: 679 case CXL_PMU_GID_QUEUE_OCC: 680 case CXL_PMU_GID_QUEUE_RESID: 681 case CXL_PMU_GID_RETRY_EVENTS: 682 break; 683 default: 684 return -EINVAL; 685 } 686 687 /* 688 * Filtering while counting multiple events is 689 * undefined behavior. 690 */ 691 if (hweight32(cxl_pmu_config_get_mask(event)) > 1) 692 return -EINVAL; 693 } 694 695 /* 696 * Verify that it is possible to count what was requested. Either must 697 * be a fixed counter that is a precise match or a configurable counter 698 * where this is a subset. 699 */ 700 rc = cxl_pmu_get_event_idx(event, NULL, NULL); 701 if (rc < 0) 702 return rc; 703 704 event->cpu = info->on_cpu; 705 706 return 0; 707 } 708 709 static void cxl_pmu_enable(struct pmu *pmu) 710 { 711 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 712 void __iomem *base = info->base; 713 714 /* Can assume frozen at this stage */ 715 WRITE_ONCE(info->enabled, true); 716 writeq(0, base + CXL_PMU_FREEZE_REG); 717 } 718 719 static void cxl_pmu_disable(struct pmu *pmu) 720 { 721 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 722 void __iomem *base = info->base; 723 724 WRITE_ONCE(info->enabled, false); 725 /* 726 * Whilst bits above number of counters are RsvdZ 727 * they are unlikely to be repurposed given 728 * number of counters is allowed to be 64 leaving 729 * no reserved bits. Hence this is only slightly 730 * naughty. 731 */ 732 writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG); 733 } 734 735 static void cxl_pmu_event_start(struct perf_event *event, int flags) 736 { 737 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 738 struct hw_perf_event *hwc = &event->hw; 739 void __iomem *base = info->base; 740 u64 cfg; 741 742 /* 743 * All paths to here should either set these flags directly or 744 * call cxl_pmu_event_stop() which will ensure the correct state. 745 */ 746 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 747 return; 748 749 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 750 hwc->state = 0; 751 752 /* 753 * Filter ID=0: HDM decoder filter 754 * Filter ID=1: Channel/Rank/Bank (CRB) filter 755 */ 756 if (info->filter_hdm) { 757 if (cxl_pmu_config1_hdm_filter_en(event)) 758 cfg = cxl_pmu_config2_get_hdm_decoder(event); 759 else 760 cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ 761 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); 762 } 763 764 if (info->filter_crb) { 765 if (cxl_pmu_config1_crb_filter_en(event)) 766 cfg = cxl_pmu_config2_get_crb(event); 767 else 768 cfg = GENMASK(31, 0); /* no filtering if 0xFFFF_FFFF */ 769 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 1)); 770 } 771 772 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 773 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1); 774 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1); 775 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1); 776 FIELD_MODIFY(CXL_PMU_COUNTER_CFG_EDGE, &cfg, 777 cxl_pmu_config1_get_edge(event) ? 1 : 0); 778 FIELD_MODIFY(CXL_PMU_COUNTER_CFG_INVERT, &cfg, 779 cxl_pmu_config1_get_invert(event) ? 1 : 0); 780 781 /* Fixed purpose counters have next two fields RO */ 782 if (test_bit(hwc->idx, info->conf_counter_bm)) { 783 FIELD_MODIFY(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, &cfg, 784 hwc->event_base); 785 FIELD_MODIFY(CXL_PMU_COUNTER_CFG_EVENTS_MSK, &cfg, 786 cxl_pmu_config_get_mask(event)); 787 } 788 cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK; 789 /* 790 * For events that generate only 1 count per clock the CXL 3.0 spec 791 * states the threshold shall be set to 1 but if set to 0 it will 792 * count the raw value anwyay? 793 * There is no definition of what events will count multiple per cycle 794 * and hence to which non 1 values of threshold can apply. 795 * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition) 796 */ 797 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK, 798 cxl_pmu_config1_get_threshold(event)); 799 800 /* 801 * Drop any overflow the previous owner of this counter left pending; 802 * cxl_pmu_event_stop() does not. Otherwise an interrupt latched over 803 * the handover gets charged to this event. Do it before arming Interrupt 804 * on Overflow below, and note RW1C leaves the other counters alone. 805 */ 806 writeq(BIT_ULL(hwc->idx), base + CXL_PMU_OVERFLOW_REG); 807 808 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 809 810 local64_set(&hwc->prev_count, 0); 811 writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx)); 812 813 perf_event_update_userpage(event); 814 } 815 816 static u64 cxl_pmu_read_counter(struct perf_event *event) 817 { 818 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 819 void __iomem *base = info->base; 820 821 return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx)); 822 } 823 824 static void __cxl_pmu_read(struct perf_event *event, bool overflow) 825 { 826 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 827 struct hw_perf_event *hwc = &event->hw; 828 u64 new_cnt, prev_cnt, delta, mask; 829 830 do { 831 prev_cnt = local64_read(&hwc->prev_count); 832 new_cnt = cxl_pmu_read_counter(event); 833 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt); 834 835 /* 836 * The mask discards the bit that says the counter wrapped, so a delta of 837 * one whole period reads back as 0 - the same as no events at all. Only 838 * the overflow status separates them, and new_cnt >= prev_cnt is that 839 * case, so add the period back. mask + 1 is 2^counter_width, which comes 840 * out as 0 for a 64-bit counter and avoids an undefined 1 << 64. 841 */ 842 mask = GENMASK_ULL(info->counter_width - 1, 0); 843 delta = (new_cnt - prev_cnt) & mask; 844 if (overflow && new_cnt >= prev_cnt) 845 delta += mask + 1; 846 847 local64_add(delta, &event->count); 848 } 849 850 static void cxl_pmu_read(struct perf_event *event) 851 { 852 __cxl_pmu_read(event, false); 853 } 854 855 static void cxl_pmu_event_stop(struct perf_event *event, int flags) 856 { 857 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 858 void __iomem *base = info->base; 859 struct hw_perf_event *hwc = &event->hw; 860 u64 cfg; 861 862 cxl_pmu_read(event); 863 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 864 hwc->state |= PERF_HES_STOPPED; 865 866 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 867 cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) | 868 FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1)); 869 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 870 871 hwc->state |= PERF_HES_UPTODATE; 872 } 873 874 static int cxl_pmu_event_add(struct perf_event *event, int flags) 875 { 876 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 877 struct hw_perf_event *hwc = &event->hw; 878 int idx, rc; 879 int event_idx = 0; 880 881 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 882 883 rc = cxl_pmu_get_event_idx(event, &idx, &event_idx); 884 if (rc < 0) 885 return rc; 886 887 hwc->idx = idx; 888 889 /* Only set for configurable counters */ 890 hwc->event_base = event_idx; 891 info->hw_events[idx] = event; 892 set_bit(idx, info->used_counter_bm); 893 894 if (flags & PERF_EF_START) 895 cxl_pmu_event_start(event, PERF_EF_RELOAD); 896 897 return 0; 898 } 899 900 static void cxl_pmu_event_del(struct perf_event *event, int flags) 901 { 902 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 903 struct hw_perf_event *hwc = &event->hw; 904 905 cxl_pmu_event_stop(event, PERF_EF_UPDATE); 906 clear_bit(hwc->idx, info->used_counter_bm); 907 info->hw_events[hwc->idx] = NULL; 908 perf_event_update_userpage(event); 909 } 910 911 static irqreturn_t cxl_pmu_irq(int irq, void *data) 912 { 913 struct cxl_pmu_info *info = data; 914 void __iomem *base = info->base; 915 u64 overflowed; 916 DECLARE_BITMAP(overflowedbm, 64); 917 int i; 918 919 overflowed = readq(base + CXL_PMU_OVERFLOW_REG); 920 921 /* Nothing overflowed, so the device did not raise this */ 922 if (!overflowed) 923 return IRQ_NONE; 924 925 bitmap_from_arr64(overflowedbm, &overflowed, 64); 926 for_each_set_bit(i, overflowedbm, info->num_counters) { 927 struct perf_event *event = info->hw_events[i]; 928 929 if (!event) { 930 dev_dbg(info->pmu.parent, 931 "overflow but on non enabled counter %d\n", i); 932 continue; 933 } 934 935 __cxl_pmu_read(event, true); 936 } 937 938 writeq(overflowed, base + CXL_PMU_OVERFLOW_REG); 939 940 /* 941 * An overflow freezes every counter in the CPMU, so unfreeze once the 942 * overflowed ones have been read and their status cleared. Otherwise 943 * they stay frozen until the next pmu_enable() and events are lost. 944 * 945 * Not while the PMU is disabled, so as not to undo an intentional freeze. 946 * The check is advisory, not exclusive: pmu_disable() normally runs on 947 * info->on_cpu with interrupts off, where the pinned handler cannot 948 * preempt it. In the one window where it does not - the migration in 949 * cxl_pmu_offline_cpu() - the counters are legitimately running again, 950 * so unfreezing is correct there anyway. 951 */ 952 if (READ_ONCE(info->enabled)) 953 writeq(0, base + CXL_PMU_FREEZE_REG); 954 955 return IRQ_HANDLED; 956 } 957 958 static void cxl_pmu_perf_unregister(void *_info) 959 { 960 struct cxl_pmu_info *info = _info; 961 962 perf_pmu_unregister(&info->pmu); 963 } 964 965 static void cxl_pmu_cpuhp_remove(void *_info) 966 { 967 struct cxl_pmu_info *info = _info; 968 969 cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node); 970 } 971 972 static int cxl_pmu_probe(struct device *dev) 973 { 974 struct cxl_pmu *pmu = to_cxl_pmu(dev); 975 struct pci_dev *pdev = to_pci_dev(dev->parent); 976 struct cxl_pmu_info *info; 977 char *irq_name; 978 char *dev_name; 979 int rc, irq; 980 981 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); 982 if (!info) 983 return -ENOMEM; 984 985 dev_set_drvdata(dev, info); 986 INIT_LIST_HEAD(&info->event_caps_fixed); 987 INIT_LIST_HEAD(&info->event_caps_configurable); 988 989 info->base = pmu->base; 990 991 info->on_cpu = -1; 992 rc = cxl_pmu_parse_caps(dev, info); 993 if (rc) 994 return rc; 995 996 info->hw_events = devm_kcalloc(dev, info->num_counters, 997 sizeof(*info->hw_events), GFP_KERNEL); 998 if (!info->hw_events) 999 return -ENOMEM; 1000 1001 switch (pmu->type) { 1002 case CXL_PMU_MEMDEV: 1003 dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d", 1004 pmu->assoc_id, pmu->index); 1005 break; 1006 } 1007 if (!dev_name) 1008 return -ENOMEM; 1009 1010 info->pmu = (struct pmu) { 1011 .name = dev_name, 1012 .parent = dev, 1013 .module = THIS_MODULE, 1014 .event_init = cxl_pmu_event_init, 1015 .pmu_enable = cxl_pmu_enable, 1016 .pmu_disable = cxl_pmu_disable, 1017 .add = cxl_pmu_event_add, 1018 .del = cxl_pmu_event_del, 1019 .start = cxl_pmu_event_start, 1020 .stop = cxl_pmu_event_stop, 1021 .read = cxl_pmu_read, 1022 .task_ctx_nr = perf_invalid_context, 1023 .attr_groups = cxl_pmu_attr_groups, 1024 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1025 }; 1026 1027 if (info->msi_vec < 0) 1028 return -EINVAL; 1029 1030 rc = pci_irq_vector(pdev, info->msi_vec); 1031 if (rc < 0) 1032 return rc; 1033 irq = rc; 1034 1035 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name); 1036 if (!irq_name) 1037 return -ENOMEM; 1038 1039 /* 1040 * Same for counters no event owns yet: clear whatever firmware or a 1041 * previous kernel left set before the handler goes live. Bits above the 1042 * implemented counters are reserved, so only write those. 1043 */ 1044 writeq(GENMASK_ULL(info->num_counters - 1, 0), 1045 info->base + CXL_PMU_OVERFLOW_REG); 1046 1047 /* 1048 * The handler must run on info->on_cpu, so the interrupt cannot be 1049 * shared - IRQF_NOBALANCING is only honoured for the first action on a 1050 * line, and a co-owner would keep taking the interrupt wherever its own 1051 * affinity points. 1052 */ 1053 rc = devm_request_irq(dev, irq, cxl_pmu_irq, 1054 IRQF_NO_THREAD | IRQF_NOBALANCING, 1055 irq_name, info); 1056 if (rc) 1057 return rc; 1058 info->irq = irq; 1059 1060 rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node); 1061 if (rc) 1062 return rc; 1063 1064 rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info); 1065 if (rc) 1066 return rc; 1067 1068 rc = perf_pmu_register(&info->pmu, info->pmu.name, -1); 1069 if (rc) 1070 return rc; 1071 1072 rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info); 1073 if (rc) 1074 return rc; 1075 1076 return 0; 1077 } 1078 1079 static struct cxl_driver cxl_pmu_driver = { 1080 .name = "cxl_pmu", 1081 .probe = cxl_pmu_probe, 1082 .id = CXL_DEVICE_PMU, 1083 }; 1084 1085 static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 1086 { 1087 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 1088 1089 if (info->on_cpu != -1) 1090 return 0; 1091 1092 info->on_cpu = cpu; 1093 /* 1094 * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet 1095 * gone away again. 1096 */ 1097 WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu))); 1098 1099 return 0; 1100 } 1101 1102 static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1103 { 1104 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 1105 unsigned int target; 1106 1107 if (info->on_cpu != cpu) 1108 return 0; 1109 1110 info->on_cpu = -1; 1111 target = cpumask_any_but(cpu_online_mask, cpu); 1112 if (target >= nr_cpu_ids) { 1113 dev_err(info->pmu.dev, "Unable to find a suitable CPU\n"); 1114 return 0; 1115 } 1116 1117 perf_pmu_migrate_context(&info->pmu, cpu, target); 1118 info->on_cpu = target; 1119 /* 1120 * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet 1121 * gone away. 1122 */ 1123 WARN_ON(irq_set_affinity(info->irq, cpumask_of(target))); 1124 1125 return 0; 1126 } 1127 1128 static __init int cxl_pmu_init(void) 1129 { 1130 int rc; 1131 1132 rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1133 "AP_PERF_CXL_PMU_ONLINE", 1134 cxl_pmu_online_cpu, cxl_pmu_offline_cpu); 1135 if (rc < 0) 1136 return rc; 1137 cxl_pmu_cpuhp_state_num = rc; 1138 1139 rc = cxl_driver_register(&cxl_pmu_driver); 1140 if (rc) 1141 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 1142 1143 return rc; 1144 } 1145 1146 static __exit void cxl_pmu_exit(void) 1147 { 1148 cxl_driver_unregister(&cxl_pmu_driver); 1149 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 1150 } 1151 1152 MODULE_DESCRIPTION("CXL Performance Monitor Driver"); 1153 MODULE_LICENSE("GPL"); 1154 MODULE_IMPORT_NS("CXL"); 1155 module_init(cxl_pmu_init); 1156 module_exit(cxl_pmu_exit); 1157 MODULE_ALIAS_CXL(CXL_DEVICE_PMU); 1158