1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright(c) 2023 Huawei 5 * 6 * The CXL 3.0 specification includes a standard Performance Monitoring Unit, 7 * called the CXL PMU, or CPMU. In order to allow a high degree of 8 * implementation flexibility the specification provides a wide range of 9 * options all of which are self describing. 10 * 11 * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface 12 */ 13 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/perf_event.h> 16 #include <linux/bitops.h> 17 #include <linux/device.h> 18 #include <linux/bits.h> 19 #include <linux/list.h> 20 #include <linux/bug.h> 21 #include <linux/pci.h> 22 23 #include "../cxl/cxlpci.h" 24 #include "../cxl/cxl.h" 25 #include "../cxl/pmu.h" 26 27 #define CXL_PMU_CAP_REG 0x0 28 #define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(5, 0) 29 #define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8) 30 #define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20) 31 #define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32) 32 #define CXL_PMU_FILTER_HDM BIT(0) 33 #define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1) 34 #define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44) 35 #define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48) 36 #define CXL_PMU_CAP_FREEZE BIT_ULL(49) 37 #define CXL_PMU_CAP_INT BIT_ULL(50) 38 #define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60) 39 40 #define CXL_PMU_OVERFLOW_REG 0x10 41 #define CXL_PMU_FREEZE_REG 0x18 42 #define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n)) 43 #define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0) 44 #define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32) 45 #define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48) 46 47 #define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n)) 48 #define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0) 49 #define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0 50 #define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1 51 #define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2 52 #define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8) 53 #define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9) 54 #define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10) 55 #define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11) 56 #define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12) 57 #define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16) 58 #define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24) 59 #define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59) 60 61 #define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8)) 62 #define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(31, 0) 63 64 #define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n)) 65 66 /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */ 67 #define CXL_PMU_GID_CLOCK_TICKS 0x00 68 #define CXL_PMU_GID_D2H_REQ 0x0010 69 #define CXL_PMU_GID_D2H_RSP 0x0011 70 #define CXL_PMU_GID_H2D_REQ 0x0012 71 #define CXL_PMU_GID_H2D_RSP 0x0013 72 #define CXL_PMU_GID_CACHE_DATA 0x0014 73 #define CXL_PMU_GID_M2S_REQ 0x0020 74 #define CXL_PMU_GID_M2S_RWD 0x0021 75 #define CXL_PMU_GID_M2S_BIRSP 0x0022 76 #define CXL_PMU_GID_S2M_BISNP 0x0023 77 #define CXL_PMU_GID_S2M_NDR 0x0024 78 #define CXL_PMU_GID_S2M_DRS 0x0025 79 #define CXL_PMU_GID_DDR 0x8000 80 #define CXL_PMU_GID_QUEUE_OCC 0x8001 81 #define CXL_PMU_GID_QUEUE_RESID 0x8002 82 #define CXL_PMU_GID_RETRY_EVENTS 0x8003 83 #define CXL_PMU_GID_THROTTLE 0x8004 84 85 static int cxl_pmu_cpuhp_state_num; 86 87 struct cxl_pmu_ev_cap { 88 u16 vid; 89 u16 gid; 90 u32 msk; 91 union { 92 int counter_idx; /* fixed counters */ 93 int event_idx; /* configurable counters */ 94 }; 95 struct list_head node; 96 }; 97 98 #define CXL_PMU_MAX_COUNTERS 64 99 struct cxl_pmu_info { 100 struct pmu pmu; 101 void __iomem *base; 102 struct perf_event **hw_events; 103 struct list_head event_caps_configurable; 104 struct list_head event_caps_fixed; 105 DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS); 106 DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS); 107 u16 counter_width; 108 u8 num_counters; 109 u8 num_event_capabilities; 110 int on_cpu; 111 struct hlist_node node; 112 bool filter_hdm; 113 bool filter_crb; 114 int irq; 115 }; 116 117 #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu) 118 119 /* 120 * All CPMU counters are discoverable via the Event Capabilities Registers. 121 * Each Event Capability register contains a VID / GroupID. 122 * A counter may then count any combination (by summing) of events in 123 * that group which are in the Supported Events Bitmask. 124 * However, there are some complexities to the scheme. 125 * - Fixed function counters refer to an Event Capabilities register. 126 * That event capability register is not then used for Configurable 127 * counters. 128 */ 129 static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) 130 { 131 unsigned long fixed_counter_event_cap_bm = 0; 132 void __iomem *base = info->base; 133 bool freeze_for_enable; 134 u64 val, eval; 135 int i; 136 137 val = readq(base + CXL_PMU_CAP_REG); 138 freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) && 139 FIELD_GET(CXL_PMU_CAP_FREEZE, val); 140 if (!freeze_for_enable) { 141 dev_err(dev, "Counters not writable while frozen\n"); 142 return -ENODEV; 143 } 144 145 info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1; 146 info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val); 147 info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; 148 149 info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; 150 info->filter_crb = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & 151 CXL_PMU_FILTER_CHAN_RANK_BANK; 152 if (FIELD_GET(CXL_PMU_CAP_INT, val)) 153 info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); 154 else 155 info->irq = -1; 156 157 /* First handle fixed function counters; note if configurable counters found */ 158 for (i = 0; i < info->num_counters; i++) { 159 struct cxl_pmu_ev_cap *pmu_ev; 160 u32 events_msk; 161 u8 group_idx; 162 163 val = readq(base + CXL_PMU_COUNTER_CFG_REG(i)); 164 165 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) == 166 CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) { 167 set_bit(i, info->conf_counter_bm); 168 } 169 170 if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) != 171 CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN) 172 continue; 173 174 /* In this case we know which fields are const */ 175 group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val); 176 events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val); 177 eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx)); 178 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 179 if (!pmu_ev) 180 return -ENOMEM; 181 182 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 183 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 184 /* For a fixed purpose counter use the events mask from the counter CFG */ 185 pmu_ev->msk = events_msk; 186 pmu_ev->counter_idx = i; 187 /* This list add is never unwound as all entries deleted on remove */ 188 list_add(&pmu_ev->node, &info->event_caps_fixed); 189 /* 190 * Configurable counters must not use an Event Capability registers that 191 * is in use for a Fixed counter 192 */ 193 set_bit(group_idx, &fixed_counter_event_cap_bm); 194 } 195 196 if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) { 197 struct cxl_pmu_ev_cap *pmu_ev; 198 int j; 199 /* Walk event capabilities unused by fixed counters */ 200 for_each_clear_bit(j, &fixed_counter_event_cap_bm, 201 info->num_event_capabilities) { 202 pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 203 if (!pmu_ev) 204 return -ENOMEM; 205 206 eval = readq(base + CXL_PMU_EVENT_CAP_REG(j)); 207 pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 208 pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 209 pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval); 210 pmu_ev->event_idx = j; 211 list_add(&pmu_ev->node, &info->event_caps_configurable); 212 } 213 } 214 215 return 0; 216 } 217 218 #define CXL_PMU_FORMAT_ATTR(_name, _format)\ 219 (&((struct dev_ext_attribute[]) { \ 220 { \ 221 .attr = __ATTR(_name, 0444, device_show_string, NULL), \ 222 .var = (void *)_format \ 223 } \ 224 })[0].attr.attr) 225 226 enum { 227 cxl_pmu_mask_attr, 228 cxl_pmu_gid_attr, 229 cxl_pmu_vid_attr, 230 cxl_pmu_threshold_attr, 231 cxl_pmu_invert_attr, 232 cxl_pmu_edge_attr, 233 cxl_pmu_hdm_filter_en_attr, 234 cxl_pmu_hdm_attr, 235 cxl_pmu_crb_filter_en_attr, 236 cxl_pmu_crb_attr, 237 }; 238 239 static struct attribute *cxl_pmu_format_attr[] = { 240 [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"), 241 [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"), 242 [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"), 243 [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"), 244 [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"), 245 [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), 246 [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), 247 [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), 248 [cxl_pmu_crb_filter_en_attr] = CXL_PMU_FORMAT_ATTR(crb_filter_en, "config1:19"), 249 [cxl_pmu_crb_attr] = CXL_PMU_FORMAT_ATTR(crb, "config2:32-63"), 250 NULL 251 }; 252 253 #define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0) 254 #define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32) 255 #define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48) 256 #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0) 257 #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) 258 #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) 259 #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) 260 #define CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK BIT(19) 261 #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) 262 #define CXL_PMU_ATTR_CONFIG2_CRB_MSK GENMASK_ULL(63, 32) 263 264 static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, 265 struct attribute *attr, int a) 266 { 267 struct device *dev = kobj_to_dev(kobj); 268 struct cxl_pmu_info *info = dev_get_drvdata(dev); 269 270 /* 271 * Filter capability at the CPMU level, so hide the attributes if the particular 272 * filter is not supported. 273 */ 274 if (!info->filter_hdm && 275 (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] || 276 attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) 277 return 0; 278 279 if (!info->filter_crb && 280 (attr == cxl_pmu_format_attr[cxl_pmu_crb_filter_en_attr] || 281 attr == cxl_pmu_format_attr[cxl_pmu_crb_attr])) 282 return 0; 283 284 return attr->mode; 285 } 286 287 static const struct attribute_group cxl_pmu_format_group = { 288 .name = "format", 289 .attrs = cxl_pmu_format_attr, 290 .is_visible = cxl_pmu_format_is_visible, 291 }; 292 293 static u32 cxl_pmu_config_get_mask(struct perf_event *event) 294 { 295 return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config); 296 } 297 298 static u16 cxl_pmu_config_get_gid(struct perf_event *event) 299 { 300 return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config); 301 } 302 303 static u16 cxl_pmu_config_get_vid(struct perf_event *event) 304 { 305 return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config); 306 } 307 308 static u8 cxl_pmu_config1_get_threshold(struct perf_event *event) 309 { 310 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1); 311 } 312 313 static bool cxl_pmu_config1_get_invert(struct perf_event *event) 314 { 315 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1); 316 } 317 318 static bool cxl_pmu_config1_get_edge(struct perf_event *event) 319 { 320 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1); 321 } 322 323 /* 324 * CPMU specification allows for 8 filters, each with a 32 bit value... 325 * So we need to find 8x32bits to store it in. 326 * As the value used for disable is 0xffff_ffff, a separate enable switch 327 * is needed. 328 */ 329 330 static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event) 331 { 332 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1); 333 } 334 335 static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) 336 { 337 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); 338 } 339 340 static u16 cxl_pmu_config1_crb_filter_en(struct perf_event *event) 341 { 342 return FIELD_GET(CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK, 343 event->attr.config1); 344 } 345 346 static u32 cxl_pmu_config2_get_crb(struct perf_event *event) 347 { 348 return FIELD_GET(CXL_PMU_ATTR_CONFIG2_CRB_MSK, event->attr.config2); 349 } 350 351 static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, 352 struct device_attribute *attr, char *buf) 353 { 354 struct perf_pmu_events_attr *pmu_attr = 355 container_of(attr, struct perf_pmu_events_attr, attr); 356 357 return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id); 358 } 359 360 #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \ 361 PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \ 362 ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk)) 363 364 /* For CXL spec defined events */ 365 #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \ 366 CXL_PMU_EVENT_ATTR(_name, PCI_VENDOR_ID_CXL, _gid, _msk) 367 368 static struct attribute *cxl_pmu_event_attrs[] = { 369 CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)), 370 /* CXL rev 3.0 Table 3-17 - Device to Host Requests */ 371 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)), 372 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)), 373 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)), 374 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)), 375 CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)), 376 CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)), 377 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)), 378 CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)), 379 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)), 380 CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)), 381 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)), 382 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)), 383 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)), 384 CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)), 385 CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)), 386 /* CXL rev 3.0 Table 3-20 - D2H Response Encodings */ 387 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)), 388 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)), 389 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)), 390 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)), 391 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)), 392 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)), 393 CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)), 394 /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */ 395 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)), 396 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)), 397 CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)), 398 /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */ 399 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)), 400 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)), 401 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)), 402 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)), 403 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)), 404 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)), 405 CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)), 406 /* CXL rev 3.0 Table 13-5 directly lists these */ 407 CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)), 408 CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)), 409 /* CXL rev 3.1 Table 3-35 M2S Req Memory Opcodes */ 410 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)), 411 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)), 412 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)), 413 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)), 414 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), 415 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)), 416 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)), 417 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvtee, CXL_PMU_GID_M2S_REQ, BIT(7)), 418 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), 419 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), 420 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), 421 CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvptee, CXL_PMU_GID_M2S_REQ, BIT(11)), 422 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrdtee, CXL_PMU_GID_M2S_REQ, BIT(12)), 423 CXL_PMU_EVENT_CXL_ATTR(m2s_req_teupdate, CXL_PMU_GID_M2S_REQ, BIT(13)), 424 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevcttee, CXL_PMU_GID_M2S_REQ, BIT(14)), 425 CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevctu, CXL_PMU_GID_M2S_REQ, BIT(15)), 426 /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ 427 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), 428 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), 429 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), 430 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfill, CXL_PMU_GID_M2S_RWD, BIT(5)), 431 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrtee, CXL_PMU_GID_M2S_RWD, BIT(9)), 432 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptltee, CXL_PMU_GID_M2S_RWD, BIT(10)), 433 CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfilltee, CXL_PMU_GID_M2S_RWD, BIT(13)), 434 /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ 435 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), 436 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), 437 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)), 438 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)), 439 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)), 440 CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)), 441 /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */ 442 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)), 443 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)), 444 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)), 445 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), 446 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), 447 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), 448 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curtee, CXL_PMU_GID_S2M_BISNP, BIT(8)), 449 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datatee, CXL_PMU_GID_S2M_BISNP, BIT(9)), 450 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invtee, CXL_PMU_GID_S2M_BISNP, BIT(10)), 451 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblktee, CXL_PMU_GID_S2M_BISNP, BIT(12)), 452 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datablktee, CXL_PMU_GID_S2M_BISNP, BIT(13)), 453 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblktee, CXL_PMU_GID_S2M_BISNP, BIT(14)), 454 /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */ 455 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), 456 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), 457 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), 458 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)), 459 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)), 460 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee, CXL_PMU_GID_S2M_NDR, BIT(5)), 461 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_s, CXL_PMU_GID_S2M_NDR, BIT(6)), 462 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_e, CXL_PMU_GID_S2M_NDR, BIT(7)), 463 /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ 464 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), 465 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), 466 CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatatee, CXL_PMU_GID_S2M_DRS, BIT(2)), 467 /* CXL rev 3.0 Table 13-5 directly lists these */ 468 CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), 469 CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), 470 CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)), 471 CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)), 472 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), 473 CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), 474 CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), 475 /* CXL 4.0 Table 13-5 DDR add-on events opcodes */ 476 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_rd_ap, CXL_PMU_GID_DDR, BIT(7)), 477 CXL_PMU_EVENT_CXL_ATTR(ddr_cas_wr_ap, CXL_PMU_GID_DDR, BIT(8)), 478 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_all_banks, CXL_PMU_GID_DDR, BIT(9)), 479 CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_same_bank, CXL_PMU_GID_DDR, BIT(10)), 480 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_entry, CXL_PMU_GID_DDR, BIT(11)), 481 CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_exit, CXL_PMU_GID_DDR, BIT(12)), 482 CXL_PMU_EVENT_CXL_ATTR(ddr_rd_wr_ddr_bus_switching, CXL_PMU_GID_DDR, BIT(13)), 483 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_rd_req, CXL_PMU_GID_DDR, BIT(14)), 484 CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_wr_req, CXL_PMU_GID_DDR, BIT(15)), 485 /* CXL 4.0 Table 13-5 QUEUE OCCUPANCY events opcodes */ 486 CXL_PMU_EVENT_CXL_ATTR(rd_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(0)), 487 CXL_PMU_EVENT_CXL_ATTR(wr_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(1)), 488 CXL_PMU_EVENT_CXL_ATTR(rd_wr_merged_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(2)), 489 CXL_PMU_EVENT_CXL_ATTR(pwrdn_event, CXL_PMU_GID_QUEUE_OCC, BIT(3)), 490 /* CXL 4.0 Table 13-5 QUEUE RESIDENCY events opcodes */ 491 CXL_PMU_EVENT_CXL_ATTR(mc_rd_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(0)), 492 CXL_PMU_EVENT_CXL_ATTR(mc_wr_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(1)), 493 /* CXL 4.0 Table 13-5 RETRY events opcodes */ 494 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_rd_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(0)), 495 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_wr_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(1)), 496 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ca_parity, CXL_PMU_GID_RETRY_EVENTS, BIT(2)), 497 CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ecc, CXL_PMU_GID_RETRY_EVENTS, BIT(3)), 498 /* CXL 4.0 Table 13-5 THROTTLE events opcodes */ 499 CXL_PMU_EVENT_CXL_ATTR(thermal_throttle_event, CXL_PMU_GID_THROTTLE, BIT(0)), 500 CXL_PMU_EVENT_CXL_ATTR(power_throttle_event, CXL_PMU_GID_THROTTLE, BIT(1)), 501 NULL 502 }; 503 504 static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info, 505 int vid, int gid, int msk) 506 { 507 struct cxl_pmu_ev_cap *pmu_ev; 508 509 list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) { 510 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 511 continue; 512 513 /* Precise match for fixed counter */ 514 if (msk == pmu_ev->msk) 515 return pmu_ev; 516 } 517 518 return ERR_PTR(-EINVAL); 519 } 520 521 static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info, 522 int vid, int gid, int msk) 523 { 524 struct cxl_pmu_ev_cap *pmu_ev; 525 526 list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) { 527 if (vid != pmu_ev->vid || gid != pmu_ev->gid) 528 continue; 529 530 /* Request mask must be subset of supported */ 531 if (msk & ~pmu_ev->msk) 532 continue; 533 534 return pmu_ev; 535 } 536 537 return ERR_PTR(-EINVAL); 538 } 539 540 static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a) 541 { 542 struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr); 543 struct perf_pmu_events_attr *pmu_attr = 544 container_of(dev_attr, struct perf_pmu_events_attr, attr); 545 struct device *dev = kobj_to_dev(kobj); 546 struct cxl_pmu_info *info = dev_get_drvdata(dev); 547 int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id); 548 int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id); 549 int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id); 550 551 if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk))) 552 return attr->mode; 553 554 if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk))) 555 return attr->mode; 556 557 return 0; 558 } 559 560 static const struct attribute_group cxl_pmu_events = { 561 .name = "events", 562 .attrs = cxl_pmu_event_attrs, 563 .is_visible = cxl_pmu_event_is_visible, 564 }; 565 566 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 567 char *buf) 568 { 569 struct cxl_pmu_info *info = dev_get_drvdata(dev); 570 571 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(info->on_cpu))); 572 } 573 static DEVICE_ATTR_RO(cpumask); 574 575 static struct attribute *cxl_pmu_cpumask_attrs[] = { 576 &dev_attr_cpumask.attr, 577 NULL 578 }; 579 580 static const struct attribute_group cxl_pmu_cpumask_group = { 581 .attrs = cxl_pmu_cpumask_attrs, 582 }; 583 584 static const struct attribute_group *cxl_pmu_attr_groups[] = { 585 &cxl_pmu_events, 586 &cxl_pmu_format_group, 587 &cxl_pmu_cpumask_group, 588 NULL 589 }; 590 591 /* If counter_idx == NULL, don't try to allocate a counter. */ 592 static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx, 593 int *event_idx) 594 { 595 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 596 DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS); 597 struct cxl_pmu_ev_cap *pmu_ev; 598 u32 mask; 599 u16 gid, vid; 600 int i; 601 602 vid = cxl_pmu_config_get_vid(event); 603 gid = cxl_pmu_config_get_gid(event); 604 mask = cxl_pmu_config_get_mask(event); 605 606 pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask); 607 if (!IS_ERR(pmu_ev)) { 608 if (!counter_idx) 609 return 0; 610 if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) { 611 *counter_idx = pmu_ev->counter_idx; 612 return 0; 613 } 614 /* Fixed counter is in use, but maybe a configurable one? */ 615 } 616 617 pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask); 618 if (!IS_ERR(pmu_ev)) { 619 if (!counter_idx) 620 return 0; 621 622 bitmap_andnot(configurable_and_free, info->conf_counter_bm, 623 info->used_counter_bm, CXL_PMU_MAX_COUNTERS); 624 625 i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS); 626 if (i == CXL_PMU_MAX_COUNTERS) 627 return -EINVAL; 628 629 *counter_idx = i; 630 return 0; 631 } 632 633 return -EINVAL; 634 } 635 636 static int cxl_pmu_event_init(struct perf_event *event) 637 { 638 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 639 int rc; 640 641 /* Top level type sanity check - is this a Hardware Event being requested */ 642 if (event->attr.type != event->pmu->type) 643 return -ENOENT; 644 645 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 646 return -EOPNOTSUPP; 647 /* TODO: Validation of any filter */ 648 649 if (cxl_pmu_config1_crb_filter_en(event)) { 650 if (!info->filter_crb) 651 return -EINVAL; 652 /* event group IDs are scoped by the CXL vendor ID */ 653 if (cxl_pmu_config_get_vid(event) != PCI_VENDOR_ID_CXL) 654 return -EINVAL; 655 656 /* 657 * CRB filtering (Filter ID 1) is only valid for the DDR 658 * Interface, Queue Occupancy, Queue Residency and Retry 659 * event groups (CXL 4.0 Table 13-5). 660 */ 661 switch (cxl_pmu_config_get_gid(event)) { 662 case CXL_PMU_GID_DDR: 663 case CXL_PMU_GID_QUEUE_OCC: 664 case CXL_PMU_GID_QUEUE_RESID: 665 case CXL_PMU_GID_RETRY_EVENTS: 666 break; 667 default: 668 return -EINVAL; 669 } 670 671 /* 672 * Filtering while counting multiple events is 673 * undefined behavior. 674 */ 675 if (hweight32(cxl_pmu_config_get_mask(event)) > 1) 676 return -EINVAL; 677 } 678 679 /* 680 * Verify that it is possible to count what was requested. Either must 681 * be a fixed counter that is a precise match or a configurable counter 682 * where this is a subset. 683 */ 684 rc = cxl_pmu_get_event_idx(event, NULL, NULL); 685 if (rc < 0) 686 return rc; 687 688 event->cpu = info->on_cpu; 689 690 return 0; 691 } 692 693 static void cxl_pmu_enable(struct pmu *pmu) 694 { 695 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 696 void __iomem *base = info->base; 697 698 /* Can assume frozen at this stage */ 699 writeq(0, base + CXL_PMU_FREEZE_REG); 700 } 701 702 static void cxl_pmu_disable(struct pmu *pmu) 703 { 704 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 705 void __iomem *base = info->base; 706 707 /* 708 * Whilst bits above number of counters are RsvdZ 709 * they are unlikely to be repurposed given 710 * number of counters is allowed to be 64 leaving 711 * no reserved bits. Hence this is only slightly 712 * naughty. 713 */ 714 writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG); 715 } 716 717 static void cxl_pmu_event_start(struct perf_event *event, int flags) 718 { 719 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 720 struct hw_perf_event *hwc = &event->hw; 721 void __iomem *base = info->base; 722 u64 cfg; 723 724 /* 725 * All paths to here should either set these flags directly or 726 * call cxl_pmu_event_stop() which will ensure the correct state. 727 */ 728 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 729 return; 730 731 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 732 hwc->state = 0; 733 734 /* 735 * Filter ID=0: HDM decoder filter 736 * Filter ID=1: Channel/Rank/Bank (CRB) filter 737 */ 738 if (info->filter_hdm) { 739 if (cxl_pmu_config1_hdm_filter_en(event)) 740 cfg = cxl_pmu_config2_get_hdm_decoder(event); 741 else 742 cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ 743 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); 744 } 745 746 if (info->filter_crb) { 747 if (cxl_pmu_config1_crb_filter_en(event)) 748 cfg = cxl_pmu_config2_get_crb(event); 749 else 750 cfg = GENMASK(31, 0); /* no filtering if 0xFFFF_FFFF */ 751 writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 1)); 752 } 753 754 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 755 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1); 756 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1); 757 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1); 758 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE, 759 cxl_pmu_config1_get_edge(event) ? 1 : 0); 760 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT, 761 cxl_pmu_config1_get_invert(event) ? 1 : 0); 762 763 /* Fixed purpose counters have next two fields RO */ 764 if (test_bit(hwc->idx, info->conf_counter_bm)) { 765 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, 766 hwc->event_base); 767 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK, 768 cxl_pmu_config_get_mask(event)); 769 } 770 cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK; 771 /* 772 * For events that generate only 1 count per clock the CXL 3.0 spec 773 * states the threshold shall be set to 1 but if set to 0 it will 774 * count the raw value anwyay? 775 * There is no definition of what events will count multiple per cycle 776 * and hence to which non 1 values of threshold can apply. 777 * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition) 778 */ 779 cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK, 780 cxl_pmu_config1_get_threshold(event)); 781 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 782 783 local64_set(&hwc->prev_count, 0); 784 writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx)); 785 786 perf_event_update_userpage(event); 787 } 788 789 static u64 cxl_pmu_read_counter(struct perf_event *event) 790 { 791 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 792 void __iomem *base = info->base; 793 794 return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx)); 795 } 796 797 static void __cxl_pmu_read(struct perf_event *event, bool overflow) 798 { 799 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 800 struct hw_perf_event *hwc = &event->hw; 801 u64 new_cnt, prev_cnt, delta; 802 803 do { 804 prev_cnt = local64_read(&hwc->prev_count); 805 new_cnt = cxl_pmu_read_counter(event); 806 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt); 807 808 /* 809 * If we know an overflow occur then take that into account. 810 * Note counter is not reset as that would lose events 811 */ 812 delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0); 813 if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0)) 814 delta += (1UL << info->counter_width); 815 816 local64_add(delta, &event->count); 817 } 818 819 static void cxl_pmu_read(struct perf_event *event) 820 { 821 __cxl_pmu_read(event, false); 822 } 823 824 static void cxl_pmu_event_stop(struct perf_event *event, int flags) 825 { 826 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 827 void __iomem *base = info->base; 828 struct hw_perf_event *hwc = &event->hw; 829 u64 cfg; 830 831 cxl_pmu_read(event); 832 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 833 hwc->state |= PERF_HES_STOPPED; 834 835 cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 836 cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) | 837 FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1)); 838 writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 839 840 hwc->state |= PERF_HES_UPTODATE; 841 } 842 843 static int cxl_pmu_event_add(struct perf_event *event, int flags) 844 { 845 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 846 struct hw_perf_event *hwc = &event->hw; 847 int idx, rc; 848 int event_idx = 0; 849 850 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 851 852 rc = cxl_pmu_get_event_idx(event, &idx, &event_idx); 853 if (rc < 0) 854 return rc; 855 856 hwc->idx = idx; 857 858 /* Only set for configurable counters */ 859 hwc->event_base = event_idx; 860 info->hw_events[idx] = event; 861 set_bit(idx, info->used_counter_bm); 862 863 if (flags & PERF_EF_START) 864 cxl_pmu_event_start(event, PERF_EF_RELOAD); 865 866 return 0; 867 } 868 869 static void cxl_pmu_event_del(struct perf_event *event, int flags) 870 { 871 struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 872 struct hw_perf_event *hwc = &event->hw; 873 874 cxl_pmu_event_stop(event, PERF_EF_UPDATE); 875 clear_bit(hwc->idx, info->used_counter_bm); 876 info->hw_events[hwc->idx] = NULL; 877 perf_event_update_userpage(event); 878 } 879 880 static irqreturn_t cxl_pmu_irq(int irq, void *data) 881 { 882 struct cxl_pmu_info *info = data; 883 void __iomem *base = info->base; 884 u64 overflowed; 885 DECLARE_BITMAP(overflowedbm, 64); 886 int i; 887 888 overflowed = readq(base + CXL_PMU_OVERFLOW_REG); 889 890 /* Interrupt may be shared, so maybe it isn't ours */ 891 if (!overflowed) 892 return IRQ_NONE; 893 894 bitmap_from_arr64(overflowedbm, &overflowed, 64); 895 for_each_set_bit(i, overflowedbm, info->num_counters) { 896 struct perf_event *event = info->hw_events[i]; 897 898 if (!event) { 899 dev_dbg(info->pmu.dev, 900 "overflow but on non enabled counter %d\n", i); 901 continue; 902 } 903 904 __cxl_pmu_read(event, true); 905 } 906 907 writeq(overflowed, base + CXL_PMU_OVERFLOW_REG); 908 909 return IRQ_HANDLED; 910 } 911 912 static void cxl_pmu_perf_unregister(void *_info) 913 { 914 struct cxl_pmu_info *info = _info; 915 916 perf_pmu_unregister(&info->pmu); 917 } 918 919 static void cxl_pmu_cpuhp_remove(void *_info) 920 { 921 struct cxl_pmu_info *info = _info; 922 923 cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node); 924 } 925 926 static int cxl_pmu_probe(struct device *dev) 927 { 928 struct cxl_pmu *pmu = to_cxl_pmu(dev); 929 struct pci_dev *pdev = to_pci_dev(dev->parent); 930 struct cxl_pmu_info *info; 931 char *irq_name; 932 char *dev_name; 933 int rc, irq; 934 935 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); 936 if (!info) 937 return -ENOMEM; 938 939 dev_set_drvdata(dev, info); 940 INIT_LIST_HEAD(&info->event_caps_fixed); 941 INIT_LIST_HEAD(&info->event_caps_configurable); 942 943 info->base = pmu->base; 944 945 info->on_cpu = -1; 946 rc = cxl_pmu_parse_caps(dev, info); 947 if (rc) 948 return rc; 949 950 info->hw_events = devm_kcalloc(dev, info->num_counters, 951 sizeof(*info->hw_events), GFP_KERNEL); 952 if (!info->hw_events) 953 return -ENOMEM; 954 955 switch (pmu->type) { 956 case CXL_PMU_MEMDEV: 957 dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d", 958 pmu->assoc_id, pmu->index); 959 break; 960 } 961 if (!dev_name) 962 return -ENOMEM; 963 964 info->pmu = (struct pmu) { 965 .name = dev_name, 966 .parent = dev, 967 .module = THIS_MODULE, 968 .event_init = cxl_pmu_event_init, 969 .pmu_enable = cxl_pmu_enable, 970 .pmu_disable = cxl_pmu_disable, 971 .add = cxl_pmu_event_add, 972 .del = cxl_pmu_event_del, 973 .start = cxl_pmu_event_start, 974 .stop = cxl_pmu_event_stop, 975 .read = cxl_pmu_read, 976 .task_ctx_nr = perf_invalid_context, 977 .attr_groups = cxl_pmu_attr_groups, 978 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 979 }; 980 981 if (info->irq <= 0) 982 return -EINVAL; 983 984 rc = pci_irq_vector(pdev, info->irq); 985 if (rc < 0) 986 return rc; 987 irq = rc; 988 989 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name); 990 if (!irq_name) 991 return -ENOMEM; 992 993 rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_NO_THREAD, 994 irq_name, info); 995 if (rc) 996 return rc; 997 info->irq = irq; 998 999 rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node); 1000 if (rc) 1001 return rc; 1002 1003 rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info); 1004 if (rc) 1005 return rc; 1006 1007 rc = perf_pmu_register(&info->pmu, info->pmu.name, -1); 1008 if (rc) 1009 return rc; 1010 1011 rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info); 1012 if (rc) 1013 return rc; 1014 1015 return 0; 1016 } 1017 1018 static struct cxl_driver cxl_pmu_driver = { 1019 .name = "cxl_pmu", 1020 .probe = cxl_pmu_probe, 1021 .id = CXL_DEVICE_PMU, 1022 }; 1023 1024 static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 1025 { 1026 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 1027 1028 if (info->on_cpu != -1) 1029 return 0; 1030 1031 info->on_cpu = cpu; 1032 /* 1033 * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet 1034 * gone away again. 1035 */ 1036 WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu))); 1037 1038 return 0; 1039 } 1040 1041 static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1042 { 1043 struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 1044 unsigned int target; 1045 1046 if (info->on_cpu != cpu) 1047 return 0; 1048 1049 info->on_cpu = -1; 1050 target = cpumask_any_but(cpu_online_mask, cpu); 1051 if (target >= nr_cpu_ids) { 1052 dev_err(info->pmu.dev, "Unable to find a suitable CPU\n"); 1053 return 0; 1054 } 1055 1056 perf_pmu_migrate_context(&info->pmu, cpu, target); 1057 info->on_cpu = target; 1058 /* 1059 * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet 1060 * gone away. 1061 */ 1062 WARN_ON(irq_set_affinity(info->irq, cpumask_of(target))); 1063 1064 return 0; 1065 } 1066 1067 static __init int cxl_pmu_init(void) 1068 { 1069 int rc; 1070 1071 rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1072 "AP_PERF_CXL_PMU_ONLINE", 1073 cxl_pmu_online_cpu, cxl_pmu_offline_cpu); 1074 if (rc < 0) 1075 return rc; 1076 cxl_pmu_cpuhp_state_num = rc; 1077 1078 rc = cxl_driver_register(&cxl_pmu_driver); 1079 if (rc) 1080 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 1081 1082 return rc; 1083 } 1084 1085 static __exit void cxl_pmu_exit(void) 1086 { 1087 cxl_driver_unregister(&cxl_pmu_driver); 1088 cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 1089 } 1090 1091 MODULE_DESCRIPTION("CXL Performance Monitor Driver"); 1092 MODULE_LICENSE("GPL"); 1093 MODULE_IMPORT_NS("CXL"); 1094 module_init(cxl_pmu_init); 1095 module_exit(cxl_pmu_exit); 1096 MODULE_ALIAS_CXL(CXL_DEVICE_PMU); 1097