1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 * 5 */ 6 7 /* Support for NVIDIA specific attributes. */ 8 9 #include <linux/module.h> 10 #include <linux/topology.h> 11 12 #include "arm_cspmu.h" 13 14 #define NV_PCIE_PORT_COUNT 10ULL 15 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0) 16 17 #define NV_NVL_C2C_PORT_COUNT 2ULL 18 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0) 19 20 #define NV_CNVL_PORT_COUNT 4ULL 21 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0) 22 23 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0) 24 25 #define NV_PRODID_MASK GENMASK(31, 0) 26 27 #define NV_FORMAT_NAME_GENERIC 0 28 29 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx)) 30 31 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \ 32 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config) 33 34 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \ 35 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \ 36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \ 37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \ 38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3) 39 40 struct nv_cspmu_ctx { 41 const char *name; 42 u32 filter_mask; 43 u32 filter_default_val; 44 struct attribute **event_attr; 45 struct attribute **format_attr; 46 }; 47 48 static struct attribute *scf_pmu_event_attrs[] = { 49 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d), 50 51 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0), 52 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1), 53 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2), 54 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3), 55 56 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101), 57 NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp, 0x105), 58 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109), 59 NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp, 0x10d), 60 NV_CSPMU_EVENT_ATTR_4(socket, prb_data, 0x111), 61 62 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115), 63 NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding, 0x119), 64 NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding, 0x11d), 65 NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding, 0x121), 66 NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding, 0x125), 67 NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding, 0x129), 68 69 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d), 70 NV_CSPMU_EVENT_ATTR_4(socket, dl_access, 0x131), 71 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135), 72 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139), 73 NV_CSPMU_EVENT_ATTR_4(socket, ev_access, 0x13d), 74 NV_CSPMU_EVENT_ATTR_4(socket, prb_access, 0x141), 75 76 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data, 0x145), 77 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access, 0x149), 78 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access, 0x14d), 79 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding, 0x151), 80 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding, 0x155), 81 82 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data, 0x159), 83 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access, 0x15d), 84 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access, 0x161), 85 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding, 0x165), 86 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding, 0x169), 87 88 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d), 89 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e), 90 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f), 91 ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp, 0x170), 92 ARM_CSPMU_EVENT_ATTR(gmem_dl_access, 0x171), 93 ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding, 0x172), 94 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173), 95 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174), 96 ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding, 0x175), 97 ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp, 0x176), 98 ARM_CSPMU_EVENT_ATTR(gmem_ev_access, 0x177), 99 ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding, 0x178), 100 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179), 101 ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding, 0x17a), 102 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b), 103 104 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c), 105 106 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data, 0x180), 107 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data, 0x184), 108 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access, 0x188), 109 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding, 0x18c), 110 111 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data, 0x190), 112 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data, 0x194), 113 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access, 0x198), 114 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding, 0x19c), 115 116 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0), 117 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1), 118 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2), 119 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3), 120 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4), 121 122 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5), 123 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6), 124 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7), 125 ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp, 0x1a8), 126 ARM_CSPMU_EVENT_ATTR(cmem_dl_access, 0x1a9), 127 ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding, 0x1aa), 128 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab), 129 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac), 130 ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding, 0x1ad), 131 ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp, 0x1ae), 132 ARM_CSPMU_EVENT_ATTR(cmem_ev_access, 0x1af), 133 ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding, 0x1b0), 134 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1), 135 ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding, 0x1b2), 136 137 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data, 0x1b3), 138 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access, 0x1b7), 139 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access, 0x1bb), 140 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding, 0x1bf), 141 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding, 0x1c3), 142 143 ARM_CSPMU_EVENT_ATTR(ocu_prb_access, 0x1c7), 144 ARM_CSPMU_EVENT_ATTR(ocu_prb_data, 0x1c8), 145 ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding, 0x1c9), 146 147 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca), 148 149 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access, 0x1cb), 150 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data, 0x1cf), 151 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data, 0x1d3), 152 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding, 0x1d7), 153 154 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db), 155 156 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 157 NULL, 158 }; 159 160 static struct attribute *mcf_pmu_event_attrs[] = { 161 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0), 162 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1), 163 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2), 164 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3), 165 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4), 166 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5), 167 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6), 168 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7), 169 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8), 170 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9), 171 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa), 172 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb), 173 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc), 174 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd), 175 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 176 NULL, 177 }; 178 179 static struct attribute *generic_pmu_event_attrs[] = { 180 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 181 NULL, 182 }; 183 184 static struct attribute *scf_pmu_format_attrs[] = { 185 ARM_CSPMU_FORMAT_EVENT_ATTR, 186 NULL, 187 }; 188 189 static struct attribute *pcie_pmu_format_attrs[] = { 190 ARM_CSPMU_FORMAT_EVENT_ATTR, 191 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"), 192 NULL, 193 }; 194 195 static struct attribute *nvlink_c2c_pmu_format_attrs[] = { 196 ARM_CSPMU_FORMAT_EVENT_ATTR, 197 NULL, 198 }; 199 200 static struct attribute *cnvlink_pmu_format_attrs[] = { 201 ARM_CSPMU_FORMAT_EVENT_ATTR, 202 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"), 203 NULL, 204 }; 205 206 static struct attribute *generic_pmu_format_attrs[] = { 207 ARM_CSPMU_FORMAT_EVENT_ATTR, 208 ARM_CSPMU_FORMAT_FILTER_ATTR, 209 NULL, 210 }; 211 212 static struct attribute ** 213 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu) 214 { 215 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 216 217 return ctx->event_attr; 218 } 219 220 static struct attribute ** 221 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) 222 { 223 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 224 225 return ctx->format_attr; 226 } 227 228 static const char * 229 nv_cspmu_get_name(const struct arm_cspmu *cspmu) 230 { 231 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 232 233 return ctx->name; 234 } 235 236 static u32 nv_cspmu_event_filter(const struct perf_event *event) 237 { 238 const struct nv_cspmu_ctx *ctx = 239 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu)); 240 241 if (ctx->filter_mask == 0) 242 return ctx->filter_default_val; 243 244 return event->attr.config1 & ctx->filter_mask; 245 } 246 247 enum nv_cspmu_name_fmt { 248 NAME_FMT_GENERIC, 249 NAME_FMT_SOCKET 250 }; 251 252 struct nv_cspmu_match { 253 u32 prodid; 254 u32 prodid_mask; 255 u64 filter_mask; 256 u32 filter_default_val; 257 const char *name_pattern; 258 enum nv_cspmu_name_fmt name_fmt; 259 struct attribute **event_attr; 260 struct attribute **format_attr; 261 }; 262 263 static const struct nv_cspmu_match nv_cspmu_match[] = { 264 { 265 .prodid = 0x103, 266 .prodid_mask = NV_PRODID_MASK, 267 .filter_mask = NV_PCIE_FILTER_ID_MASK, 268 .filter_default_val = NV_PCIE_FILTER_ID_MASK, 269 .name_pattern = "nvidia_pcie_pmu_%u", 270 .name_fmt = NAME_FMT_SOCKET, 271 .event_attr = mcf_pmu_event_attrs, 272 .format_attr = pcie_pmu_format_attrs 273 }, 274 { 275 .prodid = 0x104, 276 .prodid_mask = NV_PRODID_MASK, 277 .filter_mask = 0x0, 278 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 279 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u", 280 .name_fmt = NAME_FMT_SOCKET, 281 .event_attr = mcf_pmu_event_attrs, 282 .format_attr = nvlink_c2c_pmu_format_attrs 283 }, 284 { 285 .prodid = 0x105, 286 .prodid_mask = NV_PRODID_MASK, 287 .filter_mask = 0x0, 288 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 289 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u", 290 .name_fmt = NAME_FMT_SOCKET, 291 .event_attr = mcf_pmu_event_attrs, 292 .format_attr = nvlink_c2c_pmu_format_attrs 293 }, 294 { 295 .prodid = 0x106, 296 .prodid_mask = NV_PRODID_MASK, 297 .filter_mask = NV_CNVL_FILTER_ID_MASK, 298 .filter_default_val = NV_CNVL_FILTER_ID_MASK, 299 .name_pattern = "nvidia_cnvlink_pmu_%u", 300 .name_fmt = NAME_FMT_SOCKET, 301 .event_attr = mcf_pmu_event_attrs, 302 .format_attr = cnvlink_pmu_format_attrs 303 }, 304 { 305 .prodid = 0x2CF, 306 .prodid_mask = NV_PRODID_MASK, 307 .filter_mask = 0x0, 308 .filter_default_val = 0x0, 309 .name_pattern = "nvidia_scf_pmu_%u", 310 .name_fmt = NAME_FMT_SOCKET, 311 .event_attr = scf_pmu_event_attrs, 312 .format_attr = scf_pmu_format_attrs 313 }, 314 { 315 .prodid = 0, 316 .prodid_mask = 0, 317 .filter_mask = NV_GENERIC_FILTER_ID_MASK, 318 .filter_default_val = NV_GENERIC_FILTER_ID_MASK, 319 .name_pattern = "nvidia_uncore_pmu_%u", 320 .name_fmt = NAME_FMT_GENERIC, 321 .event_attr = generic_pmu_event_attrs, 322 .format_attr = generic_pmu_format_attrs 323 }, 324 }; 325 326 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu, 327 const struct nv_cspmu_match *match) 328 { 329 char *name; 330 struct device *dev = cspmu->dev; 331 332 static atomic_t pmu_generic_idx = {0}; 333 334 switch (match->name_fmt) { 335 case NAME_FMT_SOCKET: { 336 const int cpu = cpumask_first(&cspmu->associated_cpus); 337 const int socket = cpu_to_node(cpu); 338 339 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 340 socket); 341 break; 342 } 343 case NAME_FMT_GENERIC: 344 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 345 atomic_fetch_inc(&pmu_generic_idx)); 346 break; 347 default: 348 name = NULL; 349 break; 350 } 351 352 return name; 353 } 354 355 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) 356 { 357 u32 prodid; 358 struct nv_cspmu_ctx *ctx; 359 struct device *dev = cspmu->dev; 360 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; 361 const struct nv_cspmu_match *match = nv_cspmu_match; 362 363 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL); 364 if (!ctx) 365 return -ENOMEM; 366 367 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr); 368 369 /* Find matching PMU. */ 370 for (; match->prodid; match++) { 371 const u32 prodid_mask = match->prodid_mask; 372 373 if ((match->prodid & prodid_mask) == (prodid & prodid_mask)) 374 break; 375 } 376 377 ctx->name = nv_cspmu_format_name(cspmu, match); 378 ctx->filter_mask = match->filter_mask; 379 ctx->filter_default_val = match->filter_default_val; 380 ctx->event_attr = match->event_attr; 381 ctx->format_attr = match->format_attr; 382 383 cspmu->impl.ctx = ctx; 384 385 /* NVIDIA specific callbacks. */ 386 impl_ops->event_filter = nv_cspmu_event_filter; 387 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs; 388 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs; 389 impl_ops->get_name = nv_cspmu_get_name; 390 391 return 0; 392 } 393 394 /* Match all NVIDIA Coresight PMU devices */ 395 static const struct arm_cspmu_impl_match nv_cspmu_param = { 396 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA, 397 .module = THIS_MODULE, 398 .impl_init_ops = nv_cspmu_init_ops 399 }; 400 401 static int __init nvidia_cspmu_init(void) 402 { 403 int ret; 404 405 ret = arm_cspmu_impl_register(&nv_cspmu_param); 406 if (ret) 407 pr_err("nvidia_cspmu backend registration error: %d\n", ret); 408 409 return ret; 410 } 411 412 static void __exit nvidia_cspmu_exit(void) 413 { 414 arm_cspmu_impl_unregister(&nv_cspmu_param); 415 } 416 417 module_init(nvidia_cspmu_init); 418 module_exit(nvidia_cspmu_exit); 419 420 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver"); 421 MODULE_LICENSE("GPL v2"); 422