1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 * 5 */ 6 7 /* Support for NVIDIA specific attributes. */ 8 9 #include <linux/io.h> 10 #include <linux/module.h> 11 #include <linux/topology.h> 12 13 #include "arm_cspmu.h" 14 15 #define NV_PCIE_PORT_COUNT 10ULL 16 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0) 17 18 #define NV_NVL_C2C_PORT_COUNT 2ULL 19 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0) 20 21 #define NV_CNVL_PORT_COUNT 4ULL 22 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0) 23 24 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0) 25 26 #define NV_PRODID_MASK GENMASK(31, 0) 27 28 #define NV_FORMAT_NAME_GENERIC 0 29 30 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx)) 31 32 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \ 33 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config) 34 35 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \ 36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \ 37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \ 38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \ 39 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3) 40 41 struct nv_cspmu_ctx { 42 const char *name; 43 u32 filter_mask; 44 u32 filter_default_val; 45 struct attribute **event_attr; 46 struct attribute **format_attr; 47 }; 48 49 static struct attribute *scf_pmu_event_attrs[] = { 50 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d), 51 52 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0), 53 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1), 54 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2), 55 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3), 56 57 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101), 58 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109), 59 60 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115), 61 62 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d), 63 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135), 64 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139), 65 66 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d), 67 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e), 68 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f), 69 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173), 70 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174), 71 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179), 72 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b), 73 74 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c), 75 76 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0), 77 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1), 78 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2), 79 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3), 80 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4), 81 82 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5), 83 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6), 84 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7), 85 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab), 86 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac), 87 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1), 88 89 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca), 90 91 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db), 92 93 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 94 NULL, 95 }; 96 97 static struct attribute *mcf_pmu_event_attrs[] = { 98 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0), 99 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1), 100 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2), 101 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3), 102 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4), 103 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5), 104 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6), 105 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7), 106 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8), 107 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9), 108 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa), 109 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb), 110 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc), 111 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd), 112 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 113 NULL, 114 }; 115 116 static struct attribute *generic_pmu_event_attrs[] = { 117 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 118 NULL, 119 }; 120 121 static struct attribute *scf_pmu_format_attrs[] = { 122 ARM_CSPMU_FORMAT_EVENT_ATTR, 123 NULL, 124 }; 125 126 static struct attribute *pcie_pmu_format_attrs[] = { 127 ARM_CSPMU_FORMAT_EVENT_ATTR, 128 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"), 129 NULL, 130 }; 131 132 static struct attribute *nvlink_c2c_pmu_format_attrs[] = { 133 ARM_CSPMU_FORMAT_EVENT_ATTR, 134 ARM_CSPMU_FORMAT_ATTR(port, "config1:0-1"), 135 NULL, 136 }; 137 138 static struct attribute *cnvlink_pmu_format_attrs[] = { 139 ARM_CSPMU_FORMAT_EVENT_ATTR, 140 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"), 141 NULL, 142 }; 143 144 static struct attribute *generic_pmu_format_attrs[] = { 145 ARM_CSPMU_FORMAT_EVENT_ATTR, 146 ARM_CSPMU_FORMAT_FILTER_ATTR, 147 NULL, 148 }; 149 150 static struct attribute ** 151 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu) 152 { 153 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 154 155 return ctx->event_attr; 156 } 157 158 static struct attribute ** 159 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) 160 { 161 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 162 163 return ctx->format_attr; 164 } 165 166 static const char * 167 nv_cspmu_get_name(const struct arm_cspmu *cspmu) 168 { 169 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 170 171 return ctx->name; 172 } 173 174 static u32 nv_cspmu_event_filter(const struct perf_event *event) 175 { 176 const struct nv_cspmu_ctx *ctx = 177 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu)); 178 179 const u32 filter_val = event->attr.config1 & ctx->filter_mask; 180 181 if (filter_val == 0) 182 return ctx->filter_default_val; 183 184 return filter_val; 185 } 186 187 static void nv_cspmu_set_ev_filter(struct arm_cspmu *cspmu, 188 const struct perf_event *event) 189 { 190 u32 filter = nv_cspmu_event_filter(event); 191 u32 offset = PMEVFILTR + (4 * event->hw.idx); 192 193 writel(filter, cspmu->base0 + offset); 194 } 195 196 static void nv_cspmu_set_cc_filter(struct arm_cspmu *cspmu, 197 const struct perf_event *event) 198 { 199 u32 filter = nv_cspmu_event_filter(event); 200 201 writel(filter, cspmu->base0 + PMCCFILTR); 202 } 203 204 205 enum nv_cspmu_name_fmt { 206 NAME_FMT_GENERIC, 207 NAME_FMT_SOCKET 208 }; 209 210 struct nv_cspmu_match { 211 u32 prodid; 212 u32 prodid_mask; 213 u64 filter_mask; 214 u32 filter_default_val; 215 const char *name_pattern; 216 enum nv_cspmu_name_fmt name_fmt; 217 struct attribute **event_attr; 218 struct attribute **format_attr; 219 }; 220 221 static const struct nv_cspmu_match nv_cspmu_match[] = { 222 { 223 .prodid = 0x103, 224 .prodid_mask = NV_PRODID_MASK, 225 .filter_mask = NV_PCIE_FILTER_ID_MASK, 226 .filter_default_val = NV_PCIE_FILTER_ID_MASK, 227 .name_pattern = "nvidia_pcie_pmu_%u", 228 .name_fmt = NAME_FMT_SOCKET, 229 .event_attr = mcf_pmu_event_attrs, 230 .format_attr = pcie_pmu_format_attrs 231 }, 232 { 233 .prodid = 0x104, 234 .prodid_mask = NV_PRODID_MASK, 235 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, 236 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 237 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u", 238 .name_fmt = NAME_FMT_SOCKET, 239 .event_attr = mcf_pmu_event_attrs, 240 .format_attr = nvlink_c2c_pmu_format_attrs 241 }, 242 { 243 .prodid = 0x105, 244 .prodid_mask = NV_PRODID_MASK, 245 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, 246 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 247 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u", 248 .name_fmt = NAME_FMT_SOCKET, 249 .event_attr = mcf_pmu_event_attrs, 250 .format_attr = nvlink_c2c_pmu_format_attrs 251 }, 252 { 253 .prodid = 0x106, 254 .prodid_mask = NV_PRODID_MASK, 255 .filter_mask = NV_CNVL_FILTER_ID_MASK, 256 .filter_default_val = NV_CNVL_FILTER_ID_MASK, 257 .name_pattern = "nvidia_cnvlink_pmu_%u", 258 .name_fmt = NAME_FMT_SOCKET, 259 .event_attr = mcf_pmu_event_attrs, 260 .format_attr = cnvlink_pmu_format_attrs 261 }, 262 { 263 .prodid = 0x2CF, 264 .prodid_mask = NV_PRODID_MASK, 265 .filter_mask = 0x0, 266 .filter_default_val = 0x0, 267 .name_pattern = "nvidia_scf_pmu_%u", 268 .name_fmt = NAME_FMT_SOCKET, 269 .event_attr = scf_pmu_event_attrs, 270 .format_attr = scf_pmu_format_attrs 271 }, 272 { 273 .prodid = 0, 274 .prodid_mask = 0, 275 .filter_mask = NV_GENERIC_FILTER_ID_MASK, 276 .filter_default_val = NV_GENERIC_FILTER_ID_MASK, 277 .name_pattern = "nvidia_uncore_pmu_%u", 278 .name_fmt = NAME_FMT_GENERIC, 279 .event_attr = generic_pmu_event_attrs, 280 .format_attr = generic_pmu_format_attrs 281 }, 282 }; 283 284 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu, 285 const struct nv_cspmu_match *match) 286 { 287 char *name; 288 struct device *dev = cspmu->dev; 289 290 static atomic_t pmu_generic_idx = {0}; 291 292 switch (match->name_fmt) { 293 case NAME_FMT_SOCKET: { 294 const int cpu = cpumask_first(&cspmu->associated_cpus); 295 const int socket = cpu_to_node(cpu); 296 297 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 298 socket); 299 break; 300 } 301 case NAME_FMT_GENERIC: 302 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 303 atomic_fetch_inc(&pmu_generic_idx)); 304 break; 305 default: 306 name = NULL; 307 break; 308 } 309 310 return name; 311 } 312 313 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) 314 { 315 u32 prodid; 316 struct nv_cspmu_ctx *ctx; 317 struct device *dev = cspmu->dev; 318 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; 319 const struct nv_cspmu_match *match = nv_cspmu_match; 320 321 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL); 322 if (!ctx) 323 return -ENOMEM; 324 325 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr); 326 327 /* Find matching PMU. */ 328 for (; match->prodid; match++) { 329 const u32 prodid_mask = match->prodid_mask; 330 331 if ((match->prodid & prodid_mask) == (prodid & prodid_mask)) 332 break; 333 } 334 335 ctx->name = nv_cspmu_format_name(cspmu, match); 336 ctx->filter_mask = match->filter_mask; 337 ctx->filter_default_val = match->filter_default_val; 338 ctx->event_attr = match->event_attr; 339 ctx->format_attr = match->format_attr; 340 341 cspmu->impl.ctx = ctx; 342 343 /* NVIDIA specific callbacks. */ 344 impl_ops->set_cc_filter = nv_cspmu_set_cc_filter; 345 impl_ops->set_ev_filter = nv_cspmu_set_ev_filter; 346 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs; 347 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs; 348 impl_ops->get_name = nv_cspmu_get_name; 349 350 return 0; 351 } 352 353 /* Match all NVIDIA Coresight PMU devices */ 354 static const struct arm_cspmu_impl_match nv_cspmu_param = { 355 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA, 356 .module = THIS_MODULE, 357 .impl_init_ops = nv_cspmu_init_ops 358 }; 359 360 static int __init nvidia_cspmu_init(void) 361 { 362 int ret; 363 364 ret = arm_cspmu_impl_register(&nv_cspmu_param); 365 if (ret) 366 pr_err("nvidia_cspmu backend registration error: %d\n", ret); 367 368 return ret; 369 } 370 371 static void __exit nvidia_cspmu_exit(void) 372 { 373 arm_cspmu_impl_unregister(&nv_cspmu_param); 374 } 375 376 module_init(nvidia_cspmu_init); 377 module_exit(nvidia_cspmu_exit); 378 379 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver"); 380 MODULE_LICENSE("GPL v2"); 381