1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 * 5 */ 6 7 /* Support for NVIDIA specific attributes. */ 8 9 #include <linux/module.h> 10 #include <linux/topology.h> 11 12 #include "arm_cspmu.h" 13 14 #define NV_PCIE_PORT_COUNT 10ULL 15 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0) 16 17 #define NV_NVL_C2C_PORT_COUNT 2ULL 18 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0) 19 20 #define NV_CNVL_PORT_COUNT 4ULL 21 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0) 22 23 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0) 24 25 #define NV_PRODID_MASK GENMASK(31, 0) 26 27 #define NV_FORMAT_NAME_GENERIC 0 28 29 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx)) 30 31 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \ 32 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config) 33 34 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \ 35 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \ 36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \ 37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \ 38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3) 39 40 struct nv_cspmu_ctx { 41 const char *name; 42 u32 filter_mask; 43 u32 filter_default_val; 44 struct attribute **event_attr; 45 struct attribute **format_attr; 46 }; 47 48 static struct attribute *scf_pmu_event_attrs[] = { 49 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d), 50 51 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0), 52 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1), 53 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2), 54 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3), 55 56 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101), 57 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109), 58 59 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115), 60 61 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d), 62 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135), 63 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139), 64 65 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d), 66 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e), 67 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f), 68 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173), 69 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174), 70 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179), 71 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b), 72 73 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c), 74 75 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0), 76 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1), 77 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2), 78 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3), 79 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4), 80 81 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5), 82 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6), 83 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7), 84 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab), 85 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac), 86 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1), 87 88 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca), 89 90 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db), 91 92 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 93 NULL, 94 }; 95 96 static struct attribute *mcf_pmu_event_attrs[] = { 97 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0), 98 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1), 99 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2), 100 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3), 101 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4), 102 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5), 103 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6), 104 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7), 105 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8), 106 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9), 107 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa), 108 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb), 109 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc), 110 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd), 111 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 112 NULL, 113 }; 114 115 static struct attribute *generic_pmu_event_attrs[] = { 116 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT), 117 NULL, 118 }; 119 120 static struct attribute *scf_pmu_format_attrs[] = { 121 ARM_CSPMU_FORMAT_EVENT_ATTR, 122 NULL, 123 }; 124 125 static struct attribute *pcie_pmu_format_attrs[] = { 126 ARM_CSPMU_FORMAT_EVENT_ATTR, 127 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"), 128 NULL, 129 }; 130 131 static struct attribute *nvlink_c2c_pmu_format_attrs[] = { 132 ARM_CSPMU_FORMAT_EVENT_ATTR, 133 ARM_CSPMU_FORMAT_ATTR(port, "config1:0-1"), 134 NULL, 135 }; 136 137 static struct attribute *cnvlink_pmu_format_attrs[] = { 138 ARM_CSPMU_FORMAT_EVENT_ATTR, 139 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"), 140 NULL, 141 }; 142 143 static struct attribute *generic_pmu_format_attrs[] = { 144 ARM_CSPMU_FORMAT_EVENT_ATTR, 145 ARM_CSPMU_FORMAT_FILTER_ATTR, 146 NULL, 147 }; 148 149 static struct attribute ** 150 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu) 151 { 152 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 153 154 return ctx->event_attr; 155 } 156 157 static struct attribute ** 158 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) 159 { 160 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 161 162 return ctx->format_attr; 163 } 164 165 static const char * 166 nv_cspmu_get_name(const struct arm_cspmu *cspmu) 167 { 168 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu); 169 170 return ctx->name; 171 } 172 173 static u32 nv_cspmu_event_filter(const struct perf_event *event) 174 { 175 const struct nv_cspmu_ctx *ctx = 176 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu)); 177 178 const u32 filter_val = event->attr.config1 & ctx->filter_mask; 179 180 if (filter_val == 0) 181 return ctx->filter_default_val; 182 183 return filter_val; 184 } 185 186 enum nv_cspmu_name_fmt { 187 NAME_FMT_GENERIC, 188 NAME_FMT_SOCKET 189 }; 190 191 struct nv_cspmu_match { 192 u32 prodid; 193 u32 prodid_mask; 194 u64 filter_mask; 195 u32 filter_default_val; 196 const char *name_pattern; 197 enum nv_cspmu_name_fmt name_fmt; 198 struct attribute **event_attr; 199 struct attribute **format_attr; 200 }; 201 202 static const struct nv_cspmu_match nv_cspmu_match[] = { 203 { 204 .prodid = 0x103, 205 .prodid_mask = NV_PRODID_MASK, 206 .filter_mask = NV_PCIE_FILTER_ID_MASK, 207 .filter_default_val = NV_PCIE_FILTER_ID_MASK, 208 .name_pattern = "nvidia_pcie_pmu_%u", 209 .name_fmt = NAME_FMT_SOCKET, 210 .event_attr = mcf_pmu_event_attrs, 211 .format_attr = pcie_pmu_format_attrs 212 }, 213 { 214 .prodid = 0x104, 215 .prodid_mask = NV_PRODID_MASK, 216 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, 217 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 218 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u", 219 .name_fmt = NAME_FMT_SOCKET, 220 .event_attr = mcf_pmu_event_attrs, 221 .format_attr = nvlink_c2c_pmu_format_attrs 222 }, 223 { 224 .prodid = 0x105, 225 .prodid_mask = NV_PRODID_MASK, 226 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, 227 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, 228 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u", 229 .name_fmt = NAME_FMT_SOCKET, 230 .event_attr = mcf_pmu_event_attrs, 231 .format_attr = nvlink_c2c_pmu_format_attrs 232 }, 233 { 234 .prodid = 0x106, 235 .prodid_mask = NV_PRODID_MASK, 236 .filter_mask = NV_CNVL_FILTER_ID_MASK, 237 .filter_default_val = NV_CNVL_FILTER_ID_MASK, 238 .name_pattern = "nvidia_cnvlink_pmu_%u", 239 .name_fmt = NAME_FMT_SOCKET, 240 .event_attr = mcf_pmu_event_attrs, 241 .format_attr = cnvlink_pmu_format_attrs 242 }, 243 { 244 .prodid = 0x2CF, 245 .prodid_mask = NV_PRODID_MASK, 246 .filter_mask = 0x0, 247 .filter_default_val = 0x0, 248 .name_pattern = "nvidia_scf_pmu_%u", 249 .name_fmt = NAME_FMT_SOCKET, 250 .event_attr = scf_pmu_event_attrs, 251 .format_attr = scf_pmu_format_attrs 252 }, 253 { 254 .prodid = 0, 255 .prodid_mask = 0, 256 .filter_mask = NV_GENERIC_FILTER_ID_MASK, 257 .filter_default_val = NV_GENERIC_FILTER_ID_MASK, 258 .name_pattern = "nvidia_uncore_pmu_%u", 259 .name_fmt = NAME_FMT_GENERIC, 260 .event_attr = generic_pmu_event_attrs, 261 .format_attr = generic_pmu_format_attrs 262 }, 263 }; 264 265 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu, 266 const struct nv_cspmu_match *match) 267 { 268 char *name; 269 struct device *dev = cspmu->dev; 270 271 static atomic_t pmu_generic_idx = {0}; 272 273 switch (match->name_fmt) { 274 case NAME_FMT_SOCKET: { 275 const int cpu = cpumask_first(&cspmu->associated_cpus); 276 const int socket = cpu_to_node(cpu); 277 278 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 279 socket); 280 break; 281 } 282 case NAME_FMT_GENERIC: 283 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern, 284 atomic_fetch_inc(&pmu_generic_idx)); 285 break; 286 default: 287 name = NULL; 288 break; 289 } 290 291 return name; 292 } 293 294 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) 295 { 296 u32 prodid; 297 struct nv_cspmu_ctx *ctx; 298 struct device *dev = cspmu->dev; 299 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; 300 const struct nv_cspmu_match *match = nv_cspmu_match; 301 302 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL); 303 if (!ctx) 304 return -ENOMEM; 305 306 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr); 307 308 /* Find matching PMU. */ 309 for (; match->prodid; match++) { 310 const u32 prodid_mask = match->prodid_mask; 311 312 if ((match->prodid & prodid_mask) == (prodid & prodid_mask)) 313 break; 314 } 315 316 ctx->name = nv_cspmu_format_name(cspmu, match); 317 ctx->filter_mask = match->filter_mask; 318 ctx->filter_default_val = match->filter_default_val; 319 ctx->event_attr = match->event_attr; 320 ctx->format_attr = match->format_attr; 321 322 cspmu->impl.ctx = ctx; 323 324 /* NVIDIA specific callbacks. */ 325 impl_ops->event_filter = nv_cspmu_event_filter; 326 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs; 327 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs; 328 impl_ops->get_name = nv_cspmu_get_name; 329 330 return 0; 331 } 332 333 /* Match all NVIDIA Coresight PMU devices */ 334 static const struct arm_cspmu_impl_match nv_cspmu_param = { 335 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA, 336 .module = THIS_MODULE, 337 .impl_init_ops = nv_cspmu_init_ops 338 }; 339 340 static int __init nvidia_cspmu_init(void) 341 { 342 int ret; 343 344 ret = arm_cspmu_impl_register(&nv_cspmu_param); 345 if (ret) 346 pr_err("nvidia_cspmu backend registration error: %d\n", ret); 347 348 return ret; 349 } 350 351 static void __exit nvidia_cspmu_exit(void) 352 { 353 arm_cspmu_impl_unregister(&nv_cspmu_param); 354 } 355 356 module_init(nvidia_cspmu_init); 357 module_exit(nvidia_cspmu_exit); 358 359 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver"); 360 MODULE_LICENSE("GPL v2"); 361