1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4 *
5 */
6
7 /* Support for NVIDIA specific attributes. */
8
9 #include <linux/module.h>
10 #include <linux/topology.h>
11
12 #include "arm_cspmu.h"
13
14 #define NV_PCIE_PORT_COUNT 10ULL
15 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
16
17 #define NV_NVL_C2C_PORT_COUNT 2ULL
18 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
19
20 #define NV_CNVL_PORT_COUNT 4ULL
21 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
22
23 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
24
25 #define NV_PRODID_MASK GENMASK(31, 0)
26
27 #define NV_FORMAT_NAME_GENERIC 0
28
29 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
30
31 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
32 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
33
34 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
35 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
39
40 struct nv_cspmu_ctx {
41 const char *name;
42 u32 filter_mask;
43 u32 filter_default_val;
44 struct attribute **event_attr;
45 struct attribute **format_attr;
46 };
47
48 static struct attribute *scf_pmu_event_attrs[] = {
49 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
50
51 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
52 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
53 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
54 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
55
56 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
57 NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp, 0x105),
58 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
59 NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp, 0x10d),
60 NV_CSPMU_EVENT_ATTR_4(socket, prb_data, 0x111),
61
62 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
63 NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding, 0x119),
64 NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding, 0x11d),
65 NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding, 0x121),
66 NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding, 0x125),
67 NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding, 0x129),
68
69 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
70 NV_CSPMU_EVENT_ATTR_4(socket, dl_access, 0x131),
71 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
72 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
73 NV_CSPMU_EVENT_ATTR_4(socket, ev_access, 0x13d),
74 NV_CSPMU_EVENT_ATTR_4(socket, prb_access, 0x141),
75
76 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data, 0x145),
77 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access, 0x149),
78 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access, 0x14d),
79 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding, 0x151),
80 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding, 0x155),
81
82 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data, 0x159),
83 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access, 0x15d),
84 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access, 0x161),
85 NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding, 0x165),
86 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding, 0x169),
87
88 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
89 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
90 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
91 ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp, 0x170),
92 ARM_CSPMU_EVENT_ATTR(gmem_dl_access, 0x171),
93 ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding, 0x172),
94 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
95 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
96 ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding, 0x175),
97 ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp, 0x176),
98 ARM_CSPMU_EVENT_ATTR(gmem_ev_access, 0x177),
99 ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding, 0x178),
100 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
101 ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding, 0x17a),
102 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
103
104 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
105
106 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data, 0x180),
107 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data, 0x184),
108 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access, 0x188),
109 NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding, 0x18c),
110
111 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data, 0x190),
112 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data, 0x194),
113 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access, 0x198),
114 NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding, 0x19c),
115
116 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
117 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
118 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
119 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
120 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
121
122 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
123 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
124 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
125 ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp, 0x1a8),
126 ARM_CSPMU_EVENT_ATTR(cmem_dl_access, 0x1a9),
127 ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding, 0x1aa),
128 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
129 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
130 ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding, 0x1ad),
131 ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp, 0x1ae),
132 ARM_CSPMU_EVENT_ATTR(cmem_ev_access, 0x1af),
133 ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding, 0x1b0),
134 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
135 ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding, 0x1b2),
136
137 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data, 0x1b3),
138 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access, 0x1b7),
139 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access, 0x1bb),
140 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding, 0x1bf),
141 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding, 0x1c3),
142
143 ARM_CSPMU_EVENT_ATTR(ocu_prb_access, 0x1c7),
144 ARM_CSPMU_EVENT_ATTR(ocu_prb_data, 0x1c8),
145 ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding, 0x1c9),
146
147 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
148
149 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access, 0x1cb),
150 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data, 0x1cf),
151 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data, 0x1d3),
152 NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding, 0x1d7),
153
154 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
155
156 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
157 NULL,
158 };
159
160 static struct attribute *mcf_pmu_event_attrs[] = {
161 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
162 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
163 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
164 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
165 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
166 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
167 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
168 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
169 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
170 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
171 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
172 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
173 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
174 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
175 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
176 NULL,
177 };
178
179 static struct attribute *generic_pmu_event_attrs[] = {
180 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
181 NULL,
182 };
183
184 static struct attribute *scf_pmu_format_attrs[] = {
185 ARM_CSPMU_FORMAT_EVENT_ATTR,
186 NULL,
187 };
188
189 static struct attribute *pcie_pmu_format_attrs[] = {
190 ARM_CSPMU_FORMAT_EVENT_ATTR,
191 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
192 NULL,
193 };
194
195 static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
196 ARM_CSPMU_FORMAT_EVENT_ATTR,
197 NULL,
198 };
199
200 static struct attribute *cnvlink_pmu_format_attrs[] = {
201 ARM_CSPMU_FORMAT_EVENT_ATTR,
202 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
203 NULL,
204 };
205
206 static struct attribute *generic_pmu_format_attrs[] = {
207 ARM_CSPMU_FORMAT_EVENT_ATTR,
208 ARM_CSPMU_FORMAT_FILTER_ATTR,
209 NULL,
210 };
211
212 static struct attribute **
nv_cspmu_get_event_attrs(const struct arm_cspmu * cspmu)213 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
214 {
215 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
216
217 return ctx->event_attr;
218 }
219
220 static struct attribute **
nv_cspmu_get_format_attrs(const struct arm_cspmu * cspmu)221 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
222 {
223 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
224
225 return ctx->format_attr;
226 }
227
228 static const char *
nv_cspmu_get_name(const struct arm_cspmu * cspmu)229 nv_cspmu_get_name(const struct arm_cspmu *cspmu)
230 {
231 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
232
233 return ctx->name;
234 }
235
nv_cspmu_event_filter(const struct perf_event * event)236 static u32 nv_cspmu_event_filter(const struct perf_event *event)
237 {
238 const struct nv_cspmu_ctx *ctx =
239 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
240
241 if (ctx->filter_mask == 0)
242 return ctx->filter_default_val;
243
244 return event->attr.config1 & ctx->filter_mask;
245 }
246
247 enum nv_cspmu_name_fmt {
248 NAME_FMT_GENERIC,
249 NAME_FMT_SOCKET
250 };
251
252 struct nv_cspmu_match {
253 u32 prodid;
254 u32 prodid_mask;
255 u64 filter_mask;
256 u32 filter_default_val;
257 const char *name_pattern;
258 enum nv_cspmu_name_fmt name_fmt;
259 struct attribute **event_attr;
260 struct attribute **format_attr;
261 };
262
263 static const struct nv_cspmu_match nv_cspmu_match[] = {
264 {
265 .prodid = 0x103,
266 .prodid_mask = NV_PRODID_MASK,
267 .filter_mask = NV_PCIE_FILTER_ID_MASK,
268 .filter_default_val = NV_PCIE_FILTER_ID_MASK,
269 .name_pattern = "nvidia_pcie_pmu_%u",
270 .name_fmt = NAME_FMT_SOCKET,
271 .event_attr = mcf_pmu_event_attrs,
272 .format_attr = pcie_pmu_format_attrs
273 },
274 {
275 .prodid = 0x104,
276 .prodid_mask = NV_PRODID_MASK,
277 .filter_mask = 0x0,
278 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
279 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
280 .name_fmt = NAME_FMT_SOCKET,
281 .event_attr = mcf_pmu_event_attrs,
282 .format_attr = nvlink_c2c_pmu_format_attrs
283 },
284 {
285 .prodid = 0x105,
286 .prodid_mask = NV_PRODID_MASK,
287 .filter_mask = 0x0,
288 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
289 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
290 .name_fmt = NAME_FMT_SOCKET,
291 .event_attr = mcf_pmu_event_attrs,
292 .format_attr = nvlink_c2c_pmu_format_attrs
293 },
294 {
295 .prodid = 0x106,
296 .prodid_mask = NV_PRODID_MASK,
297 .filter_mask = NV_CNVL_FILTER_ID_MASK,
298 .filter_default_val = NV_CNVL_FILTER_ID_MASK,
299 .name_pattern = "nvidia_cnvlink_pmu_%u",
300 .name_fmt = NAME_FMT_SOCKET,
301 .event_attr = mcf_pmu_event_attrs,
302 .format_attr = cnvlink_pmu_format_attrs
303 },
304 {
305 .prodid = 0x2CF,
306 .prodid_mask = NV_PRODID_MASK,
307 .filter_mask = 0x0,
308 .filter_default_val = 0x0,
309 .name_pattern = "nvidia_scf_pmu_%u",
310 .name_fmt = NAME_FMT_SOCKET,
311 .event_attr = scf_pmu_event_attrs,
312 .format_attr = scf_pmu_format_attrs
313 },
314 {
315 .prodid = 0,
316 .prodid_mask = 0,
317 .filter_mask = NV_GENERIC_FILTER_ID_MASK,
318 .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
319 .name_pattern = "nvidia_uncore_pmu_%u",
320 .name_fmt = NAME_FMT_GENERIC,
321 .event_attr = generic_pmu_event_attrs,
322 .format_attr = generic_pmu_format_attrs
323 },
324 };
325
nv_cspmu_format_name(const struct arm_cspmu * cspmu,const struct nv_cspmu_match * match)326 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
327 const struct nv_cspmu_match *match)
328 {
329 char *name;
330 struct device *dev = cspmu->dev;
331
332 static atomic_t pmu_generic_idx = {0};
333
334 switch (match->name_fmt) {
335 case NAME_FMT_SOCKET: {
336 const int cpu = cpumask_first(&cspmu->associated_cpus);
337 const int socket = cpu_to_node(cpu);
338
339 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
340 socket);
341 break;
342 }
343 case NAME_FMT_GENERIC:
344 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
345 atomic_fetch_inc(&pmu_generic_idx));
346 break;
347 default:
348 name = NULL;
349 break;
350 }
351
352 return name;
353 }
354
nv_cspmu_init_ops(struct arm_cspmu * cspmu)355 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
356 {
357 u32 prodid;
358 struct nv_cspmu_ctx *ctx;
359 struct device *dev = cspmu->dev;
360 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
361 const struct nv_cspmu_match *match = nv_cspmu_match;
362
363 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
364 if (!ctx)
365 return -ENOMEM;
366
367 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
368
369 /* Find matching PMU. */
370 for (; match->prodid; match++) {
371 const u32 prodid_mask = match->prodid_mask;
372
373 if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
374 break;
375 }
376
377 ctx->name = nv_cspmu_format_name(cspmu, match);
378 ctx->filter_mask = match->filter_mask;
379 ctx->filter_default_val = match->filter_default_val;
380 ctx->event_attr = match->event_attr;
381 ctx->format_attr = match->format_attr;
382
383 cspmu->impl.ctx = ctx;
384
385 /* NVIDIA specific callbacks. */
386 impl_ops->event_filter = nv_cspmu_event_filter;
387 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
388 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
389 impl_ops->get_name = nv_cspmu_get_name;
390
391 return 0;
392 }
393
394 /* Match all NVIDIA Coresight PMU devices */
395 static const struct arm_cspmu_impl_match nv_cspmu_param = {
396 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA,
397 .module = THIS_MODULE,
398 .impl_init_ops = nv_cspmu_init_ops
399 };
400
nvidia_cspmu_init(void)401 static int __init nvidia_cspmu_init(void)
402 {
403 int ret;
404
405 ret = arm_cspmu_impl_register(&nv_cspmu_param);
406 if (ret)
407 pr_err("nvidia_cspmu backend registration error: %d\n", ret);
408
409 return ret;
410 }
411
nvidia_cspmu_exit(void)412 static void __exit nvidia_cspmu_exit(void)
413 {
414 arm_cspmu_impl_unregister(&nv_cspmu_param);
415 }
416
417 module_init(nvidia_cspmu_init);
418 module_exit(nvidia_cspmu_exit);
419
420 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver");
421 MODULE_LICENSE("GPL v2");
422