1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVIDIA Tegra410 C2C PMU driver.
4 *
5 * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6 */
7
8 #include <linux/acpi.h>
9 #include <linux/bitops.h>
10 #include <linux/cpumask.h>
11 #include <linux/device.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/module.h>
15 #include <linux/perf_event.h>
16 #include <linux/platform_device.h>
17 #include <linux/property.h>
18
19 /* The C2C interface types in Tegra410. */
20 #define C2C_TYPE_NVLINK 0x0
21 #define C2C_TYPE_NVCLINK 0x1
22 #define C2C_TYPE_NVDLINK 0x2
23 #define C2C_TYPE_COUNT 0x3
24
25 /* The type of the peer device connected to the C2C interface. */
26 #define C2C_PEER_TYPE_CPU 0x0
27 #define C2C_PEER_TYPE_GPU 0x1
28 #define C2C_PEER_TYPE_CXLMEM 0x2
29 #define C2C_PEER_TYPE_COUNT 0x3
30
31 /* The number of peer devices can be connected to the C2C interface. */
32 #define C2C_NR_PEER_CPU 0x1
33 #define C2C_NR_PEER_GPU 0x2
34 #define C2C_NR_PEER_CXLMEM 0x1
35 #define C2C_NR_PEER_MAX 0x2
36
37 /* Number of instances on each interface. */
38 #define C2C_NR_INST_NVLINK 14
39 #define C2C_NR_INST_NVCLINK 12
40 #define C2C_NR_INST_NVDLINK 16
41 #define C2C_NR_INST_MAX 16
42
43 /* Register offsets. */
44 #define C2C_CTRL 0x864
45 #define C2C_IN_STATUS 0x868
46 #define C2C_CYCLE_CNTR 0x86c
47 #define C2C_IN_RD_CUM_OUTS_CNTR 0x874
48 #define C2C_IN_RD_REQ_CNTR 0x87c
49 #define C2C_IN_WR_CUM_OUTS_CNTR 0x884
50 #define C2C_IN_WR_REQ_CNTR 0x88c
51 #define C2C_OUT_STATUS 0x890
52 #define C2C_OUT_RD_CUM_OUTS_CNTR 0x898
53 #define C2C_OUT_RD_REQ_CNTR 0x8a0
54 #define C2C_OUT_WR_CUM_OUTS_CNTR 0x8a8
55 #define C2C_OUT_WR_REQ_CNTR 0x8b0
56
57 /* C2C_IN_STATUS register field. */
58 #define C2C_IN_STATUS_CYCLE_OVF BIT(0)
59 #define C2C_IN_STATUS_IN_RD_CUM_OUTS_OVF BIT(1)
60 #define C2C_IN_STATUS_IN_RD_REQ_OVF BIT(2)
61 #define C2C_IN_STATUS_IN_WR_CUM_OUTS_OVF BIT(3)
62 #define C2C_IN_STATUS_IN_WR_REQ_OVF BIT(4)
63
64 /* C2C_OUT_STATUS register field. */
65 #define C2C_OUT_STATUS_OUT_RD_CUM_OUTS_OVF BIT(0)
66 #define C2C_OUT_STATUS_OUT_RD_REQ_OVF BIT(1)
67 #define C2C_OUT_STATUS_OUT_WR_CUM_OUTS_OVF BIT(2)
68 #define C2C_OUT_STATUS_OUT_WR_REQ_OVF BIT(3)
69
70 /* Events. */
71 #define C2C_EVENT_CYCLES 0x0
72 #define C2C_EVENT_IN_RD_CUM_OUTS 0x1
73 #define C2C_EVENT_IN_RD_REQ 0x2
74 #define C2C_EVENT_IN_WR_CUM_OUTS 0x3
75 #define C2C_EVENT_IN_WR_REQ 0x4
76 #define C2C_EVENT_OUT_RD_CUM_OUTS 0x5
77 #define C2C_EVENT_OUT_RD_REQ 0x6
78 #define C2C_EVENT_OUT_WR_CUM_OUTS 0x7
79 #define C2C_EVENT_OUT_WR_REQ 0x8
80
81 #define C2C_NUM_EVENTS 0x9
82 #define C2C_MASK_EVENT 0xFF
83 #define C2C_MAX_ACTIVE_EVENTS 32
84
85 #define C2C_ACTIVE_CPU_MASK 0x0
86 #define C2C_ASSOCIATED_CPU_MASK 0x1
87
88 /*
89 * Maximum poll count for reading counter value using high-low-high sequence.
90 */
91 #define HILOHI_MAX_POLL 1000
92
93 static unsigned long nv_c2c_pmu_cpuhp_state;
94
95 /* PMU descriptor. */
96
97 /* C2C type information. */
98 struct nv_c2c_pmu_data {
99 unsigned int c2c_type;
100 unsigned int nr_inst;
101 const char *name_fmt;
102 };
103
104 static const struct nv_c2c_pmu_data nv_c2c_pmu_data[] = {
105 [C2C_TYPE_NVLINK] = {
106 .c2c_type = C2C_TYPE_NVLINK,
107 .nr_inst = C2C_NR_INST_NVLINK,
108 .name_fmt = "nvidia_nvlink_c2c_pmu_%u",
109 },
110 [C2C_TYPE_NVCLINK] = {
111 .c2c_type = C2C_TYPE_NVCLINK,
112 .nr_inst = C2C_NR_INST_NVCLINK,
113 .name_fmt = "nvidia_nvclink_pmu_%u",
114 },
115 [C2C_TYPE_NVDLINK] = {
116 .c2c_type = C2C_TYPE_NVDLINK,
117 .nr_inst = C2C_NR_INST_NVDLINK,
118 .name_fmt = "nvidia_nvdlink_pmu_%u",
119 },
120 };
121
122 /* Tracks the events assigned to the PMU for a given logical index. */
123 struct nv_c2c_pmu_hw_events {
124 /* The events that are active. */
125 struct perf_event *events[C2C_MAX_ACTIVE_EVENTS];
126
127 /*
128 * Each bit indicates a logical counter is being used (or not) for an
129 * event.
130 */
131 DECLARE_BITMAP(used_ctrs, C2C_MAX_ACTIVE_EVENTS);
132 };
133
134 struct nv_c2c_pmu {
135 struct pmu pmu;
136 struct device *dev;
137 struct acpi_device *acpi_dev;
138
139 const char *name;
140 const char *identifier;
141
142 const struct nv_c2c_pmu_data *data;
143 unsigned int peer_type;
144 unsigned int socket;
145 unsigned int nr_peer;
146 unsigned long peer_insts[C2C_NR_PEER_MAX][BITS_TO_LONGS(C2C_NR_INST_MAX)];
147 u32 filter_default;
148
149 struct nv_c2c_pmu_hw_events hw_events;
150
151 cpumask_t associated_cpus;
152 cpumask_t active_cpu;
153
154 struct hlist_node cpuhp_node;
155
156 const struct attribute_group **attr_groups;
157
158 void __iomem *base_broadcast;
159 void __iomem *base[C2C_NR_INST_MAX];
160 };
161
162 #define to_c2c_pmu(p) (container_of(p, struct nv_c2c_pmu, pmu))
163
164 /* Get event type from perf_event. */
get_event_type(struct perf_event * event)165 static inline u32 get_event_type(struct perf_event *event)
166 {
167 return (event->attr.config) & C2C_MASK_EVENT;
168 }
169
get_filter_mask(struct perf_event * event)170 static inline u32 get_filter_mask(struct perf_event *event)
171 {
172 u32 filter;
173 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu);
174
175 filter = ((u32)event->attr.config1) & c2c_pmu->filter_default;
176 if (filter == 0)
177 filter = c2c_pmu->filter_default;
178
179 return filter;
180 }
181
182 /* PMU operations. */
183
nv_c2c_pmu_get_event_idx(struct nv_c2c_pmu_hw_events * hw_events,struct perf_event * event)184 static int nv_c2c_pmu_get_event_idx(struct nv_c2c_pmu_hw_events *hw_events,
185 struct perf_event *event)
186 {
187 u32 idx;
188
189 idx = find_first_zero_bit(hw_events->used_ctrs, C2C_MAX_ACTIVE_EVENTS);
190 if (idx >= C2C_MAX_ACTIVE_EVENTS)
191 return -EAGAIN;
192
193 set_bit(idx, hw_events->used_ctrs);
194
195 return idx;
196 }
197
198 static bool
nv_c2c_pmu_validate_event(struct pmu * pmu,struct nv_c2c_pmu_hw_events * hw_events,struct perf_event * event)199 nv_c2c_pmu_validate_event(struct pmu *pmu,
200 struct nv_c2c_pmu_hw_events *hw_events,
201 struct perf_event *event)
202 {
203 if (is_software_event(event))
204 return true;
205
206 /* Reject groups spanning multiple HW PMUs. */
207 if (event->pmu != pmu)
208 return false;
209
210 return nv_c2c_pmu_get_event_idx(hw_events, event) >= 0;
211 }
212
213 /*
214 * Make sure the group of events can be scheduled at once
215 * on the PMU.
216 */
nv_c2c_pmu_validate_group(struct perf_event * event)217 static bool nv_c2c_pmu_validate_group(struct perf_event *event)
218 {
219 struct perf_event *sibling, *leader = event->group_leader;
220 struct nv_c2c_pmu_hw_events fake_hw_events;
221
222 if (event->group_leader == event)
223 return true;
224
225 memset(&fake_hw_events, 0, sizeof(fake_hw_events));
226
227 if (!nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events, leader))
228 return false;
229
230 for_each_sibling_event(sibling, leader) {
231 if (!nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events,
232 sibling))
233 return false;
234 }
235
236 return nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events, event);
237 }
238
nv_c2c_pmu_event_init(struct perf_event * event)239 static int nv_c2c_pmu_event_init(struct perf_event *event)
240 {
241 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu);
242 struct hw_perf_event *hwc = &event->hw;
243 u32 event_type = get_event_type(event);
244
245 if (event->attr.type != event->pmu->type ||
246 event_type >= C2C_NUM_EVENTS)
247 return -ENOENT;
248
249 /*
250 * Following other "uncore" PMUs, we do not support sampling mode or
251 * attach to a task (per-process mode).
252 */
253 if (is_sampling_event(event)) {
254 dev_dbg(c2c_pmu->pmu.dev, "Can't support sampling events\n");
255 return -EOPNOTSUPP;
256 }
257
258 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
259 dev_dbg(c2c_pmu->pmu.dev, "Can't support per-task counters\n");
260 return -EINVAL;
261 }
262
263 /*
264 * Make sure the CPU assignment is on one of the CPUs associated with
265 * this PMU.
266 */
267 if (!cpumask_test_cpu(event->cpu, &c2c_pmu->associated_cpus)) {
268 dev_dbg(c2c_pmu->pmu.dev,
269 "Requested cpu is not associated with the PMU\n");
270 return -EINVAL;
271 }
272
273 /* Enforce the current active CPU to handle the events in this PMU. */
274 event->cpu = cpumask_first(&c2c_pmu->active_cpu);
275 if (event->cpu >= nr_cpu_ids)
276 return -EINVAL;
277
278 if (!nv_c2c_pmu_validate_group(event))
279 return -EINVAL;
280
281 hwc->idx = -1;
282 hwc->config = event_type;
283
284 return 0;
285 }
286
287 /*
288 * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
289 */
read_reg64_hilohi(const void __iomem * addr,u32 max_poll_count)290 static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
291 {
292 u32 val_lo, val_hi;
293 u64 val;
294
295 /* Use high-low-high sequence to avoid tearing */
296 do {
297 if (max_poll_count-- == 0) {
298 pr_err("NV C2C PMU: timeout hi-low-high sequence\n");
299 return 0;
300 }
301
302 val_hi = readl(addr + 4);
303 val_lo = readl(addr);
304 } while (val_hi != readl(addr + 4));
305
306 val = (((u64)val_hi << 32) | val_lo);
307
308 return val;
309 }
310
nv_c2c_pmu_check_status(struct nv_c2c_pmu * c2c_pmu,u32 instance)311 static void nv_c2c_pmu_check_status(struct nv_c2c_pmu *c2c_pmu, u32 instance)
312 {
313 u32 in_status, out_status;
314
315 in_status = readl(c2c_pmu->base[instance] + C2C_IN_STATUS);
316 out_status = readl(c2c_pmu->base[instance] + C2C_OUT_STATUS);
317
318 if (in_status || out_status)
319 dev_warn(c2c_pmu->dev,
320 "C2C PMU overflow in: 0x%x, out: 0x%x\n",
321 in_status, out_status);
322 }
323
324 static u32 nv_c2c_ctr_offset[C2C_NUM_EVENTS] = {
325 [C2C_EVENT_CYCLES] = C2C_CYCLE_CNTR,
326 [C2C_EVENT_IN_RD_CUM_OUTS] = C2C_IN_RD_CUM_OUTS_CNTR,
327 [C2C_EVENT_IN_RD_REQ] = C2C_IN_RD_REQ_CNTR,
328 [C2C_EVENT_IN_WR_CUM_OUTS] = C2C_IN_WR_CUM_OUTS_CNTR,
329 [C2C_EVENT_IN_WR_REQ] = C2C_IN_WR_REQ_CNTR,
330 [C2C_EVENT_OUT_RD_CUM_OUTS] = C2C_OUT_RD_CUM_OUTS_CNTR,
331 [C2C_EVENT_OUT_RD_REQ] = C2C_OUT_RD_REQ_CNTR,
332 [C2C_EVENT_OUT_WR_CUM_OUTS] = C2C_OUT_WR_CUM_OUTS_CNTR,
333 [C2C_EVENT_OUT_WR_REQ] = C2C_OUT_WR_REQ_CNTR,
334 };
335
nv_c2c_pmu_read_counter(struct perf_event * event)336 static u64 nv_c2c_pmu_read_counter(struct perf_event *event)
337 {
338 u32 ctr_id, ctr_offset, filter_mask, filter_idx, inst_idx;
339 unsigned long *inst_mask;
340 DECLARE_BITMAP(filter_bitmap, C2C_NR_PEER_MAX);
341 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu);
342 u64 val = 0;
343
344 filter_mask = get_filter_mask(event);
345 bitmap_from_arr32(filter_bitmap, &filter_mask, c2c_pmu->nr_peer);
346
347 ctr_id = event->hw.config;
348 ctr_offset = nv_c2c_ctr_offset[ctr_id];
349
350 for_each_set_bit(filter_idx, filter_bitmap, c2c_pmu->nr_peer) {
351 inst_mask = c2c_pmu->peer_insts[filter_idx];
352 for_each_set_bit(inst_idx, inst_mask, c2c_pmu->data->nr_inst) {
353 nv_c2c_pmu_check_status(c2c_pmu, inst_idx);
354
355 /*
356 * Each instance share same clock and the driver always
357 * enables all instances. So we can use the counts from
358 * one instance for cycle counter.
359 */
360 if (ctr_id == C2C_EVENT_CYCLES)
361 return read_reg64_hilohi(
362 c2c_pmu->base[inst_idx] + ctr_offset,
363 HILOHI_MAX_POLL);
364
365 /*
366 * For other events, sum up the counts from all instances.
367 */
368 val += read_reg64_hilohi(
369 c2c_pmu->base[inst_idx] + ctr_offset,
370 HILOHI_MAX_POLL);
371 }
372 }
373
374 return val;
375 }
376
nv_c2c_pmu_event_update(struct perf_event * event)377 static void nv_c2c_pmu_event_update(struct perf_event *event)
378 {
379 struct hw_perf_event *hwc = &event->hw;
380 u64 prev, now;
381
382 do {
383 prev = local64_read(&hwc->prev_count);
384 now = nv_c2c_pmu_read_counter(event);
385 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
386
387 local64_add(now - prev, &event->count);
388 }
389
nv_c2c_pmu_start(struct perf_event * event,int pmu_flags)390 static void nv_c2c_pmu_start(struct perf_event *event, int pmu_flags)
391 {
392 event->hw.state = 0;
393 }
394
nv_c2c_pmu_stop(struct perf_event * event,int pmu_flags)395 static void nv_c2c_pmu_stop(struct perf_event *event, int pmu_flags)
396 {
397 event->hw.state |= PERF_HES_STOPPED;
398 }
399
nv_c2c_pmu_add(struct perf_event * event,int flags)400 static int nv_c2c_pmu_add(struct perf_event *event, int flags)
401 {
402 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu);
403 struct nv_c2c_pmu_hw_events *hw_events = &c2c_pmu->hw_events;
404 struct hw_perf_event *hwc = &event->hw;
405 int idx;
406
407 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
408 &c2c_pmu->associated_cpus)))
409 return -ENOENT;
410
411 idx = nv_c2c_pmu_get_event_idx(hw_events, event);
412 if (idx < 0)
413 return idx;
414
415 hw_events->events[idx] = event;
416 hwc->idx = idx;
417 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
418
419 if (flags & PERF_EF_START)
420 nv_c2c_pmu_start(event, PERF_EF_RELOAD);
421
422 /* Propagate changes to the userspace mapping. */
423 perf_event_update_userpage(event);
424
425 return 0;
426 }
427
nv_c2c_pmu_del(struct perf_event * event,int flags)428 static void nv_c2c_pmu_del(struct perf_event *event, int flags)
429 {
430 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu);
431 struct nv_c2c_pmu_hw_events *hw_events = &c2c_pmu->hw_events;
432 struct hw_perf_event *hwc = &event->hw;
433 int idx = hwc->idx;
434
435 nv_c2c_pmu_stop(event, PERF_EF_UPDATE);
436
437 hw_events->events[idx] = NULL;
438
439 clear_bit(idx, hw_events->used_ctrs);
440
441 perf_event_update_userpage(event);
442 }
443
nv_c2c_pmu_read(struct perf_event * event)444 static void nv_c2c_pmu_read(struct perf_event *event)
445 {
446 nv_c2c_pmu_event_update(event);
447 }
448
nv_c2c_pmu_enable(struct pmu * pmu)449 static void nv_c2c_pmu_enable(struct pmu *pmu)
450 {
451 void __iomem *bcast;
452 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu);
453
454 /* Check if any filter is enabled. */
455 if (bitmap_empty(c2c_pmu->hw_events.used_ctrs, C2C_MAX_ACTIVE_EVENTS))
456 return;
457
458 /* Enable all the counters. */
459 bcast = c2c_pmu->base_broadcast;
460 writel(0x1UL, bcast + C2C_CTRL);
461 }
462
nv_c2c_pmu_disable(struct pmu * pmu)463 static void nv_c2c_pmu_disable(struct pmu *pmu)
464 {
465 unsigned int idx;
466 void __iomem *bcast;
467 struct perf_event *event;
468 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu);
469
470 /* Disable all the counters. */
471 bcast = c2c_pmu->base_broadcast;
472 writel(0x0UL, bcast + C2C_CTRL);
473
474 /*
475 * The counters will start from 0 again on restart.
476 * Update the events immediately to avoid losing the counts.
477 */
478 for_each_set_bit(idx, c2c_pmu->hw_events.used_ctrs,
479 C2C_MAX_ACTIVE_EVENTS) {
480 event = c2c_pmu->hw_events.events[idx];
481
482 if (!event)
483 continue;
484
485 nv_c2c_pmu_event_update(event);
486
487 local64_set(&event->hw.prev_count, 0ULL);
488 }
489 }
490
491 /* PMU identifier attribute. */
492
nv_c2c_pmu_identifier_show(struct device * dev,struct device_attribute * attr,char * page)493 static ssize_t nv_c2c_pmu_identifier_show(struct device *dev,
494 struct device_attribute *attr,
495 char *page)
496 {
497 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(dev_get_drvdata(dev));
498
499 return sysfs_emit(page, "%s\n", c2c_pmu->identifier);
500 }
501
502 static struct device_attribute nv_c2c_pmu_identifier_attr =
503 __ATTR(identifier, 0444, nv_c2c_pmu_identifier_show, NULL);
504
505 static struct attribute *nv_c2c_pmu_identifier_attrs[] = {
506 &nv_c2c_pmu_identifier_attr.attr,
507 NULL,
508 };
509
510 static struct attribute_group nv_c2c_pmu_identifier_attr_group = {
511 .attrs = nv_c2c_pmu_identifier_attrs,
512 };
513
514 /* Peer attribute. */
515
nv_c2c_pmu_peer_show(struct device * dev,struct device_attribute * attr,char * page)516 static ssize_t nv_c2c_pmu_peer_show(struct device *dev,
517 struct device_attribute *attr,
518 char *page)
519 {
520 const char *peer_type[C2C_PEER_TYPE_COUNT] = {
521 [C2C_PEER_TYPE_CPU] = "cpu",
522 [C2C_PEER_TYPE_GPU] = "gpu",
523 [C2C_PEER_TYPE_CXLMEM] = "cxlmem",
524 };
525
526 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(dev_get_drvdata(dev));
527 return sysfs_emit(page, "nr_%s=%u\n", peer_type[c2c_pmu->peer_type],
528 c2c_pmu->nr_peer);
529 }
530
531 static struct device_attribute nv_c2c_pmu_peer_attr =
532 __ATTR(peer, 0444, nv_c2c_pmu_peer_show, NULL);
533
534 static struct attribute *nv_c2c_pmu_peer_attrs[] = {
535 &nv_c2c_pmu_peer_attr.attr,
536 NULL,
537 };
538
539 static struct attribute_group nv_c2c_pmu_peer_attr_group = {
540 .attrs = nv_c2c_pmu_peer_attrs,
541 };
542
543 /* Format attributes. */
544
545 #define NV_C2C_PMU_EXT_ATTR(_name, _func, _config) \
546 (&((struct dev_ext_attribute[]){ \
547 { \
548 .attr = __ATTR(_name, 0444, _func, NULL), \
549 .var = (void *)_config \
550 } \
551 })[0].attr.attr)
552
553 #define NV_C2C_PMU_FORMAT_ATTR(_name, _config) \
554 NV_C2C_PMU_EXT_ATTR(_name, device_show_string, _config)
555
556 #define NV_C2C_PMU_FORMAT_EVENT_ATTR \
557 NV_C2C_PMU_FORMAT_ATTR(event, "config:0-3")
558
559 static struct attribute *nv_c2c_pmu_gpu_formats[] = {
560 NV_C2C_PMU_FORMAT_EVENT_ATTR,
561 NV_C2C_PMU_FORMAT_ATTR(gpu_mask, "config1:0-1"),
562 NULL,
563 };
564
565 static const struct attribute_group nv_c2c_pmu_gpu_format_group = {
566 .name = "format",
567 .attrs = nv_c2c_pmu_gpu_formats,
568 };
569
570 static struct attribute *nv_c2c_pmu_formats[] = {
571 NV_C2C_PMU_FORMAT_EVENT_ATTR,
572 NULL,
573 };
574
575 static const struct attribute_group nv_c2c_pmu_format_group = {
576 .name = "format",
577 .attrs = nv_c2c_pmu_formats,
578 };
579
580 /* Event attributes. */
581
nv_c2c_pmu_sysfs_event_show(struct device * dev,struct device_attribute * attr,char * buf)582 static ssize_t nv_c2c_pmu_sysfs_event_show(struct device *dev,
583 struct device_attribute *attr,
584 char *buf)
585 {
586 struct perf_pmu_events_attr *pmu_attr;
587
588 pmu_attr = container_of(attr, typeof(*pmu_attr), attr);
589 return sysfs_emit(buf, "event=0x%llx\n", pmu_attr->id);
590 }
591
592 #define NV_C2C_PMU_EVENT_ATTR(_name, _config) \
593 PMU_EVENT_ATTR_ID(_name, nv_c2c_pmu_sysfs_event_show, _config)
594
595 static struct attribute *nv_c2c_pmu_gpu_events[] = {
596 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES),
597 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS),
598 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ),
599 NV_C2C_PMU_EVENT_ATTR(in_wr_cum_outs, C2C_EVENT_IN_WR_CUM_OUTS),
600 NV_C2C_PMU_EVENT_ATTR(in_wr_req, C2C_EVENT_IN_WR_REQ),
601 NV_C2C_PMU_EVENT_ATTR(out_rd_cum_outs, C2C_EVENT_OUT_RD_CUM_OUTS),
602 NV_C2C_PMU_EVENT_ATTR(out_rd_req, C2C_EVENT_OUT_RD_REQ),
603 NV_C2C_PMU_EVENT_ATTR(out_wr_cum_outs, C2C_EVENT_OUT_WR_CUM_OUTS),
604 NV_C2C_PMU_EVENT_ATTR(out_wr_req, C2C_EVENT_OUT_WR_REQ),
605 NULL
606 };
607
608 static const struct attribute_group nv_c2c_pmu_gpu_events_group = {
609 .name = "events",
610 .attrs = nv_c2c_pmu_gpu_events,
611 };
612
613 static struct attribute *nv_c2c_pmu_cpu_events[] = {
614 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES),
615 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS),
616 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ),
617 NV_C2C_PMU_EVENT_ATTR(out_rd_cum_outs, C2C_EVENT_OUT_RD_CUM_OUTS),
618 NV_C2C_PMU_EVENT_ATTR(out_rd_req, C2C_EVENT_OUT_RD_REQ),
619 NULL
620 };
621
622 static const struct attribute_group nv_c2c_pmu_cpu_events_group = {
623 .name = "events",
624 .attrs = nv_c2c_pmu_cpu_events,
625 };
626
627 static struct attribute *nv_c2c_pmu_cxlmem_events[] = {
628 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES),
629 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS),
630 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ),
631 NULL
632 };
633
634 static const struct attribute_group nv_c2c_pmu_cxlmem_events_group = {
635 .name = "events",
636 .attrs = nv_c2c_pmu_cxlmem_events,
637 };
638
639 /* Cpumask attributes. */
640
nv_c2c_pmu_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)641 static ssize_t nv_c2c_pmu_cpumask_show(struct device *dev,
642 struct device_attribute *attr, char *buf)
643 {
644 struct pmu *pmu = dev_get_drvdata(dev);
645 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu);
646 struct dev_ext_attribute *eattr =
647 container_of(attr, struct dev_ext_attribute, attr);
648 unsigned long mask_id = (unsigned long)eattr->var;
649 const cpumask_t *cpumask;
650
651 switch (mask_id) {
652 case C2C_ACTIVE_CPU_MASK:
653 cpumask = &c2c_pmu->active_cpu;
654 break;
655 case C2C_ASSOCIATED_CPU_MASK:
656 cpumask = &c2c_pmu->associated_cpus;
657 break;
658 default:
659 return 0;
660 }
661 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
662 }
663
664 #define NV_C2C_PMU_CPUMASK_ATTR(_name, _config) \
665 NV_C2C_PMU_EXT_ATTR(_name, nv_c2c_pmu_cpumask_show, \
666 (unsigned long)_config)
667
668 static struct attribute *nv_c2c_pmu_cpumask_attrs[] = {
669 NV_C2C_PMU_CPUMASK_ATTR(cpumask, C2C_ACTIVE_CPU_MASK),
670 NV_C2C_PMU_CPUMASK_ATTR(associated_cpus, C2C_ASSOCIATED_CPU_MASK),
671 NULL,
672 };
673
674 static const struct attribute_group nv_c2c_pmu_cpumask_attr_group = {
675 .attrs = nv_c2c_pmu_cpumask_attrs,
676 };
677
678 /* Attribute groups for C2C PMU connecting SoC and GPU */
679 static const struct attribute_group *nv_c2c_pmu_gpu_attr_groups[] = {
680 &nv_c2c_pmu_gpu_format_group,
681 &nv_c2c_pmu_gpu_events_group,
682 &nv_c2c_pmu_cpumask_attr_group,
683 &nv_c2c_pmu_identifier_attr_group,
684 &nv_c2c_pmu_peer_attr_group,
685 NULL
686 };
687
688 /* Attribute groups for C2C PMU connecting multiple SoCs */
689 static const struct attribute_group *nv_c2c_pmu_cpu_attr_groups[] = {
690 &nv_c2c_pmu_format_group,
691 &nv_c2c_pmu_cpu_events_group,
692 &nv_c2c_pmu_cpumask_attr_group,
693 &nv_c2c_pmu_identifier_attr_group,
694 &nv_c2c_pmu_peer_attr_group,
695 NULL
696 };
697
698 /* Attribute groups for C2C PMU connecting SoC and CXLMEM */
699 static const struct attribute_group *nv_c2c_pmu_cxlmem_attr_groups[] = {
700 &nv_c2c_pmu_format_group,
701 &nv_c2c_pmu_cxlmem_events_group,
702 &nv_c2c_pmu_cpumask_attr_group,
703 &nv_c2c_pmu_identifier_attr_group,
704 &nv_c2c_pmu_peer_attr_group,
705 NULL
706 };
707
nv_c2c_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)708 static int nv_c2c_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
709 {
710 struct nv_c2c_pmu *c2c_pmu =
711 hlist_entry_safe(node, struct nv_c2c_pmu, cpuhp_node);
712
713 if (!cpumask_test_cpu(cpu, &c2c_pmu->associated_cpus))
714 return 0;
715
716 /* If the PMU is already managed, there is nothing to do */
717 if (!cpumask_empty(&c2c_pmu->active_cpu))
718 return 0;
719
720 /* Use this CPU for event counting */
721 cpumask_set_cpu(cpu, &c2c_pmu->active_cpu);
722
723 return 0;
724 }
725
nv_c2c_pmu_cpu_teardown(unsigned int cpu,struct hlist_node * node)726 static int nv_c2c_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
727 {
728 unsigned int dst;
729
730 struct nv_c2c_pmu *c2c_pmu =
731 hlist_entry_safe(node, struct nv_c2c_pmu, cpuhp_node);
732
733 /* Nothing to do if this CPU doesn't own the PMU */
734 if (!cpumask_test_and_clear_cpu(cpu, &c2c_pmu->active_cpu))
735 return 0;
736
737 /* Choose a new CPU to migrate ownership of the PMU to */
738 dst = cpumask_any_and_but(&c2c_pmu->associated_cpus,
739 cpu_online_mask, cpu);
740 if (dst >= nr_cpu_ids)
741 return 0;
742
743 /* Use this CPU for event counting */
744 perf_pmu_migrate_context(&c2c_pmu->pmu, cpu, dst);
745 cpumask_set_cpu(dst, &c2c_pmu->active_cpu);
746
747 return 0;
748 }
749
nv_c2c_pmu_get_cpus(struct nv_c2c_pmu * c2c_pmu)750 static int nv_c2c_pmu_get_cpus(struct nv_c2c_pmu *c2c_pmu)
751 {
752 int socket = c2c_pmu->socket, cpu;
753
754 for_each_possible_cpu(cpu) {
755 if (cpu_to_node(cpu) == socket)
756 cpumask_set_cpu(cpu, &c2c_pmu->associated_cpus);
757 }
758
759 if (cpumask_empty(&c2c_pmu->associated_cpus)) {
760 dev_dbg(c2c_pmu->dev,
761 "No cpu associated with C2C PMU socket-%u\n", socket);
762 return -ENODEV;
763 }
764
765 return 0;
766 }
767
nv_c2c_pmu_init_socket(struct nv_c2c_pmu * c2c_pmu)768 static int nv_c2c_pmu_init_socket(struct nv_c2c_pmu *c2c_pmu)
769 {
770 const char *uid_str;
771 int ret, socket;
772
773 uid_str = acpi_device_uid(c2c_pmu->acpi_dev);
774 if (!uid_str) {
775 dev_err(c2c_pmu->dev, "No ACPI device UID\n");
776 return -ENODEV;
777 }
778
779 ret = kstrtou32(uid_str, 0, &socket);
780 if (ret) {
781 dev_err(c2c_pmu->dev, "Failed to parse ACPI device UID\n");
782 return ret;
783 }
784
785 c2c_pmu->socket = socket;
786 return 0;
787 }
788
nv_c2c_pmu_init_id(struct nv_c2c_pmu * c2c_pmu)789 static int nv_c2c_pmu_init_id(struct nv_c2c_pmu *c2c_pmu)
790 {
791 char *name;
792
793 name = devm_kasprintf(c2c_pmu->dev, GFP_KERNEL, c2c_pmu->data->name_fmt,
794 c2c_pmu->socket);
795 if (!name)
796 return -ENOMEM;
797
798 c2c_pmu->name = name;
799
800 c2c_pmu->identifier = acpi_device_hid(c2c_pmu->acpi_dev);
801
802 return 0;
803 }
804
nv_c2c_pmu_init_filter(struct nv_c2c_pmu * c2c_pmu)805 static int nv_c2c_pmu_init_filter(struct nv_c2c_pmu *c2c_pmu)
806 {
807 u32 cpu_en = 0;
808 struct device *dev = c2c_pmu->dev;
809 const struct nv_c2c_pmu_data *data = c2c_pmu->data;
810
811 if (data->c2c_type == C2C_TYPE_NVDLINK) {
812 c2c_pmu->peer_type = C2C_PEER_TYPE_CXLMEM;
813
814 c2c_pmu->peer_insts[0][0] = (1UL << data->nr_inst) - 1;
815
816 c2c_pmu->nr_peer = C2C_NR_PEER_CXLMEM;
817 c2c_pmu->filter_default = (1 << c2c_pmu->nr_peer) - 1;
818
819 c2c_pmu->attr_groups = nv_c2c_pmu_cxlmem_attr_groups;
820
821 return 0;
822 }
823
824 if (device_property_read_u32(dev, "cpu_en_mask", &cpu_en))
825 dev_dbg(dev, "no cpu_en_mask property\n");
826
827 if (cpu_en) {
828 c2c_pmu->peer_type = C2C_PEER_TYPE_CPU;
829
830 /* Fill peer_insts bitmap with instances connected to peer CPU. */
831 bitmap_from_arr32(c2c_pmu->peer_insts[0], &cpu_en, data->nr_inst);
832
833 c2c_pmu->nr_peer = 1;
834 c2c_pmu->attr_groups = nv_c2c_pmu_cpu_attr_groups;
835 } else {
836 u32 i;
837 const char *props[C2C_NR_PEER_MAX] = {
838 "gpu0_en_mask", "gpu1_en_mask"
839 };
840
841 for (i = 0; i < C2C_NR_PEER_MAX; i++) {
842 u32 gpu_en = 0;
843
844 if (device_property_read_u32(dev, props[i], &gpu_en))
845 dev_dbg(dev, "no %s property\n", props[i]);
846
847 if (gpu_en) {
848 /* Fill peer_insts bitmap with instances connected to peer GPU. */
849 bitmap_from_arr32(c2c_pmu->peer_insts[i], &gpu_en,
850 data->nr_inst);
851
852 c2c_pmu->nr_peer++;
853 }
854 }
855
856 if (c2c_pmu->nr_peer == 0) {
857 dev_err(dev, "No GPU is enabled\n");
858 return -EINVAL;
859 }
860
861 c2c_pmu->peer_type = C2C_PEER_TYPE_GPU;
862 c2c_pmu->attr_groups = nv_c2c_pmu_gpu_attr_groups;
863 }
864
865 c2c_pmu->filter_default = (1 << c2c_pmu->nr_peer) - 1;
866
867 return 0;
868 }
869
nv_c2c_pmu_init_pmu(struct platform_device * pdev)870 static void *nv_c2c_pmu_init_pmu(struct platform_device *pdev)
871 {
872 int ret;
873 struct nv_c2c_pmu *c2c_pmu;
874 struct acpi_device *acpi_dev;
875 struct device *dev = &pdev->dev;
876
877 acpi_dev = ACPI_COMPANION(dev);
878 if (!acpi_dev)
879 return ERR_PTR(-ENODEV);
880
881 c2c_pmu = devm_kzalloc(dev, sizeof(*c2c_pmu), GFP_KERNEL);
882 if (!c2c_pmu)
883 return ERR_PTR(-ENOMEM);
884
885 c2c_pmu->dev = dev;
886 c2c_pmu->acpi_dev = acpi_dev;
887 c2c_pmu->data = (const struct nv_c2c_pmu_data *)device_get_match_data(dev);
888 if (!c2c_pmu->data)
889 return ERR_PTR(-EINVAL);
890
891 platform_set_drvdata(pdev, c2c_pmu);
892
893 ret = nv_c2c_pmu_init_socket(c2c_pmu);
894 if (ret)
895 return ERR_PTR(ret);
896
897 ret = nv_c2c_pmu_init_id(c2c_pmu);
898 if (ret)
899 return ERR_PTR(ret);
900
901 ret = nv_c2c_pmu_init_filter(c2c_pmu);
902 if (ret)
903 return ERR_PTR(ret);
904
905 return c2c_pmu;
906 }
907
nv_c2c_pmu_init_mmio(struct nv_c2c_pmu * c2c_pmu)908 static int nv_c2c_pmu_init_mmio(struct nv_c2c_pmu *c2c_pmu)
909 {
910 int i;
911 struct device *dev = c2c_pmu->dev;
912 struct platform_device *pdev = to_platform_device(dev);
913 const struct nv_c2c_pmu_data *data = c2c_pmu->data;
914
915 /* Map the address of all the instances. */
916 for (i = 0; i < data->nr_inst; i++) {
917 c2c_pmu->base[i] = devm_platform_ioremap_resource(pdev, i);
918 if (IS_ERR(c2c_pmu->base[i])) {
919 dev_err(dev, "Failed map address for instance %d\n", i);
920 return PTR_ERR(c2c_pmu->base[i]);
921 }
922 }
923
924 /* Map broadcast address. */
925 c2c_pmu->base_broadcast = devm_platform_ioremap_resource(pdev,
926 data->nr_inst);
927 if (IS_ERR(c2c_pmu->base_broadcast)) {
928 dev_err(dev, "Failed map broadcast address\n");
929 return PTR_ERR(c2c_pmu->base_broadcast);
930 }
931
932 return 0;
933 }
934
nv_c2c_pmu_register_pmu(struct nv_c2c_pmu * c2c_pmu)935 static int nv_c2c_pmu_register_pmu(struct nv_c2c_pmu *c2c_pmu)
936 {
937 int ret;
938
939 ret = cpuhp_state_add_instance(nv_c2c_pmu_cpuhp_state,
940 &c2c_pmu->cpuhp_node);
941 if (ret) {
942 dev_err(c2c_pmu->dev, "Error %d registering hotplug\n", ret);
943 return ret;
944 }
945
946 c2c_pmu->pmu = (struct pmu) {
947 .parent = c2c_pmu->dev,
948 .task_ctx_nr = perf_invalid_context,
949 .pmu_enable = nv_c2c_pmu_enable,
950 .pmu_disable = nv_c2c_pmu_disable,
951 .event_init = nv_c2c_pmu_event_init,
952 .add = nv_c2c_pmu_add,
953 .del = nv_c2c_pmu_del,
954 .start = nv_c2c_pmu_start,
955 .stop = nv_c2c_pmu_stop,
956 .read = nv_c2c_pmu_read,
957 .attr_groups = c2c_pmu->attr_groups,
958 .capabilities = PERF_PMU_CAP_NO_EXCLUDE |
959 PERF_PMU_CAP_NO_INTERRUPT,
960 };
961
962 ret = perf_pmu_register(&c2c_pmu->pmu, c2c_pmu->name, -1);
963 if (ret) {
964 dev_err(c2c_pmu->dev, "Failed to register C2C PMU: %d\n", ret);
965 cpuhp_state_remove_instance(nv_c2c_pmu_cpuhp_state,
966 &c2c_pmu->cpuhp_node);
967 return ret;
968 }
969
970 return 0;
971 }
972
nv_c2c_pmu_probe(struct platform_device * pdev)973 static int nv_c2c_pmu_probe(struct platform_device *pdev)
974 {
975 int ret;
976 struct nv_c2c_pmu *c2c_pmu;
977
978 c2c_pmu = nv_c2c_pmu_init_pmu(pdev);
979 if (IS_ERR(c2c_pmu))
980 return PTR_ERR(c2c_pmu);
981
982 ret = nv_c2c_pmu_init_mmio(c2c_pmu);
983 if (ret)
984 return ret;
985
986 ret = nv_c2c_pmu_get_cpus(c2c_pmu);
987 if (ret)
988 return ret;
989
990 ret = nv_c2c_pmu_register_pmu(c2c_pmu);
991 if (ret)
992 return ret;
993
994 dev_dbg(c2c_pmu->dev, "Registered %s PMU\n", c2c_pmu->name);
995
996 return 0;
997 }
998
nv_c2c_pmu_device_remove(struct platform_device * pdev)999 static void nv_c2c_pmu_device_remove(struct platform_device *pdev)
1000 {
1001 struct nv_c2c_pmu *c2c_pmu = platform_get_drvdata(pdev);
1002
1003 perf_pmu_unregister(&c2c_pmu->pmu);
1004 cpuhp_state_remove_instance(nv_c2c_pmu_cpuhp_state, &c2c_pmu->cpuhp_node);
1005 }
1006
1007 static const struct acpi_device_id nv_c2c_pmu_acpi_match[] = {
1008 { "NVDA2023", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVLINK] },
1009 { "NVDA2022", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVCLINK] },
1010 { "NVDA2020", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVDLINK] },
1011 { }
1012 };
1013 MODULE_DEVICE_TABLE(acpi, nv_c2c_pmu_acpi_match);
1014
1015 static struct platform_driver nv_c2c_pmu_driver = {
1016 .driver = {
1017 .name = "nvidia-t410-c2c-pmu",
1018 .acpi_match_table = nv_c2c_pmu_acpi_match,
1019 .suppress_bind_attrs = true,
1020 },
1021 .probe = nv_c2c_pmu_probe,
1022 .remove = nv_c2c_pmu_device_remove,
1023 };
1024
nv_c2c_pmu_init(void)1025 static int __init nv_c2c_pmu_init(void)
1026 {
1027 int ret;
1028
1029 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1030 "perf/nvidia/c2c:online",
1031 nv_c2c_pmu_online_cpu,
1032 nv_c2c_pmu_cpu_teardown);
1033 if (ret < 0)
1034 return ret;
1035
1036 nv_c2c_pmu_cpuhp_state = ret;
1037 return platform_driver_register(&nv_c2c_pmu_driver);
1038 }
1039
nv_c2c_pmu_exit(void)1040 static void __exit nv_c2c_pmu_exit(void)
1041 {
1042 platform_driver_unregister(&nv_c2c_pmu_driver);
1043 cpuhp_remove_multi_state(nv_c2c_pmu_cpuhp_state);
1044 }
1045
1046 module_init(nv_c2c_pmu_init);
1047 module_exit(nv_c2c_pmu_exit);
1048
1049 MODULE_LICENSE("GPL");
1050 MODULE_DESCRIPTION("NVIDIA Tegra410 C2C PMU driver");
1051 MODULE_AUTHOR("Besar Wicaksono <bwicaksono@nvidia.com>");
1052