1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVIDIA Tegra410 C2C PMU driver. 4 * 5 * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 6 */ 7 8 #include <linux/acpi.h> 9 #include <linux/bitops.h> 10 #include <linux/cpumask.h> 11 #include <linux/device.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/module.h> 15 #include <linux/perf_event.h> 16 #include <linux/platform_device.h> 17 #include <linux/property.h> 18 19 /* The C2C interface types in Tegra410. */ 20 #define C2C_TYPE_NVLINK 0x0 21 #define C2C_TYPE_NVCLINK 0x1 22 #define C2C_TYPE_NVDLINK 0x2 23 #define C2C_TYPE_COUNT 0x3 24 25 /* The type of the peer device connected to the C2C interface. */ 26 #define C2C_PEER_TYPE_CPU 0x0 27 #define C2C_PEER_TYPE_GPU 0x1 28 #define C2C_PEER_TYPE_CXLMEM 0x2 29 #define C2C_PEER_TYPE_COUNT 0x3 30 31 /* The number of peer devices can be connected to the C2C interface. */ 32 #define C2C_NR_PEER_CPU 0x1 33 #define C2C_NR_PEER_GPU 0x2 34 #define C2C_NR_PEER_CXLMEM 0x1 35 #define C2C_NR_PEER_MAX 0x2 36 37 /* Number of instances on each interface. */ 38 #define C2C_NR_INST_NVLINK 14 39 #define C2C_NR_INST_NVCLINK 12 40 #define C2C_NR_INST_NVDLINK 16 41 #define C2C_NR_INST_MAX 16 42 43 /* Register offsets. */ 44 #define C2C_CTRL 0x864 45 #define C2C_IN_STATUS 0x868 46 #define C2C_CYCLE_CNTR 0x86c 47 #define C2C_IN_RD_CUM_OUTS_CNTR 0x874 48 #define C2C_IN_RD_REQ_CNTR 0x87c 49 #define C2C_IN_WR_CUM_OUTS_CNTR 0x884 50 #define C2C_IN_WR_REQ_CNTR 0x88c 51 #define C2C_OUT_STATUS 0x890 52 #define C2C_OUT_RD_CUM_OUTS_CNTR 0x898 53 #define C2C_OUT_RD_REQ_CNTR 0x8a0 54 #define C2C_OUT_WR_CUM_OUTS_CNTR 0x8a8 55 #define C2C_OUT_WR_REQ_CNTR 0x8b0 56 57 /* C2C_IN_STATUS register field. */ 58 #define C2C_IN_STATUS_CYCLE_OVF BIT(0) 59 #define C2C_IN_STATUS_IN_RD_CUM_OUTS_OVF BIT(1) 60 #define C2C_IN_STATUS_IN_RD_REQ_OVF BIT(2) 61 #define C2C_IN_STATUS_IN_WR_CUM_OUTS_OVF BIT(3) 62 #define C2C_IN_STATUS_IN_WR_REQ_OVF BIT(4) 63 64 /* C2C_OUT_STATUS register field. */ 65 #define C2C_OUT_STATUS_OUT_RD_CUM_OUTS_OVF BIT(0) 66 #define C2C_OUT_STATUS_OUT_RD_REQ_OVF BIT(1) 67 #define C2C_OUT_STATUS_OUT_WR_CUM_OUTS_OVF BIT(2) 68 #define C2C_OUT_STATUS_OUT_WR_REQ_OVF BIT(3) 69 70 /* Events. */ 71 #define C2C_EVENT_CYCLES 0x0 72 #define C2C_EVENT_IN_RD_CUM_OUTS 0x1 73 #define C2C_EVENT_IN_RD_REQ 0x2 74 #define C2C_EVENT_IN_WR_CUM_OUTS 0x3 75 #define C2C_EVENT_IN_WR_REQ 0x4 76 #define C2C_EVENT_OUT_RD_CUM_OUTS 0x5 77 #define C2C_EVENT_OUT_RD_REQ 0x6 78 #define C2C_EVENT_OUT_WR_CUM_OUTS 0x7 79 #define C2C_EVENT_OUT_WR_REQ 0x8 80 81 #define C2C_NUM_EVENTS 0x9 82 #define C2C_MASK_EVENT 0xFF 83 #define C2C_MAX_ACTIVE_EVENTS 32 84 85 #define C2C_ACTIVE_CPU_MASK 0x0 86 #define C2C_ASSOCIATED_CPU_MASK 0x1 87 88 /* 89 * Maximum poll count for reading counter value using high-low-high sequence. 90 */ 91 #define HILOHI_MAX_POLL 1000 92 93 static unsigned long nv_c2c_pmu_cpuhp_state; 94 95 /* PMU descriptor. */ 96 97 /* C2C type information. */ 98 struct nv_c2c_pmu_data { 99 unsigned int c2c_type; 100 unsigned int nr_inst; 101 const char *name_fmt; 102 }; 103 104 static const struct nv_c2c_pmu_data nv_c2c_pmu_data[] = { 105 [C2C_TYPE_NVLINK] = { 106 .c2c_type = C2C_TYPE_NVLINK, 107 .nr_inst = C2C_NR_INST_NVLINK, 108 .name_fmt = "nvidia_nvlink_c2c_pmu_%u", 109 }, 110 [C2C_TYPE_NVCLINK] = { 111 .c2c_type = C2C_TYPE_NVCLINK, 112 .nr_inst = C2C_NR_INST_NVCLINK, 113 .name_fmt = "nvidia_nvclink_pmu_%u", 114 }, 115 [C2C_TYPE_NVDLINK] = { 116 .c2c_type = C2C_TYPE_NVDLINK, 117 .nr_inst = C2C_NR_INST_NVDLINK, 118 .name_fmt = "nvidia_nvdlink_pmu_%u", 119 }, 120 }; 121 122 /* Tracks the events assigned to the PMU for a given logical index. */ 123 struct nv_c2c_pmu_hw_events { 124 /* The events that are active. */ 125 struct perf_event *events[C2C_MAX_ACTIVE_EVENTS]; 126 127 /* 128 * Each bit indicates a logical counter is being used (or not) for an 129 * event. 130 */ 131 DECLARE_BITMAP(used_ctrs, C2C_MAX_ACTIVE_EVENTS); 132 }; 133 134 struct nv_c2c_pmu { 135 struct pmu pmu; 136 struct device *dev; 137 struct acpi_device *acpi_dev; 138 139 const char *name; 140 const char *identifier; 141 142 const struct nv_c2c_pmu_data *data; 143 unsigned int peer_type; 144 unsigned int socket; 145 unsigned int nr_peer; 146 unsigned long peer_insts[C2C_NR_PEER_MAX][BITS_TO_LONGS(C2C_NR_INST_MAX)]; 147 u32 filter_default; 148 149 struct nv_c2c_pmu_hw_events hw_events; 150 151 cpumask_t associated_cpus; 152 cpumask_t active_cpu; 153 154 struct hlist_node cpuhp_node; 155 156 const struct attribute_group **attr_groups; 157 158 void __iomem *base_broadcast; 159 void __iomem *base[C2C_NR_INST_MAX]; 160 }; 161 162 #define to_c2c_pmu(p) (container_of(p, struct nv_c2c_pmu, pmu)) 163 164 /* Get event type from perf_event. */ 165 static inline u32 get_event_type(struct perf_event *event) 166 { 167 return (event->attr.config) & C2C_MASK_EVENT; 168 } 169 170 static inline u32 get_filter_mask(struct perf_event *event) 171 { 172 u32 filter; 173 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu); 174 175 filter = ((u32)event->attr.config1) & c2c_pmu->filter_default; 176 if (filter == 0) 177 filter = c2c_pmu->filter_default; 178 179 return filter; 180 } 181 182 /* PMU operations. */ 183 184 static int nv_c2c_pmu_get_event_idx(struct nv_c2c_pmu_hw_events *hw_events, 185 struct perf_event *event) 186 { 187 u32 idx; 188 189 idx = find_first_zero_bit(hw_events->used_ctrs, C2C_MAX_ACTIVE_EVENTS); 190 if (idx >= C2C_MAX_ACTIVE_EVENTS) 191 return -EAGAIN; 192 193 set_bit(idx, hw_events->used_ctrs); 194 195 return idx; 196 } 197 198 static bool 199 nv_c2c_pmu_validate_event(struct pmu *pmu, 200 struct nv_c2c_pmu_hw_events *hw_events, 201 struct perf_event *event) 202 { 203 if (is_software_event(event)) 204 return true; 205 206 /* Reject groups spanning multiple HW PMUs. */ 207 if (event->pmu != pmu) 208 return false; 209 210 return nv_c2c_pmu_get_event_idx(hw_events, event) >= 0; 211 } 212 213 /* 214 * Make sure the group of events can be scheduled at once 215 * on the PMU. 216 */ 217 static bool nv_c2c_pmu_validate_group(struct perf_event *event) 218 { 219 struct perf_event *sibling, *leader = event->group_leader; 220 struct nv_c2c_pmu_hw_events fake_hw_events; 221 222 if (event->group_leader == event) 223 return true; 224 225 memset(&fake_hw_events, 0, sizeof(fake_hw_events)); 226 227 if (!nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events, leader)) 228 return false; 229 230 for_each_sibling_event(sibling, leader) { 231 if (!nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events, 232 sibling)) 233 return false; 234 } 235 236 return nv_c2c_pmu_validate_event(event->pmu, &fake_hw_events, event); 237 } 238 239 static int nv_c2c_pmu_event_init(struct perf_event *event) 240 { 241 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu); 242 struct hw_perf_event *hwc = &event->hw; 243 u32 event_type = get_event_type(event); 244 245 if (event->attr.type != event->pmu->type || 246 event_type >= C2C_NUM_EVENTS) 247 return -ENOENT; 248 249 /* 250 * Following other "uncore" PMUs, we do not support sampling mode or 251 * attach to a task (per-process mode). 252 */ 253 if (is_sampling_event(event)) { 254 dev_dbg(c2c_pmu->pmu.dev, "Can't support sampling events\n"); 255 return -EOPNOTSUPP; 256 } 257 258 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) { 259 dev_dbg(c2c_pmu->pmu.dev, "Can't support per-task counters\n"); 260 return -EINVAL; 261 } 262 263 /* 264 * Make sure the CPU assignment is on one of the CPUs associated with 265 * this PMU. 266 */ 267 if (!cpumask_test_cpu(event->cpu, &c2c_pmu->associated_cpus)) { 268 dev_dbg(c2c_pmu->pmu.dev, 269 "Requested cpu is not associated with the PMU\n"); 270 return -EINVAL; 271 } 272 273 /* Enforce the current active CPU to handle the events in this PMU. */ 274 event->cpu = cpumask_first(&c2c_pmu->active_cpu); 275 if (event->cpu >= nr_cpu_ids) 276 return -EINVAL; 277 278 if (!nv_c2c_pmu_validate_group(event)) 279 return -EINVAL; 280 281 hwc->idx = -1; 282 hwc->config = event_type; 283 284 return 0; 285 } 286 287 /* 288 * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence. 289 */ 290 static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count) 291 { 292 u32 val_lo, val_hi; 293 u64 val; 294 295 /* Use high-low-high sequence to avoid tearing */ 296 do { 297 if (max_poll_count-- == 0) { 298 pr_err("NV C2C PMU: timeout hi-low-high sequence\n"); 299 return 0; 300 } 301 302 val_hi = readl(addr + 4); 303 val_lo = readl(addr); 304 } while (val_hi != readl(addr + 4)); 305 306 val = (((u64)val_hi << 32) | val_lo); 307 308 return val; 309 } 310 311 static void nv_c2c_pmu_check_status(struct nv_c2c_pmu *c2c_pmu, u32 instance) 312 { 313 u32 in_status, out_status; 314 315 in_status = readl(c2c_pmu->base[instance] + C2C_IN_STATUS); 316 out_status = readl(c2c_pmu->base[instance] + C2C_OUT_STATUS); 317 318 if (in_status || out_status) 319 dev_warn(c2c_pmu->dev, 320 "C2C PMU overflow in: 0x%x, out: 0x%x\n", 321 in_status, out_status); 322 } 323 324 static u32 nv_c2c_ctr_offset[C2C_NUM_EVENTS] = { 325 [C2C_EVENT_CYCLES] = C2C_CYCLE_CNTR, 326 [C2C_EVENT_IN_RD_CUM_OUTS] = C2C_IN_RD_CUM_OUTS_CNTR, 327 [C2C_EVENT_IN_RD_REQ] = C2C_IN_RD_REQ_CNTR, 328 [C2C_EVENT_IN_WR_CUM_OUTS] = C2C_IN_WR_CUM_OUTS_CNTR, 329 [C2C_EVENT_IN_WR_REQ] = C2C_IN_WR_REQ_CNTR, 330 [C2C_EVENT_OUT_RD_CUM_OUTS] = C2C_OUT_RD_CUM_OUTS_CNTR, 331 [C2C_EVENT_OUT_RD_REQ] = C2C_OUT_RD_REQ_CNTR, 332 [C2C_EVENT_OUT_WR_CUM_OUTS] = C2C_OUT_WR_CUM_OUTS_CNTR, 333 [C2C_EVENT_OUT_WR_REQ] = C2C_OUT_WR_REQ_CNTR, 334 }; 335 336 static u64 nv_c2c_pmu_read_counter(struct perf_event *event) 337 { 338 u32 ctr_id, ctr_offset, filter_mask, filter_idx, inst_idx; 339 unsigned long *inst_mask; 340 DECLARE_BITMAP(filter_bitmap, C2C_NR_PEER_MAX); 341 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu); 342 u64 val = 0; 343 344 filter_mask = get_filter_mask(event); 345 bitmap_from_arr32(filter_bitmap, &filter_mask, c2c_pmu->nr_peer); 346 347 ctr_id = event->hw.config; 348 ctr_offset = nv_c2c_ctr_offset[ctr_id]; 349 350 for_each_set_bit(filter_idx, filter_bitmap, c2c_pmu->nr_peer) { 351 inst_mask = c2c_pmu->peer_insts[filter_idx]; 352 for_each_set_bit(inst_idx, inst_mask, c2c_pmu->data->nr_inst) { 353 nv_c2c_pmu_check_status(c2c_pmu, inst_idx); 354 355 /* 356 * Each instance share same clock and the driver always 357 * enables all instances. So we can use the counts from 358 * one instance for cycle counter. 359 */ 360 if (ctr_id == C2C_EVENT_CYCLES) 361 return read_reg64_hilohi( 362 c2c_pmu->base[inst_idx] + ctr_offset, 363 HILOHI_MAX_POLL); 364 365 /* 366 * For other events, sum up the counts from all instances. 367 */ 368 val += read_reg64_hilohi( 369 c2c_pmu->base[inst_idx] + ctr_offset, 370 HILOHI_MAX_POLL); 371 } 372 } 373 374 return val; 375 } 376 377 static void nv_c2c_pmu_event_update(struct perf_event *event) 378 { 379 struct hw_perf_event *hwc = &event->hw; 380 u64 prev, now; 381 382 do { 383 prev = local64_read(&hwc->prev_count); 384 now = nv_c2c_pmu_read_counter(event); 385 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 386 387 local64_add(now - prev, &event->count); 388 } 389 390 static void nv_c2c_pmu_start(struct perf_event *event, int pmu_flags) 391 { 392 event->hw.state = 0; 393 } 394 395 static void nv_c2c_pmu_stop(struct perf_event *event, int pmu_flags) 396 { 397 event->hw.state |= PERF_HES_STOPPED; 398 } 399 400 static int nv_c2c_pmu_add(struct perf_event *event, int flags) 401 { 402 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu); 403 struct nv_c2c_pmu_hw_events *hw_events = &c2c_pmu->hw_events; 404 struct hw_perf_event *hwc = &event->hw; 405 int idx; 406 407 if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), 408 &c2c_pmu->associated_cpus))) 409 return -ENOENT; 410 411 idx = nv_c2c_pmu_get_event_idx(hw_events, event); 412 if (idx < 0) 413 return idx; 414 415 hw_events->events[idx] = event; 416 hwc->idx = idx; 417 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 418 419 if (flags & PERF_EF_START) 420 nv_c2c_pmu_start(event, PERF_EF_RELOAD); 421 422 /* Propagate changes to the userspace mapping. */ 423 perf_event_update_userpage(event); 424 425 return 0; 426 } 427 428 static void nv_c2c_pmu_del(struct perf_event *event, int flags) 429 { 430 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(event->pmu); 431 struct nv_c2c_pmu_hw_events *hw_events = &c2c_pmu->hw_events; 432 struct hw_perf_event *hwc = &event->hw; 433 int idx = hwc->idx; 434 435 nv_c2c_pmu_stop(event, PERF_EF_UPDATE); 436 437 hw_events->events[idx] = NULL; 438 439 clear_bit(idx, hw_events->used_ctrs); 440 441 perf_event_update_userpage(event); 442 } 443 444 static void nv_c2c_pmu_read(struct perf_event *event) 445 { 446 nv_c2c_pmu_event_update(event); 447 } 448 449 static void nv_c2c_pmu_enable(struct pmu *pmu) 450 { 451 void __iomem *bcast; 452 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu); 453 454 /* Check if any filter is enabled. */ 455 if (bitmap_empty(c2c_pmu->hw_events.used_ctrs, C2C_MAX_ACTIVE_EVENTS)) 456 return; 457 458 /* Enable all the counters. */ 459 bcast = c2c_pmu->base_broadcast; 460 writel(0x1UL, bcast + C2C_CTRL); 461 } 462 463 static void nv_c2c_pmu_disable(struct pmu *pmu) 464 { 465 unsigned int idx; 466 void __iomem *bcast; 467 struct perf_event *event; 468 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu); 469 470 /* Disable all the counters. */ 471 bcast = c2c_pmu->base_broadcast; 472 writel(0x0UL, bcast + C2C_CTRL); 473 474 /* 475 * The counters will start from 0 again on restart. 476 * Update the events immediately to avoid losing the counts. 477 */ 478 for_each_set_bit(idx, c2c_pmu->hw_events.used_ctrs, 479 C2C_MAX_ACTIVE_EVENTS) { 480 event = c2c_pmu->hw_events.events[idx]; 481 482 if (!event) 483 continue; 484 485 nv_c2c_pmu_event_update(event); 486 487 local64_set(&event->hw.prev_count, 0ULL); 488 } 489 } 490 491 /* PMU identifier attribute. */ 492 493 static ssize_t nv_c2c_pmu_identifier_show(struct device *dev, 494 struct device_attribute *attr, 495 char *page) 496 { 497 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(dev_get_drvdata(dev)); 498 499 return sysfs_emit(page, "%s\n", c2c_pmu->identifier); 500 } 501 502 static struct device_attribute nv_c2c_pmu_identifier_attr = 503 __ATTR(identifier, 0444, nv_c2c_pmu_identifier_show, NULL); 504 505 static struct attribute *nv_c2c_pmu_identifier_attrs[] = { 506 &nv_c2c_pmu_identifier_attr.attr, 507 NULL, 508 }; 509 510 static struct attribute_group nv_c2c_pmu_identifier_attr_group = { 511 .attrs = nv_c2c_pmu_identifier_attrs, 512 }; 513 514 /* Peer attribute. */ 515 516 static ssize_t nv_c2c_pmu_peer_show(struct device *dev, 517 struct device_attribute *attr, 518 char *page) 519 { 520 const char *peer_type[C2C_PEER_TYPE_COUNT] = { 521 [C2C_PEER_TYPE_CPU] = "cpu", 522 [C2C_PEER_TYPE_GPU] = "gpu", 523 [C2C_PEER_TYPE_CXLMEM] = "cxlmem", 524 }; 525 526 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(dev_get_drvdata(dev)); 527 return sysfs_emit(page, "nr_%s=%u\n", peer_type[c2c_pmu->peer_type], 528 c2c_pmu->nr_peer); 529 } 530 531 static struct device_attribute nv_c2c_pmu_peer_attr = 532 __ATTR(peer, 0444, nv_c2c_pmu_peer_show, NULL); 533 534 static struct attribute *nv_c2c_pmu_peer_attrs[] = { 535 &nv_c2c_pmu_peer_attr.attr, 536 NULL, 537 }; 538 539 static struct attribute_group nv_c2c_pmu_peer_attr_group = { 540 .attrs = nv_c2c_pmu_peer_attrs, 541 }; 542 543 /* Format attributes. */ 544 545 #define NV_C2C_PMU_EXT_ATTR(_name, _func, _config) \ 546 (&((struct dev_ext_attribute[]){ \ 547 { \ 548 .attr = __ATTR(_name, 0444, _func, NULL), \ 549 .var = (void *)_config \ 550 } \ 551 })[0].attr.attr) 552 553 #define NV_C2C_PMU_FORMAT_ATTR(_name, _config) \ 554 NV_C2C_PMU_EXT_ATTR(_name, device_show_string, _config) 555 556 #define NV_C2C_PMU_FORMAT_EVENT_ATTR \ 557 NV_C2C_PMU_FORMAT_ATTR(event, "config:0-3") 558 559 static struct attribute *nv_c2c_pmu_gpu_formats[] = { 560 NV_C2C_PMU_FORMAT_EVENT_ATTR, 561 NV_C2C_PMU_FORMAT_ATTR(gpu_mask, "config1:0-1"), 562 NULL, 563 }; 564 565 static const struct attribute_group nv_c2c_pmu_gpu_format_group = { 566 .name = "format", 567 .attrs = nv_c2c_pmu_gpu_formats, 568 }; 569 570 static struct attribute *nv_c2c_pmu_formats[] = { 571 NV_C2C_PMU_FORMAT_EVENT_ATTR, 572 NULL, 573 }; 574 575 static const struct attribute_group nv_c2c_pmu_format_group = { 576 .name = "format", 577 .attrs = nv_c2c_pmu_formats, 578 }; 579 580 /* Event attributes. */ 581 582 static ssize_t nv_c2c_pmu_sysfs_event_show(struct device *dev, 583 struct device_attribute *attr, 584 char *buf) 585 { 586 struct perf_pmu_events_attr *pmu_attr; 587 588 pmu_attr = container_of(attr, typeof(*pmu_attr), attr); 589 return sysfs_emit(buf, "event=0x%llx\n", pmu_attr->id); 590 } 591 592 #define NV_C2C_PMU_EVENT_ATTR(_name, _config) \ 593 PMU_EVENT_ATTR_ID(_name, nv_c2c_pmu_sysfs_event_show, _config) 594 595 static struct attribute *nv_c2c_pmu_gpu_events[] = { 596 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES), 597 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS), 598 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ), 599 NV_C2C_PMU_EVENT_ATTR(in_wr_cum_outs, C2C_EVENT_IN_WR_CUM_OUTS), 600 NV_C2C_PMU_EVENT_ATTR(in_wr_req, C2C_EVENT_IN_WR_REQ), 601 NV_C2C_PMU_EVENT_ATTR(out_rd_cum_outs, C2C_EVENT_OUT_RD_CUM_OUTS), 602 NV_C2C_PMU_EVENT_ATTR(out_rd_req, C2C_EVENT_OUT_RD_REQ), 603 NV_C2C_PMU_EVENT_ATTR(out_wr_cum_outs, C2C_EVENT_OUT_WR_CUM_OUTS), 604 NV_C2C_PMU_EVENT_ATTR(out_wr_req, C2C_EVENT_OUT_WR_REQ), 605 NULL 606 }; 607 608 static const struct attribute_group nv_c2c_pmu_gpu_events_group = { 609 .name = "events", 610 .attrs = nv_c2c_pmu_gpu_events, 611 }; 612 613 static struct attribute *nv_c2c_pmu_cpu_events[] = { 614 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES), 615 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS), 616 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ), 617 NV_C2C_PMU_EVENT_ATTR(out_rd_cum_outs, C2C_EVENT_OUT_RD_CUM_OUTS), 618 NV_C2C_PMU_EVENT_ATTR(out_rd_req, C2C_EVENT_OUT_RD_REQ), 619 NULL 620 }; 621 622 static const struct attribute_group nv_c2c_pmu_cpu_events_group = { 623 .name = "events", 624 .attrs = nv_c2c_pmu_cpu_events, 625 }; 626 627 static struct attribute *nv_c2c_pmu_cxlmem_events[] = { 628 NV_C2C_PMU_EVENT_ATTR(cycles, C2C_EVENT_CYCLES), 629 NV_C2C_PMU_EVENT_ATTR(in_rd_cum_outs, C2C_EVENT_IN_RD_CUM_OUTS), 630 NV_C2C_PMU_EVENT_ATTR(in_rd_req, C2C_EVENT_IN_RD_REQ), 631 NULL 632 }; 633 634 static const struct attribute_group nv_c2c_pmu_cxlmem_events_group = { 635 .name = "events", 636 .attrs = nv_c2c_pmu_cxlmem_events, 637 }; 638 639 /* Cpumask attributes. */ 640 641 static ssize_t nv_c2c_pmu_cpumask_show(struct device *dev, 642 struct device_attribute *attr, char *buf) 643 { 644 struct pmu *pmu = dev_get_drvdata(dev); 645 struct nv_c2c_pmu *c2c_pmu = to_c2c_pmu(pmu); 646 struct dev_ext_attribute *eattr = 647 container_of(attr, struct dev_ext_attribute, attr); 648 unsigned long mask_id = (unsigned long)eattr->var; 649 const cpumask_t *cpumask; 650 651 switch (mask_id) { 652 case C2C_ACTIVE_CPU_MASK: 653 cpumask = &c2c_pmu->active_cpu; 654 break; 655 case C2C_ASSOCIATED_CPU_MASK: 656 cpumask = &c2c_pmu->associated_cpus; 657 break; 658 default: 659 return 0; 660 } 661 return cpumap_print_to_pagebuf(true, buf, cpumask); 662 } 663 664 #define NV_C2C_PMU_CPUMASK_ATTR(_name, _config) \ 665 NV_C2C_PMU_EXT_ATTR(_name, nv_c2c_pmu_cpumask_show, \ 666 (unsigned long)_config) 667 668 static struct attribute *nv_c2c_pmu_cpumask_attrs[] = { 669 NV_C2C_PMU_CPUMASK_ATTR(cpumask, C2C_ACTIVE_CPU_MASK), 670 NV_C2C_PMU_CPUMASK_ATTR(associated_cpus, C2C_ASSOCIATED_CPU_MASK), 671 NULL, 672 }; 673 674 static const struct attribute_group nv_c2c_pmu_cpumask_attr_group = { 675 .attrs = nv_c2c_pmu_cpumask_attrs, 676 }; 677 678 /* Attribute groups for C2C PMU connecting SoC and GPU */ 679 static const struct attribute_group *nv_c2c_pmu_gpu_attr_groups[] = { 680 &nv_c2c_pmu_gpu_format_group, 681 &nv_c2c_pmu_gpu_events_group, 682 &nv_c2c_pmu_cpumask_attr_group, 683 &nv_c2c_pmu_identifier_attr_group, 684 &nv_c2c_pmu_peer_attr_group, 685 NULL 686 }; 687 688 /* Attribute groups for C2C PMU connecting multiple SoCs */ 689 static const struct attribute_group *nv_c2c_pmu_cpu_attr_groups[] = { 690 &nv_c2c_pmu_format_group, 691 &nv_c2c_pmu_cpu_events_group, 692 &nv_c2c_pmu_cpumask_attr_group, 693 &nv_c2c_pmu_identifier_attr_group, 694 &nv_c2c_pmu_peer_attr_group, 695 NULL 696 }; 697 698 /* Attribute groups for C2C PMU connecting SoC and CXLMEM */ 699 static const struct attribute_group *nv_c2c_pmu_cxlmem_attr_groups[] = { 700 &nv_c2c_pmu_format_group, 701 &nv_c2c_pmu_cxlmem_events_group, 702 &nv_c2c_pmu_cpumask_attr_group, 703 &nv_c2c_pmu_identifier_attr_group, 704 &nv_c2c_pmu_peer_attr_group, 705 NULL 706 }; 707 708 static int nv_c2c_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 709 { 710 struct nv_c2c_pmu *c2c_pmu = 711 hlist_entry_safe(node, struct nv_c2c_pmu, cpuhp_node); 712 713 if (!cpumask_test_cpu(cpu, &c2c_pmu->associated_cpus)) 714 return 0; 715 716 /* If the PMU is already managed, there is nothing to do */ 717 if (!cpumask_empty(&c2c_pmu->active_cpu)) 718 return 0; 719 720 /* Use this CPU for event counting */ 721 cpumask_set_cpu(cpu, &c2c_pmu->active_cpu); 722 723 return 0; 724 } 725 726 static int nv_c2c_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 727 { 728 unsigned int dst; 729 730 struct nv_c2c_pmu *c2c_pmu = 731 hlist_entry_safe(node, struct nv_c2c_pmu, cpuhp_node); 732 733 /* Nothing to do if this CPU doesn't own the PMU */ 734 if (!cpumask_test_and_clear_cpu(cpu, &c2c_pmu->active_cpu)) 735 return 0; 736 737 /* Choose a new CPU to migrate ownership of the PMU to */ 738 dst = cpumask_any_and_but(&c2c_pmu->associated_cpus, 739 cpu_online_mask, cpu); 740 if (dst >= nr_cpu_ids) 741 return 0; 742 743 /* Use this CPU for event counting */ 744 perf_pmu_migrate_context(&c2c_pmu->pmu, cpu, dst); 745 cpumask_set_cpu(dst, &c2c_pmu->active_cpu); 746 747 return 0; 748 } 749 750 static int nv_c2c_pmu_get_cpus(struct nv_c2c_pmu *c2c_pmu) 751 { 752 int socket = c2c_pmu->socket, cpu; 753 754 for_each_possible_cpu(cpu) { 755 if (cpu_to_node(cpu) == socket) 756 cpumask_set_cpu(cpu, &c2c_pmu->associated_cpus); 757 } 758 759 if (cpumask_empty(&c2c_pmu->associated_cpus)) { 760 dev_dbg(c2c_pmu->dev, 761 "No cpu associated with C2C PMU socket-%u\n", socket); 762 return -ENODEV; 763 } 764 765 return 0; 766 } 767 768 static int nv_c2c_pmu_init_socket(struct nv_c2c_pmu *c2c_pmu) 769 { 770 const char *uid_str; 771 int ret, socket; 772 773 uid_str = acpi_device_uid(c2c_pmu->acpi_dev); 774 if (!uid_str) { 775 dev_err(c2c_pmu->dev, "No ACPI device UID\n"); 776 return -ENODEV; 777 } 778 779 ret = kstrtou32(uid_str, 0, &socket); 780 if (ret) { 781 dev_err(c2c_pmu->dev, "Failed to parse ACPI device UID\n"); 782 return ret; 783 } 784 785 c2c_pmu->socket = socket; 786 return 0; 787 } 788 789 static int nv_c2c_pmu_init_id(struct nv_c2c_pmu *c2c_pmu) 790 { 791 char *name; 792 793 name = devm_kasprintf(c2c_pmu->dev, GFP_KERNEL, c2c_pmu->data->name_fmt, 794 c2c_pmu->socket); 795 if (!name) 796 return -ENOMEM; 797 798 c2c_pmu->name = name; 799 800 c2c_pmu->identifier = acpi_device_hid(c2c_pmu->acpi_dev); 801 802 return 0; 803 } 804 805 static int nv_c2c_pmu_init_filter(struct nv_c2c_pmu *c2c_pmu) 806 { 807 u32 cpu_en = 0; 808 struct device *dev = c2c_pmu->dev; 809 const struct nv_c2c_pmu_data *data = c2c_pmu->data; 810 811 if (data->c2c_type == C2C_TYPE_NVDLINK) { 812 c2c_pmu->peer_type = C2C_PEER_TYPE_CXLMEM; 813 814 c2c_pmu->peer_insts[0][0] = (1UL << data->nr_inst) - 1; 815 816 c2c_pmu->nr_peer = C2C_NR_PEER_CXLMEM; 817 c2c_pmu->filter_default = (1 << c2c_pmu->nr_peer) - 1; 818 819 c2c_pmu->attr_groups = nv_c2c_pmu_cxlmem_attr_groups; 820 821 return 0; 822 } 823 824 if (device_property_read_u32(dev, "cpu_en_mask", &cpu_en)) 825 dev_dbg(dev, "no cpu_en_mask property\n"); 826 827 if (cpu_en) { 828 c2c_pmu->peer_type = C2C_PEER_TYPE_CPU; 829 830 /* Fill peer_insts bitmap with instances connected to peer CPU. */ 831 bitmap_from_arr32(c2c_pmu->peer_insts[0], &cpu_en, data->nr_inst); 832 833 c2c_pmu->nr_peer = 1; 834 c2c_pmu->attr_groups = nv_c2c_pmu_cpu_attr_groups; 835 } else { 836 u32 i; 837 const char *props[C2C_NR_PEER_MAX] = { 838 "gpu0_en_mask", "gpu1_en_mask" 839 }; 840 841 for (i = 0; i < C2C_NR_PEER_MAX; i++) { 842 u32 gpu_en = 0; 843 844 if (device_property_read_u32(dev, props[i], &gpu_en)) 845 dev_dbg(dev, "no %s property\n", props[i]); 846 847 if (gpu_en) { 848 /* Fill peer_insts bitmap with instances connected to peer GPU. */ 849 bitmap_from_arr32(c2c_pmu->peer_insts[i], &gpu_en, 850 data->nr_inst); 851 852 c2c_pmu->nr_peer++; 853 } 854 } 855 856 if (c2c_pmu->nr_peer == 0) { 857 dev_err(dev, "No GPU is enabled\n"); 858 return -EINVAL; 859 } 860 861 c2c_pmu->peer_type = C2C_PEER_TYPE_GPU; 862 c2c_pmu->attr_groups = nv_c2c_pmu_gpu_attr_groups; 863 } 864 865 c2c_pmu->filter_default = (1 << c2c_pmu->nr_peer) - 1; 866 867 return 0; 868 } 869 870 static void *nv_c2c_pmu_init_pmu(struct platform_device *pdev) 871 { 872 int ret; 873 struct nv_c2c_pmu *c2c_pmu; 874 struct acpi_device *acpi_dev; 875 struct device *dev = &pdev->dev; 876 877 acpi_dev = ACPI_COMPANION(dev); 878 if (!acpi_dev) 879 return ERR_PTR(-ENODEV); 880 881 c2c_pmu = devm_kzalloc(dev, sizeof(*c2c_pmu), GFP_KERNEL); 882 if (!c2c_pmu) 883 return ERR_PTR(-ENOMEM); 884 885 c2c_pmu->dev = dev; 886 c2c_pmu->acpi_dev = acpi_dev; 887 c2c_pmu->data = (const struct nv_c2c_pmu_data *)device_get_match_data(dev); 888 if (!c2c_pmu->data) 889 return ERR_PTR(-EINVAL); 890 891 platform_set_drvdata(pdev, c2c_pmu); 892 893 ret = nv_c2c_pmu_init_socket(c2c_pmu); 894 if (ret) 895 return ERR_PTR(ret); 896 897 ret = nv_c2c_pmu_init_id(c2c_pmu); 898 if (ret) 899 return ERR_PTR(ret); 900 901 ret = nv_c2c_pmu_init_filter(c2c_pmu); 902 if (ret) 903 return ERR_PTR(ret); 904 905 return c2c_pmu; 906 } 907 908 static int nv_c2c_pmu_init_mmio(struct nv_c2c_pmu *c2c_pmu) 909 { 910 int i; 911 struct device *dev = c2c_pmu->dev; 912 struct platform_device *pdev = to_platform_device(dev); 913 const struct nv_c2c_pmu_data *data = c2c_pmu->data; 914 915 /* Map the address of all the instances. */ 916 for (i = 0; i < data->nr_inst; i++) { 917 c2c_pmu->base[i] = devm_platform_ioremap_resource(pdev, i); 918 if (IS_ERR(c2c_pmu->base[i])) { 919 dev_err(dev, "Failed map address for instance %d\n", i); 920 return PTR_ERR(c2c_pmu->base[i]); 921 } 922 } 923 924 /* Map broadcast address. */ 925 c2c_pmu->base_broadcast = devm_platform_ioremap_resource(pdev, 926 data->nr_inst); 927 if (IS_ERR(c2c_pmu->base_broadcast)) { 928 dev_err(dev, "Failed map broadcast address\n"); 929 return PTR_ERR(c2c_pmu->base_broadcast); 930 } 931 932 return 0; 933 } 934 935 static int nv_c2c_pmu_register_pmu(struct nv_c2c_pmu *c2c_pmu) 936 { 937 int ret; 938 939 ret = cpuhp_state_add_instance(nv_c2c_pmu_cpuhp_state, 940 &c2c_pmu->cpuhp_node); 941 if (ret) { 942 dev_err(c2c_pmu->dev, "Error %d registering hotplug\n", ret); 943 return ret; 944 } 945 946 c2c_pmu->pmu = (struct pmu) { 947 .parent = c2c_pmu->dev, 948 .task_ctx_nr = perf_invalid_context, 949 .pmu_enable = nv_c2c_pmu_enable, 950 .pmu_disable = nv_c2c_pmu_disable, 951 .event_init = nv_c2c_pmu_event_init, 952 .add = nv_c2c_pmu_add, 953 .del = nv_c2c_pmu_del, 954 .start = nv_c2c_pmu_start, 955 .stop = nv_c2c_pmu_stop, 956 .read = nv_c2c_pmu_read, 957 .attr_groups = c2c_pmu->attr_groups, 958 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | 959 PERF_PMU_CAP_NO_INTERRUPT, 960 }; 961 962 ret = perf_pmu_register(&c2c_pmu->pmu, c2c_pmu->name, -1); 963 if (ret) { 964 dev_err(c2c_pmu->dev, "Failed to register C2C PMU: %d\n", ret); 965 cpuhp_state_remove_instance(nv_c2c_pmu_cpuhp_state, 966 &c2c_pmu->cpuhp_node); 967 return ret; 968 } 969 970 return 0; 971 } 972 973 static int nv_c2c_pmu_probe(struct platform_device *pdev) 974 { 975 int ret; 976 struct nv_c2c_pmu *c2c_pmu; 977 978 c2c_pmu = nv_c2c_pmu_init_pmu(pdev); 979 if (IS_ERR(c2c_pmu)) 980 return PTR_ERR(c2c_pmu); 981 982 ret = nv_c2c_pmu_init_mmio(c2c_pmu); 983 if (ret) 984 return ret; 985 986 ret = nv_c2c_pmu_get_cpus(c2c_pmu); 987 if (ret) 988 return ret; 989 990 ret = nv_c2c_pmu_register_pmu(c2c_pmu); 991 if (ret) 992 return ret; 993 994 dev_dbg(c2c_pmu->dev, "Registered %s PMU\n", c2c_pmu->name); 995 996 return 0; 997 } 998 999 static void nv_c2c_pmu_device_remove(struct platform_device *pdev) 1000 { 1001 struct nv_c2c_pmu *c2c_pmu = platform_get_drvdata(pdev); 1002 1003 perf_pmu_unregister(&c2c_pmu->pmu); 1004 cpuhp_state_remove_instance(nv_c2c_pmu_cpuhp_state, &c2c_pmu->cpuhp_node); 1005 } 1006 1007 static const struct acpi_device_id nv_c2c_pmu_acpi_match[] = { 1008 { "NVDA2023", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVLINK] }, 1009 { "NVDA2022", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVCLINK] }, 1010 { "NVDA2020", (kernel_ulong_t)&nv_c2c_pmu_data[C2C_TYPE_NVDLINK] }, 1011 { } 1012 }; 1013 MODULE_DEVICE_TABLE(acpi, nv_c2c_pmu_acpi_match); 1014 1015 static struct platform_driver nv_c2c_pmu_driver = { 1016 .driver = { 1017 .name = "nvidia-t410-c2c-pmu", 1018 .acpi_match_table = nv_c2c_pmu_acpi_match, 1019 .suppress_bind_attrs = true, 1020 }, 1021 .probe = nv_c2c_pmu_probe, 1022 .remove = nv_c2c_pmu_device_remove, 1023 }; 1024 1025 static int __init nv_c2c_pmu_init(void) 1026 { 1027 int ret; 1028 1029 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1030 "perf/nvidia/c2c:online", 1031 nv_c2c_pmu_online_cpu, 1032 nv_c2c_pmu_cpu_teardown); 1033 if (ret < 0) 1034 return ret; 1035 1036 nv_c2c_pmu_cpuhp_state = ret; 1037 return platform_driver_register(&nv_c2c_pmu_driver); 1038 } 1039 1040 static void __exit nv_c2c_pmu_exit(void) 1041 { 1042 platform_driver_unregister(&nv_c2c_pmu_driver); 1043 cpuhp_remove_multi_state(nv_c2c_pmu_cpuhp_state); 1044 } 1045 1046 module_init(nv_c2c_pmu_init); 1047 module_exit(nv_c2c_pmu_exit); 1048 1049 MODULE_LICENSE("GPL"); 1050 MODULE_DESCRIPTION("NVIDIA Tegra410 C2C PMU driver"); 1051 MODULE_AUTHOR("Besar Wicaksono <bwicaksono@nvidia.com>"); 1052