1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * 5 * Author: Jacob Shin <jacob.shin@amd.com> 6 */ 7 8 #include <linux/perf_event.h> 9 #include <linux/percpu.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/init.h> 13 #include <linux/cpu.h> 14 #include <linux/cpumask.h> 15 #include <linux/cpufeature.h> 16 #include <linux/smp.h> 17 18 #include <asm/perf_event.h> 19 #include <asm/cpuid/api.h> 20 #include <asm/msr.h> 21 22 #define NUM_COUNTERS_NB 4 23 #define NUM_COUNTERS_L2 4 24 #define NUM_COUNTERS_L3 6 25 #define NUM_COUNTERS_MAX 64 26 27 #define RDPMC_BASE_NB 6 28 #define RDPMC_BASE_LLC 10 29 30 #define COUNTER_SHIFT 16 31 #define UNCORE_NAME_LEN 16 32 #define UNCORE_GROUP_MAX 256 33 34 #undef pr_fmt 35 #define pr_fmt(fmt) "amd_uncore: " fmt 36 37 static int pmu_version; 38 39 struct amd_uncore_ctx { 40 int refcnt; 41 int cpu; 42 struct perf_event **events; 43 unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)]; 44 int nr_active; 45 struct hrtimer hrtimer; 46 u64 hrtimer_duration; 47 }; 48 49 struct amd_uncore_pmu { 50 char name[UNCORE_NAME_LEN]; 51 int num_counters; 52 int rdpmc_base; 53 u32 msr_base; 54 int group; 55 cpumask_t active_mask; 56 struct pmu pmu; 57 struct amd_uncore_ctx * __percpu *ctx; 58 }; 59 60 enum { 61 UNCORE_TYPE_DF, 62 UNCORE_TYPE_L3, 63 UNCORE_TYPE_UMC, 64 65 UNCORE_TYPE_MAX 66 }; 67 68 union amd_uncore_info { 69 struct { 70 u64 aux_data:32; /* auxiliary data */ 71 u64 num_pmcs:8; /* number of counters */ 72 u64 gid:8; /* group id */ 73 u64 cid:8; /* context id */ 74 } split; 75 u64 full; 76 }; 77 78 struct amd_uncore { 79 union amd_uncore_info __percpu *info; 80 struct amd_uncore_pmu *pmus; 81 unsigned int num_pmus; 82 bool init_done; 83 void (*scan)(struct amd_uncore *uncore, unsigned int cpu); 84 int (*init)(struct amd_uncore *uncore, unsigned int cpu); 85 void (*move)(struct amd_uncore *uncore, unsigned int cpu); 86 void (*free)(struct amd_uncore *uncore, unsigned int cpu); 87 }; 88 89 static struct amd_uncore uncores[UNCORE_TYPE_MAX]; 90 91 /* Interval for hrtimer, defaults to 60000 milliseconds */ 92 static unsigned int update_interval = 60 * MSEC_PER_SEC; 93 module_param(update_interval, uint, 0444); 94 95 static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event) 96 { 97 return container_of(event->pmu, struct amd_uncore_pmu, pmu); 98 } 99 100 static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer) 101 { 102 struct amd_uncore_ctx *ctx; 103 struct perf_event *event; 104 int bit; 105 106 ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer); 107 108 if (!ctx->nr_active || ctx->cpu != smp_processor_id()) 109 return HRTIMER_NORESTART; 110 111 for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) { 112 event = ctx->events[bit]; 113 event->pmu->read(event); 114 } 115 116 hrtimer_forward_now(hrtimer, ns_to_ktime(ctx->hrtimer_duration)); 117 return HRTIMER_RESTART; 118 } 119 120 static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx) 121 { 122 hrtimer_start(&ctx->hrtimer, ns_to_ktime(ctx->hrtimer_duration), 123 HRTIMER_MODE_REL_PINNED_HARD); 124 } 125 126 static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx) 127 { 128 hrtimer_cancel(&ctx->hrtimer); 129 } 130 131 static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx) 132 { 133 hrtimer_setup(&ctx->hrtimer, amd_uncore_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 134 } 135 136 static void amd_uncore_read(struct perf_event *event) 137 { 138 struct hw_perf_event *hwc = &event->hw; 139 u64 prev, new; 140 s64 delta; 141 142 /* 143 * since we do not enable counter overflow interrupts, 144 * we do not have to worry about prev_count changing on us 145 */ 146 147 prev = local64_read(&hwc->prev_count); 148 149 /* 150 * Some uncore PMUs do not have RDPMC assignments. In such cases, 151 * read counts directly from the corresponding PERF_CTR. 152 */ 153 if (hwc->event_base_rdpmc < 0) 154 rdmsrq(hwc->event_base, new); 155 else 156 new = rdpmc(hwc->event_base_rdpmc); 157 158 local64_set(&hwc->prev_count, new); 159 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); 160 delta >>= COUNTER_SHIFT; 161 local64_add(delta, &event->count); 162 } 163 164 static void amd_uncore_start(struct perf_event *event, int flags) 165 { 166 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 167 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 168 struct hw_perf_event *hwc = &event->hw; 169 170 if (!ctx->nr_active++) 171 amd_uncore_start_hrtimer(ctx); 172 173 if (flags & PERF_EF_RELOAD) 174 wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count)); 175 176 hwc->state = 0; 177 __set_bit(hwc->idx, ctx->active_mask); 178 wrmsrq(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); 179 perf_event_update_userpage(event); 180 } 181 182 static void amd_uncore_stop(struct perf_event *event, int flags) 183 { 184 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 185 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 186 struct hw_perf_event *hwc = &event->hw; 187 188 wrmsrq(hwc->config_base, hwc->config); 189 hwc->state |= PERF_HES_STOPPED; 190 191 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 192 event->pmu->read(event); 193 hwc->state |= PERF_HES_UPTODATE; 194 } 195 196 if (!--ctx->nr_active) 197 amd_uncore_cancel_hrtimer(ctx); 198 199 __clear_bit(hwc->idx, ctx->active_mask); 200 } 201 202 static int amd_uncore_add(struct perf_event *event, int flags) 203 { 204 int i; 205 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 206 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 207 struct hw_perf_event *hwc = &event->hw; 208 209 /* are we already assigned? */ 210 if (hwc->idx != -1 && ctx->events[hwc->idx] == event) 211 goto out; 212 213 for (i = 0; i < pmu->num_counters; i++) { 214 if (ctx->events[i] == event) { 215 hwc->idx = i; 216 goto out; 217 } 218 } 219 220 /* if not, take the first available counter */ 221 hwc->idx = -1; 222 for (i = 0; i < pmu->num_counters; i++) { 223 struct perf_event *tmp = NULL; 224 225 if (try_cmpxchg(&ctx->events[i], &tmp, event)) { 226 hwc->idx = i; 227 break; 228 } 229 } 230 231 out: 232 if (hwc->idx == -1) 233 return -EBUSY; 234 235 hwc->config_base = pmu->msr_base + (2 * hwc->idx); 236 hwc->event_base = pmu->msr_base + 1 + (2 * hwc->idx); 237 hwc->event_base_rdpmc = pmu->rdpmc_base + hwc->idx; 238 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 239 240 if (pmu->rdpmc_base < 0) 241 hwc->event_base_rdpmc = -1; 242 243 if (flags & PERF_EF_START) 244 event->pmu->start(event, PERF_EF_RELOAD); 245 246 return 0; 247 } 248 249 static void amd_uncore_del(struct perf_event *event, int flags) 250 { 251 int i; 252 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 253 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 254 struct hw_perf_event *hwc = &event->hw; 255 256 event->pmu->stop(event, PERF_EF_UPDATE); 257 258 for (i = 0; i < pmu->num_counters; i++) { 259 struct perf_event *tmp = event; 260 261 if (try_cmpxchg(&ctx->events[i], &tmp, NULL)) 262 break; 263 } 264 265 hwc->idx = -1; 266 } 267 268 static bool amd_uncore_group_valid(struct perf_event *event) 269 { 270 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 271 struct perf_event *leader = event->group_leader; 272 struct perf_event *sibling; 273 int counters = 0; 274 275 if (leader->pmu == event->pmu) 276 counters++; 277 278 for_each_sibling_event(sibling, leader) { 279 if (sibling->pmu == event->pmu && 280 sibling->state > PERF_EVENT_STATE_OFF) 281 counters++; 282 } 283 284 /* 285 * When pmu->event_init() is called, the event is yet to be linked to 286 * its leader's sibling list, so it is counted separately 287 */ 288 return (counters + 1) <= pmu->num_counters; 289 } 290 291 static int amd_uncore_event_init(struct perf_event *event) 292 { 293 struct amd_uncore_pmu *pmu; 294 struct amd_uncore_ctx *ctx; 295 struct hw_perf_event *hwc = &event->hw; 296 297 if (event->attr.type != event->pmu->type) 298 return -ENOENT; 299 300 if (event->cpu < 0) 301 return -EINVAL; 302 303 pmu = event_to_amd_uncore_pmu(event); 304 ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 305 if (!ctx) 306 return -ENODEV; 307 308 /* 309 * Ensure that all events in a group can be scheduled together so that 310 * a failure can be reported at perf_event_open() time rather than 311 * silently at pmu->add() time when no free counter is found 312 */ 313 if (event->group_leader != event && !amd_uncore_group_valid(event)) 314 return -EINVAL; 315 316 /* 317 * NB and Last level cache counters (MSRs) are shared across all cores 318 * that share the same NB / Last level cache. On family 16h and below, 319 * Interrupts can be directed to a single target core, however, event 320 * counts generated by processes running on other cores cannot be masked 321 * out. So we do not support sampling and per-thread events via 322 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts: 323 */ 324 hwc->config = event->attr.config; 325 hwc->idx = -1; 326 327 /* 328 * since request can come in to any of the shared cores, we will remap 329 * to a single common cpu. 330 */ 331 event->cpu = ctx->cpu; 332 333 return 0; 334 } 335 336 static umode_t 337 amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i) 338 { 339 return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ? 340 attr->mode : 0; 341 } 342 343 static umode_t 344 amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i) 345 { 346 return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0; 347 } 348 349 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, 350 struct device_attribute *attr, 351 char *buf) 352 { 353 struct pmu *ptr = dev_get_drvdata(dev); 354 struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu); 355 356 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu->active_mask)); 357 } 358 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL); 359 360 static struct attribute *amd_uncore_attrs[] = { 361 &dev_attr_cpumask.attr, 362 NULL, 363 }; 364 365 static struct attribute_group amd_uncore_attr_group = { 366 .attrs = amd_uncore_attrs, 367 }; 368 369 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ 370 static ssize_t __uncore_##_var##_show(struct device *dev, \ 371 struct device_attribute *attr, \ 372 char *page) \ 373 { \ 374 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 375 return sprintf(page, _format "\n"); \ 376 } \ 377 static struct device_attribute format_attr_##_var = \ 378 __ATTR(_name, 0444, __uncore_##_var##_show, NULL) 379 380 DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); 381 DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */ 382 DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */ 383 DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3, PerfMonV2 UMC */ 384 DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15"); 385 DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */ 386 DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */ 387 DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */ 388 DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */ 389 DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */ 390 DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */ 391 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ 392 DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ 393 DEFINE_UNCORE_FORMAT_ATTR(rdwrmask, rdwrmask, "config:8-9"); /* PerfMonV2 UMC */ 394 395 /* Common DF and NB attributes */ 396 static struct attribute *amd_uncore_df_format_attr[] = { 397 &format_attr_event12.attr, /* event */ 398 &format_attr_umask8.attr, /* umask */ 399 NULL, 400 }; 401 402 /* Common L2 and L3 attributes */ 403 static struct attribute *amd_uncore_l3_format_attr[] = { 404 &format_attr_event12.attr, /* event */ 405 &format_attr_umask8.attr, /* umask */ 406 NULL, /* threadmask */ 407 NULL, 408 }; 409 410 /* Common UMC attributes */ 411 static struct attribute *amd_uncore_umc_format_attr[] = { 412 &format_attr_event8.attr, /* event */ 413 &format_attr_rdwrmask.attr, /* rdwrmask */ 414 NULL, 415 }; 416 417 /* F17h unique L3 attributes */ 418 static struct attribute *amd_f17h_uncore_l3_format_attr[] = { 419 &format_attr_slicemask.attr, /* slicemask */ 420 NULL, 421 }; 422 423 /* F19h unique L3 attributes */ 424 static struct attribute *amd_f19h_uncore_l3_format_attr[] = { 425 &format_attr_coreid.attr, /* coreid */ 426 &format_attr_enallslices.attr, /* enallslices */ 427 &format_attr_enallcores.attr, /* enallcores */ 428 &format_attr_sliceid.attr, /* sliceid */ 429 NULL, 430 }; 431 432 static struct attribute_group amd_uncore_df_format_group = { 433 .name = "format", 434 .attrs = amd_uncore_df_format_attr, 435 }; 436 437 static struct attribute_group amd_uncore_l3_format_group = { 438 .name = "format", 439 .attrs = amd_uncore_l3_format_attr, 440 }; 441 442 static struct attribute_group amd_f17h_uncore_l3_format_group = { 443 .name = "format", 444 .attrs = amd_f17h_uncore_l3_format_attr, 445 .is_visible = amd_f17h_uncore_is_visible, 446 }; 447 448 static struct attribute_group amd_f19h_uncore_l3_format_group = { 449 .name = "format", 450 .attrs = amd_f19h_uncore_l3_format_attr, 451 .is_visible = amd_f19h_uncore_is_visible, 452 }; 453 454 static struct attribute_group amd_uncore_umc_format_group = { 455 .name = "format", 456 .attrs = amd_uncore_umc_format_attr, 457 }; 458 459 static const struct attribute_group *amd_uncore_df_attr_groups[] = { 460 &amd_uncore_attr_group, 461 &amd_uncore_df_format_group, 462 NULL, 463 }; 464 465 static const struct attribute_group *amd_uncore_l3_attr_groups[] = { 466 &amd_uncore_attr_group, 467 &amd_uncore_l3_format_group, 468 NULL, 469 }; 470 471 static const struct attribute_group *amd_uncore_l3_attr_update[] = { 472 &amd_f17h_uncore_l3_format_group, 473 &amd_f19h_uncore_l3_format_group, 474 NULL, 475 }; 476 477 static const struct attribute_group *amd_uncore_umc_attr_groups[] = { 478 &amd_uncore_attr_group, 479 &amd_uncore_umc_format_group, 480 NULL, 481 }; 482 483 static __always_inline 484 int amd_uncore_ctx_cid(struct amd_uncore *uncore, unsigned int cpu) 485 { 486 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu); 487 return info->split.cid; 488 } 489 490 static __always_inline 491 int amd_uncore_ctx_gid(struct amd_uncore *uncore, unsigned int cpu) 492 { 493 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu); 494 return info->split.gid; 495 } 496 497 static __always_inline 498 int amd_uncore_ctx_num_pmcs(struct amd_uncore *uncore, unsigned int cpu) 499 { 500 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu); 501 return info->split.num_pmcs; 502 } 503 504 static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu) 505 { 506 struct amd_uncore_pmu *pmu; 507 struct amd_uncore_ctx *ctx; 508 int i; 509 510 if (!uncore->init_done) 511 return; 512 513 for (i = 0; i < uncore->num_pmus; i++) { 514 pmu = &uncore->pmus[i]; 515 ctx = *per_cpu_ptr(pmu->ctx, cpu); 516 if (!ctx) 517 continue; 518 519 if (cpu == ctx->cpu) 520 cpumask_clear_cpu(cpu, &pmu->active_mask); 521 522 if (!--ctx->refcnt) { 523 kfree(ctx->events); 524 kfree(ctx); 525 } 526 527 *per_cpu_ptr(pmu->ctx, cpu) = NULL; 528 } 529 } 530 531 static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 532 { 533 struct amd_uncore_ctx *curr, *prev; 534 struct amd_uncore_pmu *pmu; 535 int node, cid, gid, i, j; 536 537 if (!uncore->init_done || !uncore->num_pmus) 538 return 0; 539 540 cid = amd_uncore_ctx_cid(uncore, cpu); 541 gid = amd_uncore_ctx_gid(uncore, cpu); 542 543 for (i = 0; i < uncore->num_pmus; i++) { 544 pmu = &uncore->pmus[i]; 545 *per_cpu_ptr(pmu->ctx, cpu) = NULL; 546 curr = NULL; 547 548 /* Check for group exclusivity */ 549 if (gid != pmu->group) 550 continue; 551 552 /* Find a sibling context */ 553 for_each_online_cpu(j) { 554 if (cpu == j) 555 continue; 556 557 prev = *per_cpu_ptr(pmu->ctx, j); 558 if (!prev) 559 continue; 560 561 if (cid == amd_uncore_ctx_cid(uncore, j)) { 562 curr = prev; 563 break; 564 } 565 } 566 567 /* Allocate context if sibling does not exist */ 568 if (!curr) { 569 node = cpu_to_node(cpu); 570 curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node); 571 if (!curr) 572 goto fail; 573 574 curr->cpu = cpu; 575 curr->events = kzalloc_node(sizeof(*curr->events) * 576 pmu->num_counters, 577 GFP_KERNEL, node); 578 if (!curr->events) { 579 kfree(curr); 580 goto fail; 581 } 582 583 amd_uncore_init_hrtimer(curr); 584 curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC; 585 586 cpumask_set_cpu(cpu, &pmu->active_mask); 587 } 588 589 curr->refcnt++; 590 *per_cpu_ptr(pmu->ctx, cpu) = curr; 591 } 592 593 return 0; 594 595 fail: 596 amd_uncore_ctx_free(uncore, cpu); 597 598 return -ENOMEM; 599 } 600 601 static void amd_uncore_ctx_move(struct amd_uncore *uncore, unsigned int cpu) 602 { 603 struct amd_uncore_ctx *curr, *next; 604 struct amd_uncore_pmu *pmu; 605 int i, j; 606 607 if (!uncore->init_done) 608 return; 609 610 for (i = 0; i < uncore->num_pmus; i++) { 611 pmu = &uncore->pmus[i]; 612 curr = *per_cpu_ptr(pmu->ctx, cpu); 613 if (!curr) 614 continue; 615 616 /* Migrate to a shared sibling if possible */ 617 for_each_online_cpu(j) { 618 next = *per_cpu_ptr(pmu->ctx, j); 619 if (!next || cpu == j) 620 continue; 621 622 if (curr == next) { 623 perf_pmu_migrate_context(&pmu->pmu, cpu, j); 624 cpumask_clear_cpu(cpu, &pmu->active_mask); 625 cpumask_set_cpu(j, &pmu->active_mask); 626 next->cpu = j; 627 break; 628 } 629 } 630 } 631 } 632 633 static int amd_uncore_cpu_starting(unsigned int cpu) 634 { 635 struct amd_uncore *uncore; 636 int i; 637 638 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 639 uncore = &uncores[i]; 640 uncore->scan(uncore, cpu); 641 } 642 643 return 0; 644 } 645 646 static int amd_uncore_cpu_online(unsigned int cpu) 647 { 648 struct amd_uncore *uncore; 649 int i; 650 651 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 652 uncore = &uncores[i]; 653 if (uncore->init(uncore, cpu)) 654 break; 655 } 656 657 return 0; 658 } 659 660 static int amd_uncore_cpu_down_prepare(unsigned int cpu) 661 { 662 struct amd_uncore *uncore; 663 int i; 664 665 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 666 uncore = &uncores[i]; 667 uncore->move(uncore, cpu); 668 } 669 670 return 0; 671 } 672 673 static int amd_uncore_cpu_dead(unsigned int cpu) 674 { 675 struct amd_uncore *uncore; 676 int i; 677 678 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 679 uncore = &uncores[i]; 680 uncore->free(uncore, cpu); 681 } 682 683 return 0; 684 } 685 686 static int amd_uncore_df_event_init(struct perf_event *event) 687 { 688 struct hw_perf_event *hwc = &event->hw; 689 int ret = amd_uncore_event_init(event); 690 691 hwc->config = event->attr.config & 692 (pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB : 693 AMD64_RAW_EVENT_MASK_NB); 694 695 return ret; 696 } 697 698 static int amd_uncore_df_add(struct perf_event *event, int flags) 699 { 700 int ret = amd_uncore_add(event, flags & ~PERF_EF_START); 701 struct hw_perf_event *hwc = &event->hw; 702 703 if (ret) 704 return ret; 705 706 /* 707 * The first four DF counters are accessible via RDPMC index 6 to 9 708 * followed by the L3 counters from index 10 to 15. For processors 709 * with more than four DF counters, the DF RDPMC assignments become 710 * discontiguous as the additional counters are accessible starting 711 * from index 16. 712 */ 713 if (hwc->idx >= NUM_COUNTERS_NB) 714 hwc->event_base_rdpmc += NUM_COUNTERS_L3; 715 716 /* Delayed start after rdpmc base update */ 717 if (flags & PERF_EF_START) 718 amd_uncore_start(event, PERF_EF_RELOAD); 719 720 return 0; 721 } 722 723 static 724 void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) 725 { 726 union cpuid_0x80000022_ebx ebx; 727 union amd_uncore_info info; 728 729 if (!boot_cpu_has(X86_FEATURE_PERFCTR_NB)) 730 return; 731 732 info.split.aux_data = 0; 733 info.split.num_pmcs = NUM_COUNTERS_NB; 734 info.split.gid = 0; 735 info.split.cid = topology_amd_node_id(cpu); 736 737 if (pmu_version >= 2) { 738 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); 739 info.split.num_pmcs = ebx.split.num_df_pmc; 740 } 741 742 *per_cpu_ptr(uncore->info, cpu) = info; 743 } 744 745 static 746 int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 747 { 748 struct attribute **df_attr = amd_uncore_df_format_attr; 749 struct amd_uncore_pmu *pmu; 750 int num_counters; 751 752 /* Run just once */ 753 if (uncore->init_done) 754 return amd_uncore_ctx_init(uncore, cpu); 755 756 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu); 757 if (!num_counters) 758 goto done; 759 760 /* No grouping, single instance for a system */ 761 uncore->pmus = kzalloc_obj(*uncore->pmus); 762 if (!uncore->pmus) 763 goto done; 764 765 /* 766 * For Family 17h and above, the Northbridge counters are repurposed 767 * as Data Fabric counters. The PMUs are exported based on family as 768 * either NB or DF. 769 */ 770 pmu = &uncore->pmus[0]; 771 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb", 772 sizeof(pmu->name)); 773 pmu->num_counters = num_counters; 774 pmu->msr_base = MSR_F15H_NB_PERF_CTL; 775 pmu->rdpmc_base = RDPMC_BASE_NB; 776 pmu->group = amd_uncore_ctx_gid(uncore, cpu); 777 778 if (pmu_version >= 2) { 779 *df_attr++ = &format_attr_event14v2.attr; 780 *df_attr++ = &format_attr_umask12.attr; 781 } else if (boot_cpu_data.x86 >= 0x17) { 782 *df_attr = &format_attr_event14.attr; 783 } 784 785 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *); 786 if (!pmu->ctx) 787 goto done; 788 789 pmu->pmu = (struct pmu) { 790 .task_ctx_nr = perf_invalid_context, 791 .attr_groups = amd_uncore_df_attr_groups, 792 .name = pmu->name, 793 .event_init = amd_uncore_df_event_init, 794 .add = amd_uncore_df_add, 795 .del = amd_uncore_del, 796 .start = amd_uncore_start, 797 .stop = amd_uncore_stop, 798 .read = amd_uncore_read, 799 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 800 .module = THIS_MODULE, 801 }; 802 803 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) { 804 free_percpu(pmu->ctx); 805 pmu->ctx = NULL; 806 goto done; 807 } 808 809 pr_info("%d %s%s counters detected\n", pmu->num_counters, 810 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "", 811 pmu->pmu.name); 812 813 uncore->num_pmus = 1; 814 815 done: 816 uncore->init_done = true; 817 818 return amd_uncore_ctx_init(uncore, cpu); 819 } 820 821 static int amd_uncore_l3_event_init(struct perf_event *event) 822 { 823 int ret = amd_uncore_event_init(event); 824 struct hw_perf_event *hwc = &event->hw; 825 u64 config = event->attr.config; 826 u64 mask; 827 828 hwc->config = config & AMD64_RAW_EVENT_MASK_NB; 829 830 /* 831 * SliceMask and ThreadMask need to be set for certain L3 events. 832 * For other events, the two fields do not affect the count. 833 */ 834 if (ret || boot_cpu_data.x86 < 0x17) 835 return ret; 836 837 mask = config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK | 838 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES | 839 AMD64_L3_COREID_MASK); 840 841 if (boot_cpu_data.x86 <= 0x18) 842 mask = ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) | 843 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK); 844 845 /* 846 * If the user doesn't specify a ThreadMask, they're not trying to 847 * count core 0, so we enable all cores & threads. 848 * We'll also assume that they want to count slice 0 if they specify 849 * a ThreadMask and leave SliceId and EnAllSlices unpopulated. 850 */ 851 else if (!(config & AMD64_L3_F19H_THREAD_MASK)) 852 mask = AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES | 853 AMD64_L3_EN_ALL_CORES; 854 855 hwc->config |= mask; 856 857 return 0; 858 } 859 860 static 861 void amd_uncore_l3_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) 862 { 863 union amd_uncore_info info; 864 865 if (!boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) 866 return; 867 868 info.split.aux_data = 0; 869 info.split.num_pmcs = NUM_COUNTERS_L2; 870 info.split.gid = 0; 871 info.split.cid = per_cpu_llc_id(cpu); 872 873 if (boot_cpu_data.x86 >= 0x17) 874 info.split.num_pmcs = NUM_COUNTERS_L3; 875 876 *per_cpu_ptr(uncore->info, cpu) = info; 877 } 878 879 static 880 int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 881 { 882 struct attribute **l3_attr = amd_uncore_l3_format_attr; 883 struct amd_uncore_pmu *pmu; 884 int num_counters; 885 886 /* Run just once */ 887 if (uncore->init_done) 888 return amd_uncore_ctx_init(uncore, cpu); 889 890 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu); 891 if (!num_counters) 892 goto done; 893 894 /* No grouping, single instance for a system */ 895 uncore->pmus = kzalloc_obj(*uncore->pmus); 896 if (!uncore->pmus) 897 goto done; 898 899 /* 900 * For Family 17h and above, L3 cache counters are available instead 901 * of L2 cache counters. The PMUs are exported based on family as 902 * either L2 or L3. 903 */ 904 pmu = &uncore->pmus[0]; 905 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2", 906 sizeof(pmu->name)); 907 pmu->num_counters = num_counters; 908 pmu->msr_base = MSR_F16H_L2I_PERF_CTL; 909 pmu->rdpmc_base = RDPMC_BASE_LLC; 910 pmu->group = amd_uncore_ctx_gid(uncore, cpu); 911 912 if (boot_cpu_data.x86 >= 0x17) { 913 *l3_attr++ = &format_attr_event8.attr; 914 *l3_attr++ = &format_attr_umask8.attr; 915 *l3_attr++ = boot_cpu_data.x86 >= 0x19 ? 916 &format_attr_threadmask2.attr : 917 &format_attr_threadmask8.attr; 918 } 919 920 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *); 921 if (!pmu->ctx) 922 goto done; 923 924 pmu->pmu = (struct pmu) { 925 .task_ctx_nr = perf_invalid_context, 926 .attr_groups = amd_uncore_l3_attr_groups, 927 .attr_update = amd_uncore_l3_attr_update, 928 .name = pmu->name, 929 .event_init = amd_uncore_l3_event_init, 930 .add = amd_uncore_add, 931 .del = amd_uncore_del, 932 .start = amd_uncore_start, 933 .stop = amd_uncore_stop, 934 .read = amd_uncore_read, 935 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 936 .module = THIS_MODULE, 937 }; 938 939 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) { 940 free_percpu(pmu->ctx); 941 pmu->ctx = NULL; 942 goto done; 943 } 944 945 pr_info("%d %s%s counters detected\n", pmu->num_counters, 946 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "", 947 pmu->pmu.name); 948 949 uncore->num_pmus = 1; 950 951 done: 952 uncore->init_done = true; 953 954 return amd_uncore_ctx_init(uncore, cpu); 955 } 956 957 static int amd_uncore_umc_event_init(struct perf_event *event) 958 { 959 struct hw_perf_event *hwc = &event->hw; 960 int ret = amd_uncore_event_init(event); 961 962 if (ret) 963 return ret; 964 965 hwc->config = event->attr.config & AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC; 966 967 return 0; 968 } 969 970 static void amd_uncore_umc_start(struct perf_event *event, int flags) 971 { 972 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); 973 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); 974 struct hw_perf_event *hwc = &event->hw; 975 976 if (!ctx->nr_active++) 977 amd_uncore_start_hrtimer(ctx); 978 979 if (flags & PERF_EF_RELOAD) 980 wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count)); 981 982 hwc->state = 0; 983 __set_bit(hwc->idx, ctx->active_mask); 984 wrmsrq(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC)); 985 perf_event_update_userpage(event); 986 } 987 988 static void amd_uncore_umc_read(struct perf_event *event) 989 { 990 struct hw_perf_event *hwc = &event->hw; 991 u64 prev, new, shift; 992 s64 delta; 993 994 shift = COUNTER_SHIFT + 1; 995 prev = local64_read(&hwc->prev_count); 996 997 /* 998 * UMC counters do not have RDPMC assignments. Read counts directly 999 * from the corresponding PERF_CTR. 1000 */ 1001 rdmsrq(hwc->event_base, new); 1002 1003 /* 1004 * Unlike the other uncore counters, UMC counters saturate and set the 1005 * Overflow bit (bit 48) on overflow. Since they do not roll over, 1006 * proactively reset the corresponding PERF_CTR when bit 47 is set so 1007 * that the counter never gets a chance to saturate. 1008 */ 1009 if (new & BIT_ULL(63 - COUNTER_SHIFT)) { 1010 wrmsrq(hwc->event_base, 0); 1011 local64_set(&hwc->prev_count, 0); 1012 } else { 1013 local64_set(&hwc->prev_count, new); 1014 } 1015 1016 delta = (new << shift) - (prev << shift); 1017 delta >>= shift; 1018 local64_add(delta, &event->count); 1019 } 1020 1021 static 1022 void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) 1023 { 1024 union cpuid_0x80000022_ebx ebx; 1025 union amd_uncore_info info; 1026 unsigned int eax, ecx, edx; 1027 1028 if (pmu_version < 2) 1029 return; 1030 1031 cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx); 1032 info.split.aux_data = ecx; /* stash active mask */ 1033 info.split.num_pmcs = ebx.split.num_umc_pmc; 1034 info.split.gid = topology_amd_node_id(cpu); 1035 info.split.cid = topology_amd_node_id(cpu); 1036 *per_cpu_ptr(uncore->info, cpu) = info; 1037 } 1038 1039 static 1040 int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 1041 { 1042 DECLARE_BITMAP(gmask, UNCORE_GROUP_MAX) = { 0 }; 1043 u8 group_num_pmus[UNCORE_GROUP_MAX] = { 0 }; 1044 u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 }; 1045 union amd_uncore_info info; 1046 struct amd_uncore_pmu *pmu; 1047 int gid, i; 1048 u16 index = 0; 1049 1050 if (pmu_version < 2) 1051 return 0; 1052 1053 /* Run just once */ 1054 if (uncore->init_done) 1055 return amd_uncore_ctx_init(uncore, cpu); 1056 1057 /* Find unique groups */ 1058 for_each_online_cpu(i) { 1059 info = *per_cpu_ptr(uncore->info, i); 1060 gid = info.split.gid; 1061 if (test_bit(gid, gmask)) 1062 continue; 1063 1064 __set_bit(gid, gmask); 1065 group_num_pmus[gid] = hweight32(info.split.aux_data); 1066 group_num_pmcs[gid] = info.split.num_pmcs; 1067 uncore->num_pmus += group_num_pmus[gid]; 1068 } 1069 1070 uncore->pmus = kzalloc(sizeof(*uncore->pmus) * uncore->num_pmus, 1071 GFP_KERNEL); 1072 if (!uncore->pmus) { 1073 uncore->num_pmus = 0; 1074 goto done; 1075 } 1076 1077 for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) { 1078 for (i = 0; i < group_num_pmus[gid]; i++) { 1079 pmu = &uncore->pmus[index]; 1080 snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%hu", index); 1081 pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid]; 1082 pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2; 1083 pmu->rdpmc_base = -1; 1084 pmu->group = gid; 1085 1086 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *); 1087 if (!pmu->ctx) 1088 goto done; 1089 1090 pmu->pmu = (struct pmu) { 1091 .task_ctx_nr = perf_invalid_context, 1092 .attr_groups = amd_uncore_umc_attr_groups, 1093 .name = pmu->name, 1094 .event_init = amd_uncore_umc_event_init, 1095 .add = amd_uncore_add, 1096 .del = amd_uncore_del, 1097 .start = amd_uncore_umc_start, 1098 .stop = amd_uncore_stop, 1099 .read = amd_uncore_umc_read, 1100 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 1101 .module = THIS_MODULE, 1102 }; 1103 1104 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) { 1105 free_percpu(pmu->ctx); 1106 pmu->ctx = NULL; 1107 goto done; 1108 } 1109 1110 pr_info("%d %s counters detected\n", pmu->num_counters, 1111 pmu->pmu.name); 1112 1113 index++; 1114 } 1115 } 1116 1117 done: 1118 uncore->num_pmus = index; 1119 uncore->init_done = true; 1120 1121 return amd_uncore_ctx_init(uncore, cpu); 1122 } 1123 1124 static struct amd_uncore uncores[UNCORE_TYPE_MAX] = { 1125 /* UNCORE_TYPE_DF */ 1126 { 1127 .scan = amd_uncore_df_ctx_scan, 1128 .init = amd_uncore_df_ctx_init, 1129 .move = amd_uncore_ctx_move, 1130 .free = amd_uncore_ctx_free, 1131 }, 1132 /* UNCORE_TYPE_L3 */ 1133 { 1134 .scan = amd_uncore_l3_ctx_scan, 1135 .init = amd_uncore_l3_ctx_init, 1136 .move = amd_uncore_ctx_move, 1137 .free = amd_uncore_ctx_free, 1138 }, 1139 /* UNCORE_TYPE_UMC */ 1140 { 1141 .scan = amd_uncore_umc_ctx_scan, 1142 .init = amd_uncore_umc_ctx_init, 1143 .move = amd_uncore_ctx_move, 1144 .free = amd_uncore_ctx_free, 1145 }, 1146 }; 1147 1148 static int __init amd_uncore_init(void) 1149 { 1150 struct amd_uncore *uncore; 1151 int ret = -ENODEV; 1152 int i; 1153 1154 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 1155 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 1156 return -ENODEV; 1157 1158 if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) 1159 return -ENODEV; 1160 1161 if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) 1162 pmu_version = 2; 1163 1164 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 1165 uncore = &uncores[i]; 1166 1167 BUG_ON(!uncore->scan); 1168 BUG_ON(!uncore->init); 1169 BUG_ON(!uncore->move); 1170 BUG_ON(!uncore->free); 1171 1172 uncore->info = alloc_percpu(union amd_uncore_info); 1173 if (!uncore->info) { 1174 ret = -ENOMEM; 1175 goto fail; 1176 } 1177 }; 1178 1179 /* 1180 * Install callbacks. Core will call them for each online cpu. 1181 */ 1182 ret = cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP, 1183 "perf/x86/amd/uncore:prepare", 1184 NULL, amd_uncore_cpu_dead); 1185 if (ret) 1186 goto fail; 1187 1188 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING, 1189 "perf/x86/amd/uncore:starting", 1190 amd_uncore_cpu_starting, NULL); 1191 if (ret) 1192 goto fail_prep; 1193 1194 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE, 1195 "perf/x86/amd/uncore:online", 1196 amd_uncore_cpu_online, 1197 amd_uncore_cpu_down_prepare); 1198 if (ret) 1199 goto fail_start; 1200 1201 return 0; 1202 1203 fail_start: 1204 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 1205 fail_prep: 1206 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 1207 fail: 1208 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 1209 uncore = &uncores[i]; 1210 if (uncore->info) { 1211 free_percpu(uncore->info); 1212 uncore->info = NULL; 1213 } 1214 } 1215 1216 return ret; 1217 } 1218 1219 static void __exit amd_uncore_exit(void) 1220 { 1221 struct amd_uncore *uncore; 1222 struct amd_uncore_pmu *pmu; 1223 int i, j; 1224 1225 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE); 1226 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 1227 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 1228 1229 for (i = 0; i < UNCORE_TYPE_MAX; i++) { 1230 uncore = &uncores[i]; 1231 if (!uncore->info) 1232 continue; 1233 1234 free_percpu(uncore->info); 1235 uncore->info = NULL; 1236 1237 for (j = 0; j < uncore->num_pmus; j++) { 1238 pmu = &uncore->pmus[j]; 1239 if (!pmu->ctx) 1240 continue; 1241 1242 perf_pmu_unregister(&pmu->pmu); 1243 free_percpu(pmu->ctx); 1244 pmu->ctx = NULL; 1245 } 1246 1247 kfree(uncore->pmus); 1248 uncore->pmus = NULL; 1249 } 1250 } 1251 1252 module_init(amd_uncore_init); 1253 module_exit(amd_uncore_exit); 1254 1255 MODULE_DESCRIPTION("AMD Uncore Driver"); 1256 MODULE_LICENSE("GPL v2"); 1257