1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/module.h> 3 4 #include <asm/cpu_device_id.h> 5 #include <asm/intel-family.h> 6 #include "uncore.h" 7 #include "uncore_discovery.h" 8 9 static bool uncore_no_discover; 10 module_param(uncore_no_discover, bool, 0); 11 MODULE_PARM_DESC(uncore_no_discover, "Don't enable the Intel uncore PerfMon discovery mechanism " 12 "(default: enable the discovery mechanism)."); 13 struct intel_uncore_type *empty_uncore[] = { NULL, }; 14 struct intel_uncore_type **uncore_msr_uncores = empty_uncore; 15 struct intel_uncore_type **uncore_pci_uncores = empty_uncore; 16 struct intel_uncore_type **uncore_mmio_uncores = empty_uncore; 17 18 static bool pcidrv_registered; 19 struct pci_driver *uncore_pci_driver; 20 /* The PCI driver for the device which the uncore doesn't own. */ 21 struct pci_driver *uncore_pci_sub_driver; 22 /* pci bus to socket mapping */ 23 DEFINE_RAW_SPINLOCK(pci2phy_map_lock); 24 struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head); 25 struct pci_extra_dev *uncore_extra_pci_dev; 26 int __uncore_max_dies; 27 28 /* mask of cpus that collect uncore events */ 29 static cpumask_t uncore_cpu_mask; 30 31 /* constraint for the fixed counter */ 32 static struct event_constraint uncore_constraint_fixed = 33 EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL); 34 struct event_constraint uncore_constraint_empty = 35 EVENT_CONSTRAINT(0, 0, 0); 36 37 MODULE_LICENSE("GPL"); 38 39 int uncore_pcibus_to_dieid(struct pci_bus *bus) 40 { 41 struct pci2phy_map *map; 42 int die_id = -1; 43 44 raw_spin_lock(&pci2phy_map_lock); 45 list_for_each_entry(map, &pci2phy_map_head, list) { 46 if (map->segment == pci_domain_nr(bus)) { 47 die_id = map->pbus_to_dieid[bus->number]; 48 break; 49 } 50 } 51 raw_spin_unlock(&pci2phy_map_lock); 52 53 return die_id; 54 } 55 56 int uncore_die_to_segment(int die) 57 { 58 struct pci_bus *bus = NULL; 59 60 /* Find first pci bus which attributes to specified die. */ 61 while ((bus = pci_find_next_bus(bus)) && 62 (die != uncore_pcibus_to_dieid(bus))) 63 ; 64 65 return bus ? pci_domain_nr(bus) : -EINVAL; 66 } 67 68 static void uncore_free_pcibus_map(void) 69 { 70 struct pci2phy_map *map, *tmp; 71 72 list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) { 73 list_del(&map->list); 74 kfree(map); 75 } 76 } 77 78 struct pci2phy_map *__find_pci2phy_map(int segment) 79 { 80 struct pci2phy_map *map, *alloc = NULL; 81 int i; 82 83 lockdep_assert_held(&pci2phy_map_lock); 84 85 lookup: 86 list_for_each_entry(map, &pci2phy_map_head, list) { 87 if (map->segment == segment) 88 goto end; 89 } 90 91 if (!alloc) { 92 raw_spin_unlock(&pci2phy_map_lock); 93 alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL); 94 raw_spin_lock(&pci2phy_map_lock); 95 96 if (!alloc) 97 return NULL; 98 99 goto lookup; 100 } 101 102 map = alloc; 103 alloc = NULL; 104 map->segment = segment; 105 for (i = 0; i < 256; i++) 106 map->pbus_to_dieid[i] = -1; 107 list_add_tail(&map->list, &pci2phy_map_head); 108 109 end: 110 kfree(alloc); 111 return map; 112 } 113 114 ssize_t uncore_event_show(struct device *dev, 115 struct device_attribute *attr, char *buf) 116 { 117 struct uncore_event_desc *event = 118 container_of(attr, struct uncore_event_desc, attr); 119 return sprintf(buf, "%s", event->config); 120 } 121 122 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu) 123 { 124 unsigned int dieid = topology_logical_die_id(cpu); 125 126 /* 127 * The unsigned check also catches the '-1' return value for non 128 * existent mappings in the topology map. 129 */ 130 return dieid < uncore_max_dies() ? pmu->boxes[dieid] : NULL; 131 } 132 133 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event) 134 { 135 u64 count; 136 137 rdmsrl(event->hw.event_base, count); 138 139 return count; 140 } 141 142 void uncore_mmio_exit_box(struct intel_uncore_box *box) 143 { 144 if (box->io_addr) 145 iounmap(box->io_addr); 146 } 147 148 u64 uncore_mmio_read_counter(struct intel_uncore_box *box, 149 struct perf_event *event) 150 { 151 if (!box->io_addr) 152 return 0; 153 154 if (!uncore_mmio_is_valid_offset(box, event->hw.event_base)) 155 return 0; 156 157 return readq(box->io_addr + event->hw.event_base); 158 } 159 160 /* 161 * generic get constraint function for shared match/mask registers. 162 */ 163 struct event_constraint * 164 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 165 { 166 struct intel_uncore_extra_reg *er; 167 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 168 struct hw_perf_event_extra *reg2 = &event->hw.branch_reg; 169 unsigned long flags; 170 bool ok = false; 171 172 /* 173 * reg->alloc can be set due to existing state, so for fake box we 174 * need to ignore this, otherwise we might fail to allocate proper 175 * fake state for this extra reg constraint. 176 */ 177 if (reg1->idx == EXTRA_REG_NONE || 178 (!uncore_box_is_fake(box) && reg1->alloc)) 179 return NULL; 180 181 er = &box->shared_regs[reg1->idx]; 182 raw_spin_lock_irqsave(&er->lock, flags); 183 if (!atomic_read(&er->ref) || 184 (er->config1 == reg1->config && er->config2 == reg2->config)) { 185 atomic_inc(&er->ref); 186 er->config1 = reg1->config; 187 er->config2 = reg2->config; 188 ok = true; 189 } 190 raw_spin_unlock_irqrestore(&er->lock, flags); 191 192 if (ok) { 193 if (!uncore_box_is_fake(box)) 194 reg1->alloc = 1; 195 return NULL; 196 } 197 198 return &uncore_constraint_empty; 199 } 200 201 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event) 202 { 203 struct intel_uncore_extra_reg *er; 204 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 205 206 /* 207 * Only put constraint if extra reg was actually allocated. Also 208 * takes care of event which do not use an extra shared reg. 209 * 210 * Also, if this is a fake box we shouldn't touch any event state 211 * (reg->alloc) and we don't care about leaving inconsistent box 212 * state either since it will be thrown out. 213 */ 214 if (uncore_box_is_fake(box) || !reg1->alloc) 215 return; 216 217 er = &box->shared_regs[reg1->idx]; 218 atomic_dec(&er->ref); 219 reg1->alloc = 0; 220 } 221 222 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx) 223 { 224 struct intel_uncore_extra_reg *er; 225 unsigned long flags; 226 u64 config; 227 228 er = &box->shared_regs[idx]; 229 230 raw_spin_lock_irqsave(&er->lock, flags); 231 config = er->config; 232 raw_spin_unlock_irqrestore(&er->lock, flags); 233 234 return config; 235 } 236 237 static void uncore_assign_hw_event(struct intel_uncore_box *box, 238 struct perf_event *event, int idx) 239 { 240 struct hw_perf_event *hwc = &event->hw; 241 242 hwc->idx = idx; 243 hwc->last_tag = ++box->tags[idx]; 244 245 if (uncore_pmc_fixed(hwc->idx)) { 246 hwc->event_base = uncore_fixed_ctr(box); 247 hwc->config_base = uncore_fixed_ctl(box); 248 return; 249 } 250 251 hwc->config_base = uncore_event_ctl(box, hwc->idx); 252 hwc->event_base = uncore_perf_ctr(box, hwc->idx); 253 } 254 255 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event) 256 { 257 u64 prev_count, new_count, delta; 258 int shift; 259 260 if (uncore_pmc_freerunning(event->hw.idx)) 261 shift = 64 - uncore_freerunning_bits(box, event); 262 else if (uncore_pmc_fixed(event->hw.idx)) 263 shift = 64 - uncore_fixed_ctr_bits(box); 264 else 265 shift = 64 - uncore_perf_ctr_bits(box); 266 267 /* the hrtimer might modify the previous event value */ 268 again: 269 prev_count = local64_read(&event->hw.prev_count); 270 new_count = uncore_read_counter(box, event); 271 if (local64_xchg(&event->hw.prev_count, new_count) != prev_count) 272 goto again; 273 274 delta = (new_count << shift) - (prev_count << shift); 275 delta >>= shift; 276 277 local64_add(delta, &event->count); 278 } 279 280 /* 281 * The overflow interrupt is unavailable for SandyBridge-EP, is broken 282 * for SandyBridge. So we use hrtimer to periodically poll the counter 283 * to avoid overflow. 284 */ 285 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) 286 { 287 struct intel_uncore_box *box; 288 struct perf_event *event; 289 unsigned long flags; 290 int bit; 291 292 box = container_of(hrtimer, struct intel_uncore_box, hrtimer); 293 if (!box->n_active || box->cpu != smp_processor_id()) 294 return HRTIMER_NORESTART; 295 /* 296 * disable local interrupt to prevent uncore_pmu_event_start/stop 297 * to interrupt the update process 298 */ 299 local_irq_save(flags); 300 301 /* 302 * handle boxes with an active event list as opposed to active 303 * counters 304 */ 305 list_for_each_entry(event, &box->active_list, active_entry) { 306 uncore_perf_event_update(box, event); 307 } 308 309 for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX) 310 uncore_perf_event_update(box, box->events[bit]); 311 312 local_irq_restore(flags); 313 314 hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration)); 315 return HRTIMER_RESTART; 316 } 317 318 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box) 319 { 320 hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration), 321 HRTIMER_MODE_REL_PINNED); 322 } 323 324 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box) 325 { 326 hrtimer_cancel(&box->hrtimer); 327 } 328 329 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box) 330 { 331 hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 332 box->hrtimer.function = uncore_pmu_hrtimer; 333 } 334 335 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, 336 int node) 337 { 338 int i, size, numshared = type->num_shared_regs ; 339 struct intel_uncore_box *box; 340 341 size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg); 342 343 box = kzalloc_node(size, GFP_KERNEL, node); 344 if (!box) 345 return NULL; 346 347 for (i = 0; i < numshared; i++) 348 raw_spin_lock_init(&box->shared_regs[i].lock); 349 350 uncore_pmu_init_hrtimer(box); 351 box->cpu = -1; 352 box->dieid = -1; 353 354 /* set default hrtimer timeout */ 355 box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL; 356 357 INIT_LIST_HEAD(&box->active_list); 358 359 return box; 360 } 361 362 /* 363 * Using uncore_pmu_event_init pmu event_init callback 364 * as a detection point for uncore events. 365 */ 366 static int uncore_pmu_event_init(struct perf_event *event); 367 368 static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event) 369 { 370 return &box->pmu->pmu == event->pmu; 371 } 372 373 static int 374 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader, 375 bool dogrp) 376 { 377 struct perf_event *event; 378 int n, max_count; 379 380 max_count = box->pmu->type->num_counters; 381 if (box->pmu->type->fixed_ctl) 382 max_count++; 383 384 if (box->n_events >= max_count) 385 return -EINVAL; 386 387 n = box->n_events; 388 389 if (is_box_event(box, leader)) { 390 box->event_list[n] = leader; 391 n++; 392 } 393 394 if (!dogrp) 395 return n; 396 397 for_each_sibling_event(event, leader) { 398 if (!is_box_event(box, event) || 399 event->state <= PERF_EVENT_STATE_OFF) 400 continue; 401 402 if (n >= max_count) 403 return -EINVAL; 404 405 box->event_list[n] = event; 406 n++; 407 } 408 return n; 409 } 410 411 static struct event_constraint * 412 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event) 413 { 414 struct intel_uncore_type *type = box->pmu->type; 415 struct event_constraint *c; 416 417 if (type->ops->get_constraint) { 418 c = type->ops->get_constraint(box, event); 419 if (c) 420 return c; 421 } 422 423 if (event->attr.config == UNCORE_FIXED_EVENT) 424 return &uncore_constraint_fixed; 425 426 if (type->constraints) { 427 for_each_event_constraint(c, type->constraints) { 428 if ((event->hw.config & c->cmask) == c->code) 429 return c; 430 } 431 } 432 433 return &type->unconstrainted; 434 } 435 436 static void uncore_put_event_constraint(struct intel_uncore_box *box, 437 struct perf_event *event) 438 { 439 if (box->pmu->type->ops->put_constraint) 440 box->pmu->type->ops->put_constraint(box, event); 441 } 442 443 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n) 444 { 445 unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)]; 446 struct event_constraint *c; 447 int i, wmin, wmax, ret = 0; 448 struct hw_perf_event *hwc; 449 450 bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX); 451 452 for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) { 453 c = uncore_get_event_constraint(box, box->event_list[i]); 454 box->event_constraint[i] = c; 455 wmin = min(wmin, c->weight); 456 wmax = max(wmax, c->weight); 457 } 458 459 /* fastpath, try to reuse previous register */ 460 for (i = 0; i < n; i++) { 461 hwc = &box->event_list[i]->hw; 462 c = box->event_constraint[i]; 463 464 /* never assigned */ 465 if (hwc->idx == -1) 466 break; 467 468 /* constraint still honored */ 469 if (!test_bit(hwc->idx, c->idxmsk)) 470 break; 471 472 /* not already used */ 473 if (test_bit(hwc->idx, used_mask)) 474 break; 475 476 __set_bit(hwc->idx, used_mask); 477 if (assign) 478 assign[i] = hwc->idx; 479 } 480 /* slow path */ 481 if (i != n) 482 ret = perf_assign_events(box->event_constraint, n, 483 wmin, wmax, n, assign); 484 485 if (!assign || ret) { 486 for (i = 0; i < n; i++) 487 uncore_put_event_constraint(box, box->event_list[i]); 488 } 489 return ret ? -EINVAL : 0; 490 } 491 492 void uncore_pmu_event_start(struct perf_event *event, int flags) 493 { 494 struct intel_uncore_box *box = uncore_event_to_box(event); 495 int idx = event->hw.idx; 496 497 if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX)) 498 return; 499 500 /* 501 * Free running counter is read-only and always active. 502 * Use the current counter value as start point. 503 * There is no overflow interrupt for free running counter. 504 * Use hrtimer to periodically poll the counter to avoid overflow. 505 */ 506 if (uncore_pmc_freerunning(event->hw.idx)) { 507 list_add_tail(&event->active_entry, &box->active_list); 508 local64_set(&event->hw.prev_count, 509 uncore_read_counter(box, event)); 510 if (box->n_active++ == 0) 511 uncore_pmu_start_hrtimer(box); 512 return; 513 } 514 515 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 516 return; 517 518 event->hw.state = 0; 519 box->events[idx] = event; 520 box->n_active++; 521 __set_bit(idx, box->active_mask); 522 523 local64_set(&event->hw.prev_count, uncore_read_counter(box, event)); 524 uncore_enable_event(box, event); 525 526 if (box->n_active == 1) 527 uncore_pmu_start_hrtimer(box); 528 } 529 530 void uncore_pmu_event_stop(struct perf_event *event, int flags) 531 { 532 struct intel_uncore_box *box = uncore_event_to_box(event); 533 struct hw_perf_event *hwc = &event->hw; 534 535 /* Cannot disable free running counter which is read-only */ 536 if (uncore_pmc_freerunning(hwc->idx)) { 537 list_del(&event->active_entry); 538 if (--box->n_active == 0) 539 uncore_pmu_cancel_hrtimer(box); 540 uncore_perf_event_update(box, event); 541 return; 542 } 543 544 if (__test_and_clear_bit(hwc->idx, box->active_mask)) { 545 uncore_disable_event(box, event); 546 box->n_active--; 547 box->events[hwc->idx] = NULL; 548 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 549 hwc->state |= PERF_HES_STOPPED; 550 551 if (box->n_active == 0) 552 uncore_pmu_cancel_hrtimer(box); 553 } 554 555 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 556 /* 557 * Drain the remaining delta count out of a event 558 * that we are disabling: 559 */ 560 uncore_perf_event_update(box, event); 561 hwc->state |= PERF_HES_UPTODATE; 562 } 563 } 564 565 int uncore_pmu_event_add(struct perf_event *event, int flags) 566 { 567 struct intel_uncore_box *box = uncore_event_to_box(event); 568 struct hw_perf_event *hwc = &event->hw; 569 int assign[UNCORE_PMC_IDX_MAX]; 570 int i, n, ret; 571 572 if (!box) 573 return -ENODEV; 574 575 /* 576 * The free funning counter is assigned in event_init(). 577 * The free running counter event and free running counter 578 * are 1:1 mapped. It doesn't need to be tracked in event_list. 579 */ 580 if (uncore_pmc_freerunning(hwc->idx)) { 581 if (flags & PERF_EF_START) 582 uncore_pmu_event_start(event, 0); 583 return 0; 584 } 585 586 ret = n = uncore_collect_events(box, event, false); 587 if (ret < 0) 588 return ret; 589 590 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 591 if (!(flags & PERF_EF_START)) 592 hwc->state |= PERF_HES_ARCH; 593 594 ret = uncore_assign_events(box, assign, n); 595 if (ret) 596 return ret; 597 598 /* save events moving to new counters */ 599 for (i = 0; i < box->n_events; i++) { 600 event = box->event_list[i]; 601 hwc = &event->hw; 602 603 if (hwc->idx == assign[i] && 604 hwc->last_tag == box->tags[assign[i]]) 605 continue; 606 /* 607 * Ensure we don't accidentally enable a stopped 608 * counter simply because we rescheduled. 609 */ 610 if (hwc->state & PERF_HES_STOPPED) 611 hwc->state |= PERF_HES_ARCH; 612 613 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 614 } 615 616 /* reprogram moved events into new counters */ 617 for (i = 0; i < n; i++) { 618 event = box->event_list[i]; 619 hwc = &event->hw; 620 621 if (hwc->idx != assign[i] || 622 hwc->last_tag != box->tags[assign[i]]) 623 uncore_assign_hw_event(box, event, assign[i]); 624 else if (i < box->n_events) 625 continue; 626 627 if (hwc->state & PERF_HES_ARCH) 628 continue; 629 630 uncore_pmu_event_start(event, 0); 631 } 632 box->n_events = n; 633 634 return 0; 635 } 636 637 void uncore_pmu_event_del(struct perf_event *event, int flags) 638 { 639 struct intel_uncore_box *box = uncore_event_to_box(event); 640 int i; 641 642 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 643 644 /* 645 * The event for free running counter is not tracked by event_list. 646 * It doesn't need to force event->hw.idx = -1 to reassign the counter. 647 * Because the event and the free running counter are 1:1 mapped. 648 */ 649 if (uncore_pmc_freerunning(event->hw.idx)) 650 return; 651 652 for (i = 0; i < box->n_events; i++) { 653 if (event == box->event_list[i]) { 654 uncore_put_event_constraint(box, event); 655 656 for (++i; i < box->n_events; i++) 657 box->event_list[i - 1] = box->event_list[i]; 658 659 --box->n_events; 660 break; 661 } 662 } 663 664 event->hw.idx = -1; 665 event->hw.last_tag = ~0ULL; 666 } 667 668 void uncore_pmu_event_read(struct perf_event *event) 669 { 670 struct intel_uncore_box *box = uncore_event_to_box(event); 671 uncore_perf_event_update(box, event); 672 } 673 674 /* 675 * validation ensures the group can be loaded onto the 676 * PMU if it was the only group available. 677 */ 678 static int uncore_validate_group(struct intel_uncore_pmu *pmu, 679 struct perf_event *event) 680 { 681 struct perf_event *leader = event->group_leader; 682 struct intel_uncore_box *fake_box; 683 int ret = -EINVAL, n; 684 685 /* The free running counter is always active. */ 686 if (uncore_pmc_freerunning(event->hw.idx)) 687 return 0; 688 689 fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE); 690 if (!fake_box) 691 return -ENOMEM; 692 693 fake_box->pmu = pmu; 694 /* 695 * the event is not yet connected with its 696 * siblings therefore we must first collect 697 * existing siblings, then add the new event 698 * before we can simulate the scheduling 699 */ 700 n = uncore_collect_events(fake_box, leader, true); 701 if (n < 0) 702 goto out; 703 704 fake_box->n_events = n; 705 n = uncore_collect_events(fake_box, event, false); 706 if (n < 0) 707 goto out; 708 709 fake_box->n_events = n; 710 711 ret = uncore_assign_events(fake_box, NULL, n); 712 out: 713 kfree(fake_box); 714 return ret; 715 } 716 717 static int uncore_pmu_event_init(struct perf_event *event) 718 { 719 struct intel_uncore_pmu *pmu; 720 struct intel_uncore_box *box; 721 struct hw_perf_event *hwc = &event->hw; 722 int ret; 723 724 if (event->attr.type != event->pmu->type) 725 return -ENOENT; 726 727 pmu = uncore_event_to_pmu(event); 728 /* no device found for this pmu */ 729 if (pmu->func_id < 0) 730 return -ENOENT; 731 732 /* Sampling not supported yet */ 733 if (hwc->sample_period) 734 return -EINVAL; 735 736 /* 737 * Place all uncore events for a particular physical package 738 * onto a single cpu 739 */ 740 if (event->cpu < 0) 741 return -EINVAL; 742 box = uncore_pmu_to_box(pmu, event->cpu); 743 if (!box || box->cpu < 0) 744 return -EINVAL; 745 event->cpu = box->cpu; 746 event->pmu_private = box; 747 748 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; 749 750 event->hw.idx = -1; 751 event->hw.last_tag = ~0ULL; 752 event->hw.extra_reg.idx = EXTRA_REG_NONE; 753 event->hw.branch_reg.idx = EXTRA_REG_NONE; 754 755 if (event->attr.config == UNCORE_FIXED_EVENT) { 756 /* no fixed counter */ 757 if (!pmu->type->fixed_ctl) 758 return -EINVAL; 759 /* 760 * if there is only one fixed counter, only the first pmu 761 * can access the fixed counter 762 */ 763 if (pmu->type->single_fixed && pmu->pmu_idx > 0) 764 return -EINVAL; 765 766 /* fixed counters have event field hardcoded to zero */ 767 hwc->config = 0ULL; 768 } else if (is_freerunning_event(event)) { 769 hwc->config = event->attr.config; 770 if (!check_valid_freerunning_event(box, event)) 771 return -EINVAL; 772 event->hw.idx = UNCORE_PMC_IDX_FREERUNNING; 773 /* 774 * The free running counter event and free running counter 775 * are always 1:1 mapped. 776 * The free running counter is always active. 777 * Assign the free running counter here. 778 */ 779 event->hw.event_base = uncore_freerunning_counter(box, event); 780 } else { 781 hwc->config = event->attr.config & 782 (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32)); 783 if (pmu->type->ops->hw_config) { 784 ret = pmu->type->ops->hw_config(box, event); 785 if (ret) 786 return ret; 787 } 788 } 789 790 if (event->group_leader != event) 791 ret = uncore_validate_group(pmu, event); 792 else 793 ret = 0; 794 795 return ret; 796 } 797 798 static void uncore_pmu_enable(struct pmu *pmu) 799 { 800 struct intel_uncore_pmu *uncore_pmu; 801 struct intel_uncore_box *box; 802 803 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 804 805 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 806 if (!box) 807 return; 808 809 if (uncore_pmu->type->ops->enable_box) 810 uncore_pmu->type->ops->enable_box(box); 811 } 812 813 static void uncore_pmu_disable(struct pmu *pmu) 814 { 815 struct intel_uncore_pmu *uncore_pmu; 816 struct intel_uncore_box *box; 817 818 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 819 820 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 821 if (!box) 822 return; 823 824 if (uncore_pmu->type->ops->disable_box) 825 uncore_pmu->type->ops->disable_box(box); 826 } 827 828 static ssize_t uncore_get_attr_cpumask(struct device *dev, 829 struct device_attribute *attr, char *buf) 830 { 831 return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask); 832 } 833 834 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL); 835 836 static struct attribute *uncore_pmu_attrs[] = { 837 &dev_attr_cpumask.attr, 838 NULL, 839 }; 840 841 static const struct attribute_group uncore_pmu_attr_group = { 842 .attrs = uncore_pmu_attrs, 843 }; 844 845 void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu) 846 { 847 struct intel_uncore_type *type = pmu->type; 848 849 if (type->num_boxes == 1) 850 sprintf(pmu_name, "uncore_type_%u", type->type_id); 851 else { 852 sprintf(pmu_name, "uncore_type_%u_%d", 853 type->type_id, type->box_ids[pmu->pmu_idx]); 854 } 855 } 856 857 static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu) 858 { 859 struct intel_uncore_type *type = pmu->type; 860 861 /* 862 * No uncore block name in discovery table. 863 * Use uncore_type_&typeid_&boxid as name. 864 */ 865 if (!type->name) { 866 uncore_get_alias_name(pmu->name, pmu); 867 return; 868 } 869 870 if (type->num_boxes == 1) { 871 if (strlen(type->name) > 0) 872 sprintf(pmu->name, "uncore_%s", type->name); 873 else 874 sprintf(pmu->name, "uncore"); 875 } else { 876 /* 877 * Use the box ID from the discovery table if applicable. 878 */ 879 sprintf(pmu->name, "uncore_%s_%d", type->name, 880 type->box_ids ? type->box_ids[pmu->pmu_idx] : pmu->pmu_idx); 881 } 882 } 883 884 static int uncore_pmu_register(struct intel_uncore_pmu *pmu) 885 { 886 int ret; 887 888 if (!pmu->type->pmu) { 889 pmu->pmu = (struct pmu) { 890 .attr_groups = pmu->type->attr_groups, 891 .task_ctx_nr = perf_invalid_context, 892 .pmu_enable = uncore_pmu_enable, 893 .pmu_disable = uncore_pmu_disable, 894 .event_init = uncore_pmu_event_init, 895 .add = uncore_pmu_event_add, 896 .del = uncore_pmu_event_del, 897 .start = uncore_pmu_event_start, 898 .stop = uncore_pmu_event_stop, 899 .read = uncore_pmu_event_read, 900 .module = THIS_MODULE, 901 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 902 .attr_update = pmu->type->attr_update, 903 }; 904 } else { 905 pmu->pmu = *pmu->type->pmu; 906 pmu->pmu.attr_groups = pmu->type->attr_groups; 907 pmu->pmu.attr_update = pmu->type->attr_update; 908 } 909 910 uncore_get_pmu_name(pmu); 911 912 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); 913 if (!ret) 914 pmu->registered = true; 915 return ret; 916 } 917 918 static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) 919 { 920 if (!pmu->registered) 921 return; 922 perf_pmu_unregister(&pmu->pmu); 923 pmu->registered = false; 924 } 925 926 static void uncore_free_boxes(struct intel_uncore_pmu *pmu) 927 { 928 int die; 929 930 for (die = 0; die < uncore_max_dies(); die++) 931 kfree(pmu->boxes[die]); 932 kfree(pmu->boxes); 933 } 934 935 static void uncore_type_exit(struct intel_uncore_type *type) 936 { 937 struct intel_uncore_pmu *pmu = type->pmus; 938 int i; 939 940 if (type->cleanup_mapping) 941 type->cleanup_mapping(type); 942 943 if (pmu) { 944 for (i = 0; i < type->num_boxes; i++, pmu++) { 945 uncore_pmu_unregister(pmu); 946 uncore_free_boxes(pmu); 947 } 948 kfree(type->pmus); 949 type->pmus = NULL; 950 } 951 if (type->box_ids) { 952 kfree(type->box_ids); 953 type->box_ids = NULL; 954 } 955 kfree(type->events_group); 956 type->events_group = NULL; 957 } 958 959 static void uncore_types_exit(struct intel_uncore_type **types) 960 { 961 for (; *types; types++) 962 uncore_type_exit(*types); 963 } 964 965 static int __init uncore_type_init(struct intel_uncore_type *type, bool setid) 966 { 967 struct intel_uncore_pmu *pmus; 968 size_t size; 969 int i, j; 970 971 pmus = kcalloc(type->num_boxes, sizeof(*pmus), GFP_KERNEL); 972 if (!pmus) 973 return -ENOMEM; 974 975 size = uncore_max_dies() * sizeof(struct intel_uncore_box *); 976 977 for (i = 0; i < type->num_boxes; i++) { 978 pmus[i].func_id = setid ? i : -1; 979 pmus[i].pmu_idx = i; 980 pmus[i].type = type; 981 pmus[i].boxes = kzalloc(size, GFP_KERNEL); 982 if (!pmus[i].boxes) 983 goto err; 984 } 985 986 type->pmus = pmus; 987 type->unconstrainted = (struct event_constraint) 988 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1, 989 0, type->num_counters, 0, 0); 990 991 if (type->event_descs) { 992 struct { 993 struct attribute_group group; 994 struct attribute *attrs[]; 995 } *attr_group; 996 for (i = 0; type->event_descs[i].attr.attr.name; i++); 997 998 attr_group = kzalloc(struct_size(attr_group, attrs, i + 1), 999 GFP_KERNEL); 1000 if (!attr_group) 1001 goto err; 1002 1003 attr_group->group.name = "events"; 1004 attr_group->group.attrs = attr_group->attrs; 1005 1006 for (j = 0; j < i; j++) 1007 attr_group->attrs[j] = &type->event_descs[j].attr.attr; 1008 1009 type->events_group = &attr_group->group; 1010 } 1011 1012 type->pmu_group = &uncore_pmu_attr_group; 1013 1014 if (type->set_mapping) 1015 type->set_mapping(type); 1016 1017 return 0; 1018 1019 err: 1020 for (i = 0; i < type->num_boxes; i++) 1021 kfree(pmus[i].boxes); 1022 kfree(pmus); 1023 1024 return -ENOMEM; 1025 } 1026 1027 static int __init 1028 uncore_types_init(struct intel_uncore_type **types, bool setid) 1029 { 1030 int ret; 1031 1032 for (; *types; types++) { 1033 ret = uncore_type_init(*types, setid); 1034 if (ret) 1035 return ret; 1036 } 1037 return 0; 1038 } 1039 1040 /* 1041 * Get the die information of a PCI device. 1042 * @pdev: The PCI device. 1043 * @die: The die id which the device maps to. 1044 */ 1045 static int uncore_pci_get_dev_die_info(struct pci_dev *pdev, int *die) 1046 { 1047 *die = uncore_pcibus_to_dieid(pdev->bus); 1048 if (*die < 0) 1049 return -EINVAL; 1050 1051 return 0; 1052 } 1053 1054 static struct intel_uncore_pmu * 1055 uncore_pci_find_dev_pmu_from_types(struct pci_dev *pdev) 1056 { 1057 struct intel_uncore_type **types = uncore_pci_uncores; 1058 struct intel_uncore_type *type; 1059 u64 box_ctl; 1060 int i, die; 1061 1062 for (; *types; types++) { 1063 type = *types; 1064 for (die = 0; die < __uncore_max_dies; die++) { 1065 for (i = 0; i < type->num_boxes; i++) { 1066 if (!type->box_ctls[die]) 1067 continue; 1068 box_ctl = type->box_ctls[die] + type->pci_offsets[i]; 1069 if (pdev->devfn == UNCORE_DISCOVERY_PCI_DEVFN(box_ctl) && 1070 pdev->bus->number == UNCORE_DISCOVERY_PCI_BUS(box_ctl) && 1071 pci_domain_nr(pdev->bus) == UNCORE_DISCOVERY_PCI_DOMAIN(box_ctl)) 1072 return &type->pmus[i]; 1073 } 1074 } 1075 } 1076 1077 return NULL; 1078 } 1079 1080 /* 1081 * Find the PMU of a PCI device. 1082 * @pdev: The PCI device. 1083 * @ids: The ID table of the available PCI devices with a PMU. 1084 * If NULL, search the whole uncore_pci_uncores. 1085 */ 1086 static struct intel_uncore_pmu * 1087 uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) 1088 { 1089 struct intel_uncore_pmu *pmu = NULL; 1090 struct intel_uncore_type *type; 1091 kernel_ulong_t data; 1092 unsigned int devfn; 1093 1094 if (!ids) 1095 return uncore_pci_find_dev_pmu_from_types(pdev); 1096 1097 while (ids && ids->vendor) { 1098 if ((ids->vendor == pdev->vendor) && 1099 (ids->device == pdev->device)) { 1100 data = ids->driver_data; 1101 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(data), 1102 UNCORE_PCI_DEV_FUNC(data)); 1103 if (devfn == pdev->devfn) { 1104 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(data)]; 1105 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(data)]; 1106 break; 1107 } 1108 } 1109 ids++; 1110 } 1111 return pmu; 1112 } 1113 1114 /* 1115 * Register the PMU for a PCI device 1116 * @pdev: The PCI device. 1117 * @type: The corresponding PMU type of the device. 1118 * @pmu: The corresponding PMU of the device. 1119 * @die: The die id which the device maps to. 1120 */ 1121 static int uncore_pci_pmu_register(struct pci_dev *pdev, 1122 struct intel_uncore_type *type, 1123 struct intel_uncore_pmu *pmu, 1124 int die) 1125 { 1126 struct intel_uncore_box *box; 1127 int ret; 1128 1129 if (WARN_ON_ONCE(pmu->boxes[die] != NULL)) 1130 return -EINVAL; 1131 1132 box = uncore_alloc_box(type, NUMA_NO_NODE); 1133 if (!box) 1134 return -ENOMEM; 1135 1136 if (pmu->func_id < 0) 1137 pmu->func_id = pdev->devfn; 1138 else 1139 WARN_ON_ONCE(pmu->func_id != pdev->devfn); 1140 1141 atomic_inc(&box->refcnt); 1142 box->dieid = die; 1143 box->pci_dev = pdev; 1144 box->pmu = pmu; 1145 uncore_box_init(box); 1146 1147 pmu->boxes[die] = box; 1148 if (atomic_inc_return(&pmu->activeboxes) > 1) 1149 return 0; 1150 1151 /* First active box registers the pmu */ 1152 ret = uncore_pmu_register(pmu); 1153 if (ret) { 1154 pmu->boxes[die] = NULL; 1155 uncore_box_exit(box); 1156 kfree(box); 1157 } 1158 return ret; 1159 } 1160 1161 /* 1162 * add a pci uncore device 1163 */ 1164 static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1165 { 1166 struct intel_uncore_type *type; 1167 struct intel_uncore_pmu *pmu = NULL; 1168 int die, ret; 1169 1170 ret = uncore_pci_get_dev_die_info(pdev, &die); 1171 if (ret) 1172 return ret; 1173 1174 if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) { 1175 int idx = UNCORE_PCI_DEV_IDX(id->driver_data); 1176 1177 uncore_extra_pci_dev[die].dev[idx] = pdev; 1178 pci_set_drvdata(pdev, NULL); 1179 return 0; 1180 } 1181 1182 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; 1183 1184 /* 1185 * Some platforms, e.g. Knights Landing, use a common PCI device ID 1186 * for multiple instances of an uncore PMU device type. We should check 1187 * PCI slot and func to indicate the uncore box. 1188 */ 1189 if (id->driver_data & ~0xffff) { 1190 struct pci_driver *pci_drv = to_pci_driver(pdev->dev.driver); 1191 1192 pmu = uncore_pci_find_dev_pmu(pdev, pci_drv->id_table); 1193 if (pmu == NULL) 1194 return -ENODEV; 1195 } else { 1196 /* 1197 * for performance monitoring unit with multiple boxes, 1198 * each box has a different function id. 1199 */ 1200 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; 1201 } 1202 1203 ret = uncore_pci_pmu_register(pdev, type, pmu, die); 1204 1205 pci_set_drvdata(pdev, pmu->boxes[die]); 1206 1207 return ret; 1208 } 1209 1210 /* 1211 * Unregister the PMU of a PCI device 1212 * @pmu: The corresponding PMU is unregistered. 1213 * @die: The die id which the device maps to. 1214 */ 1215 static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) 1216 { 1217 struct intel_uncore_box *box = pmu->boxes[die]; 1218 1219 pmu->boxes[die] = NULL; 1220 if (atomic_dec_return(&pmu->activeboxes) == 0) 1221 uncore_pmu_unregister(pmu); 1222 uncore_box_exit(box); 1223 kfree(box); 1224 } 1225 1226 static void uncore_pci_remove(struct pci_dev *pdev) 1227 { 1228 struct intel_uncore_box *box; 1229 struct intel_uncore_pmu *pmu; 1230 int i, die; 1231 1232 if (uncore_pci_get_dev_die_info(pdev, &die)) 1233 return; 1234 1235 box = pci_get_drvdata(pdev); 1236 if (!box) { 1237 for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) { 1238 if (uncore_extra_pci_dev[die].dev[i] == pdev) { 1239 uncore_extra_pci_dev[die].dev[i] = NULL; 1240 break; 1241 } 1242 } 1243 WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); 1244 return; 1245 } 1246 1247 pmu = box->pmu; 1248 1249 pci_set_drvdata(pdev, NULL); 1250 1251 uncore_pci_pmu_unregister(pmu, die); 1252 } 1253 1254 static int uncore_bus_notify(struct notifier_block *nb, 1255 unsigned long action, void *data, 1256 const struct pci_device_id *ids) 1257 { 1258 struct device *dev = data; 1259 struct pci_dev *pdev = to_pci_dev(dev); 1260 struct intel_uncore_pmu *pmu; 1261 int die; 1262 1263 /* Unregister the PMU when the device is going to be deleted. */ 1264 if (action != BUS_NOTIFY_DEL_DEVICE) 1265 return NOTIFY_DONE; 1266 1267 pmu = uncore_pci_find_dev_pmu(pdev, ids); 1268 if (!pmu) 1269 return NOTIFY_DONE; 1270 1271 if (uncore_pci_get_dev_die_info(pdev, &die)) 1272 return NOTIFY_DONE; 1273 1274 uncore_pci_pmu_unregister(pmu, die); 1275 1276 return NOTIFY_OK; 1277 } 1278 1279 static int uncore_pci_sub_bus_notify(struct notifier_block *nb, 1280 unsigned long action, void *data) 1281 { 1282 return uncore_bus_notify(nb, action, data, 1283 uncore_pci_sub_driver->id_table); 1284 } 1285 1286 static struct notifier_block uncore_pci_sub_notifier = { 1287 .notifier_call = uncore_pci_sub_bus_notify, 1288 }; 1289 1290 static void uncore_pci_sub_driver_init(void) 1291 { 1292 const struct pci_device_id *ids = uncore_pci_sub_driver->id_table; 1293 struct intel_uncore_type *type; 1294 struct intel_uncore_pmu *pmu; 1295 struct pci_dev *pci_sub_dev; 1296 bool notify = false; 1297 unsigned int devfn; 1298 int die; 1299 1300 while (ids && ids->vendor) { 1301 pci_sub_dev = NULL; 1302 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(ids->driver_data)]; 1303 /* 1304 * Search the available device, and register the 1305 * corresponding PMU. 1306 */ 1307 while ((pci_sub_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 1308 ids->device, pci_sub_dev))) { 1309 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data), 1310 UNCORE_PCI_DEV_FUNC(ids->driver_data)); 1311 if (devfn != pci_sub_dev->devfn) 1312 continue; 1313 1314 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)]; 1315 if (!pmu) 1316 continue; 1317 1318 if (uncore_pci_get_dev_die_info(pci_sub_dev, &die)) 1319 continue; 1320 1321 if (!uncore_pci_pmu_register(pci_sub_dev, type, pmu, 1322 die)) 1323 notify = true; 1324 } 1325 ids++; 1326 } 1327 1328 if (notify && bus_register_notifier(&pci_bus_type, &uncore_pci_sub_notifier)) 1329 notify = false; 1330 1331 if (!notify) 1332 uncore_pci_sub_driver = NULL; 1333 } 1334 1335 static int uncore_pci_bus_notify(struct notifier_block *nb, 1336 unsigned long action, void *data) 1337 { 1338 return uncore_bus_notify(nb, action, data, NULL); 1339 } 1340 1341 static struct notifier_block uncore_pci_notifier = { 1342 .notifier_call = uncore_pci_bus_notify, 1343 }; 1344 1345 1346 static void uncore_pci_pmus_register(void) 1347 { 1348 struct intel_uncore_type **types = uncore_pci_uncores; 1349 struct intel_uncore_type *type; 1350 struct intel_uncore_pmu *pmu; 1351 struct pci_dev *pdev; 1352 u64 box_ctl; 1353 int i, die; 1354 1355 for (; *types; types++) { 1356 type = *types; 1357 for (die = 0; die < __uncore_max_dies; die++) { 1358 for (i = 0; i < type->num_boxes; i++) { 1359 if (!type->box_ctls[die]) 1360 continue; 1361 box_ctl = type->box_ctls[die] + type->pci_offsets[i]; 1362 pdev = pci_get_domain_bus_and_slot(UNCORE_DISCOVERY_PCI_DOMAIN(box_ctl), 1363 UNCORE_DISCOVERY_PCI_BUS(box_ctl), 1364 UNCORE_DISCOVERY_PCI_DEVFN(box_ctl)); 1365 if (!pdev) 1366 continue; 1367 pmu = &type->pmus[i]; 1368 1369 uncore_pci_pmu_register(pdev, type, pmu, die); 1370 } 1371 } 1372 } 1373 1374 bus_register_notifier(&pci_bus_type, &uncore_pci_notifier); 1375 } 1376 1377 static int __init uncore_pci_init(void) 1378 { 1379 size_t size; 1380 int ret; 1381 1382 size = uncore_max_dies() * sizeof(struct pci_extra_dev); 1383 uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL); 1384 if (!uncore_extra_pci_dev) { 1385 ret = -ENOMEM; 1386 goto err; 1387 } 1388 1389 ret = uncore_types_init(uncore_pci_uncores, false); 1390 if (ret) 1391 goto errtype; 1392 1393 if (uncore_pci_driver) { 1394 uncore_pci_driver->probe = uncore_pci_probe; 1395 uncore_pci_driver->remove = uncore_pci_remove; 1396 1397 ret = pci_register_driver(uncore_pci_driver); 1398 if (ret) 1399 goto errtype; 1400 } else 1401 uncore_pci_pmus_register(); 1402 1403 if (uncore_pci_sub_driver) 1404 uncore_pci_sub_driver_init(); 1405 1406 pcidrv_registered = true; 1407 return 0; 1408 1409 errtype: 1410 uncore_types_exit(uncore_pci_uncores); 1411 kfree(uncore_extra_pci_dev); 1412 uncore_extra_pci_dev = NULL; 1413 uncore_free_pcibus_map(); 1414 err: 1415 uncore_pci_uncores = empty_uncore; 1416 return ret; 1417 } 1418 1419 static void uncore_pci_exit(void) 1420 { 1421 if (pcidrv_registered) { 1422 pcidrv_registered = false; 1423 if (uncore_pci_sub_driver) 1424 bus_unregister_notifier(&pci_bus_type, &uncore_pci_sub_notifier); 1425 if (uncore_pci_driver) 1426 pci_unregister_driver(uncore_pci_driver); 1427 else 1428 bus_unregister_notifier(&pci_bus_type, &uncore_pci_notifier); 1429 uncore_types_exit(uncore_pci_uncores); 1430 kfree(uncore_extra_pci_dev); 1431 uncore_free_pcibus_map(); 1432 } 1433 } 1434 1435 static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, 1436 int new_cpu) 1437 { 1438 struct intel_uncore_pmu *pmu = type->pmus; 1439 struct intel_uncore_box *box; 1440 int i, die; 1441 1442 die = topology_logical_die_id(old_cpu < 0 ? new_cpu : old_cpu); 1443 for (i = 0; i < type->num_boxes; i++, pmu++) { 1444 box = pmu->boxes[die]; 1445 if (!box) 1446 continue; 1447 1448 if (old_cpu < 0) { 1449 WARN_ON_ONCE(box->cpu != -1); 1450 box->cpu = new_cpu; 1451 continue; 1452 } 1453 1454 WARN_ON_ONCE(box->cpu != old_cpu); 1455 box->cpu = -1; 1456 if (new_cpu < 0) 1457 continue; 1458 1459 uncore_pmu_cancel_hrtimer(box); 1460 perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); 1461 box->cpu = new_cpu; 1462 } 1463 } 1464 1465 static void uncore_change_context(struct intel_uncore_type **uncores, 1466 int old_cpu, int new_cpu) 1467 { 1468 for (; *uncores; uncores++) 1469 uncore_change_type_ctx(*uncores, old_cpu, new_cpu); 1470 } 1471 1472 static void uncore_box_unref(struct intel_uncore_type **types, int id) 1473 { 1474 struct intel_uncore_type *type; 1475 struct intel_uncore_pmu *pmu; 1476 struct intel_uncore_box *box; 1477 int i; 1478 1479 for (; *types; types++) { 1480 type = *types; 1481 pmu = type->pmus; 1482 for (i = 0; i < type->num_boxes; i++, pmu++) { 1483 box = pmu->boxes[id]; 1484 if (box && atomic_dec_return(&box->refcnt) == 0) 1485 uncore_box_exit(box); 1486 } 1487 } 1488 } 1489 1490 static int uncore_event_cpu_offline(unsigned int cpu) 1491 { 1492 int die, target; 1493 1494 /* Check if exiting cpu is used for collecting uncore events */ 1495 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) 1496 goto unref; 1497 /* Find a new cpu to collect uncore events */ 1498 target = cpumask_any_but(topology_die_cpumask(cpu), cpu); 1499 1500 /* Migrate uncore events to the new target */ 1501 if (target < nr_cpu_ids) 1502 cpumask_set_cpu(target, &uncore_cpu_mask); 1503 else 1504 target = -1; 1505 1506 uncore_change_context(uncore_msr_uncores, cpu, target); 1507 uncore_change_context(uncore_mmio_uncores, cpu, target); 1508 uncore_change_context(uncore_pci_uncores, cpu, target); 1509 1510 unref: 1511 /* Clear the references */ 1512 die = topology_logical_die_id(cpu); 1513 uncore_box_unref(uncore_msr_uncores, die); 1514 uncore_box_unref(uncore_mmio_uncores, die); 1515 return 0; 1516 } 1517 1518 static int allocate_boxes(struct intel_uncore_type **types, 1519 unsigned int die, unsigned int cpu) 1520 { 1521 struct intel_uncore_box *box, *tmp; 1522 struct intel_uncore_type *type; 1523 struct intel_uncore_pmu *pmu; 1524 LIST_HEAD(allocated); 1525 int i; 1526 1527 /* Try to allocate all required boxes */ 1528 for (; *types; types++) { 1529 type = *types; 1530 pmu = type->pmus; 1531 for (i = 0; i < type->num_boxes; i++, pmu++) { 1532 if (pmu->boxes[die]) 1533 continue; 1534 box = uncore_alloc_box(type, cpu_to_node(cpu)); 1535 if (!box) 1536 goto cleanup; 1537 box->pmu = pmu; 1538 box->dieid = die; 1539 list_add(&box->active_list, &allocated); 1540 } 1541 } 1542 /* Install them in the pmus */ 1543 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1544 list_del_init(&box->active_list); 1545 box->pmu->boxes[die] = box; 1546 } 1547 return 0; 1548 1549 cleanup: 1550 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1551 list_del_init(&box->active_list); 1552 kfree(box); 1553 } 1554 return -ENOMEM; 1555 } 1556 1557 static int uncore_box_ref(struct intel_uncore_type **types, 1558 int id, unsigned int cpu) 1559 { 1560 struct intel_uncore_type *type; 1561 struct intel_uncore_pmu *pmu; 1562 struct intel_uncore_box *box; 1563 int i, ret; 1564 1565 ret = allocate_boxes(types, id, cpu); 1566 if (ret) 1567 return ret; 1568 1569 for (; *types; types++) { 1570 type = *types; 1571 pmu = type->pmus; 1572 for (i = 0; i < type->num_boxes; i++, pmu++) { 1573 box = pmu->boxes[id]; 1574 if (box && atomic_inc_return(&box->refcnt) == 1) 1575 uncore_box_init(box); 1576 } 1577 } 1578 return 0; 1579 } 1580 1581 static int uncore_event_cpu_online(unsigned int cpu) 1582 { 1583 int die, target, msr_ret, mmio_ret; 1584 1585 die = topology_logical_die_id(cpu); 1586 msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); 1587 mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); 1588 if (msr_ret && mmio_ret) 1589 return -ENOMEM; 1590 1591 /* 1592 * Check if there is an online cpu in the package 1593 * which collects uncore events already. 1594 */ 1595 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); 1596 if (target < nr_cpu_ids) 1597 return 0; 1598 1599 cpumask_set_cpu(cpu, &uncore_cpu_mask); 1600 1601 if (!msr_ret) 1602 uncore_change_context(uncore_msr_uncores, -1, cpu); 1603 if (!mmio_ret) 1604 uncore_change_context(uncore_mmio_uncores, -1, cpu); 1605 uncore_change_context(uncore_pci_uncores, -1, cpu); 1606 return 0; 1607 } 1608 1609 static int __init type_pmu_register(struct intel_uncore_type *type) 1610 { 1611 int i, ret; 1612 1613 for (i = 0; i < type->num_boxes; i++) { 1614 ret = uncore_pmu_register(&type->pmus[i]); 1615 if (ret) 1616 return ret; 1617 } 1618 return 0; 1619 } 1620 1621 static int __init uncore_msr_pmus_register(void) 1622 { 1623 struct intel_uncore_type **types = uncore_msr_uncores; 1624 int ret; 1625 1626 for (; *types; types++) { 1627 ret = type_pmu_register(*types); 1628 if (ret) 1629 return ret; 1630 } 1631 return 0; 1632 } 1633 1634 static int __init uncore_cpu_init(void) 1635 { 1636 int ret; 1637 1638 ret = uncore_types_init(uncore_msr_uncores, true); 1639 if (ret) 1640 goto err; 1641 1642 ret = uncore_msr_pmus_register(); 1643 if (ret) 1644 goto err; 1645 return 0; 1646 err: 1647 uncore_types_exit(uncore_msr_uncores); 1648 uncore_msr_uncores = empty_uncore; 1649 return ret; 1650 } 1651 1652 static int __init uncore_mmio_init(void) 1653 { 1654 struct intel_uncore_type **types = uncore_mmio_uncores; 1655 int ret; 1656 1657 ret = uncore_types_init(types, true); 1658 if (ret) 1659 goto err; 1660 1661 for (; *types; types++) { 1662 ret = type_pmu_register(*types); 1663 if (ret) 1664 goto err; 1665 } 1666 return 0; 1667 err: 1668 uncore_types_exit(uncore_mmio_uncores); 1669 uncore_mmio_uncores = empty_uncore; 1670 return ret; 1671 } 1672 1673 struct intel_uncore_init_fun { 1674 void (*cpu_init)(void); 1675 int (*pci_init)(void); 1676 void (*mmio_init)(void); 1677 bool use_discovery; 1678 }; 1679 1680 static const struct intel_uncore_init_fun nhm_uncore_init __initconst = { 1681 .cpu_init = nhm_uncore_cpu_init, 1682 }; 1683 1684 static const struct intel_uncore_init_fun snb_uncore_init __initconst = { 1685 .cpu_init = snb_uncore_cpu_init, 1686 .pci_init = snb_uncore_pci_init, 1687 }; 1688 1689 static const struct intel_uncore_init_fun ivb_uncore_init __initconst = { 1690 .cpu_init = snb_uncore_cpu_init, 1691 .pci_init = ivb_uncore_pci_init, 1692 }; 1693 1694 static const struct intel_uncore_init_fun hsw_uncore_init __initconst = { 1695 .cpu_init = snb_uncore_cpu_init, 1696 .pci_init = hsw_uncore_pci_init, 1697 }; 1698 1699 static const struct intel_uncore_init_fun bdw_uncore_init __initconst = { 1700 .cpu_init = snb_uncore_cpu_init, 1701 .pci_init = bdw_uncore_pci_init, 1702 }; 1703 1704 static const struct intel_uncore_init_fun snbep_uncore_init __initconst = { 1705 .cpu_init = snbep_uncore_cpu_init, 1706 .pci_init = snbep_uncore_pci_init, 1707 }; 1708 1709 static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = { 1710 .cpu_init = nhmex_uncore_cpu_init, 1711 }; 1712 1713 static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = { 1714 .cpu_init = ivbep_uncore_cpu_init, 1715 .pci_init = ivbep_uncore_pci_init, 1716 }; 1717 1718 static const struct intel_uncore_init_fun hswep_uncore_init __initconst = { 1719 .cpu_init = hswep_uncore_cpu_init, 1720 .pci_init = hswep_uncore_pci_init, 1721 }; 1722 1723 static const struct intel_uncore_init_fun bdx_uncore_init __initconst = { 1724 .cpu_init = bdx_uncore_cpu_init, 1725 .pci_init = bdx_uncore_pci_init, 1726 }; 1727 1728 static const struct intel_uncore_init_fun knl_uncore_init __initconst = { 1729 .cpu_init = knl_uncore_cpu_init, 1730 .pci_init = knl_uncore_pci_init, 1731 }; 1732 1733 static const struct intel_uncore_init_fun skl_uncore_init __initconst = { 1734 .cpu_init = skl_uncore_cpu_init, 1735 .pci_init = skl_uncore_pci_init, 1736 }; 1737 1738 static const struct intel_uncore_init_fun skx_uncore_init __initconst = { 1739 .cpu_init = skx_uncore_cpu_init, 1740 .pci_init = skx_uncore_pci_init, 1741 }; 1742 1743 static const struct intel_uncore_init_fun icl_uncore_init __initconst = { 1744 .cpu_init = icl_uncore_cpu_init, 1745 .pci_init = skl_uncore_pci_init, 1746 }; 1747 1748 static const struct intel_uncore_init_fun tgl_uncore_init __initconst = { 1749 .cpu_init = tgl_uncore_cpu_init, 1750 .mmio_init = tgl_uncore_mmio_init, 1751 }; 1752 1753 static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = { 1754 .cpu_init = tgl_uncore_cpu_init, 1755 .mmio_init = tgl_l_uncore_mmio_init, 1756 }; 1757 1758 static const struct intel_uncore_init_fun rkl_uncore_init __initconst = { 1759 .cpu_init = tgl_uncore_cpu_init, 1760 .pci_init = skl_uncore_pci_init, 1761 }; 1762 1763 static const struct intel_uncore_init_fun adl_uncore_init __initconst = { 1764 .cpu_init = adl_uncore_cpu_init, 1765 .mmio_init = adl_uncore_mmio_init, 1766 }; 1767 1768 static const struct intel_uncore_init_fun icx_uncore_init __initconst = { 1769 .cpu_init = icx_uncore_cpu_init, 1770 .pci_init = icx_uncore_pci_init, 1771 .mmio_init = icx_uncore_mmio_init, 1772 }; 1773 1774 static const struct intel_uncore_init_fun snr_uncore_init __initconst = { 1775 .cpu_init = snr_uncore_cpu_init, 1776 .pci_init = snr_uncore_pci_init, 1777 .mmio_init = snr_uncore_mmio_init, 1778 }; 1779 1780 static const struct intel_uncore_init_fun spr_uncore_init __initconst = { 1781 .cpu_init = spr_uncore_cpu_init, 1782 .pci_init = spr_uncore_pci_init, 1783 .mmio_init = spr_uncore_mmio_init, 1784 .use_discovery = true, 1785 }; 1786 1787 static const struct intel_uncore_init_fun generic_uncore_init __initconst = { 1788 .cpu_init = intel_uncore_generic_uncore_cpu_init, 1789 .pci_init = intel_uncore_generic_uncore_pci_init, 1790 .mmio_init = intel_uncore_generic_uncore_mmio_init, 1791 }; 1792 1793 static const struct x86_cpu_id intel_uncore_match[] __initconst = { 1794 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_uncore_init), 1795 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_uncore_init), 1796 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_uncore_init), 1797 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_uncore_init), 1798 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_uncore_init), 1799 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &ivb_uncore_init), 1800 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &hsw_uncore_init), 1801 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hsw_uncore_init), 1802 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &hsw_uncore_init), 1803 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &bdw_uncore_init), 1804 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &bdw_uncore_init), 1805 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snbep_uncore_init), 1806 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhmex_uncore_init), 1807 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhmex_uncore_init), 1808 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ivbep_uncore_init), 1809 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &hswep_uncore_init), 1810 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &bdx_uncore_init), 1811 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &bdx_uncore_init), 1812 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_uncore_init), 1813 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_uncore_init), 1814 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &skl_uncore_init), 1815 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &skl_uncore_init), 1816 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &skx_uncore_init), 1817 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &skl_uncore_init), 1818 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &skl_uncore_init), 1819 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &skl_uncore_init), 1820 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &skl_uncore_init), 1821 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init), 1822 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init), 1823 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init), 1824 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_uncore_init), 1825 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_uncore_init), 1826 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init), 1827 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init), 1828 X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &rkl_uncore_init), 1829 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_uncore_init), 1830 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_uncore_init), 1831 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &adl_uncore_init), 1832 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_uncore_init), 1833 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_uncore_init), 1834 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_uncore_init), 1835 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &spr_uncore_init), 1836 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &spr_uncore_init), 1837 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init), 1838 {}, 1839 }; 1840 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match); 1841 1842 static int __init intel_uncore_init(void) 1843 { 1844 const struct x86_cpu_id *id; 1845 struct intel_uncore_init_fun *uncore_init; 1846 int pret = 0, cret = 0, mret = 0, ret; 1847 1848 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 1849 return -ENODEV; 1850 1851 __uncore_max_dies = 1852 topology_max_packages() * topology_max_die_per_package(); 1853 1854 id = x86_match_cpu(intel_uncore_match); 1855 if (!id) { 1856 if (!uncore_no_discover && intel_uncore_has_discovery_tables()) 1857 uncore_init = (struct intel_uncore_init_fun *)&generic_uncore_init; 1858 else 1859 return -ENODEV; 1860 } else { 1861 uncore_init = (struct intel_uncore_init_fun *)id->driver_data; 1862 if (uncore_no_discover && uncore_init->use_discovery) 1863 return -ENODEV; 1864 if (uncore_init->use_discovery && !intel_uncore_has_discovery_tables()) 1865 return -ENODEV; 1866 } 1867 1868 if (uncore_init->pci_init) { 1869 pret = uncore_init->pci_init(); 1870 if (!pret) 1871 pret = uncore_pci_init(); 1872 } 1873 1874 if (uncore_init->cpu_init) { 1875 uncore_init->cpu_init(); 1876 cret = uncore_cpu_init(); 1877 } 1878 1879 if (uncore_init->mmio_init) { 1880 uncore_init->mmio_init(); 1881 mret = uncore_mmio_init(); 1882 } 1883 1884 if (cret && pret && mret) { 1885 ret = -ENODEV; 1886 goto free_discovery; 1887 } 1888 1889 /* Install hotplug callbacks to setup the targets for each package */ 1890 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE, 1891 "perf/x86/intel/uncore:online", 1892 uncore_event_cpu_online, 1893 uncore_event_cpu_offline); 1894 if (ret) 1895 goto err; 1896 return 0; 1897 1898 err: 1899 uncore_types_exit(uncore_msr_uncores); 1900 uncore_types_exit(uncore_mmio_uncores); 1901 uncore_pci_exit(); 1902 free_discovery: 1903 intel_uncore_clear_discovery_tables(); 1904 return ret; 1905 } 1906 module_init(intel_uncore_init); 1907 1908 static void __exit intel_uncore_exit(void) 1909 { 1910 cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE); 1911 uncore_types_exit(uncore_msr_uncores); 1912 uncore_types_exit(uncore_mmio_uncores); 1913 uncore_pci_exit(); 1914 intel_uncore_clear_discovery_tables(); 1915 } 1916 module_exit(intel_uncore_exit); 1917