1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Support Intel uncore PerfMon discovery mechanism. 4 * Copyright(c) 2021 Intel Corporation. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include "uncore.h" 9 #include "uncore_discovery.h" 10 11 static struct rb_root discovery_tables = RB_ROOT; 12 static int num_discovered_types[UNCORE_ACCESS_MAX]; 13 14 static bool has_generic_discovery_table(void) 15 { 16 struct pci_dev *dev; 17 int dvsec; 18 19 dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL); 20 if (!dev) 21 return false; 22 23 /* A discovery table device has the unique capability ID. */ 24 dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY); 25 pci_dev_put(dev); 26 if (dvsec) 27 return true; 28 29 return false; 30 } 31 32 static int logical_die_id; 33 34 static int get_device_die_id(struct pci_dev *dev) 35 { 36 int node = pcibus_to_node(dev->bus); 37 38 /* 39 * If the NUMA info is not available, assume that the logical die id is 40 * continuous in the order in which the discovery table devices are 41 * detected. 42 */ 43 if (node < 0) 44 return logical_die_id++; 45 46 return uncore_device_to_die(dev); 47 } 48 49 #define __node_2_type(cur) \ 50 rb_entry((cur), struct intel_uncore_discovery_type, node) 51 52 static inline int __type_cmp(const void *key, const struct rb_node *b) 53 { 54 struct intel_uncore_discovery_type *type_b = __node_2_type(b); 55 const u16 *type_id = key; 56 57 if (type_b->type > *type_id) 58 return -1; 59 else if (type_b->type < *type_id) 60 return 1; 61 62 return 0; 63 } 64 65 static inline struct intel_uncore_discovery_type * 66 search_uncore_discovery_type(u16 type_id) 67 { 68 struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp); 69 70 return (node) ? __node_2_type(node) : NULL; 71 } 72 73 static inline bool __type_less(struct rb_node *a, const struct rb_node *b) 74 { 75 return (__node_2_type(a)->type < __node_2_type(b)->type); 76 } 77 78 static struct intel_uncore_discovery_type * 79 add_uncore_discovery_type(struct uncore_unit_discovery *unit) 80 { 81 struct intel_uncore_discovery_type *type; 82 83 if (unit->access_type >= UNCORE_ACCESS_MAX) { 84 pr_warn("Unsupported access type %d\n", unit->access_type); 85 return NULL; 86 } 87 88 type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL); 89 if (!type) 90 return NULL; 91 92 type->units = RB_ROOT; 93 94 type->access_type = unit->access_type; 95 num_discovered_types[type->access_type]++; 96 type->type = unit->box_type; 97 98 rb_add(&type->node, &discovery_tables, __type_less); 99 100 return type; 101 } 102 103 static struct intel_uncore_discovery_type * 104 get_uncore_discovery_type(struct uncore_unit_discovery *unit) 105 { 106 struct intel_uncore_discovery_type *type; 107 108 type = search_uncore_discovery_type(unit->box_type); 109 if (type) 110 return type; 111 112 return add_uncore_discovery_type(unit); 113 } 114 115 static inline int pmu_idx_cmp(const void *key, const struct rb_node *b) 116 { 117 struct intel_uncore_discovery_unit *unit; 118 const unsigned int *id = key; 119 120 unit = rb_entry(b, struct intel_uncore_discovery_unit, node); 121 122 if (unit->pmu_idx > *id) 123 return -1; 124 else if (unit->pmu_idx < *id) 125 return 1; 126 127 return 0; 128 } 129 130 static struct intel_uncore_discovery_unit * 131 intel_uncore_find_discovery_unit(struct rb_root *units, int die, 132 unsigned int pmu_idx) 133 { 134 struct intel_uncore_discovery_unit *unit; 135 struct rb_node *pos; 136 137 if (!units) 138 return NULL; 139 140 pos = rb_find_first(&pmu_idx, units, pmu_idx_cmp); 141 if (!pos) 142 return NULL; 143 unit = rb_entry(pos, struct intel_uncore_discovery_unit, node); 144 145 if (die < 0) 146 return unit; 147 148 for (; pos; pos = rb_next(pos)) { 149 unit = rb_entry(pos, struct intel_uncore_discovery_unit, node); 150 151 if (unit->pmu_idx != pmu_idx) 152 break; 153 154 if (unit->die == die) 155 return unit; 156 } 157 158 return NULL; 159 } 160 161 int intel_uncore_find_discovery_unit_id(struct rb_root *units, int die, 162 unsigned int pmu_idx) 163 { 164 struct intel_uncore_discovery_unit *unit; 165 166 unit = intel_uncore_find_discovery_unit(units, die, pmu_idx); 167 if (unit) 168 return unit->id; 169 170 return -1; 171 } 172 173 static inline bool unit_less(struct rb_node *a, const struct rb_node *b) 174 { 175 struct intel_uncore_discovery_unit *a_node, *b_node; 176 177 a_node = rb_entry(a, struct intel_uncore_discovery_unit, node); 178 b_node = rb_entry(b, struct intel_uncore_discovery_unit, node); 179 180 if (a_node->pmu_idx < b_node->pmu_idx) 181 return true; 182 if (a_node->pmu_idx > b_node->pmu_idx) 183 return false; 184 185 if (a_node->die < b_node->die) 186 return true; 187 if (a_node->die > b_node->die) 188 return false; 189 190 return 0; 191 } 192 193 static inline struct intel_uncore_discovery_unit * 194 uncore_find_unit(struct rb_root *root, unsigned int id) 195 { 196 struct intel_uncore_discovery_unit *unit; 197 struct rb_node *node; 198 199 for (node = rb_first(root); node; node = rb_next(node)) { 200 unit = rb_entry(node, struct intel_uncore_discovery_unit, node); 201 if (unit->id == id) 202 return unit; 203 } 204 205 return NULL; 206 } 207 208 void uncore_find_add_unit(struct intel_uncore_discovery_unit *node, 209 struct rb_root *root, u16 *num_units) 210 { 211 struct intel_uncore_discovery_unit *unit = uncore_find_unit(root, node->id); 212 213 if (unit) 214 node->pmu_idx = unit->pmu_idx; 215 else if (num_units) 216 node->pmu_idx = (*num_units)++; 217 218 rb_add(&node->node, root, unit_less); 219 } 220 221 static void 222 uncore_insert_box_info(struct uncore_unit_discovery *unit, 223 int die) 224 { 225 struct intel_uncore_discovery_unit *node; 226 struct intel_uncore_discovery_type *type; 227 228 if (!unit->ctl || !unit->ctl_offset || !unit->ctr_offset) { 229 pr_info("Invalid address is detected for uncore type %d box %d, " 230 "Disable the uncore unit.\n", 231 unit->box_type, unit->box_id); 232 return; 233 } 234 235 node = kzalloc(sizeof(*node), GFP_KERNEL); 236 if (!node) 237 return; 238 239 node->die = die; 240 node->id = unit->box_id; 241 node->addr = unit->ctl; 242 243 type = get_uncore_discovery_type(unit); 244 if (!type) { 245 kfree(node); 246 return; 247 } 248 249 uncore_find_add_unit(node, &type->units, &type->num_units); 250 251 /* Store generic information for the first box */ 252 if (type->num_units == 1) { 253 type->num_counters = unit->num_regs; 254 type->counter_width = unit->bit_width; 255 type->ctl_offset = unit->ctl_offset; 256 type->ctr_offset = unit->ctr_offset; 257 } 258 } 259 260 static bool 261 uncore_ignore_unit(struct uncore_unit_discovery *unit, int *ignore) 262 { 263 int i; 264 265 if (!ignore) 266 return false; 267 268 for (i = 0; ignore[i] != UNCORE_IGNORE_END ; i++) { 269 if (unit->box_type == ignore[i]) 270 return true; 271 } 272 273 return false; 274 } 275 276 static int parse_discovery_table(struct pci_dev *dev, int die, 277 u32 bar_offset, bool *parsed, 278 int *ignore) 279 { 280 struct uncore_global_discovery global; 281 struct uncore_unit_discovery unit; 282 void __iomem *io_addr; 283 resource_size_t addr; 284 unsigned long size; 285 u32 val; 286 int i; 287 288 pci_read_config_dword(dev, bar_offset, &val); 289 290 if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64) 291 return -EINVAL; 292 293 addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK); 294 #ifdef CONFIG_PHYS_ADDR_T_64BIT 295 if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) { 296 u32 val2; 297 298 pci_read_config_dword(dev, bar_offset + 4, &val2); 299 addr |= ((resource_size_t)val2) << 32; 300 } 301 #endif 302 size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE; 303 io_addr = ioremap(addr, size); 304 if (!io_addr) 305 return -ENOMEM; 306 307 /* Read Global Discovery State */ 308 memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery)); 309 if (uncore_discovery_invalid_unit(global)) { 310 pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n", 311 global.table1, global.ctl, global.table3); 312 iounmap(io_addr); 313 return -EINVAL; 314 } 315 iounmap(io_addr); 316 317 size = (1 + global.max_units) * global.stride * 8; 318 io_addr = ioremap(addr, size); 319 if (!io_addr) 320 return -ENOMEM; 321 322 /* Parsing Unit Discovery State */ 323 for (i = 0; i < global.max_units; i++) { 324 memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8), 325 sizeof(struct uncore_unit_discovery)); 326 327 if (uncore_discovery_invalid_unit(unit)) 328 continue; 329 330 if (unit.access_type >= UNCORE_ACCESS_MAX) 331 continue; 332 333 if (uncore_ignore_unit(&unit, ignore)) 334 continue; 335 336 uncore_insert_box_info(&unit, die); 337 } 338 339 *parsed = true; 340 iounmap(io_addr); 341 return 0; 342 } 343 344 bool intel_uncore_has_discovery_tables(int *ignore) 345 { 346 u32 device, val, entry_id, bar_offset; 347 int die, dvsec = 0, ret = true; 348 struct pci_dev *dev = NULL; 349 bool parsed = false; 350 351 if (has_generic_discovery_table()) 352 device = UNCORE_DISCOVERY_TABLE_DEVICE; 353 else 354 device = PCI_ANY_ID; 355 356 /* 357 * Start a new search and iterates through the list of 358 * the discovery table devices. 359 */ 360 while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) { 361 while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) { 362 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val); 363 entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK; 364 if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON) 365 continue; 366 367 pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val); 368 369 if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) { 370 ret = false; 371 goto err; 372 } 373 bar_offset = UNCORE_DISCOVERY_BIR_BASE + 374 (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP; 375 376 die = get_device_die_id(dev); 377 if (die < 0) 378 continue; 379 380 parse_discovery_table(dev, die, bar_offset, &parsed, ignore); 381 } 382 } 383 384 /* None of the discovery tables are available */ 385 if (!parsed) 386 ret = false; 387 err: 388 pci_dev_put(dev); 389 390 return ret; 391 } 392 393 void intel_uncore_clear_discovery_tables(void) 394 { 395 struct intel_uncore_discovery_type *type, *next; 396 struct intel_uncore_discovery_unit *pos; 397 struct rb_node *node; 398 399 rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) { 400 while (!RB_EMPTY_ROOT(&type->units)) { 401 node = rb_first(&type->units); 402 pos = rb_entry(node, struct intel_uncore_discovery_unit, node); 403 rb_erase(node, &type->units); 404 kfree(pos); 405 } 406 kfree(type); 407 } 408 } 409 410 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); 411 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 412 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); 413 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); 414 DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31"); 415 416 static struct attribute *generic_uncore_formats_attr[] = { 417 &format_attr_event.attr, 418 &format_attr_umask.attr, 419 &format_attr_edge.attr, 420 &format_attr_inv.attr, 421 &format_attr_thresh.attr, 422 NULL, 423 }; 424 425 static const struct attribute_group generic_uncore_format_group = { 426 .name = "format", 427 .attrs = generic_uncore_formats_attr, 428 }; 429 430 static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) 431 { 432 struct intel_uncore_discovery_unit *unit; 433 434 unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes, 435 -1, box->pmu->pmu_idx); 436 if (WARN_ON_ONCE(!unit)) 437 return 0; 438 439 return unit->addr; 440 } 441 442 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) 443 { 444 wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); 445 } 446 447 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) 448 { 449 wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); 450 } 451 452 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) 453 { 454 wrmsrl(intel_generic_uncore_box_ctl(box), 0); 455 } 456 457 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, 458 struct perf_event *event) 459 { 460 struct hw_perf_event *hwc = &event->hw; 461 462 wrmsrl(hwc->config_base, hwc->config); 463 } 464 465 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box, 466 struct perf_event *event) 467 { 468 struct hw_perf_event *hwc = &event->hw; 469 470 wrmsrl(hwc->config_base, 0); 471 } 472 473 static struct intel_uncore_ops generic_uncore_msr_ops = { 474 .init_box = intel_generic_uncore_msr_init_box, 475 .disable_box = intel_generic_uncore_msr_disable_box, 476 .enable_box = intel_generic_uncore_msr_enable_box, 477 .disable_event = intel_generic_uncore_msr_disable_event, 478 .enable_event = intel_generic_uncore_msr_enable_event, 479 .read_counter = uncore_msr_read_counter, 480 }; 481 482 bool intel_generic_uncore_assign_hw_event(struct perf_event *event, 483 struct intel_uncore_box *box) 484 { 485 struct hw_perf_event *hwc = &event->hw; 486 u64 box_ctl; 487 488 if (!box->pmu->type->boxes) 489 return false; 490 491 if (box->io_addr) { 492 hwc->config_base = uncore_pci_event_ctl(box, hwc->idx); 493 hwc->event_base = uncore_pci_perf_ctr(box, hwc->idx); 494 return true; 495 } 496 497 box_ctl = intel_generic_uncore_box_ctl(box); 498 if (!box_ctl) 499 return false; 500 501 if (box->pci_dev) { 502 box_ctl = UNCORE_DISCOVERY_PCI_BOX_CTRL(box_ctl); 503 hwc->config_base = box_ctl + uncore_pci_event_ctl(box, hwc->idx); 504 hwc->event_base = box_ctl + uncore_pci_perf_ctr(box, hwc->idx); 505 return true; 506 } 507 508 hwc->config_base = box_ctl + box->pmu->type->event_ctl + hwc->idx; 509 hwc->event_base = box_ctl + box->pmu->type->perf_ctr + hwc->idx; 510 511 return true; 512 } 513 514 static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) 515 { 516 return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box)); 517 } 518 519 void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) 520 { 521 struct pci_dev *pdev = box->pci_dev; 522 int box_ctl = intel_pci_uncore_box_ctl(box); 523 524 __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); 525 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT); 526 } 527 528 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) 529 { 530 struct pci_dev *pdev = box->pci_dev; 531 int box_ctl = intel_pci_uncore_box_ctl(box); 532 533 pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ); 534 } 535 536 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) 537 { 538 struct pci_dev *pdev = box->pci_dev; 539 int box_ctl = intel_pci_uncore_box_ctl(box); 540 541 pci_write_config_dword(pdev, box_ctl, 0); 542 } 543 544 static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box, 545 struct perf_event *event) 546 { 547 struct pci_dev *pdev = box->pci_dev; 548 struct hw_perf_event *hwc = &event->hw; 549 550 pci_write_config_dword(pdev, hwc->config_base, hwc->config); 551 } 552 553 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, 554 struct perf_event *event) 555 { 556 struct pci_dev *pdev = box->pci_dev; 557 struct hw_perf_event *hwc = &event->hw; 558 559 pci_write_config_dword(pdev, hwc->config_base, 0); 560 } 561 562 u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, 563 struct perf_event *event) 564 { 565 struct pci_dev *pdev = box->pci_dev; 566 struct hw_perf_event *hwc = &event->hw; 567 u64 count = 0; 568 569 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count); 570 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1); 571 572 return count; 573 } 574 575 static struct intel_uncore_ops generic_uncore_pci_ops = { 576 .init_box = intel_generic_uncore_pci_init_box, 577 .disable_box = intel_generic_uncore_pci_disable_box, 578 .enable_box = intel_generic_uncore_pci_enable_box, 579 .disable_event = intel_generic_uncore_pci_disable_event, 580 .enable_event = intel_generic_uncore_pci_enable_event, 581 .read_counter = intel_generic_uncore_pci_read_counter, 582 }; 583 584 #define UNCORE_GENERIC_MMIO_SIZE 0x4000 585 586 void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) 587 { 588 static struct intel_uncore_discovery_unit *unit; 589 struct intel_uncore_type *type = box->pmu->type; 590 resource_size_t addr; 591 592 unit = intel_uncore_find_discovery_unit(type->boxes, box->dieid, box->pmu->pmu_idx); 593 if (!unit) { 594 pr_warn("Uncore type %d id %d: Cannot find box control address.\n", 595 type->type_id, box->pmu->pmu_idx); 596 return; 597 } 598 599 if (!unit->addr) { 600 pr_warn("Uncore type %d box %d: Invalid box control address.\n", 601 type->type_id, unit->id); 602 return; 603 } 604 605 addr = unit->addr; 606 box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE); 607 if (!box->io_addr) { 608 pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", 609 type->type_id, unit->id, (unsigned long long)addr); 610 return; 611 } 612 613 writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); 614 } 615 616 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) 617 { 618 if (!box->io_addr) 619 return; 620 621 writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr); 622 } 623 624 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box) 625 { 626 if (!box->io_addr) 627 return; 628 629 writel(0, box->io_addr); 630 } 631 632 void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, 633 struct perf_event *event) 634 { 635 struct hw_perf_event *hwc = &event->hw; 636 637 if (!box->io_addr) 638 return; 639 640 writel(hwc->config, box->io_addr + hwc->config_base); 641 } 642 643 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, 644 struct perf_event *event) 645 { 646 struct hw_perf_event *hwc = &event->hw; 647 648 if (!box->io_addr) 649 return; 650 651 writel(0, box->io_addr + hwc->config_base); 652 } 653 654 static struct intel_uncore_ops generic_uncore_mmio_ops = { 655 .init_box = intel_generic_uncore_mmio_init_box, 656 .exit_box = uncore_mmio_exit_box, 657 .disable_box = intel_generic_uncore_mmio_disable_box, 658 .enable_box = intel_generic_uncore_mmio_enable_box, 659 .disable_event = intel_generic_uncore_mmio_disable_event, 660 .enable_event = intel_generic_uncore_mmio_enable_event, 661 .read_counter = uncore_mmio_read_counter, 662 }; 663 664 static bool uncore_update_uncore_type(enum uncore_access_type type_id, 665 struct intel_uncore_type *uncore, 666 struct intel_uncore_discovery_type *type) 667 { 668 uncore->type_id = type->type; 669 uncore->num_counters = type->num_counters; 670 uncore->perf_ctr_bits = type->counter_width; 671 uncore->perf_ctr = (unsigned int)type->ctr_offset; 672 uncore->event_ctl = (unsigned int)type->ctl_offset; 673 uncore->boxes = &type->units; 674 uncore->num_boxes = type->num_units; 675 676 switch (type_id) { 677 case UNCORE_ACCESS_MSR: 678 uncore->ops = &generic_uncore_msr_ops; 679 break; 680 case UNCORE_ACCESS_PCI: 681 uncore->ops = &generic_uncore_pci_ops; 682 break; 683 case UNCORE_ACCESS_MMIO: 684 uncore->ops = &generic_uncore_mmio_ops; 685 uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE; 686 break; 687 default: 688 return false; 689 } 690 691 return true; 692 } 693 694 struct intel_uncore_type ** 695 intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra) 696 { 697 struct intel_uncore_discovery_type *type; 698 struct intel_uncore_type **uncores; 699 struct intel_uncore_type *uncore; 700 struct rb_node *node; 701 int i = 0; 702 703 uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1, 704 sizeof(struct intel_uncore_type *), GFP_KERNEL); 705 if (!uncores) 706 return empty_uncore; 707 708 for (node = rb_first(&discovery_tables); node; node = rb_next(node)) { 709 type = rb_entry(node, struct intel_uncore_discovery_type, node); 710 if (type->access_type != type_id) 711 continue; 712 713 uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL); 714 if (!uncore) 715 break; 716 717 uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK; 718 uncore->format_group = &generic_uncore_format_group; 719 720 if (!uncore_update_uncore_type(type_id, uncore, type)) { 721 kfree(uncore); 722 continue; 723 } 724 uncores[i++] = uncore; 725 } 726 727 return uncores; 728 } 729 730 void intel_uncore_generic_uncore_cpu_init(void) 731 { 732 uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0); 733 } 734 735 int intel_uncore_generic_uncore_pci_init(void) 736 { 737 uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0); 738 739 return 0; 740 } 741 742 void intel_uncore_generic_uncore_mmio_init(void) 743 { 744 uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0); 745 } 746