xref: /linux/arch/x86/events/intel/uncore_discovery.c (revision 527a0f2bdcfe77fce22f006b97e42e4da3137c86)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Support Intel uncore PerfMon discovery mechanism.
4  * Copyright(c) 2021 Intel Corporation.
5  */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include "uncore.h"
9 #include "uncore_discovery.h"
10 
11 static struct rb_root discovery_tables = RB_ROOT;
12 static int num_discovered_types[UNCORE_ACCESS_MAX];
13 
14 static bool has_generic_discovery_table(void)
15 {
16 	struct pci_dev *dev;
17 	int dvsec;
18 
19 	dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL);
20 	if (!dev)
21 		return false;
22 
23 	/* A discovery table device has the unique capability ID. */
24 	dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY);
25 	pci_dev_put(dev);
26 	if (dvsec)
27 		return true;
28 
29 	return false;
30 }
31 
32 static int logical_die_id;
33 
34 static int get_device_die_id(struct pci_dev *dev)
35 {
36 	int node = pcibus_to_node(dev->bus);
37 
38 	/*
39 	 * If the NUMA info is not available, assume that the logical die id is
40 	 * continuous in the order in which the discovery table devices are
41 	 * detected.
42 	 */
43 	if (node < 0)
44 		return logical_die_id++;
45 
46 	return uncore_device_to_die(dev);
47 }
48 
49 #define __node_2_type(cur)	\
50 	rb_entry((cur), struct intel_uncore_discovery_type, node)
51 
52 static inline int __type_cmp(const void *key, const struct rb_node *b)
53 {
54 	struct intel_uncore_discovery_type *type_b = __node_2_type(b);
55 	const u16 *type_id = key;
56 
57 	if (type_b->type > *type_id)
58 		return -1;
59 	else if (type_b->type < *type_id)
60 		return 1;
61 
62 	return 0;
63 }
64 
65 static inline struct intel_uncore_discovery_type *
66 search_uncore_discovery_type(u16 type_id)
67 {
68 	struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp);
69 
70 	return (node) ? __node_2_type(node) : NULL;
71 }
72 
73 static inline bool __type_less(struct rb_node *a, const struct rb_node *b)
74 {
75 	return (__node_2_type(a)->type < __node_2_type(b)->type);
76 }
77 
78 static struct intel_uncore_discovery_type *
79 add_uncore_discovery_type(struct uncore_unit_discovery *unit)
80 {
81 	struct intel_uncore_discovery_type *type;
82 
83 	if (unit->access_type >= UNCORE_ACCESS_MAX) {
84 		pr_warn("Unsupported access type %d\n", unit->access_type);
85 		return NULL;
86 	}
87 
88 	type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL);
89 	if (!type)
90 		return NULL;
91 
92 	type->units = RB_ROOT;
93 
94 	type->access_type = unit->access_type;
95 	num_discovered_types[type->access_type]++;
96 	type->type = unit->box_type;
97 
98 	rb_add(&type->node, &discovery_tables, __type_less);
99 
100 	return type;
101 }
102 
103 static struct intel_uncore_discovery_type *
104 get_uncore_discovery_type(struct uncore_unit_discovery *unit)
105 {
106 	struct intel_uncore_discovery_type *type;
107 
108 	type = search_uncore_discovery_type(unit->box_type);
109 	if (type)
110 		return type;
111 
112 	return add_uncore_discovery_type(unit);
113 }
114 
115 static inline int pmu_idx_cmp(const void *key, const struct rb_node *b)
116 {
117 	struct intel_uncore_discovery_unit *unit;
118 	const unsigned int *id = key;
119 
120 	unit = rb_entry(b, struct intel_uncore_discovery_unit, node);
121 
122 	if (unit->pmu_idx > *id)
123 		return -1;
124 	else if (unit->pmu_idx < *id)
125 		return 1;
126 
127 	return 0;
128 }
129 
130 static struct intel_uncore_discovery_unit *
131 intel_uncore_find_discovery_unit(struct rb_root *units, int die,
132 				 unsigned int pmu_idx)
133 {
134 	struct intel_uncore_discovery_unit *unit;
135 	struct rb_node *pos;
136 
137 	if (!units)
138 		return NULL;
139 
140 	pos = rb_find_first(&pmu_idx, units, pmu_idx_cmp);
141 	if (!pos)
142 		return NULL;
143 	unit = rb_entry(pos, struct intel_uncore_discovery_unit, node);
144 
145 	if (die < 0)
146 		return unit;
147 
148 	for (; pos; pos = rb_next(pos)) {
149 		unit = rb_entry(pos, struct intel_uncore_discovery_unit, node);
150 
151 		if (unit->pmu_idx != pmu_idx)
152 			break;
153 
154 		if (unit->die == die)
155 			return unit;
156 	}
157 
158 	return NULL;
159 }
160 
161 int intel_uncore_find_discovery_unit_id(struct rb_root *units, int die,
162 					unsigned int pmu_idx)
163 {
164 	struct intel_uncore_discovery_unit *unit;
165 
166 	unit = intel_uncore_find_discovery_unit(units, die, pmu_idx);
167 	if (unit)
168 		return unit->id;
169 
170 	return -1;
171 }
172 
173 static inline bool unit_less(struct rb_node *a, const struct rb_node *b)
174 {
175 	struct intel_uncore_discovery_unit *a_node, *b_node;
176 
177 	a_node = rb_entry(a, struct intel_uncore_discovery_unit, node);
178 	b_node = rb_entry(b, struct intel_uncore_discovery_unit, node);
179 
180 	if (a_node->pmu_idx < b_node->pmu_idx)
181 		return true;
182 	if (a_node->pmu_idx > b_node->pmu_idx)
183 		return false;
184 
185 	if (a_node->die < b_node->die)
186 		return true;
187 	if (a_node->die > b_node->die)
188 		return false;
189 
190 	return 0;
191 }
192 
193 static inline struct intel_uncore_discovery_unit *
194 uncore_find_unit(struct rb_root *root, unsigned int id)
195 {
196 	struct intel_uncore_discovery_unit *unit;
197 	struct rb_node *node;
198 
199 	for (node = rb_first(root); node; node = rb_next(node)) {
200 		unit = rb_entry(node, struct intel_uncore_discovery_unit, node);
201 		if (unit->id == id)
202 			return unit;
203 	}
204 
205 	return NULL;
206 }
207 
208 void uncore_find_add_unit(struct intel_uncore_discovery_unit *node,
209 			  struct rb_root *root, u16 *num_units)
210 {
211 	struct intel_uncore_discovery_unit *unit = uncore_find_unit(root, node->id);
212 
213 	if (unit)
214 		node->pmu_idx = unit->pmu_idx;
215 	else if (num_units)
216 		node->pmu_idx = (*num_units)++;
217 
218 	rb_add(&node->node, root, unit_less);
219 }
220 
221 static void
222 uncore_insert_box_info(struct uncore_unit_discovery *unit,
223 		       int die)
224 {
225 	struct intel_uncore_discovery_unit *node;
226 	struct intel_uncore_discovery_type *type;
227 
228 	if (!unit->ctl || !unit->ctl_offset || !unit->ctr_offset) {
229 		pr_info("Invalid address is detected for uncore type %d box %d, "
230 			"Disable the uncore unit.\n",
231 			unit->box_type, unit->box_id);
232 		return;
233 	}
234 
235 	node = kzalloc(sizeof(*node), GFP_KERNEL);
236 	if (!node)
237 		return;
238 
239 	node->die = die;
240 	node->id = unit->box_id;
241 	node->addr = unit->ctl;
242 
243 	type = get_uncore_discovery_type(unit);
244 	if (!type) {
245 		kfree(node);
246 		return;
247 	}
248 
249 	uncore_find_add_unit(node, &type->units, &type->num_units);
250 
251 	/* Store generic information for the first box */
252 	if (type->num_units == 1) {
253 		type->num_counters = unit->num_regs;
254 		type->counter_width = unit->bit_width;
255 		type->ctl_offset = unit->ctl_offset;
256 		type->ctr_offset = unit->ctr_offset;
257 	}
258 }
259 
260 static bool
261 uncore_ignore_unit(struct uncore_unit_discovery *unit, int *ignore)
262 {
263 	int i;
264 
265 	if (!ignore)
266 		return false;
267 
268 	for (i = 0; ignore[i] != UNCORE_IGNORE_END ; i++) {
269 		if (unit->box_type == ignore[i])
270 			return true;
271 	}
272 
273 	return false;
274 }
275 
276 static int parse_discovery_table(struct pci_dev *dev, int die,
277 				 u32 bar_offset, bool *parsed,
278 				 int *ignore)
279 {
280 	struct uncore_global_discovery global;
281 	struct uncore_unit_discovery unit;
282 	void __iomem *io_addr;
283 	resource_size_t addr;
284 	unsigned long size;
285 	u32 val;
286 	int i;
287 
288 	pci_read_config_dword(dev, bar_offset, &val);
289 
290 	if (val & ~PCI_BASE_ADDRESS_MEM_MASK & ~PCI_BASE_ADDRESS_MEM_TYPE_64)
291 		return -EINVAL;
292 
293 	addr = (resource_size_t)(val & PCI_BASE_ADDRESS_MEM_MASK);
294 #ifdef CONFIG_PHYS_ADDR_T_64BIT
295 	if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) {
296 		u32 val2;
297 
298 		pci_read_config_dword(dev, bar_offset + 4, &val2);
299 		addr |= ((resource_size_t)val2) << 32;
300 	}
301 #endif
302 	size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE;
303 	io_addr = ioremap(addr, size);
304 	if (!io_addr)
305 		return -ENOMEM;
306 
307 	/* Read Global Discovery State */
308 	memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery));
309 	if (uncore_discovery_invalid_unit(global)) {
310 		pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n",
311 			global.table1, global.ctl, global.table3);
312 		iounmap(io_addr);
313 		return -EINVAL;
314 	}
315 	iounmap(io_addr);
316 
317 	size = (1 + global.max_units) * global.stride * 8;
318 	io_addr = ioremap(addr, size);
319 	if (!io_addr)
320 		return -ENOMEM;
321 
322 	/* Parsing Unit Discovery State */
323 	for (i = 0; i < global.max_units; i++) {
324 		memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8),
325 			      sizeof(struct uncore_unit_discovery));
326 
327 		if (uncore_discovery_invalid_unit(unit))
328 			continue;
329 
330 		if (unit.access_type >= UNCORE_ACCESS_MAX)
331 			continue;
332 
333 		if (uncore_ignore_unit(&unit, ignore))
334 			continue;
335 
336 		uncore_insert_box_info(&unit, die);
337 	}
338 
339 	*parsed = true;
340 	iounmap(io_addr);
341 	return 0;
342 }
343 
344 bool intel_uncore_has_discovery_tables(int *ignore)
345 {
346 	u32 device, val, entry_id, bar_offset;
347 	int die, dvsec = 0, ret = true;
348 	struct pci_dev *dev = NULL;
349 	bool parsed = false;
350 
351 	if (has_generic_discovery_table())
352 		device = UNCORE_DISCOVERY_TABLE_DEVICE;
353 	else
354 		device = PCI_ANY_ID;
355 
356 	/*
357 	 * Start a new search and iterates through the list of
358 	 * the discovery table devices.
359 	 */
360 	while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) {
361 		while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) {
362 			pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val);
363 			entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK;
364 			if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON)
365 				continue;
366 
367 			pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val);
368 
369 			if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) {
370 				ret = false;
371 				goto err;
372 			}
373 			bar_offset = UNCORE_DISCOVERY_BIR_BASE +
374 				     (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP;
375 
376 			die = get_device_die_id(dev);
377 			if (die < 0)
378 				continue;
379 
380 			parse_discovery_table(dev, die, bar_offset, &parsed, ignore);
381 		}
382 	}
383 
384 	/* None of the discovery tables are available */
385 	if (!parsed)
386 		ret = false;
387 err:
388 	pci_dev_put(dev);
389 
390 	return ret;
391 }
392 
393 void intel_uncore_clear_discovery_tables(void)
394 {
395 	struct intel_uncore_discovery_type *type, *next;
396 	struct intel_uncore_discovery_unit *pos;
397 	struct rb_node *node;
398 
399 	rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) {
400 		while (!RB_EMPTY_ROOT(&type->units)) {
401 			node = rb_first(&type->units);
402 			pos = rb_entry(node, struct intel_uncore_discovery_unit, node);
403 			rb_erase(node, &type->units);
404 			kfree(pos);
405 		}
406 		kfree(type);
407 	}
408 }
409 
410 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
411 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
412 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
413 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
414 DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31");
415 
416 static struct attribute *generic_uncore_formats_attr[] = {
417 	&format_attr_event.attr,
418 	&format_attr_umask.attr,
419 	&format_attr_edge.attr,
420 	&format_attr_inv.attr,
421 	&format_attr_thresh.attr,
422 	NULL,
423 };
424 
425 static const struct attribute_group generic_uncore_format_group = {
426 	.name = "format",
427 	.attrs = generic_uncore_formats_attr,
428 };
429 
430 static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box)
431 {
432 	struct intel_uncore_discovery_unit *unit;
433 
434 	unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes,
435 						-1, box->pmu->pmu_idx);
436 	if (WARN_ON_ONCE(!unit))
437 		return 0;
438 
439 	return unit->addr;
440 }
441 
442 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
443 {
444 	wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
445 }
446 
447 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
448 {
449 	wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
450 }
451 
452 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box)
453 {
454 	wrmsrl(intel_generic_uncore_box_ctl(box), 0);
455 }
456 
457 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
458 					    struct perf_event *event)
459 {
460 	struct hw_perf_event *hwc = &event->hw;
461 
462 	wrmsrl(hwc->config_base, hwc->config);
463 }
464 
465 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
466 					     struct perf_event *event)
467 {
468 	struct hw_perf_event *hwc = &event->hw;
469 
470 	wrmsrl(hwc->config_base, 0);
471 }
472 
473 static struct intel_uncore_ops generic_uncore_msr_ops = {
474 	.init_box		= intel_generic_uncore_msr_init_box,
475 	.disable_box		= intel_generic_uncore_msr_disable_box,
476 	.enable_box		= intel_generic_uncore_msr_enable_box,
477 	.disable_event		= intel_generic_uncore_msr_disable_event,
478 	.enable_event		= intel_generic_uncore_msr_enable_event,
479 	.read_counter		= uncore_msr_read_counter,
480 };
481 
482 bool intel_generic_uncore_assign_hw_event(struct perf_event *event,
483 					  struct intel_uncore_box *box)
484 {
485 	struct hw_perf_event *hwc = &event->hw;
486 	u64 box_ctl;
487 
488 	if (!box->pmu->type->boxes)
489 		return false;
490 
491 	if (box->io_addr) {
492 		hwc->config_base = uncore_pci_event_ctl(box, hwc->idx);
493 		hwc->event_base  = uncore_pci_perf_ctr(box, hwc->idx);
494 		return true;
495 	}
496 
497 	box_ctl = intel_generic_uncore_box_ctl(box);
498 	if (!box_ctl)
499 		return false;
500 
501 	if (box->pci_dev) {
502 		box_ctl = UNCORE_DISCOVERY_PCI_BOX_CTRL(box_ctl);
503 		hwc->config_base = box_ctl + uncore_pci_event_ctl(box, hwc->idx);
504 		hwc->event_base  = box_ctl + uncore_pci_perf_ctr(box, hwc->idx);
505 		return true;
506 	}
507 
508 	hwc->config_base = box_ctl + box->pmu->type->event_ctl + hwc->idx;
509 	hwc->event_base  = box_ctl + box->pmu->type->perf_ctr + hwc->idx;
510 
511 	return true;
512 }
513 
514 static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box)
515 {
516 	return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box));
517 }
518 
519 void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
520 {
521 	struct pci_dev *pdev = box->pci_dev;
522 	int box_ctl = intel_pci_uncore_box_ctl(box);
523 
524 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
525 	pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT);
526 }
527 
528 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box)
529 {
530 	struct pci_dev *pdev = box->pci_dev;
531 	int box_ctl = intel_pci_uncore_box_ctl(box);
532 
533 	pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ);
534 }
535 
536 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box)
537 {
538 	struct pci_dev *pdev = box->pci_dev;
539 	int box_ctl = intel_pci_uncore_box_ctl(box);
540 
541 	pci_write_config_dword(pdev, box_ctl, 0);
542 }
543 
544 static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box,
545 					    struct perf_event *event)
546 {
547 	struct pci_dev *pdev = box->pci_dev;
548 	struct hw_perf_event *hwc = &event->hw;
549 
550 	pci_write_config_dword(pdev, hwc->config_base, hwc->config);
551 }
552 
553 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box,
554 					    struct perf_event *event)
555 {
556 	struct pci_dev *pdev = box->pci_dev;
557 	struct hw_perf_event *hwc = &event->hw;
558 
559 	pci_write_config_dword(pdev, hwc->config_base, 0);
560 }
561 
562 u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box,
563 					  struct perf_event *event)
564 {
565 	struct pci_dev *pdev = box->pci_dev;
566 	struct hw_perf_event *hwc = &event->hw;
567 	u64 count = 0;
568 
569 	pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
570 	pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
571 
572 	return count;
573 }
574 
575 static struct intel_uncore_ops generic_uncore_pci_ops = {
576 	.init_box	= intel_generic_uncore_pci_init_box,
577 	.disable_box	= intel_generic_uncore_pci_disable_box,
578 	.enable_box	= intel_generic_uncore_pci_enable_box,
579 	.disable_event	= intel_generic_uncore_pci_disable_event,
580 	.enable_event	= intel_generic_uncore_pci_enable_event,
581 	.read_counter	= intel_generic_uncore_pci_read_counter,
582 };
583 
584 #define UNCORE_GENERIC_MMIO_SIZE		0x4000
585 
586 void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
587 {
588 	static struct intel_uncore_discovery_unit *unit;
589 	struct intel_uncore_type *type = box->pmu->type;
590 	resource_size_t addr;
591 
592 	unit = intel_uncore_find_discovery_unit(type->boxes, box->dieid, box->pmu->pmu_idx);
593 	if (!unit) {
594 		pr_warn("Uncore type %d id %d: Cannot find box control address.\n",
595 			type->type_id, box->pmu->pmu_idx);
596 		return;
597 	}
598 
599 	if (!unit->addr) {
600 		pr_warn("Uncore type %d box %d: Invalid box control address.\n",
601 			type->type_id, unit->id);
602 		return;
603 	}
604 
605 	addr = unit->addr;
606 	box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE);
607 	if (!box->io_addr) {
608 		pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
609 			type->type_id, unit->id, (unsigned long long)addr);
610 		return;
611 	}
612 
613 	writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr);
614 }
615 
616 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box)
617 {
618 	if (!box->io_addr)
619 		return;
620 
621 	writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr);
622 }
623 
624 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box)
625 {
626 	if (!box->io_addr)
627 		return;
628 
629 	writel(0, box->io_addr);
630 }
631 
632 void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box,
633 					    struct perf_event *event)
634 {
635 	struct hw_perf_event *hwc = &event->hw;
636 
637 	if (!box->io_addr)
638 		return;
639 
640 	writel(hwc->config, box->io_addr + hwc->config_base);
641 }
642 
643 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
644 					     struct perf_event *event)
645 {
646 	struct hw_perf_event *hwc = &event->hw;
647 
648 	if (!box->io_addr)
649 		return;
650 
651 	writel(0, box->io_addr + hwc->config_base);
652 }
653 
654 static struct intel_uncore_ops generic_uncore_mmio_ops = {
655 	.init_box	= intel_generic_uncore_mmio_init_box,
656 	.exit_box	= uncore_mmio_exit_box,
657 	.disable_box	= intel_generic_uncore_mmio_disable_box,
658 	.enable_box	= intel_generic_uncore_mmio_enable_box,
659 	.disable_event	= intel_generic_uncore_mmio_disable_event,
660 	.enable_event	= intel_generic_uncore_mmio_enable_event,
661 	.read_counter	= uncore_mmio_read_counter,
662 };
663 
664 static bool uncore_update_uncore_type(enum uncore_access_type type_id,
665 				      struct intel_uncore_type *uncore,
666 				      struct intel_uncore_discovery_type *type)
667 {
668 	uncore->type_id = type->type;
669 	uncore->num_counters = type->num_counters;
670 	uncore->perf_ctr_bits = type->counter_width;
671 	uncore->perf_ctr = (unsigned int)type->ctr_offset;
672 	uncore->event_ctl = (unsigned int)type->ctl_offset;
673 	uncore->boxes = &type->units;
674 	uncore->num_boxes = type->num_units;
675 
676 	switch (type_id) {
677 	case UNCORE_ACCESS_MSR:
678 		uncore->ops = &generic_uncore_msr_ops;
679 		break;
680 	case UNCORE_ACCESS_PCI:
681 		uncore->ops = &generic_uncore_pci_ops;
682 		break;
683 	case UNCORE_ACCESS_MMIO:
684 		uncore->ops = &generic_uncore_mmio_ops;
685 		uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE;
686 		break;
687 	default:
688 		return false;
689 	}
690 
691 	return true;
692 }
693 
694 struct intel_uncore_type **
695 intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra)
696 {
697 	struct intel_uncore_discovery_type *type;
698 	struct intel_uncore_type **uncores;
699 	struct intel_uncore_type *uncore;
700 	struct rb_node *node;
701 	int i = 0;
702 
703 	uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1,
704 			  sizeof(struct intel_uncore_type *), GFP_KERNEL);
705 	if (!uncores)
706 		return empty_uncore;
707 
708 	for (node = rb_first(&discovery_tables); node; node = rb_next(node)) {
709 		type = rb_entry(node, struct intel_uncore_discovery_type, node);
710 		if (type->access_type != type_id)
711 			continue;
712 
713 		uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL);
714 		if (!uncore)
715 			break;
716 
717 		uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK;
718 		uncore->format_group = &generic_uncore_format_group;
719 
720 		if (!uncore_update_uncore_type(type_id, uncore, type)) {
721 			kfree(uncore);
722 			continue;
723 		}
724 		uncores[i++] = uncore;
725 	}
726 
727 	return uncores;
728 }
729 
730 void intel_uncore_generic_uncore_cpu_init(void)
731 {
732 	uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0);
733 }
734 
735 int intel_uncore_generic_uncore_pci_init(void)
736 {
737 	uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0);
738 
739 	return 0;
740 }
741 
742 void intel_uncore_generic_uncore_mmio_init(void)
743 {
744 	uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0);
745 }
746