1 // SPDX-License-Identifier: GPL-2.0
30 	INIT_LIST_HEAD(&evlist->entries);  in perf_evlist__init()
31 	evlist->nr_entries = 0;  in perf_evlist__init()
32 	fdarray__init(&evlist->pollfd, 64);  in perf_evlist__init()
39 	if (evsel->system_wide) {  in __perf_evlist__propagate_maps()
40 		/* System wide: set the cpu map of the evsel to all online CPUs. */  in __perf_evlist__propagate_maps()
41 		perf_cpu_map__put(evsel->cpus);  in __perf_evlist__propagate_maps()
42 		evsel->cpus = perf_cpu_map__new_online_cpus();  in __perf_evlist__propagate_maps()
43 	} else if (evlist->has_user_cpus && evsel->is_pmu_core) {  in __perf_evlist__propagate_maps()
48 		perf_cpu_map__put(evsel->cpus);  in __perf_evlist__propagate_maps()
49 		evsel->cpus = perf_cpu_map__intersect(evlist->user_requested_cpus, evsel->own_cpus);  in __perf_evlist__propagate_maps()
52 		 * Empty cpu lists would eventually get opened as "any" so remove  in __perf_evlist__propagate_maps()
55 		if (perf_cpu_map__is_empty(evsel->cpus)) {  in __perf_evlist__propagate_maps()
61 				list_for_each_entry_from(next, &evlist->entries, node)  in __perf_evlist__propagate_maps()
62 					next->idx--;  in __perf_evlist__propagate_maps()
64 	} else if (!evsel->own_cpus || evlist->has_user_cpus ||  in __perf_evlist__propagate_maps()
65 		(!evsel->requires_cpu && perf_cpu_map__has_any_cpu(evlist->user_requested_cpus))) {  in __perf_evlist__propagate_maps()
67 		 * The PMU didn't specify a default cpu map, this isn't a core  in __perf_evlist__propagate_maps()
69 		 * requested CPUs have the "any CPU" (aka dummy) CPU value. In  in __perf_evlist__propagate_maps()
73 		perf_cpu_map__put(evsel->cpus);  in __perf_evlist__propagate_maps()
74 		evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus);  in __perf_evlist__propagate_maps()
75 	} else if (evsel->cpus != evsel->own_cpus) {  in __perf_evlist__propagate_maps()
77 		 * No user requested cpu map but the PMU cpu map doesn't match  in __perf_evlist__propagate_maps()
78 		 * the evsel's. Reset it back to the PMU cpu map.  in __perf_evlist__propagate_maps()
80 		perf_cpu_map__put(evsel->cpus);  in __perf_evlist__propagate_maps()
81 		evsel->cpus = perf_cpu_map__get(evsel->own_cpus);  in __perf_evlist__propagate_maps()
84 	if (evsel->system_wide) {  in __perf_evlist__propagate_maps()
85 		perf_thread_map__put(evsel->threads);  in __perf_evlist__propagate_maps()
86 		evsel->threads = perf_thread_map__new_dummy();  in __perf_evlist__propagate_maps()
88 		perf_thread_map__put(evsel->threads);  in __perf_evlist__propagate_maps()
89 		evsel->threads = perf_thread_map__get(evlist->threads);  in __perf_evlist__propagate_maps()
92 	perf_cpu_map__merge(&evlist->all_cpus, evsel->cpus);  in __perf_evlist__propagate_maps()
99 	evlist->needs_map_propagation = true;  in perf_evlist__propagate_maps()
101 	list_for_each_entry_safe(evsel, n, &evlist->entries, node)  in perf_evlist__propagate_maps()
108 	evsel->idx = evlist->nr_entries;  in perf_evlist__add()
109 	list_add_tail(&evsel->node, &evlist->entries);  in perf_evlist__add()
110 	evlist->nr_entries += 1;  in perf_evlist__add()
112 	if (evlist->needs_map_propagation)  in perf_evlist__add()
119 	list_del_init(&evsel->node);  in perf_evlist__remove()
120 	evlist->nr_entries -= 1;  in perf_evlist__remove()
139 		next = list_first_entry(&evlist->entries,  in perf_evlist__next()
147 	if (&next->node == &evlist->entries)  in perf_evlist__next()
158 		list_del_init(&pos->node);  in perf_evlist__purge()
162 	evlist->nr_entries = 0;  in perf_evlist__purge()
167 	perf_cpu_map__put(evlist->user_requested_cpus);  in perf_evlist__exit()
168 	perf_cpu_map__put(evlist->all_cpus);  in perf_evlist__exit()
169 	perf_thread_map__put(evlist->threads);  in perf_evlist__exit()
170 	evlist->user_requested_cpus = NULL;  in perf_evlist__exit()
171 	evlist->all_cpus = NULL;  in perf_evlist__exit()
172 	evlist->threads = NULL;  in perf_evlist__exit()
173 	fdarray__exit(&evlist->pollfd);  in perf_evlist__exit()
199 	if (cpus != evlist->user_requested_cpus) {  in perf_evlist__set_maps()
200 		perf_cpu_map__put(evlist->user_requested_cpus);  in perf_evlist__set_maps()
201 		evlist->user_requested_cpus = perf_cpu_map__get(cpus);  in perf_evlist__set_maps()
204 	if (threads != evlist->threads) {  in perf_evlist__set_maps()
205 		perf_thread_map__put(evlist->threads);  in perf_evlist__set_maps()
206 		evlist->threads = perf_thread_map__get(threads);  in perf_evlist__set_maps()
218 		err = perf_evsel__open(evsel, evsel->cpus, evsel->threads);  in perf_evlist__open()
258 	return first->attr.read_format;  in perf_evlist__read_format()
261 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
270 	sid->id = id;  in perf_evlist__id_hash()
271 	sid->evsel = evsel;  in perf_evlist__id_hash()
272 	hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);  in perf_evlist__id_hash()
273 	hlist_add_head(&sid->node, &evlist->heads[hash]);  in perf_evlist__id_hash()
281 		INIT_HLIST_HEAD(&evlist->heads[i]);  in perf_evlist__reset_id_hash()
292 	evsel->id[evsel->ids++] = id;  in perf_evlist__id_add()
305 		return -1;  in perf_evlist__id_add_fd()
312 		return -1;  in perf_evlist__id_add_fd()
321 		return -1;  in perf_evlist__id_add_fd()
323 	if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||  in perf_evlist__id_add_fd()
324 	    read(fd, &read_data, sizeof(read_data)) == -1)  in perf_evlist__id_add_fd()
325 		return -1;  in perf_evlist__id_add_fd()
327 	if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)  in perf_evlist__id_add_fd()
329 	if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)  in perf_evlist__id_add_fd()
341 	int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);  in perf_evlist__alloc_pollfd()
342 	int nr_threads = perf_thread_map__nr(evlist->threads);  in perf_evlist__alloc_pollfd()
347 		if (evsel->system_wide)  in perf_evlist__alloc_pollfd()
353 	if (fdarray__available_entries(&evlist->pollfd) < nfds &&  in perf_evlist__alloc_pollfd()
354 	    fdarray__grow(&evlist->pollfd, nfds) < 0)  in perf_evlist__alloc_pollfd()
355 		return -ENOMEM;  in perf_evlist__alloc_pollfd()
363 	int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags);  in perf_evlist__add_pollfd()
366 		evlist->pollfd.priv[pos].ptr = ptr;  in perf_evlist__add_pollfd()
376 	struct perf_mmap *map = fda->priv[fd].ptr;  in perf_evlist__munmap_filtered()  local
378 	if (map)  in perf_evlist__munmap_filtered()
379 		perf_mmap__put(map);  in perf_evlist__munmap_filtered()
384 	return fdarray__filter(&evlist->pollfd, revents_and_mask,  in perf_evlist__filter_pollfd()
390 	return fdarray__poll(&evlist->pollfd, timeout);  in perf_evlist__poll()
396 	struct perf_mmap *map;  in perf_evlist__alloc_mmap()  local
398 	map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));  in perf_evlist__alloc_mmap()
399 	if (!map)  in perf_evlist__alloc_mmap()
402 	for (i = 0; i < evlist->nr_mmaps; i++) {  in perf_evlist__alloc_mmap()
403 		struct perf_mmap *prev = i ? &map[i - 1] : NULL;  in perf_evlist__alloc_mmap()
414 		perf_mmap__init(&map[i], prev, overwrite, NULL);  in perf_evlist__alloc_mmap()
417 	return map;  in perf_evlist__alloc_mmap()
420 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread)  in perf_evsel__set_sid_idx()  argument
422 	struct perf_sample_id *sid = SID(evsel, cpu, thread);  in perf_evsel__set_sid_idx()
424 	sid->idx = idx;  in perf_evsel__set_sid_idx()
425 	sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu);  in perf_evsel__set_sid_idx()
426 	sid->tid = perf_thread_map__pid(evsel->threads, thread);  in perf_evsel__set_sid_idx()
434 	maps = overwrite ? evlist->mmap_ovw : evlist->mmap;  in perf_evlist__mmap_cb_get()
442 			evlist->mmap_ovw = maps;  in perf_evlist__mmap_cb_get()
444 			evlist->mmap = maps;  in perf_evlist__mmap_cb_get()
450 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y))
453 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp,  in perf_evlist__mmap_cb_mmap()  argument
454 			  int output, struct perf_cpu cpu)  in perf_evlist__mmap_cb_mmap()  argument
456 	return perf_mmap__mmap(map, mp, output, cpu);  in perf_evlist__mmap_cb_mmap()
459 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map,  in perf_evlist__set_mmap_first()  argument
463 		evlist->mmap_ovw_first = map;  in perf_evlist__set_mmap_first()
465 		evlist->mmap_first = map;  in perf_evlist__set_mmap_first()
473 	struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->all_cpus, cpu_idx);  in mmap_per_evsel()
478 		bool overwrite = evsel->attr.write_backward;  in mmap_per_evsel()
480 		struct perf_mmap *map;  in mmap_per_evsel()  local
481 		int *output, fd, cpu;  in mmap_per_evsel()  local
483 		if (evsel->system_wide && thread)  in mmap_per_evsel()
486 		cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu);  in mmap_per_evsel()
487 		if (cpu == -1)  in mmap_per_evsel()
490 		map = ops->get(evlist, overwrite, idx);  in mmap_per_evsel()
491 		if (map == NULL)  in mmap_per_evsel()
492 			return -ENOMEM;  in mmap_per_evsel()
495 			mp->prot = PROT_READ;  in mmap_per_evsel()
498 			mp->prot = PROT_READ | PROT_WRITE;  in mmap_per_evsel()
502 		fd = FD(evsel, cpu, thread);  in mmap_per_evsel()
504 		if (*output == -1) {  in mmap_per_evsel()
520 			refcount_set(&map->refcnt, 2);  in mmap_per_evsel()
522 			if (ops->idx)  in mmap_per_evsel()
523 				ops->idx(evlist, evsel, mp, idx);  in mmap_per_evsel()
527 			if (ops->mmap(map, mp, *output, evlist_cpu) < 0)  in mmap_per_evsel()
528 				return -1;  in mmap_per_evsel()
533 				perf_evlist__set_mmap_first(evlist, map, overwrite);  in mmap_per_evsel()
536 			pr_debug("idx %d: set output fd %d -> %d\n", idx, fd, *output);  in mmap_per_evsel()
538 				return -1;  in mmap_per_evsel()
540 			perf_mmap__get(map);  in mmap_per_evsel()
545 		flgs = evsel->system_wide ? fdarray_flag__nonfilterable : fdarray_flag__default;  in mmap_per_evsel()
546 		if (perf_evlist__add_pollfd(evlist, fd, map, revent, flgs) < 0) {  in mmap_per_evsel()
547 			perf_mmap__put(map);  in mmap_per_evsel()
548 			return -1;  in mmap_per_evsel()
551 		if (evsel->attr.read_format & PERF_FORMAT_ID) {  in mmap_per_evsel()
552 			if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,  in mmap_per_evsel()
554 				return -1;  in mmap_per_evsel()
555 			perf_evsel__set_sid_idx(evsel, idx, cpu, thread);  in mmap_per_evsel()
566 	int nr_threads = perf_thread_map__nr(evlist->threads);  in mmap_per_thread()
567 	int nr_cpus    = perf_cpu_map__nr(evlist->all_cpus);  in mmap_per_thread()
568 	int cpu, thread, idx = 0;  in mmap_per_thread()  local
571 	pr_debug("%s: nr cpu values (may include -1) %d nr threads %d\n",  in mmap_per_thread()
574 	/* per-thread mmaps */  in mmap_per_thread()
576 		int output = -1;  in mmap_per_thread()
577 		int output_overwrite = -1;  in mmap_per_thread()
584 	/* system-wide mmaps i.e. per-cpu */  in mmap_per_thread()
585 	for (cpu = 1; cpu < nr_cpus; cpu++, idx++) {  in mmap_per_thread()
586 		int output = -1;  in mmap_per_thread()
587 		int output_overwrite = -1;  in mmap_per_thread()
589 		if (mmap_per_evsel(evlist, ops, idx, mp, cpu, 0, &output,  in mmap_per_thread()
594 	if (nr_mmaps != evlist->nr_mmaps)  in mmap_per_thread()
595 		pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);  in mmap_per_thread()
601 	return -1;  in mmap_per_thread()
608 	int nr_threads = perf_thread_map__nr(evlist->threads);  in mmap_per_cpu()
609 	int nr_cpus    = perf_cpu_map__nr(evlist->all_cpus);  in mmap_per_cpu()
611 	int cpu, thread;  in mmap_per_cpu()  local
613 	pr_debug("%s: nr cpu values %d nr threads %d\n", __func__, nr_cpus, nr_threads);  in mmap_per_cpu()
615 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in mmap_per_cpu()
616 		int output = -1;  in mmap_per_cpu()
617 		int output_overwrite = -1;  in mmap_per_cpu()
620 			if (mmap_per_evsel(evlist, ops, cpu, mp, cpu,  in mmap_per_cpu()
626 	if (nr_mmaps != evlist->nr_mmaps)  in mmap_per_cpu()
627 		pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);  in mmap_per_cpu()
633 	return -1;  in mmap_per_cpu()
640 	/* One for each CPU */  in perf_evlist__nr_mmaps()
641 	nr_mmaps = perf_cpu_map__nr(evlist->all_cpus);  in perf_evlist__nr_mmaps()
642 	if (perf_cpu_map__has_any_cpu_or_is_empty(evlist->all_cpus)) {  in perf_evlist__nr_mmaps()
644 		nr_mmaps += perf_thread_map__nr(evlist->threads);  in perf_evlist__nr_mmaps()
645 		/* Minus the per-thread CPU (-1) */  in perf_evlist__nr_mmaps()
646 		nr_mmaps -= 1;  in perf_evlist__nr_mmaps()
656 	const struct perf_cpu_map *cpus = evlist->all_cpus;  in perf_evlist__mmap_ops()
659 	if (!ops || !ops->get || !ops->mmap)  in perf_evlist__mmap_ops()
660 		return -EINVAL;  in perf_evlist__mmap_ops()
662 	mp->mask = evlist->mmap_len - page_size - 1;  in perf_evlist__mmap_ops()
664 	evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist);  in perf_evlist__mmap_ops()
667 		if ((evsel->attr.read_format & PERF_FORMAT_ID) &&  in perf_evlist__mmap_ops()
668 		    evsel->sample_id == NULL &&  in perf_evlist__mmap_ops()
669 		    perf_evsel__alloc_id(evsel, evsel->fd->max_x, evsel->fd->max_y) < 0)  in perf_evlist__mmap_ops()
670 			return -ENOMEM;  in perf_evlist__mmap_ops()
673 	if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)  in perf_evlist__mmap_ops()
674 		return -ENOMEM;  in perf_evlist__mmap_ops()
690 	evlist->mmap_len = (pages + 1) * page_size;  in perf_evlist__mmap()
699 	if (evlist->mmap) {  in perf_evlist__munmap()
700 		for (i = 0; i < evlist->nr_mmaps; i++)  in perf_evlist__munmap()
701 			perf_mmap__munmap(&evlist->mmap[i]);  in perf_evlist__munmap()
704 	if (evlist->mmap_ovw) {  in perf_evlist__munmap()
705 		for (i = 0; i < evlist->nr_mmaps; i++)  in perf_evlist__munmap()
706 			perf_mmap__munmap(&evlist->mmap_ovw[i]);  in perf_evlist__munmap()
709 	zfree(&evlist->mmap);  in perf_evlist__munmap()
710 	zfree(&evlist->mmap_ovw);  in perf_evlist__munmap()
714 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map,  in perf_evlist__next_mmap()  argument
717 	if (map)  in perf_evlist__next_mmap()
718 		return map->next;  in perf_evlist__next_mmap()
720 	return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first;  in perf_evlist__next_mmap()
729 		evsel->leader = leader;  in __perf_evlist__set_leader()
732 	leader->nr_members = n;  in __perf_evlist__set_leader()
737 	if (evlist->nr_entries) {  in perf_evlist__set_leader()
738 		struct perf_evsel *first = list_entry(evlist->entries.next,  in perf_evlist__set_leader()
741 		__perf_evlist__set_leader(&evlist->entries, first);  in perf_evlist__set_leader()
756 		if (evsel->leader == evsel && evsel->nr_members > 1)  in perf_evlist__nr_groups()
764 	if (!evsel->system_wide) {  in perf_evlist__go_system_wide()
765 		evsel->system_wide = true;  in perf_evlist__go_system_wide()
766 		if (evlist->needs_map_propagation)  in perf_evlist__go_system_wide()