xref: /linux/tools/perf/util/auxtrace.c (revision 39f75da7bcc829ddc4d40bb60d0e95520de7898b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * auxtrace.c: AUX area trace support
4  * Copyright (c) 2013-2015, Intel Corporation.
5  */
6 
7 #include <inttypes.h>
8 #include <sys/types.h>
9 #include <sys/mman.h>
10 #include <stdbool.h>
11 #include <string.h>
12 #include <limits.h>
13 #include <errno.h>
14 
15 #include <linux/kernel.h>
16 #include <linux/perf_event.h>
17 #include <linux/types.h>
18 #include <linux/bitops.h>
19 #include <linux/log2.h>
20 #include <linux/string.h>
21 #include <linux/time64.h>
22 
23 #include <sys/param.h>
24 #include <stdlib.h>
25 #include <stdio.h>
26 #include <linux/list.h>
27 #include <linux/zalloc.h>
28 
29 #include "evlist.h"
30 #include "dso.h"
31 #include "map.h"
32 #include "pmu.h"
33 #include "evsel.h"
34 #include "evsel_config.h"
35 #include "symbol.h"
36 #include "util/perf_api_probe.h"
37 #include "util/synthetic-events.h"
38 #include "thread_map.h"
39 #include "asm/bug.h"
40 #include "auxtrace.h"
41 
42 #include <linux/hash.h>
43 
44 #include "event.h"
45 #include "record.h"
46 #include "session.h"
47 #include "debug.h"
48 #include <subcmd/parse-options.h>
49 
50 #include "cs-etm.h"
51 #include "intel-pt.h"
52 #include "intel-bts.h"
53 #include "arm-spe.h"
54 #include "s390-cpumsf.h"
55 #include "util/mmap.h"
56 
57 #include <linux/ctype.h>
58 #include "symbol/kallsyms.h"
59 #include <internal/lib.h>
60 
61 /*
62  * Make a group from 'leader' to 'last', requiring that the events were not
63  * already grouped to a different leader.
64  */
65 static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct evsel *last)
66 {
67 	struct evsel *evsel;
68 	bool grp;
69 
70 	if (!evsel__is_group_leader(leader))
71 		return -EINVAL;
72 
73 	grp = false;
74 	evlist__for_each_entry(evlist, evsel) {
75 		if (grp) {
76 			if (!(evsel__leader(evsel) == leader ||
77 			     (evsel__leader(evsel) == evsel &&
78 			      evsel->core.nr_members <= 1)))
79 				return -EINVAL;
80 		} else if (evsel == leader) {
81 			grp = true;
82 		}
83 		if (evsel == last)
84 			break;
85 	}
86 
87 	grp = false;
88 	evlist__for_each_entry(evlist, evsel) {
89 		if (grp) {
90 			if (!evsel__has_leader(evsel, leader)) {
91 				evsel__set_leader(evsel, leader);
92 				if (leader->core.nr_members < 1)
93 					leader->core.nr_members = 1;
94 				leader->core.nr_members += 1;
95 			}
96 		} else if (evsel == leader) {
97 			grp = true;
98 		}
99 		if (evsel == last)
100 			break;
101 	}
102 
103 	return 0;
104 }
105 
106 static bool auxtrace__dont_decode(struct perf_session *session)
107 {
108 	return !session->itrace_synth_opts ||
109 	       session->itrace_synth_opts->dont_decode;
110 }
111 
112 int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
113 			struct auxtrace_mmap_params *mp,
114 			void *userpg, int fd)
115 {
116 	struct perf_event_mmap_page *pc = userpg;
117 
118 	WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");
119 
120 	mm->userpg = userpg;
121 	mm->mask = mp->mask;
122 	mm->len = mp->len;
123 	mm->prev = 0;
124 	mm->idx = mp->idx;
125 	mm->tid = mp->tid;
126 	mm->cpu = mp->cpu;
127 
128 	if (!mp->len) {
129 		mm->base = NULL;
130 		return 0;
131 	}
132 
133 #if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
134 	pr_err("Cannot use AUX area tracing mmaps\n");
135 	return -1;
136 #endif
137 
138 	pc->aux_offset = mp->offset;
139 	pc->aux_size = mp->len;
140 
141 	mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);
142 	if (mm->base == MAP_FAILED) {
143 		pr_debug2("failed to mmap AUX area\n");
144 		mm->base = NULL;
145 		return -1;
146 	}
147 
148 	return 0;
149 }
150 
151 void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)
152 {
153 	if (mm->base) {
154 		munmap(mm->base, mm->len);
155 		mm->base = NULL;
156 	}
157 }
158 
159 void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
160 				off_t auxtrace_offset,
161 				unsigned int auxtrace_pages,
162 				bool auxtrace_overwrite)
163 {
164 	if (auxtrace_pages) {
165 		mp->offset = auxtrace_offset;
166 		mp->len = auxtrace_pages * (size_t)page_size;
167 		mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;
168 		mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);
169 		pr_debug2("AUX area mmap length %zu\n", mp->len);
170 	} else {
171 		mp->len = 0;
172 	}
173 }
174 
175 void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
176 				   struct evlist *evlist, int idx,
177 				   bool per_cpu)
178 {
179 	mp->idx = idx;
180 
181 	if (per_cpu) {
182 		mp->cpu = evlist->core.cpus->map[idx];
183 		if (evlist->core.threads)
184 			mp->tid = perf_thread_map__pid(evlist->core.threads, 0);
185 		else
186 			mp->tid = -1;
187 	} else {
188 		mp->cpu = -1;
189 		mp->tid = perf_thread_map__pid(evlist->core.threads, idx);
190 	}
191 }
192 
193 #define AUXTRACE_INIT_NR_QUEUES	32
194 
195 static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)
196 {
197 	struct auxtrace_queue *queue_array;
198 	unsigned int max_nr_queues, i;
199 
200 	max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);
201 	if (nr_queues > max_nr_queues)
202 		return NULL;
203 
204 	queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));
205 	if (!queue_array)
206 		return NULL;
207 
208 	for (i = 0; i < nr_queues; i++) {
209 		INIT_LIST_HEAD(&queue_array[i].head);
210 		queue_array[i].priv = NULL;
211 	}
212 
213 	return queue_array;
214 }
215 
216 int auxtrace_queues__init(struct auxtrace_queues *queues)
217 {
218 	queues->nr_queues = AUXTRACE_INIT_NR_QUEUES;
219 	queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);
220 	if (!queues->queue_array)
221 		return -ENOMEM;
222 	return 0;
223 }
224 
225 static int auxtrace_queues__grow(struct auxtrace_queues *queues,
226 				 unsigned int new_nr_queues)
227 {
228 	unsigned int nr_queues = queues->nr_queues;
229 	struct auxtrace_queue *queue_array;
230 	unsigned int i;
231 
232 	if (!nr_queues)
233 		nr_queues = AUXTRACE_INIT_NR_QUEUES;
234 
235 	while (nr_queues && nr_queues < new_nr_queues)
236 		nr_queues <<= 1;
237 
238 	if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)
239 		return -EINVAL;
240 
241 	queue_array = auxtrace_alloc_queue_array(nr_queues);
242 	if (!queue_array)
243 		return -ENOMEM;
244 
245 	for (i = 0; i < queues->nr_queues; i++) {
246 		list_splice_tail(&queues->queue_array[i].head,
247 				 &queue_array[i].head);
248 		queue_array[i].tid = queues->queue_array[i].tid;
249 		queue_array[i].cpu = queues->queue_array[i].cpu;
250 		queue_array[i].set = queues->queue_array[i].set;
251 		queue_array[i].priv = queues->queue_array[i].priv;
252 	}
253 
254 	queues->nr_queues = nr_queues;
255 	queues->queue_array = queue_array;
256 
257 	return 0;
258 }
259 
260 static void *auxtrace_copy_data(u64 size, struct perf_session *session)
261 {
262 	int fd = perf_data__fd(session->data);
263 	void *p;
264 	ssize_t ret;
265 
266 	if (size > SSIZE_MAX)
267 		return NULL;
268 
269 	p = malloc(size);
270 	if (!p)
271 		return NULL;
272 
273 	ret = readn(fd, p, size);
274 	if (ret != (ssize_t)size) {
275 		free(p);
276 		return NULL;
277 	}
278 
279 	return p;
280 }
281 
282 static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues,
283 					 unsigned int idx,
284 					 struct auxtrace_buffer *buffer)
285 {
286 	struct auxtrace_queue *queue;
287 	int err;
288 
289 	if (idx >= queues->nr_queues) {
290 		err = auxtrace_queues__grow(queues, idx + 1);
291 		if (err)
292 			return err;
293 	}
294 
295 	queue = &queues->queue_array[idx];
296 
297 	if (!queue->set) {
298 		queue->set = true;
299 		queue->tid = buffer->tid;
300 		queue->cpu = buffer->cpu;
301 	}
302 
303 	buffer->buffer_nr = queues->next_buffer_nr++;
304 
305 	list_add_tail(&buffer->list, &queue->head);
306 
307 	queues->new_data = true;
308 	queues->populated = true;
309 
310 	return 0;
311 }
312 
313 /* Limit buffers to 32MiB on 32-bit */
314 #define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)
315 
316 static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,
317 					 unsigned int idx,
318 					 struct auxtrace_buffer *buffer)
319 {
320 	u64 sz = buffer->size;
321 	bool consecutive = false;
322 	struct auxtrace_buffer *b;
323 	int err;
324 
325 	while (sz > BUFFER_LIMIT_FOR_32_BIT) {
326 		b = memdup(buffer, sizeof(struct auxtrace_buffer));
327 		if (!b)
328 			return -ENOMEM;
329 		b->size = BUFFER_LIMIT_FOR_32_BIT;
330 		b->consecutive = consecutive;
331 		err = auxtrace_queues__queue_buffer(queues, idx, b);
332 		if (err) {
333 			auxtrace_buffer__free(b);
334 			return err;
335 		}
336 		buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;
337 		sz -= BUFFER_LIMIT_FOR_32_BIT;
338 		consecutive = true;
339 	}
340 
341 	buffer->size = sz;
342 	buffer->consecutive = consecutive;
343 
344 	return 0;
345 }
346 
347 static bool filter_cpu(struct perf_session *session, int cpu)
348 {
349 	unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap;
350 
351 	return cpu_bitmap && cpu != -1 && !test_bit(cpu, cpu_bitmap);
352 }
353 
354 static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,
355 				       struct perf_session *session,
356 				       unsigned int idx,
357 				       struct auxtrace_buffer *buffer,
358 				       struct auxtrace_buffer **buffer_ptr)
359 {
360 	int err = -ENOMEM;
361 
362 	if (filter_cpu(session, buffer->cpu))
363 		return 0;
364 
365 	buffer = memdup(buffer, sizeof(*buffer));
366 	if (!buffer)
367 		return -ENOMEM;
368 
369 	if (session->one_mmap) {
370 		buffer->data = buffer->data_offset - session->one_mmap_offset +
371 			       session->one_mmap_addr;
372 	} else if (perf_data__is_pipe(session->data)) {
373 		buffer->data = auxtrace_copy_data(buffer->size, session);
374 		if (!buffer->data)
375 			goto out_free;
376 		buffer->data_needs_freeing = true;
377 	} else if (BITS_PER_LONG == 32 &&
378 		   buffer->size > BUFFER_LIMIT_FOR_32_BIT) {
379 		err = auxtrace_queues__split_buffer(queues, idx, buffer);
380 		if (err)
381 			goto out_free;
382 	}
383 
384 	err = auxtrace_queues__queue_buffer(queues, idx, buffer);
385 	if (err)
386 		goto out_free;
387 
388 	/* FIXME: Doesn't work for split buffer */
389 	if (buffer_ptr)
390 		*buffer_ptr = buffer;
391 
392 	return 0;
393 
394 out_free:
395 	auxtrace_buffer__free(buffer);
396 	return err;
397 }
398 
399 int auxtrace_queues__add_event(struct auxtrace_queues *queues,
400 			       struct perf_session *session,
401 			       union perf_event *event, off_t data_offset,
402 			       struct auxtrace_buffer **buffer_ptr)
403 {
404 	struct auxtrace_buffer buffer = {
405 		.pid = -1,
406 		.tid = event->auxtrace.tid,
407 		.cpu = event->auxtrace.cpu,
408 		.data_offset = data_offset,
409 		.offset = event->auxtrace.offset,
410 		.reference = event->auxtrace.reference,
411 		.size = event->auxtrace.size,
412 	};
413 	unsigned int idx = event->auxtrace.idx;
414 
415 	return auxtrace_queues__add_buffer(queues, session, idx, &buffer,
416 					   buffer_ptr);
417 }
418 
419 static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,
420 					      struct perf_session *session,
421 					      off_t file_offset, size_t sz)
422 {
423 	union perf_event *event;
424 	int err;
425 	char buf[PERF_SAMPLE_MAX_SIZE];
426 
427 	err = perf_session__peek_event(session, file_offset, buf,
428 				       PERF_SAMPLE_MAX_SIZE, &event, NULL);
429 	if (err)
430 		return err;
431 
432 	if (event->header.type == PERF_RECORD_AUXTRACE) {
433 		if (event->header.size < sizeof(struct perf_record_auxtrace) ||
434 		    event->header.size != sz) {
435 			err = -EINVAL;
436 			goto out;
437 		}
438 		file_offset += event->header.size;
439 		err = auxtrace_queues__add_event(queues, session, event,
440 						 file_offset, NULL);
441 	}
442 out:
443 	return err;
444 }
445 
446 void auxtrace_queues__free(struct auxtrace_queues *queues)
447 {
448 	unsigned int i;
449 
450 	for (i = 0; i < queues->nr_queues; i++) {
451 		while (!list_empty(&queues->queue_array[i].head)) {
452 			struct auxtrace_buffer *buffer;
453 
454 			buffer = list_entry(queues->queue_array[i].head.next,
455 					    struct auxtrace_buffer, list);
456 			list_del_init(&buffer->list);
457 			auxtrace_buffer__free(buffer);
458 		}
459 	}
460 
461 	zfree(&queues->queue_array);
462 	queues->nr_queues = 0;
463 }
464 
465 static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,
466 			     unsigned int pos, unsigned int queue_nr,
467 			     u64 ordinal)
468 {
469 	unsigned int parent;
470 
471 	while (pos) {
472 		parent = (pos - 1) >> 1;
473 		if (heap_array[parent].ordinal <= ordinal)
474 			break;
475 		heap_array[pos] = heap_array[parent];
476 		pos = parent;
477 	}
478 	heap_array[pos].queue_nr = queue_nr;
479 	heap_array[pos].ordinal = ordinal;
480 }
481 
482 int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
483 		       u64 ordinal)
484 {
485 	struct auxtrace_heap_item *heap_array;
486 
487 	if (queue_nr >= heap->heap_sz) {
488 		unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;
489 
490 		while (heap_sz <= queue_nr)
491 			heap_sz <<= 1;
492 		heap_array = realloc(heap->heap_array,
493 				     heap_sz * sizeof(struct auxtrace_heap_item));
494 		if (!heap_array)
495 			return -ENOMEM;
496 		heap->heap_array = heap_array;
497 		heap->heap_sz = heap_sz;
498 	}
499 
500 	auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);
501 
502 	return 0;
503 }
504 
505 void auxtrace_heap__free(struct auxtrace_heap *heap)
506 {
507 	zfree(&heap->heap_array);
508 	heap->heap_cnt = 0;
509 	heap->heap_sz = 0;
510 }
511 
512 void auxtrace_heap__pop(struct auxtrace_heap *heap)
513 {
514 	unsigned int pos, last, heap_cnt = heap->heap_cnt;
515 	struct auxtrace_heap_item *heap_array;
516 
517 	if (!heap_cnt)
518 		return;
519 
520 	heap->heap_cnt -= 1;
521 
522 	heap_array = heap->heap_array;
523 
524 	pos = 0;
525 	while (1) {
526 		unsigned int left, right;
527 
528 		left = (pos << 1) + 1;
529 		if (left >= heap_cnt)
530 			break;
531 		right = left + 1;
532 		if (right >= heap_cnt) {
533 			heap_array[pos] = heap_array[left];
534 			return;
535 		}
536 		if (heap_array[left].ordinal < heap_array[right].ordinal) {
537 			heap_array[pos] = heap_array[left];
538 			pos = left;
539 		} else {
540 			heap_array[pos] = heap_array[right];
541 			pos = right;
542 		}
543 	}
544 
545 	last = heap_cnt - 1;
546 	auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,
547 			 heap_array[last].ordinal);
548 }
549 
550 size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
551 				       struct evlist *evlist)
552 {
553 	if (itr)
554 		return itr->info_priv_size(itr, evlist);
555 	return 0;
556 }
557 
558 static int auxtrace_not_supported(void)
559 {
560 	pr_err("AUX area tracing is not supported on this architecture\n");
561 	return -EINVAL;
562 }
563 
564 int auxtrace_record__info_fill(struct auxtrace_record *itr,
565 			       struct perf_session *session,
566 			       struct perf_record_auxtrace_info *auxtrace_info,
567 			       size_t priv_size)
568 {
569 	if (itr)
570 		return itr->info_fill(itr, session, auxtrace_info, priv_size);
571 	return auxtrace_not_supported();
572 }
573 
574 void auxtrace_record__free(struct auxtrace_record *itr)
575 {
576 	if (itr)
577 		itr->free(itr);
578 }
579 
580 int auxtrace_record__snapshot_start(struct auxtrace_record *itr)
581 {
582 	if (itr && itr->snapshot_start)
583 		return itr->snapshot_start(itr);
584 	return 0;
585 }
586 
587 int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit)
588 {
589 	if (!on_exit && itr && itr->snapshot_finish)
590 		return itr->snapshot_finish(itr);
591 	return 0;
592 }
593 
594 int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
595 				   struct auxtrace_mmap *mm,
596 				   unsigned char *data, u64 *head, u64 *old)
597 {
598 	if (itr && itr->find_snapshot)
599 		return itr->find_snapshot(itr, idx, mm, data, head, old);
600 	return 0;
601 }
602 
603 int auxtrace_record__options(struct auxtrace_record *itr,
604 			     struct evlist *evlist,
605 			     struct record_opts *opts)
606 {
607 	if (itr) {
608 		itr->evlist = evlist;
609 		return itr->recording_options(itr, evlist, opts);
610 	}
611 	return 0;
612 }
613 
614 u64 auxtrace_record__reference(struct auxtrace_record *itr)
615 {
616 	if (itr)
617 		return itr->reference(itr);
618 	return 0;
619 }
620 
621 int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
622 				    struct record_opts *opts, const char *str)
623 {
624 	if (!str)
625 		return 0;
626 
627 	/* PMU-agnostic options */
628 	switch (*str) {
629 	case 'e':
630 		opts->auxtrace_snapshot_on_exit = true;
631 		str++;
632 		break;
633 	default:
634 		break;
635 	}
636 
637 	if (itr && itr->parse_snapshot_options)
638 		return itr->parse_snapshot_options(itr, opts, str);
639 
640 	pr_err("No AUX area tracing to snapshot\n");
641 	return -EINVAL;
642 }
643 
644 int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx)
645 {
646 	struct evsel *evsel;
647 
648 	if (!itr->evlist || !itr->pmu)
649 		return -EINVAL;
650 
651 	evlist__for_each_entry(itr->evlist, evsel) {
652 		if (evsel->core.attr.type == itr->pmu->type) {
653 			if (evsel->disabled)
654 				return 0;
655 			return evlist__enable_event_idx(itr->evlist, evsel, idx);
656 		}
657 	}
658 	return -EINVAL;
659 }
660 
661 /*
662  * Event record size is 16-bit which results in a maximum size of about 64KiB.
663  * Allow about 4KiB for the rest of the sample record, to give a maximum
664  * AUX area sample size of 60KiB.
665  */
666 #define MAX_AUX_SAMPLE_SIZE (60 * 1024)
667 
668 /* Arbitrary default size if no other default provided */
669 #define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)
670 
671 static int auxtrace_validate_aux_sample_size(struct evlist *evlist,
672 					     struct record_opts *opts)
673 {
674 	struct evsel *evsel;
675 	bool has_aux_leader = false;
676 	u32 sz;
677 
678 	evlist__for_each_entry(evlist, evsel) {
679 		sz = evsel->core.attr.aux_sample_size;
680 		if (evsel__is_group_leader(evsel)) {
681 			has_aux_leader = evsel__is_aux_event(evsel);
682 			if (sz) {
683 				if (has_aux_leader)
684 					pr_err("Cannot add AUX area sampling to an AUX area event\n");
685 				else
686 					pr_err("Cannot add AUX area sampling to a group leader\n");
687 				return -EINVAL;
688 			}
689 		}
690 		if (sz > MAX_AUX_SAMPLE_SIZE) {
691 			pr_err("AUX area sample size %u too big, max. %d\n",
692 			       sz, MAX_AUX_SAMPLE_SIZE);
693 			return -EINVAL;
694 		}
695 		if (sz) {
696 			if (!has_aux_leader) {
697 				pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");
698 				return -EINVAL;
699 			}
700 			evsel__set_sample_bit(evsel, AUX);
701 			opts->auxtrace_sample_mode = true;
702 		} else {
703 			evsel__reset_sample_bit(evsel, AUX);
704 		}
705 	}
706 
707 	if (!opts->auxtrace_sample_mode) {
708 		pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");
709 		return -EINVAL;
710 	}
711 
712 	if (!perf_can_aux_sample()) {
713 		pr_err("AUX area sampling is not supported by kernel\n");
714 		return -EINVAL;
715 	}
716 
717 	return 0;
718 }
719 
720 int auxtrace_parse_sample_options(struct auxtrace_record *itr,
721 				  struct evlist *evlist,
722 				  struct record_opts *opts, const char *str)
723 {
724 	struct evsel_config_term *term;
725 	struct evsel *aux_evsel;
726 	bool has_aux_sample_size = false;
727 	bool has_aux_leader = false;
728 	struct evsel *evsel;
729 	char *endptr;
730 	unsigned long sz;
731 
732 	if (!str)
733 		goto no_opt;
734 
735 	if (!itr) {
736 		pr_err("No AUX area event to sample\n");
737 		return -EINVAL;
738 	}
739 
740 	sz = strtoul(str, &endptr, 0);
741 	if (*endptr || sz > UINT_MAX) {
742 		pr_err("Bad AUX area sampling option: '%s'\n", str);
743 		return -EINVAL;
744 	}
745 
746 	if (!sz)
747 		sz = itr->default_aux_sample_size;
748 
749 	if (!sz)
750 		sz = DEFAULT_AUX_SAMPLE_SIZE;
751 
752 	/* Set aux_sample_size based on --aux-sample option */
753 	evlist__for_each_entry(evlist, evsel) {
754 		if (evsel__is_group_leader(evsel)) {
755 			has_aux_leader = evsel__is_aux_event(evsel);
756 		} else if (has_aux_leader) {
757 			evsel->core.attr.aux_sample_size = sz;
758 		}
759 	}
760 no_opt:
761 	aux_evsel = NULL;
762 	/* Override with aux_sample_size from config term */
763 	evlist__for_each_entry(evlist, evsel) {
764 		if (evsel__is_aux_event(evsel))
765 			aux_evsel = evsel;
766 		term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE);
767 		if (term) {
768 			has_aux_sample_size = true;
769 			evsel->core.attr.aux_sample_size = term->val.aux_sample_size;
770 			/* If possible, group with the AUX event */
771 			if (aux_evsel && evsel->core.attr.aux_sample_size)
772 				evlist__regroup(evlist, aux_evsel, evsel);
773 		}
774 	}
775 
776 	if (!str && !has_aux_sample_size)
777 		return 0;
778 
779 	if (!itr) {
780 		pr_err("No AUX area event to sample\n");
781 		return -EINVAL;
782 	}
783 
784 	return auxtrace_validate_aux_sample_size(evlist, opts);
785 }
786 
787 void auxtrace_regroup_aux_output(struct evlist *evlist)
788 {
789 	struct evsel *evsel, *aux_evsel = NULL;
790 	struct evsel_config_term *term;
791 
792 	evlist__for_each_entry(evlist, evsel) {
793 		if (evsel__is_aux_event(evsel))
794 			aux_evsel = evsel;
795 		term = evsel__get_config_term(evsel, AUX_OUTPUT);
796 		/* If possible, group with the AUX event */
797 		if (term && aux_evsel)
798 			evlist__regroup(evlist, aux_evsel, evsel);
799 	}
800 }
801 
802 struct auxtrace_record *__weak
803 auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err)
804 {
805 	*err = 0;
806 	return NULL;
807 }
808 
809 static int auxtrace_index__alloc(struct list_head *head)
810 {
811 	struct auxtrace_index *auxtrace_index;
812 
813 	auxtrace_index = malloc(sizeof(struct auxtrace_index));
814 	if (!auxtrace_index)
815 		return -ENOMEM;
816 
817 	auxtrace_index->nr = 0;
818 	INIT_LIST_HEAD(&auxtrace_index->list);
819 
820 	list_add_tail(&auxtrace_index->list, head);
821 
822 	return 0;
823 }
824 
825 void auxtrace_index__free(struct list_head *head)
826 {
827 	struct auxtrace_index *auxtrace_index, *n;
828 
829 	list_for_each_entry_safe(auxtrace_index, n, head, list) {
830 		list_del_init(&auxtrace_index->list);
831 		free(auxtrace_index);
832 	}
833 }
834 
835 static struct auxtrace_index *auxtrace_index__last(struct list_head *head)
836 {
837 	struct auxtrace_index *auxtrace_index;
838 	int err;
839 
840 	if (list_empty(head)) {
841 		err = auxtrace_index__alloc(head);
842 		if (err)
843 			return NULL;
844 	}
845 
846 	auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);
847 
848 	if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {
849 		err = auxtrace_index__alloc(head);
850 		if (err)
851 			return NULL;
852 		auxtrace_index = list_entry(head->prev, struct auxtrace_index,
853 					    list);
854 	}
855 
856 	return auxtrace_index;
857 }
858 
859 int auxtrace_index__auxtrace_event(struct list_head *head,
860 				   union perf_event *event, off_t file_offset)
861 {
862 	struct auxtrace_index *auxtrace_index;
863 	size_t nr;
864 
865 	auxtrace_index = auxtrace_index__last(head);
866 	if (!auxtrace_index)
867 		return -ENOMEM;
868 
869 	nr = auxtrace_index->nr;
870 	auxtrace_index->entries[nr].file_offset = file_offset;
871 	auxtrace_index->entries[nr].sz = event->header.size;
872 	auxtrace_index->nr += 1;
873 
874 	return 0;
875 }
876 
877 static int auxtrace_index__do_write(int fd,
878 				    struct auxtrace_index *auxtrace_index)
879 {
880 	struct auxtrace_index_entry ent;
881 	size_t i;
882 
883 	for (i = 0; i < auxtrace_index->nr; i++) {
884 		ent.file_offset = auxtrace_index->entries[i].file_offset;
885 		ent.sz = auxtrace_index->entries[i].sz;
886 		if (writen(fd, &ent, sizeof(ent)) != sizeof(ent))
887 			return -errno;
888 	}
889 	return 0;
890 }
891 
892 int auxtrace_index__write(int fd, struct list_head *head)
893 {
894 	struct auxtrace_index *auxtrace_index;
895 	u64 total = 0;
896 	int err;
897 
898 	list_for_each_entry(auxtrace_index, head, list)
899 		total += auxtrace_index->nr;
900 
901 	if (writen(fd, &total, sizeof(total)) != sizeof(total))
902 		return -errno;
903 
904 	list_for_each_entry(auxtrace_index, head, list) {
905 		err = auxtrace_index__do_write(fd, auxtrace_index);
906 		if (err)
907 			return err;
908 	}
909 
910 	return 0;
911 }
912 
913 static int auxtrace_index__process_entry(int fd, struct list_head *head,
914 					 bool needs_swap)
915 {
916 	struct auxtrace_index *auxtrace_index;
917 	struct auxtrace_index_entry ent;
918 	size_t nr;
919 
920 	if (readn(fd, &ent, sizeof(ent)) != sizeof(ent))
921 		return -1;
922 
923 	auxtrace_index = auxtrace_index__last(head);
924 	if (!auxtrace_index)
925 		return -1;
926 
927 	nr = auxtrace_index->nr;
928 	if (needs_swap) {
929 		auxtrace_index->entries[nr].file_offset =
930 						bswap_64(ent.file_offset);
931 		auxtrace_index->entries[nr].sz = bswap_64(ent.sz);
932 	} else {
933 		auxtrace_index->entries[nr].file_offset = ent.file_offset;
934 		auxtrace_index->entries[nr].sz = ent.sz;
935 	}
936 
937 	auxtrace_index->nr = nr + 1;
938 
939 	return 0;
940 }
941 
942 int auxtrace_index__process(int fd, u64 size, struct perf_session *session,
943 			    bool needs_swap)
944 {
945 	struct list_head *head = &session->auxtrace_index;
946 	u64 nr;
947 
948 	if (readn(fd, &nr, sizeof(u64)) != sizeof(u64))
949 		return -1;
950 
951 	if (needs_swap)
952 		nr = bswap_64(nr);
953 
954 	if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size)
955 		return -1;
956 
957 	while (nr--) {
958 		int err;
959 
960 		err = auxtrace_index__process_entry(fd, head, needs_swap);
961 		if (err)
962 			return -1;
963 	}
964 
965 	return 0;
966 }
967 
968 static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues,
969 						struct perf_session *session,
970 						struct auxtrace_index_entry *ent)
971 {
972 	return auxtrace_queues__add_indexed_event(queues, session,
973 						  ent->file_offset, ent->sz);
974 }
975 
976 int auxtrace_queues__process_index(struct auxtrace_queues *queues,
977 				   struct perf_session *session)
978 {
979 	struct auxtrace_index *auxtrace_index;
980 	struct auxtrace_index_entry *ent;
981 	size_t i;
982 	int err;
983 
984 	if (auxtrace__dont_decode(session))
985 		return 0;
986 
987 	list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
988 		for (i = 0; i < auxtrace_index->nr; i++) {
989 			ent = &auxtrace_index->entries[i];
990 			err = auxtrace_queues__process_index_entry(queues,
991 								   session,
992 								   ent);
993 			if (err)
994 				return err;
995 		}
996 	}
997 	return 0;
998 }
999 
1000 struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,
1001 					      struct auxtrace_buffer *buffer)
1002 {
1003 	if (buffer) {
1004 		if (list_is_last(&buffer->list, &queue->head))
1005 			return NULL;
1006 		return list_entry(buffer->list.next, struct auxtrace_buffer,
1007 				  list);
1008 	} else {
1009 		if (list_empty(&queue->head))
1010 			return NULL;
1011 		return list_entry(queue->head.next, struct auxtrace_buffer,
1012 				  list);
1013 	}
1014 }
1015 
1016 struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues,
1017 						     struct perf_sample *sample,
1018 						     struct perf_session *session)
1019 {
1020 	struct perf_sample_id *sid;
1021 	unsigned int idx;
1022 	u64 id;
1023 
1024 	id = sample->id;
1025 	if (!id)
1026 		return NULL;
1027 
1028 	sid = evlist__id2sid(session->evlist, id);
1029 	if (!sid)
1030 		return NULL;
1031 
1032 	idx = sid->idx;
1033 
1034 	if (idx >= queues->nr_queues)
1035 		return NULL;
1036 
1037 	return &queues->queue_array[idx];
1038 }
1039 
1040 int auxtrace_queues__add_sample(struct auxtrace_queues *queues,
1041 				struct perf_session *session,
1042 				struct perf_sample *sample, u64 data_offset,
1043 				u64 reference)
1044 {
1045 	struct auxtrace_buffer buffer = {
1046 		.pid = -1,
1047 		.data_offset = data_offset,
1048 		.reference = reference,
1049 		.size = sample->aux_sample.size,
1050 	};
1051 	struct perf_sample_id *sid;
1052 	u64 id = sample->id;
1053 	unsigned int idx;
1054 
1055 	if (!id)
1056 		return -EINVAL;
1057 
1058 	sid = evlist__id2sid(session->evlist, id);
1059 	if (!sid)
1060 		return -ENOENT;
1061 
1062 	idx = sid->idx;
1063 	buffer.tid = sid->tid;
1064 	buffer.cpu = sid->cpu;
1065 
1066 	return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL);
1067 }
1068 
1069 struct queue_data {
1070 	bool samples;
1071 	bool events;
1072 };
1073 
1074 static int auxtrace_queue_data_cb(struct perf_session *session,
1075 				  union perf_event *event, u64 offset,
1076 				  void *data)
1077 {
1078 	struct queue_data *qd = data;
1079 	struct perf_sample sample;
1080 	int err;
1081 
1082 	if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) {
1083 		if (event->header.size < sizeof(struct perf_record_auxtrace))
1084 			return -EINVAL;
1085 		offset += event->header.size;
1086 		return session->auxtrace->queue_data(session, NULL, event,
1087 						     offset);
1088 	}
1089 
1090 	if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE)
1091 		return 0;
1092 
1093 	err = evlist__parse_sample(session->evlist, event, &sample);
1094 	if (err)
1095 		return err;
1096 
1097 	if (!sample.aux_sample.size)
1098 		return 0;
1099 
1100 	offset += sample.aux_sample.data - (void *)event;
1101 
1102 	return session->auxtrace->queue_data(session, &sample, NULL, offset);
1103 }
1104 
1105 int auxtrace_queue_data(struct perf_session *session, bool samples, bool events)
1106 {
1107 	struct queue_data qd = {
1108 		.samples = samples,
1109 		.events = events,
1110 	};
1111 
1112 	if (auxtrace__dont_decode(session))
1113 		return 0;
1114 
1115 	if (!session->auxtrace || !session->auxtrace->queue_data)
1116 		return -EINVAL;
1117 
1118 	return perf_session__peek_events(session, session->header.data_offset,
1119 					 session->header.data_size,
1120 					 auxtrace_queue_data_cb, &qd);
1121 }
1122 
1123 void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw)
1124 {
1125 	int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ;
1126 	size_t adj = buffer->data_offset & (page_size - 1);
1127 	size_t size = buffer->size + adj;
1128 	off_t file_offset = buffer->data_offset - adj;
1129 	void *addr;
1130 
1131 	if (buffer->data)
1132 		return buffer->data;
1133 
1134 	addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset);
1135 	if (addr == MAP_FAILED)
1136 		return NULL;
1137 
1138 	buffer->mmap_addr = addr;
1139 	buffer->mmap_size = size;
1140 
1141 	buffer->data = addr + adj;
1142 
1143 	return buffer->data;
1144 }
1145 
1146 void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer)
1147 {
1148 	if (!buffer->data || !buffer->mmap_addr)
1149 		return;
1150 	munmap(buffer->mmap_addr, buffer->mmap_size);
1151 	buffer->mmap_addr = NULL;
1152 	buffer->mmap_size = 0;
1153 	buffer->data = NULL;
1154 	buffer->use_data = NULL;
1155 }
1156 
1157 void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer)
1158 {
1159 	auxtrace_buffer__put_data(buffer);
1160 	if (buffer->data_needs_freeing) {
1161 		buffer->data_needs_freeing = false;
1162 		zfree(&buffer->data);
1163 		buffer->use_data = NULL;
1164 		buffer->size = 0;
1165 	}
1166 }
1167 
1168 void auxtrace_buffer__free(struct auxtrace_buffer *buffer)
1169 {
1170 	auxtrace_buffer__drop_data(buffer);
1171 	free(buffer);
1172 }
1173 
1174 void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,
1175 			  int code, int cpu, pid_t pid, pid_t tid, u64 ip,
1176 			  const char *msg, u64 timestamp)
1177 {
1178 	size_t size;
1179 
1180 	memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error));
1181 
1182 	auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR;
1183 	auxtrace_error->type = type;
1184 	auxtrace_error->code = code;
1185 	auxtrace_error->cpu = cpu;
1186 	auxtrace_error->pid = pid;
1187 	auxtrace_error->tid = tid;
1188 	auxtrace_error->fmt = 1;
1189 	auxtrace_error->ip = ip;
1190 	auxtrace_error->time = timestamp;
1191 	strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG);
1192 
1193 	size = (void *)auxtrace_error->msg - (void *)auxtrace_error +
1194 	       strlen(auxtrace_error->msg) + 1;
1195 	auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64));
1196 }
1197 
1198 int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr,
1199 					 struct perf_tool *tool,
1200 					 struct perf_session *session,
1201 					 perf_event__handler_t process)
1202 {
1203 	union perf_event *ev;
1204 	size_t priv_size;
1205 	int err;
1206 
1207 	pr_debug2("Synthesizing auxtrace information\n");
1208 	priv_size = auxtrace_record__info_priv_size(itr, session->evlist);
1209 	ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size);
1210 	if (!ev)
1211 		return -ENOMEM;
1212 
1213 	ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO;
1214 	ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) +
1215 					priv_size;
1216 	err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info,
1217 					 priv_size);
1218 	if (err)
1219 		goto out_free;
1220 
1221 	err = process(tool, ev, NULL, NULL);
1222 out_free:
1223 	free(ev);
1224 	return err;
1225 }
1226 
1227 static void unleader_evsel(struct evlist *evlist, struct evsel *leader)
1228 {
1229 	struct evsel *new_leader = NULL;
1230 	struct evsel *evsel;
1231 
1232 	/* Find new leader for the group */
1233 	evlist__for_each_entry(evlist, evsel) {
1234 		if (!evsel__has_leader(evsel, leader) || evsel == leader)
1235 			continue;
1236 		if (!new_leader)
1237 			new_leader = evsel;
1238 		evsel__set_leader(evsel, new_leader);
1239 	}
1240 
1241 	/* Update group information */
1242 	if (new_leader) {
1243 		zfree(&new_leader->group_name);
1244 		new_leader->group_name = leader->group_name;
1245 		leader->group_name = NULL;
1246 
1247 		new_leader->core.nr_members = leader->core.nr_members - 1;
1248 		leader->core.nr_members = 1;
1249 	}
1250 }
1251 
1252 static void unleader_auxtrace(struct perf_session *session)
1253 {
1254 	struct evsel *evsel;
1255 
1256 	evlist__for_each_entry(session->evlist, evsel) {
1257 		if (auxtrace__evsel_is_auxtrace(session, evsel) &&
1258 		    evsel__is_group_leader(evsel)) {
1259 			unleader_evsel(session->evlist, evsel);
1260 		}
1261 	}
1262 }
1263 
1264 int perf_event__process_auxtrace_info(struct perf_session *session,
1265 				      union perf_event *event)
1266 {
1267 	enum auxtrace_type type = event->auxtrace_info.type;
1268 	int err;
1269 
1270 	if (dump_trace)
1271 		fprintf(stdout, " type: %u\n", type);
1272 
1273 	switch (type) {
1274 	case PERF_AUXTRACE_INTEL_PT:
1275 		err = intel_pt_process_auxtrace_info(event, session);
1276 		break;
1277 	case PERF_AUXTRACE_INTEL_BTS:
1278 		err = intel_bts_process_auxtrace_info(event, session);
1279 		break;
1280 	case PERF_AUXTRACE_ARM_SPE:
1281 		err = arm_spe_process_auxtrace_info(event, session);
1282 		break;
1283 	case PERF_AUXTRACE_CS_ETM:
1284 		err = cs_etm__process_auxtrace_info(event, session);
1285 		break;
1286 	case PERF_AUXTRACE_S390_CPUMSF:
1287 		err = s390_cpumsf_process_auxtrace_info(event, session);
1288 		break;
1289 	case PERF_AUXTRACE_UNKNOWN:
1290 	default:
1291 		return -EINVAL;
1292 	}
1293 
1294 	if (err)
1295 		return err;
1296 
1297 	unleader_auxtrace(session);
1298 
1299 	return 0;
1300 }
1301 
1302 s64 perf_event__process_auxtrace(struct perf_session *session,
1303 				 union perf_event *event)
1304 {
1305 	s64 err;
1306 
1307 	if (dump_trace)
1308 		fprintf(stdout, " size: %#"PRI_lx64"  offset: %#"PRI_lx64"  ref: %#"PRI_lx64"  idx: %u  tid: %d  cpu: %d\n",
1309 			event->auxtrace.size, event->auxtrace.offset,
1310 			event->auxtrace.reference, event->auxtrace.idx,
1311 			event->auxtrace.tid, event->auxtrace.cpu);
1312 
1313 	if (auxtrace__dont_decode(session))
1314 		return event->auxtrace.size;
1315 
1316 	if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE)
1317 		return -EINVAL;
1318 
1319 	err = session->auxtrace->process_auxtrace_event(session, event, session->tool);
1320 	if (err < 0)
1321 		return err;
1322 
1323 	return event->auxtrace.size;
1324 }
1325 
1326 #define PERF_ITRACE_DEFAULT_PERIOD_TYPE		PERF_ITRACE_PERIOD_NANOSECS
1327 #define PERF_ITRACE_DEFAULT_PERIOD		100000
1328 #define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ	16
1329 #define PERF_ITRACE_MAX_CALLCHAIN_SZ		1024
1330 #define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ	64
1331 #define PERF_ITRACE_MAX_LAST_BRANCH_SZ		1024
1332 
1333 void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,
1334 				    bool no_sample)
1335 {
1336 	synth_opts->branches = true;
1337 	synth_opts->transactions = true;
1338 	synth_opts->ptwrites = true;
1339 	synth_opts->pwr_events = true;
1340 	synth_opts->other_events = true;
1341 	synth_opts->errors = true;
1342 	synth_opts->flc = true;
1343 	synth_opts->llc = true;
1344 	synth_opts->tlb = true;
1345 	synth_opts->mem = true;
1346 	synth_opts->remote_access = true;
1347 
1348 	if (no_sample) {
1349 		synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS;
1350 		synth_opts->period = 1;
1351 		synth_opts->calls = true;
1352 	} else {
1353 		synth_opts->instructions = true;
1354 		synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE;
1355 		synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1356 	}
1357 	synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
1358 	synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
1359 	synth_opts->initial_skip = 0;
1360 }
1361 
1362 static int get_flag(const char **ptr, unsigned int *flags)
1363 {
1364 	while (1) {
1365 		char c = **ptr;
1366 
1367 		if (c >= 'a' && c <= 'z') {
1368 			*flags |= 1 << (c - 'a');
1369 			++*ptr;
1370 			return 0;
1371 		} else if (c == ' ') {
1372 			++*ptr;
1373 			continue;
1374 		} else {
1375 			return -1;
1376 		}
1377 	}
1378 }
1379 
1380 static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags)
1381 {
1382 	while (1) {
1383 		switch (**ptr) {
1384 		case '+':
1385 			++*ptr;
1386 			if (get_flag(ptr, plus_flags))
1387 				return -1;
1388 			break;
1389 		case '-':
1390 			++*ptr;
1391 			if (get_flag(ptr, minus_flags))
1392 				return -1;
1393 			break;
1394 		case ' ':
1395 			++*ptr;
1396 			break;
1397 		default:
1398 			return 0;
1399 		}
1400 	}
1401 }
1402 
1403 /*
1404  * Please check tools/perf/Documentation/perf-script.txt for information
1405  * about the options parsed here, which is introduced after this cset,
1406  * when support in 'perf script' for these options is introduced.
1407  */
1408 int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts,
1409 			       const char *str, int unset)
1410 {
1411 	const char *p;
1412 	char *endptr;
1413 	bool period_type_set = false;
1414 	bool period_set = false;
1415 
1416 	synth_opts->set = true;
1417 
1418 	if (unset) {
1419 		synth_opts->dont_decode = true;
1420 		return 0;
1421 	}
1422 
1423 	if (!str) {
1424 		itrace_synth_opts__set_default(synth_opts,
1425 					       synth_opts->default_no_sample);
1426 		return 0;
1427 	}
1428 
1429 	for (p = str; *p;) {
1430 		switch (*p++) {
1431 		case 'i':
1432 			synth_opts->instructions = true;
1433 			while (*p == ' ' || *p == ',')
1434 				p += 1;
1435 			if (isdigit(*p)) {
1436 				synth_opts->period = strtoull(p, &endptr, 10);
1437 				period_set = true;
1438 				p = endptr;
1439 				while (*p == ' ' || *p == ',')
1440 					p += 1;
1441 				switch (*p++) {
1442 				case 'i':
1443 					synth_opts->period_type =
1444 						PERF_ITRACE_PERIOD_INSTRUCTIONS;
1445 					period_type_set = true;
1446 					break;
1447 				case 't':
1448 					synth_opts->period_type =
1449 						PERF_ITRACE_PERIOD_TICKS;
1450 					period_type_set = true;
1451 					break;
1452 				case 'm':
1453 					synth_opts->period *= 1000;
1454 					/* Fall through */
1455 				case 'u':
1456 					synth_opts->period *= 1000;
1457 					/* Fall through */
1458 				case 'n':
1459 					if (*p++ != 's')
1460 						goto out_err;
1461 					synth_opts->period_type =
1462 						PERF_ITRACE_PERIOD_NANOSECS;
1463 					period_type_set = true;
1464 					break;
1465 				case '\0':
1466 					goto out;
1467 				default:
1468 					goto out_err;
1469 				}
1470 			}
1471 			break;
1472 		case 'b':
1473 			synth_opts->branches = true;
1474 			break;
1475 		case 'x':
1476 			synth_opts->transactions = true;
1477 			break;
1478 		case 'w':
1479 			synth_opts->ptwrites = true;
1480 			break;
1481 		case 'p':
1482 			synth_opts->pwr_events = true;
1483 			break;
1484 		case 'o':
1485 			synth_opts->other_events = true;
1486 			break;
1487 		case 'e':
1488 			synth_opts->errors = true;
1489 			if (get_flags(&p, &synth_opts->error_plus_flags,
1490 				      &synth_opts->error_minus_flags))
1491 				goto out_err;
1492 			break;
1493 		case 'd':
1494 			synth_opts->log = true;
1495 			if (get_flags(&p, &synth_opts->log_plus_flags,
1496 				      &synth_opts->log_minus_flags))
1497 				goto out_err;
1498 			break;
1499 		case 'c':
1500 			synth_opts->branches = true;
1501 			synth_opts->calls = true;
1502 			break;
1503 		case 'r':
1504 			synth_opts->branches = true;
1505 			synth_opts->returns = true;
1506 			break;
1507 		case 'G':
1508 		case 'g':
1509 			if (p[-1] == 'G')
1510 				synth_opts->add_callchain = true;
1511 			else
1512 				synth_opts->callchain = true;
1513 			synth_opts->callchain_sz =
1514 					PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
1515 			while (*p == ' ' || *p == ',')
1516 				p += 1;
1517 			if (isdigit(*p)) {
1518 				unsigned int val;
1519 
1520 				val = strtoul(p, &endptr, 10);
1521 				p = endptr;
1522 				if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ)
1523 					goto out_err;
1524 				synth_opts->callchain_sz = val;
1525 			}
1526 			break;
1527 		case 'L':
1528 		case 'l':
1529 			if (p[-1] == 'L')
1530 				synth_opts->add_last_branch = true;
1531 			else
1532 				synth_opts->last_branch = true;
1533 			synth_opts->last_branch_sz =
1534 					PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
1535 			while (*p == ' ' || *p == ',')
1536 				p += 1;
1537 			if (isdigit(*p)) {
1538 				unsigned int val;
1539 
1540 				val = strtoul(p, &endptr, 10);
1541 				p = endptr;
1542 				if (!val ||
1543 				    val > PERF_ITRACE_MAX_LAST_BRANCH_SZ)
1544 					goto out_err;
1545 				synth_opts->last_branch_sz = val;
1546 			}
1547 			break;
1548 		case 's':
1549 			synth_opts->initial_skip = strtoul(p, &endptr, 10);
1550 			if (p == endptr)
1551 				goto out_err;
1552 			p = endptr;
1553 			break;
1554 		case 'f':
1555 			synth_opts->flc = true;
1556 			break;
1557 		case 'm':
1558 			synth_opts->llc = true;
1559 			break;
1560 		case 't':
1561 			synth_opts->tlb = true;
1562 			break;
1563 		case 'a':
1564 			synth_opts->remote_access = true;
1565 			break;
1566 		case 'M':
1567 			synth_opts->mem = true;
1568 			break;
1569 		case 'q':
1570 			synth_opts->quick += 1;
1571 			break;
1572 		case 'Z':
1573 			synth_opts->timeless_decoding = true;
1574 			break;
1575 		case ' ':
1576 		case ',':
1577 			break;
1578 		default:
1579 			goto out_err;
1580 		}
1581 	}
1582 out:
1583 	if (synth_opts->instructions) {
1584 		if (!period_type_set)
1585 			synth_opts->period_type =
1586 					PERF_ITRACE_DEFAULT_PERIOD_TYPE;
1587 		if (!period_set)
1588 			synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1589 	}
1590 
1591 	return 0;
1592 
1593 out_err:
1594 	pr_err("Bad Instruction Tracing options '%s'\n", str);
1595 	return -EINVAL;
1596 }
1597 
1598 int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset)
1599 {
1600 	return itrace_do_parse_synth_opts(opt->value, str, unset);
1601 }
1602 
1603 static const char * const auxtrace_error_type_name[] = {
1604 	[PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace",
1605 };
1606 
1607 static const char *auxtrace_error_name(int type)
1608 {
1609 	const char *error_type_name = NULL;
1610 
1611 	if (type < PERF_AUXTRACE_ERROR_MAX)
1612 		error_type_name = auxtrace_error_type_name[type];
1613 	if (!error_type_name)
1614 		error_type_name = "unknown AUX";
1615 	return error_type_name;
1616 }
1617 
1618 size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp)
1619 {
1620 	struct perf_record_auxtrace_error *e = &event->auxtrace_error;
1621 	unsigned long long nsecs = e->time;
1622 	const char *msg = e->msg;
1623 	int ret;
1624 
1625 	ret = fprintf(fp, " %s error type %u",
1626 		      auxtrace_error_name(e->type), e->type);
1627 
1628 	if (e->fmt && nsecs) {
1629 		unsigned long secs = nsecs / NSEC_PER_SEC;
1630 
1631 		nsecs -= secs * NSEC_PER_SEC;
1632 		ret += fprintf(fp, " time %lu.%09llu", secs, nsecs);
1633 	} else {
1634 		ret += fprintf(fp, " time 0");
1635 	}
1636 
1637 	if (!e->fmt)
1638 		msg = (const char *)&e->time;
1639 
1640 	ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n",
1641 		       e->cpu, e->pid, e->tid, e->ip, e->code, msg);
1642 	return ret;
1643 }
1644 
1645 void perf_session__auxtrace_error_inc(struct perf_session *session,
1646 				      union perf_event *event)
1647 {
1648 	struct perf_record_auxtrace_error *e = &event->auxtrace_error;
1649 
1650 	if (e->type < PERF_AUXTRACE_ERROR_MAX)
1651 		session->evlist->stats.nr_auxtrace_errors[e->type] += 1;
1652 }
1653 
1654 void events_stats__auxtrace_error_warn(const struct events_stats *stats)
1655 {
1656 	int i;
1657 
1658 	for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) {
1659 		if (!stats->nr_auxtrace_errors[i])
1660 			continue;
1661 		ui__warning("%u %s errors\n",
1662 			    stats->nr_auxtrace_errors[i],
1663 			    auxtrace_error_name(i));
1664 	}
1665 }
1666 
1667 int perf_event__process_auxtrace_error(struct perf_session *session,
1668 				       union perf_event *event)
1669 {
1670 	if (auxtrace__dont_decode(session))
1671 		return 0;
1672 
1673 	perf_event__fprintf_auxtrace_error(event, stdout);
1674 	return 0;
1675 }
1676 
1677 static int __auxtrace_mmap__read(struct mmap *map,
1678 				 struct auxtrace_record *itr,
1679 				 struct perf_tool *tool, process_auxtrace_t fn,
1680 				 bool snapshot, size_t snapshot_size)
1681 {
1682 	struct auxtrace_mmap *mm = &map->auxtrace_mmap;
1683 	u64 head, old = mm->prev, offset, ref;
1684 	unsigned char *data = mm->base;
1685 	size_t size, head_off, old_off, len1, len2, padding;
1686 	union perf_event ev;
1687 	void *data1, *data2;
1688 
1689 	if (snapshot) {
1690 		head = auxtrace_mmap__read_snapshot_head(mm);
1691 		if (auxtrace_record__find_snapshot(itr, mm->idx, mm, data,
1692 						   &head, &old))
1693 			return -1;
1694 	} else {
1695 		head = auxtrace_mmap__read_head(mm);
1696 	}
1697 
1698 	if (old == head)
1699 		return 0;
1700 
1701 	pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n",
1702 		  mm->idx, old, head, head - old);
1703 
1704 	if (mm->mask) {
1705 		head_off = head & mm->mask;
1706 		old_off = old & mm->mask;
1707 	} else {
1708 		head_off = head % mm->len;
1709 		old_off = old % mm->len;
1710 	}
1711 
1712 	if (head_off > old_off)
1713 		size = head_off - old_off;
1714 	else
1715 		size = mm->len - (old_off - head_off);
1716 
1717 	if (snapshot && size > snapshot_size)
1718 		size = snapshot_size;
1719 
1720 	ref = auxtrace_record__reference(itr);
1721 
1722 	if (head > old || size <= head || mm->mask) {
1723 		offset = head - size;
1724 	} else {
1725 		/*
1726 		 * When the buffer size is not a power of 2, 'head' wraps at the
1727 		 * highest multiple of the buffer size, so we have to subtract
1728 		 * the remainder here.
1729 		 */
1730 		u64 rem = (0ULL - mm->len) % mm->len;
1731 
1732 		offset = head - size - rem;
1733 	}
1734 
1735 	if (size > head_off) {
1736 		len1 = size - head_off;
1737 		data1 = &data[mm->len - len1];
1738 		len2 = head_off;
1739 		data2 = &data[0];
1740 	} else {
1741 		len1 = size;
1742 		data1 = &data[head_off - len1];
1743 		len2 = 0;
1744 		data2 = NULL;
1745 	}
1746 
1747 	if (itr->alignment) {
1748 		unsigned int unwanted = len1 % itr->alignment;
1749 
1750 		len1 -= unwanted;
1751 		size -= unwanted;
1752 	}
1753 
1754 	/* padding must be written by fn() e.g. record__process_auxtrace() */
1755 	padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1);
1756 	if (padding)
1757 		padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding;
1758 
1759 	memset(&ev, 0, sizeof(ev));
1760 	ev.auxtrace.header.type = PERF_RECORD_AUXTRACE;
1761 	ev.auxtrace.header.size = sizeof(ev.auxtrace);
1762 	ev.auxtrace.size = size + padding;
1763 	ev.auxtrace.offset = offset;
1764 	ev.auxtrace.reference = ref;
1765 	ev.auxtrace.idx = mm->idx;
1766 	ev.auxtrace.tid = mm->tid;
1767 	ev.auxtrace.cpu = mm->cpu;
1768 
1769 	if (fn(tool, map, &ev, data1, len1, data2, len2))
1770 		return -1;
1771 
1772 	mm->prev = head;
1773 
1774 	if (!snapshot) {
1775 		auxtrace_mmap__write_tail(mm, head);
1776 		if (itr->read_finish) {
1777 			int err;
1778 
1779 			err = itr->read_finish(itr, mm->idx);
1780 			if (err < 0)
1781 				return err;
1782 		}
1783 	}
1784 
1785 	return 1;
1786 }
1787 
1788 int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,
1789 			struct perf_tool *tool, process_auxtrace_t fn)
1790 {
1791 	return __auxtrace_mmap__read(map, itr, tool, fn, false, 0);
1792 }
1793 
1794 int auxtrace_mmap__read_snapshot(struct mmap *map,
1795 				 struct auxtrace_record *itr,
1796 				 struct perf_tool *tool, process_auxtrace_t fn,
1797 				 size_t snapshot_size)
1798 {
1799 	return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size);
1800 }
1801 
1802 /**
1803  * struct auxtrace_cache - hash table to implement a cache
1804  * @hashtable: the hashtable
1805  * @sz: hashtable size (number of hlists)
1806  * @entry_size: size of an entry
1807  * @limit: limit the number of entries to this maximum, when reached the cache
1808  *         is dropped and caching begins again with an empty cache
1809  * @cnt: current number of entries
1810  * @bits: hashtable size (@sz = 2^@bits)
1811  */
1812 struct auxtrace_cache {
1813 	struct hlist_head *hashtable;
1814 	size_t sz;
1815 	size_t entry_size;
1816 	size_t limit;
1817 	size_t cnt;
1818 	unsigned int bits;
1819 };
1820 
1821 struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,
1822 					   unsigned int limit_percent)
1823 {
1824 	struct auxtrace_cache *c;
1825 	struct hlist_head *ht;
1826 	size_t sz, i;
1827 
1828 	c = zalloc(sizeof(struct auxtrace_cache));
1829 	if (!c)
1830 		return NULL;
1831 
1832 	sz = 1UL << bits;
1833 
1834 	ht = calloc(sz, sizeof(struct hlist_head));
1835 	if (!ht)
1836 		goto out_free;
1837 
1838 	for (i = 0; i < sz; i++)
1839 		INIT_HLIST_HEAD(&ht[i]);
1840 
1841 	c->hashtable = ht;
1842 	c->sz = sz;
1843 	c->entry_size = entry_size;
1844 	c->limit = (c->sz * limit_percent) / 100;
1845 	c->bits = bits;
1846 
1847 	return c;
1848 
1849 out_free:
1850 	free(c);
1851 	return NULL;
1852 }
1853 
1854 static void auxtrace_cache__drop(struct auxtrace_cache *c)
1855 {
1856 	struct auxtrace_cache_entry *entry;
1857 	struct hlist_node *tmp;
1858 	size_t i;
1859 
1860 	if (!c)
1861 		return;
1862 
1863 	for (i = 0; i < c->sz; i++) {
1864 		hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) {
1865 			hlist_del(&entry->hash);
1866 			auxtrace_cache__free_entry(c, entry);
1867 		}
1868 	}
1869 
1870 	c->cnt = 0;
1871 }
1872 
1873 void auxtrace_cache__free(struct auxtrace_cache *c)
1874 {
1875 	if (!c)
1876 		return;
1877 
1878 	auxtrace_cache__drop(c);
1879 	zfree(&c->hashtable);
1880 	free(c);
1881 }
1882 
1883 void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c)
1884 {
1885 	return malloc(c->entry_size);
1886 }
1887 
1888 void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused,
1889 				void *entry)
1890 {
1891 	free(entry);
1892 }
1893 
1894 int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,
1895 			struct auxtrace_cache_entry *entry)
1896 {
1897 	if (c->limit && ++c->cnt > c->limit)
1898 		auxtrace_cache__drop(c);
1899 
1900 	entry->key = key;
1901 	hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]);
1902 
1903 	return 0;
1904 }
1905 
1906 static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c,
1907 						       u32 key)
1908 {
1909 	struct auxtrace_cache_entry *entry;
1910 	struct hlist_head *hlist;
1911 	struct hlist_node *n;
1912 
1913 	if (!c)
1914 		return NULL;
1915 
1916 	hlist = &c->hashtable[hash_32(key, c->bits)];
1917 	hlist_for_each_entry_safe(entry, n, hlist, hash) {
1918 		if (entry->key == key) {
1919 			hlist_del(&entry->hash);
1920 			return entry;
1921 		}
1922 	}
1923 
1924 	return NULL;
1925 }
1926 
1927 void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key)
1928 {
1929 	struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key);
1930 
1931 	auxtrace_cache__free_entry(c, entry);
1932 }
1933 
1934 void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key)
1935 {
1936 	struct auxtrace_cache_entry *entry;
1937 	struct hlist_head *hlist;
1938 
1939 	if (!c)
1940 		return NULL;
1941 
1942 	hlist = &c->hashtable[hash_32(key, c->bits)];
1943 	hlist_for_each_entry(entry, hlist, hash) {
1944 		if (entry->key == key)
1945 			return entry;
1946 	}
1947 
1948 	return NULL;
1949 }
1950 
1951 static void addr_filter__free_str(struct addr_filter *filt)
1952 {
1953 	zfree(&filt->str);
1954 	filt->action   = NULL;
1955 	filt->sym_from = NULL;
1956 	filt->sym_to   = NULL;
1957 	filt->filename = NULL;
1958 }
1959 
1960 static struct addr_filter *addr_filter__new(void)
1961 {
1962 	struct addr_filter *filt = zalloc(sizeof(*filt));
1963 
1964 	if (filt)
1965 		INIT_LIST_HEAD(&filt->list);
1966 
1967 	return filt;
1968 }
1969 
1970 static void addr_filter__free(struct addr_filter *filt)
1971 {
1972 	if (filt)
1973 		addr_filter__free_str(filt);
1974 	free(filt);
1975 }
1976 
1977 static void addr_filters__add(struct addr_filters *filts,
1978 			      struct addr_filter *filt)
1979 {
1980 	list_add_tail(&filt->list, &filts->head);
1981 	filts->cnt += 1;
1982 }
1983 
1984 static void addr_filters__del(struct addr_filters *filts,
1985 			      struct addr_filter *filt)
1986 {
1987 	list_del_init(&filt->list);
1988 	filts->cnt -= 1;
1989 }
1990 
1991 void addr_filters__init(struct addr_filters *filts)
1992 {
1993 	INIT_LIST_HEAD(&filts->head);
1994 	filts->cnt = 0;
1995 }
1996 
1997 void addr_filters__exit(struct addr_filters *filts)
1998 {
1999 	struct addr_filter *filt, *n;
2000 
2001 	list_for_each_entry_safe(filt, n, &filts->head, list) {
2002 		addr_filters__del(filts, filt);
2003 		addr_filter__free(filt);
2004 	}
2005 }
2006 
2007 static int parse_num_or_str(char **inp, u64 *num, const char **str,
2008 			    const char *str_delim)
2009 {
2010 	*inp += strspn(*inp, " ");
2011 
2012 	if (isdigit(**inp)) {
2013 		char *endptr;
2014 
2015 		if (!num)
2016 			return -EINVAL;
2017 		errno = 0;
2018 		*num = strtoull(*inp, &endptr, 0);
2019 		if (errno)
2020 			return -errno;
2021 		if (endptr == *inp)
2022 			return -EINVAL;
2023 		*inp = endptr;
2024 	} else {
2025 		size_t n;
2026 
2027 		if (!str)
2028 			return -EINVAL;
2029 		*inp += strspn(*inp, " ");
2030 		*str = *inp;
2031 		n = strcspn(*inp, str_delim);
2032 		if (!n)
2033 			return -EINVAL;
2034 		*inp += n;
2035 		if (**inp) {
2036 			**inp = '\0';
2037 			*inp += 1;
2038 		}
2039 	}
2040 	return 0;
2041 }
2042 
2043 static int parse_action(struct addr_filter *filt)
2044 {
2045 	if (!strcmp(filt->action, "filter")) {
2046 		filt->start = true;
2047 		filt->range = true;
2048 	} else if (!strcmp(filt->action, "start")) {
2049 		filt->start = true;
2050 	} else if (!strcmp(filt->action, "stop")) {
2051 		filt->start = false;
2052 	} else if (!strcmp(filt->action, "tracestop")) {
2053 		filt->start = false;
2054 		filt->range = true;
2055 		filt->action += 5; /* Change 'tracestop' to 'stop' */
2056 	} else {
2057 		return -EINVAL;
2058 	}
2059 	return 0;
2060 }
2061 
2062 static int parse_sym_idx(char **inp, int *idx)
2063 {
2064 	*idx = -1;
2065 
2066 	*inp += strspn(*inp, " ");
2067 
2068 	if (**inp != '#')
2069 		return 0;
2070 
2071 	*inp += 1;
2072 
2073 	if (**inp == 'g' || **inp == 'G') {
2074 		*inp += 1;
2075 		*idx = 0;
2076 	} else {
2077 		unsigned long num;
2078 		char *endptr;
2079 
2080 		errno = 0;
2081 		num = strtoul(*inp, &endptr, 0);
2082 		if (errno)
2083 			return -errno;
2084 		if (endptr == *inp || num > INT_MAX)
2085 			return -EINVAL;
2086 		*inp = endptr;
2087 		*idx = num;
2088 	}
2089 
2090 	return 0;
2091 }
2092 
2093 static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx)
2094 {
2095 	int err = parse_num_or_str(inp, num, str, " ");
2096 
2097 	if (!err && *str)
2098 		err = parse_sym_idx(inp, idx);
2099 
2100 	return err;
2101 }
2102 
2103 static int parse_one_filter(struct addr_filter *filt, const char **filter_inp)
2104 {
2105 	char *fstr;
2106 	int err;
2107 
2108 	filt->str = fstr = strdup(*filter_inp);
2109 	if (!fstr)
2110 		return -ENOMEM;
2111 
2112 	err = parse_num_or_str(&fstr, NULL, &filt->action, " ");
2113 	if (err)
2114 		goto out_err;
2115 
2116 	err = parse_action(filt);
2117 	if (err)
2118 		goto out_err;
2119 
2120 	err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from,
2121 			      &filt->sym_from_idx);
2122 	if (err)
2123 		goto out_err;
2124 
2125 	fstr += strspn(fstr, " ");
2126 
2127 	if (*fstr == '/') {
2128 		fstr += 1;
2129 		err = parse_addr_size(&fstr, &filt->size, &filt->sym_to,
2130 				      &filt->sym_to_idx);
2131 		if (err)
2132 			goto out_err;
2133 		filt->range = true;
2134 	}
2135 
2136 	fstr += strspn(fstr, " ");
2137 
2138 	if (*fstr == '@') {
2139 		fstr += 1;
2140 		err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,");
2141 		if (err)
2142 			goto out_err;
2143 	}
2144 
2145 	fstr += strspn(fstr, " ,");
2146 
2147 	*filter_inp += fstr - filt->str;
2148 
2149 	return 0;
2150 
2151 out_err:
2152 	addr_filter__free_str(filt);
2153 
2154 	return err;
2155 }
2156 
2157 int addr_filters__parse_bare_filter(struct addr_filters *filts,
2158 				    const char *filter)
2159 {
2160 	struct addr_filter *filt;
2161 	const char *fstr = filter;
2162 	int err;
2163 
2164 	while (*fstr) {
2165 		filt = addr_filter__new();
2166 		err = parse_one_filter(filt, &fstr);
2167 		if (err) {
2168 			addr_filter__free(filt);
2169 			addr_filters__exit(filts);
2170 			return err;
2171 		}
2172 		addr_filters__add(filts, filt);
2173 	}
2174 
2175 	return 0;
2176 }
2177 
2178 struct sym_args {
2179 	const char	*name;
2180 	u64		start;
2181 	u64		size;
2182 	int		idx;
2183 	int		cnt;
2184 	bool		started;
2185 	bool		global;
2186 	bool		selected;
2187 	bool		duplicate;
2188 	bool		near;
2189 };
2190 
2191 static bool kern_sym_match(struct sym_args *args, const char *name, char type)
2192 {
2193 	/* A function with the same name, and global or the n'th found or any */
2194 	return kallsyms__is_function(type) &&
2195 	       !strcmp(name, args->name) &&
2196 	       ((args->global && isupper(type)) ||
2197 		(args->selected && ++(args->cnt) == args->idx) ||
2198 		(!args->global && !args->selected));
2199 }
2200 
2201 static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2202 {
2203 	struct sym_args *args = arg;
2204 
2205 	if (args->started) {
2206 		if (!args->size)
2207 			args->size = start - args->start;
2208 		if (args->selected) {
2209 			if (args->size)
2210 				return 1;
2211 		} else if (kern_sym_match(args, name, type)) {
2212 			args->duplicate = true;
2213 			return 1;
2214 		}
2215 	} else if (kern_sym_match(args, name, type)) {
2216 		args->started = true;
2217 		args->start = start;
2218 	}
2219 
2220 	return 0;
2221 }
2222 
2223 static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2224 {
2225 	struct sym_args *args = arg;
2226 
2227 	if (kern_sym_match(args, name, type)) {
2228 		pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2229 		       ++args->cnt, start, type, name);
2230 		args->near = true;
2231 	} else if (args->near) {
2232 		args->near = false;
2233 		pr_err("\t\twhich is near\t\t%s\n", name);
2234 	}
2235 
2236 	return 0;
2237 }
2238 
2239 static int sym_not_found_error(const char *sym_name, int idx)
2240 {
2241 	if (idx > 0) {
2242 		pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",
2243 		       idx, sym_name);
2244 	} else if (!idx) {
2245 		pr_err("Global symbol '%s' not found.\n", sym_name);
2246 	} else {
2247 		pr_err("Symbol '%s' not found.\n", sym_name);
2248 	}
2249 	pr_err("Note that symbols must be functions.\n");
2250 
2251 	return -EINVAL;
2252 }
2253 
2254 static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx)
2255 {
2256 	struct sym_args args = {
2257 		.name = sym_name,
2258 		.idx = idx,
2259 		.global = !idx,
2260 		.selected = idx > 0,
2261 	};
2262 	int err;
2263 
2264 	*start = 0;
2265 	*size = 0;
2266 
2267 	err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb);
2268 	if (err < 0) {
2269 		pr_err("Failed to parse /proc/kallsyms\n");
2270 		return err;
2271 	}
2272 
2273 	if (args.duplicate) {
2274 		pr_err("Multiple kernel symbols with name '%s'\n", sym_name);
2275 		args.cnt = 0;
2276 		kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb);
2277 		pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2278 		       sym_name);
2279 		pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2280 		return -EINVAL;
2281 	}
2282 
2283 	if (!args.started) {
2284 		pr_err("Kernel symbol lookup: ");
2285 		return sym_not_found_error(sym_name, idx);
2286 	}
2287 
2288 	*start = args.start;
2289 	*size = args.size;
2290 
2291 	return 0;
2292 }
2293 
2294 static int find_entire_kern_cb(void *arg, const char *name __maybe_unused,
2295 			       char type, u64 start)
2296 {
2297 	struct sym_args *args = arg;
2298 
2299 	if (!kallsyms__is_function(type))
2300 		return 0;
2301 
2302 	if (!args->started) {
2303 		args->started = true;
2304 		args->start = start;
2305 	}
2306 	/* Don't know exactly where the kernel ends, so we add a page */
2307 	args->size = round_up(start, page_size) + page_size - args->start;
2308 
2309 	return 0;
2310 }
2311 
2312 static int addr_filter__entire_kernel(struct addr_filter *filt)
2313 {
2314 	struct sym_args args = { .started = false };
2315 	int err;
2316 
2317 	err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb);
2318 	if (err < 0 || !args.started) {
2319 		pr_err("Failed to parse /proc/kallsyms\n");
2320 		return err;
2321 	}
2322 
2323 	filt->addr = args.start;
2324 	filt->size = args.size;
2325 
2326 	return 0;
2327 }
2328 
2329 static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size)
2330 {
2331 	if (start + size >= filt->addr)
2332 		return 0;
2333 
2334 	if (filt->sym_from) {
2335 		pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n",
2336 		       filt->sym_to, start, filt->sym_from, filt->addr);
2337 	} else {
2338 		pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n",
2339 		       filt->sym_to, start, filt->addr);
2340 	}
2341 
2342 	return -EINVAL;
2343 }
2344 
2345 static int addr_filter__resolve_kernel_syms(struct addr_filter *filt)
2346 {
2347 	bool no_size = false;
2348 	u64 start, size;
2349 	int err;
2350 
2351 	if (symbol_conf.kptr_restrict) {
2352 		pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");
2353 		return -EINVAL;
2354 	}
2355 
2356 	if (filt->sym_from && !strcmp(filt->sym_from, "*"))
2357 		return addr_filter__entire_kernel(filt);
2358 
2359 	if (filt->sym_from) {
2360 		err = find_kern_sym(filt->sym_from, &start, &size,
2361 				    filt->sym_from_idx);
2362 		if (err)
2363 			return err;
2364 		filt->addr = start;
2365 		if (filt->range && !filt->size && !filt->sym_to) {
2366 			filt->size = size;
2367 			no_size = !size;
2368 		}
2369 	}
2370 
2371 	if (filt->sym_to) {
2372 		err = find_kern_sym(filt->sym_to, &start, &size,
2373 				    filt->sym_to_idx);
2374 		if (err)
2375 			return err;
2376 
2377 		err = check_end_after_start(filt, start, size);
2378 		if (err)
2379 			return err;
2380 		filt->size = start + size - filt->addr;
2381 		no_size = !size;
2382 	}
2383 
2384 	/* The very last symbol in kallsyms does not imply a particular size */
2385 	if (no_size) {
2386 		pr_err("Cannot determine size of symbol '%s'\n",
2387 		       filt->sym_to ? filt->sym_to : filt->sym_from);
2388 		return -EINVAL;
2389 	}
2390 
2391 	return 0;
2392 }
2393 
2394 static struct dso *load_dso(const char *name)
2395 {
2396 	struct map *map;
2397 	struct dso *dso;
2398 
2399 	map = dso__new_map(name);
2400 	if (!map)
2401 		return NULL;
2402 
2403 	if (map__load(map) < 0)
2404 		pr_err("File '%s' not found or has no symbols.\n", name);
2405 
2406 	dso = dso__get(map->dso);
2407 
2408 	map__put(map);
2409 
2410 	return dso;
2411 }
2412 
2413 static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt,
2414 			  int idx)
2415 {
2416 	/* Same name, and global or the n'th found or any */
2417 	return !arch__compare_symbol_names(name, sym->name) &&
2418 	       ((!idx && sym->binding == STB_GLOBAL) ||
2419 		(idx > 0 && ++*cnt == idx) ||
2420 		idx < 0);
2421 }
2422 
2423 static void print_duplicate_syms(struct dso *dso, const char *sym_name)
2424 {
2425 	struct symbol *sym;
2426 	bool near = false;
2427 	int cnt = 0;
2428 
2429 	pr_err("Multiple symbols with name '%s'\n", sym_name);
2430 
2431 	sym = dso__first_symbol(dso);
2432 	while (sym) {
2433 		if (dso_sym_match(sym, sym_name, &cnt, -1)) {
2434 			pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2435 			       ++cnt, sym->start,
2436 			       sym->binding == STB_GLOBAL ? 'g' :
2437 			       sym->binding == STB_LOCAL  ? 'l' : 'w',
2438 			       sym->name);
2439 			near = true;
2440 		} else if (near) {
2441 			near = false;
2442 			pr_err("\t\twhich is near\t\t%s\n", sym->name);
2443 		}
2444 		sym = dso__next_symbol(sym);
2445 	}
2446 
2447 	pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2448 	       sym_name);
2449 	pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2450 }
2451 
2452 static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start,
2453 			u64 *size, int idx)
2454 {
2455 	struct symbol *sym;
2456 	int cnt = 0;
2457 
2458 	*start = 0;
2459 	*size = 0;
2460 
2461 	sym = dso__first_symbol(dso);
2462 	while (sym) {
2463 		if (*start) {
2464 			if (!*size)
2465 				*size = sym->start - *start;
2466 			if (idx > 0) {
2467 				if (*size)
2468 					return 1;
2469 			} else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2470 				print_duplicate_syms(dso, sym_name);
2471 				return -EINVAL;
2472 			}
2473 		} else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2474 			*start = sym->start;
2475 			*size = sym->end - sym->start;
2476 		}
2477 		sym = dso__next_symbol(sym);
2478 	}
2479 
2480 	if (!*start)
2481 		return sym_not_found_error(sym_name, idx);
2482 
2483 	return 0;
2484 }
2485 
2486 static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso)
2487 {
2488 	if (dso__data_file_size(dso, NULL)) {
2489 		pr_err("Failed to determine filter for %s\nCannot determine file size.\n",
2490 		       filt->filename);
2491 		return -EINVAL;
2492 	}
2493 
2494 	filt->addr = 0;
2495 	filt->size = dso->data.file_size;
2496 
2497 	return 0;
2498 }
2499 
2500 static int addr_filter__resolve_syms(struct addr_filter *filt)
2501 {
2502 	u64 start, size;
2503 	struct dso *dso;
2504 	int err = 0;
2505 
2506 	if (!filt->sym_from && !filt->sym_to)
2507 		return 0;
2508 
2509 	if (!filt->filename)
2510 		return addr_filter__resolve_kernel_syms(filt);
2511 
2512 	dso = load_dso(filt->filename);
2513 	if (!dso) {
2514 		pr_err("Failed to load symbols from: %s\n", filt->filename);
2515 		return -EINVAL;
2516 	}
2517 
2518 	if (filt->sym_from && !strcmp(filt->sym_from, "*")) {
2519 		err = addr_filter__entire_dso(filt, dso);
2520 		goto put_dso;
2521 	}
2522 
2523 	if (filt->sym_from) {
2524 		err = find_dso_sym(dso, filt->sym_from, &start, &size,
2525 				   filt->sym_from_idx);
2526 		if (err)
2527 			goto put_dso;
2528 		filt->addr = start;
2529 		if (filt->range && !filt->size && !filt->sym_to)
2530 			filt->size = size;
2531 	}
2532 
2533 	if (filt->sym_to) {
2534 		err = find_dso_sym(dso, filt->sym_to, &start, &size,
2535 				   filt->sym_to_idx);
2536 		if (err)
2537 			goto put_dso;
2538 
2539 		err = check_end_after_start(filt, start, size);
2540 		if (err)
2541 			return err;
2542 
2543 		filt->size = start + size - filt->addr;
2544 	}
2545 
2546 put_dso:
2547 	dso__put(dso);
2548 
2549 	return err;
2550 }
2551 
2552 static char *addr_filter__to_str(struct addr_filter *filt)
2553 {
2554 	char filename_buf[PATH_MAX];
2555 	const char *at = "";
2556 	const char *fn = "";
2557 	char *filter;
2558 	int err;
2559 
2560 	if (filt->filename) {
2561 		at = "@";
2562 		fn = realpath(filt->filename, filename_buf);
2563 		if (!fn)
2564 			return NULL;
2565 	}
2566 
2567 	if (filt->range) {
2568 		err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s",
2569 			       filt->action, filt->addr, filt->size, at, fn);
2570 	} else {
2571 		err = asprintf(&filter, "%s 0x%"PRIx64"%s%s",
2572 			       filt->action, filt->addr, at, fn);
2573 	}
2574 
2575 	return err < 0 ? NULL : filter;
2576 }
2577 
2578 static int parse_addr_filter(struct evsel *evsel, const char *filter,
2579 			     int max_nr)
2580 {
2581 	struct addr_filters filts;
2582 	struct addr_filter *filt;
2583 	int err;
2584 
2585 	addr_filters__init(&filts);
2586 
2587 	err = addr_filters__parse_bare_filter(&filts, filter);
2588 	if (err)
2589 		goto out_exit;
2590 
2591 	if (filts.cnt > max_nr) {
2592 		pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",
2593 		       filts.cnt, max_nr);
2594 		err = -EINVAL;
2595 		goto out_exit;
2596 	}
2597 
2598 	list_for_each_entry(filt, &filts.head, list) {
2599 		char *new_filter;
2600 
2601 		err = addr_filter__resolve_syms(filt);
2602 		if (err)
2603 			goto out_exit;
2604 
2605 		new_filter = addr_filter__to_str(filt);
2606 		if (!new_filter) {
2607 			err = -ENOMEM;
2608 			goto out_exit;
2609 		}
2610 
2611 		if (evsel__append_addr_filter(evsel, new_filter)) {
2612 			err = -ENOMEM;
2613 			goto out_exit;
2614 		}
2615 	}
2616 
2617 out_exit:
2618 	addr_filters__exit(&filts);
2619 
2620 	if (err) {
2621 		pr_err("Failed to parse address filter: '%s'\n", filter);
2622 		pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");
2623 		pr_err("Where multiple filters are separated by space or comma.\n");
2624 	}
2625 
2626 	return err;
2627 }
2628 
2629 static int evsel__nr_addr_filter(struct evsel *evsel)
2630 {
2631 	struct perf_pmu *pmu = evsel__find_pmu(evsel);
2632 	int nr_addr_filters = 0;
2633 
2634 	if (!pmu)
2635 		return 0;
2636 
2637 	perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters);
2638 
2639 	return nr_addr_filters;
2640 }
2641 
2642 int auxtrace_parse_filters(struct evlist *evlist)
2643 {
2644 	struct evsel *evsel;
2645 	char *filter;
2646 	int err, max_nr;
2647 
2648 	evlist__for_each_entry(evlist, evsel) {
2649 		filter = evsel->filter;
2650 		max_nr = evsel__nr_addr_filter(evsel);
2651 		if (!filter || !max_nr)
2652 			continue;
2653 		evsel->filter = NULL;
2654 		err = parse_addr_filter(evsel, filter, max_nr);
2655 		free(filter);
2656 		if (err)
2657 			return err;
2658 		pr_debug("Address filter: %s\n", evsel->filter);
2659 	}
2660 
2661 	return 0;
2662 }
2663 
2664 int auxtrace__process_event(struct perf_session *session, union perf_event *event,
2665 			    struct perf_sample *sample, struct perf_tool *tool)
2666 {
2667 	if (!session->auxtrace)
2668 		return 0;
2669 
2670 	return session->auxtrace->process_event(session, event, sample, tool);
2671 }
2672 
2673 void auxtrace__dump_auxtrace_sample(struct perf_session *session,
2674 				    struct perf_sample *sample)
2675 {
2676 	if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample ||
2677 	    auxtrace__dont_decode(session))
2678 		return;
2679 
2680 	session->auxtrace->dump_auxtrace_sample(session, sample);
2681 }
2682 
2683 int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool)
2684 {
2685 	if (!session->auxtrace)
2686 		return 0;
2687 
2688 	return session->auxtrace->flush_events(session, tool);
2689 }
2690 
2691 void auxtrace__free_events(struct perf_session *session)
2692 {
2693 	if (!session->auxtrace)
2694 		return;
2695 
2696 	return session->auxtrace->free_events(session);
2697 }
2698 
2699 void auxtrace__free(struct perf_session *session)
2700 {
2701 	if (!session->auxtrace)
2702 		return;
2703 
2704 	return session->auxtrace->free(session);
2705 }
2706 
2707 bool auxtrace__evsel_is_auxtrace(struct perf_session *session,
2708 				 struct evsel *evsel)
2709 {
2710 	if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace)
2711 		return false;
2712 
2713 	return session->auxtrace->evsel_is_auxtrace(session, evsel);
2714 }
2715