xref: /linux/tools/perf/arch/x86/util/intel-pt.c (revision b8321ed4a40c02054f930ca59d3570caa27bc86c)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * intel_pt.c: Intel Processor Trace support
4  * Copyright (c) 2013-2015, Intel Corporation.
5  */
6 
7 #include <errno.h>
8 #include <stdbool.h>
9 #include <linux/kernel.h>
10 #include <linux/types.h>
11 #include <linux/bitops.h>
12 #include <linux/log2.h>
13 #include <linux/zalloc.h>
14 #include <cpuid.h>
15 
16 #include "../../../util/session.h"
17 #include "../../../util/event.h"
18 #include "../../../util/evlist.h"
19 #include "../../../util/evsel.h"
20 #include "../../../util/evsel_config.h"
21 #include "../../../util/cpumap.h"
22 #include "../../../util/mmap.h"
23 #include <subcmd/parse-options.h>
24 #include "../../../util/parse-events.h"
25 #include "../../../util/pmu.h"
26 #include "../../../util/debug.h"
27 #include "../../../util/auxtrace.h"
28 #include "../../../util/perf_api_probe.h"
29 #include "../../../util/record.h"
30 #include "../../../util/target.h"
31 #include "../../../util/tsc.h"
32 #include <internal/lib.h> // page_size
33 #include "../../../util/intel-pt.h"
34 
35 #define KiB(x) ((x) * 1024)
36 #define MiB(x) ((x) * 1024 * 1024)
37 #define KiB_MASK(x) (KiB(x) - 1)
38 #define MiB_MASK(x) (MiB(x) - 1)
39 
40 #define INTEL_PT_PSB_PERIOD_NEAR	256
41 
42 struct intel_pt_snapshot_ref {
43 	void *ref_buf;
44 	size_t ref_offset;
45 	bool wrapped;
46 };
47 
48 struct intel_pt_recording {
49 	struct auxtrace_record		itr;
50 	struct perf_pmu			*intel_pt_pmu;
51 	int				have_sched_switch;
52 	struct evlist		*evlist;
53 	bool				snapshot_mode;
54 	bool				snapshot_init_done;
55 	size_t				snapshot_size;
56 	size_t				snapshot_ref_buf_size;
57 	int				snapshot_ref_cnt;
58 	struct intel_pt_snapshot_ref	*snapshot_refs;
59 	size_t				priv_size;
60 };
61 
62 static int intel_pt_parse_terms_with_default(const char *pmu_name,
63 					     struct list_head *formats,
64 					     const char *str,
65 					     u64 *config)
66 {
67 	struct list_head *terms;
68 	struct perf_event_attr attr = { .size = 0, };
69 	int err;
70 
71 	terms = malloc(sizeof(struct list_head));
72 	if (!terms)
73 		return -ENOMEM;
74 
75 	INIT_LIST_HEAD(terms);
76 
77 	err = parse_events_terms(terms, str);
78 	if (err)
79 		goto out_free;
80 
81 	attr.config = *config;
82 	err = perf_pmu__config_terms(pmu_name, formats, &attr, terms, true,
83 				     NULL);
84 	if (err)
85 		goto out_free;
86 
87 	*config = attr.config;
88 out_free:
89 	parse_events_terms__delete(terms);
90 	return err;
91 }
92 
93 static int intel_pt_parse_terms(const char *pmu_name, struct list_head *formats,
94 				const char *str, u64 *config)
95 {
96 	*config = 0;
97 	return intel_pt_parse_terms_with_default(pmu_name, formats, str,
98 						 config);
99 }
100 
101 static u64 intel_pt_masked_bits(u64 mask, u64 bits)
102 {
103 	const u64 top_bit = 1ULL << 63;
104 	u64 res = 0;
105 	int i;
106 
107 	for (i = 0; i < 64; i++) {
108 		if (mask & top_bit) {
109 			res <<= 1;
110 			if (bits & top_bit)
111 				res |= 1;
112 		}
113 		mask <<= 1;
114 		bits <<= 1;
115 	}
116 
117 	return res;
118 }
119 
120 static int intel_pt_read_config(struct perf_pmu *intel_pt_pmu, const char *str,
121 				struct evlist *evlist, u64 *res)
122 {
123 	struct evsel *evsel;
124 	u64 mask;
125 
126 	*res = 0;
127 
128 	mask = perf_pmu__format_bits(&intel_pt_pmu->format, str);
129 	if (!mask)
130 		return -EINVAL;
131 
132 	evlist__for_each_entry(evlist, evsel) {
133 		if (evsel->core.attr.type == intel_pt_pmu->type) {
134 			*res = intel_pt_masked_bits(mask, evsel->core.attr.config);
135 			return 0;
136 		}
137 	}
138 
139 	return -EINVAL;
140 }
141 
142 static size_t intel_pt_psb_period(struct perf_pmu *intel_pt_pmu,
143 				  struct evlist *evlist)
144 {
145 	u64 val;
146 	int err, topa_multiple_entries;
147 	size_t psb_period;
148 
149 	if (perf_pmu__scan_file(intel_pt_pmu, "caps/topa_multiple_entries",
150 				"%d", &topa_multiple_entries) != 1)
151 		topa_multiple_entries = 0;
152 
153 	/*
154 	 * Use caps/topa_multiple_entries to indicate early hardware that had
155 	 * extra frequent PSBs.
156 	 */
157 	if (!topa_multiple_entries) {
158 		psb_period = 256;
159 		goto out;
160 	}
161 
162 	err = intel_pt_read_config(intel_pt_pmu, "psb_period", evlist, &val);
163 	if (err)
164 		val = 0;
165 
166 	psb_period = 1 << (val + 11);
167 out:
168 	pr_debug2("%s psb_period %zu\n", intel_pt_pmu->name, psb_period);
169 	return psb_period;
170 }
171 
172 static int intel_pt_pick_bit(int bits, int target)
173 {
174 	int pos, pick = -1;
175 
176 	for (pos = 0; bits; bits >>= 1, pos++) {
177 		if (bits & 1) {
178 			if (pos <= target || pick < 0)
179 				pick = pos;
180 			if (pos >= target)
181 				break;
182 		}
183 	}
184 
185 	return pick;
186 }
187 
188 static u64 intel_pt_default_config(struct perf_pmu *intel_pt_pmu)
189 {
190 	char buf[256];
191 	int mtc, mtc_periods = 0, mtc_period;
192 	int psb_cyc, psb_periods, psb_period;
193 	int pos = 0;
194 	u64 config;
195 	char c;
196 
197 	pos += scnprintf(buf + pos, sizeof(buf) - pos, "tsc");
198 
199 	if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc", "%d",
200 				&mtc) != 1)
201 		mtc = 1;
202 
203 	if (mtc) {
204 		if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc_periods", "%x",
205 					&mtc_periods) != 1)
206 			mtc_periods = 0;
207 		if (mtc_periods) {
208 			mtc_period = intel_pt_pick_bit(mtc_periods, 3);
209 			pos += scnprintf(buf + pos, sizeof(buf) - pos,
210 					 ",mtc,mtc_period=%d", mtc_period);
211 		}
212 	}
213 
214 	if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_cyc", "%d",
215 				&psb_cyc) != 1)
216 		psb_cyc = 1;
217 
218 	if (psb_cyc && mtc_periods) {
219 		if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_periods", "%x",
220 					&psb_periods) != 1)
221 			psb_periods = 0;
222 		if (psb_periods) {
223 			psb_period = intel_pt_pick_bit(psb_periods, 3);
224 			pos += scnprintf(buf + pos, sizeof(buf) - pos,
225 					 ",psb_period=%d", psb_period);
226 		}
227 	}
228 
229 	if (perf_pmu__scan_file(intel_pt_pmu, "format/pt", "%c", &c) == 1 &&
230 	    perf_pmu__scan_file(intel_pt_pmu, "format/branch", "%c", &c) == 1)
231 		pos += scnprintf(buf + pos, sizeof(buf) - pos, ",pt,branch");
232 
233 	pr_debug2("%s default config: %s\n", intel_pt_pmu->name, buf);
234 
235 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format, buf,
236 			     &config);
237 
238 	return config;
239 }
240 
241 static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr,
242 					   struct record_opts *opts,
243 					   const char *str)
244 {
245 	struct intel_pt_recording *ptr =
246 			container_of(itr, struct intel_pt_recording, itr);
247 	unsigned long long snapshot_size = 0;
248 	char *endptr;
249 
250 	if (str) {
251 		snapshot_size = strtoull(str, &endptr, 0);
252 		if (*endptr || snapshot_size > SIZE_MAX)
253 			return -1;
254 	}
255 
256 	opts->auxtrace_snapshot_mode = true;
257 	opts->auxtrace_snapshot_size = snapshot_size;
258 
259 	ptr->snapshot_size = snapshot_size;
260 
261 	return 0;
262 }
263 
264 struct perf_event_attr *
265 intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu)
266 {
267 	struct perf_event_attr *attr;
268 
269 	attr = zalloc(sizeof(struct perf_event_attr));
270 	if (!attr)
271 		return NULL;
272 
273 	attr->config = intel_pt_default_config(intel_pt_pmu);
274 
275 	intel_pt_pmu->selectable = true;
276 
277 	return attr;
278 }
279 
280 static const char *intel_pt_find_filter(struct evlist *evlist,
281 					struct perf_pmu *intel_pt_pmu)
282 {
283 	struct evsel *evsel;
284 
285 	evlist__for_each_entry(evlist, evsel) {
286 		if (evsel->core.attr.type == intel_pt_pmu->type)
287 			return evsel->filter;
288 	}
289 
290 	return NULL;
291 }
292 
293 static size_t intel_pt_filter_bytes(const char *filter)
294 {
295 	size_t len = filter ? strlen(filter) : 0;
296 
297 	return len ? roundup(len + 1, 8) : 0;
298 }
299 
300 static size_t
301 intel_pt_info_priv_size(struct auxtrace_record *itr, struct evlist *evlist)
302 {
303 	struct intel_pt_recording *ptr =
304 			container_of(itr, struct intel_pt_recording, itr);
305 	const char *filter = intel_pt_find_filter(evlist, ptr->intel_pt_pmu);
306 
307 	ptr->priv_size = (INTEL_PT_AUXTRACE_PRIV_MAX * sizeof(u64)) +
308 			 intel_pt_filter_bytes(filter);
309 	ptr->priv_size += sizeof(u64); /* Cap Event Trace */
310 
311 	return ptr->priv_size;
312 }
313 
314 static void intel_pt_tsc_ctc_ratio(u32 *n, u32 *d)
315 {
316 	unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
317 
318 	__get_cpuid(0x15, &eax, &ebx, &ecx, &edx);
319 	*n = ebx;
320 	*d = eax;
321 }
322 
323 static int intel_pt_info_fill(struct auxtrace_record *itr,
324 			      struct perf_session *session,
325 			      struct perf_record_auxtrace_info *auxtrace_info,
326 			      size_t priv_size)
327 {
328 	struct intel_pt_recording *ptr =
329 			container_of(itr, struct intel_pt_recording, itr);
330 	struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu;
331 	struct perf_event_mmap_page *pc;
332 	struct perf_tsc_conversion tc = { .time_mult = 0, };
333 	bool cap_user_time_zero = false, per_cpu_mmaps;
334 	u64 tsc_bit, mtc_bit, mtc_freq_bits, cyc_bit, noretcomp_bit;
335 	u32 tsc_ctc_ratio_n, tsc_ctc_ratio_d;
336 	unsigned long max_non_turbo_ratio;
337 	size_t filter_str_len;
338 	const char *filter;
339 	int event_trace;
340 	__u64 *info;
341 	int err;
342 
343 	if (priv_size != ptr->priv_size)
344 		return -EINVAL;
345 
346 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format,
347 			     "tsc", &tsc_bit);
348 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format,
349 			     "noretcomp", &noretcomp_bit);
350 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format,
351 			     "mtc", &mtc_bit);
352 	mtc_freq_bits = perf_pmu__format_bits(&intel_pt_pmu->format,
353 					      "mtc_period");
354 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format,
355 			     "cyc", &cyc_bit);
356 
357 	intel_pt_tsc_ctc_ratio(&tsc_ctc_ratio_n, &tsc_ctc_ratio_d);
358 
359 	if (perf_pmu__scan_file(intel_pt_pmu, "max_nonturbo_ratio",
360 				"%lu", &max_non_turbo_ratio) != 1)
361 		max_non_turbo_ratio = 0;
362 	if (perf_pmu__scan_file(intel_pt_pmu, "caps/event_trace",
363 				"%d", &event_trace) != 1)
364 		event_trace = 0;
365 
366 	filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu);
367 	filter_str_len = filter ? strlen(filter) : 0;
368 
369 	if (!session->evlist->core.nr_mmaps)
370 		return -EINVAL;
371 
372 	pc = session->evlist->mmap[0].core.base;
373 	if (pc) {
374 		err = perf_read_tsc_conversion(pc, &tc);
375 		if (err) {
376 			if (err != -EOPNOTSUPP)
377 				return err;
378 		} else {
379 			cap_user_time_zero = tc.time_mult != 0;
380 		}
381 		if (!cap_user_time_zero)
382 			ui__warning("Intel Processor Trace: TSC not available\n");
383 	}
384 
385 	per_cpu_mmaps = !perf_cpu_map__empty(session->evlist->core.cpus);
386 
387 	auxtrace_info->type = PERF_AUXTRACE_INTEL_PT;
388 	auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type;
389 	auxtrace_info->priv[INTEL_PT_TIME_SHIFT] = tc.time_shift;
390 	auxtrace_info->priv[INTEL_PT_TIME_MULT] = tc.time_mult;
391 	auxtrace_info->priv[INTEL_PT_TIME_ZERO] = tc.time_zero;
392 	auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO] = cap_user_time_zero;
393 	auxtrace_info->priv[INTEL_PT_TSC_BIT] = tsc_bit;
394 	auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT] = noretcomp_bit;
395 	auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH] = ptr->have_sched_switch;
396 	auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE] = ptr->snapshot_mode;
397 	auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS] = per_cpu_mmaps;
398 	auxtrace_info->priv[INTEL_PT_MTC_BIT] = mtc_bit;
399 	auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS] = mtc_freq_bits;
400 	auxtrace_info->priv[INTEL_PT_TSC_CTC_N] = tsc_ctc_ratio_n;
401 	auxtrace_info->priv[INTEL_PT_TSC_CTC_D] = tsc_ctc_ratio_d;
402 	auxtrace_info->priv[INTEL_PT_CYC_BIT] = cyc_bit;
403 	auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO] = max_non_turbo_ratio;
404 	auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] = filter_str_len;
405 
406 	info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
407 
408 	if (filter_str_len) {
409 		size_t len = intel_pt_filter_bytes(filter);
410 
411 		strncpy((char *)info, filter, len);
412 		info += len >> 3;
413 	}
414 
415 	*info++ = event_trace;
416 
417 	return 0;
418 }
419 
420 static int intel_pt_track_switches(struct evlist *evlist)
421 {
422 	const char *sched_switch = "sched:sched_switch";
423 	struct evsel *evsel;
424 	int err;
425 
426 	if (!evlist__can_select_event(evlist, sched_switch))
427 		return -EPERM;
428 
429 	err = parse_events(evlist, sched_switch, NULL);
430 	if (err) {
431 		pr_debug2("%s: failed to parse %s, error %d\n",
432 			  __func__, sched_switch, err);
433 		return err;
434 	}
435 
436 	evsel = evlist__last(evlist);
437 
438 	evsel__set_sample_bit(evsel, CPU);
439 	evsel__set_sample_bit(evsel, TIME);
440 
441 	evsel->core.system_wide = true;
442 	evsel->no_aux_samples = true;
443 	evsel->immediate = true;
444 
445 	return 0;
446 }
447 
448 static void intel_pt_valid_str(char *str, size_t len, u64 valid)
449 {
450 	unsigned int val, last = 0, state = 1;
451 	int p = 0;
452 
453 	str[0] = '\0';
454 
455 	for (val = 0; val <= 64; val++, valid >>= 1) {
456 		if (valid & 1) {
457 			last = val;
458 			switch (state) {
459 			case 0:
460 				p += scnprintf(str + p, len - p, ",");
461 				/* Fall through */
462 			case 1:
463 				p += scnprintf(str + p, len - p, "%u", val);
464 				state = 2;
465 				break;
466 			case 2:
467 				state = 3;
468 				break;
469 			case 3:
470 				state = 4;
471 				break;
472 			default:
473 				break;
474 			}
475 		} else {
476 			switch (state) {
477 			case 3:
478 				p += scnprintf(str + p, len - p, ",%u", last);
479 				state = 0;
480 				break;
481 			case 4:
482 				p += scnprintf(str + p, len - p, "-%u", last);
483 				state = 0;
484 				break;
485 			default:
486 				break;
487 			}
488 			if (state != 1)
489 				state = 0;
490 		}
491 	}
492 }
493 
494 static int intel_pt_val_config_term(struct perf_pmu *intel_pt_pmu,
495 				    const char *caps, const char *name,
496 				    const char *supported, u64 config)
497 {
498 	char valid_str[256];
499 	unsigned int shift;
500 	unsigned long long valid;
501 	u64 bits;
502 	int ok;
503 
504 	if (perf_pmu__scan_file(intel_pt_pmu, caps, "%llx", &valid) != 1)
505 		valid = 0;
506 
507 	if (supported &&
508 	    perf_pmu__scan_file(intel_pt_pmu, supported, "%d", &ok) == 1 && !ok)
509 		valid = 0;
510 
511 	valid |= 1;
512 
513 	bits = perf_pmu__format_bits(&intel_pt_pmu->format, name);
514 
515 	config &= bits;
516 
517 	for (shift = 0; bits && !(bits & 1); shift++)
518 		bits >>= 1;
519 
520 	config >>= shift;
521 
522 	if (config > 63)
523 		goto out_err;
524 
525 	if (valid & (1 << config))
526 		return 0;
527 out_err:
528 	intel_pt_valid_str(valid_str, sizeof(valid_str), valid);
529 	pr_err("Invalid %s for %s. Valid values are: %s\n",
530 	       name, INTEL_PT_PMU_NAME, valid_str);
531 	return -EINVAL;
532 }
533 
534 static int intel_pt_validate_config(struct perf_pmu *intel_pt_pmu,
535 				    struct evsel *evsel)
536 {
537 	int err;
538 	char c;
539 
540 	if (!evsel)
541 		return 0;
542 
543 	/*
544 	 * If supported, force pass-through config term (pt=1) even if user
545 	 * sets pt=0, which avoids senseless kernel errors.
546 	 */
547 	if (perf_pmu__scan_file(intel_pt_pmu, "format/pt", "%c", &c) == 1 &&
548 	    !(evsel->core.attr.config & 1)) {
549 		pr_warning("pt=0 doesn't make sense, forcing pt=1\n");
550 		evsel->core.attr.config |= 1;
551 	}
552 
553 	err = intel_pt_val_config_term(intel_pt_pmu, "caps/cycle_thresholds",
554 				       "cyc_thresh", "caps/psb_cyc",
555 				       evsel->core.attr.config);
556 	if (err)
557 		return err;
558 
559 	err = intel_pt_val_config_term(intel_pt_pmu, "caps/mtc_periods",
560 				       "mtc_period", "caps/mtc",
561 				       evsel->core.attr.config);
562 	if (err)
563 		return err;
564 
565 	return intel_pt_val_config_term(intel_pt_pmu, "caps/psb_periods",
566 					"psb_period", "caps/psb_cyc",
567 					evsel->core.attr.config);
568 }
569 
570 static void intel_pt_config_sample_mode(struct perf_pmu *intel_pt_pmu,
571 					struct evsel *evsel)
572 {
573 	u64 user_bits = 0, bits;
574 	struct evsel_config_term *term = evsel__get_config_term(evsel, CFG_CHG);
575 
576 	if (term)
577 		user_bits = term->val.cfg_chg;
578 
579 	bits = perf_pmu__format_bits(&intel_pt_pmu->format, "psb_period");
580 
581 	/* Did user change psb_period */
582 	if (bits & user_bits)
583 		return;
584 
585 	/* Set psb_period to 0 */
586 	evsel->core.attr.config &= ~bits;
587 }
588 
589 static void intel_pt_min_max_sample_sz(struct evlist *evlist,
590 				       size_t *min_sz, size_t *max_sz)
591 {
592 	struct evsel *evsel;
593 
594 	evlist__for_each_entry(evlist, evsel) {
595 		size_t sz = evsel->core.attr.aux_sample_size;
596 
597 		if (!sz)
598 			continue;
599 		if (min_sz && (sz < *min_sz || !*min_sz))
600 			*min_sz = sz;
601 		if (max_sz && sz > *max_sz)
602 			*max_sz = sz;
603 	}
604 }
605 
606 /*
607  * Currently, there is not enough information to disambiguate different PEBS
608  * events, so only allow one.
609  */
610 static bool intel_pt_too_many_aux_output(struct evlist *evlist)
611 {
612 	struct evsel *evsel;
613 	int aux_output_cnt = 0;
614 
615 	evlist__for_each_entry(evlist, evsel)
616 		aux_output_cnt += !!evsel->core.attr.aux_output;
617 
618 	if (aux_output_cnt > 1) {
619 		pr_err(INTEL_PT_PMU_NAME " supports at most one event with aux-output\n");
620 		return true;
621 	}
622 
623 	return false;
624 }
625 
626 static int intel_pt_recording_options(struct auxtrace_record *itr,
627 				      struct evlist *evlist,
628 				      struct record_opts *opts)
629 {
630 	struct intel_pt_recording *ptr =
631 			container_of(itr, struct intel_pt_recording, itr);
632 	struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu;
633 	bool have_timing_info, need_immediate = false;
634 	struct evsel *evsel, *intel_pt_evsel = NULL;
635 	const struct perf_cpu_map *cpus = evlist->core.cpus;
636 	bool privileged = perf_event_paranoid_check(-1);
637 	u64 tsc_bit;
638 	int err;
639 
640 	ptr->evlist = evlist;
641 	ptr->snapshot_mode = opts->auxtrace_snapshot_mode;
642 
643 	evlist__for_each_entry(evlist, evsel) {
644 		if (evsel->core.attr.type == intel_pt_pmu->type) {
645 			if (intel_pt_evsel) {
646 				pr_err("There may be only one " INTEL_PT_PMU_NAME " event\n");
647 				return -EINVAL;
648 			}
649 			evsel->core.attr.freq = 0;
650 			evsel->core.attr.sample_period = 1;
651 			evsel->no_aux_samples = true;
652 			intel_pt_evsel = evsel;
653 			opts->full_auxtrace = true;
654 		}
655 	}
656 
657 	if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
658 		pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME " PMU event (-e " INTEL_PT_PMU_NAME ")\n");
659 		return -EINVAL;
660 	}
661 
662 	if (opts->auxtrace_snapshot_mode && opts->auxtrace_sample_mode) {
663 		pr_err("Snapshot mode (" INTEL_PT_PMU_NAME " PMU) and sample trace cannot be used together\n");
664 		return -EINVAL;
665 	}
666 
667 	if (opts->use_clockid) {
668 		pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME "\n");
669 		return -EINVAL;
670 	}
671 
672 	if (intel_pt_too_many_aux_output(evlist))
673 		return -EINVAL;
674 
675 	if (!opts->full_auxtrace)
676 		return 0;
677 
678 	if (opts->auxtrace_sample_mode)
679 		intel_pt_config_sample_mode(intel_pt_pmu, intel_pt_evsel);
680 
681 	err = intel_pt_validate_config(intel_pt_pmu, intel_pt_evsel);
682 	if (err)
683 		return err;
684 
685 	/* Set default sizes for snapshot mode */
686 	if (opts->auxtrace_snapshot_mode) {
687 		size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist);
688 
689 		if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
690 			if (privileged) {
691 				opts->auxtrace_mmap_pages = MiB(4) / page_size;
692 			} else {
693 				opts->auxtrace_mmap_pages = KiB(128) / page_size;
694 				if (opts->mmap_pages == UINT_MAX)
695 					opts->mmap_pages = KiB(256) / page_size;
696 			}
697 		} else if (!opts->auxtrace_mmap_pages && !privileged &&
698 			   opts->mmap_pages == UINT_MAX) {
699 			opts->mmap_pages = KiB(256) / page_size;
700 		}
701 		if (!opts->auxtrace_snapshot_size)
702 			opts->auxtrace_snapshot_size =
703 				opts->auxtrace_mmap_pages * (size_t)page_size;
704 		if (!opts->auxtrace_mmap_pages) {
705 			size_t sz = opts->auxtrace_snapshot_size;
706 
707 			sz = round_up(sz, page_size) / page_size;
708 			opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
709 		}
710 		if (opts->auxtrace_snapshot_size >
711 				opts->auxtrace_mmap_pages * (size_t)page_size) {
712 			pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
713 			       opts->auxtrace_snapshot_size,
714 			       opts->auxtrace_mmap_pages * (size_t)page_size);
715 			return -EINVAL;
716 		}
717 		if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) {
718 			pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
719 			return -EINVAL;
720 		}
721 		pr_debug2("Intel PT snapshot size: %zu\n",
722 			  opts->auxtrace_snapshot_size);
723 		if (psb_period &&
724 		    opts->auxtrace_snapshot_size <= psb_period +
725 						  INTEL_PT_PSB_PERIOD_NEAR)
726 			ui__warning("Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n",
727 				    opts->auxtrace_snapshot_size, psb_period);
728 	}
729 
730 	/* Set default sizes for sample mode */
731 	if (opts->auxtrace_sample_mode) {
732 		size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist);
733 		size_t min_sz = 0, max_sz = 0;
734 
735 		intel_pt_min_max_sample_sz(evlist, &min_sz, &max_sz);
736 		if (!opts->auxtrace_mmap_pages && !privileged &&
737 		    opts->mmap_pages == UINT_MAX)
738 			opts->mmap_pages = KiB(256) / page_size;
739 		if (!opts->auxtrace_mmap_pages) {
740 			size_t sz = round_up(max_sz, page_size) / page_size;
741 
742 			opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
743 		}
744 		if (max_sz > opts->auxtrace_mmap_pages * (size_t)page_size) {
745 			pr_err("Sample size %zu must not be greater than AUX area tracing mmap size %zu\n",
746 			       max_sz,
747 			       opts->auxtrace_mmap_pages * (size_t)page_size);
748 			return -EINVAL;
749 		}
750 		pr_debug2("Intel PT min. sample size: %zu max. sample size: %zu\n",
751 			  min_sz, max_sz);
752 		if (psb_period &&
753 		    min_sz <= psb_period + INTEL_PT_PSB_PERIOD_NEAR)
754 			ui__warning("Intel PT sample size (%zu) may be too small for PSB period (%zu)\n",
755 				    min_sz, psb_period);
756 	}
757 
758 	/* Set default sizes for full trace mode */
759 	if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
760 		if (privileged) {
761 			opts->auxtrace_mmap_pages = MiB(4) / page_size;
762 		} else {
763 			opts->auxtrace_mmap_pages = KiB(128) / page_size;
764 			if (opts->mmap_pages == UINT_MAX)
765 				opts->mmap_pages = KiB(256) / page_size;
766 		}
767 	}
768 
769 	/* Validate auxtrace_mmap_pages */
770 	if (opts->auxtrace_mmap_pages) {
771 		size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
772 		size_t min_sz;
773 
774 		if (opts->auxtrace_snapshot_mode || opts->auxtrace_sample_mode)
775 			min_sz = KiB(4);
776 		else
777 			min_sz = KiB(8);
778 
779 		if (sz < min_sz || !is_power_of_2(sz)) {
780 			pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n",
781 			       min_sz / 1024);
782 			return -EINVAL;
783 		}
784 	}
785 
786 	if (!opts->auxtrace_snapshot_mode && !opts->auxtrace_sample_mode) {
787 		u32 aux_watermark = opts->auxtrace_mmap_pages * page_size / 4;
788 
789 		intel_pt_evsel->core.attr.aux_watermark = aux_watermark;
790 	}
791 
792 	intel_pt_parse_terms(intel_pt_pmu->name, &intel_pt_pmu->format,
793 			     "tsc", &tsc_bit);
794 
795 	if (opts->full_auxtrace && (intel_pt_evsel->core.attr.config & tsc_bit))
796 		have_timing_info = true;
797 	else
798 		have_timing_info = false;
799 
800 	/*
801 	 * Per-cpu recording needs sched_switch events to distinguish different
802 	 * threads.
803 	 */
804 	if (have_timing_info && !perf_cpu_map__empty(cpus) &&
805 	    !record_opts__no_switch_events(opts)) {
806 		if (perf_can_record_switch_events()) {
807 			bool cpu_wide = !target__none(&opts->target) &&
808 					!target__has_task(&opts->target);
809 
810 			if (!cpu_wide && perf_can_record_cpu_wide()) {
811 				struct evsel *switch_evsel;
812 
813 				err = parse_events(evlist, "dummy:u", NULL);
814 				if (err)
815 					return err;
816 
817 				switch_evsel = evlist__last(evlist);
818 
819 				switch_evsel->core.attr.freq = 0;
820 				switch_evsel->core.attr.sample_period = 1;
821 				switch_evsel->core.attr.context_switch = 1;
822 
823 				switch_evsel->core.system_wide = true;
824 				switch_evsel->no_aux_samples = true;
825 				switch_evsel->immediate = true;
826 
827 				evsel__set_sample_bit(switch_evsel, TID);
828 				evsel__set_sample_bit(switch_evsel, TIME);
829 				evsel__set_sample_bit(switch_evsel, CPU);
830 				evsel__reset_sample_bit(switch_evsel, BRANCH_STACK);
831 
832 				opts->record_switch_events = false;
833 				ptr->have_sched_switch = 3;
834 			} else {
835 				opts->record_switch_events = true;
836 				need_immediate = true;
837 				if (cpu_wide)
838 					ptr->have_sched_switch = 3;
839 				else
840 					ptr->have_sched_switch = 2;
841 			}
842 		} else {
843 			err = intel_pt_track_switches(evlist);
844 			if (err == -EPERM)
845 				pr_debug2("Unable to select sched:sched_switch\n");
846 			else if (err)
847 				return err;
848 			else
849 				ptr->have_sched_switch = 1;
850 		}
851 	}
852 
853 	if (have_timing_info && !intel_pt_evsel->core.attr.exclude_kernel &&
854 	    perf_can_record_text_poke_events() && perf_can_record_cpu_wide())
855 		opts->text_poke = true;
856 
857 	if (intel_pt_evsel) {
858 		/*
859 		 * To obtain the auxtrace buffer file descriptor, the auxtrace
860 		 * event must come first.
861 		 */
862 		evlist__to_front(evlist, intel_pt_evsel);
863 		/*
864 		 * In the case of per-cpu mmaps, we need the CPU on the
865 		 * AUX event.
866 		 */
867 		if (!perf_cpu_map__empty(cpus))
868 			evsel__set_sample_bit(intel_pt_evsel, CPU);
869 	}
870 
871 	/* Add dummy event to keep tracking */
872 	if (opts->full_auxtrace) {
873 		struct evsel *tracking_evsel;
874 
875 		err = parse_events(evlist, "dummy:u", NULL);
876 		if (err)
877 			return err;
878 
879 		tracking_evsel = evlist__last(evlist);
880 
881 		evlist__set_tracking_event(evlist, tracking_evsel);
882 
883 		tracking_evsel->core.attr.freq = 0;
884 		tracking_evsel->core.attr.sample_period = 1;
885 
886 		tracking_evsel->no_aux_samples = true;
887 		if (need_immediate)
888 			tracking_evsel->immediate = true;
889 
890 		/* In per-cpu case, always need the time of mmap events etc */
891 		if (!perf_cpu_map__empty(cpus)) {
892 			evsel__set_sample_bit(tracking_evsel, TIME);
893 			/* And the CPU for switch events */
894 			evsel__set_sample_bit(tracking_evsel, CPU);
895 		}
896 		evsel__reset_sample_bit(tracking_evsel, BRANCH_STACK);
897 	}
898 
899 	/*
900 	 * Warn the user when we do not have enough information to decode i.e.
901 	 * per-cpu with no sched_switch (except workload-only).
902 	 */
903 	if (!ptr->have_sched_switch && !perf_cpu_map__empty(cpus) &&
904 	    !target__none(&opts->target) &&
905 	    !intel_pt_evsel->core.attr.exclude_user)
906 		ui__warning("Intel Processor Trace decoding will not be possible except for kernel tracing!\n");
907 
908 	return 0;
909 }
910 
911 static int intel_pt_snapshot_start(struct auxtrace_record *itr)
912 {
913 	struct intel_pt_recording *ptr =
914 			container_of(itr, struct intel_pt_recording, itr);
915 	struct evsel *evsel;
916 
917 	evlist__for_each_entry(ptr->evlist, evsel) {
918 		if (evsel->core.attr.type == ptr->intel_pt_pmu->type)
919 			return evsel__disable(evsel);
920 	}
921 	return -EINVAL;
922 }
923 
924 static int intel_pt_snapshot_finish(struct auxtrace_record *itr)
925 {
926 	struct intel_pt_recording *ptr =
927 			container_of(itr, struct intel_pt_recording, itr);
928 	struct evsel *evsel;
929 
930 	evlist__for_each_entry(ptr->evlist, evsel) {
931 		if (evsel->core.attr.type == ptr->intel_pt_pmu->type)
932 			return evsel__enable(evsel);
933 	}
934 	return -EINVAL;
935 }
936 
937 static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording *ptr, int idx)
938 {
939 	const size_t sz = sizeof(struct intel_pt_snapshot_ref);
940 	int cnt = ptr->snapshot_ref_cnt, new_cnt = cnt * 2;
941 	struct intel_pt_snapshot_ref *refs;
942 
943 	if (!new_cnt)
944 		new_cnt = 16;
945 
946 	while (new_cnt <= idx)
947 		new_cnt *= 2;
948 
949 	refs = calloc(new_cnt, sz);
950 	if (!refs)
951 		return -ENOMEM;
952 
953 	memcpy(refs, ptr->snapshot_refs, cnt * sz);
954 
955 	ptr->snapshot_refs = refs;
956 	ptr->snapshot_ref_cnt = new_cnt;
957 
958 	return 0;
959 }
960 
961 static void intel_pt_free_snapshot_refs(struct intel_pt_recording *ptr)
962 {
963 	int i;
964 
965 	for (i = 0; i < ptr->snapshot_ref_cnt; i++)
966 		zfree(&ptr->snapshot_refs[i].ref_buf);
967 	zfree(&ptr->snapshot_refs);
968 }
969 
970 static void intel_pt_recording_free(struct auxtrace_record *itr)
971 {
972 	struct intel_pt_recording *ptr =
973 			container_of(itr, struct intel_pt_recording, itr);
974 
975 	intel_pt_free_snapshot_refs(ptr);
976 	free(ptr);
977 }
978 
979 static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording *ptr, int idx,
980 				       size_t snapshot_buf_size)
981 {
982 	size_t ref_buf_size = ptr->snapshot_ref_buf_size;
983 	void *ref_buf;
984 
985 	ref_buf = zalloc(ref_buf_size);
986 	if (!ref_buf)
987 		return -ENOMEM;
988 
989 	ptr->snapshot_refs[idx].ref_buf = ref_buf;
990 	ptr->snapshot_refs[idx].ref_offset = snapshot_buf_size - ref_buf_size;
991 
992 	return 0;
993 }
994 
995 static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording *ptr,
996 					     size_t snapshot_buf_size)
997 {
998 	const size_t max_size = 256 * 1024;
999 	size_t buf_size = 0, psb_period;
1000 
1001 	if (ptr->snapshot_size <= 64 * 1024)
1002 		return 0;
1003 
1004 	psb_period = intel_pt_psb_period(ptr->intel_pt_pmu, ptr->evlist);
1005 	if (psb_period)
1006 		buf_size = psb_period * 2;
1007 
1008 	if (!buf_size || buf_size > max_size)
1009 		buf_size = max_size;
1010 
1011 	if (buf_size >= snapshot_buf_size)
1012 		return 0;
1013 
1014 	if (buf_size >= ptr->snapshot_size / 2)
1015 		return 0;
1016 
1017 	return buf_size;
1018 }
1019 
1020 static int intel_pt_snapshot_init(struct intel_pt_recording *ptr,
1021 				  size_t snapshot_buf_size)
1022 {
1023 	if (ptr->snapshot_init_done)
1024 		return 0;
1025 
1026 	ptr->snapshot_init_done = true;
1027 
1028 	ptr->snapshot_ref_buf_size = intel_pt_snapshot_ref_buf_size(ptr,
1029 							snapshot_buf_size);
1030 
1031 	return 0;
1032 }
1033 
1034 /**
1035  * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer.
1036  * @buf1: first buffer
1037  * @compare_size: number of bytes to compare
1038  * @buf2: second buffer (a circular buffer)
1039  * @offs2: offset in second buffer
1040  * @buf2_size: size of second buffer
1041  *
1042  * The comparison allows for the possibility that the bytes to compare in the
1043  * circular buffer are not contiguous.  It is assumed that @compare_size <=
1044  * @buf2_size.  This function returns %false if the bytes are identical, %true
1045  * otherwise.
1046  */
1047 static bool intel_pt_compare_buffers(void *buf1, size_t compare_size,
1048 				     void *buf2, size_t offs2, size_t buf2_size)
1049 {
1050 	size_t end2 = offs2 + compare_size, part_size;
1051 
1052 	if (end2 <= buf2_size)
1053 		return memcmp(buf1, buf2 + offs2, compare_size);
1054 
1055 	part_size = end2 - buf2_size;
1056 	if (memcmp(buf1, buf2 + offs2, part_size))
1057 		return true;
1058 
1059 	compare_size -= part_size;
1060 
1061 	return memcmp(buf1 + part_size, buf2, compare_size);
1062 }
1063 
1064 static bool intel_pt_compare_ref(void *ref_buf, size_t ref_offset,
1065 				 size_t ref_size, size_t buf_size,
1066 				 void *data, size_t head)
1067 {
1068 	size_t ref_end = ref_offset + ref_size;
1069 
1070 	if (ref_end > buf_size) {
1071 		if (head > ref_offset || head < ref_end - buf_size)
1072 			return true;
1073 	} else if (head > ref_offset && head < ref_end) {
1074 		return true;
1075 	}
1076 
1077 	return intel_pt_compare_buffers(ref_buf, ref_size, data, ref_offset,
1078 					buf_size);
1079 }
1080 
1081 static void intel_pt_copy_ref(void *ref_buf, size_t ref_size, size_t buf_size,
1082 			      void *data, size_t head)
1083 {
1084 	if (head >= ref_size) {
1085 		memcpy(ref_buf, data + head - ref_size, ref_size);
1086 	} else {
1087 		memcpy(ref_buf, data, head);
1088 		ref_size -= head;
1089 		memcpy(ref_buf + head, data + buf_size - ref_size, ref_size);
1090 	}
1091 }
1092 
1093 static bool intel_pt_wrapped(struct intel_pt_recording *ptr, int idx,
1094 			     struct auxtrace_mmap *mm, unsigned char *data,
1095 			     u64 head)
1096 {
1097 	struct intel_pt_snapshot_ref *ref = &ptr->snapshot_refs[idx];
1098 	bool wrapped;
1099 
1100 	wrapped = intel_pt_compare_ref(ref->ref_buf, ref->ref_offset,
1101 				       ptr->snapshot_ref_buf_size, mm->len,
1102 				       data, head);
1103 
1104 	intel_pt_copy_ref(ref->ref_buf, ptr->snapshot_ref_buf_size, mm->len,
1105 			  data, head);
1106 
1107 	return wrapped;
1108 }
1109 
1110 static bool intel_pt_first_wrap(u64 *data, size_t buf_size)
1111 {
1112 	int i, a, b;
1113 
1114 	b = buf_size >> 3;
1115 	a = b - 512;
1116 	if (a < 0)
1117 		a = 0;
1118 
1119 	for (i = a; i < b; i++) {
1120 		if (data[i])
1121 			return true;
1122 	}
1123 
1124 	return false;
1125 }
1126 
1127 static int intel_pt_find_snapshot(struct auxtrace_record *itr, int idx,
1128 				  struct auxtrace_mmap *mm, unsigned char *data,
1129 				  u64 *head, u64 *old)
1130 {
1131 	struct intel_pt_recording *ptr =
1132 			container_of(itr, struct intel_pt_recording, itr);
1133 	bool wrapped;
1134 	int err;
1135 
1136 	pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
1137 		  __func__, idx, (size_t)*old, (size_t)*head);
1138 
1139 	err = intel_pt_snapshot_init(ptr, mm->len);
1140 	if (err)
1141 		goto out_err;
1142 
1143 	if (idx >= ptr->snapshot_ref_cnt) {
1144 		err = intel_pt_alloc_snapshot_refs(ptr, idx);
1145 		if (err)
1146 			goto out_err;
1147 	}
1148 
1149 	if (ptr->snapshot_ref_buf_size) {
1150 		if (!ptr->snapshot_refs[idx].ref_buf) {
1151 			err = intel_pt_alloc_snapshot_ref(ptr, idx, mm->len);
1152 			if (err)
1153 				goto out_err;
1154 		}
1155 		wrapped = intel_pt_wrapped(ptr, idx, mm, data, *head);
1156 	} else {
1157 		wrapped = ptr->snapshot_refs[idx].wrapped;
1158 		if (!wrapped && intel_pt_first_wrap((u64 *)data, mm->len)) {
1159 			ptr->snapshot_refs[idx].wrapped = true;
1160 			wrapped = true;
1161 		}
1162 	}
1163 
1164 	/*
1165 	 * In full trace mode 'head' continually increases.  However in snapshot
1166 	 * mode 'head' is an offset within the buffer.  Here 'old' and 'head'
1167 	 * are adjusted to match the full trace case which expects that 'old' is
1168 	 * always less than 'head'.
1169 	 */
1170 	if (wrapped) {
1171 		*old = *head;
1172 		*head += mm->len;
1173 	} else {
1174 		if (mm->mask)
1175 			*old &= mm->mask;
1176 		else
1177 			*old %= mm->len;
1178 		if (*old > *head)
1179 			*head += mm->len;
1180 	}
1181 
1182 	pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
1183 		  __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head);
1184 
1185 	return 0;
1186 
1187 out_err:
1188 	pr_err("%s: failed, error %d\n", __func__, err);
1189 	return err;
1190 }
1191 
1192 static u64 intel_pt_reference(struct auxtrace_record *itr __maybe_unused)
1193 {
1194 	return rdtsc();
1195 }
1196 
1197 struct auxtrace_record *intel_pt_recording_init(int *err)
1198 {
1199 	struct perf_pmu *intel_pt_pmu = perf_pmu__find(INTEL_PT_PMU_NAME);
1200 	struct intel_pt_recording *ptr;
1201 
1202 	if (!intel_pt_pmu)
1203 		return NULL;
1204 
1205 	if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
1206 		*err = -errno;
1207 		return NULL;
1208 	}
1209 
1210 	ptr = zalloc(sizeof(struct intel_pt_recording));
1211 	if (!ptr) {
1212 		*err = -ENOMEM;
1213 		return NULL;
1214 	}
1215 
1216 	ptr->intel_pt_pmu = intel_pt_pmu;
1217 	ptr->itr.pmu = intel_pt_pmu;
1218 	ptr->itr.recording_options = intel_pt_recording_options;
1219 	ptr->itr.info_priv_size = intel_pt_info_priv_size;
1220 	ptr->itr.info_fill = intel_pt_info_fill;
1221 	ptr->itr.free = intel_pt_recording_free;
1222 	ptr->itr.snapshot_start = intel_pt_snapshot_start;
1223 	ptr->itr.snapshot_finish = intel_pt_snapshot_finish;
1224 	ptr->itr.find_snapshot = intel_pt_find_snapshot;
1225 	ptr->itr.parse_snapshot_options = intel_pt_parse_snapshot_options;
1226 	ptr->itr.reference = intel_pt_reference;
1227 	ptr->itr.read_finish = auxtrace_record__read_finish;
1228 	/*
1229 	 * Decoding starts at a PSB packet. Minimum PSB period is 2K so 4K
1230 	 * should give at least 1 PSB per sample.
1231 	 */
1232 	ptr->itr.default_aux_sample_size = 4096;
1233 	return &ptr->itr;
1234 }
1235