xref: /linux/tools/perf/util/evsel.c (revision 72932371e78012cea96edb9e833d81f1c32dd892)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
4  *
5  * Parts came from builtin-{top,stat,record}.c, see those files for further
6  * copyright notes.
7  */
8 
9 #include <byteswap.h>
10 #include <errno.h>
11 #include <inttypes.h>
12 #include <linux/bitops.h>
13 #include <api/fs/fs.h>
14 #include <api/fs/tracing_path.h>
15 #include <traceevent/event-parse.h>
16 #include <linux/hw_breakpoint.h>
17 #include <linux/perf_event.h>
18 #include <linux/compiler.h>
19 #include <linux/err.h>
20 #include <linux/zalloc.h>
21 #include <sys/ioctl.h>
22 #include <sys/resource.h>
23 #include <sys/types.h>
24 #include <dirent.h>
25 #include <perf/evsel.h>
26 #include "asm/bug.h"
27 #include "callchain.h"
28 #include "cgroup.h"
29 #include "counts.h"
30 #include "event.h"
31 #include "evsel.h"
32 #include "evlist.h"
33 #include "cpumap.h"
34 #include "thread_map.h"
35 #include "target.h"
36 #include "perf_regs.h"
37 #include "record.h"
38 #include "debug.h"
39 #include "trace-event.h"
40 #include "stat.h"
41 #include "string2.h"
42 #include "memswap.h"
43 #include "util.h"
44 #include "util/parse-branch-options.h"
45 #include <internal/xyarray.h>
46 
47 #include <linux/ctype.h>
48 
49 struct perf_missing_features perf_missing_features;
50 
51 static clockid_t clockid;
52 
53 static int perf_evsel__no_extra_init(struct evsel *evsel __maybe_unused)
54 {
55 	return 0;
56 }
57 
58 void __weak test_attr__ready(void) { }
59 
60 static void perf_evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
61 {
62 }
63 
64 static struct {
65 	size_t	size;
66 	int	(*init)(struct evsel *evsel);
67 	void	(*fini)(struct evsel *evsel);
68 } perf_evsel__object = {
69 	.size = sizeof(struct evsel),
70 	.init = perf_evsel__no_extra_init,
71 	.fini = perf_evsel__no_extra_fini,
72 };
73 
74 int perf_evsel__object_config(size_t object_size,
75 			      int (*init)(struct evsel *evsel),
76 			      void (*fini)(struct evsel *evsel))
77 {
78 
79 	if (object_size == 0)
80 		goto set_methods;
81 
82 	if (perf_evsel__object.size > object_size)
83 		return -EINVAL;
84 
85 	perf_evsel__object.size = object_size;
86 
87 set_methods:
88 	if (init != NULL)
89 		perf_evsel__object.init = init;
90 
91 	if (fini != NULL)
92 		perf_evsel__object.fini = fini;
93 
94 	return 0;
95 }
96 
97 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
98 
99 int __perf_evsel__sample_size(u64 sample_type)
100 {
101 	u64 mask = sample_type & PERF_SAMPLE_MASK;
102 	int size = 0;
103 	int i;
104 
105 	for (i = 0; i < 64; i++) {
106 		if (mask & (1ULL << i))
107 			size++;
108 	}
109 
110 	size *= sizeof(u64);
111 
112 	return size;
113 }
114 
115 /**
116  * __perf_evsel__calc_id_pos - calculate id_pos.
117  * @sample_type: sample type
118  *
119  * This function returns the position of the event id (PERF_SAMPLE_ID or
120  * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
121  * perf_record_sample.
122  */
123 static int __perf_evsel__calc_id_pos(u64 sample_type)
124 {
125 	int idx = 0;
126 
127 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
128 		return 0;
129 
130 	if (!(sample_type & PERF_SAMPLE_ID))
131 		return -1;
132 
133 	if (sample_type & PERF_SAMPLE_IP)
134 		idx += 1;
135 
136 	if (sample_type & PERF_SAMPLE_TID)
137 		idx += 1;
138 
139 	if (sample_type & PERF_SAMPLE_TIME)
140 		idx += 1;
141 
142 	if (sample_type & PERF_SAMPLE_ADDR)
143 		idx += 1;
144 
145 	return idx;
146 }
147 
148 /**
149  * __perf_evsel__calc_is_pos - calculate is_pos.
150  * @sample_type: sample type
151  *
152  * This function returns the position (counting backwards) of the event id
153  * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
154  * sample_id_all is used there is an id sample appended to non-sample events.
155  */
156 static int __perf_evsel__calc_is_pos(u64 sample_type)
157 {
158 	int idx = 1;
159 
160 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
161 		return 1;
162 
163 	if (!(sample_type & PERF_SAMPLE_ID))
164 		return -1;
165 
166 	if (sample_type & PERF_SAMPLE_CPU)
167 		idx += 1;
168 
169 	if (sample_type & PERF_SAMPLE_STREAM_ID)
170 		idx += 1;
171 
172 	return idx;
173 }
174 
175 void perf_evsel__calc_id_pos(struct evsel *evsel)
176 {
177 	evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
178 	evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
179 }
180 
181 void __perf_evsel__set_sample_bit(struct evsel *evsel,
182 				  enum perf_event_sample_format bit)
183 {
184 	if (!(evsel->core.attr.sample_type & bit)) {
185 		evsel->core.attr.sample_type |= bit;
186 		evsel->sample_size += sizeof(u64);
187 		perf_evsel__calc_id_pos(evsel);
188 	}
189 }
190 
191 void __perf_evsel__reset_sample_bit(struct evsel *evsel,
192 				    enum perf_event_sample_format bit)
193 {
194 	if (evsel->core.attr.sample_type & bit) {
195 		evsel->core.attr.sample_type &= ~bit;
196 		evsel->sample_size -= sizeof(u64);
197 		perf_evsel__calc_id_pos(evsel);
198 	}
199 }
200 
201 void perf_evsel__set_sample_id(struct evsel *evsel,
202 			       bool can_sample_identifier)
203 {
204 	if (can_sample_identifier) {
205 		perf_evsel__reset_sample_bit(evsel, ID);
206 		perf_evsel__set_sample_bit(evsel, IDENTIFIER);
207 	} else {
208 		perf_evsel__set_sample_bit(evsel, ID);
209 	}
210 	evsel->core.attr.read_format |= PERF_FORMAT_ID;
211 }
212 
213 /**
214  * perf_evsel__is_function_event - Return whether given evsel is a function
215  * trace event
216  *
217  * @evsel - evsel selector to be tested
218  *
219  * Return %true if event is function trace event
220  */
221 bool perf_evsel__is_function_event(struct evsel *evsel)
222 {
223 #define FUNCTION_EVENT "ftrace:function"
224 
225 	return evsel->name &&
226 	       !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
227 
228 #undef FUNCTION_EVENT
229 }
230 
231 void evsel__init(struct evsel *evsel,
232 		 struct perf_event_attr *attr, int idx)
233 {
234 	perf_evsel__init(&evsel->core, attr);
235 	evsel->idx	   = idx;
236 	evsel->tracking	   = !idx;
237 	evsel->leader	   = evsel;
238 	evsel->unit	   = "";
239 	evsel->scale	   = 1.0;
240 	evsel->max_events  = ULONG_MAX;
241 	evsel->evlist	   = NULL;
242 	evsel->bpf_obj	   = NULL;
243 	evsel->bpf_fd	   = -1;
244 	INIT_LIST_HEAD(&evsel->config_terms);
245 	perf_evsel__object.init(evsel);
246 	evsel->sample_size = __perf_evsel__sample_size(attr->sample_type);
247 	perf_evsel__calc_id_pos(evsel);
248 	evsel->cmdline_group_boundary = false;
249 	evsel->metric_expr   = NULL;
250 	evsel->metric_name   = NULL;
251 	evsel->metric_events = NULL;
252 	evsel->collect_stat  = false;
253 	evsel->pmu_name      = NULL;
254 }
255 
256 struct evsel *perf_evsel__new_idx(struct perf_event_attr *attr, int idx)
257 {
258 	struct evsel *evsel = zalloc(perf_evsel__object.size);
259 
260 	if (!evsel)
261 		return NULL;
262 	evsel__init(evsel, attr, idx);
263 
264 	if (perf_evsel__is_bpf_output(evsel)) {
265 		evsel->core.attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
266 					    PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
267 		evsel->core.attr.sample_period = 1;
268 	}
269 
270 	if (perf_evsel__is_clock(evsel)) {
271 		/*
272 		 * The evsel->unit points to static alias->unit
273 		 * so it's ok to use static string in here.
274 		 */
275 		static const char *unit = "msec";
276 
277 		evsel->unit = unit;
278 		evsel->scale = 1e-6;
279 	}
280 
281 	return evsel;
282 }
283 
284 static bool perf_event_can_profile_kernel(void)
285 {
286 	return perf_event_paranoid_check(1);
287 }
288 
289 struct evsel *perf_evsel__new_cycles(bool precise)
290 {
291 	struct perf_event_attr attr = {
292 		.type	= PERF_TYPE_HARDWARE,
293 		.config	= PERF_COUNT_HW_CPU_CYCLES,
294 		.exclude_kernel	= !perf_event_can_profile_kernel(),
295 	};
296 	struct evsel *evsel;
297 
298 	event_attr_init(&attr);
299 
300 	if (!precise)
301 		goto new_event;
302 
303 	/*
304 	 * Now let the usual logic to set up the perf_event_attr defaults
305 	 * to kick in when we return and before perf_evsel__open() is called.
306 	 */
307 new_event:
308 	evsel = evsel__new(&attr);
309 	if (evsel == NULL)
310 		goto out;
311 
312 	evsel->precise_max = true;
313 
314 	/* use asprintf() because free(evsel) assumes name is allocated */
315 	if (asprintf(&evsel->name, "cycles%s%s%.*s",
316 		     (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
317 		     attr.exclude_kernel ? "u" : "",
318 		     attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
319 		goto error_free;
320 out:
321 	return evsel;
322 error_free:
323 	evsel__delete(evsel);
324 	evsel = NULL;
325 	goto out;
326 }
327 
328 /*
329  * Returns pointer with encoded error via <linux/err.h> interface.
330  */
331 struct evsel *perf_evsel__newtp_idx(const char *sys, const char *name, int idx)
332 {
333 	struct evsel *evsel = zalloc(perf_evsel__object.size);
334 	int err = -ENOMEM;
335 
336 	if (evsel == NULL) {
337 		goto out_err;
338 	} else {
339 		struct perf_event_attr attr = {
340 			.type	       = PERF_TYPE_TRACEPOINT,
341 			.sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
342 					  PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
343 		};
344 
345 		if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
346 			goto out_free;
347 
348 		evsel->tp_format = trace_event__tp_format(sys, name);
349 		if (IS_ERR(evsel->tp_format)) {
350 			err = PTR_ERR(evsel->tp_format);
351 			goto out_free;
352 		}
353 
354 		event_attr_init(&attr);
355 		attr.config = evsel->tp_format->id;
356 		attr.sample_period = 1;
357 		evsel__init(evsel, &attr, idx);
358 	}
359 
360 	return evsel;
361 
362 out_free:
363 	zfree(&evsel->name);
364 	free(evsel);
365 out_err:
366 	return ERR_PTR(err);
367 }
368 
369 const char *perf_evsel__hw_names[PERF_COUNT_HW_MAX] = {
370 	"cycles",
371 	"instructions",
372 	"cache-references",
373 	"cache-misses",
374 	"branches",
375 	"branch-misses",
376 	"bus-cycles",
377 	"stalled-cycles-frontend",
378 	"stalled-cycles-backend",
379 	"ref-cycles",
380 };
381 
382 static const char *__perf_evsel__hw_name(u64 config)
383 {
384 	if (config < PERF_COUNT_HW_MAX && perf_evsel__hw_names[config])
385 		return perf_evsel__hw_names[config];
386 
387 	return "unknown-hardware";
388 }
389 
390 static int perf_evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
391 {
392 	int colon = 0, r = 0;
393 	struct perf_event_attr *attr = &evsel->core.attr;
394 	bool exclude_guest_default = false;
395 
396 #define MOD_PRINT(context, mod)	do {					\
397 		if (!attr->exclude_##context) {				\
398 			if (!colon) colon = ++r;			\
399 			r += scnprintf(bf + r, size - r, "%c", mod);	\
400 		} } while(0)
401 
402 	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
403 		MOD_PRINT(kernel, 'k');
404 		MOD_PRINT(user, 'u');
405 		MOD_PRINT(hv, 'h');
406 		exclude_guest_default = true;
407 	}
408 
409 	if (attr->precise_ip) {
410 		if (!colon)
411 			colon = ++r;
412 		r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
413 		exclude_guest_default = true;
414 	}
415 
416 	if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
417 		MOD_PRINT(host, 'H');
418 		MOD_PRINT(guest, 'G');
419 	}
420 #undef MOD_PRINT
421 	if (colon)
422 		bf[colon - 1] = ':';
423 	return r;
424 }
425 
426 static int perf_evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
427 {
428 	int r = scnprintf(bf, size, "%s", __perf_evsel__hw_name(evsel->core.attr.config));
429 	return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
430 }
431 
432 const char *perf_evsel__sw_names[PERF_COUNT_SW_MAX] = {
433 	"cpu-clock",
434 	"task-clock",
435 	"page-faults",
436 	"context-switches",
437 	"cpu-migrations",
438 	"minor-faults",
439 	"major-faults",
440 	"alignment-faults",
441 	"emulation-faults",
442 	"dummy",
443 };
444 
445 static const char *__perf_evsel__sw_name(u64 config)
446 {
447 	if (config < PERF_COUNT_SW_MAX && perf_evsel__sw_names[config])
448 		return perf_evsel__sw_names[config];
449 	return "unknown-software";
450 }
451 
452 static int perf_evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
453 {
454 	int r = scnprintf(bf, size, "%s", __perf_evsel__sw_name(evsel->core.attr.config));
455 	return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
456 }
457 
458 static int __perf_evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
459 {
460 	int r;
461 
462 	r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
463 
464 	if (type & HW_BREAKPOINT_R)
465 		r += scnprintf(bf + r, size - r, "r");
466 
467 	if (type & HW_BREAKPOINT_W)
468 		r += scnprintf(bf + r, size - r, "w");
469 
470 	if (type & HW_BREAKPOINT_X)
471 		r += scnprintf(bf + r, size - r, "x");
472 
473 	return r;
474 }
475 
476 static int perf_evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
477 {
478 	struct perf_event_attr *attr = &evsel->core.attr;
479 	int r = __perf_evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
480 	return r + perf_evsel__add_modifiers(evsel, bf + r, size - r);
481 }
482 
483 const char *perf_evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX]
484 				[PERF_EVSEL__MAX_ALIASES] = {
485  { "L1-dcache",	"l1-d",		"l1d",		"L1-data",		},
486  { "L1-icache",	"l1-i",		"l1i",		"L1-instruction",	},
487  { "LLC",	"L2",							},
488  { "dTLB",	"d-tlb",	"Data-TLB",				},
489  { "iTLB",	"i-tlb",	"Instruction-TLB",			},
490  { "branch",	"branches",	"bpu",		"btb",		"bpc",	},
491  { "node",								},
492 };
493 
494 const char *perf_evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX]
495 				   [PERF_EVSEL__MAX_ALIASES] = {
496  { "load",	"loads",	"read",					},
497  { "store",	"stores",	"write",				},
498  { "prefetch",	"prefetches",	"speculative-read", "speculative-load",	},
499 };
500 
501 const char *perf_evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX]
502 				       [PERF_EVSEL__MAX_ALIASES] = {
503  { "refs",	"Reference",	"ops",		"access",		},
504  { "misses",	"miss",							},
505 };
506 
507 #define C(x)		PERF_COUNT_HW_CACHE_##x
508 #define CACHE_READ	(1 << C(OP_READ))
509 #define CACHE_WRITE	(1 << C(OP_WRITE))
510 #define CACHE_PREFETCH	(1 << C(OP_PREFETCH))
511 #define COP(x)		(1 << x)
512 
513 /*
514  * cache operartion stat
515  * L1I : Read and prefetch only
516  * ITLB and BPU : Read-only
517  */
518 static unsigned long perf_evsel__hw_cache_stat[C(MAX)] = {
519  [C(L1D)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
520  [C(L1I)]	= (CACHE_READ | CACHE_PREFETCH),
521  [C(LL)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
522  [C(DTLB)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
523  [C(ITLB)]	= (CACHE_READ),
524  [C(BPU)]	= (CACHE_READ),
525  [C(NODE)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
526 };
527 
528 bool perf_evsel__is_cache_op_valid(u8 type, u8 op)
529 {
530 	if (perf_evsel__hw_cache_stat[type] & COP(op))
531 		return true;	/* valid */
532 	else
533 		return false;	/* invalid */
534 }
535 
536 int __perf_evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result,
537 					    char *bf, size_t size)
538 {
539 	if (result) {
540 		return scnprintf(bf, size, "%s-%s-%s", perf_evsel__hw_cache[type][0],
541 				 perf_evsel__hw_cache_op[op][0],
542 				 perf_evsel__hw_cache_result[result][0]);
543 	}
544 
545 	return scnprintf(bf, size, "%s-%s", perf_evsel__hw_cache[type][0],
546 			 perf_evsel__hw_cache_op[op][1]);
547 }
548 
549 static int __perf_evsel__hw_cache_name(u64 config, char *bf, size_t size)
550 {
551 	u8 op, result, type = (config >>  0) & 0xff;
552 	const char *err = "unknown-ext-hardware-cache-type";
553 
554 	if (type >= PERF_COUNT_HW_CACHE_MAX)
555 		goto out_err;
556 
557 	op = (config >>  8) & 0xff;
558 	err = "unknown-ext-hardware-cache-op";
559 	if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
560 		goto out_err;
561 
562 	result = (config >> 16) & 0xff;
563 	err = "unknown-ext-hardware-cache-result";
564 	if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
565 		goto out_err;
566 
567 	err = "invalid-cache";
568 	if (!perf_evsel__is_cache_op_valid(type, op))
569 		goto out_err;
570 
571 	return __perf_evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
572 out_err:
573 	return scnprintf(bf, size, "%s", err);
574 }
575 
576 static int perf_evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
577 {
578 	int ret = __perf_evsel__hw_cache_name(evsel->core.attr.config, bf, size);
579 	return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
580 }
581 
582 static int perf_evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
583 {
584 	int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
585 	return ret + perf_evsel__add_modifiers(evsel, bf + ret, size - ret);
586 }
587 
588 static int perf_evsel__tool_name(char *bf, size_t size)
589 {
590 	int ret = scnprintf(bf, size, "duration_time");
591 	return ret;
592 }
593 
594 const char *perf_evsel__name(struct evsel *evsel)
595 {
596 	char bf[128];
597 
598 	if (!evsel)
599 		goto out_unknown;
600 
601 	if (evsel->name)
602 		return evsel->name;
603 
604 	switch (evsel->core.attr.type) {
605 	case PERF_TYPE_RAW:
606 		perf_evsel__raw_name(evsel, bf, sizeof(bf));
607 		break;
608 
609 	case PERF_TYPE_HARDWARE:
610 		perf_evsel__hw_name(evsel, bf, sizeof(bf));
611 		break;
612 
613 	case PERF_TYPE_HW_CACHE:
614 		perf_evsel__hw_cache_name(evsel, bf, sizeof(bf));
615 		break;
616 
617 	case PERF_TYPE_SOFTWARE:
618 		if (evsel->tool_event)
619 			perf_evsel__tool_name(bf, sizeof(bf));
620 		else
621 			perf_evsel__sw_name(evsel, bf, sizeof(bf));
622 		break;
623 
624 	case PERF_TYPE_TRACEPOINT:
625 		scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
626 		break;
627 
628 	case PERF_TYPE_BREAKPOINT:
629 		perf_evsel__bp_name(evsel, bf, sizeof(bf));
630 		break;
631 
632 	default:
633 		scnprintf(bf, sizeof(bf), "unknown attr type: %d",
634 			  evsel->core.attr.type);
635 		break;
636 	}
637 
638 	evsel->name = strdup(bf);
639 
640 	if (evsel->name)
641 		return evsel->name;
642 out_unknown:
643 	return "unknown";
644 }
645 
646 const char *perf_evsel__group_name(struct evsel *evsel)
647 {
648 	return evsel->group_name ?: "anon group";
649 }
650 
651 /*
652  * Returns the group details for the specified leader,
653  * with following rules.
654  *
655  *  For record -e '{cycles,instructions}'
656  *    'anon group { cycles:u, instructions:u }'
657  *
658  *  For record -e 'cycles,instructions' and report --group
659  *    'cycles:u, instructions:u'
660  */
661 int perf_evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
662 {
663 	int ret = 0;
664 	struct evsel *pos;
665 	const char *group_name = perf_evsel__group_name(evsel);
666 
667 	if (!evsel->forced_leader)
668 		ret = scnprintf(buf, size, "%s { ", group_name);
669 
670 	ret += scnprintf(buf + ret, size - ret, "%s",
671 			 perf_evsel__name(evsel));
672 
673 	for_each_group_member(pos, evsel)
674 		ret += scnprintf(buf + ret, size - ret, ", %s",
675 				 perf_evsel__name(pos));
676 
677 	if (!evsel->forced_leader)
678 		ret += scnprintf(buf + ret, size - ret, " }");
679 
680 	return ret;
681 }
682 
683 static void __perf_evsel__config_callchain(struct evsel *evsel,
684 					   struct record_opts *opts,
685 					   struct callchain_param *param)
686 {
687 	bool function = perf_evsel__is_function_event(evsel);
688 	struct perf_event_attr *attr = &evsel->core.attr;
689 
690 	perf_evsel__set_sample_bit(evsel, CALLCHAIN);
691 
692 	attr->sample_max_stack = param->max_stack;
693 
694 	if (opts->kernel_callchains)
695 		attr->exclude_callchain_user = 1;
696 	if (opts->user_callchains)
697 		attr->exclude_callchain_kernel = 1;
698 	if (param->record_mode == CALLCHAIN_LBR) {
699 		if (!opts->branch_stack) {
700 			if (attr->exclude_user) {
701 				pr_warning("LBR callstack option is only available "
702 					   "to get user callchain information. "
703 					   "Falling back to framepointers.\n");
704 			} else {
705 				perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
706 				attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
707 							PERF_SAMPLE_BRANCH_CALL_STACK |
708 							PERF_SAMPLE_BRANCH_NO_CYCLES |
709 							PERF_SAMPLE_BRANCH_NO_FLAGS;
710 			}
711 		} else
712 			 pr_warning("Cannot use LBR callstack with branch stack. "
713 				    "Falling back to framepointers.\n");
714 	}
715 
716 	if (param->record_mode == CALLCHAIN_DWARF) {
717 		if (!function) {
718 			perf_evsel__set_sample_bit(evsel, REGS_USER);
719 			perf_evsel__set_sample_bit(evsel, STACK_USER);
720 			if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
721 				attr->sample_regs_user |= DWARF_MINIMAL_REGS;
722 				pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
723 					   "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
724 					   "so the minimal registers set (IP, SP) is explicitly forced.\n");
725 			} else {
726 				attr->sample_regs_user |= PERF_REGS_MASK;
727 			}
728 			attr->sample_stack_user = param->dump_size;
729 			attr->exclude_callchain_user = 1;
730 		} else {
731 			pr_info("Cannot use DWARF unwind for function trace event,"
732 				" falling back to framepointers.\n");
733 		}
734 	}
735 
736 	if (function) {
737 		pr_info("Disabling user space callchains for function trace event.\n");
738 		attr->exclude_callchain_user = 1;
739 	}
740 }
741 
742 void perf_evsel__config_callchain(struct evsel *evsel,
743 				  struct record_opts *opts,
744 				  struct callchain_param *param)
745 {
746 	if (param->enabled)
747 		return __perf_evsel__config_callchain(evsel, opts, param);
748 }
749 
750 static void
751 perf_evsel__reset_callgraph(struct evsel *evsel,
752 			    struct callchain_param *param)
753 {
754 	struct perf_event_attr *attr = &evsel->core.attr;
755 
756 	perf_evsel__reset_sample_bit(evsel, CALLCHAIN);
757 	if (param->record_mode == CALLCHAIN_LBR) {
758 		perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
759 		attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
760 					      PERF_SAMPLE_BRANCH_CALL_STACK);
761 	}
762 	if (param->record_mode == CALLCHAIN_DWARF) {
763 		perf_evsel__reset_sample_bit(evsel, REGS_USER);
764 		perf_evsel__reset_sample_bit(evsel, STACK_USER);
765 	}
766 }
767 
768 static void apply_config_terms(struct evsel *evsel,
769 			       struct record_opts *opts, bool track)
770 {
771 	struct perf_evsel_config_term *term;
772 	struct list_head *config_terms = &evsel->config_terms;
773 	struct perf_event_attr *attr = &evsel->core.attr;
774 	/* callgraph default */
775 	struct callchain_param param = {
776 		.record_mode = callchain_param.record_mode,
777 	};
778 	u32 dump_size = 0;
779 	int max_stack = 0;
780 	const char *callgraph_buf = NULL;
781 
782 	list_for_each_entry(term, config_terms, list) {
783 		switch (term->type) {
784 		case PERF_EVSEL__CONFIG_TERM_PERIOD:
785 			if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
786 				attr->sample_period = term->val.period;
787 				attr->freq = 0;
788 				perf_evsel__reset_sample_bit(evsel, PERIOD);
789 			}
790 			break;
791 		case PERF_EVSEL__CONFIG_TERM_FREQ:
792 			if (!(term->weak && opts->user_freq != UINT_MAX)) {
793 				attr->sample_freq = term->val.freq;
794 				attr->freq = 1;
795 				perf_evsel__set_sample_bit(evsel, PERIOD);
796 			}
797 			break;
798 		case PERF_EVSEL__CONFIG_TERM_TIME:
799 			if (term->val.time)
800 				perf_evsel__set_sample_bit(evsel, TIME);
801 			else
802 				perf_evsel__reset_sample_bit(evsel, TIME);
803 			break;
804 		case PERF_EVSEL__CONFIG_TERM_CALLGRAPH:
805 			callgraph_buf = term->val.callgraph;
806 			break;
807 		case PERF_EVSEL__CONFIG_TERM_BRANCH:
808 			if (term->val.branch && strcmp(term->val.branch, "no")) {
809 				perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
810 				parse_branch_str(term->val.branch,
811 						 &attr->branch_sample_type);
812 			} else
813 				perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
814 			break;
815 		case PERF_EVSEL__CONFIG_TERM_STACK_USER:
816 			dump_size = term->val.stack_user;
817 			break;
818 		case PERF_EVSEL__CONFIG_TERM_MAX_STACK:
819 			max_stack = term->val.max_stack;
820 			break;
821 		case PERF_EVSEL__CONFIG_TERM_MAX_EVENTS:
822 			evsel->max_events = term->val.max_events;
823 			break;
824 		case PERF_EVSEL__CONFIG_TERM_INHERIT:
825 			/*
826 			 * attr->inherit should has already been set by
827 			 * perf_evsel__config. If user explicitly set
828 			 * inherit using config terms, override global
829 			 * opt->no_inherit setting.
830 			 */
831 			attr->inherit = term->val.inherit ? 1 : 0;
832 			break;
833 		case PERF_EVSEL__CONFIG_TERM_OVERWRITE:
834 			attr->write_backward = term->val.overwrite ? 1 : 0;
835 			break;
836 		case PERF_EVSEL__CONFIG_TERM_DRV_CFG:
837 			break;
838 		case PERF_EVSEL__CONFIG_TERM_PERCORE:
839 			break;
840 		case PERF_EVSEL__CONFIG_TERM_AUX_OUTPUT:
841 			attr->aux_output = term->val.aux_output ? 1 : 0;
842 			break;
843 		default:
844 			break;
845 		}
846 	}
847 
848 	/* User explicitly set per-event callgraph, clear the old setting and reset. */
849 	if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
850 		bool sample_address = false;
851 
852 		if (max_stack) {
853 			param.max_stack = max_stack;
854 			if (callgraph_buf == NULL)
855 				callgraph_buf = "fp";
856 		}
857 
858 		/* parse callgraph parameters */
859 		if (callgraph_buf != NULL) {
860 			if (!strcmp(callgraph_buf, "no")) {
861 				param.enabled = false;
862 				param.record_mode = CALLCHAIN_NONE;
863 			} else {
864 				param.enabled = true;
865 				if (parse_callchain_record(callgraph_buf, &param)) {
866 					pr_err("per-event callgraph setting for %s failed. "
867 					       "Apply callgraph global setting for it\n",
868 					       evsel->name);
869 					return;
870 				}
871 				if (param.record_mode == CALLCHAIN_DWARF)
872 					sample_address = true;
873 			}
874 		}
875 		if (dump_size > 0) {
876 			dump_size = round_up(dump_size, sizeof(u64));
877 			param.dump_size = dump_size;
878 		}
879 
880 		/* If global callgraph set, clear it */
881 		if (callchain_param.enabled)
882 			perf_evsel__reset_callgraph(evsel, &callchain_param);
883 
884 		/* set perf-event callgraph */
885 		if (param.enabled) {
886 			if (sample_address) {
887 				perf_evsel__set_sample_bit(evsel, ADDR);
888 				perf_evsel__set_sample_bit(evsel, DATA_SRC);
889 				evsel->core.attr.mmap_data = track;
890 			}
891 			perf_evsel__config_callchain(evsel, opts, &param);
892 		}
893 	}
894 }
895 
896 static bool is_dummy_event(struct evsel *evsel)
897 {
898 	return (evsel->core.attr.type == PERF_TYPE_SOFTWARE) &&
899 	       (evsel->core.attr.config == PERF_COUNT_SW_DUMMY);
900 }
901 
902 /*
903  * The enable_on_exec/disabled value strategy:
904  *
905  *  1) For any type of traced program:
906  *    - all independent events and group leaders are disabled
907  *    - all group members are enabled
908  *
909  *     Group members are ruled by group leaders. They need to
910  *     be enabled, because the group scheduling relies on that.
911  *
912  *  2) For traced programs executed by perf:
913  *     - all independent events and group leaders have
914  *       enable_on_exec set
915  *     - we don't specifically enable or disable any event during
916  *       the record command
917  *
918  *     Independent events and group leaders are initially disabled
919  *     and get enabled by exec. Group members are ruled by group
920  *     leaders as stated in 1).
921  *
922  *  3) For traced programs attached by perf (pid/tid):
923  *     - we specifically enable or disable all events during
924  *       the record command
925  *
926  *     When attaching events to already running traced we
927  *     enable/disable events specifically, as there's no
928  *     initial traced exec call.
929  */
930 void perf_evsel__config(struct evsel *evsel, struct record_opts *opts,
931 			struct callchain_param *callchain)
932 {
933 	struct evsel *leader = evsel->leader;
934 	struct perf_event_attr *attr = &evsel->core.attr;
935 	int track = evsel->tracking;
936 	bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
937 
938 	attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
939 	attr->inherit	    = !opts->no_inherit;
940 	attr->write_backward = opts->overwrite ? 1 : 0;
941 
942 	perf_evsel__set_sample_bit(evsel, IP);
943 	perf_evsel__set_sample_bit(evsel, TID);
944 
945 	if (evsel->sample_read) {
946 		perf_evsel__set_sample_bit(evsel, READ);
947 
948 		/*
949 		 * We need ID even in case of single event, because
950 		 * PERF_SAMPLE_READ process ID specific data.
951 		 */
952 		perf_evsel__set_sample_id(evsel, false);
953 
954 		/*
955 		 * Apply group format only if we belong to group
956 		 * with more than one members.
957 		 */
958 		if (leader->core.nr_members > 1) {
959 			attr->read_format |= PERF_FORMAT_GROUP;
960 			attr->inherit = 0;
961 		}
962 	}
963 
964 	/*
965 	 * We default some events to have a default interval. But keep
966 	 * it a weak assumption overridable by the user.
967 	 */
968 	if (!attr->sample_period || (opts->user_freq != UINT_MAX ||
969 				     opts->user_interval != ULLONG_MAX)) {
970 		if (opts->freq) {
971 			perf_evsel__set_sample_bit(evsel, PERIOD);
972 			attr->freq		= 1;
973 			attr->sample_freq	= opts->freq;
974 		} else {
975 			attr->sample_period = opts->default_interval;
976 		}
977 	}
978 
979 	/*
980 	 * Disable sampling for all group members other
981 	 * than leader in case leader 'leads' the sampling.
982 	 */
983 	if ((leader != evsel) && leader->sample_read) {
984 		attr->freq           = 0;
985 		attr->sample_freq    = 0;
986 		attr->sample_period  = 0;
987 		attr->write_backward = 0;
988 
989 		/*
990 		 * We don't get sample for slave events, we make them
991 		 * when delivering group leader sample. Set the slave
992 		 * event to follow the master sample_type to ease up
993 		 * report.
994 		 */
995 		attr->sample_type = leader->core.attr.sample_type;
996 	}
997 
998 	if (opts->no_samples)
999 		attr->sample_freq = 0;
1000 
1001 	if (opts->inherit_stat) {
1002 		evsel->core.attr.read_format |=
1003 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1004 			PERF_FORMAT_TOTAL_TIME_RUNNING |
1005 			PERF_FORMAT_ID;
1006 		attr->inherit_stat = 1;
1007 	}
1008 
1009 	if (opts->sample_address) {
1010 		perf_evsel__set_sample_bit(evsel, ADDR);
1011 		attr->mmap_data = track;
1012 	}
1013 
1014 	/*
1015 	 * We don't allow user space callchains for  function trace
1016 	 * event, due to issues with page faults while tracing page
1017 	 * fault handler and its overall trickiness nature.
1018 	 */
1019 	if (perf_evsel__is_function_event(evsel))
1020 		evsel->core.attr.exclude_callchain_user = 1;
1021 
1022 	if (callchain && callchain->enabled && !evsel->no_aux_samples)
1023 		perf_evsel__config_callchain(evsel, opts, callchain);
1024 
1025 	if (opts->sample_intr_regs) {
1026 		attr->sample_regs_intr = opts->sample_intr_regs;
1027 		perf_evsel__set_sample_bit(evsel, REGS_INTR);
1028 	}
1029 
1030 	if (opts->sample_user_regs) {
1031 		attr->sample_regs_user |= opts->sample_user_regs;
1032 		perf_evsel__set_sample_bit(evsel, REGS_USER);
1033 	}
1034 
1035 	if (target__has_cpu(&opts->target) || opts->sample_cpu)
1036 		perf_evsel__set_sample_bit(evsel, CPU);
1037 
1038 	/*
1039 	 * When the user explicitly disabled time don't force it here.
1040 	 */
1041 	if (opts->sample_time &&
1042 	    (!perf_missing_features.sample_id_all &&
1043 	    (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1044 	     opts->sample_time_set)))
1045 		perf_evsel__set_sample_bit(evsel, TIME);
1046 
1047 	if (opts->raw_samples && !evsel->no_aux_samples) {
1048 		perf_evsel__set_sample_bit(evsel, TIME);
1049 		perf_evsel__set_sample_bit(evsel, RAW);
1050 		perf_evsel__set_sample_bit(evsel, CPU);
1051 	}
1052 
1053 	if (opts->sample_address)
1054 		perf_evsel__set_sample_bit(evsel, DATA_SRC);
1055 
1056 	if (opts->sample_phys_addr)
1057 		perf_evsel__set_sample_bit(evsel, PHYS_ADDR);
1058 
1059 	if (opts->no_buffering) {
1060 		attr->watermark = 0;
1061 		attr->wakeup_events = 1;
1062 	}
1063 	if (opts->branch_stack && !evsel->no_aux_samples) {
1064 		perf_evsel__set_sample_bit(evsel, BRANCH_STACK);
1065 		attr->branch_sample_type = opts->branch_stack;
1066 	}
1067 
1068 	if (opts->sample_weight)
1069 		perf_evsel__set_sample_bit(evsel, WEIGHT);
1070 
1071 	attr->task  = track;
1072 	attr->mmap  = track;
1073 	attr->mmap2 = track && !perf_missing_features.mmap2;
1074 	attr->comm  = track;
1075 	attr->ksymbol = track && !perf_missing_features.ksymbol;
1076 	attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1077 
1078 	if (opts->record_namespaces)
1079 		attr->namespaces  = track;
1080 
1081 	if (opts->record_switch_events)
1082 		attr->context_switch = track;
1083 
1084 	if (opts->sample_transaction)
1085 		perf_evsel__set_sample_bit(evsel, TRANSACTION);
1086 
1087 	if (opts->running_time) {
1088 		evsel->core.attr.read_format |=
1089 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1090 			PERF_FORMAT_TOTAL_TIME_RUNNING;
1091 	}
1092 
1093 	/*
1094 	 * XXX see the function comment above
1095 	 *
1096 	 * Disabling only independent events or group leaders,
1097 	 * keeping group members enabled.
1098 	 */
1099 	if (perf_evsel__is_group_leader(evsel))
1100 		attr->disabled = 1;
1101 
1102 	/*
1103 	 * Setting enable_on_exec for independent events and
1104 	 * group leaders for traced executed by perf.
1105 	 */
1106 	if (target__none(&opts->target) && perf_evsel__is_group_leader(evsel) &&
1107 		!opts->initial_delay)
1108 		attr->enable_on_exec = 1;
1109 
1110 	if (evsel->immediate) {
1111 		attr->disabled = 0;
1112 		attr->enable_on_exec = 0;
1113 	}
1114 
1115 	clockid = opts->clockid;
1116 	if (opts->use_clockid) {
1117 		attr->use_clockid = 1;
1118 		attr->clockid = opts->clockid;
1119 	}
1120 
1121 	if (evsel->precise_max)
1122 		attr->precise_ip = 3;
1123 
1124 	if (opts->all_user) {
1125 		attr->exclude_kernel = 1;
1126 		attr->exclude_user   = 0;
1127 	}
1128 
1129 	if (opts->all_kernel) {
1130 		attr->exclude_kernel = 0;
1131 		attr->exclude_user   = 1;
1132 	}
1133 
1134 	if (evsel->core.own_cpus || evsel->unit)
1135 		evsel->core.attr.read_format |= PERF_FORMAT_ID;
1136 
1137 	/*
1138 	 * Apply event specific term settings,
1139 	 * it overloads any global configuration.
1140 	 */
1141 	apply_config_terms(evsel, opts, track);
1142 
1143 	evsel->ignore_missing_thread = opts->ignore_missing_thread;
1144 
1145 	/* The --period option takes the precedence. */
1146 	if (opts->period_set) {
1147 		if (opts->period)
1148 			perf_evsel__set_sample_bit(evsel, PERIOD);
1149 		else
1150 			perf_evsel__reset_sample_bit(evsel, PERIOD);
1151 	}
1152 
1153 	/*
1154 	 * For initial_delay, a dummy event is added implicitly.
1155 	 * The software event will trigger -EOPNOTSUPP error out,
1156 	 * if BRANCH_STACK bit is set.
1157 	 */
1158 	if (opts->initial_delay && is_dummy_event(evsel))
1159 		perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
1160 }
1161 
1162 int perf_evsel__set_filter(struct evsel *evsel, const char *filter)
1163 {
1164 	char *new_filter = strdup(filter);
1165 
1166 	if (new_filter != NULL) {
1167 		free(evsel->filter);
1168 		evsel->filter = new_filter;
1169 		return 0;
1170 	}
1171 
1172 	return -1;
1173 }
1174 
1175 static int perf_evsel__append_filter(struct evsel *evsel,
1176 				     const char *fmt, const char *filter)
1177 {
1178 	char *new_filter;
1179 
1180 	if (evsel->filter == NULL)
1181 		return perf_evsel__set_filter(evsel, filter);
1182 
1183 	if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1184 		free(evsel->filter);
1185 		evsel->filter = new_filter;
1186 		return 0;
1187 	}
1188 
1189 	return -1;
1190 }
1191 
1192 int perf_evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1193 {
1194 	return perf_evsel__append_filter(evsel, "(%s) && (%s)", filter);
1195 }
1196 
1197 int perf_evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1198 {
1199 	return perf_evsel__append_filter(evsel, "%s,%s", filter);
1200 }
1201 
1202 int evsel__enable(struct evsel *evsel)
1203 {
1204 	int err = perf_evsel__enable(&evsel->core);
1205 
1206 	if (!err)
1207 		evsel->disabled = false;
1208 
1209 	return err;
1210 }
1211 
1212 int evsel__disable(struct evsel *evsel)
1213 {
1214 	int err = perf_evsel__disable(&evsel->core);
1215 	/*
1216 	 * We mark it disabled here so that tools that disable a event can
1217 	 * ignore events after they disable it. I.e. the ring buffer may have
1218 	 * already a few more events queued up before the kernel got the stop
1219 	 * request.
1220 	 */
1221 	if (!err)
1222 		evsel->disabled = true;
1223 
1224 	return err;
1225 }
1226 
1227 int perf_evsel__alloc_id(struct evsel *evsel, int ncpus, int nthreads)
1228 {
1229 	if (ncpus == 0 || nthreads == 0)
1230 		return 0;
1231 
1232 	if (evsel->system_wide)
1233 		nthreads = 1;
1234 
1235 	evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
1236 	if (evsel->sample_id == NULL)
1237 		return -ENOMEM;
1238 
1239 	evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
1240 	if (evsel->id == NULL) {
1241 		xyarray__delete(evsel->sample_id);
1242 		evsel->sample_id = NULL;
1243 		return -ENOMEM;
1244 	}
1245 
1246 	return 0;
1247 }
1248 
1249 static void perf_evsel__free_id(struct evsel *evsel)
1250 {
1251 	xyarray__delete(evsel->sample_id);
1252 	evsel->sample_id = NULL;
1253 	zfree(&evsel->id);
1254 	evsel->ids = 0;
1255 }
1256 
1257 static void perf_evsel__free_config_terms(struct evsel *evsel)
1258 {
1259 	struct perf_evsel_config_term *term, *h;
1260 
1261 	list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1262 		list_del_init(&term->list);
1263 		free(term);
1264 	}
1265 }
1266 
1267 void perf_evsel__exit(struct evsel *evsel)
1268 {
1269 	assert(list_empty(&evsel->core.node));
1270 	assert(evsel->evlist == NULL);
1271 	perf_evsel__free_counts(evsel);
1272 	perf_evsel__free_fd(&evsel->core);
1273 	perf_evsel__free_id(evsel);
1274 	perf_evsel__free_config_terms(evsel);
1275 	cgroup__put(evsel->cgrp);
1276 	perf_cpu_map__put(evsel->core.cpus);
1277 	perf_cpu_map__put(evsel->core.own_cpus);
1278 	perf_thread_map__put(evsel->core.threads);
1279 	zfree(&evsel->group_name);
1280 	zfree(&evsel->name);
1281 	perf_evsel__object.fini(evsel);
1282 }
1283 
1284 void evsel__delete(struct evsel *evsel)
1285 {
1286 	perf_evsel__exit(evsel);
1287 	free(evsel);
1288 }
1289 
1290 void perf_evsel__compute_deltas(struct evsel *evsel, int cpu, int thread,
1291 				struct perf_counts_values *count)
1292 {
1293 	struct perf_counts_values tmp;
1294 
1295 	if (!evsel->prev_raw_counts)
1296 		return;
1297 
1298 	if (cpu == -1) {
1299 		tmp = evsel->prev_raw_counts->aggr;
1300 		evsel->prev_raw_counts->aggr = *count;
1301 	} else {
1302 		tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1303 		*perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1304 	}
1305 
1306 	count->val = count->val - tmp.val;
1307 	count->ena = count->ena - tmp.ena;
1308 	count->run = count->run - tmp.run;
1309 }
1310 
1311 void perf_counts_values__scale(struct perf_counts_values *count,
1312 			       bool scale, s8 *pscaled)
1313 {
1314 	s8 scaled = 0;
1315 
1316 	if (scale) {
1317 		if (count->run == 0) {
1318 			scaled = -1;
1319 			count->val = 0;
1320 		} else if (count->run < count->ena) {
1321 			scaled = 1;
1322 			count->val = (u64)((double) count->val * count->ena / count->run);
1323 		}
1324 	}
1325 
1326 	if (pscaled)
1327 		*pscaled = scaled;
1328 }
1329 
1330 static int
1331 perf_evsel__read_one(struct evsel *evsel, int cpu, int thread)
1332 {
1333 	struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1334 
1335 	return perf_evsel__read(&evsel->core, cpu, thread, count);
1336 }
1337 
1338 static void
1339 perf_evsel__set_count(struct evsel *counter, int cpu, int thread,
1340 		      u64 val, u64 ena, u64 run)
1341 {
1342 	struct perf_counts_values *count;
1343 
1344 	count = perf_counts(counter->counts, cpu, thread);
1345 
1346 	count->val    = val;
1347 	count->ena    = ena;
1348 	count->run    = run;
1349 
1350 	perf_counts__set_loaded(counter->counts, cpu, thread, true);
1351 }
1352 
1353 static int
1354 perf_evsel__process_group_data(struct evsel *leader,
1355 			       int cpu, int thread, u64 *data)
1356 {
1357 	u64 read_format = leader->core.attr.read_format;
1358 	struct sample_read_value *v;
1359 	u64 nr, ena = 0, run = 0, i;
1360 
1361 	nr = *data++;
1362 
1363 	if (nr != (u64) leader->core.nr_members)
1364 		return -EINVAL;
1365 
1366 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1367 		ena = *data++;
1368 
1369 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1370 		run = *data++;
1371 
1372 	v = (struct sample_read_value *) data;
1373 
1374 	perf_evsel__set_count(leader, cpu, thread,
1375 			      v[0].value, ena, run);
1376 
1377 	for (i = 1; i < nr; i++) {
1378 		struct evsel *counter;
1379 
1380 		counter = perf_evlist__id2evsel(leader->evlist, v[i].id);
1381 		if (!counter)
1382 			return -EINVAL;
1383 
1384 		perf_evsel__set_count(counter, cpu, thread,
1385 				      v[i].value, ena, run);
1386 	}
1387 
1388 	return 0;
1389 }
1390 
1391 static int
1392 perf_evsel__read_group(struct evsel *leader, int cpu, int thread)
1393 {
1394 	struct perf_stat_evsel *ps = leader->stats;
1395 	u64 read_format = leader->core.attr.read_format;
1396 	int size = perf_evsel__read_size(&leader->core);
1397 	u64 *data = ps->group_data;
1398 
1399 	if (!(read_format & PERF_FORMAT_ID))
1400 		return -EINVAL;
1401 
1402 	if (!perf_evsel__is_group_leader(leader))
1403 		return -EINVAL;
1404 
1405 	if (!data) {
1406 		data = zalloc(size);
1407 		if (!data)
1408 			return -ENOMEM;
1409 
1410 		ps->group_data = data;
1411 	}
1412 
1413 	if (FD(leader, cpu, thread) < 0)
1414 		return -EINVAL;
1415 
1416 	if (readn(FD(leader, cpu, thread), data, size) <= 0)
1417 		return -errno;
1418 
1419 	return perf_evsel__process_group_data(leader, cpu, thread, data);
1420 }
1421 
1422 int perf_evsel__read_counter(struct evsel *evsel, int cpu, int thread)
1423 {
1424 	u64 read_format = evsel->core.attr.read_format;
1425 
1426 	if (read_format & PERF_FORMAT_GROUP)
1427 		return perf_evsel__read_group(evsel, cpu, thread);
1428 	else
1429 		return perf_evsel__read_one(evsel, cpu, thread);
1430 }
1431 
1432 int __perf_evsel__read_on_cpu(struct evsel *evsel,
1433 			      int cpu, int thread, bool scale)
1434 {
1435 	struct perf_counts_values count;
1436 	size_t nv = scale ? 3 : 1;
1437 
1438 	if (FD(evsel, cpu, thread) < 0)
1439 		return -EINVAL;
1440 
1441 	if (evsel->counts == NULL && perf_evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1442 		return -ENOMEM;
1443 
1444 	if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1445 		return -errno;
1446 
1447 	perf_evsel__compute_deltas(evsel, cpu, thread, &count);
1448 	perf_counts_values__scale(&count, scale, NULL);
1449 	*perf_counts(evsel->counts, cpu, thread) = count;
1450 	return 0;
1451 }
1452 
1453 static int get_group_fd(struct evsel *evsel, int cpu, int thread)
1454 {
1455 	struct evsel *leader = evsel->leader;
1456 	int fd;
1457 
1458 	if (perf_evsel__is_group_leader(evsel))
1459 		return -1;
1460 
1461 	/*
1462 	 * Leader must be already processed/open,
1463 	 * if not it's a bug.
1464 	 */
1465 	BUG_ON(!leader->core.fd);
1466 
1467 	fd = FD(leader, cpu, thread);
1468 	BUG_ON(fd == -1);
1469 
1470 	return fd;
1471 }
1472 
1473 struct bit_names {
1474 	int bit;
1475 	const char *name;
1476 };
1477 
1478 static void __p_bits(char *buf, size_t size, u64 value, struct bit_names *bits)
1479 {
1480 	bool first_bit = true;
1481 	int i = 0;
1482 
1483 	do {
1484 		if (value & bits[i].bit) {
1485 			buf += scnprintf(buf, size, "%s%s", first_bit ? "" : "|", bits[i].name);
1486 			first_bit = false;
1487 		}
1488 	} while (bits[++i].name != NULL);
1489 }
1490 
1491 static void __p_sample_type(char *buf, size_t size, u64 value)
1492 {
1493 #define bit_name(n) { PERF_SAMPLE_##n, #n }
1494 	struct bit_names bits[] = {
1495 		bit_name(IP), bit_name(TID), bit_name(TIME), bit_name(ADDR),
1496 		bit_name(READ), bit_name(CALLCHAIN), bit_name(ID), bit_name(CPU),
1497 		bit_name(PERIOD), bit_name(STREAM_ID), bit_name(RAW),
1498 		bit_name(BRANCH_STACK), bit_name(REGS_USER), bit_name(STACK_USER),
1499 		bit_name(IDENTIFIER), bit_name(REGS_INTR), bit_name(DATA_SRC),
1500 		bit_name(WEIGHT), bit_name(PHYS_ADDR),
1501 		{ .name = NULL, }
1502 	};
1503 #undef bit_name
1504 	__p_bits(buf, size, value, bits);
1505 }
1506 
1507 static void __p_branch_sample_type(char *buf, size_t size, u64 value)
1508 {
1509 #define bit_name(n) { PERF_SAMPLE_BRANCH_##n, #n }
1510 	struct bit_names bits[] = {
1511 		bit_name(USER), bit_name(KERNEL), bit_name(HV), bit_name(ANY),
1512 		bit_name(ANY_CALL), bit_name(ANY_RETURN), bit_name(IND_CALL),
1513 		bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
1514 		bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
1515 		bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
1516 		{ .name = NULL, }
1517 	};
1518 #undef bit_name
1519 	__p_bits(buf, size, value, bits);
1520 }
1521 
1522 static void __p_read_format(char *buf, size_t size, u64 value)
1523 {
1524 #define bit_name(n) { PERF_FORMAT_##n, #n }
1525 	struct bit_names bits[] = {
1526 		bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
1527 		bit_name(ID), bit_name(GROUP),
1528 		{ .name = NULL, }
1529 	};
1530 #undef bit_name
1531 	__p_bits(buf, size, value, bits);
1532 }
1533 
1534 #define BUF_SIZE		1024
1535 
1536 #define p_hex(val)		snprintf(buf, BUF_SIZE, "%#"PRIx64, (uint64_t)(val))
1537 #define p_unsigned(val)		snprintf(buf, BUF_SIZE, "%"PRIu64, (uint64_t)(val))
1538 #define p_signed(val)		snprintf(buf, BUF_SIZE, "%"PRId64, (int64_t)(val))
1539 #define p_sample_type(val)	__p_sample_type(buf, BUF_SIZE, val)
1540 #define p_branch_sample_type(val) __p_branch_sample_type(buf, BUF_SIZE, val)
1541 #define p_read_format(val)	__p_read_format(buf, BUF_SIZE, val)
1542 
1543 #define PRINT_ATTRn(_n, _f, _p)				\
1544 do {							\
1545 	if (attr->_f) {					\
1546 		_p(attr->_f);				\
1547 		ret += attr__fprintf(fp, _n, buf, priv);\
1548 	}						\
1549 } while (0)
1550 
1551 #define PRINT_ATTRf(_f, _p)	PRINT_ATTRn(#_f, _f, _p)
1552 
1553 int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr,
1554 			     attr__fprintf_f attr__fprintf, void *priv)
1555 {
1556 	char buf[BUF_SIZE];
1557 	int ret = 0;
1558 
1559 	PRINT_ATTRf(type, p_unsigned);
1560 	PRINT_ATTRf(size, p_unsigned);
1561 	PRINT_ATTRf(config, p_hex);
1562 	PRINT_ATTRn("{ sample_period, sample_freq }", sample_period, p_unsigned);
1563 	PRINT_ATTRf(sample_type, p_sample_type);
1564 	PRINT_ATTRf(read_format, p_read_format);
1565 
1566 	PRINT_ATTRf(disabled, p_unsigned);
1567 	PRINT_ATTRf(inherit, p_unsigned);
1568 	PRINT_ATTRf(pinned, p_unsigned);
1569 	PRINT_ATTRf(exclusive, p_unsigned);
1570 	PRINT_ATTRf(exclude_user, p_unsigned);
1571 	PRINT_ATTRf(exclude_kernel, p_unsigned);
1572 	PRINT_ATTRf(exclude_hv, p_unsigned);
1573 	PRINT_ATTRf(exclude_idle, p_unsigned);
1574 	PRINT_ATTRf(mmap, p_unsigned);
1575 	PRINT_ATTRf(comm, p_unsigned);
1576 	PRINT_ATTRf(freq, p_unsigned);
1577 	PRINT_ATTRf(inherit_stat, p_unsigned);
1578 	PRINT_ATTRf(enable_on_exec, p_unsigned);
1579 	PRINT_ATTRf(task, p_unsigned);
1580 	PRINT_ATTRf(watermark, p_unsigned);
1581 	PRINT_ATTRf(precise_ip, p_unsigned);
1582 	PRINT_ATTRf(mmap_data, p_unsigned);
1583 	PRINT_ATTRf(sample_id_all, p_unsigned);
1584 	PRINT_ATTRf(exclude_host, p_unsigned);
1585 	PRINT_ATTRf(exclude_guest, p_unsigned);
1586 	PRINT_ATTRf(exclude_callchain_kernel, p_unsigned);
1587 	PRINT_ATTRf(exclude_callchain_user, p_unsigned);
1588 	PRINT_ATTRf(mmap2, p_unsigned);
1589 	PRINT_ATTRf(comm_exec, p_unsigned);
1590 	PRINT_ATTRf(use_clockid, p_unsigned);
1591 	PRINT_ATTRf(context_switch, p_unsigned);
1592 	PRINT_ATTRf(write_backward, p_unsigned);
1593 	PRINT_ATTRf(namespaces, p_unsigned);
1594 	PRINT_ATTRf(ksymbol, p_unsigned);
1595 	PRINT_ATTRf(bpf_event, p_unsigned);
1596 	PRINT_ATTRf(aux_output, p_unsigned);
1597 
1598 	PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned);
1599 	PRINT_ATTRf(bp_type, p_unsigned);
1600 	PRINT_ATTRn("{ bp_addr, config1 }", bp_addr, p_hex);
1601 	PRINT_ATTRn("{ bp_len, config2 }", bp_len, p_hex);
1602 	PRINT_ATTRf(branch_sample_type, p_branch_sample_type);
1603 	PRINT_ATTRf(sample_regs_user, p_hex);
1604 	PRINT_ATTRf(sample_stack_user, p_unsigned);
1605 	PRINT_ATTRf(clockid, p_signed);
1606 	PRINT_ATTRf(sample_regs_intr, p_hex);
1607 	PRINT_ATTRf(aux_watermark, p_unsigned);
1608 	PRINT_ATTRf(sample_max_stack, p_unsigned);
1609 
1610 	return ret;
1611 }
1612 
1613 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1614 				void *priv __maybe_unused)
1615 {
1616 	return fprintf(fp, "  %-32s %s\n", name, val);
1617 }
1618 
1619 static void perf_evsel__remove_fd(struct evsel *pos,
1620 				  int nr_cpus, int nr_threads,
1621 				  int thread_idx)
1622 {
1623 	for (int cpu = 0; cpu < nr_cpus; cpu++)
1624 		for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1625 			FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1626 }
1627 
1628 static int update_fds(struct evsel *evsel,
1629 		      int nr_cpus, int cpu_idx,
1630 		      int nr_threads, int thread_idx)
1631 {
1632 	struct evsel *pos;
1633 
1634 	if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1635 		return -EINVAL;
1636 
1637 	evlist__for_each_entry(evsel->evlist, pos) {
1638 		nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1639 
1640 		perf_evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1641 
1642 		/*
1643 		 * Since fds for next evsel has not been created,
1644 		 * there is no need to iterate whole event list.
1645 		 */
1646 		if (pos == evsel)
1647 			break;
1648 	}
1649 	return 0;
1650 }
1651 
1652 static bool ignore_missing_thread(struct evsel *evsel,
1653 				  int nr_cpus, int cpu,
1654 				  struct perf_thread_map *threads,
1655 				  int thread, int err)
1656 {
1657 	pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1658 
1659 	if (!evsel->ignore_missing_thread)
1660 		return false;
1661 
1662 	/* The system wide setup does not work with threads. */
1663 	if (evsel->system_wide)
1664 		return false;
1665 
1666 	/* The -ESRCH is perf event syscall errno for pid's not found. */
1667 	if (err != -ESRCH)
1668 		return false;
1669 
1670 	/* If there's only one thread, let it fail. */
1671 	if (threads->nr == 1)
1672 		return false;
1673 
1674 	/*
1675 	 * We should remove fd for missing_thread first
1676 	 * because thread_map__remove() will decrease threads->nr.
1677 	 */
1678 	if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1679 		return false;
1680 
1681 	if (thread_map__remove(threads, thread))
1682 		return false;
1683 
1684 	pr_warning("WARNING: Ignored open failure for pid %d\n",
1685 		   ignore_pid);
1686 	return true;
1687 }
1688 
1689 static void display_attr(struct perf_event_attr *attr)
1690 {
1691 	if (verbose >= 2) {
1692 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1693 		fprintf(stderr, "perf_event_attr:\n");
1694 		perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1695 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1696 	}
1697 }
1698 
1699 static int perf_event_open(struct evsel *evsel,
1700 			   pid_t pid, int cpu, int group_fd,
1701 			   unsigned long flags)
1702 {
1703 	int precise_ip = evsel->core.attr.precise_ip;
1704 	int fd;
1705 
1706 	while (1) {
1707 		pr_debug2("sys_perf_event_open: pid %d  cpu %d  group_fd %d  flags %#lx",
1708 			  pid, cpu, group_fd, flags);
1709 
1710 		fd = sys_perf_event_open(&evsel->core.attr, pid, cpu, group_fd, flags);
1711 		if (fd >= 0)
1712 			break;
1713 
1714 		/* Do not try less precise if not requested. */
1715 		if (!evsel->precise_max)
1716 			break;
1717 
1718 		/*
1719 		 * We tried all the precise_ip values, and it's
1720 		 * still failing, so leave it to standard fallback.
1721 		 */
1722 		if (!evsel->core.attr.precise_ip) {
1723 			evsel->core.attr.precise_ip = precise_ip;
1724 			break;
1725 		}
1726 
1727 		pr_debug2("\nsys_perf_event_open failed, error %d\n", -ENOTSUP);
1728 		evsel->core.attr.precise_ip--;
1729 		pr_debug2("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1730 		display_attr(&evsel->core.attr);
1731 	}
1732 
1733 	return fd;
1734 }
1735 
1736 int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
1737 		struct perf_thread_map *threads)
1738 {
1739 	int cpu, thread, nthreads;
1740 	unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1741 	int pid = -1, err;
1742 	enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1743 
1744 	if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1745 	    (perf_missing_features.aux_output     && evsel->core.attr.aux_output))
1746 		return -EINVAL;
1747 
1748 	if (cpus == NULL) {
1749 		static struct perf_cpu_map *empty_cpu_map;
1750 
1751 		if (empty_cpu_map == NULL) {
1752 			empty_cpu_map = perf_cpu_map__dummy_new();
1753 			if (empty_cpu_map == NULL)
1754 				return -ENOMEM;
1755 		}
1756 
1757 		cpus = empty_cpu_map;
1758 	}
1759 
1760 	if (threads == NULL) {
1761 		static struct perf_thread_map *empty_thread_map;
1762 
1763 		if (empty_thread_map == NULL) {
1764 			empty_thread_map = thread_map__new_by_tid(-1);
1765 			if (empty_thread_map == NULL)
1766 				return -ENOMEM;
1767 		}
1768 
1769 		threads = empty_thread_map;
1770 	}
1771 
1772 	if (evsel->system_wide)
1773 		nthreads = 1;
1774 	else
1775 		nthreads = threads->nr;
1776 
1777 	if (evsel->core.fd == NULL &&
1778 	    perf_evsel__alloc_fd(&evsel->core, cpus->nr, nthreads) < 0)
1779 		return -ENOMEM;
1780 
1781 	if (evsel->cgrp) {
1782 		flags |= PERF_FLAG_PID_CGROUP;
1783 		pid = evsel->cgrp->fd;
1784 	}
1785 
1786 fallback_missing_features:
1787 	if (perf_missing_features.clockid_wrong)
1788 		evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */
1789 	if (perf_missing_features.clockid) {
1790 		evsel->core.attr.use_clockid = 0;
1791 		evsel->core.attr.clockid = 0;
1792 	}
1793 	if (perf_missing_features.cloexec)
1794 		flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1795 	if (perf_missing_features.mmap2)
1796 		evsel->core.attr.mmap2 = 0;
1797 	if (perf_missing_features.exclude_guest)
1798 		evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1799 	if (perf_missing_features.lbr_flags)
1800 		evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1801 				     PERF_SAMPLE_BRANCH_NO_CYCLES);
1802 	if (perf_missing_features.group_read && evsel->core.attr.inherit)
1803 		evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1804 	if (perf_missing_features.ksymbol)
1805 		evsel->core.attr.ksymbol = 0;
1806 	if (perf_missing_features.bpf)
1807 		evsel->core.attr.bpf_event = 0;
1808 retry_sample_id:
1809 	if (perf_missing_features.sample_id_all)
1810 		evsel->core.attr.sample_id_all = 0;
1811 
1812 	display_attr(&evsel->core.attr);
1813 
1814 	for (cpu = 0; cpu < cpus->nr; cpu++) {
1815 
1816 		for (thread = 0; thread < nthreads; thread++) {
1817 			int fd, group_fd;
1818 
1819 			if (!evsel->cgrp && !evsel->system_wide)
1820 				pid = perf_thread_map__pid(threads, thread);
1821 
1822 			group_fd = get_group_fd(evsel, cpu, thread);
1823 retry_open:
1824 			test_attr__ready();
1825 
1826 			fd = perf_event_open(evsel, pid, cpus->map[cpu],
1827 					     group_fd, flags);
1828 
1829 			FD(evsel, cpu, thread) = fd;
1830 
1831 			if (fd < 0) {
1832 				err = -errno;
1833 
1834 				if (ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
1835 					/*
1836 					 * We just removed 1 thread, so take a step
1837 					 * back on thread index and lower the upper
1838 					 * nthreads limit.
1839 					 */
1840 					nthreads--;
1841 					thread--;
1842 
1843 					/* ... and pretend like nothing have happened. */
1844 					err = 0;
1845 					continue;
1846 				}
1847 
1848 				pr_debug2("\nsys_perf_event_open failed, error %d\n",
1849 					  err);
1850 				goto try_fallback;
1851 			}
1852 
1853 			pr_debug2(" = %d\n", fd);
1854 
1855 			if (evsel->bpf_fd >= 0) {
1856 				int evt_fd = fd;
1857 				int bpf_fd = evsel->bpf_fd;
1858 
1859 				err = ioctl(evt_fd,
1860 					    PERF_EVENT_IOC_SET_BPF,
1861 					    bpf_fd);
1862 				if (err && errno != EEXIST) {
1863 					pr_err("failed to attach bpf fd %d: %s\n",
1864 					       bpf_fd, strerror(errno));
1865 					err = -EINVAL;
1866 					goto out_close;
1867 				}
1868 			}
1869 
1870 			set_rlimit = NO_CHANGE;
1871 
1872 			/*
1873 			 * If we succeeded but had to kill clockid, fail and
1874 			 * have perf_evsel__open_strerror() print us a nice
1875 			 * error.
1876 			 */
1877 			if (perf_missing_features.clockid ||
1878 			    perf_missing_features.clockid_wrong) {
1879 				err = -EINVAL;
1880 				goto out_close;
1881 			}
1882 		}
1883 	}
1884 
1885 	return 0;
1886 
1887 try_fallback:
1888 	/*
1889 	 * perf stat needs between 5 and 22 fds per CPU. When we run out
1890 	 * of them try to increase the limits.
1891 	 */
1892 	if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1893 		struct rlimit l;
1894 		int old_errno = errno;
1895 
1896 		if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1897 			if (set_rlimit == NO_CHANGE)
1898 				l.rlim_cur = l.rlim_max;
1899 			else {
1900 				l.rlim_cur = l.rlim_max + 1000;
1901 				l.rlim_max = l.rlim_cur;
1902 			}
1903 			if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1904 				set_rlimit++;
1905 				errno = old_errno;
1906 				goto retry_open;
1907 			}
1908 		}
1909 		errno = old_errno;
1910 	}
1911 
1912 	if (err != -EINVAL || cpu > 0 || thread > 0)
1913 		goto out_close;
1914 
1915 	/*
1916 	 * Must probe features in the order they were added to the
1917 	 * perf_event_attr interface.
1918 	 */
1919 	if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1920 		perf_missing_features.aux_output = true;
1921 		pr_debug2("Kernel has no attr.aux_output support, bailing out\n");
1922 		goto out_close;
1923 	} else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1924 		perf_missing_features.bpf = true;
1925 		pr_debug2("switching off bpf_event\n");
1926 		goto fallback_missing_features;
1927 	} else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1928 		perf_missing_features.ksymbol = true;
1929 		pr_debug2("switching off ksymbol\n");
1930 		goto fallback_missing_features;
1931 	} else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1932 		perf_missing_features.write_backward = true;
1933 		pr_debug2("switching off write_backward\n");
1934 		goto out_close;
1935 	} else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1936 		perf_missing_features.clockid_wrong = true;
1937 		pr_debug2("switching off clockid\n");
1938 		goto fallback_missing_features;
1939 	} else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1940 		perf_missing_features.clockid = true;
1941 		pr_debug2("switching off use_clockid\n");
1942 		goto fallback_missing_features;
1943 	} else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1944 		perf_missing_features.cloexec = true;
1945 		pr_debug2("switching off cloexec flag\n");
1946 		goto fallback_missing_features;
1947 	} else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1948 		perf_missing_features.mmap2 = true;
1949 		pr_debug2("switching off mmap2\n");
1950 		goto fallback_missing_features;
1951 	} else if (!perf_missing_features.exclude_guest &&
1952 		   (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host)) {
1953 		perf_missing_features.exclude_guest = true;
1954 		pr_debug2("switching off exclude_guest, exclude_host\n");
1955 		goto fallback_missing_features;
1956 	} else if (!perf_missing_features.sample_id_all) {
1957 		perf_missing_features.sample_id_all = true;
1958 		pr_debug2("switching off sample_id_all\n");
1959 		goto retry_sample_id;
1960 	} else if (!perf_missing_features.lbr_flags &&
1961 			(evsel->core.attr.branch_sample_type &
1962 			 (PERF_SAMPLE_BRANCH_NO_CYCLES |
1963 			  PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1964 		perf_missing_features.lbr_flags = true;
1965 		pr_debug2("switching off branch sample type no (cycles/flags)\n");
1966 		goto fallback_missing_features;
1967 	} else if (!perf_missing_features.group_read &&
1968 		    evsel->core.attr.inherit &&
1969 		   (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
1970 		   perf_evsel__is_group_leader(evsel)) {
1971 		perf_missing_features.group_read = true;
1972 		pr_debug2("switching off group read\n");
1973 		goto fallback_missing_features;
1974 	}
1975 out_close:
1976 	if (err)
1977 		threads->err_thread = thread;
1978 
1979 	do {
1980 		while (--thread >= 0) {
1981 			close(FD(evsel, cpu, thread));
1982 			FD(evsel, cpu, thread) = -1;
1983 		}
1984 		thread = nthreads;
1985 	} while (--cpu >= 0);
1986 	return err;
1987 }
1988 
1989 void evsel__close(struct evsel *evsel)
1990 {
1991 	perf_evsel__close(&evsel->core);
1992 	perf_evsel__free_id(evsel);
1993 }
1994 
1995 int perf_evsel__open_per_cpu(struct evsel *evsel,
1996 			     struct perf_cpu_map *cpus)
1997 {
1998 	return evsel__open(evsel, cpus, NULL);
1999 }
2000 
2001 int perf_evsel__open_per_thread(struct evsel *evsel,
2002 				struct perf_thread_map *threads)
2003 {
2004 	return evsel__open(evsel, NULL, threads);
2005 }
2006 
2007 static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2008 				       const union perf_event *event,
2009 				       struct perf_sample *sample)
2010 {
2011 	u64 type = evsel->core.attr.sample_type;
2012 	const __u64 *array = event->sample.array;
2013 	bool swapped = evsel->needs_swap;
2014 	union u64_swap u;
2015 
2016 	array += ((event->header.size -
2017 		   sizeof(event->header)) / sizeof(u64)) - 1;
2018 
2019 	if (type & PERF_SAMPLE_IDENTIFIER) {
2020 		sample->id = *array;
2021 		array--;
2022 	}
2023 
2024 	if (type & PERF_SAMPLE_CPU) {
2025 		u.val64 = *array;
2026 		if (swapped) {
2027 			/* undo swap of u64, then swap on individual u32s */
2028 			u.val64 = bswap_64(u.val64);
2029 			u.val32[0] = bswap_32(u.val32[0]);
2030 		}
2031 
2032 		sample->cpu = u.val32[0];
2033 		array--;
2034 	}
2035 
2036 	if (type & PERF_SAMPLE_STREAM_ID) {
2037 		sample->stream_id = *array;
2038 		array--;
2039 	}
2040 
2041 	if (type & PERF_SAMPLE_ID) {
2042 		sample->id = *array;
2043 		array--;
2044 	}
2045 
2046 	if (type & PERF_SAMPLE_TIME) {
2047 		sample->time = *array;
2048 		array--;
2049 	}
2050 
2051 	if (type & PERF_SAMPLE_TID) {
2052 		u.val64 = *array;
2053 		if (swapped) {
2054 			/* undo swap of u64, then swap on individual u32s */
2055 			u.val64 = bswap_64(u.val64);
2056 			u.val32[0] = bswap_32(u.val32[0]);
2057 			u.val32[1] = bswap_32(u.val32[1]);
2058 		}
2059 
2060 		sample->pid = u.val32[0];
2061 		sample->tid = u.val32[1];
2062 		array--;
2063 	}
2064 
2065 	return 0;
2066 }
2067 
2068 static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2069 			    u64 size)
2070 {
2071 	return size > max_size || offset + size > endp;
2072 }
2073 
2074 #define OVERFLOW_CHECK(offset, size, max_size)				\
2075 	do {								\
2076 		if (overflow(endp, (max_size), (offset), (size)))	\
2077 			return -EFAULT;					\
2078 	} while (0)
2079 
2080 #define OVERFLOW_CHECK_u64(offset) \
2081 	OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2082 
2083 static int
2084 perf_event__check_size(union perf_event *event, unsigned int sample_size)
2085 {
2086 	/*
2087 	 * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
2088 	 * up to PERF_SAMPLE_PERIOD.  After that overflow() must be used to
2089 	 * check the format does not go past the end of the event.
2090 	 */
2091 	if (sample_size + sizeof(event->header) > event->header.size)
2092 		return -EFAULT;
2093 
2094 	return 0;
2095 }
2096 
2097 int perf_evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2098 			     struct perf_sample *data)
2099 {
2100 	u64 type = evsel->core.attr.sample_type;
2101 	bool swapped = evsel->needs_swap;
2102 	const __u64 *array;
2103 	u16 max_size = event->header.size;
2104 	const void *endp = (void *)event + max_size;
2105 	u64 sz;
2106 
2107 	/*
2108 	 * used for cross-endian analysis. See git commit 65014ab3
2109 	 * for why this goofiness is needed.
2110 	 */
2111 	union u64_swap u;
2112 
2113 	memset(data, 0, sizeof(*data));
2114 	data->cpu = data->pid = data->tid = -1;
2115 	data->stream_id = data->id = data->time = -1ULL;
2116 	data->period = evsel->core.attr.sample_period;
2117 	data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2118 	data->misc    = event->header.misc;
2119 	data->id = -1ULL;
2120 	data->data_src = PERF_MEM_DATA_SRC_NONE;
2121 
2122 	if (event->header.type != PERF_RECORD_SAMPLE) {
2123 		if (!evsel->core.attr.sample_id_all)
2124 			return 0;
2125 		return perf_evsel__parse_id_sample(evsel, event, data);
2126 	}
2127 
2128 	array = event->sample.array;
2129 
2130 	if (perf_event__check_size(event, evsel->sample_size))
2131 		return -EFAULT;
2132 
2133 	if (type & PERF_SAMPLE_IDENTIFIER) {
2134 		data->id = *array;
2135 		array++;
2136 	}
2137 
2138 	if (type & PERF_SAMPLE_IP) {
2139 		data->ip = *array;
2140 		array++;
2141 	}
2142 
2143 	if (type & PERF_SAMPLE_TID) {
2144 		u.val64 = *array;
2145 		if (swapped) {
2146 			/* undo swap of u64, then swap on individual u32s */
2147 			u.val64 = bswap_64(u.val64);
2148 			u.val32[0] = bswap_32(u.val32[0]);
2149 			u.val32[1] = bswap_32(u.val32[1]);
2150 		}
2151 
2152 		data->pid = u.val32[0];
2153 		data->tid = u.val32[1];
2154 		array++;
2155 	}
2156 
2157 	if (type & PERF_SAMPLE_TIME) {
2158 		data->time = *array;
2159 		array++;
2160 	}
2161 
2162 	if (type & PERF_SAMPLE_ADDR) {
2163 		data->addr = *array;
2164 		array++;
2165 	}
2166 
2167 	if (type & PERF_SAMPLE_ID) {
2168 		data->id = *array;
2169 		array++;
2170 	}
2171 
2172 	if (type & PERF_SAMPLE_STREAM_ID) {
2173 		data->stream_id = *array;
2174 		array++;
2175 	}
2176 
2177 	if (type & PERF_SAMPLE_CPU) {
2178 
2179 		u.val64 = *array;
2180 		if (swapped) {
2181 			/* undo swap of u64, then swap on individual u32s */
2182 			u.val64 = bswap_64(u.val64);
2183 			u.val32[0] = bswap_32(u.val32[0]);
2184 		}
2185 
2186 		data->cpu = u.val32[0];
2187 		array++;
2188 	}
2189 
2190 	if (type & PERF_SAMPLE_PERIOD) {
2191 		data->period = *array;
2192 		array++;
2193 	}
2194 
2195 	if (type & PERF_SAMPLE_READ) {
2196 		u64 read_format = evsel->core.attr.read_format;
2197 
2198 		OVERFLOW_CHECK_u64(array);
2199 		if (read_format & PERF_FORMAT_GROUP)
2200 			data->read.group.nr = *array;
2201 		else
2202 			data->read.one.value = *array;
2203 
2204 		array++;
2205 
2206 		if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2207 			OVERFLOW_CHECK_u64(array);
2208 			data->read.time_enabled = *array;
2209 			array++;
2210 		}
2211 
2212 		if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2213 			OVERFLOW_CHECK_u64(array);
2214 			data->read.time_running = *array;
2215 			array++;
2216 		}
2217 
2218 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2219 		if (read_format & PERF_FORMAT_GROUP) {
2220 			const u64 max_group_nr = UINT64_MAX /
2221 					sizeof(struct sample_read_value);
2222 
2223 			if (data->read.group.nr > max_group_nr)
2224 				return -EFAULT;
2225 			sz = data->read.group.nr *
2226 			     sizeof(struct sample_read_value);
2227 			OVERFLOW_CHECK(array, sz, max_size);
2228 			data->read.group.values =
2229 					(struct sample_read_value *)array;
2230 			array = (void *)array + sz;
2231 		} else {
2232 			OVERFLOW_CHECK_u64(array);
2233 			data->read.one.id = *array;
2234 			array++;
2235 		}
2236 	}
2237 
2238 	if (evsel__has_callchain(evsel)) {
2239 		const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2240 
2241 		OVERFLOW_CHECK_u64(array);
2242 		data->callchain = (struct ip_callchain *)array++;
2243 		if (data->callchain->nr > max_callchain_nr)
2244 			return -EFAULT;
2245 		sz = data->callchain->nr * sizeof(u64);
2246 		OVERFLOW_CHECK(array, sz, max_size);
2247 		array = (void *)array + sz;
2248 	}
2249 
2250 	if (type & PERF_SAMPLE_RAW) {
2251 		OVERFLOW_CHECK_u64(array);
2252 		u.val64 = *array;
2253 
2254 		/*
2255 		 * Undo swap of u64, then swap on individual u32s,
2256 		 * get the size of the raw area and undo all of the
2257 		 * swap. The pevent interface handles endianity by
2258 		 * itself.
2259 		 */
2260 		if (swapped) {
2261 			u.val64 = bswap_64(u.val64);
2262 			u.val32[0] = bswap_32(u.val32[0]);
2263 			u.val32[1] = bswap_32(u.val32[1]);
2264 		}
2265 		data->raw_size = u.val32[0];
2266 
2267 		/*
2268 		 * The raw data is aligned on 64bits including the
2269 		 * u32 size, so it's safe to use mem_bswap_64.
2270 		 */
2271 		if (swapped)
2272 			mem_bswap_64((void *) array, data->raw_size);
2273 
2274 		array = (void *)array + sizeof(u32);
2275 
2276 		OVERFLOW_CHECK(array, data->raw_size, max_size);
2277 		data->raw_data = (void *)array;
2278 		array = (void *)array + data->raw_size;
2279 	}
2280 
2281 	if (type & PERF_SAMPLE_BRANCH_STACK) {
2282 		const u64 max_branch_nr = UINT64_MAX /
2283 					  sizeof(struct branch_entry);
2284 
2285 		OVERFLOW_CHECK_u64(array);
2286 		data->branch_stack = (struct branch_stack *)array++;
2287 
2288 		if (data->branch_stack->nr > max_branch_nr)
2289 			return -EFAULT;
2290 		sz = data->branch_stack->nr * sizeof(struct branch_entry);
2291 		OVERFLOW_CHECK(array, sz, max_size);
2292 		array = (void *)array + sz;
2293 	}
2294 
2295 	if (type & PERF_SAMPLE_REGS_USER) {
2296 		OVERFLOW_CHECK_u64(array);
2297 		data->user_regs.abi = *array;
2298 		array++;
2299 
2300 		if (data->user_regs.abi) {
2301 			u64 mask = evsel->core.attr.sample_regs_user;
2302 
2303 			sz = hweight64(mask) * sizeof(u64);
2304 			OVERFLOW_CHECK(array, sz, max_size);
2305 			data->user_regs.mask = mask;
2306 			data->user_regs.regs = (u64 *)array;
2307 			array = (void *)array + sz;
2308 		}
2309 	}
2310 
2311 	if (type & PERF_SAMPLE_STACK_USER) {
2312 		OVERFLOW_CHECK_u64(array);
2313 		sz = *array++;
2314 
2315 		data->user_stack.offset = ((char *)(array - 1)
2316 					  - (char *) event);
2317 
2318 		if (!sz) {
2319 			data->user_stack.size = 0;
2320 		} else {
2321 			OVERFLOW_CHECK(array, sz, max_size);
2322 			data->user_stack.data = (char *)array;
2323 			array = (void *)array + sz;
2324 			OVERFLOW_CHECK_u64(array);
2325 			data->user_stack.size = *array++;
2326 			if (WARN_ONCE(data->user_stack.size > sz,
2327 				      "user stack dump failure\n"))
2328 				return -EFAULT;
2329 		}
2330 	}
2331 
2332 	if (type & PERF_SAMPLE_WEIGHT) {
2333 		OVERFLOW_CHECK_u64(array);
2334 		data->weight = *array;
2335 		array++;
2336 	}
2337 
2338 	if (type & PERF_SAMPLE_DATA_SRC) {
2339 		OVERFLOW_CHECK_u64(array);
2340 		data->data_src = *array;
2341 		array++;
2342 	}
2343 
2344 	if (type & PERF_SAMPLE_TRANSACTION) {
2345 		OVERFLOW_CHECK_u64(array);
2346 		data->transaction = *array;
2347 		array++;
2348 	}
2349 
2350 	data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2351 	if (type & PERF_SAMPLE_REGS_INTR) {
2352 		OVERFLOW_CHECK_u64(array);
2353 		data->intr_regs.abi = *array;
2354 		array++;
2355 
2356 		if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2357 			u64 mask = evsel->core.attr.sample_regs_intr;
2358 
2359 			sz = hweight64(mask) * sizeof(u64);
2360 			OVERFLOW_CHECK(array, sz, max_size);
2361 			data->intr_regs.mask = mask;
2362 			data->intr_regs.regs = (u64 *)array;
2363 			array = (void *)array + sz;
2364 		}
2365 	}
2366 
2367 	data->phys_addr = 0;
2368 	if (type & PERF_SAMPLE_PHYS_ADDR) {
2369 		data->phys_addr = *array;
2370 		array++;
2371 	}
2372 
2373 	return 0;
2374 }
2375 
2376 int perf_evsel__parse_sample_timestamp(struct evsel *evsel,
2377 				       union perf_event *event,
2378 				       u64 *timestamp)
2379 {
2380 	u64 type = evsel->core.attr.sample_type;
2381 	const __u64 *array;
2382 
2383 	if (!(type & PERF_SAMPLE_TIME))
2384 		return -1;
2385 
2386 	if (event->header.type != PERF_RECORD_SAMPLE) {
2387 		struct perf_sample data = {
2388 			.time = -1ULL,
2389 		};
2390 
2391 		if (!evsel->core.attr.sample_id_all)
2392 			return -1;
2393 		if (perf_evsel__parse_id_sample(evsel, event, &data))
2394 			return -1;
2395 
2396 		*timestamp = data.time;
2397 		return 0;
2398 	}
2399 
2400 	array = event->sample.array;
2401 
2402 	if (perf_event__check_size(event, evsel->sample_size))
2403 		return -EFAULT;
2404 
2405 	if (type & PERF_SAMPLE_IDENTIFIER)
2406 		array++;
2407 
2408 	if (type & PERF_SAMPLE_IP)
2409 		array++;
2410 
2411 	if (type & PERF_SAMPLE_TID)
2412 		array++;
2413 
2414 	if (type & PERF_SAMPLE_TIME)
2415 		*timestamp = *array;
2416 
2417 	return 0;
2418 }
2419 
2420 size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type,
2421 				     u64 read_format)
2422 {
2423 	size_t sz, result = sizeof(struct perf_record_sample);
2424 
2425 	if (type & PERF_SAMPLE_IDENTIFIER)
2426 		result += sizeof(u64);
2427 
2428 	if (type & PERF_SAMPLE_IP)
2429 		result += sizeof(u64);
2430 
2431 	if (type & PERF_SAMPLE_TID)
2432 		result += sizeof(u64);
2433 
2434 	if (type & PERF_SAMPLE_TIME)
2435 		result += sizeof(u64);
2436 
2437 	if (type & PERF_SAMPLE_ADDR)
2438 		result += sizeof(u64);
2439 
2440 	if (type & PERF_SAMPLE_ID)
2441 		result += sizeof(u64);
2442 
2443 	if (type & PERF_SAMPLE_STREAM_ID)
2444 		result += sizeof(u64);
2445 
2446 	if (type & PERF_SAMPLE_CPU)
2447 		result += sizeof(u64);
2448 
2449 	if (type & PERF_SAMPLE_PERIOD)
2450 		result += sizeof(u64);
2451 
2452 	if (type & PERF_SAMPLE_READ) {
2453 		result += sizeof(u64);
2454 		if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2455 			result += sizeof(u64);
2456 		if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2457 			result += sizeof(u64);
2458 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2459 		if (read_format & PERF_FORMAT_GROUP) {
2460 			sz = sample->read.group.nr *
2461 			     sizeof(struct sample_read_value);
2462 			result += sz;
2463 		} else {
2464 			result += sizeof(u64);
2465 		}
2466 	}
2467 
2468 	if (type & PERF_SAMPLE_CALLCHAIN) {
2469 		sz = (sample->callchain->nr + 1) * sizeof(u64);
2470 		result += sz;
2471 	}
2472 
2473 	if (type & PERF_SAMPLE_RAW) {
2474 		result += sizeof(u32);
2475 		result += sample->raw_size;
2476 	}
2477 
2478 	if (type & PERF_SAMPLE_BRANCH_STACK) {
2479 		sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2480 		sz += sizeof(u64);
2481 		result += sz;
2482 	}
2483 
2484 	if (type & PERF_SAMPLE_REGS_USER) {
2485 		if (sample->user_regs.abi) {
2486 			result += sizeof(u64);
2487 			sz = hweight64(sample->user_regs.mask) * sizeof(u64);
2488 			result += sz;
2489 		} else {
2490 			result += sizeof(u64);
2491 		}
2492 	}
2493 
2494 	if (type & PERF_SAMPLE_STACK_USER) {
2495 		sz = sample->user_stack.size;
2496 		result += sizeof(u64);
2497 		if (sz) {
2498 			result += sz;
2499 			result += sizeof(u64);
2500 		}
2501 	}
2502 
2503 	if (type & PERF_SAMPLE_WEIGHT)
2504 		result += sizeof(u64);
2505 
2506 	if (type & PERF_SAMPLE_DATA_SRC)
2507 		result += sizeof(u64);
2508 
2509 	if (type & PERF_SAMPLE_TRANSACTION)
2510 		result += sizeof(u64);
2511 
2512 	if (type & PERF_SAMPLE_REGS_INTR) {
2513 		if (sample->intr_regs.abi) {
2514 			result += sizeof(u64);
2515 			sz = hweight64(sample->intr_regs.mask) * sizeof(u64);
2516 			result += sz;
2517 		} else {
2518 			result += sizeof(u64);
2519 		}
2520 	}
2521 
2522 	if (type & PERF_SAMPLE_PHYS_ADDR)
2523 		result += sizeof(u64);
2524 
2525 	return result;
2526 }
2527 
2528 int perf_event__synthesize_sample(union perf_event *event, u64 type,
2529 				  u64 read_format,
2530 				  const struct perf_sample *sample)
2531 {
2532 	__u64 *array;
2533 	size_t sz;
2534 	/*
2535 	 * used for cross-endian analysis. See git commit 65014ab3
2536 	 * for why this goofiness is needed.
2537 	 */
2538 	union u64_swap u;
2539 
2540 	array = event->sample.array;
2541 
2542 	if (type & PERF_SAMPLE_IDENTIFIER) {
2543 		*array = sample->id;
2544 		array++;
2545 	}
2546 
2547 	if (type & PERF_SAMPLE_IP) {
2548 		*array = sample->ip;
2549 		array++;
2550 	}
2551 
2552 	if (type & PERF_SAMPLE_TID) {
2553 		u.val32[0] = sample->pid;
2554 		u.val32[1] = sample->tid;
2555 		*array = u.val64;
2556 		array++;
2557 	}
2558 
2559 	if (type & PERF_SAMPLE_TIME) {
2560 		*array = sample->time;
2561 		array++;
2562 	}
2563 
2564 	if (type & PERF_SAMPLE_ADDR) {
2565 		*array = sample->addr;
2566 		array++;
2567 	}
2568 
2569 	if (type & PERF_SAMPLE_ID) {
2570 		*array = sample->id;
2571 		array++;
2572 	}
2573 
2574 	if (type & PERF_SAMPLE_STREAM_ID) {
2575 		*array = sample->stream_id;
2576 		array++;
2577 	}
2578 
2579 	if (type & PERF_SAMPLE_CPU) {
2580 		u.val32[0] = sample->cpu;
2581 		u.val32[1] = 0;
2582 		*array = u.val64;
2583 		array++;
2584 	}
2585 
2586 	if (type & PERF_SAMPLE_PERIOD) {
2587 		*array = sample->period;
2588 		array++;
2589 	}
2590 
2591 	if (type & PERF_SAMPLE_READ) {
2592 		if (read_format & PERF_FORMAT_GROUP)
2593 			*array = sample->read.group.nr;
2594 		else
2595 			*array = sample->read.one.value;
2596 		array++;
2597 
2598 		if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2599 			*array = sample->read.time_enabled;
2600 			array++;
2601 		}
2602 
2603 		if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2604 			*array = sample->read.time_running;
2605 			array++;
2606 		}
2607 
2608 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2609 		if (read_format & PERF_FORMAT_GROUP) {
2610 			sz = sample->read.group.nr *
2611 			     sizeof(struct sample_read_value);
2612 			memcpy(array, sample->read.group.values, sz);
2613 			array = (void *)array + sz;
2614 		} else {
2615 			*array = sample->read.one.id;
2616 			array++;
2617 		}
2618 	}
2619 
2620 	if (type & PERF_SAMPLE_CALLCHAIN) {
2621 		sz = (sample->callchain->nr + 1) * sizeof(u64);
2622 		memcpy(array, sample->callchain, sz);
2623 		array = (void *)array + sz;
2624 	}
2625 
2626 	if (type & PERF_SAMPLE_RAW) {
2627 		u.val32[0] = sample->raw_size;
2628 		*array = u.val64;
2629 		array = (void *)array + sizeof(u32);
2630 
2631 		memcpy(array, sample->raw_data, sample->raw_size);
2632 		array = (void *)array + sample->raw_size;
2633 	}
2634 
2635 	if (type & PERF_SAMPLE_BRANCH_STACK) {
2636 		sz = sample->branch_stack->nr * sizeof(struct branch_entry);
2637 		sz += sizeof(u64);
2638 		memcpy(array, sample->branch_stack, sz);
2639 		array = (void *)array + sz;
2640 	}
2641 
2642 	if (type & PERF_SAMPLE_REGS_USER) {
2643 		if (sample->user_regs.abi) {
2644 			*array++ = sample->user_regs.abi;
2645 			sz = hweight64(sample->user_regs.mask) * sizeof(u64);
2646 			memcpy(array, sample->user_regs.regs, sz);
2647 			array = (void *)array + sz;
2648 		} else {
2649 			*array++ = 0;
2650 		}
2651 	}
2652 
2653 	if (type & PERF_SAMPLE_STACK_USER) {
2654 		sz = sample->user_stack.size;
2655 		*array++ = sz;
2656 		if (sz) {
2657 			memcpy(array, sample->user_stack.data, sz);
2658 			array = (void *)array + sz;
2659 			*array++ = sz;
2660 		}
2661 	}
2662 
2663 	if (type & PERF_SAMPLE_WEIGHT) {
2664 		*array = sample->weight;
2665 		array++;
2666 	}
2667 
2668 	if (type & PERF_SAMPLE_DATA_SRC) {
2669 		*array = sample->data_src;
2670 		array++;
2671 	}
2672 
2673 	if (type & PERF_SAMPLE_TRANSACTION) {
2674 		*array = sample->transaction;
2675 		array++;
2676 	}
2677 
2678 	if (type & PERF_SAMPLE_REGS_INTR) {
2679 		if (sample->intr_regs.abi) {
2680 			*array++ = sample->intr_regs.abi;
2681 			sz = hweight64(sample->intr_regs.mask) * sizeof(u64);
2682 			memcpy(array, sample->intr_regs.regs, sz);
2683 			array = (void *)array + sz;
2684 		} else {
2685 			*array++ = 0;
2686 		}
2687 	}
2688 
2689 	if (type & PERF_SAMPLE_PHYS_ADDR) {
2690 		*array = sample->phys_addr;
2691 		array++;
2692 	}
2693 
2694 	return 0;
2695 }
2696 
2697 struct tep_format_field *perf_evsel__field(struct evsel *evsel, const char *name)
2698 {
2699 	return tep_find_field(evsel->tp_format, name);
2700 }
2701 
2702 void *perf_evsel__rawptr(struct evsel *evsel, struct perf_sample *sample,
2703 			 const char *name)
2704 {
2705 	struct tep_format_field *field = perf_evsel__field(evsel, name);
2706 	int offset;
2707 
2708 	if (!field)
2709 		return NULL;
2710 
2711 	offset = field->offset;
2712 
2713 	if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2714 		offset = *(int *)(sample->raw_data + field->offset);
2715 		offset &= 0xffff;
2716 	}
2717 
2718 	return sample->raw_data + offset;
2719 }
2720 
2721 u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2722 			 bool needs_swap)
2723 {
2724 	u64 value;
2725 	void *ptr = sample->raw_data + field->offset;
2726 
2727 	switch (field->size) {
2728 	case 1:
2729 		return *(u8 *)ptr;
2730 	case 2:
2731 		value = *(u16 *)ptr;
2732 		break;
2733 	case 4:
2734 		value = *(u32 *)ptr;
2735 		break;
2736 	case 8:
2737 		memcpy(&value, ptr, sizeof(u64));
2738 		break;
2739 	default:
2740 		return 0;
2741 	}
2742 
2743 	if (!needs_swap)
2744 		return value;
2745 
2746 	switch (field->size) {
2747 	case 2:
2748 		return bswap_16(value);
2749 	case 4:
2750 		return bswap_32(value);
2751 	case 8:
2752 		return bswap_64(value);
2753 	default:
2754 		return 0;
2755 	}
2756 
2757 	return 0;
2758 }
2759 
2760 u64 perf_evsel__intval(struct evsel *evsel, struct perf_sample *sample,
2761 		       const char *name)
2762 {
2763 	struct tep_format_field *field = perf_evsel__field(evsel, name);
2764 
2765 	if (!field)
2766 		return 0;
2767 
2768 	return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2769 }
2770 
2771 bool perf_evsel__fallback(struct evsel *evsel, int err,
2772 			  char *msg, size_t msgsize)
2773 {
2774 	int paranoid;
2775 
2776 	if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2777 	    evsel->core.attr.type   == PERF_TYPE_HARDWARE &&
2778 	    evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2779 		/*
2780 		 * If it's cycles then fall back to hrtimer based
2781 		 * cpu-clock-tick sw counter, which is always available even if
2782 		 * no PMU support.
2783 		 *
2784 		 * PPC returns ENXIO until 2.6.37 (behavior changed with commit
2785 		 * b0a873e).
2786 		 */
2787 		scnprintf(msg, msgsize, "%s",
2788 "The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2789 
2790 		evsel->core.attr.type   = PERF_TYPE_SOFTWARE;
2791 		evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2792 
2793 		zfree(&evsel->name);
2794 		return true;
2795 	} else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2796 		   (paranoid = perf_event_paranoid()) > 1) {
2797 		const char *name = perf_evsel__name(evsel);
2798 		char *new_name;
2799 		const char *sep = ":";
2800 
2801 		/* Is there already the separator in the name. */
2802 		if (strchr(name, '/') ||
2803 		    strchr(name, ':'))
2804 			sep = "";
2805 
2806 		if (asprintf(&new_name, "%s%su", name, sep) < 0)
2807 			return false;
2808 
2809 		if (evsel->name)
2810 			free(evsel->name);
2811 		evsel->name = new_name;
2812 		scnprintf(msg, msgsize,
2813 "kernel.perf_event_paranoid=%d, trying to fall back to excluding kernel samples", paranoid);
2814 		evsel->core.attr.exclude_kernel = 1;
2815 
2816 		return true;
2817 	}
2818 
2819 	return false;
2820 }
2821 
2822 static bool find_process(const char *name)
2823 {
2824 	size_t len = strlen(name);
2825 	DIR *dir;
2826 	struct dirent *d;
2827 	int ret = -1;
2828 
2829 	dir = opendir(procfs__mountpoint());
2830 	if (!dir)
2831 		return false;
2832 
2833 	/* Walk through the directory. */
2834 	while (ret && (d = readdir(dir)) != NULL) {
2835 		char path[PATH_MAX];
2836 		char *data;
2837 		size_t size;
2838 
2839 		if ((d->d_type != DT_DIR) ||
2840 		     !strcmp(".", d->d_name) ||
2841 		     !strcmp("..", d->d_name))
2842 			continue;
2843 
2844 		scnprintf(path, sizeof(path), "%s/%s/comm",
2845 			  procfs__mountpoint(), d->d_name);
2846 
2847 		if (filename__read_str(path, &data, &size))
2848 			continue;
2849 
2850 		ret = strncmp(name, data, len);
2851 		free(data);
2852 	}
2853 
2854 	closedir(dir);
2855 	return ret ? false : true;
2856 }
2857 
2858 int perf_evsel__open_strerror(struct evsel *evsel, struct target *target,
2859 			      int err, char *msg, size_t size)
2860 {
2861 	char sbuf[STRERR_BUFSIZE];
2862 	int printed = 0;
2863 
2864 	switch (err) {
2865 	case EPERM:
2866 	case EACCES:
2867 		if (err == EPERM)
2868 			printed = scnprintf(msg, size,
2869 				"No permission to enable %s event.\n\n",
2870 				perf_evsel__name(evsel));
2871 
2872 		return scnprintf(msg + printed, size - printed,
2873 		 "You may not have permission to collect %sstats.\n\n"
2874 		 "Consider tweaking /proc/sys/kernel/perf_event_paranoid,\n"
2875 		 "which controls use of the performance events system by\n"
2876 		 "unprivileged users (without CAP_SYS_ADMIN).\n\n"
2877 		 "The current value is %d:\n\n"
2878 		 "  -1: Allow use of (almost) all events by all users\n"
2879 		 "      Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2880 		 ">= 0: Disallow ftrace function tracepoint by users without CAP_SYS_ADMIN\n"
2881 		 "      Disallow raw tracepoint access by users without CAP_SYS_ADMIN\n"
2882 		 ">= 1: Disallow CPU event access by users without CAP_SYS_ADMIN\n"
2883 		 ">= 2: Disallow kernel profiling by users without CAP_SYS_ADMIN\n\n"
2884 		 "To make this setting permanent, edit /etc/sysctl.conf too, e.g.:\n\n"
2885 		 "	kernel.perf_event_paranoid = -1\n" ,
2886 				 target->system_wide ? "system-wide " : "",
2887 				 perf_event_paranoid());
2888 	case ENOENT:
2889 		return scnprintf(msg, size, "The %s event is not supported.",
2890 				 perf_evsel__name(evsel));
2891 	case EMFILE:
2892 		return scnprintf(msg, size, "%s",
2893 			 "Too many events are opened.\n"
2894 			 "Probably the maximum number of open file descriptors has been reached.\n"
2895 			 "Hint: Try again after reducing the number of events.\n"
2896 			 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2897 	case ENOMEM:
2898 		if (evsel__has_callchain(evsel) &&
2899 		    access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2900 			return scnprintf(msg, size,
2901 					 "Not enough memory to setup event with callchain.\n"
2902 					 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2903 					 "Hint: Current value: %d", sysctl__max_stack());
2904 		break;
2905 	case ENODEV:
2906 		if (target->cpu_list)
2907 			return scnprintf(msg, size, "%s",
2908 	 "No such device - did you specify an out-of-range profile CPU?");
2909 		break;
2910 	case EOPNOTSUPP:
2911 		if (evsel->core.attr.sample_period != 0)
2912 			return scnprintf(msg, size,
2913 	"%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2914 					 perf_evsel__name(evsel));
2915 		if (evsel->core.attr.precise_ip)
2916 			return scnprintf(msg, size, "%s",
2917 	"\'precise\' request may not be supported. Try removing 'p' modifier.");
2918 #if defined(__i386__) || defined(__x86_64__)
2919 		if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
2920 			return scnprintf(msg, size, "%s",
2921 	"No hardware sampling interrupt available.\n");
2922 #endif
2923 		break;
2924 	case EBUSY:
2925 		if (find_process("oprofiled"))
2926 			return scnprintf(msg, size,
2927 	"The PMU counters are busy/taken by another profiler.\n"
2928 	"We found oprofile daemon running, please stop it and try again.");
2929 		break;
2930 	case EINVAL:
2931 		if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
2932 			return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2933 		if (perf_missing_features.clockid)
2934 			return scnprintf(msg, size, "clockid feature not supported.");
2935 		if (perf_missing_features.clockid_wrong)
2936 			return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2937 		if (perf_missing_features.aux_output)
2938 			return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
2939 		break;
2940 	default:
2941 		break;
2942 	}
2943 
2944 	return scnprintf(msg, size,
2945 	"The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2946 	"/bin/dmesg | grep -i perf may provide additional information.\n",
2947 			 err, str_error_r(err, sbuf, sizeof(sbuf)),
2948 			 perf_evsel__name(evsel));
2949 }
2950 
2951 struct perf_env *perf_evsel__env(struct evsel *evsel)
2952 {
2953 	if (evsel && evsel->evlist)
2954 		return evsel->evlist->env;
2955 	return NULL;
2956 }
2957 
2958 static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
2959 {
2960 	int cpu, thread;
2961 
2962 	for (cpu = 0; cpu < xyarray__max_x(evsel->core.fd); cpu++) {
2963 		for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
2964 		     thread++) {
2965 			int fd = FD(evsel, cpu, thread);
2966 
2967 			if (perf_evlist__id_add_fd(evlist, evsel,
2968 						   cpu, thread, fd) < 0)
2969 				return -1;
2970 		}
2971 	}
2972 
2973 	return 0;
2974 }
2975 
2976 int perf_evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
2977 {
2978 	struct perf_cpu_map *cpus = evsel->core.cpus;
2979 	struct perf_thread_map *threads = evsel->core.threads;
2980 
2981 	if (perf_evsel__alloc_id(evsel, cpus->nr, threads->nr))
2982 		return -ENOMEM;
2983 
2984 	return store_evsel_ids(evsel, evlist);
2985 }
2986