1 // SPDX-License-Identifier: GPL-2.0
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
75 	struct task_struct *p = tfc->p;  in remote_function()
78 		/* -EAGAIN */  in remote_function()
87 		tfc->ret = -ESRCH; /* No such (running) process */  in remote_function()
92 	tfc->ret = tfc->func(tfc->info);  in remote_function()
96  * task_function_call - call a function on the cpu on which a task runs
106  * returns @func return value or -ESRCH or -ENXIO when the process isn't running
115 		.ret	= -EAGAIN,  in task_function_call()
125 		if (ret != -EAGAIN)  in task_function_call()
135  * cpu_function_call - call a function on the cpu
142  * returns: @func return value or -ENXIO when the cpu is offline
150 		.ret	= -ENXIO, /* No such CPU */  in cpu_function_call()
174 	raw_spin_lock(&ctx->lock);  in __perf_ctx_lock()
175 	WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN);  in __perf_ctx_lock()
181 	__perf_ctx_lock(&cpuctx->ctx);  in perf_ctx_lock()
192 	if (ctx->is_active & EVENT_FROZEN) {  in __perf_ctx_unlock()
193 		if (!(ctx->is_active & EVENT_ALL))  in __perf_ctx_unlock()
194 			ctx->is_active = 0;  in __perf_ctx_unlock()
196 			ctx->is_active &= ~EVENT_FROZEN;  in __perf_ctx_unlock()
198 	raw_spin_unlock(&ctx->lock);  in __perf_ctx_unlock()
206 	__perf_ctx_unlock(&cpuctx->ctx);  in perf_ctx_unlock()
209 #define TASK_TOMBSTONE ((void *)-1L)
213 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;  in is_kernel_event()
221 	return this_cpu_ptr(&perf_cpu_context)->task_ctx;  in perf_cpu_task_ctx()
227  * When !ctx->nr_events a task context will not be scheduled. This means
233  *  - removing the last event from a task ctx; this is relatively straight
236  *  - adding the first event to a task ctx; this is tricky because we cannot
237  *    rely on ctx->is_active and therefore cannot use event_function_call().
240  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
255 	struct perf_event *event = efs->event;  in event_function()
256 	struct perf_event_context *ctx = event->ctx;  in event_function()
258 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in event_function()
265 	 * Since we do the IPI call without holding ctx->lock things can have  in event_function()
268 	if (ctx->task) {  in event_function()
269 		if (ctx->task != current) {  in event_function()
270 			ret = -ESRCH;  in event_function()
278 		 * above ctx->task != current test), therefore we must have  in event_function()
279 		 * ctx->is_active here.  in event_function()
281 		WARN_ON_ONCE(!ctx->is_active);  in event_function()
283 		 * And since we have ctx->is_active, cpuctx->task_ctx must  in event_function()
288 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function()
291 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
300 	struct perf_event_context *ctx = event->ctx;  in event_function_call()
301 	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */  in event_function_call()
309 	if (!event->parent) {  in event_function_call()
312 		 * stabilize the event->ctx relation. See  in event_function_call()
315 		lockdep_assert_held(&ctx->mutex);  in event_function_call()
319 		cpu_function_call(event->cpu, event_function, &efs);  in event_function_call()
337 	task = ctx->task;  in event_function_call()
340 	if (ctx->is_active) {  in event_function_call()
357 	struct perf_event_context *ctx = event->ctx;  in event_function_local()
359 	struct task_struct *task = READ_ONCE(ctx->task);  in event_function_local()
373 	task = ctx->task;  in event_function_local()
383 		if (ctx->is_active) {  in event_function_local()
387 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))  in event_function_local()
391 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function_local()
448  *  -1 - not paranoid at all
449  *   0 - disallow raw tracepoint access for unpriv
450  *   1 - disallow cpu events for unpriv
451  *   2 - disallow kernel profiling for unpriv
496 		return -EINVAL;  in perf_event_max_sample_rate_handler()
566 	running_len -= running_len/NR_ACCUMULATED_SAMPLES;  in perf_sample_event_took()
620 	return event->clock();  in perf_event_clock()
626  * The basic idea is to use event->state to determine which (if any) time
648 	struct perf_event *leader = event->group_leader;  in __perf_effective_state()
650 	if (leader->state <= PERF_EVENT_STATE_OFF)  in __perf_effective_state()
651 		return leader->state;  in __perf_effective_state()
653 	return event->state;  in __perf_effective_state()
660 	u64 delta = now - event->tstamp;  in __perf_update_times()
662 	*enabled = event->total_time_enabled;  in __perf_update_times()
666 	*running = event->total_time_running;  in __perf_update_times()
675 	__perf_update_times(event, now, &event->total_time_enabled,  in perf_event_update_time()
676 					&event->total_time_running);  in perf_event_update_time()
677 	event->tstamp = now;  in perf_event_update_time()
691 	if (event->state == state)  in perf_event_set_state()
699 	if ((event->state < 0) ^ (state < 0))  in perf_event_set_state()
702 	WRITE_ONCE(event->state, state);  in perf_event_set_state()
706  * UP store-release, load-acquire
723 	list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
724 		if (_cgroup && !_epc->nr_cgroups)			\
726 		else if (_pmu && _epc->pmu != _pmu)			\
735 		perf_pmu_disable(pmu_ctx->pmu);  in perf_ctx_disable()
743 		perf_pmu_enable(pmu_ctx->pmu);  in perf_ctx_enable()
757 	if (!event->cgrp)  in perf_cgroup_match()
761 	if (!cpuctx->cgrp)  in perf_cgroup_match()
770 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
771 				    event->cgrp->css.cgroup);  in perf_cgroup_match()
776 	css_put(&event->cgrp->css);  in perf_detach_cgroup()
777 	event->cgrp = NULL;  in perf_detach_cgroup()
782 	return event->cgrp != NULL;  in is_cgroup_event()
789 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time()
790 	return t->time;  in perf_cgroup_event_time()
797 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time_now()
798 	if (!__load_acquire(&t->active))  in perf_cgroup_event_time_now()
799 		return t->time;  in perf_cgroup_event_time_now()
800 	now += READ_ONCE(t->timeoffset);  in perf_cgroup_event_time_now()
807 		info->time += now - info->timestamp;  in __update_cgrp_time()
808 	info->timestamp = now;  in __update_cgrp_time()
812 	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);  in __update_cgrp_time()
817 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
824 		for (css = &cgrp->css; css; css = css->parent) {  in update_cgrp_time_from_cpuctx()
826 			info = this_cpu_ptr(cgrp->info);  in update_cgrp_time_from_cpuctx()
830 				__store_release(&info->active, 0);  in update_cgrp_time_from_cpuctx()
846 	info = this_cpu_ptr(event->cgrp->info);  in update_cgrp_time_from_event()
850 	if (info->active)  in update_cgrp_time_from_event()
857 	struct perf_event_context *ctx = &cpuctx->ctx;  in perf_cgroup_set_timestamp()
858 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in perf_cgroup_set_timestamp()
863 	 * ctx->lock held by caller  in perf_cgroup_set_timestamp()
870 	WARN_ON_ONCE(!ctx->nr_cgroups);  in perf_cgroup_set_timestamp()
872 	for (css = &cgrp->css; css; css = css->parent) {  in perf_cgroup_set_timestamp()
874 		info = this_cpu_ptr(cgrp->info);  in perf_cgroup_set_timestamp()
875 		__update_cgrp_time(info, ctx->timestamp, false);  in perf_cgroup_set_timestamp()
876 		__store_release(&info->active, 1);  in perf_cgroup_set_timestamp()
889 	 * cpuctx->cgrp is set when the first cgroup event enabled,  in perf_cgroup_switch()
892 	if (READ_ONCE(cpuctx->cgrp) == NULL)  in perf_cgroup_switch()
895 	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);  in perf_cgroup_switch()
898 	if (READ_ONCE(cpuctx->cgrp) == cgrp)  in perf_cgroup_switch()
901 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
902 	perf_ctx_disable(&cpuctx->ctx, true);  in perf_cgroup_switch()
904 	ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
910 	cpuctx->cgrp = cgrp;  in perf_cgroup_switch()
916 	ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);  in perf_cgroup_switch()
918 	perf_ctx_enable(&cpuctx->ctx, true);  in perf_cgroup_switch()
919 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
933 	for (heap_size = 1; css; css = css->parent)  in perf_cgroup_ensure_storage()
938 		if (heap_size <= cpuctx->heap_size)  in perf_cgroup_ensure_storage()
944 			ret = -ENOMEM;  in perf_cgroup_ensure_storage()
948 		raw_spin_lock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
949 		if (cpuctx->heap_size < heap_size) {  in perf_cgroup_ensure_storage()
950 			swap(cpuctx->heap, storage);  in perf_cgroup_ensure_storage()
951 			if (storage == cpuctx->heap_default)  in perf_cgroup_ensure_storage()
953 			cpuctx->heap_size = heap_size;  in perf_cgroup_ensure_storage()
955 		raw_spin_unlock_irq(&cpuctx->ctx.lock);  in perf_cgroup_ensure_storage()
973 		return -EBADF;  in perf_cgroup_connect()
975 	css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry,  in perf_cgroup_connect()
985 	event->cgrp = cgrp;  in perf_cgroup_connect()
992 	if (group_leader && group_leader->cgrp != cgrp) {  in perf_cgroup_connect()
994 		ret = -EINVAL;  in perf_cgroup_connect()
1007 	event->pmu_ctx->nr_cgroups++;  in perf_cgroup_event_enable()
1010 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_enable()
1011 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_enable()
1015 	if (ctx->nr_cgroups++)  in perf_cgroup_event_enable()
1018 	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);  in perf_cgroup_event_enable()
1029 	event->pmu_ctx->nr_cgroups--;  in perf_cgroup_event_disable()
1032 	 * Because cgroup events are always per-cpu events,  in perf_cgroup_event_disable()
1033 	 * @ctx == &cpuctx->ctx.  in perf_cgroup_event_disable()
1037 	if (--ctx->nr_cgroups)  in perf_cgroup_event_disable()
1040 	cpuctx->cgrp = NULL;  in perf_cgroup_event_disable()
1072 	return -EINVAL;  in perf_cgroup_connect()
1123 	raw_spin_lock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1125 		hrtimer_forward_now(hr, cpc->hrtimer_interval);  in perf_mux_hrtimer_handler()
1127 		cpc->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1128 	raw_spin_unlock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1135 	struct hrtimer *timer = &cpc->hrtimer;  in __perf_mux_hrtimer_init()
1136 	struct pmu *pmu = cpc->epc.pmu;  in __perf_mux_hrtimer_init()
1143 	interval = pmu->hrtimer_interval_ms;  in __perf_mux_hrtimer_init()
1145 		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;  in __perf_mux_hrtimer_init()
1147 	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1149 	raw_spin_lock_init(&cpc->hrtimer_lock);  in __perf_mux_hrtimer_init()
1151 	timer->function = perf_mux_hrtimer_handler;  in __perf_mux_hrtimer_init()
1156 	struct hrtimer *timer = &cpc->hrtimer;  in perf_mux_hrtimer_restart()
1159 	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1160 	if (!cpc->hrtimer_active) {  in perf_mux_hrtimer_restart()
1161 		cpc->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1162 		hrtimer_forward_now(timer, cpc->hrtimer_interval);  in perf_mux_hrtimer_restart()
1165 	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1177 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_disable()
1179 		pmu->pmu_disable(pmu);  in perf_pmu_disable()
1184 	int *count = this_cpu_ptr(pmu->pmu_disable_count);  in perf_pmu_enable()
1185 	if (!--(*count))  in perf_pmu_enable()
1186 		pmu->pmu_enable(pmu);  in perf_pmu_enable()
1191 	WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0);  in perf_assert_pmu_disabled()
1196 	refcount_inc(&ctx->refcount);  in get_ctx()
1201 	if (pmu->task_ctx_cache)  in alloc_task_ctx_data()
1202 		return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);  in alloc_task_ctx_data()
1209 	if (pmu->task_ctx_cache && task_ctx_data)  in free_task_ctx_data()
1210 		kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);  in free_task_ctx_data()
1223 	if (refcount_dec_and_test(&ctx->refcount)) {  in put_ctx()
1224 		if (ctx->parent_ctx)  in put_ctx()
1225 			put_ctx(ctx->parent_ctx);  in put_ctx()
1226 		if (ctx->task && ctx->task != TASK_TOMBSTONE)  in put_ctx()
1227 			put_task_struct(ctx->task);  in put_ctx()
1228 		call_rcu(&ctx->rcu_head, free_ctx);  in put_ctx()
1242  *  - perf_event_exit_task_context()	[ child , 0 ]
1246  *  - perf_event_init_context()		[ parent, 0 ]
1254  * While it appears there is an obvious deadlock here -- the parent and child
1256  * life-time rules separate them. That is an exiting task cannot fork, and a
1259  * But remember that these are parent<->child context relations, and
1265  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1280  * However; because event->ctx can change while we're waiting to acquire
1281  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1297  *	  cpuctx->mutex / perf_event_context::mutex
1306 	ctx = READ_ONCE(event->ctx);  in perf_event_ctx_lock_nested()
1307 	if (!refcount_inc_not_zero(&ctx->refcount)) {  in perf_event_ctx_lock_nested()
1313 	mutex_lock_nested(&ctx->mutex, nesting);  in perf_event_ctx_lock_nested()
1314 	if (event->ctx != ctx) {  in perf_event_ctx_lock_nested()
1315 		mutex_unlock(&ctx->mutex);  in perf_event_ctx_lock_nested()
1332 	mutex_unlock(&ctx->mutex);  in perf_event_ctx_unlock()
1337  * This must be done under the ctx->lock, such as to serialize against
1339  * calling scheduler related locks and ctx->lock nests inside those.
1344 	struct perf_event_context *parent_ctx = ctx->parent_ctx;  in unclone_ctx()
1346 	lockdep_assert_held(&ctx->lock);  in unclone_ctx()
1349 		ctx->parent_ctx = NULL;  in unclone_ctx()
1350 	ctx->generation++;  in unclone_ctx()
1362 	if (event->parent)  in perf_event_pid_type()
1363 		event = event->parent;  in perf_event_pid_type()
1365 	nr = __task_pid_nr_ns(p, type, event->ns);  in perf_event_pid_type()
1366 	/* avoid -1 if it is idle thread or runs in another ns */  in perf_event_pid_type()
1368 		nr = -1;  in perf_event_pid_type()
1388 	u64 id = event->id;  in primary_event_id()
1390 	if (event->parent)  in primary_event_id()
1391 		id = event->parent->id;  in primary_event_id()
1411 	 * part of the read side critical section was irqs-enabled -- see  in perf_lock_task_context()
1414 	 * Since ctx->lock nests under rq->lock we must ensure the entire read  in perf_lock_task_context()
1419 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_lock_task_context()
1431 		raw_spin_lock(&ctx->lock);  in perf_lock_task_context()
1432 		if (ctx != rcu_dereference(task->perf_event_ctxp)) {  in perf_lock_task_context()
1433 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1439 		if (ctx->task == TASK_TOMBSTONE ||  in perf_lock_task_context()
1440 		    !refcount_inc_not_zero(&ctx->refcount)) {  in perf_lock_task_context()
1441 			raw_spin_unlock(&ctx->lock);  in perf_lock_task_context()
1444 			WARN_ON_ONCE(ctx->task != task);  in perf_lock_task_context()
1466 		++ctx->pin_count;  in perf_pin_task_context()
1467 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_pin_task_context()
1476 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_unpin_context()
1477 	--ctx->pin_count;  in perf_unpin_context()
1478 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_unpin_context()
1488 	lockdep_assert_held(&ctx->lock);  in __update_context_time()
1491 		ctx->time += now - ctx->timestamp;  in __update_context_time()
1492 	ctx->timestamp = now;  in __update_context_time()
1495 	 * The above: time' = time + (now - timestamp), can be re-arranged  in __update_context_time()
1496 	 * into: time` = now + (time - timestamp), which gives a single value  in __update_context_time()
1500 	 * it's (obviously) not possible to acquire ctx->lock in order to read  in __update_context_time()
1503 	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);  in __update_context_time()
1513 	struct perf_event_context *ctx = event->ctx;  in perf_event_time()
1521 	return ctx->time;  in perf_event_time()
1526 	struct perf_event_context *ctx = event->ctx;  in perf_event_time_now()
1534 	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))  in perf_event_time_now()
1535 		return ctx->time;  in perf_event_time_now()
1537 	now += READ_ONCE(ctx->timeoffset);  in perf_event_time_now()
1543 	struct perf_event_context *ctx = event->ctx;  in get_event_type()
1546 	lockdep_assert_held(&ctx->lock);  in get_event_type()
1552 	if (event->group_leader != event)  in get_event_type()
1553 		event = event->group_leader;  in get_event_type()
1555 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;  in get_event_type()
1556 	if (!ctx->task)  in get_event_type()
1567 	RB_CLEAR_NODE(&event->group_node);  in init_event_group()
1568 	event->group_index = 0;  in init_event_group()
1578 	if (event->attr.pinned)  in get_event_groups()
1579 		return &ctx->pinned_groups;  in get_event_groups()
1581 		return &ctx->flexible_groups;  in get_event_groups()
1589 	groups->tree = RB_ROOT;  in perf_event_groups_init()
1590 	groups->index = 0;  in perf_event_groups_init()
1598 	if (event->cgrp)  in event_cgroup()
1599 		cgroup = event->cgrp->css.cgroup;  in event_cgroup()
1616 	if (left_cpu < right->cpu)  in perf_event_groups_cmp()
1617 		return -1;  in perf_event_groups_cmp()
1618 	if (left_cpu > right->cpu)  in perf_event_groups_cmp()
1622 		if (left_pmu < right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1623 			return -1;  in perf_event_groups_cmp()
1624 		if (left_pmu > right->pmu_ctx->pmu)  in perf_event_groups_cmp()
1638 				return -1;  in perf_event_groups_cmp()
1649 				return -1;  in perf_event_groups_cmp()
1656 	if (left_group_index < right->group_index)  in perf_event_groups_cmp()
1657 		return -1;  in perf_event_groups_cmp()
1658 	if (left_group_index > right->group_index)  in perf_event_groups_cmp()
1670 	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),  in __group_less()
1671 				     e->group_index, __node_2_pe(b)) < 0;  in __group_less()
1686 	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);  in __group_cmp()
1696 	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),  in __group_cmp_ignore_cgroup()
1697 				     b->group_index, b);  in __group_cmp_ignore_cgroup()
1702  *   {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1709 	event->group_index = ++groups->index;  in perf_event_groups_insert()
1711 	rb_add(&event->group_node, &groups->tree, __group_less);  in perf_event_groups_insert()
1733 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||  in perf_event_groups_delete()
1734 		     RB_EMPTY_ROOT(&groups->tree));  in perf_event_groups_delete()
1736 	rb_erase(&event->group_node, &groups->tree);  in perf_event_groups_delete()
1766 	node = rb_find_first(&key, &groups->tree, __group_cmp);  in perf_event_groups_first()
1777 		.cpu = event->cpu,  in perf_event_groups_next()
1783 	next = rb_next_match(&key, &event->group_node, __group_cmp);  in perf_event_groups_next()
1798 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1800 		event = rb_entry_safe(rb_next(&event->group_node),	\
1808 	return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ);  in has_inherit_and_sample_read()
1813  * Must be called with ctx->mutex and ctx->lock held.
1818 	lockdep_assert_held(&ctx->lock);  in list_add_event()
1820 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in list_add_event()
1821 	event->attach_state |= PERF_ATTACH_CONTEXT;  in list_add_event()
1823 	event->tstamp = perf_event_time(event);  in list_add_event()
1830 	if (event->group_leader == event) {  in list_add_event()
1831 		event->group_caps = event->event_caps;  in list_add_event()
1835 	list_add_rcu(&event->event_entry, &ctx->event_list);  in list_add_event()
1836 	ctx->nr_events++;  in list_add_event()
1837 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_add_event()
1838 		ctx->nr_user++;  in list_add_event()
1839 	if (event->attr.inherit_stat)  in list_add_event()
1840 		ctx->nr_stat++;  in list_add_event()
1841 	if (has_inherit_and_sample_read(&event->attr))  in list_add_event()
1842 		local_inc(&ctx->nr_no_switch_fast);  in list_add_event()
1844 	if (event->state > PERF_EVENT_STATE_OFF)  in list_add_event()
1847 	ctx->generation++;  in list_add_event()
1848 	event->pmu_ctx->nr_events++;  in list_add_event()
1856 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :  in perf_event__state_init()
1896 		size += sizeof(data->ip);  in __perf_event_header_size()
1899 		size += sizeof(data->addr);  in __perf_event_header_size()
1902 		size += sizeof(data->period);  in __perf_event_header_size()
1905 		size += sizeof(data->weight.full);  in __perf_event_header_size()
1908 		size += event->read_size;  in __perf_event_header_size()
1911 		size += sizeof(data->data_src.val);  in __perf_event_header_size()
1914 		size += sizeof(data->txn);  in __perf_event_header_size()
1917 		size += sizeof(data->phys_addr);  in __perf_event_header_size()
1920 		size += sizeof(data->cgroup);  in __perf_event_header_size()
1923 		size += sizeof(data->data_page_size);  in __perf_event_header_size()
1926 		size += sizeof(data->code_page_size);  in __perf_event_header_size()
1928 	event->header_size = size;  in __perf_event_header_size()
1937 	event->read_size =  in perf_event__header_size()
1938 		__perf_event_read_size(event->attr.read_format,  in perf_event__header_size()
1939 				       event->group_leader->nr_siblings);  in perf_event__header_size()
1940 	__perf_event_header_size(event, event->attr.sample_type);  in perf_event__header_size()
1946 	u64 sample_type = event->attr.sample_type;  in perf_event__id_header_size()
1950 		size += sizeof(data->tid_entry);  in perf_event__id_header_size()
1953 		size += sizeof(data->time);  in perf_event__id_header_size()
1956 		size += sizeof(data->id);  in perf_event__id_header_size()
1959 		size += sizeof(data->id);  in perf_event__id_header_size()
1962 		size += sizeof(data->stream_id);  in perf_event__id_header_size()
1965 		size += sizeof(data->cpu_entry);  in perf_event__id_header_size()
1967 	event->id_header_size = size;  in perf_event__id_header_size()
1976  * depends on per-event read_format, also (re)check the existing events.
1983 	struct perf_event *sibling, *group_leader = event->group_leader;  in perf_event_validate_size()
1985 	if (__perf_event_read_size(event->attr.read_format,  in perf_event_validate_size()
1986 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1989 	if (__perf_event_read_size(group_leader->attr.read_format,  in perf_event_validate_size()
1990 				   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
1994 	 * When creating a new group leader, group_leader->ctx is initialized  in perf_event_validate_size()
1996 	 * for_each_sibling_event() until group_leader->ctx is set. A new group  in perf_event_validate_size()
1998 	 * the non-existent siblings.  in perf_event_validate_size()
2004 		if (__perf_event_read_size(sibling->attr.read_format,  in perf_event_validate_size()
2005 					   group_leader->nr_siblings + 1) > 16*1024)  in perf_event_validate_size()
2014 	struct perf_event *group_leader = event->group_leader, *pos;  in perf_group_attach()
2016 	lockdep_assert_held(&event->ctx->lock);  in perf_group_attach()
2022 	if (event->attach_state & PERF_ATTACH_GROUP)  in perf_group_attach()
2025 	event->attach_state |= PERF_ATTACH_GROUP;  in perf_group_attach()
2030 	WARN_ON_ONCE(group_leader->ctx != event->ctx);  in perf_group_attach()
2032 	group_leader->group_caps &= event->event_caps;  in perf_group_attach()
2034 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);  in perf_group_attach()
2035 	group_leader->nr_siblings++;  in perf_group_attach()
2036 	group_leader->group_generation++;  in perf_group_attach()
2046  * Must be called with ctx->mutex and ctx->lock held.
2051 	WARN_ON_ONCE(event->ctx != ctx);  in list_del_event()
2052 	lockdep_assert_held(&ctx->lock);  in list_del_event()
2055 	 * We can have double detach due to exit/hot-unplug + close.  in list_del_event()
2057 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))  in list_del_event()
2060 	event->attach_state &= ~PERF_ATTACH_CONTEXT;  in list_del_event()
2062 	ctx->nr_events--;  in list_del_event()
2063 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_del_event()
2064 		ctx->nr_user--;  in list_del_event()
2065 	if (event->attr.inherit_stat)  in list_del_event()
2066 		ctx->nr_stat--;  in list_del_event()
2067 	if (has_inherit_and_sample_read(&event->attr))  in list_del_event()
2068 		local_dec(&ctx->nr_no_switch_fast);  in list_del_event()
2070 	list_del_rcu(&event->event_entry);  in list_del_event()
2072 	if (event->group_leader == event)  in list_del_event()
2079 	 * of error state is by explicit re-enabling  in list_del_event()
2082 	if (event->state > PERF_EVENT_STATE_OFF) {  in list_del_event()
2087 	ctx->generation++;  in list_del_event()
2088 	event->pmu_ctx->nr_events--;  in list_del_event()
2097 	if (!event->pmu->aux_output_match)  in perf_aux_output_match()
2100 	return event->pmu->aux_output_match(aux_event);  in perf_aux_output_match()
2109 	struct perf_event_context *ctx = event->ctx;  in perf_put_aux_event()
2115 	if (event->aux_event) {  in perf_put_aux_event()
2116 		iter = event->aux_event;  in perf_put_aux_event()
2117 		event->aux_event = NULL;  in perf_put_aux_event()
2126 	for_each_sibling_event(iter, event->group_leader) {  in perf_put_aux_event()
2127 		if (iter->aux_event != event)  in perf_put_aux_event()
2130 		iter->aux_event = NULL;  in perf_put_aux_event()
2145 	return event->attr.aux_output || has_aux_action(event);  in perf_need_aux_event()
2163 	if (event->attr.aux_output && event->attr.aux_sample_size)  in perf_get_aux_event()
2166 	if (event->attr.aux_output &&  in perf_get_aux_event()
2170 	if ((event->attr.aux_pause || event->attr.aux_resume) &&  in perf_get_aux_event()
2171 	    !(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE))  in perf_get_aux_event()
2174 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)  in perf_get_aux_event()
2177 	if (!atomic_long_inc_not_zero(&group_leader->refcount))  in perf_get_aux_event()
2186 	event->aux_event = group_leader;  in perf_get_aux_event()
2193 	return event->attr.pinned ? &event->pmu_ctx->pinned_active :  in get_event_list()
2194 				    &event->pmu_ctx->flexible_active;  in get_event_list()
2205 	event_sched_out(event, event->ctx);  in perf_remove_sibling_event()
2211 	struct perf_event *leader = event->group_leader;  in perf_group_detach()
2213 	struct perf_event_context *ctx = event->ctx;  in perf_group_detach()
2215 	lockdep_assert_held(&ctx->lock);  in perf_group_detach()
2218 	 * We can have double detach due to exit/hot-unplug + close.  in perf_group_detach()
2220 	if (!(event->attach_state & PERF_ATTACH_GROUP))  in perf_group_detach()
2223 	event->attach_state &= ~PERF_ATTACH_GROUP;  in perf_group_detach()
2231 		list_del_init(&event->sibling_list);  in perf_group_detach()
2232 		event->group_leader->nr_siblings--;  in perf_group_detach()
2233 		event->group_leader->group_generation++;  in perf_group_detach()
2242 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {  in perf_group_detach()
2244 		if (sibling->event_caps & PERF_EV_CAP_SIBLING)  in perf_group_detach()
2247 		sibling->group_leader = sibling;  in perf_group_detach()
2248 		list_del_init(&sibling->sibling_list);  in perf_group_detach()
2251 		sibling->group_caps = event->group_caps;  in perf_group_detach()
2253 		if (sibling->attach_state & PERF_ATTACH_CONTEXT) {  in perf_group_detach()
2254 			add_event_to_groups(sibling, event->ctx);  in perf_group_detach()
2256 			if (sibling->state == PERF_EVENT_STATE_ACTIVE)  in perf_group_detach()
2257 				list_add_tail(&sibling->active_list, get_event_list(sibling));  in perf_group_detach()
2260 		WARN_ON_ONCE(sibling->ctx != event->ctx);  in perf_group_detach()
2274 	struct perf_event *parent_event = event->parent;  in perf_child_detach()
2276 	if (!(event->attach_state & PERF_ATTACH_CHILD))  in perf_child_detach()
2279 	event->attach_state &= ~PERF_ATTACH_CHILD;  in perf_child_detach()
2284 	lockdep_assert_held(&parent_event->child_mutex);  in perf_child_detach()
2287 	list_del_init(&event->child_list);  in perf_child_detach()
2292 	return event->state == PERF_EVENT_STATE_DEAD;  in is_orphaned_event()
2298 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&  in event_filter_match()
2305 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_out()
2306 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_out()
2309 	// XXX cpc serialization, probably per-cpu IRQ disabled  in event_sched_out()
2311 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_out()
2312 	lockdep_assert_held(&ctx->lock);  in event_sched_out()
2314 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in event_sched_out()
2322 	list_del_init(&event->active_list);  in event_sched_out()
2324 	perf_pmu_disable(event->pmu);  in event_sched_out()
2326 	event->pmu->del(event, 0);  in event_sched_out()
2327 	event->oncpu = -1;  in event_sched_out()
2329 	if (event->pending_disable) {  in event_sched_out()
2330 		event->pending_disable = 0;  in event_sched_out()
2338 		cpc->active_oncpu--;  in event_sched_out()
2339 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_out()
2340 		ctx->nr_freq--;  in event_sched_out()
2341 		epc->nr_freq--;  in event_sched_out()
2343 	if (event->attr.exclusive || !cpc->active_oncpu)  in event_sched_out()
2344 		cpc->exclusive = 0;  in event_sched_out()
2346 	perf_pmu_enable(event->pmu);  in event_sched_out()
2354 	if (group_event->state != PERF_EVENT_STATE_ACTIVE)  in group_sched_out()
2357 	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);  in group_sched_out()
2371 	if (ctx->is_active & EVENT_TIME) {  in __ctx_time_update()
2372 		if (ctx->is_active & EVENT_FROZEN)  in __ctx_time_update()
2392 	if (ctx->is_active & EVENT_TIME)  in ctx_time_freeze()
2393 		ctx->is_active |= EVENT_FROZEN;  in ctx_time_freeze()
2399 	if (ctx->is_active & EVENT_TIME) {  in ctx_time_update_event()
2400 		if (ctx->is_active & EVENT_FROZEN)  in ctx_time_update_event()
2423 	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;  in __perf_remove_from_context()
2433 		event->pending_disable = 1;  in __perf_remove_from_context()
2441 		event->state = PERF_EVENT_STATE_DEAD;  in __perf_remove_from_context()
2443 	if (!pmu_ctx->nr_events) {  in __perf_remove_from_context()
2444 		pmu_ctx->rotate_necessary = 0;  in __perf_remove_from_context()
2446 		if (ctx->task && ctx->is_active) {  in __perf_remove_from_context()
2449 			cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __perf_remove_from_context()
2450 			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __perf_remove_from_context()
2451 			cpc->task_epc = NULL;  in __perf_remove_from_context()
2455 	if (!ctx->nr_events && ctx->is_active) {  in __perf_remove_from_context()
2456 		if (ctx == &cpuctx->ctx)  in __perf_remove_from_context()
2459 		ctx->is_active = 0;  in __perf_remove_from_context()
2460 		if (ctx->task) {  in __perf_remove_from_context()
2461 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in __perf_remove_from_context()
2462 			cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
2470  * If event->ctx is a cloned context, callers must make sure that
2471  * every task struct that event->ctx->task could possibly point to
2473  * that only calls us on the top-level context, which can't be a clone.
2479 	struct perf_event_context *ctx = event->ctx;  in perf_remove_from_context()
2481 	lockdep_assert_held(&ctx->mutex);  in perf_remove_from_context()
2488 	raw_spin_lock_irq(&ctx->lock);  in perf_remove_from_context()
2489 	if (!ctx->is_active) {  in perf_remove_from_context()
2492 		raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2495 	raw_spin_unlock_irq(&ctx->lock);  in perf_remove_from_context()
2508 	if (event->state < PERF_EVENT_STATE_INACTIVE)  in __perf_event_disable()
2511 	perf_pmu_disable(event->pmu_ctx->pmu);  in __perf_event_disable()
2514 	if (event == event->group_leader)  in __perf_event_disable()
2522 	perf_pmu_enable(event->pmu_ctx->pmu);  in __perf_event_disable()
2528  * If event->ctx is a cloned context, callers must make sure that
2529  * every task struct that event->ctx->task could possibly point to
2532  * hold the top-level event's child_mutex, so any descendant that
2535  * When called from perf_pending_disable it's OK because event->ctx
2541 	struct perf_event_context *ctx = event->ctx;  in _perf_event_disable()
2543 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_disable()
2544 	if (event->state <= PERF_EVENT_STATE_OFF) {  in _perf_event_disable()
2545 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2548 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_disable()
2574 	event->pending_disable = 1;  in perf_event_disable_inatomic()
2575 	irq_work_queue(&event->pending_disable_irq);  in perf_event_disable_inatomic()
2586 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_in()
2587 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in event_sched_in()
2590 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_in()
2592 	lockdep_assert_held(&ctx->lock);  in event_sched_in()
2594 	if (event->state <= PERF_EVENT_STATE_OFF)  in event_sched_in()
2597 	WRITE_ONCE(event->oncpu, smp_processor_id());  in event_sched_in()
2601 	 * ->oncpu if it sees ACTIVE.  in event_sched_in()
2611 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {  in event_sched_in()
2613 		event->hw.interrupts = 0;  in event_sched_in()
2616 	perf_pmu_disable(event->pmu);  in event_sched_in()
2620 	if (event->pmu->add(event, PERF_EF_START)) {  in event_sched_in()
2622 		event->oncpu = -1;  in event_sched_in()
2623 		ret = -EAGAIN;  in event_sched_in()
2628 		cpc->active_oncpu++;  in event_sched_in()
2629 	if (event->attr.freq && event->attr.sample_freq) {  in event_sched_in()
2630 		ctx->nr_freq++;  in event_sched_in()
2631 		epc->nr_freq++;  in event_sched_in()
2633 	if (event->attr.exclusive)  in event_sched_in()
2634 		cpc->exclusive = 1;  in event_sched_in()
2637 	perf_pmu_enable(event->pmu);  in event_sched_in()
2646 	struct pmu *pmu = group_event->pmu_ctx->pmu;  in group_sched_in()
2648 	if (group_event->state == PERF_EVENT_STATE_OFF)  in group_sched_in()
2651 	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);  in group_sched_in()
2666 	if (!pmu->commit_txn(pmu))  in group_sched_in()
2684 	pmu->cancel_txn(pmu);  in group_sched_in()
2685 	return -EAGAIN;  in group_sched_in()
2693 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in group_can_go_on()
2694 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);  in group_can_go_on()
2699 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)  in group_can_go_on()
2705 	if (cpc->exclusive)  in group_can_go_on()
2711 	if (event->attr.exclusive && !list_empty(get_event_list(event)))  in group_can_go_on()
2733 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2736 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2746 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED);  in perf_event_sched_in()
2749 	ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in perf_event_sched_in()
2756  *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2757  *  - task pinned (EVENT_PINNED)
2758  *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2759  *  - task flexible (EVENT_FLEXIBLE).
2785 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2786 		perf_pmu_disable(epc->pmu);  in ctx_resched()
2790 			perf_pmu_disable(epc->pmu);  in ctx_resched()
2798 	 *  - EVENT_CPU: schedule out corresponding groups;  in ctx_resched()
2799 	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;  in ctx_resched()
2800 	 *  - otherwise, do nothing more.  in ctx_resched()
2803 		ctx_sched_out(&cpuctx->ctx, pmu, event_type);  in ctx_resched()
2805 		ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);  in ctx_resched()
2809 	for_each_epc(epc, &cpuctx->ctx, pmu, false)  in ctx_resched()
2810 		perf_pmu_enable(epc->pmu);  in ctx_resched()
2814 			perf_pmu_enable(epc->pmu);  in ctx_resched()
2821 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in perf_pmu_resched()
2832  * things like ctx->is_active and cpuctx->task_ctx are set.
2837 	struct perf_event_context *ctx = event->ctx;  in __perf_install_in_context()
2839 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2843 	raw_spin_lock(&cpuctx->ctx.lock);  in __perf_install_in_context()
2844 	if (ctx->task) {  in __perf_install_in_context()
2845 		raw_spin_lock(&ctx->lock);  in __perf_install_in_context()
2848 		reprogram = (ctx->task == current);  in __perf_install_in_context()
2854 		 * If its not running, we don't care, ctx->lock will  in __perf_install_in_context()
2857 		if (task_curr(ctx->task) && !reprogram) {  in __perf_install_in_context()
2858 			ret = -ESRCH;  in __perf_install_in_context()
2862 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);  in __perf_install_in_context()
2864 		raw_spin_lock(&task_ctx->lock);  in __perf_install_in_context()
2868 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {  in __perf_install_in_context()
2874 		reprogram = cgroup_is_descendant(cgrp->css.cgroup,  in __perf_install_in_context()
2875 					event->cgrp->css.cgroup);  in __perf_install_in_context()
2882 		ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu,  in __perf_install_in_context()
2907 	struct task_struct *task = READ_ONCE(ctx->task);  in perf_install_in_context()
2909 	lockdep_assert_held(&ctx->mutex);  in perf_install_in_context()
2913 	if (event->cpu != -1)  in perf_install_in_context()
2914 		WARN_ON_ONCE(event->cpu != cpu);  in perf_install_in_context()
2917 	 * Ensures that if we can observe event->ctx, both the event and ctx  in perf_install_in_context()
2920 	smp_store_release(&event->ctx, ctx);  in perf_install_in_context()
2925 	 * that case we need the magic of the IPI to set ctx->is_active.  in perf_install_in_context()
2931 	    ctx->nr_events && !is_cgroup_event(event)) {  in perf_install_in_context()
2932 		raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2933 		if (ctx->task == TASK_TOMBSTONE) {  in perf_install_in_context()
2934 			raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2938 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
2954 	 * Installing events is tricky because we cannot rely on ctx->is_active  in perf_install_in_context()
2955 	 * to be set in case this is the nr_events 0 -> 1 transition.  in perf_install_in_context()
2969 	 * our task->perf_event_ctxp[] store, such that it will in fact take  in perf_install_in_context()
2978 	 * This smp_mb() orders the task->perf_event_ctxp[] store with the  in perf_install_in_context()
2988 	raw_spin_lock_irq(&ctx->lock);  in perf_install_in_context()
2989 	task = ctx->task;  in perf_install_in_context()
2993 		 * cannot happen), and we hold ctx->mutex, which serializes us  in perf_install_in_context()
2996 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3000 	 * If the task is not running, ctx->lock will avoid it becoming so,  in perf_install_in_context()
3004 		raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3008 	raw_spin_unlock_irq(&ctx->lock);  in perf_install_in_context()
3019 	struct perf_event *leader = event->group_leader;  in __perf_event_enable()
3022 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in __perf_event_enable()
3023 	    event->state <= PERF_EVENT_STATE_ERROR)  in __perf_event_enable()
3031 	if (!ctx->is_active)  in __perf_event_enable()
3041 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_enable()
3044 	task_ctx = cpuctx->task_ctx;  in __perf_event_enable()
3045 	if (ctx->task)  in __perf_event_enable()
3048 	ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event));  in __perf_event_enable()
3054  * If event->ctx is a cloned context, callers must make sure that
3055  * every task struct that event->ctx->task could possibly point to
3062 	struct perf_event_context *ctx = event->ctx;  in _perf_event_enable()
3064 	raw_spin_lock_irq(&ctx->lock);  in _perf_event_enable()
3065 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in _perf_event_enable()
3066 	    event->state <  PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3068 		raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3077 	 * been scheduled away before the cross-call arrived.  in _perf_event_enable()
3079 	if (event->state == PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
3083 		if (event->event_caps & PERF_EV_CAP_SIBLING &&  in _perf_event_enable()
3084 		    event->group_leader == event)  in _perf_event_enable()
3087 		event->state = PERF_EVENT_STATE_OFF;  in _perf_event_enable()
3089 	raw_spin_unlock_irq(&ctx->lock);  in _perf_event_enable()
3115 	struct perf_event *event = sd->event;  in __perf_event_stop()
3118 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in __perf_event_stop()
3128 	if (READ_ONCE(event->oncpu) != smp_processor_id())  in __perf_event_stop()
3129 		return -EAGAIN;  in __perf_event_stop()
3131 	event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_stop()
3139 	 * Since this is happening on an event-local CPU, no trace is lost  in __perf_event_stop()
3142 	if (sd->restart)  in __perf_event_stop()
3143 		event->pmu->start(event, 0);  in __perf_event_stop()
3157 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in perf_event_stop()
3165 		 * inactive here (event->oncpu==-1), there's nothing more to do;  in perf_event_stop()
3166 		 * fall through with ret==-ENXIO.  in perf_event_stop()
3168 		ret = cpu_function_call(READ_ONCE(event->oncpu),  in perf_event_stop()
3170 	} while (ret == -EAGAIN);  in perf_event_stop()
3189  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3204 	raw_spin_lock(&ifh->lock);  in perf_event_addr_filters_sync()
3205 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {  in perf_event_addr_filters_sync()
3206 		event->pmu->addr_filters_sync(event);  in perf_event_addr_filters_sync()
3207 		event->hw.addr_filters_gen = event->addr_filters_gen;  in perf_event_addr_filters_sync()
3209 	raw_spin_unlock(&ifh->lock);  in perf_event_addr_filters_sync()
3218 	if (event->attr.inherit || !is_sampling_event(event))  in _perf_event_refresh()
3219 		return -EINVAL;  in _perf_event_refresh()
3221 	atomic_add(refresh, &event->event_limit);  in _perf_event_refresh()
3252 	if (!bp->attr.disabled)  in perf_event_modify_breakpoint()
3259  * Copy event-type-independent attributes that may be modified.
3264 	to->sig_data = from->sig_data;  in perf_event_modify_copy_attr()
3274 	if (event->attr.type != attr->type)  in perf_event_modify_attr()
3275 		return -EINVAL;  in perf_event_modify_attr()
3277 	switch (event->attr.type) {  in perf_event_modify_attr()
3283 		return -EOPNOTSUPP;  in perf_event_modify_attr()
3286 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_modify_attr()
3288 	mutex_lock(&event->child_mutex);  in perf_event_modify_attr()
3290 	 * Event-type-independent attributes must be copied before event-type  in perf_event_modify_attr()
3294 	perf_event_modify_copy_attr(&event->attr, attr);  in perf_event_modify_attr()
3298 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_modify_attr()
3299 		perf_event_modify_copy_attr(&child->attr, attr);  in perf_event_modify_attr()
3305 	mutex_unlock(&event->child_mutex);  in perf_event_modify_attr()
3312 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_out()
3314 	struct pmu *pmu = pmu_ctx->pmu;  in __pmu_ctx_sched_out()
3316 	if (ctx->task && !(ctx->is_active & EVENT_ALL)) {  in __pmu_ctx_sched_out()
3319 		cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in __pmu_ctx_sched_out()
3320 		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __pmu_ctx_sched_out()
3321 		cpc->task_epc = NULL;  in __pmu_ctx_sched_out()
3330 					 &pmu_ctx->pinned_active,  in __pmu_ctx_sched_out()
3337 					 &pmu_ctx->flexible_active,  in __pmu_ctx_sched_out()
3345 		pmu_ctx->rotate_necessary = 0;  in __pmu_ctx_sched_out()
3356  * state, eg. ctx->is_active cleared even though most EPCs would still actually
3364 	int is_active = ctx->is_active;  in ctx_sched_out()
3369 	lockdep_assert_held(&ctx->lock);  in ctx_sched_out()
3371 	if (likely(!ctx->nr_events)) {  in ctx_sched_out()
3375 		WARN_ON_ONCE(ctx->is_active);  in ctx_sched_out()
3376 		if (ctx->task)  in ctx_sched_out()
3377 			WARN_ON_ONCE(cpuctx->task_ctx);  in ctx_sched_out()
3391 	__ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx);  in ctx_sched_out()
3394 	 * CPU-release for the below ->is_active store,  in ctx_sched_out()
3398 	ctx->is_active &= ~event_type;  in ctx_sched_out()
3400 	if (!(ctx->is_active & EVENT_ALL)) {  in ctx_sched_out()
3405 		if (ctx->is_active & EVENT_FROZEN)  in ctx_sched_out()
3406 			ctx->is_active &= EVENT_TIME_FROZEN;  in ctx_sched_out()
3408 			ctx->is_active = 0;  in ctx_sched_out()
3411 	if (ctx->task) {  in ctx_sched_out()
3412 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_out()
3413 		if (!(ctx->is_active & EVENT_ALL))  in ctx_sched_out()
3414 			cpuctx->task_ctx = NULL;  in ctx_sched_out()
3417 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_out()
3434 	lockdep_assert_held(&ctx1->lock);  in context_equiv()
3435 	lockdep_assert_held(&ctx2->lock);  in context_equiv()
3438 	if (ctx1->pin_count || ctx2->pin_count)  in context_equiv()
3442 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)  in context_equiv()
3446 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)  in context_equiv()
3453 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&  in context_equiv()
3454 			ctx1->parent_gen == ctx2->parent_gen)  in context_equiv()
3466 	if (!event->attr.inherit_stat)  in __perf_event_sync_stat()
3476 	if (event->state == PERF_EVENT_STATE_ACTIVE)  in __perf_event_sync_stat()
3477 		event->pmu->read(event);  in __perf_event_sync_stat()
3482 	 * In order to keep per-task stats reliable we need to flip the event  in __perf_event_sync_stat()
3485 	value = local64_read(&next_event->count);  in __perf_event_sync_stat()
3486 	value = local64_xchg(&event->count, value);  in __perf_event_sync_stat()
3487 	local64_set(&next_event->count, value);  in __perf_event_sync_stat()
3489 	swap(event->total_time_enabled, next_event->total_time_enabled);  in __perf_event_sync_stat()
3490 	swap(event->total_time_running, next_event->total_time_running);  in __perf_event_sync_stat()
3504 	if (!ctx->nr_stat)  in perf_event_sync_stat()
3509 	event = list_first_entry(&ctx->event_list,  in perf_event_sync_stat()
3512 	next_event = list_first_entry(&next_ctx->event_list,  in perf_event_sync_stat()
3515 	while (&event->event_entry != &ctx->event_list &&  in perf_event_sync_stat()
3516 	       &next_event->event_entry != &next_ctx->event_list) {  in perf_event_sync_stat()
3538 	if (!prev_ctx->nr_task_data)  in perf_event_swap_task_ctx_data()
3542 				   &prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list,  in perf_event_swap_task_ctx_data()
3545 		if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu))  in perf_event_swap_task_ctx_data()
3554 		if (prev_epc->pmu->swap_task_ctx)  in perf_event_swap_task_ctx_data()
3555 			prev_epc->pmu->swap_task_ctx(prev_epc, next_epc);  in perf_event_swap_task_ctx_data()
3557 			swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);  in perf_event_swap_task_ctx_data()
3566 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_ctx_sched_task_cb()
3567 		cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in perf_ctx_sched_task_cb()
3569 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)  in perf_ctx_sched_task_cb()
3570 			pmu_ctx->pmu->sched_task(pmu_ctx, sched_in);  in perf_ctx_sched_task_cb()
3577 	struct perf_event_context *ctx = task->perf_event_ctxp;  in perf_event_context_sched_out()
3586 	next_ctx = rcu_dereference(next->perf_event_ctxp);  in perf_event_context_sched_out()
3590 	parent = rcu_dereference(ctx->parent_ctx);  in perf_event_context_sched_out()
3591 	next_parent = rcu_dereference(next_ctx->parent_ctx);  in perf_event_context_sched_out()
3602 		 * lock (including re-checking that neither has been  in perf_event_context_sched_out()
3607 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3608 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);  in perf_event_context_sched_out()
3613 			/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */  in perf_event_context_sched_out()
3614 			if (local_read(&ctx->nr_no_switch_fast) ||  in perf_event_context_sched_out()
3615 			    local_read(&next_ctx->nr_no_switch_fast)) {  in perf_event_context_sched_out()
3618 				 * events that rely on the ctx->task relation.  in perf_event_context_sched_out()
3625 				raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3630 			WRITE_ONCE(ctx->task, next);  in perf_event_context_sched_out()
3631 			WRITE_ONCE(next_ctx->task, task);  in perf_event_context_sched_out()
3641 			 * ctx->task and ctx->task_ctx_data are immaterial  in perf_event_context_sched_out()
3643 			 * ctx->lock which we're now holding.  in perf_event_context_sched_out()
3645 			RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);  in perf_event_context_sched_out()
3646 			RCU_INIT_POINTER(next->perf_event_ctxp, ctx);  in perf_event_context_sched_out()
3652 		raw_spin_unlock(&next_ctx->lock);  in perf_event_context_sched_out()
3653 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3659 		raw_spin_lock(&ctx->lock);  in perf_event_context_sched_out()
3667 		raw_spin_unlock(&ctx->lock);  in perf_event_context_sched_out()
3676 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_dec()
3681 	if (!--cpc->sched_cb_usage)  in perf_sched_cb_dec()
3682 		list_del(&cpc->sched_cb_entry);  in perf_sched_cb_dec()
3688 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);  in perf_sched_cb_inc()
3690 	if (!cpc->sched_cb_usage++)  in perf_sched_cb_inc()
3691 		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3701  * This callback is relevant even to per-cpu events; for example multi event
3710 	pmu = cpc->epc.pmu;  in __perf_pmu_sched_task()
3713 	if (WARN_ON_ONCE(!pmu->sched_task))  in __perf_pmu_sched_task()
3716 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3719 	pmu->sched_task(cpc->task_epc, sched_in);  in __perf_pmu_sched_task()
3722 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in __perf_pmu_sched_task()
3732 	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */  in perf_pmu_sched_task()
3733 	if (prev == next || cpuctx->task_ctx)  in perf_pmu_sched_task()
3747  * We stop each event and update the event value in event->count.
3768 	 * cgroup event are system-wide mode only  in __perf_event_task_sched_out()
3778 	return le->group_index < re->group_index;  in perf_less_group_idx()
3790 	struct perf_event **itrs = heap->data;  in __heap_add()
3793 		itrs[heap->nr] = event;  in __heap_add()
3794 		heap->nr++;  in __heap_add()
3802 	if (!pmu_ctx->ctx->task)  in __link_epc()
3805 	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);  in __link_epc()
3806 	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __link_epc()
3807 	cpc->task_epc = pmu_ctx;  in __link_epc()
3826 	if (pmu->filter && pmu->filter(pmu, cpu))  in visit_groups_merge()
3829 	if (!ctx->task) {  in visit_groups_merge()
3832 			.data = cpuctx->heap,  in visit_groups_merge()
3834 			.size = cpuctx->heap_size,  in visit_groups_merge()
3837 		lockdep_assert_held(&cpuctx->ctx.lock);  in visit_groups_merge()
3840 		if (cpuctx->cgrp)  in visit_groups_merge()
3841 			css = &cpuctx->cgrp->css;  in visit_groups_merge()
3850 		__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));  in visit_groups_merge()
3857 	for (; css; css = css->parent)  in visit_groups_merge()
3858 		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));  in visit_groups_merge()
3862 		__link_epc((*evt)->pmu_ctx);  in visit_groups_merge()
3863 		perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);  in visit_groups_merge()
3884  * Because the userpage is strictly per-event (there is no concept of context,
3886  * when context time starts :-(
3892 	if (likely(!atomic_read(&event->mmap_count)))  in event_update_userpage()
3914 	struct perf_event_context *ctx = event->ctx;  in merge_sched_in()
3917 	if (event->state <= PERF_EVENT_STATE_OFF)  in merge_sched_in()
3925 			list_add_tail(&event->active_list, get_event_list(event));  in merge_sched_in()
3928 	if (event->state == PERF_EVENT_STATE_INACTIVE) {  in merge_sched_in()
3930 		if (event->attr.pinned) {  in merge_sched_in()
3936 			event->pmu_ctx->rotate_necessary = 1;  in merge_sched_in()
3937 			cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);  in merge_sched_in()
3958 	struct perf_event_context *ctx = pmu_ctx->ctx;  in __pmu_ctx_sched_in()
3961 		pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3963 		pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu);  in __pmu_ctx_sched_in()
3971 	int is_active = ctx->is_active;  in ctx_sched_in()
3976 	lockdep_assert_held(&ctx->lock);  in ctx_sched_in()
3978 	if (likely(!ctx->nr_events))  in ctx_sched_in()
3986 		 * CPU-release for the below ->is_active store,  in ctx_sched_in()
3992 	ctx->is_active |= (event_type | EVENT_TIME);  in ctx_sched_in()
3993 	if (ctx->task) {  in ctx_sched_in()
3995 			cpuctx->task_ctx = ctx;  in ctx_sched_in()
3997 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_in()
4000 	is_active ^= ctx->is_active; /* changed bits */  in ctx_sched_in()
4024 	ctx = rcu_dereference(task->perf_event_ctxp);  in perf_event_context_sched_in()
4028 	if (cpuctx->task_ctx == ctx) {  in perf_event_context_sched_in()
4041 	 * We must check ctx->nr_events while holding ctx->lock, such  in perf_event_context_sched_in()
4044 	if (!ctx->nr_events)  in perf_event_context_sched_in()
4056 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {  in perf_event_context_sched_in()
4057 		perf_ctx_disable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4058 		ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
4063 	perf_ctx_sched_task_cb(cpuctx->task_ctx, true);  in perf_event_context_sched_in()
4065 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))  in perf_event_context_sched_in()
4066 		perf_ctx_enable(&cpuctx->ctx, false);  in perf_event_context_sched_in()
4101 	u64 frequency = event->attr.sample_freq;  in perf_calculate_period()
4117 	 * period = -------------------  in perf_calculate_period()
4130 		a##_fls--;		\  in perf_calculate_period()
4133 		b##_fls--;		\  in perf_calculate_period()
4177 	struct hw_perf_event *hwc = &event->hw;  in perf_adjust_period()
4183 	delta = (s64)(period - hwc->sample_period);  in perf_adjust_period()
4187 		delta -= 7;  in perf_adjust_period()
4190 	sample_period = hwc->sample_period + delta;  in perf_adjust_period()
4195 	hwc->sample_period = sample_period;  in perf_adjust_period()
4197 	if (local64_read(&hwc->period_left) > 8*sample_period) {  in perf_adjust_period()
4199 			event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_period()
4201 		local64_set(&hwc->period_left, 0);  in perf_adjust_period()
4204 			event->pmu->start(event, PERF_EF_RELOAD);  in perf_adjust_period()
4216 		if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_adjust_freq_unthr_events()
4219 		// XXX use visit thingy to avoid the -1,cpu match  in perf_adjust_freq_unthr_events()
4223 		hwc = &event->hw;  in perf_adjust_freq_unthr_events()
4225 		if (hwc->interrupts == MAX_INTERRUPTS) {  in perf_adjust_freq_unthr_events()
4226 			hwc->interrupts = 0;  in perf_adjust_freq_unthr_events()
4228 			if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4229 				event->pmu->start(event, 0);  in perf_adjust_freq_unthr_events()
4232 		if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_events()
4236 		 * stop the event and update event->count  in perf_adjust_freq_unthr_events()
4238 		event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_freq_unthr_events()
4240 		now = local64_read(&event->count);  in perf_adjust_freq_unthr_events()
4241 		delta = now - hwc->freq_count_stamp;  in perf_adjust_freq_unthr_events()
4242 		hwc->freq_count_stamp = now;  in perf_adjust_freq_unthr_events()
4254 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);  in perf_adjust_freq_unthr_events()
4270 	 * - context have events in frequency mode (needs freq adjust)  in perf_adjust_freq_unthr_context()
4271 	 * - there are events to unthrottle on this cpu  in perf_adjust_freq_unthr_context()
4273 	if (!(ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4276 	raw_spin_lock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4278 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in perf_adjust_freq_unthr_context()
4279 		if (!(pmu_ctx->nr_freq || unthrottle))  in perf_adjust_freq_unthr_context()
4283 		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)  in perf_adjust_freq_unthr_context()
4286 		perf_pmu_disable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4287 		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);  in perf_adjust_freq_unthr_context()
4288 		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);  in perf_adjust_freq_unthr_context()
4289 		perf_pmu_enable(pmu_ctx->pmu);  in perf_adjust_freq_unthr_context()
4292 	raw_spin_unlock(&ctx->lock);  in perf_adjust_freq_unthr_context()
4301 	 * Rotate the first entry last of non-pinned groups. Rotation might be  in rotate_ctx()
4304 	if (ctx->rotate_disable)  in rotate_ctx()
4307 	perf_event_groups_delete(&ctx->flexible_groups, event);  in rotate_ctx()
4308 	perf_event_groups_insert(&ctx->flexible_groups, event);  in rotate_ctx()
4319 		.pmu = pmu_ctx->pmu,  in ctx_event_to_rotate()
4323 	event = list_first_entry_or_null(&pmu_ctx->flexible_active,  in ctx_event_to_rotate()
4329 	tree = &pmu_ctx->ctx->flexible_groups.tree;  in ctx_event_to_rotate()
4331 	if (!pmu_ctx->ctx->task) {  in ctx_event_to_rotate()
4340 	key.cpu = -1;  in ctx_event_to_rotate()
4357 	pmu_ctx->rotate_necessary = 0;  in ctx_event_to_rotate()
4375 	cpu_epc = &cpc->epc;  in perf_rotate_context()
4376 	pmu = cpu_epc->pmu;  in perf_rotate_context()
4377 	task_epc = cpc->task_epc;  in perf_rotate_context()
4379 	cpu_rotate = cpu_epc->rotate_necessary;  in perf_rotate_context()
4380 	task_rotate = task_epc ? task_epc->rotate_necessary : 0;  in perf_rotate_context()
4385 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4398 		update_context_time(task_epc->ctx);  in perf_rotate_context()
4403 		update_context_time(&cpuctx->ctx);  in perf_rotate_context()
4405 		rotate_ctx(&cpuctx->ctx, cpu_event);  in perf_rotate_context()
4410 		rotate_ctx(task_epc->ctx, task_event);  in perf_rotate_context()
4416 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
4433 	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);  in perf_event_task_tick()
4436 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_event_task_tick()
4445 	if (!event->attr.enable_on_exec)  in event_enable_on_exec()
4448 	event->attr.enable_on_exec = 0;  in event_enable_on_exec()
4449 	if (event->state >= PERF_EVENT_STATE_INACTIVE)  in event_enable_on_exec()
4458  * Enable all of a task's events that have been marked enable-on-exec.
4471 	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))  in perf_event_enable_on_exec()
4474 	if (!ctx->nr_events)  in perf_event_enable_on_exec()
4481 	list_for_each_entry(event, &ctx->event_list, event_entry) {  in perf_event_enable_on_exec()
4508  * remove-on-exec, and feeds their values back to parent events.
4517 	mutex_lock(&ctx->mutex);  in perf_event_remove_on_exec()
4519 	if (WARN_ON_ONCE(ctx->task != current))  in perf_event_remove_on_exec()
4522 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {  in perf_event_remove_on_exec()
4523 		if (!event->attr.remove_on_exec)  in perf_event_remove_on_exec()
4534 	raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_remove_on_exec()
4537 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_remove_on_exec()
4540 	mutex_unlock(&ctx->mutex);  in perf_event_remove_on_exec()
4562 	if (event->group_caps & PERF_EV_CAP_READ_SCOPE) {  in __perf_event_read_cpu()
4563 		const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu);  in __perf_event_read_cpu()
4569 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {  in __perf_event_read_cpu()
4586 	struct perf_event *sub, *event = data->event;  in __perf_event_read()
4587 	struct perf_event_context *ctx = event->ctx;  in __perf_event_read()
4589 	struct pmu *pmu = event->pmu;  in __perf_event_read()
4595 	 * event->count would have been updated to a recent sample  in __perf_event_read()
4598 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
4601 	raw_spin_lock(&ctx->lock);  in __perf_event_read()
4605 	if (data->group)  in __perf_event_read()
4608 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_read()
4611 	if (!data->group) {  in __perf_event_read()
4612 		pmu->read(event);  in __perf_event_read()
4613 		data->ret = 0;  in __perf_event_read()
4617 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);  in __perf_event_read()
4619 	pmu->read(event);  in __perf_event_read()
4622 		if (sub->state == PERF_EVENT_STATE_ACTIVE) {  in __perf_event_read()
4627 			sub->pmu->read(sub);  in __perf_event_read()
4631 	data->ret = pmu->commit_txn(pmu);  in __perf_event_read()
4634 	raw_spin_unlock(&ctx->lock);  in __perf_event_read()
4640 		return local64_read(&event->count);  in perf_event_count()
4642 	return local64_read(&event->count) + atomic64_read(&event->child_count);  in perf_event_count()
4658  * NMI-safe method to read a local event, that is an event that
4660  *   - either for the current task, or for this CPU
4661  *   - does not have inherit set, for inherited task events
4663  *   - must not have a pmu::count method
4683 	if (event->attr.inherit) {  in perf_event_read_local()
4684 		ret = -EOPNOTSUPP;  in perf_event_read_local()
4688 	/* If this is a per-task event, it must be for current */  in perf_event_read_local()
4689 	if ((event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4690 	    event->hw.target != current) {  in perf_event_read_local()
4691 		ret = -EINVAL;  in perf_event_read_local()
4697 	 * a per-package event that can be read locally  in perf_event_read_local()
4699 	event_oncpu = __perf_event_read_cpu(event, event->oncpu);  in perf_event_read_local()
4700 	event_cpu = __perf_event_read_cpu(event, event->cpu);  in perf_event_read_local()
4702 	/* If this is a per-CPU event, it must be for this CPU */  in perf_event_read_local()
4703 	if (!(event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4705 		ret = -EINVAL;  in perf_event_read_local()
4710 	if (event->attr.pinned && event_oncpu != smp_processor_id()) {  in perf_event_read_local()
4711 		ret = -EBUSY;  in perf_event_read_local()
4716 	 * If the event is currently on this CPU, its either a per-task event,  in perf_event_read_local()
4718 	 * oncpu == -1).  in perf_event_read_local()
4721 		event->pmu->read(event);  in perf_event_read_local()
4723 	*value = local64_read(&event->count);  in perf_event_read_local()
4741 	enum perf_event_state state = READ_ONCE(event->state);  in perf_event_read()
4753 		 * Orders the ->state and ->oncpu loads such that if we see  in perf_event_read()
4754 		 * ACTIVE we must also see the right ->oncpu.  in perf_event_read()
4760 		event_cpu = READ_ONCE(event->oncpu);  in perf_event_read()
4780 		 * Therefore, either way, we'll have an up-to-date event count  in perf_event_read()
4788 		struct perf_event_context *ctx = event->ctx;  in perf_event_read()
4791 		raw_spin_lock_irqsave(&ctx->lock, flags);  in perf_event_read()
4792 		state = event->state;  in perf_event_read()
4794 			raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4807 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in perf_event_read()
4818 	raw_spin_lock_init(&ctx->lock);  in __perf_event_init_context()
4819 	mutex_init(&ctx->mutex);  in __perf_event_init_context()
4820 	INIT_LIST_HEAD(&ctx->pmu_ctx_list);  in __perf_event_init_context()
4821 	perf_event_groups_init(&ctx->pinned_groups);  in __perf_event_init_context()
4822 	perf_event_groups_init(&ctx->flexible_groups);  in __perf_event_init_context()
4823 	INIT_LIST_HEAD(&ctx->event_list);  in __perf_event_init_context()
4824 	refcount_set(&ctx->refcount, 1);  in __perf_event_init_context()
4830 	epc->pmu = pmu;  in __perf_init_event_pmu_context()
4831 	INIT_LIST_HEAD(&epc->pmu_ctx_entry);  in __perf_init_event_pmu_context()
4832 	INIT_LIST_HEAD(&epc->pinned_active);  in __perf_init_event_pmu_context()
4833 	INIT_LIST_HEAD(&epc->flexible_active);  in __perf_init_event_pmu_context()
4834 	atomic_set(&epc->refcount, 1);  in __perf_init_event_pmu_context()
4848 		ctx->task = get_task_struct(task);  in alloc_perf_context()
4868 		return ERR_PTR(-ESRCH);  in find_lively_task_by_vpid()
4886 		err = perf_allow_cpu(&event->attr);  in find_get_context()
4890 		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in find_get_context()
4891 		ctx = &cpuctx->ctx;  in find_get_context()
4893 		raw_spin_lock_irqsave(&ctx->lock, flags);  in find_get_context()
4894 		++ctx->pin_count;  in find_get_context()
4895 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4900 	err = -EINVAL;  in find_get_context()
4905 		++ctx->pin_count;  in find_get_context()
4907 		raw_spin_unlock_irqrestore(&ctx->lock, flags);  in find_get_context()
4913 		err = -ENOMEM;  in find_get_context()
4918 		mutex_lock(&task->perf_event_mutex);  in find_get_context()
4923 		if (task->flags & PF_EXITING)  in find_get_context()
4924 			err = -ESRCH;  in find_get_context()
4925 		else if (task->perf_event_ctxp)  in find_get_context()
4926 			err = -EAGAIN;  in find_get_context()
4929 			++ctx->pin_count;  in find_get_context()
4930 			rcu_assign_pointer(task->perf_event_ctxp, ctx);  in find_get_context()
4932 		mutex_unlock(&task->perf_event_mutex);  in find_get_context()
4937 			if (err == -EAGAIN)  in find_get_context()
4956 	if (!ctx->task) {  in find_get_pmu_context()
4964 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);  in find_get_pmu_context()
4965 		epc = &cpc->epc;  in find_get_pmu_context()
4966 		raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
4967 		if (!epc->ctx) {  in find_get_pmu_context()
4968 			atomic_set(&epc->refcount, 1);  in find_get_pmu_context()
4969 			epc->embedded = 1;  in find_get_pmu_context()
4970 			list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
4971 			epc->ctx = ctx;  in find_get_pmu_context()
4973 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
4974 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
4976 		raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
4982 		return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
4984 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {  in find_get_pmu_context()
4988 			return ERR_PTR(-ENOMEM);  in find_get_pmu_context()
4997 	 * lockdep_assert_held(&ctx->mutex);  in find_get_pmu_context()
5000 	 * child_ctx->mutex.  in find_get_pmu_context()
5003 	raw_spin_lock_irq(&ctx->lock);  in find_get_pmu_context()
5004 	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {  in find_get_pmu_context()
5005 		if (epc->pmu == pmu) {  in find_get_pmu_context()
5006 			WARN_ON_ONCE(epc->ctx != ctx);  in find_get_pmu_context()
5007 			atomic_inc(&epc->refcount);  in find_get_pmu_context()
5015 	list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);  in find_get_pmu_context()
5016 	epc->ctx = ctx;  in find_get_pmu_context()
5019 	if (task_ctx_data && !epc->task_ctx_data) {  in find_get_pmu_context()
5020 		epc->task_ctx_data = task_ctx_data;  in find_get_pmu_context()
5022 		ctx->nr_task_data++;  in find_get_pmu_context()
5024 	raw_spin_unlock_irq(&ctx->lock);  in find_get_pmu_context()
5034 	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));  in get_pmu_ctx()
5041 	kfree(epc->task_ctx_data);  in free_epc_rcu()
5047 	struct perf_event_context *ctx = epc->ctx;  in put_pmu_ctx()
5053 	 * lockdep_assert_held(&ctx->mutex);  in put_pmu_ctx()
5055 	 * can't because of the call-site in _free_event()/put_event()  in put_pmu_ctx()
5056 	 * which isn't always called under ctx->mutex.  in put_pmu_ctx()
5058 	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))  in put_pmu_ctx()
5061 	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));  in put_pmu_ctx()
5063 	list_del_init(&epc->pmu_ctx_entry);  in put_pmu_ctx()
5064 	epc->ctx = NULL;  in put_pmu_ctx()
5066 	WARN_ON_ONCE(!list_empty(&epc->pinned_active));  in put_pmu_ctx()
5067 	WARN_ON_ONCE(!list_empty(&epc->flexible_active));  in put_pmu_ctx()
5069 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in put_pmu_ctx()
5071 	if (epc->embedded)  in put_pmu_ctx()
5074 	call_rcu(&epc->rcu_head, free_epc_rcu);  in put_pmu_ctx()
5083 	if (event->ns)  in free_event_rcu()
5084 		put_pid_ns(event->ns);  in free_event_rcu()
5094 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in detach_sb_event()
5096 	raw_spin_lock(&pel->lock);  in detach_sb_event()
5097 	list_del_rcu(&event->sb_list);  in detach_sb_event()
5098 	raw_spin_unlock(&pel->lock);  in detach_sb_event()
5103 	struct perf_event_attr *attr = &event->attr;  in is_sb_event()
5105 	if (event->parent)  in is_sb_event()
5108 	if (event->attach_state & PERF_ATTACH_TASK)  in is_sb_event()
5111 	if (attr->mmap || attr->mmap_data || attr->mmap2 ||  in is_sb_event()
5112 	    attr->comm || attr->comm_exec ||  in is_sb_event()
5113 	    attr->task || attr->ksymbol ||  in is_sb_event()
5114 	    attr->context_switch || attr->text_poke ||  in is_sb_event()
5115 	    attr->bpf_event)  in is_sb_event()
5152 	if (event->parent)  in unaccount_event()
5155 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in unaccount_event()
5157 	if (event->attr.mmap || event->attr.mmap_data)  in unaccount_event()
5159 	if (event->attr.build_id)  in unaccount_event()
5161 	if (event->attr.comm)  in unaccount_event()
5163 	if (event->attr.namespaces)  in unaccount_event()
5165 	if (event->attr.cgroup)  in unaccount_event()
5167 	if (event->attr.task)  in unaccount_event()
5169 	if (event->attr.freq)  in unaccount_event()
5171 	if (event->attr.context_switch) {  in unaccount_event()
5179 	if (event->attr.ksymbol)  in unaccount_event()
5181 	if (event->attr.bpf_event)  in unaccount_event()
5183 	if (event->attr.text_poke)  in unaccount_event()
5187 		if (!atomic_add_unless(&perf_sched_count, -1, 1))  in unaccount_event()
5207  *  1) cpu-wide events in the presence of per-task events,
5208  *  2) per-task events in the presence of cpu-wide events,
5212  * _free_event()), the latter -- before the first perf_install_in_context().
5216 	struct pmu *pmu = event->pmu;  in exclusive_event_init()
5222 	 * Prevent co-existence of per-task and cpu-wide events on the  in exclusive_event_init()
5225 	 * Negative pmu::exclusive_cnt means there are cpu-wide  in exclusive_event_init()
5227 	 * per-task events.  in exclusive_event_init()
5231 	 * to mean "per-task event", because unlike other attach states it  in exclusive_event_init()
5234 	if (event->attach_state & PERF_ATTACH_TASK) {  in exclusive_event_init()
5235 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))  in exclusive_event_init()
5236 			return -EBUSY;  in exclusive_event_init()
5238 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))  in exclusive_event_init()
5239 			return -EBUSY;  in exclusive_event_init()
5247 	struct pmu *pmu = event->pmu;  in exclusive_event_destroy()
5253 	if (event->attach_state & PERF_ATTACH_TASK)  in exclusive_event_destroy()
5254 		atomic_dec(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5256 		atomic_inc(&pmu->exclusive_cnt);  in exclusive_event_destroy()
5261 	if ((e1->pmu == e2->pmu) &&  in exclusive_event_match()
5262 	    (e1->cpu == e2->cpu ||  in exclusive_event_match()
5263 	     e1->cpu == -1 ||  in exclusive_event_match()
5264 	     e2->cpu == -1))  in exclusive_event_match()
5273 	struct pmu *pmu = event->pmu;  in exclusive_event_installable()
5275 	lockdep_assert_held(&ctx->mutex);  in exclusive_event_installable()
5280 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {  in exclusive_event_installable()
5293 	struct callback_head *head = &event->pending_task;  in perf_pending_task_sync()
5295 	if (!event->pending_work)  in perf_pending_task_sync()
5299 	 * obviously can't wait for it to complete. Simply cancel it.  in perf_pending_task_sync()
5302 		event->pending_work = 0;  in perf_pending_task_sync()
5303 		local_dec(&event->ctx->nr_no_switch_fast);  in perf_pending_task_sync()
5312 	rcuwait_wait_event(&event->pending_work_wait, !event->pending_work, TASK_UNINTERRUPTIBLE);  in perf_pending_task_sync()
5317 	irq_work_sync(&event->pending_irq);  in _free_event()
5318 	irq_work_sync(&event->pending_disable_irq);  in _free_event()
5325 	if (event->rb) {  in _free_event()
5327 		 * Can happen when we close an event with re-directed output.  in _free_event()
5332 		mutex_lock(&event->mmap_mutex);  in _free_event()
5334 		mutex_unlock(&event->mmap_mutex);  in _free_event()
5340 	if (!event->parent) {  in _free_event()
5341 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in _free_event()
5347 	kfree(event->addr_filter_ranges);  in _free_event()
5349 	if (event->destroy)  in _free_event()
5350 		event->destroy(event);  in _free_event()
5353 	 * Must be after ->destroy(), due to uprobe_perf_close() using  in _free_event()
5356 	if (event->hw.target)  in _free_event()
5357 		put_task_struct(event->hw.target);  in _free_event()
5359 	if (event->pmu_ctx)  in _free_event()
5360 		put_pmu_ctx(event->pmu_ctx);  in _free_event()
5366 	if (event->ctx)  in _free_event()
5367 		put_ctx(event->ctx);  in _free_event()
5370 	module_put(event->pmu->module);  in _free_event()
5372 	call_rcu(&event->rcu_head, free_event_rcu);  in _free_event()
5381 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,  in free_event()
5383 				atomic_long_read(&event->refcount), event)) {  in free_event()
5384 		/* leak to avoid use-after-free */  in free_event()
5403 	 * owner->perf_event_mutex.  in perf_remove_from_owner()
5405 	owner = READ_ONCE(event->owner);  in perf_remove_from_owner()
5419 		 * holding ctx->mutex which would be an inversion wrt. the  in perf_remove_from_owner()
5423 		 * ctx->mutex.  in perf_remove_from_owner()
5425 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);  in perf_remove_from_owner()
5428 		 * We have to re-check the event->owner field, if it is cleared  in perf_remove_from_owner()
5433 		if (event->owner) {  in perf_remove_from_owner()
5434 			list_del_init(&event->owner_entry);  in perf_remove_from_owner()
5435 			smp_store_release(&event->owner, NULL);  in perf_remove_from_owner()
5437 		mutex_unlock(&owner->perf_event_mutex);  in perf_remove_from_owner()
5444 	if (!atomic_long_dec_and_test(&event->refcount))  in put_event()
5457 	struct perf_event_context *ctx = event->ctx;  in perf_event_release_kernel()
5466 		WARN_ON_ONCE(event->attach_state &  in perf_event_release_kernel()
5475 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_release_kernel()
5481 	 * Anybody acquiring event->child_mutex after the below loop _must_  in perf_event_release_kernel()
5493 	mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5494 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_release_kernel()
5501 		ctx = READ_ONCE(child->ctx);  in perf_event_release_kernel()
5515 		 * can re-acquire child_mutex.  in perf_event_release_kernel()
5517 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5518 		mutex_lock(&ctx->mutex);  in perf_event_release_kernel()
5519 		mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5526 		tmp = list_first_entry_or_null(&event->child_list,  in perf_event_release_kernel()
5530 			list_move(&child->child_list, &free_list);  in perf_event_release_kernel()
5537 			var = &ctx->refcount;  in perf_event_release_kernel()
5540 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5541 		mutex_unlock(&ctx->mutex);  in perf_event_release_kernel()
5555 	mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5558 		void *var = &child->ctx->refcount;  in perf_event_release_kernel()
5560 		list_del(&child->child_list);  in perf_event_release_kernel()
5582 	perf_event_release_kernel(file->private_data);  in perf_release()
5594 	mutex_lock(&event->child_mutex);  in __perf_event_read_value()
5599 	*enabled += event->total_time_enabled +  in __perf_event_read_value()
5600 			atomic64_read(&event->child_total_time_enabled);  in __perf_event_read_value()
5601 	*running += event->total_time_running +  in __perf_event_read_value()
5602 			atomic64_read(&event->child_total_time_running);  in __perf_event_read_value()
5604 	list_for_each_entry(child, &event->child_list, child_list) {  in __perf_event_read_value()
5607 		*enabled += child->total_time_enabled;  in __perf_event_read_value()
5608 		*running += child->total_time_running;  in __perf_event_read_value()
5610 	mutex_unlock(&event->child_mutex);  in __perf_event_read_value()
5631 	struct perf_event_context *ctx = leader->ctx;  in __perf_read_group_add()
5641 	raw_spin_lock_irqsave(&ctx->lock, flags);  in __perf_read_group_add()
5647 	 *  - leader->ctx->lock pins leader->sibling_list  in __perf_read_group_add()
5648 	 *  - parent->child_mutex pins parent->child_list  in __perf_read_group_add()
5649 	 *  - parent->ctx->mutex pins parent->sibling_list  in __perf_read_group_add()
5651 	 * Because parent->ctx != leader->ctx (and child_list nests inside  in __perf_read_group_add()
5652 	 * ctx->mutex), group destruction is not atomic between children, also  in __perf_read_group_add()
5662 	parent = leader->parent;  in __perf_read_group_add()
5664 	    (parent->group_generation != leader->group_generation ||  in __perf_read_group_add()
5665 	     parent->nr_siblings != leader->nr_siblings)) {  in __perf_read_group_add()
5666 		ret = -ECHILD;  in __perf_read_group_add()
5671 	 * Since we co-schedule groups, {enabled,running} times of siblings  in __perf_read_group_add()
5676 		values[n++] += leader->total_time_enabled +  in __perf_read_group_add()
5677 			atomic64_read(&leader->child_total_time_enabled);  in __perf_read_group_add()
5681 		values[n++] += leader->total_time_running +  in __perf_read_group_add()
5682 			atomic64_read(&leader->child_total_time_running);  in __perf_read_group_add()
5692 		values[n++] = atomic64_read(&leader->lost_samples);  in __perf_read_group_add()
5699 			values[n++] = atomic64_read(&sub->lost_samples);  in __perf_read_group_add()
5703 	raw_spin_unlock_irqrestore(&ctx->lock, flags);  in __perf_read_group_add()
5710 	struct perf_event *leader = event->group_leader, *child;  in perf_read_group()
5711 	struct perf_event_context *ctx = leader->ctx;  in perf_read_group()
5715 	lockdep_assert_held(&ctx->mutex);  in perf_read_group()
5717 	values = kzalloc(event->read_size, GFP_KERNEL);  in perf_read_group()
5719 		return -ENOMEM;  in perf_read_group()
5721 	values[0] = 1 + leader->nr_siblings;  in perf_read_group()
5723 	mutex_lock(&leader->child_mutex);  in perf_read_group()
5729 	list_for_each_entry(child, &leader->child_list, child_list) {  in perf_read_group()
5735 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5737 	ret = event->read_size;  in perf_read_group()
5738 	if (copy_to_user(buf, values, event->read_size))  in perf_read_group()
5739 		ret = -EFAULT;  in perf_read_group()
5743 	mutex_unlock(&leader->child_mutex);  in perf_read_group()
5764 		values[n++] = atomic64_read(&event->lost_samples);  in perf_read_one()
5767 		return -EFAULT;  in perf_read_one()
5776 	if (event->state > PERF_EVENT_STATE_EXIT)  in is_event_hup()
5779 	mutex_lock(&event->child_mutex);  in is_event_hup()
5780 	no_children = list_empty(&event->child_list);  in is_event_hup()
5781 	mutex_unlock(&event->child_mutex);  in is_event_hup()
5786  * Read the performance event - simple non blocking version for now
5791 	u64 read_format = event->attr.read_format;  in __perf_read()
5795 	 * Return end-of-file for a read on an event that is in  in __perf_read()
5799 	if (event->state == PERF_EVENT_STATE_ERROR)  in __perf_read()
5802 	if (count < event->read_size)  in __perf_read()
5803 		return -ENOSPC;  in __perf_read()
5805 	WARN_ON_ONCE(event->ctx->parent_ctx);  in __perf_read()
5817 	struct perf_event *event = file->private_data;  in perf_read()
5832 static __poll_t perf_poll(struct file *file, poll_table *wait)  in perf_poll()  argument
5834 	struct perf_event *event = file->private_data;  in perf_poll()
5838 	poll_wait(file, &event->waitq, wait);  in perf_poll()
5844 	 * Pin the event->rb by taking event->mmap_mutex; otherwise  in perf_poll()
5847 	mutex_lock(&event->mmap_mutex);  in perf_poll()
5848 	rb = event->rb;  in perf_poll()
5850 		events = atomic_xchg(&rb->poll, 0);  in perf_poll()
5851 	mutex_unlock(&event->mmap_mutex);  in perf_poll()
5858 	local64_set(&event->count, 0);  in _perf_event_reset()
5869 	WARN_ON_ONCE(event->attr.inherit);  in perf_event_pause()
5871 	count = local64_read(&event->count);  in perf_event_pause()
5873 		local64_set(&event->count, 0);  in perf_event_pause()
5881  * Holding the top-level event's child_mutex means that any
5891 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_for_each_child()
5893 	mutex_lock(&event->child_mutex);  in perf_event_for_each_child()
5895 	list_for_each_entry(child, &event->child_list, child_list)  in perf_event_for_each_child()
5897 	mutex_unlock(&event->child_mutex);  in perf_event_for_each_child()
5903 	struct perf_event_context *ctx = event->ctx;  in perf_event_for_each()
5906 	lockdep_assert_held(&ctx->mutex);  in perf_event_for_each()
5908 	event = event->group_leader;  in perf_event_for_each()
5923 	if (event->attr.freq) {  in __perf_event_period()
5924 		event->attr.sample_freq = value;  in __perf_event_period()
5926 		event->attr.sample_period = value;  in __perf_event_period()
5927 		event->hw.sample_period = value;  in __perf_event_period()
5930 	active = (event->state == PERF_EVENT_STATE_ACTIVE);  in __perf_event_period()
5932 		perf_pmu_disable(event->pmu);  in __perf_event_period()
5935 		 * trying to unthrottle while we already re-started the event.  in __perf_event_period()
5937 		if (event->hw.interrupts == MAX_INTERRUPTS) {  in __perf_event_period()
5938 			event->hw.interrupts = 0;  in __perf_event_period()
5941 		event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_period()
5944 	local64_set(&event->hw.period_left, 0);  in __perf_event_period()
5947 		event->pmu->start(event, PERF_EF_RELOAD);  in __perf_event_period()
5948 		perf_pmu_enable(event->pmu);  in __perf_event_period()
5954 	return event->pmu->check_period(event, value);  in perf_event_check_period()
5960 		return -EINVAL;  in _perf_event_period()
5963 		return -EINVAL;  in _perf_event_period()
5965 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)  in _perf_event_period()
5966 		return -EINVAL;  in _perf_event_period()
5969 		return -EINVAL;  in _perf_event_period()
5971 	if (!event->attr.freq && (value & (1ULL << 63)))  in _perf_event_period()
5972 		return -EINVAL;  in _perf_event_period()
5996 	return !fd_empty(f) && fd_file(f)->f_op == &perf_fops;  in is_perf_file()
6029 			return -EFAULT;  in _perf_ioctl()
6038 			return -EFAULT;  in _perf_ioctl()
6044 		CLASS(fd, output)(arg);	     // arg == -1 => empty  in _perf_ioctl()
6046 		if (arg != -1) {  in _perf_ioctl()
6048 				return -EBADF;  in _perf_ioctl()
6049 			output_event = fd_file(output)->private_data;  in _perf_ioctl()
6079 		rb = rcu_dereference(event->rb);  in _perf_ioctl()
6080 		if (!rb || !rb->nr_pages) {  in _perf_ioctl()
6082 			return -EINVAL;  in _perf_ioctl()
6103 		return -ENOTTY;  in _perf_ioctl()
6116 	struct perf_event *event = file->private_data;  in perf_ioctl()
6141 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */  in perf_compat_ioctl()
6159 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_enable()
6160 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_enable()
6165 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_enable()
6175 	mutex_lock(&current->perf_event_mutex);  in perf_event_task_disable()
6176 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_disable()
6181 	mutex_unlock(&current->perf_event_mutex);  in perf_event_task_disable()
6188 	if (event->hw.state & PERF_HES_STOPPED)  in perf_event_index()
6191 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_event_index()
6194 	return event->pmu->event_idx(event);  in perf_event_index()
6203 	rb = rcu_dereference(event->rb);  in perf_event_init_userpage()
6207 	userpg = rb->user_page;  in perf_event_init_userpage()
6210 	userpg->cap_bit0_is_deprecated = 1;  in perf_event_init_userpage()
6211 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);  in perf_event_init_userpage()
6212 	userpg->data_offset = PAGE_SIZE;  in perf_event_init_userpage()
6213 	userpg->data_size = perf_data_size(rb);  in perf_event_init_userpage()
6236 	rb = rcu_dereference(event->rb);  in perf_event_update_userpage()
6251 	userpg = rb->user_page;  in perf_event_update_userpage()
6257 	++userpg->lock;  in perf_event_update_userpage()
6259 	userpg->index = perf_event_index(event);  in perf_event_update_userpage()
6260 	userpg->offset = perf_event_count(event, false);  in perf_event_update_userpage()
6261 	if (userpg->index)  in perf_event_update_userpage()
6262 		userpg->offset -= local64_read(&event->hw.prev_count);  in perf_event_update_userpage()
6264 	userpg->time_enabled = enabled +  in perf_event_update_userpage()
6265 			atomic64_read(&event->child_total_time_enabled);  in perf_event_update_userpage()
6267 	userpg->time_running = running +  in perf_event_update_userpage()
6268 			atomic64_read(&event->child_total_time_running);  in perf_event_update_userpage()
6273 	++userpg->lock;  in perf_event_update_userpage()
6282 	struct perf_event *event = vmf->vma->vm_file->private_data;  in perf_mmap_fault()
6286 	if (vmf->flags & FAULT_FLAG_MKWRITE) {  in perf_mmap_fault()
6287 		if (vmf->pgoff == 0)  in perf_mmap_fault()
6293 	rb = rcu_dereference(event->rb);  in perf_mmap_fault()
6297 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))  in perf_mmap_fault()
6300 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);  in perf_mmap_fault()
6301 	if (!vmf->page)  in perf_mmap_fault()
6304 	get_page(vmf->page);  in perf_mmap_fault()
6305 	vmf->page->mapping = vmf->vma->vm_file->f_mapping;  in perf_mmap_fault()
6306 	vmf->page->index   = vmf->pgoff;  in perf_mmap_fault()
6321 	WARN_ON_ONCE(event->parent);  in ring_buffer_attach()
6323 	if (event->rb) {  in ring_buffer_attach()
6326 		 * event->rb_entry and wait/clear when adding event->rb_entry.  in ring_buffer_attach()
6328 		WARN_ON_ONCE(event->rcu_pending);  in ring_buffer_attach()
6330 		old_rb = event->rb;  in ring_buffer_attach()
6331 		spin_lock_irqsave(&old_rb->event_lock, flags);  in ring_buffer_attach()
6332 		list_del_rcu(&event->rb_entry);  in ring_buffer_attach()
6333 		spin_unlock_irqrestore(&old_rb->event_lock, flags);  in ring_buffer_attach()
6335 		event->rcu_batches = get_state_synchronize_rcu();  in ring_buffer_attach()
6336 		event->rcu_pending = 1;  in ring_buffer_attach()
6340 		if (event->rcu_pending) {  in ring_buffer_attach()
6341 			cond_synchronize_rcu(event->rcu_batches);  in ring_buffer_attach()
6342 			event->rcu_pending = 0;  in ring_buffer_attach()
6345 		spin_lock_irqsave(&rb->event_lock, flags);  in ring_buffer_attach()
6346 		list_add_rcu(&event->rb_entry, &rb->event_list);  in ring_buffer_attach()
6347 		spin_unlock_irqrestore(&rb->event_lock, flags);  in ring_buffer_attach()
6357 	 * mid-air, but then again, whoever does it like this is  in ring_buffer_attach()
6363 	rcu_assign_pointer(event->rb, rb);  in ring_buffer_attach()
6372 		wake_up_all(&event->waitq);  in ring_buffer_attach()
6380 	if (event->parent)  in ring_buffer_wakeup()
6381 		event = event->parent;  in ring_buffer_wakeup()
6384 	rb = rcu_dereference(event->rb);  in ring_buffer_wakeup()
6386 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)  in ring_buffer_wakeup()
6387 			wake_up_all(&event->waitq);  in ring_buffer_wakeup()
6396 	if (event->parent)  in ring_buffer_get()
6397 		event = event->parent;  in ring_buffer_get()
6400 	rb = rcu_dereference(event->rb);  in ring_buffer_get()
6402 		if (!refcount_inc_not_zero(&rb->refcount))  in ring_buffer_get()
6412 	if (!refcount_dec_and_test(&rb->refcount))  in ring_buffer_put()
6415 	WARN_ON_ONCE(!list_empty(&rb->event_list));  in ring_buffer_put()
6417 	call_rcu(&rb->rcu_head, rb_free_rcu);  in ring_buffer_put()
6422 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_open()
6424 	atomic_inc(&event->mmap_count);  in perf_mmap_open()
6425 	atomic_inc(&event->rb->mmap_count);  in perf_mmap_open()
6427 	if (vma->vm_pgoff)  in perf_mmap_open()
6428 		atomic_inc(&event->rb->aux_mmap_count);  in perf_mmap_open()
6430 	if (event->pmu->event_mapped)  in perf_mmap_open()
6431 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap_open()
6446 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_close()
6448 	struct user_struct *mmap_user = rb->mmap_user;  in perf_mmap_close()
6449 	int mmap_locked = rb->mmap_locked;  in perf_mmap_close()
6453 	if (event->pmu->event_unmapped)  in perf_mmap_close()
6454 		event->pmu->event_unmapped(event, vma->vm_mm);  in perf_mmap_close()
6457 	 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex  in perf_mmap_close()
6460 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&  in perf_mmap_close()
6461 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {  in perf_mmap_close()
6471 		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);  in perf_mmap_close()
6472 		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6476 		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));  in perf_mmap_close()
6478 		mutex_unlock(&rb->aux_mutex);  in perf_mmap_close()
6481 	if (atomic_dec_and_test(&rb->mmap_count))  in perf_mmap_close()
6484 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))  in perf_mmap_close()
6488 	mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6501 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {  in perf_mmap_close()
6502 		if (!atomic_long_inc_not_zero(&event->refcount)) {  in perf_mmap_close()
6504 			 * This event is en-route to free_event() which will  in perf_mmap_close()
6511 		mutex_lock(&event->mmap_mutex);  in perf_mmap_close()
6522 		if (event->rb == rb)  in perf_mmap_close()
6525 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6537 	 * It could be there's still a few 0-ref events on the list; they'll  in perf_mmap_close()
6538 	 * get cleaned up by free_event() -- they'll also still have their  in perf_mmap_close()
6545 	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,  in perf_mmap_close()
6546 			&mmap_user->locked_vm);  in perf_mmap_close()
6547 	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);  in perf_mmap_close()
6563 	struct perf_event *event = file->private_data;  in perf_mmap()
6575 	 * Don't allow mmap() of inherited per-task counters. This would  in perf_mmap()
6579 	if (event->cpu == -1 && event->attr.inherit)  in perf_mmap()
6580 		return -EINVAL;  in perf_mmap()
6582 	if (!(vma->vm_flags & VM_SHARED))  in perf_mmap()
6583 		return -EINVAL;  in perf_mmap()
6589 	vma_size = vma->vm_end - vma->vm_start;  in perf_mmap()
6591 	if (vma->vm_pgoff == 0) {  in perf_mmap()
6592 		nr_pages = (vma_size / PAGE_SIZE) - 1;  in perf_mmap()
6595 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already  in perf_mmap()
6601 		if (!event->rb)  in perf_mmap()
6602 			return -EINVAL;  in perf_mmap()
6606 			return -ENOMEM;  in perf_mmap()
6608 		mutex_lock(&event->mmap_mutex);  in perf_mmap()
6609 		ret = -EINVAL;  in perf_mmap()
6611 		rb = event->rb;  in perf_mmap()
6615 		aux_mutex = &rb->aux_mutex;  in perf_mmap()
6618 		aux_offset = READ_ONCE(rb->user_page->aux_offset);  in perf_mmap()
6619 		aux_size = READ_ONCE(rb->user_page->aux_size);  in perf_mmap()
6624 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)  in perf_mmap()
6628 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)  in perf_mmap()
6635 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)  in perf_mmap()
6641 		if (!atomic_inc_not_zero(&rb->mmap_count))  in perf_mmap()
6645 			atomic_inc(&rb->aux_mmap_count);  in perf_mmap()
6650 		atomic_set(&rb->aux_mmap_count, 1);  in perf_mmap()
6657 	 * If we have rb pages ensure they're a power-of-two number, so we  in perf_mmap()
6661 		return -EINVAL;  in perf_mmap()
6664 		return -EINVAL;  in perf_mmap()
6666 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_mmap()
6668 	mutex_lock(&event->mmap_mutex);  in perf_mmap()
6669 	if (event->rb) {  in perf_mmap()
6670 		if (data_page_nr(event->rb) != nr_pages) {  in perf_mmap()
6671 			ret = -EINVAL;  in perf_mmap()
6675 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {  in perf_mmap()
6681 			mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6691 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);  in perf_mmap()
6698 	user_locked = atomic_long_read(&user->locked_vm);  in perf_mmap()
6702 	 *     user->locked_vm > user_lock_limit  in perf_mmap()
6713 		extra = user_locked - user_lock_limit;  in perf_mmap()
6714 		user_extra -= extra;  in perf_mmap()
6719 	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;  in perf_mmap()
6723 		ret = -EPERM;  in perf_mmap()
6727 	WARN_ON(!rb && event->rb);  in perf_mmap()
6729 	if (vma->vm_flags & VM_WRITE)  in perf_mmap()
6734 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,  in perf_mmap()
6735 			      event->cpu, flags);  in perf_mmap()
6738 			ret = -ENOMEM;  in perf_mmap()
6742 		atomic_set(&rb->mmap_count, 1);  in perf_mmap()
6743 		rb->mmap_user = get_current_user();  in perf_mmap()
6744 		rb->mmap_locked = extra;  in perf_mmap()
6752 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,  in perf_mmap()
6753 				   event->attr.aux_watermark, flags);  in perf_mmap()
6755 			rb->aux_mmap_locked = extra;  in perf_mmap()
6760 		atomic_long_add(user_extra, &user->locked_vm);  in perf_mmap()
6761 		atomic64_add(extra, &vma->vm_mm->pinned_vm);  in perf_mmap()
6763 		atomic_inc(&event->mmap_count);  in perf_mmap()
6765 		atomic_dec(&rb->mmap_count);  in perf_mmap()
6770 	mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6777 	vma->vm_ops = &perf_mmap_vmops;  in perf_mmap()
6779 	if (event->pmu->event_mapped)  in perf_mmap()
6780 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap()
6788 	struct perf_event *event = filp->private_data;  in perf_fasync()
6792 	retval = fasync_helper(fd, filp, on, &event->fasync);  in perf_fasync()
6815  * to user-space before waking everybody up.
6822 	if (event->pending_kill) {  in perf_event_wakeup()
6823 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);  in perf_event_wakeup()
6824 		event->pending_kill = 0;  in perf_event_wakeup()
6832 	 * ctx->task or current has changed in the meantime. This can be the  in perf_sigtrap()
6835 	if (WARN_ON_ONCE(event->ctx->task != current))  in perf_sigtrap()
6842 	if (current->flags & PF_EXITING)  in perf_sigtrap()
6845 	send_sig_perf((void __user *)event->pending_addr,  in perf_sigtrap()
6846 		      event->orig_type, event->attr.sig_data);  in perf_sigtrap()
6850  * Deliver the pending work in-event-context or follow the context.
6854 	int cpu = READ_ONCE(event->oncpu);  in __perf_pending_disable()
6867 		if (event->pending_disable) {  in __perf_pending_disable()
6868 			event->pending_disable = 0;  in __perf_pending_disable()
6875 	 *  CPU-A			CPU-B  in __perf_pending_disable()
6878 	 *    @pending_disable = CPU-A;  in __perf_pending_disable()
6881 	 *  sched-out  in __perf_pending_disable()
6882 	 *    @pending_disable = -1;  in __perf_pending_disable()
6884 	 *				sched-in  in __perf_pending_disable()
6886 	 *				  @pending_disable = CPU-B;  in __perf_pending_disable()
6892 	 * But the event runs on CPU-B and wants disabling there.  in __perf_pending_disable()
6894 	irq_work_queue_on(&event->pending_disable_irq, cpu);  in __perf_pending_disable()
6924 	 * The wakeup isn't bound to the context of the event -- it can happen  in perf_pending_irq()
6927 	if (event->pending_wakeup) {  in perf_pending_irq()
6928 		event->pending_wakeup = 0;  in perf_pending_irq()
6942 	 * All accesses to the event must belong to the same implicit RCU read-side  in perf_pending_task()
6943 	 * critical section as the ->pending_work reset. See comment in  in perf_pending_task()
6953 	if (event->pending_work) {  in perf_pending_task()
6954 		event->pending_work = 0;  in perf_pending_task()
6956 		local_dec(&event->ctx->nr_no_switch_fast);  in perf_pending_task()
6957 		rcuwait_wake_up(&event->pending_work_wait);  in perf_pending_task()
6968 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6969 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6970 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6978 	static_call_update(__perf_guest_state, cbs->state);  in perf_register_guest_info_callbacks()
6979 	static_call_update(__perf_guest_get_ip, cbs->get_ip);  in perf_register_guest_info_callbacks()
6981 	/* Implementing ->handle_intel_pt_intr is optional. */  in perf_register_guest_info_callbacks()
6982 	if (cbs->handle_intel_pt_intr)  in perf_register_guest_info_callbacks()
6984 				   cbs->handle_intel_pt_intr);  in perf_register_guest_info_callbacks()
7005 	return !event->attr.exclude_guest && perf_guest_state();  in should_sample_guest()
7046 		regs_user->abi = perf_reg_abi(current);  in perf_sample_regs_user()
7047 		regs_user->regs = regs;  in perf_sample_regs_user()
7048 	} else if (!(current->flags & PF_KTHREAD)) {  in perf_sample_regs_user()
7051 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;  in perf_sample_regs_user()
7052 		regs_user->regs = NULL;  in perf_sample_regs_user()
7059 	regs_intr->regs = regs;  in perf_sample_regs_intr()
7060 	regs_intr->abi  = perf_reg_abi(current);  in perf_sample_regs_intr()
7078 	return TASK_SIZE - addr;  in perf_ustack_task_size()
7093 	 * - TASK_SIZE  in perf_sample_ustack_size()
7096 	 * - remaining sample size  in perf_sample_ustack_size()
7113 		stack_size = USHRT_MAX - header_size - sizeof(u64);  in perf_sample_ustack_size()
7136 		 *   - the size requested by user or the best one we can fit  in perf_output_sample_ustack()
7139 		 *   - user stack dump data  in perf_output_sample_ustack()
7141 		 *   - the actual dumped size  in perf_output_sample_ustack()
7150 		dyn_size = dump_size - rem;  in perf_output_sample_ustack()
7163 	struct perf_event *sampler = event->aux_event;  in perf_prepare_sample_aux()
7166 	data->aux_size = 0;  in perf_prepare_sample_aux()
7171 	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))  in perf_prepare_sample_aux()
7174 	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))  in perf_prepare_sample_aux()
7185 	if (READ_ONCE(rb->aux_in_sampling)) {  in perf_prepare_sample_aux()
7186 		data->aux_size = 0;  in perf_prepare_sample_aux()
7189 		data->aux_size = ALIGN(size, sizeof(u64));  in perf_prepare_sample_aux()
7194 	return data->aux_size;  in perf_prepare_sample_aux()
7206 	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler  in perf_pmu_snapshot_aux()
7208 	 * the IRQ ones, that is, for example, re-starting an event that's just  in perf_pmu_snapshot_aux()
7219 	WRITE_ONCE(rb->aux_in_sampling, 1);  in perf_pmu_snapshot_aux()
7222 	ret = event->pmu->snapshot_aux(event, handle, size);  in perf_pmu_snapshot_aux()
7225 	WRITE_ONCE(rb->aux_in_sampling, 0);  in perf_pmu_snapshot_aux()
7235 	struct perf_event *sampler = event->aux_event;  in perf_aux_sample_output()
7240 	if (WARN_ON_ONCE(!sampler || !data->aux_size))  in perf_aux_sample_output()
7247 	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);  in perf_aux_sample_output()
7251 	 * non-zero surplus that it didn't copy), which in its current  in perf_aux_sample_output()
7259 	 * The pad comes from ALIGN()ing data->aux_size up to u64 in  in perf_aux_sample_output()
7262 	pad = data->aux_size - size;  in perf_aux_sample_output()
7276  * A set of common sample data types saved even for non-sample records
7277  * when event->attr.sample_id_all is set.
7287 	data->type = event->attr.sample_type;  in __perf_event_header__init_id()
7288 	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;  in __perf_event_header__init_id()
7292 		data->tid_entry.pid = perf_event_pid(event, current);  in __perf_event_header__init_id()
7293 		data->tid_entry.tid = perf_event_tid(event, current);  in __perf_event_header__init_id()
7297 		data->time = perf_event_clock(event);  in __perf_event_header__init_id()
7300 		data->id = primary_event_id(event);  in __perf_event_header__init_id()
7303 		data->stream_id = event->id;  in __perf_event_header__init_id()
7306 		data->cpu_entry.cpu	 = raw_smp_processor_id();  in __perf_event_header__init_id()
7307 		data->cpu_entry.reserved = 0;  in __perf_event_header__init_id()
7315 	if (event->attr.sample_id_all) {  in perf_event_header__init_id()
7316 		header->size += event->id_header_size;  in perf_event_header__init_id()
7317 		__perf_event_header__init_id(data, event, event->attr.sample_type);  in perf_event_header__init_id()
7324 	u64 sample_type = data->type;  in __perf_event__output_id_sample()
7327 		perf_output_put(handle, data->tid_entry);  in __perf_event__output_id_sample()
7330 		perf_output_put(handle, data->time);  in __perf_event__output_id_sample()
7333 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7336 		perf_output_put(handle, data->stream_id);  in __perf_event__output_id_sample()
7339 		perf_output_put(handle, data->cpu_entry);  in __perf_event__output_id_sample()
7342 		perf_output_put(handle, data->id);  in __perf_event__output_id_sample()
7349 	if (event->attr.sample_id_all)  in perf_event__output_id_sample()
7357 	u64 read_format = event->attr.read_format;  in perf_output_read_one()
7361 	values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr));  in perf_output_read_one()
7364 			atomic64_read(&event->child_total_time_enabled);  in perf_output_read_one()
7368 			atomic64_read(&event->child_total_time_running);  in perf_output_read_one()
7373 		values[n++] = atomic64_read(&event->lost_samples);  in perf_output_read_one()
7382 	struct perf_event *leader = event->group_leader, *sub;  in perf_output_read_group()
7383 	u64 read_format = event->attr.read_format;  in perf_output_read_group()
7387 	bool self = has_inherit_and_sample_read(&event->attr);  in perf_output_read_group()
7395 	values[n++] = 1 + leader->nr_siblings;  in perf_output_read_group()
7404 	    (leader->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7405 		leader->pmu->read(leader);  in perf_output_read_group()
7411 		values[n++] = atomic64_read(&leader->lost_samples);  in perf_output_read_group()
7419 		    (sub->state == PERF_EVENT_STATE_ACTIVE))  in perf_output_read_group()
7420 			sub->pmu->read(sub);  in perf_output_read_group()
7426 			values[n++] = atomic64_read(&sub->lost_samples);  in perf_output_read_group()
7444  * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
7452 	u64 read_format = event->attr.read_format;  in perf_output_read()
7466 	if (event->attr.read_format & PERF_FORMAT_GROUP)  in perf_output_read()
7477 	u64 sample_type = data->type;  in perf_output_sample()
7482 		perf_output_put(handle, data->id);  in perf_output_sample()
7485 		perf_output_put(handle, data->ip);  in perf_output_sample()
7488 		perf_output_put(handle, data->tid_entry);  in perf_output_sample()
7491 		perf_output_put(handle, data->time);  in perf_output_sample()
7494 		perf_output_put(handle, data->addr);  in perf_output_sample()
7497 		perf_output_put(handle, data->id);  in perf_output_sample()
7500 		perf_output_put(handle, data->stream_id);  in perf_output_sample()
7503 		perf_output_put(handle, data->cpu_entry);  in perf_output_sample()
7506 		perf_output_put(handle, data->period);  in perf_output_sample()
7514 		size += data->callchain->nr;  in perf_output_sample()
7516 		__output_copy(handle, data->callchain, size);  in perf_output_sample()
7520 		struct perf_raw_record *raw = data->raw;  in perf_output_sample()
7523 			struct perf_raw_frag *frag = &raw->frag;  in perf_output_sample()
7525 			perf_output_put(handle, raw->size);  in perf_output_sample()
7527 				if (frag->copy) {  in perf_output_sample()
7528 					__output_custom(handle, frag->copy,  in perf_output_sample()
7529 							frag->data, frag->size);  in perf_output_sample()
7531 					__output_copy(handle, frag->data,  in perf_output_sample()
7532 						      frag->size);  in perf_output_sample()
7536 				frag = frag->next;  in perf_output_sample()
7538 			if (frag->pad)  in perf_output_sample()
7539 				__output_skip(handle, NULL, frag->pad);  in perf_output_sample()
7553 		if (data->br_stack) {  in perf_output_sample()
7556 			size = data->br_stack->nr  in perf_output_sample()
7559 			perf_output_put(handle, data->br_stack->nr);  in perf_output_sample()
7561 				perf_output_put(handle, data->br_stack->hw_idx);  in perf_output_sample()
7562 			perf_output_copy(handle, data->br_stack->entries, size);  in perf_output_sample()
7567 			if (data->br_stack_cntr) {  in perf_output_sample()
7568 				size = data->br_stack->nr * sizeof(u64);  in perf_output_sample()
7569 				perf_output_copy(handle, data->br_stack_cntr, size);  in perf_output_sample()
7581 		u64 abi = data->regs_user.abi;  in perf_output_sample()
7590 			u64 mask = event->attr.sample_regs_user;  in perf_output_sample()
7592 						data->regs_user.regs,  in perf_output_sample()
7599 					  data->stack_user_size,  in perf_output_sample()
7600 					  data->regs_user.regs);  in perf_output_sample()
7604 		perf_output_put(handle, data->weight.full);  in perf_output_sample()
7607 		perf_output_put(handle, data->data_src.val);  in perf_output_sample()
7610 		perf_output_put(handle, data->txn);  in perf_output_sample()
7613 		u64 abi = data->regs_intr.abi;  in perf_output_sample()
7621 			u64 mask = event->attr.sample_regs_intr;  in perf_output_sample()
7624 						data->regs_intr.regs,  in perf_output_sample()
7630 		perf_output_put(handle, data->phys_addr);  in perf_output_sample()
7633 		perf_output_put(handle, data->cgroup);  in perf_output_sample()
7636 		perf_output_put(handle, data->data_page_size);  in perf_output_sample()
7639 		perf_output_put(handle, data->code_page_size);  in perf_output_sample()
7642 		perf_output_put(handle, data->aux_size);  in perf_output_sample()
7644 		if (data->aux_size)  in perf_output_sample()
7648 	if (!event->attr.watermark) {  in perf_output_sample()
7649 		int wakeup_events = event->attr.wakeup_events;  in perf_output_sample()
7652 			struct perf_buffer *rb = handle->rb;  in perf_output_sample()
7653 			int events = local_inc_return(&rb->events);  in perf_output_sample()
7656 				local_sub(wakeup_events, &rb->events);  in perf_output_sample()
7657 				local_inc(&rb->wakeup);  in perf_output_sample()
7680 		 * Try IRQ-safe get_user_page_fast_only first.  in perf_virt_to_phys()
7683 		if (current->mm != NULL) {  in perf_virt_to_phys()
7768 	 * Software page-table walkers must disable IRQs,  in perf_get_page_size()
7773 	mm = current->mm;  in perf_get_page_size()
7794 	bool kernel = !event->attr.exclude_callchain_kernel;  in perf_callchain()
7795 	bool user   = !event->attr.exclude_callchain_user;  in perf_callchain()
7796 	/* Disallow cross-task user callchains. */  in perf_callchain()
7797 	bool crosstask = event->ctx->task && event->ctx->task != current;  in perf_callchain()
7798 	const u32 max_stack = event->attr.sample_max_stack;  in perf_callchain()
7818 	u64 sample_type = event->attr.sample_type;  in perf_prepare_sample()
7832 	filtered_sample_type &= ~data->sample_flags;  in perf_prepare_sample()
7835 		/* Make sure it has the correct data->type for output */  in perf_prepare_sample()
7836 		data->type = event->attr.sample_type;  in perf_prepare_sample()
7843 		data->ip = perf_instruction_pointer(event, regs);  in perf_prepare_sample()
7844 		data->sample_flags |= PERF_SAMPLE_IP;  in perf_prepare_sample()
7851 		data->raw = NULL;  in perf_prepare_sample()
7852 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7853 		data->sample_flags |= PERF_SAMPLE_RAW;  in perf_prepare_sample()
7857 		data->br_stack = NULL;  in perf_prepare_sample()
7858 		data->dyn_size += sizeof(u64);  in perf_prepare_sample()
7859 		data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;  in perf_prepare_sample()
7863 		perf_sample_regs_user(&data->regs_user, regs);  in perf_prepare_sample()
7870 	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {  in perf_prepare_sample()
7874 		if (data->regs_user.regs) {  in perf_prepare_sample()
7875 			u64 mask = event->attr.sample_regs_user;  in perf_prepare_sample()
7879 		data->dyn_size += size;  in perf_prepare_sample()
7880 		data->sample_flags |= PERF_SAMPLE_REGS_USER;  in perf_prepare_sample()
7890 		u16 stack_size = event->attr.sample_stack_user;  in perf_prepare_sample()
7895 						     data->regs_user.regs);  in perf_prepare_sample()
7905 		data->stack_user_size = stack_size;  in perf_prepare_sample()
7906 		data->dyn_size += size;  in perf_prepare_sample()
7907 		data->sample_flags |= PERF_SAMPLE_STACK_USER;  in perf_prepare_sample()
7911 		data->weight.full = 0;  in perf_prepare_sample()
7912 		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;  in perf_prepare_sample()
7916 		data->data_src.val = PERF_MEM_NA;  in perf_prepare_sample()
7917 		data->sample_flags |= PERF_SAMPLE_DATA_SRC;  in perf_prepare_sample()
7921 		data->txn = 0;  in perf_prepare_sample()
7922 		data->sample_flags |= PERF_SAMPLE_TRANSACTION;  in perf_prepare_sample()
7926 		data->addr = 0;  in perf_prepare_sample()
7927 		data->sample_flags |= PERF_SAMPLE_ADDR;  in perf_prepare_sample()
7934 		perf_sample_regs_intr(&data->regs_intr, regs);  in perf_prepare_sample()
7936 		if (data->regs_intr.regs) {  in perf_prepare_sample()
7937 			u64 mask = event->attr.sample_regs_intr;  in perf_prepare_sample()
7942 		data->dyn_size += size;  in perf_prepare_sample()
7943 		data->sample_flags |= PERF_SAMPLE_REGS_INTR;  in perf_prepare_sample()
7947 		data->phys_addr = perf_virt_to_phys(data->addr);  in perf_prepare_sample()
7948 		data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;  in perf_prepare_sample()
7956 		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;  in perf_prepare_sample()
7957 		data->cgroup = cgroup_id(cgrp);  in perf_prepare_sample()
7958 		data->sample_flags |= PERF_SAMPLE_CGROUP;  in perf_prepare_sample()
7964 	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,  in perf_prepare_sample()
7968 		data->data_page_size = perf_get_page_size(data->addr);  in perf_prepare_sample()
7969 		data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;  in perf_prepare_sample()
7973 		data->code_page_size = perf_get_page_size(data->ip);  in perf_prepare_sample()
7974 		data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;  in perf_prepare_sample()
7989 		size = min_t(size_t, U16_MAX - header_size,  in perf_prepare_sample()
7990 			     event->attr.aux_sample_size);  in perf_prepare_sample()
7995 		data->dyn_size += size + sizeof(u64); /* size above */  in perf_prepare_sample()
7996 		data->sample_flags |= PERF_SAMPLE_AUX;  in perf_prepare_sample()
8005 	header->type = PERF_RECORD_SAMPLE;  in perf_prepare_header()
8006 	header->size = perf_sample_data_size(data, event);  in perf_prepare_header()
8007 	header->misc = perf_misc_flags(event, regs);  in perf_prepare_header()
8017 	WARN_ON_ONCE(header->size & 7);  in perf_prepare_header()
8023 		if (!event->hw.aux_paused) {  in __perf_event_aux_pause()
8024 			event->hw.aux_paused = 1;  in __perf_event_aux_pause()
8025 			event->pmu->stop(event, PERF_EF_PAUSE);  in __perf_event_aux_pause()
8028 		if (event->hw.aux_paused) {  in __perf_event_aux_pause()
8029 			event->hw.aux_paused = 0;  in __perf_event_aux_pause()
8030 			event->pmu->start(event, PERF_EF_RESUME);  in __perf_event_aux_pause()
8048 		 * Guard against self-recursion here. Another event could trip  in perf_event_aux_pause()
8051 		if (READ_ONCE(rb->aux_in_pause_resume))  in perf_event_aux_pause()
8054 		WRITE_ONCE(rb->aux_in_pause_resume, 1);  in perf_event_aux_pause()
8058 		WRITE_ONCE(rb->aux_in_pause_resume, 0);  in perf_event_aux_pause()
8140 			.size = sizeof(read_event) + event->read_size,  in perf_event_read_event()
8168 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_iterate_ctx()
8170 			if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_ctx()
8185 	list_for_each_entry_rcu(event, &pel->list, sb_list) {  in perf_iterate_sb_cpu()
8188 		 * if we observe event->ctx, both event and ctx will be  in perf_iterate_sb_cpu()
8191 		if (!smp_load_acquire(&event->ctx))  in perf_iterate_sb_cpu()
8194 		if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_sb_cpu()
8203  * Iterate all events that need to receive side-band events.
8229 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_iterate_sb()
8238  * Clear all file-based filters at exec, they'll have to be
8239  * re-instated when/if these objects are mmapped again.
8251 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8252 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_exec()
8253 		if (filter->path.dentry) {  in perf_event_addr_filters_exec()
8254 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_exec()
8255 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_exec()
8263 		event->addr_filters_gen++;  in perf_event_addr_filters_exec()
8264 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_exec()
8293 	struct perf_event *parent = event->parent;  in __perf_event_output_stop()
8295 	struct perf_buffer *rb = ro->rb;  in __perf_event_output_stop()
8308 	 * ring-buffer, but it will be the child that's actually using it.  in __perf_event_output_stop()
8313 	 * So ring_buffer_attach() has to stop an aux event before re-assigning  in __perf_event_output_stop()
8316 	if (rcu_dereference(parent->rb) == rb)  in __perf_event_output_stop()
8317 		ro->err = __perf_event_stop(&sd);  in __perf_event_output_stop()
8325 		.rb	= event->rb,  in __perf_pmu_output_stop()
8329 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);  in __perf_pmu_output_stop()
8330 	if (cpuctx->task_ctx)  in __perf_pmu_output_stop()
8331 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,  in __perf_pmu_output_stop()
8345 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {  in perf_pmu_output_stop()
8347 		 * For per-CPU events, we need to make sure that neither they  in perf_pmu_output_stop()
8348 		 * nor their children are running; for cpu==-1 events it's  in perf_pmu_output_stop()
8352 		cpu = iter->cpu;  in perf_pmu_output_stop()
8353 		if (cpu == -1)  in perf_pmu_output_stop()
8354 			cpu = READ_ONCE(iter->oncpu);  in perf_pmu_output_stop()
8356 		if (cpu == -1)  in perf_pmu_output_stop()
8360 		if (err == -EAGAIN) {  in perf_pmu_output_stop()
8369  * task tracking -- fork/exit
8391 	return event->attr.comm  || event->attr.mmap ||  in perf_event_task_match()
8392 	       event->attr.mmap2 || event->attr.mmap_data ||  in perf_event_task_match()
8393 	       event->attr.task;  in perf_event_task_match()
8402 	struct task_struct *task = task_event->task;  in perf_event_task_output()
8403 	int ret, size = task_event->event_id.header.size;  in perf_event_task_output()
8408 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);  in perf_event_task_output()
8411 				task_event->event_id.header.size);  in perf_event_task_output()
8415 	task_event->event_id.pid = perf_event_pid(event, task);  in perf_event_task_output()
8416 	task_event->event_id.tid = perf_event_tid(event, task);  in perf_event_task_output()
8418 	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {  in perf_event_task_output()
8419 		task_event->event_id.ppid = perf_event_pid(event,  in perf_event_task_output()
8420 							task->real_parent);  in perf_event_task_output()
8421 		task_event->event_id.ptid = perf_event_pid(event,  in perf_event_task_output()
8422 							task->real_parent);  in perf_event_task_output()
8424 		task_event->event_id.ppid = perf_event_pid(event, current);  in perf_event_task_output()
8425 		task_event->event_id.ptid = perf_event_tid(event, current);  in perf_event_task_output()
8428 	task_event->event_id.time = perf_event_clock(event);  in perf_event_task_output()
8430 	perf_output_put(&handle, task_event->event_id);  in perf_event_task_output()
8436 	task_event->event_id.header.size = size;  in perf_event_task_output()
8497 	return event->attr.comm;  in perf_event_comm_match()
8506 	int size = comm_event->event_id.header.size;  in perf_event_comm_output()
8512 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);  in perf_event_comm_output()
8514 				comm_event->event_id.header.size);  in perf_event_comm_output()
8519 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);  in perf_event_comm_output()
8520 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);  in perf_event_comm_output()
8522 	perf_output_put(&handle, comm_event->event_id);  in perf_event_comm_output()
8523 	__output_copy(&handle, comm_event->comm,  in perf_event_comm_output()
8524 				   comm_event->comm_size);  in perf_event_comm_output()
8530 	comm_event->event_id.header.size = size;  in perf_event_comm_output()
8539 	strscpy(comm, comm_event->task->comm, sizeof(comm));  in perf_event_comm_event()
8542 	comm_event->comm = comm;  in perf_event_comm_event()
8543 	comm_event->comm_size = size;  in perf_event_comm_event()
8545 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;  in perf_event_comm_event()
8596 	return event->attr.namespaces;  in perf_event_namespaces_match()
8605 	u16 header_size = namespaces_event->event_id.header.size;  in perf_event_namespaces_output()
8611 	perf_event_header__init_id(&namespaces_event->event_id.header,  in perf_event_namespaces_output()
8614 				namespaces_event->event_id.header.size);  in perf_event_namespaces_output()
8618 	namespaces_event->event_id.pid = perf_event_pid(event,  in perf_event_namespaces_output()
8619 							namespaces_event->task);  in perf_event_namespaces_output()
8620 	namespaces_event->event_id.tid = perf_event_tid(event,  in perf_event_namespaces_output()
8621 							namespaces_event->task);  in perf_event_namespaces_output()
8623 	perf_output_put(&handle, namespaces_event->event_id);  in perf_event_namespaces_output()
8629 	namespaces_event->event_id.header.size = header_size;  in perf_event_namespaces_output()
8642 		ns_inode = ns_path.dentry->d_inode;  in perf_fill_ns_link_info()
8643 		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);  in perf_fill_ns_link_info()
8644 		ns_link_info->ino = ns_inode->i_ino;  in perf_fill_ns_link_info()
8724 	return event->attr.cgroup;  in perf_event_cgroup_match()
8732 	u16 header_size = cgroup_event->event_id.header.size;  in perf_event_cgroup_output()
8738 	perf_event_header__init_id(&cgroup_event->event_id.header,  in perf_event_cgroup_output()
8741 				cgroup_event->event_id.header.size);  in perf_event_cgroup_output()
8745 	perf_output_put(&handle, cgroup_event->event_id);  in perf_event_cgroup_output()
8746 	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);  in perf_event_cgroup_output()
8752 	cgroup_event->event_id.header.size = header_size;  in perf_event_cgroup_output()
8781 		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));  in perf_event_cgroup()
8837 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_match()
8838 	int executable = vma->vm_flags & VM_EXEC;  in perf_event_mmap_match()
8840 	return (!executable && event->attr.mmap_data) ||  in perf_event_mmap_match()
8841 	       (executable && (event->attr.mmap || event->attr.mmap2));  in perf_event_mmap_match()
8850 	int size = mmap_event->event_id.header.size;  in perf_event_mmap_output()
8851 	u32 type = mmap_event->event_id.header.type;  in perf_event_mmap_output()
8858 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8859 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;  in perf_event_mmap_output()
8860 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);  in perf_event_mmap_output()
8861 		mmap_event->event_id.header.size += sizeof(mmap_event->min);  in perf_event_mmap_output()
8862 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);  in perf_event_mmap_output()
8863 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);  in perf_event_mmap_output()
8864 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);  in perf_event_mmap_output()
8865 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);  in perf_event_mmap_output()
8868 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);  in perf_event_mmap_output()
8870 				mmap_event->event_id.header.size);  in perf_event_mmap_output()
8874 	mmap_event->event_id.pid = perf_event_pid(event, current);  in perf_event_mmap_output()
8875 	mmap_event->event_id.tid = perf_event_tid(event, current);  in perf_event_mmap_output()
8877 	use_build_id = event->attr.build_id && mmap_event->build_id_size;  in perf_event_mmap_output()
8879 	if (event->attr.mmap2 && use_build_id)  in perf_event_mmap_output()
8880 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;  in perf_event_mmap_output()
8882 	perf_output_put(&handle, mmap_event->event_id);  in perf_event_mmap_output()
8884 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8886 			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };  in perf_event_mmap_output()
8889 			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);  in perf_event_mmap_output()
8891 			perf_output_put(&handle, mmap_event->maj);  in perf_event_mmap_output()
8892 			perf_output_put(&handle, mmap_event->min);  in perf_event_mmap_output()
8893 			perf_output_put(&handle, mmap_event->ino);  in perf_event_mmap_output()
8894 			perf_output_put(&handle, mmap_event->ino_generation);  in perf_event_mmap_output()
8896 		perf_output_put(&handle, mmap_event->prot);  in perf_event_mmap_output()
8897 		perf_output_put(&handle, mmap_event->flags);  in perf_event_mmap_output()
8900 	__output_copy(&handle, mmap_event->file_name,  in perf_event_mmap_output()
8901 				   mmap_event->file_size);  in perf_event_mmap_output()
8907 	mmap_event->event_id.header.size = size;  in perf_event_mmap_output()
8908 	mmap_event->event_id.header.type = type;  in perf_event_mmap_output()
8913 	struct vm_area_struct *vma = mmap_event->vma;  in perf_event_mmap_event()
8914 	struct file *file = vma->vm_file;  in perf_event_mmap_event()
8923 	if (vma->vm_flags & VM_READ)  in perf_event_mmap_event()
8925 	if (vma->vm_flags & VM_WRITE)  in perf_event_mmap_event()
8927 	if (vma->vm_flags & VM_EXEC)  in perf_event_mmap_event()
8930 	if (vma->vm_flags & VM_MAYSHARE)  in perf_event_mmap_event()
8935 	if (vma->vm_flags & VM_LOCKED)  in perf_event_mmap_event()
8954 		name = file_path(file, buf, PATH_MAX - sizeof(u64));  in perf_event_mmap_event()
8959 		inode = file_inode(vma->vm_file);  in perf_event_mmap_event()
8960 		dev = inode->i_sb->s_dev;  in perf_event_mmap_event()
8961 		ino = inode->i_ino;  in perf_event_mmap_event()
8962 		gen = inode->i_generation;  in perf_event_mmap_event()
8968 		if (vma->vm_ops && vma->vm_ops->name)  in perf_event_mmap_event()
8969 			name = (char *) vma->vm_ops->name(vma);  in perf_event_mmap_event()
8995 	mmap_event->file_name = name;  in perf_event_mmap_event()
8996 	mmap_event->file_size = size;  in perf_event_mmap_event()
8997 	mmap_event->maj = maj;  in perf_event_mmap_event()
8998 	mmap_event->min = min;  in perf_event_mmap_event()
8999 	mmap_event->ino = ino;  in perf_event_mmap_event()
9000 	mmap_event->ino_generation = gen;  in perf_event_mmap_event()
9001 	mmap_event->prot = prot;  in perf_event_mmap_event()
9002 	mmap_event->flags = flags;  in perf_event_mmap_event()
9004 	if (!(vma->vm_flags & VM_EXEC))  in perf_event_mmap_event()
9005 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;  in perf_event_mmap_event()
9007 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;  in perf_event_mmap_event()
9010 		build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size);  in perf_event_mmap_event()
9026 	/* d_inode(NULL) won't be equal to any mapped user-space file */  in perf_addr_filter_match()
9027 	if (!filter->path.dentry)  in perf_addr_filter_match()
9030 	if (d_inode(filter->path.dentry) != file_inode(file))  in perf_addr_filter_match()
9033 	if (filter->offset > offset + size)  in perf_addr_filter_match()
9036 	if (filter->offset + filter->size < offset)  in perf_addr_filter_match()
9046 	unsigned long vma_size = vma->vm_end - vma->vm_start;  in perf_addr_filter_vma_adjust()
9047 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;  in perf_addr_filter_vma_adjust()
9048 	struct file *file = vma->vm_file;  in perf_addr_filter_vma_adjust()
9053 	if (filter->offset < off) {  in perf_addr_filter_vma_adjust()
9054 		fr->start = vma->vm_start;  in perf_addr_filter_vma_adjust()
9055 		fr->size = min(vma_size, filter->size - (off - filter->offset));  in perf_addr_filter_vma_adjust()
9057 		fr->start = vma->vm_start + filter->offset - off;  in perf_addr_filter_vma_adjust()
9058 		fr->size = min(vma->vm_end - fr->start, filter->size);  in perf_addr_filter_vma_adjust()
9075 	if (!vma->vm_file)  in __perf_addr_filters_adjust()
9078 	raw_spin_lock_irqsave(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9079 	list_for_each_entry(filter, &ifh->list, entry) {  in __perf_addr_filters_adjust()
9081 						&event->addr_filter_ranges[count]))  in __perf_addr_filters_adjust()
9088 		event->addr_filters_gen++;  in __perf_addr_filters_adjust()
9089 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in __perf_addr_filters_adjust()
9106 	if (!(vma->vm_flags & VM_EXEC))  in perf_addr_filters_adjust()
9110 	ctx = rcu_dereference(current->perf_event_ctxp);  in perf_addr_filters_adjust()
9135 			.start  = vma->vm_start,  in perf_event_mmap()
9136 			.len    = vma->vm_end - vma->vm_start,  in perf_event_mmap()
9137 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,  in perf_event_mmap()
9235 	return event->attr.context_switch;  in perf_event_switch_match()
9248 	/* Only CPU-wide events are allowed to see next/prev pid/tid */  in perf_event_switch_output()
9249 	if (event->ctx->task) {  in perf_event_switch_output()
9250 		se->event_id.header.type = PERF_RECORD_SWITCH;  in perf_event_switch_output()
9251 		se->event_id.header.size = sizeof(se->event_id.header);  in perf_event_switch_output()
9253 		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;  in perf_event_switch_output()
9254 		se->event_id.header.size = sizeof(se->event_id);  in perf_event_switch_output()
9255 		se->event_id.next_prev_pid =  in perf_event_switch_output()
9256 					perf_event_pid(event, se->next_prev);  in perf_event_switch_output()
9257 		se->event_id.next_prev_tid =  in perf_event_switch_output()
9258 					perf_event_tid(event, se->next_prev);  in perf_event_switch_output()
9261 	perf_event_header__init_id(&se->event_id.header, &sample, event);  in perf_event_switch_output()
9263 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);  in perf_event_switch_output()
9267 	if (event->ctx->task)  in perf_event_switch_output()
9268 		perf_output_put(&handle, se->event_id.header);  in perf_event_switch_output()
9270 		perf_output_put(&handle, se->event_id);  in perf_event_switch_output()
9329 		.stream_id	= event->id,  in perf_log_throttle()
9365 	return event->attr.ksymbol;  in perf_event_ksymbol_match()
9378 	perf_event_header__init_id(&ksymbol_event->event_id.header,  in perf_event_ksymbol_output()
9381 				ksymbol_event->event_id.header.size);  in perf_event_ksymbol_output()
9385 	perf_output_put(&handle, ksymbol_event->event_id);  in perf_event_ksymbol_output()
9386 	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);  in perf_event_ksymbol_output()
9455 	return event->attr.bpf_event;  in perf_event_bpf_match()
9468 	perf_event_header__init_id(&bpf_event->event_id.header,  in perf_event_bpf_output()
9471 				bpf_event->event_id.header.size);  in perf_event_bpf_output()
9475 	perf_output_put(&handle, bpf_event->event_id);  in perf_event_bpf_output()
9488 			   (u64)(unsigned long)prog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9489 			   prog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9490 			   prog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9492 	for (i = 1; i < prog->aux->func_cnt; i++) {  in perf_event_bpf_emit_ksymbols()
9493 		struct bpf_prog *subprog = prog->aux->func[i];  in perf_event_bpf_emit_ksymbols()
9497 			(u64)(unsigned long)subprog->bpf_func,  in perf_event_bpf_emit_ksymbols()
9498 			subprog->jited_len, unregister,  in perf_event_bpf_emit_ksymbols()
9499 			subprog->aux->ksym.name);  in perf_event_bpf_emit_ksymbols()
9531 			.id = prog->aux->id,  in perf_event_bpf_event()
9537 	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);  in perf_event_bpf_event()
9557 	return event->attr.text_poke;  in perf_event_text_poke_match()
9571 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);  in perf_event_text_poke_output()
9574 				text_poke_event->event_id.header.size);  in perf_event_text_poke_output()
9578 	perf_output_put(&handle, text_poke_event->event_id);  in perf_event_text_poke_output()
9579 	perf_output_put(&handle, text_poke_event->old_len);  in perf_event_text_poke_output()
9580 	perf_output_put(&handle, text_poke_event->new_len);  in perf_event_text_poke_output()
9582 	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);  in perf_event_text_poke_output()
9583 	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);  in perf_event_text_poke_output()
9585 	if (text_poke_event->pad)  in perf_event_text_poke_output()
9586 		__output_copy(&handle, &padding, text_poke_event->pad);  in perf_event_text_poke_output()
9604 	pad  = ALIGN(tot, sizeof(u64)) - tot;  in perf_event_text_poke()
9627 	event->attach_state |= PERF_ATTACH_ITRACE;  in perf_event_itrace_started()
9641 	if (event->parent)  in perf_log_itrace_start()
9642 		event = event->parent;  in perf_log_itrace_start()
9644 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||  in perf_log_itrace_start()
9645 	    event->attach_state & PERF_ATTACH_ITRACE)  in perf_log_itrace_start()
9676 	if (event->parent)  in perf_report_aux_output_id()
9677 		event = event->parent;  in perf_report_aux_output_id()
9700 	struct hw_perf_event *hwc = &event->hw;  in __perf_event_account_interrupt()
9705 	if (seq != hwc->interrupts_seq) {  in __perf_event_account_interrupt()
9706 		hwc->interrupts_seq = seq;  in __perf_event_account_interrupt()
9707 		hwc->interrupts = 1;  in __perf_event_account_interrupt()
9709 		hwc->interrupts++;  in __perf_event_account_interrupt()
9711 			     hwc->interrupts > max_samples_per_tick)) {  in __perf_event_account_interrupt()
9714 			hwc->interrupts = MAX_INTERRUPTS;  in __perf_event_account_interrupt()
9720 	if (event->attr.freq) {  in __perf_event_account_interrupt()
9722 		s64 delta = now - hwc->freq_time_stamp;  in __perf_event_account_interrupt()
9724 		hwc->freq_time_stamp = now;  in __perf_event_account_interrupt()
9727 			perf_adjust_period(event, delta, hwc->last_period, true);  in __perf_event_account_interrupt()
9745 	if (event->attr.exclude_kernel && !user_mode(regs))  in sample_is_allowed()
9767 	prog = READ_ONCE(event->prog);  in bpf_overflow_handler()
9783 	if (event->overflow_handler_context)  in perf_event_set_bpf_handler()
9785 		return -EINVAL;  in perf_event_set_bpf_handler()
9787 	if (event->prog)  in perf_event_set_bpf_handler()
9788 		return -EEXIST;  in perf_event_set_bpf_handler()
9790 	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)  in perf_event_set_bpf_handler()
9791 		return -EINVAL;  in perf_event_set_bpf_handler()
9793 	if (event->attr.precise_ip &&  in perf_event_set_bpf_handler()
9794 	    prog->call_get_stack &&  in perf_event_set_bpf_handler()
9795 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||  in perf_event_set_bpf_handler()
9796 	     event->attr.exclude_callchain_kernel ||  in perf_event_set_bpf_handler()
9797 	     event->attr.exclude_callchain_user)) {  in perf_event_set_bpf_handler()
9807 		return -EPROTO;  in perf_event_set_bpf_handler()
9810 	event->prog = prog;  in perf_event_set_bpf_handler()
9811 	event->bpf_cookie = bpf_cookie;  in perf_event_set_bpf_handler()
9817 	struct bpf_prog *prog = event->prog;  in perf_event_free_bpf_handler()
9822 	event->prog = NULL;  in perf_event_free_bpf_handler()
9837 	return -EOPNOTSUPP;  in perf_event_set_bpf_handler()
9853 	int events = atomic_read(&event->event_limit);  in __perf_event_overflow()
9857 	 * Non-sampling counters might still use the PMI to fold short  in __perf_event_overflow()
9865 	if (event->attr.aux_pause)  in __perf_event_overflow()
9866 		perf_event_aux_pause(event->aux_event, true);  in __perf_event_overflow()
9868 	if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT &&  in __perf_event_overflow()
9877 	event->pending_kill = POLL_IN;  in __perf_event_overflow()
9878 	if (events && atomic_dec_and_test(&event->event_limit)) {  in __perf_event_overflow()
9880 		event->pending_kill = POLL_HUP;  in __perf_event_overflow()
9884 	if (event->attr.sigtrap) {  in __perf_event_overflow()
9900 		if (!event->pending_work &&  in __perf_event_overflow()
9901 		    !task_work_add(current, &event->pending_task, notify_mode)) {  in __perf_event_overflow()
9902 			event->pending_work = pending_id;  in __perf_event_overflow()
9903 			local_inc(&event->ctx->nr_no_switch_fast);  in __perf_event_overflow()
9905 			event->pending_addr = 0;  in __perf_event_overflow()
9906 			if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))  in __perf_event_overflow()
9907 				event->pending_addr = data->addr;  in __perf_event_overflow()
9909 		} else if (event->attr.exclude_kernel && valid_sample) {  in __perf_event_overflow()
9917 			 *  2. Events that can overflow again before the IRQ-  in __perf_event_overflow()
9920 			 *     check 32-bit hash of the current IP.  in __perf_event_overflow()
9922 			WARN_ON_ONCE(event->pending_work != pending_id);  in __perf_event_overflow()
9926 	READ_ONCE(event->overflow_handler)(event, data, regs);  in __perf_event_overflow()
9928 	if (*perf_event_fasync(event) && event->pending_kill) {  in __perf_event_overflow()
9929 		event->pending_wakeup = 1;  in __perf_event_overflow()
9930 		irq_work_queue(&event->pending_irq);  in __perf_event_overflow()
9933 	if (event->attr.aux_resume)  in __perf_event_overflow()
9934 		perf_event_aux_pause(event->aux_event, false);  in __perf_event_overflow()
9958  * We directly increment event->count and keep a second value in
9959  * event->hw.period_left to count intervals. This period event
9960  * is kept in the range [-sample_period, 0] so that we can use the
9966 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_set_period()
9967 	u64 period = hwc->last_period;  in perf_swevent_set_period()
9971 	hwc->last_period = hwc->sample_period;  in perf_swevent_set_period()
9973 	old = local64_read(&hwc->period_left);  in perf_swevent_set_period()
9981 		val -= offset;  in perf_swevent_set_period()
9982 	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));  in perf_swevent_set_period()
9991 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_overflow()
9997 	if (hwc->interrupts == MAX_INTERRUPTS)  in perf_swevent_overflow()
10000 	for (; overflow; overflow--) {  in perf_swevent_overflow()
10005 			 * hwc->interrupts == MAX_INTERRUPTS.  in perf_swevent_overflow()
10017 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_event()
10019 	local64_add(nr, &event->count);  in perf_swevent_event()
10027 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {  in perf_swevent_event()
10028 		data->period = nr;  in perf_swevent_event()
10031 		data->period = event->hw.last_period;  in perf_swevent_event()
10033 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)  in perf_swevent_event()
10036 	if (local64_add_negative(nr, &hwc->period_left))  in perf_swevent_event()
10045 	if (event->hw.state & PERF_HES_STOPPED)  in perf_exclude_event()
10049 		if (event->attr.exclude_user && user_mode(regs))  in perf_exclude_event()
10052 		if (event->attr.exclude_kernel && !user_mode(regs))  in perf_exclude_event()
10065 	if (event->attr.type != type)  in perf_swevent_match()
10068 	if (event->attr.config != event_id)  in perf_swevent_match()
10089 	return &hlist->heads[hash];  in __find_swevent_head()
10098 	hlist = rcu_dereference(swhash->swevent_hlist);  in find_swevent_head_rcu()
10110 	u32 event_id = event->attr.config;  in find_swevent_head()
10111 	u64 type = event->attr.type;  in find_swevent_head()
10118 	hlist = rcu_dereference_protected(swhash->swevent_hlist,  in find_swevent_head()
10119 					  lockdep_is_held(&event->ctx->lock));  in find_swevent_head()
10152 	return get_recursion_context(current->perf_recursion);  in perf_swevent_get_recursion_context()
10158 	put_recursion_context(current->perf_recursion, rctx);  in perf_swevent_put_recursion_context()
10195 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_add()
10199 		hwc->last_period = hwc->sample_period;  in perf_swevent_add()
10203 	hwc->state = !(flags & PERF_EF_START);  in perf_swevent_add()
10207 		return -EINVAL;  in perf_swevent_add()
10209 	hlist_add_head_rcu(&event->hlist_entry, head);  in perf_swevent_add()
10217 	hlist_del_rcu(&event->hlist_entry);  in perf_swevent_del()
10222 	event->hw.state = 0;  in perf_swevent_start()
10227 	event->hw.state = PERF_HES_STOPPED;  in perf_swevent_stop()
10234 	return rcu_dereference_protected(swhash->swevent_hlist,  in swevent_hlist_deref()
10235 					 lockdep_is_held(&swhash->hlist_mutex));  in swevent_hlist_deref()
10245 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);  in swevent_hlist_release()
10253 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10255 	if (!--swhash->hlist_refcount)  in swevent_hlist_put_cpu()
10258 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_put_cpu()
10274 	mutex_lock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10281 			err = -ENOMEM;  in swevent_hlist_get_cpu()
10284 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in swevent_hlist_get_cpu()
10286 	swhash->hlist_refcount++;  in swevent_hlist_get_cpu()
10288 	mutex_unlock(&swhash->hlist_mutex);  in swevent_hlist_get_cpu()
10321 	u64 event_id = event->attr.config;  in sw_perf_event_destroy()
10323 	WARN_ON(event->parent);  in sw_perf_event_destroy()
10334 	u64 event_id = event->attr.config;  in perf_swevent_init()
10336 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in perf_swevent_init()
10337 		return -ENOENT;  in perf_swevent_init()
10343 		return -EOPNOTSUPP;  in perf_swevent_init()
10347 		event->attr.type = perf_cpu_clock.type;  in perf_swevent_init()
10348 		return -ENOENT;  in perf_swevent_init()
10350 		event->attr.type = perf_task_clock.type;  in perf_swevent_init()
10351 		return -ENOENT;  in perf_swevent_init()
10358 		return -ENOENT;  in perf_swevent_init()
10360 	if (!event->parent) {  in perf_swevent_init()
10368 		event->destroy = sw_perf_event_destroy;  in perf_swevent_init()
10398 	if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event_init()
10399 		return -ENOENT;  in perf_tp_event_init()
10405 		return -EOPNOTSUPP;  in perf_tp_event_init()
10411 	event->destroy = tp_perf_event_destroy;  in perf_tp_event_init()
10430 	void *record = data->raw->frag.data;  in perf_tp_filter_match()
10433 	if (event->parent)  in perf_tp_filter_match()
10434 		event = event->parent;  in perf_tp_filter_match()
10436 	if (likely(!event->filter) || filter_match_preds(event->filter, record))  in perf_tp_filter_match()
10445 	if (event->hw.state & PERF_HES_STOPPED)  in perf_tp_event_match()
10448 	 * If exclude_kernel, only trace user-space tracepoints (uprobes)  in perf_tp_event_match()
10450 	if (event->attr.exclude_kernel && !user_mode(regs))  in perf_tp_event_match()
10471 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,  in perf_trace_run_bpf_submit()
10483 	if (event->attr.config != entry->type)  in __perf_tp_event_target_task()
10486 	if (event->attr.sigtrap)  in __perf_tp_event_target_task()
10501 	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {  in perf_tp_event_target_task()
10507 	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {  in perf_tp_event_target_task()
10538 			 * Here use the same on-stack perf_sample_data,  in perf_tp_event()
10539 			 * some members in data are event-specific and  in perf_tp_event()
10540 			 * need to be re-computed for different sweveents.  in perf_tp_event()
10541 			 * Re-initialize data->sample_flags safely to avoid  in perf_tp_event()
10543 			 * because data->sample_flags is set.  in perf_tp_event()
10558 		ctx = rcu_dereference(task->perf_event_ctxp);  in perf_tp_event()
10562 		raw_spin_lock(&ctx->lock);  in perf_tp_event()
10564 		raw_spin_unlock(&ctx->lock);  in perf_tp_event()
10591 	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10630 	if (event->attr.type != perf_kprobe.type)  in perf_kprobe_event_init()
10631 		return -ENOENT;  in perf_kprobe_event_init()
10634 		return -EACCES;  in perf_kprobe_event_init()
10640 		return -EOPNOTSUPP;  in perf_kprobe_event_init()
10642 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_kprobe_event_init()
10647 	event->destroy = perf_kprobe_destroy;  in perf_kprobe_event_init()
10654 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10690 	if (event->attr.type != perf_uprobe.type)  in perf_uprobe_event_init()
10691 		return -ENOENT;  in perf_uprobe_event_init()
10694 		return -EACCES;  in perf_uprobe_event_init()
10700 		return -EOPNOTSUPP;  in perf_uprobe_event_init()
10702 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_uprobe_event_init()
10703 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;  in perf_uprobe_event_init()
10708 	event->destroy = perf_uprobe_destroy;  in perf_uprobe_event_init()
10718 	perf_pmu_register(&perf_kprobe, "kprobe", -1);  in perf_tp_register()
10721 	perf_pmu_register(&perf_uprobe, "uprobe", -1);  in perf_tp_register()
10736 	if (event->pmu == &perf_tracepoint)  in perf_event_is_tracing()
10739 	if (event->pmu == &perf_kprobe)  in perf_event_is_tracing()
10743 	if (event->pmu == &perf_uprobe)  in perf_event_is_tracing()
10757 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;  in perf_event_set_bpf_prog()
10758 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;  in perf_event_set_bpf_prog()
10759 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;  in perf_event_set_bpf_prog()
10760 	is_syscall_tp = is_syscall_trace_event(event->tp_event);  in perf_event_set_bpf_prog()
10763 		return -EINVAL;  in perf_event_set_bpf_prog()
10765 	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||  in perf_event_set_bpf_prog()
10766 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||  in perf_event_set_bpf_prog()
10767 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))  in perf_event_set_bpf_prog()
10768 		return -EINVAL;  in perf_event_set_bpf_prog()
10770 	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe)  in perf_event_set_bpf_prog()
10772 		return -EINVAL;  in perf_event_set_bpf_prog()
10775 	if (prog->kprobe_override && !is_kprobe)  in perf_event_set_bpf_prog()
10776 		return -EINVAL;  in perf_event_set_bpf_prog()
10779 		int off = trace_event_get_offsets(event->tp_event);  in perf_event_set_bpf_prog()
10781 		if (prog->aux->max_ctx_offset > off)  in perf_event_set_bpf_prog()
10782 			return -EACCES;  in perf_event_set_bpf_prog()
10810 	return -ENOENT;  in perf_event_set_bpf_prog()
10824 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);  in perf_bp_event()
10826 	if (!bp->hw.state && !perf_exclude_event(bp, regs))  in perf_bp_event()
10837 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);  in perf_addr_filter_new()
10844 	INIT_LIST_HEAD(&filter->entry);  in perf_addr_filter_new()
10845 	list_add_tail(&filter->entry, filters);  in perf_addr_filter_new()
10855 		path_put(&filter->path);  in free_filters_list()
10856 		list_del(&filter->entry);  in free_filters_list()
10874 	if (event->parent)  in perf_addr_filters_splice()
10877 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10879 	list_splice_init(&event->addr_filters.list, &list);  in perf_addr_filters_splice()
10881 		list_splice(head, &event->addr_filters.list);  in perf_addr_filters_splice()
10883 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10901 		if (!vma->vm_file)  in perf_addr_filter_apply()
10916 	struct task_struct *task = READ_ONCE(event->ctx->task);  in perf_event_addr_filters_apply()
10923 	 * We may observe TASK_TOMBSTONE, which means that the event tear-down  in perf_event_addr_filters_apply()
10929 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
10937 	raw_spin_lock_irqsave(&ifh->lock, flags);  in perf_event_addr_filters_apply()
10938 	list_for_each_entry(filter, &ifh->list, entry) {  in perf_event_addr_filters_apply()
10939 		if (filter->path.dentry) {  in perf_event_addr_filters_apply()
10944 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_apply()
10945 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_apply()
10947 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);  in perf_event_addr_filters_apply()
10949 			event->addr_filter_ranges[count].start = filter->offset;  in perf_event_addr_filters_apply()
10950 			event->addr_filter_ranges[count].size  = filter->size;  in perf_event_addr_filters_apply()
10956 	event->addr_filters_gen++;  in perf_event_addr_filters_apply()
10957 	raw_spin_unlock_irqrestore(&ifh->lock, flags);  in perf_event_addr_filters_apply()
10959 	if (ifh->nr_file_filters) {  in perf_event_addr_filters_apply()
10989 	IF_ACT_NONE = -1,
11028 	int ret = -EINVAL;  in perf_event_parse_addr_filter()
11032 		return -ENOMEM;  in perf_event_parse_addr_filter()
11040 		ret = -EINVAL;  in perf_event_parse_addr_filter()
11060 			filter->action = actions[token];  in perf_event_parse_addr_filter()
11075 			ret = kstrtoul(args[0].from, 0, &filter->offset);  in perf_event_parse_addr_filter()
11081 				ret = kstrtoul(args[1].from, 0, &filter->size);  in perf_event_parse_addr_filter()
11092 					ret = -ENOMEM;  in perf_event_parse_addr_filter()
11110 			ret = -EINVAL;  in perf_event_parse_addr_filter()
11113 			 * ACTION "filter" must have a non-zero length region  in perf_event_parse_addr_filter()
11116 			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&  in perf_event_parse_addr_filter()
11117 			    !filter->size)  in perf_event_parse_addr_filter()
11125 				 * For now, we only support file-based filters  in perf_event_parse_addr_filter()
11126 				 * in per-task events; doing so for CPU-wide  in perf_event_parse_addr_filter()
11132 				ret = -EOPNOTSUPP;  in perf_event_parse_addr_filter()
11133 				if (!event->ctx->task)  in perf_event_parse_addr_filter()
11138 						&filter->path);  in perf_event_parse_addr_filter()
11142 				ret = -EINVAL;  in perf_event_parse_addr_filter()
11143 				if (!filter->path.dentry ||  in perf_event_parse_addr_filter()
11144 				    !S_ISREG(d_inode(filter->path.dentry)  in perf_event_parse_addr_filter()
11145 					     ->i_mode))  in perf_event_parse_addr_filter()
11148 				event->addr_filters.nr_file_filters++;  in perf_event_parse_addr_filter()
11186 	lockdep_assert_held(&event->ctx->mutex);  in perf_event_set_addr_filter()
11188 	if (WARN_ON_ONCE(event->parent))  in perf_event_set_addr_filter()
11189 		return -EINVAL;  in perf_event_set_addr_filter()
11195 	ret = event->pmu->addr_filters_validate(&filters);  in perf_event_set_addr_filter()
11211 	event->addr_filters.nr_file_filters = 0;  in perf_event_set_addr_filter()
11218 	int ret = -EINVAL;  in perf_event_set_filter()
11227 		struct perf_event_context *ctx = event->ctx;  in perf_event_set_filter()
11232 		 * the tracepoint muck will deadlock against ctx->mutex, but  in perf_event_set_filter()
11234 		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we  in perf_event_set_filter()
11240 		mutex_unlock(&ctx->mutex);  in perf_event_set_filter()
11241 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);  in perf_event_set_filter()
11242 		mutex_lock(&ctx->mutex);  in perf_event_set_filter()
11266 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_swevent_hrtimer()
11269 	event->pmu->read(event);  in perf_swevent_hrtimer()
11271 	perf_sample_data_init(&data, 0, event->hw.last_period);  in perf_swevent_hrtimer()
11275 		if (!(event->attr.exclude_idle && is_idle_task(current)))  in perf_swevent_hrtimer()
11280 	period = max_t(u64, 10000, event->hw.sample_period);  in perf_swevent_hrtimer()
11288 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_start_hrtimer()
11294 	period = local64_read(&hwc->period_left);  in perf_swevent_start_hrtimer()
11299 		local64_set(&hwc->period_left, 0);  in perf_swevent_start_hrtimer()
11301 		period = max_t(u64, 10000, hwc->sample_period);  in perf_swevent_start_hrtimer()
11303 	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),  in perf_swevent_start_hrtimer()
11309 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_cancel_hrtimer()
11312 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11313 		local64_set(&hwc->period_left, ktime_to_ns(remaining));  in perf_swevent_cancel_hrtimer()
11315 		hrtimer_cancel(&hwc->hrtimer);  in perf_swevent_cancel_hrtimer()
11321 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_init_hrtimer()
11326 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);  in perf_swevent_init_hrtimer()
11327 	hwc->hrtimer.function = perf_swevent_hrtimer;  in perf_swevent_init_hrtimer()
11330 	 * Since hrtimers have a fixed rate, we can do a static freq->period  in perf_swevent_init_hrtimer()
11333 	if (event->attr.freq) {  in perf_swevent_init_hrtimer()
11334 		long freq = event->attr.sample_freq;  in perf_swevent_init_hrtimer()
11336 		event->attr.sample_period = NSEC_PER_SEC / freq;  in perf_swevent_init_hrtimer()
11337 		hwc->sample_period = event->attr.sample_period;  in perf_swevent_init_hrtimer()
11338 		local64_set(&hwc->period_left, hwc->sample_period);  in perf_swevent_init_hrtimer()
11339 		hwc->last_period = hwc->sample_period;  in perf_swevent_init_hrtimer()
11340 		event->attr.freq = 0;  in perf_swevent_init_hrtimer()
11354 	prev = local64_xchg(&event->hw.prev_count, now);  in cpu_clock_event_update()
11355 	local64_add(now - prev, &event->count);  in cpu_clock_event_update()
11360 	local64_set(&event->hw.prev_count, local_clock());  in cpu_clock_event_start()
11391 	if (event->attr.type != perf_cpu_clock.type)  in cpu_clock_event_init()
11392 		return -ENOENT;  in cpu_clock_event_init()
11394 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)  in cpu_clock_event_init()
11395 		return -ENOENT;  in cpu_clock_event_init()
11401 		return -EOPNOTSUPP;  in cpu_clock_event_init()
11431 	prev = local64_xchg(&event->hw.prev_count, now);  in task_clock_event_update()
11432 	delta = now - prev;  in task_clock_event_update()
11433 	local64_add(delta, &event->count);  in task_clock_event_update()
11438 	local64_set(&event->hw.prev_count, event->ctx->time);  in task_clock_event_start()
11445 	task_clock_event_update(event, event->ctx->time);  in task_clock_event_stop()
11465 	u64 delta = now - event->ctx->timestamp;  in task_clock_event_read()
11466 	u64 time = event->ctx->time + delta;  in task_clock_event_read()
11473 	if (event->attr.type != perf_task_clock.type)  in task_clock_event_init()
11474 		return -ENOENT;  in task_clock_event_init()
11476 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)  in task_clock_event_init()
11477 		return -ENOENT;  in task_clock_event_init()
11483 		return -EOPNOTSUPP;  in task_clock_event_init()
11566 	free_percpu(pmu->cpu_pmu_context);  in free_pmu_context()
11578 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);  in nr_addr_filters_show()
11589 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type);  in type_show()
11600 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms);  in perf_event_mux_interval_ms_show()
11618 		return -EINVAL;  in perf_event_mux_interval_ms_store()
11621 	if (timer == pmu->hrtimer_interval_ms)  in perf_event_mux_interval_ms_store()
11625 	pmu->hrtimer_interval_ms = timer;  in perf_event_mux_interval_ms_store()
11631 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_event_mux_interval_ms_store()
11632 		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
11683 	struct cpumask *mask = perf_scope_cpumask(pmu->scope);  in cpumask_show()
11705 	if (n == 2 && !pmu->nr_addr_filters)  in pmu_dev_is_visible()
11709 	if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE)  in pmu_dev_is_visible()
11712 	return a->mode;  in pmu_dev_is_visible()
11738 	int ret = -ENOMEM;  in pmu_dev_alloc()
11740 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);  in pmu_dev_alloc()
11741 	if (!pmu->dev)  in pmu_dev_alloc()
11744 	pmu->dev->groups = pmu->attr_groups;  in pmu_dev_alloc()
11745 	device_initialize(pmu->dev);  in pmu_dev_alloc()
11747 	dev_set_drvdata(pmu->dev, pmu);  in pmu_dev_alloc()
11748 	pmu->dev->bus = &pmu_bus;  in pmu_dev_alloc()
11749 	pmu->dev->parent = pmu->parent;  in pmu_dev_alloc()
11750 	pmu->dev->release = pmu_dev_release;  in pmu_dev_alloc()
11752 	ret = dev_set_name(pmu->dev, "%s", pmu->name);  in pmu_dev_alloc()
11756 	ret = device_add(pmu->dev);  in pmu_dev_alloc()
11760 	if (pmu->attr_update) {  in pmu_dev_alloc()
11761 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);  in pmu_dev_alloc()
11770 	device_del(pmu->dev);  in pmu_dev_alloc()
11773 	put_device(pmu->dev);  in pmu_dev_alloc()
11785 	ret = -ENOMEM;  in perf_pmu_register()
11786 	pmu->pmu_disable_count = alloc_percpu(int);  in perf_pmu_register()
11787 	if (!pmu->pmu_disable_count)  in perf_pmu_register()
11790 	pmu->type = -1;  in perf_pmu_register()
11792 		ret = -EINVAL;  in perf_pmu_register()
11796 …if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE, "Can not register a pmu with an invalid scope.\n")…  in perf_pmu_register()
11797 		ret = -EINVAL;  in perf_pmu_register()
11801 	pmu->name = name;  in perf_pmu_register()
11813 	pmu->type = type;  in perf_pmu_register()
11815 	if (pmu_bus_running && !pmu->dev) {  in perf_pmu_register()
11821 	ret = -ENOMEM;  in perf_pmu_register()
11822 	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context);  in perf_pmu_register()
11823 	if (!pmu->cpu_pmu_context)  in perf_pmu_register()
11829 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_pmu_register()
11830 		__perf_init_event_pmu_context(&cpc->epc, pmu);  in perf_pmu_register()
11834 	if (!pmu->start_txn) {  in perf_pmu_register()
11835 		if (pmu->pmu_enable) {  in perf_pmu_register()
11841 			pmu->start_txn  = perf_pmu_start_txn;  in perf_pmu_register()
11842 			pmu->commit_txn = perf_pmu_commit_txn;  in perf_pmu_register()
11843 			pmu->cancel_txn = perf_pmu_cancel_txn;  in perf_pmu_register()
11845 			pmu->start_txn  = perf_pmu_nop_txn;  in perf_pmu_register()
11846 			pmu->commit_txn = perf_pmu_nop_int;  in perf_pmu_register()
11847 			pmu->cancel_txn = perf_pmu_nop_void;  in perf_pmu_register()
11851 	if (!pmu->pmu_enable) {  in perf_pmu_register()
11852 		pmu->pmu_enable  = perf_pmu_nop_void;  in perf_pmu_register()
11853 		pmu->pmu_disable = perf_pmu_nop_void;  in perf_pmu_register()
11856 	if (!pmu->check_period)  in perf_pmu_register()
11857 		pmu->check_period = perf_event_nop_int;  in perf_pmu_register()
11859 	if (!pmu->event_idx)  in perf_pmu_register()
11860 		pmu->event_idx = perf_event_idx_default;  in perf_pmu_register()
11862 	list_add_rcu(&pmu->entry, &pmus);  in perf_pmu_register()
11863 	atomic_set(&pmu->exclusive_cnt, 0);  in perf_pmu_register()
11871 	if (pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_register()
11872 		device_del(pmu->dev);  in perf_pmu_register()
11873 		put_device(pmu->dev);  in perf_pmu_register()
11877 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_register()
11880 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_register()
11888 	list_del_rcu(&pmu->entry);  in perf_pmu_unregister()
11897 	free_percpu(pmu->pmu_disable_count);  in perf_pmu_unregister()
11898 	idr_remove(&pmu_idr, pmu->type);  in perf_pmu_unregister()
11899 	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {  in perf_pmu_unregister()
11900 		if (pmu->nr_addr_filters)  in perf_pmu_unregister()
11901 			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);  in perf_pmu_unregister()
11902 		device_del(pmu->dev);  in perf_pmu_unregister()
11903 		put_device(pmu->dev);  in perf_pmu_unregister()
11912 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||  in has_extended_regs()
11913 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);  in has_extended_regs()
11921 	if (!try_module_get(pmu->module))  in perf_try_init_event()
11922 		return -ENODEV;  in perf_try_init_event()
11925 	 * A number of pmu->event_init() methods iterate the sibling_list to,  in perf_try_init_event()
11927 	 * if this is a sibling event, acquire the ctx->mutex to protect  in perf_try_init_event()
11930 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {  in perf_try_init_event()
11932 		 * This ctx->mutex can nest when we're called through  in perf_try_init_event()
11935 		ctx = perf_event_ctx_lock_nested(event->group_leader,  in perf_try_init_event()
11940 	event->pmu = pmu;  in perf_try_init_event()
11941 	ret = pmu->event_init(event);  in perf_try_init_event()
11944 		perf_event_ctx_unlock(event->group_leader, ctx);  in perf_try_init_event()
11947 		if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&  in perf_try_init_event()
11949 			ret = -EOPNOTSUPP;  in perf_try_init_event()
11951 		if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&  in perf_try_init_event()
11953 			ret = -EINVAL;  in perf_try_init_event()
11955 		if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) {  in perf_try_init_event()
11956 			const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu);  in perf_try_init_event()
11957 			struct cpumask *pmu_cpumask = perf_scope_cpumask(pmu->scope);  in perf_try_init_event()
11963 					ret = -ENODEV;  in perf_try_init_event()
11965 					event->event_caps |= PERF_EV_CAP_READ_SCOPE;  in perf_try_init_event()
11967 				ret = -ENODEV;  in perf_try_init_event()
11971 		if (ret && event->destroy)  in perf_try_init_event()
11972 			event->destroy(event);  in perf_try_init_event()
11976 		module_put(pmu->module);  in perf_try_init_event()
11990 	 * Save original type before calling pmu->event_init() since certain  in perf_init_event()
11991 	 * pmus overwrites event->attr.type to forward event to another pmu.  in perf_init_event()
11993 	event->orig_type = event->attr.type;  in perf_init_event()
11996 	if (event->parent && event->parent->pmu) {  in perf_init_event()
11997 		pmu = event->parent->pmu;  in perf_init_event()
12007 	type = event->attr.type;  in perf_init_event()
12009 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;  in perf_init_event()
12014 			event->attr.config &= PERF_HW_EVENT_MASK;  in perf_init_event()
12023 		if (event->attr.type != type && type != PERF_TYPE_RAW &&  in perf_init_event()
12024 		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))  in perf_init_event()
12028 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {  in perf_init_event()
12029 			type = event->attr.type;  in perf_init_event()
12044 		if (ret != -ENOENT) {  in perf_init_event()
12050 	pmu = ERR_PTR(-ENOENT);  in perf_init_event()
12059 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in attach_sb_event()
12061 	raw_spin_lock(&pel->lock);  in attach_sb_event()
12062 	list_add_rcu(&event->sb_list, &pel->list);  in attach_sb_event()
12063 	raw_spin_unlock(&pel->lock);  in attach_sb_event()
12067  * We keep a list of all !task (and therefore per-cpu) events
12068  * that need to receive side-band records.
12070  * This avoids having to scan all the various PMU per-cpu contexts
12104 	if (event->parent)  in account_event()
12107 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in account_event()
12109 	if (event->attr.mmap || event->attr.mmap_data)  in account_event()
12111 	if (event->attr.build_id)  in account_event()
12113 	if (event->attr.comm)  in account_event()
12115 	if (event->attr.namespaces)  in account_event()
12117 	if (event->attr.cgroup)  in account_event()
12119 	if (event->attr.task)  in account_event()
12121 	if (event->attr.freq)  in account_event()
12123 	if (event->attr.context_switch) {  in account_event()
12131 	if (event->attr.ksymbol)  in account_event()
12133 	if (event->attr.bpf_event)  in account_event()
12135 	if (event->attr.text_poke)  in account_event()
12159 		 * increments to by-pass the mutex.  in account_event()
12183 	long err = -EINVAL;  in perf_event_alloc()
12187 		if (!task || cpu != -1)  in perf_event_alloc()
12188 			return ERR_PTR(-EINVAL);  in perf_event_alloc()
12190 	if (attr->sigtrap && !task) {  in perf_event_alloc()
12192 		return ERR_PTR(-EINVAL);  in perf_event_alloc()
12195 	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;  in perf_event_alloc()
12199 		return ERR_PTR(-ENOMEM);  in perf_event_alloc()
12208 	mutex_init(&event->child_mutex);  in perf_event_alloc()
12209 	INIT_LIST_HEAD(&event->child_list);  in perf_event_alloc()
12211 	INIT_LIST_HEAD(&event->event_entry);  in perf_event_alloc()
12212 	INIT_LIST_HEAD(&event->sibling_list);  in perf_event_alloc()
12213 	INIT_LIST_HEAD(&event->active_list);  in perf_event_alloc()
12215 	INIT_LIST_HEAD(&event->rb_entry);  in perf_event_alloc()
12216 	INIT_LIST_HEAD(&event->active_entry);  in perf_event_alloc()
12217 	INIT_LIST_HEAD(&event->addr_filters.list);  in perf_event_alloc()
12218 	INIT_HLIST_NODE(&event->hlist_entry);  in perf_event_alloc()
12221 	init_waitqueue_head(&event->waitq);  in perf_event_alloc()
12222 	init_irq_work(&event->pending_irq, perf_pending_irq);  in perf_event_alloc()
12223 	event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);  in perf_event_alloc()
12224 	init_task_work(&event->pending_task, perf_pending_task);  in perf_event_alloc()
12225 	rcuwait_init(&event->pending_work_wait);  in perf_event_alloc()
12227 	mutex_init(&event->mmap_mutex);  in perf_event_alloc()
12228 	raw_spin_lock_init(&event->addr_filters.lock);  in perf_event_alloc()
12230 	atomic_long_set(&event->refcount, 1);  in perf_event_alloc()
12231 	event->cpu		= cpu;  in perf_event_alloc()
12232 	event->attr		= *attr;  in perf_event_alloc()
12233 	event->group_leader	= group_leader;  in perf_event_alloc()
12234 	event->pmu		= NULL;  in perf_event_alloc()
12235 	event->oncpu		= -1;  in perf_event_alloc()
12237 	event->parent		= parent_event;  in perf_event_alloc()
12239 	event->ns		= get_pid_ns(task_active_pid_ns(current));  in perf_event_alloc()
12240 	event->id		= atomic64_inc_return(&perf_event_id);  in perf_event_alloc()
12242 	event->state		= PERF_EVENT_STATE_INACTIVE;  in perf_event_alloc()
12245 		event->event_caps = parent_event->event_caps;  in perf_event_alloc()
12248 		event->attach_state = PERF_ATTACH_TASK;  in perf_event_alloc()
12254 		event->hw.target = get_task_struct(task);  in perf_event_alloc()
12257 	event->clock = &local_clock;  in perf_event_alloc()
12259 		event->clock = parent_event->clock;  in perf_event_alloc()
12262 		overflow_handler = parent_event->overflow_handler;  in perf_event_alloc()
12263 		context = parent_event->overflow_handler_context;  in perf_event_alloc()
12265 		if (parent_event->prog) {  in perf_event_alloc()
12266 			struct bpf_prog *prog = parent_event->prog;  in perf_event_alloc()
12269 			event->prog = prog;  in perf_event_alloc()
12275 		event->overflow_handler	= overflow_handler;  in perf_event_alloc()
12276 		event->overflow_handler_context = context;  in perf_event_alloc()
12278 		event->overflow_handler = perf_event_output_backward;  in perf_event_alloc()
12279 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12281 		event->overflow_handler = perf_event_output_forward;  in perf_event_alloc()
12282 		event->overflow_handler_context = NULL;  in perf_event_alloc()
12289 	hwc = &event->hw;  in perf_event_alloc()
12290 	hwc->sample_period = attr->sample_period;  in perf_event_alloc()
12291 	if (attr->freq && attr->sample_freq)  in perf_event_alloc()
12292 		hwc->sample_period = 1;  in perf_event_alloc()
12293 	hwc->last_period = hwc->sample_period;  in perf_event_alloc()
12295 	local64_set(&hwc->period_left, hwc->sample_period);  in perf_event_alloc()
12300 	 * collect per-thread samples.  in perf_event_alloc()
12303 	if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))  in perf_event_alloc()
12307 		event->attr.branch_sample_type = 0;  in perf_event_alloc()
12316 	 * Disallow uncore-task events. Similarly, disallow uncore-cgroup  in perf_event_alloc()
12320 	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) {  in perf_event_alloc()
12321 		err = -EINVAL;  in perf_event_alloc()
12325 	if (event->attr.aux_output &&  in perf_event_alloc()
12326 	    (!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) ||  in perf_event_alloc()
12327 	     event->attr.aux_pause || event->attr.aux_resume)) {  in perf_event_alloc()
12328 		err = -EOPNOTSUPP;  in perf_event_alloc()
12332 	if (event->attr.aux_pause && event->attr.aux_resume) {  in perf_event_alloc()
12333 		err = -EINVAL;  in perf_event_alloc()
12337 	if (event->attr.aux_start_paused) {  in perf_event_alloc()
12338 		if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) {  in perf_event_alloc()
12339 			err = -EOPNOTSUPP;  in perf_event_alloc()
12342 		event->hw.aux_paused = 1;  in perf_event_alloc()
12345 	if (cgroup_fd != -1) {  in perf_event_alloc()
12356 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,  in perf_event_alloc()
12359 		if (!event->addr_filter_ranges) {  in perf_event_alloc()
12360 			err = -ENOMEM;  in perf_event_alloc()
12368 		if (event->parent) {  in perf_event_alloc()
12371 			raw_spin_lock_irq(&ifh->lock);  in perf_event_alloc()
12372 			memcpy(event->addr_filter_ranges,  in perf_event_alloc()
12373 			       event->parent->addr_filter_ranges,  in perf_event_alloc()
12374 			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));  in perf_event_alloc()
12375 			raw_spin_unlock_irq(&ifh->lock);  in perf_event_alloc()
12379 		event->addr_filters_gen = 1;  in perf_event_alloc()
12382 	if (!event->parent) {  in perf_event_alloc()
12383 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {  in perf_event_alloc()
12384 			err = get_callchain_buffers(attr->sample_max_stack);  in perf_event_alloc()
12400 	if (!event->parent) {  in perf_event_alloc()
12401 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in perf_event_alloc()
12405 	kfree(event->addr_filter_ranges);  in perf_event_alloc()
12413 	if (event->destroy)  in perf_event_alloc()
12414 		event->destroy(event);  in perf_event_alloc()
12415 	module_put(pmu->module);  in perf_event_alloc()
12417 	if (event->hw.target)  in perf_event_alloc()
12418 		put_task_struct(event->hw.target);  in perf_event_alloc()
12419 	call_rcu(&event->rcu_head, free_event_rcu);  in perf_event_alloc()
12433 	ret = get_user(size, &uattr->size);  in perf_copy_attr()
12445 		if (ret == -E2BIG)  in perf_copy_attr()
12450 	attr->size = size;  in perf_copy_attr()
12452 	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)  in perf_copy_attr()
12453 		return -EINVAL;  in perf_copy_attr()
12455 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))  in perf_copy_attr()
12456 		return -EINVAL;  in perf_copy_attr()
12458 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))  in perf_copy_attr()
12459 		return -EINVAL;  in perf_copy_attr()
12461 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {  in perf_copy_attr()
12462 		u64 mask = attr->branch_sample_type;  in perf_copy_attr()
12465 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))  in perf_copy_attr()
12466 			return -EINVAL;  in perf_copy_attr()
12470 			return -EINVAL;  in perf_copy_attr()
12476 			if (!attr->exclude_kernel)  in perf_copy_attr()
12479 			if (!attr->exclude_user)  in perf_copy_attr()
12482 			if (!attr->exclude_hv)  in perf_copy_attr()
12487 			attr->branch_sample_type = mask;  in perf_copy_attr()
12497 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {  in perf_copy_attr()
12498 		ret = perf_reg_validate(attr->sample_regs_user);  in perf_copy_attr()
12503 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {  in perf_copy_attr()
12505 			return -ENOSYS;  in perf_copy_attr()
12512 		if (attr->sample_stack_user >= USHRT_MAX)  in perf_copy_attr()
12513 			return -EINVAL;  in perf_copy_attr()
12514 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))  in perf_copy_attr()
12515 			return -EINVAL;  in perf_copy_attr()
12518 	if (!attr->sample_max_stack)  in perf_copy_attr()
12519 		attr->sample_max_stack = sysctl_perf_event_max_stack;  in perf_copy_attr()
12521 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)  in perf_copy_attr()
12522 		ret = perf_reg_validate(attr->sample_regs_intr);  in perf_copy_attr()
12525 	if (attr->sample_type & PERF_SAMPLE_CGROUP)  in perf_copy_attr()
12526 		return -EINVAL;  in perf_copy_attr()
12528 	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&  in perf_copy_attr()
12529 	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))  in perf_copy_attr()
12530 		return -EINVAL;  in perf_copy_attr()
12532 	if (!attr->inherit && attr->inherit_thread)  in perf_copy_attr()
12533 		return -EINVAL;  in perf_copy_attr()
12535 	if (attr->remove_on_exec && attr->enable_on_exec)  in perf_copy_attr()
12536 		return -EINVAL;  in perf_copy_attr()
12538 	if (attr->sigtrap && !attr->remove_on_exec)  in perf_copy_attr()
12539 		return -EINVAL;  in perf_copy_attr()
12545 	put_user(sizeof(*attr), &uattr->size);  in perf_copy_attr()
12546 	ret = -E2BIG;  in perf_copy_attr()
12563 	int ret = -EINVAL;  in perf_event_set_output()
12566 		mutex_lock(&event->mmap_mutex);  in perf_event_set_output()
12575 	 * Don't allow cross-cpu buffers  in perf_event_set_output()
12577 	if (output_event->cpu != event->cpu)  in perf_event_set_output()
12581 	 * If its not a per-cpu rb, it must be the same task.  in perf_event_set_output()
12583 	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)  in perf_event_set_output()
12589 	if (output_event->clock != event->clock)  in perf_event_set_output()
12603 	    event->pmu != output_event->pmu)  in perf_event_set_output()
12608 	 * output_event is already on rb->event_list, and the list iteration  in perf_event_set_output()
12611 	 * observe !rb->mmap_count.  in perf_event_set_output()
12613 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);  in perf_event_set_output()
12616 	if (atomic_read(&event->mmap_count))  in perf_event_set_output()
12626 		if (!atomic_read(&rb->mmap_count)) {  in perf_event_set_output()
12636 	mutex_unlock(&event->mmap_mutex);  in perf_event_set_output()
12638 		mutex_unlock(&output_event->mmap_mutex);  in perf_event_set_output()
12650 		event->clock = &ktime_get_mono_fast_ns;  in perf_event_set_clock()
12655 		event->clock = &ktime_get_raw_fast_ns;  in perf_event_set_clock()
12660 		event->clock = &ktime_get_real_ns;  in perf_event_set_clock()
12664 		event->clock = &ktime_get_boottime_ns;  in perf_event_set_clock()
12668 		event->clock = &ktime_get_clocktai_ns;  in perf_event_set_clock()
12672 		return -EINVAL;  in perf_event_set_clock()
12675 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))  in perf_event_set_clock()
12676 		return -EINVAL;  in perf_event_set_clock()
12687 	if (attr->sigtrap) {  in perf_check_permission()
12693 		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);  in perf_check_permission()
12713  * sys_perf_event_open - open a performance event, associate it to a task/cpu
12715  * @attr_uptr:	event_id type attributes for monitoring/sampling
12737 	int cgroup_fd = -1;  in SYSCALL_DEFINE5()
12741 		return -EINVAL;  in SYSCALL_DEFINE5()
12760 			return -EACCES;  in SYSCALL_DEFINE5()
12765 			return -EINVAL;  in SYSCALL_DEFINE5()
12768 			return -EINVAL;  in SYSCALL_DEFINE5()
12791 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))  in SYSCALL_DEFINE5()
12792 		return -EINVAL;  in SYSCALL_DEFINE5()
12801 	CLASS(fd, group)(group_fd);     // group_fd == -1 => empty  in SYSCALL_DEFINE5()
12802 	if (group_fd != -1) {  in SYSCALL_DEFINE5()
12804 			err = -EBADF;  in SYSCALL_DEFINE5()
12807 		group_leader = fd_file(group)->private_data;  in SYSCALL_DEFINE5()
12814 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {  in SYSCALL_DEFINE5()
12823 	    group_leader->attr.inherit != attr.inherit) {  in SYSCALL_DEFINE5()
12824 		err = -EINVAL;  in SYSCALL_DEFINE5()
12839 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {  in SYSCALL_DEFINE5()
12840 			err = -EOPNOTSUPP;  in SYSCALL_DEFINE5()
12849 	pmu = event->pmu;  in SYSCALL_DEFINE5()
12857 	if (pmu->task_ctx_nr == perf_sw_context)  in SYSCALL_DEFINE5()
12858 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in SYSCALL_DEFINE5()
12861 		err = down_read_interruptible(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
12871 		err = -EACCES;  in SYSCALL_DEFINE5()
12885 	mutex_lock(&ctx->mutex);  in SYSCALL_DEFINE5()
12887 	if (ctx->task == TASK_TOMBSTONE) {  in SYSCALL_DEFINE5()
12888 		err = -ESRCH;  in SYSCALL_DEFINE5()
12899 		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in SYSCALL_DEFINE5()
12901 		if (!cpuctx->online) {  in SYSCALL_DEFINE5()
12902 			err = -ENODEV;  in SYSCALL_DEFINE5()
12908 		err = -EINVAL;  in SYSCALL_DEFINE5()
12912 		 * becoming part of another group-sibling):  in SYSCALL_DEFINE5()
12914 		if (group_leader->group_leader != group_leader)  in SYSCALL_DEFINE5()
12918 		if (group_leader->clock != event->clock)  in SYSCALL_DEFINE5()
12926 		if (group_leader->cpu != event->cpu)  in SYSCALL_DEFINE5()
12932 		if (group_leader->ctx != ctx)  in SYSCALL_DEFINE5()
12954 			pmu = group_leader->pmu_ctx->pmu;  in SYSCALL_DEFINE5()
12957 			    (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {  in SYSCALL_DEFINE5()
12968 			    group_leader->pmu_ctx->pmu != pmu)  in SYSCALL_DEFINE5()
12981 	event->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
12990 		err = -E2BIG;  in SYSCALL_DEFINE5()
12995 		err = -EINVAL;  in SYSCALL_DEFINE5()
13004 		err = -EBUSY;  in SYSCALL_DEFINE5()
13008 	WARN_ON_ONCE(ctx->parent_ctx);  in SYSCALL_DEFINE5()
13024 		put_pmu_ctx(group_leader->pmu_ctx);  in SYSCALL_DEFINE5()
13028 			put_pmu_ctx(sibling->pmu_ctx);  in SYSCALL_DEFINE5()
13035 		 * (through the sibling list, which is still in-tact), we can  in SYSCALL_DEFINE5()
13038 		 * By installing siblings first we NO-OP because they're not  in SYSCALL_DEFINE5()
13042 			sibling->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
13045 			perf_install_in_context(ctx, sibling, sibling->cpu);  in SYSCALL_DEFINE5()
13053 		group_leader->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
13056 		perf_install_in_context(ctx, group_leader, group_leader->cpu);  in SYSCALL_DEFINE5()
13068 	event->owner = current;  in SYSCALL_DEFINE5()
13070 	perf_install_in_context(ctx, event, event->cpu);  in SYSCALL_DEFINE5()
13073 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13076 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13080 	mutex_lock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13081 	list_add_tail(&event->owner_entry, &current->perf_event_list);  in SYSCALL_DEFINE5()
13082 	mutex_unlock(&current->perf_event_mutex);  in SYSCALL_DEFINE5()
13094 	put_pmu_ctx(event->pmu_ctx);  in SYSCALL_DEFINE5()
13095 	event->pmu_ctx = NULL; /* _free_event() */  in SYSCALL_DEFINE5()
13097 	mutex_unlock(&ctx->mutex);  in SYSCALL_DEFINE5()
13102 		up_read(&task->signal->exec_update_lock);  in SYSCALL_DEFINE5()
13138 	if (attr->aux_output || attr->aux_action)  in perf_event_create_kernel_counter()
13139 		return ERR_PTR(-EINVAL);  in perf_event_create_kernel_counter()
13142 				 overflow_handler, context, -1);  in perf_event_create_kernel_counter()
13149 	event->owner = TASK_TOMBSTONE;  in perf_event_create_kernel_counter()
13150 	pmu = event->pmu;  in perf_event_create_kernel_counter()
13152 	if (pmu->task_ctx_nr == perf_sw_context)  in perf_event_create_kernel_counter()
13153 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in perf_event_create_kernel_counter()
13164 	WARN_ON_ONCE(ctx->parent_ctx);  in perf_event_create_kernel_counter()
13165 	mutex_lock(&ctx->mutex);  in perf_event_create_kernel_counter()
13166 	if (ctx->task == TASK_TOMBSTONE) {  in perf_event_create_kernel_counter()
13167 		err = -ESRCH;  in perf_event_create_kernel_counter()
13176 	event->pmu_ctx = pmu_ctx;  in perf_event_create_kernel_counter()
13187 		if (!cpuctx->online) {  in perf_event_create_kernel_counter()
13188 			err = -ENODEV;  in perf_event_create_kernel_counter()
13194 		err = -EBUSY;  in perf_event_create_kernel_counter()
13198 	perf_install_in_context(ctx, event, event->cpu);  in perf_event_create_kernel_counter()
13200 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13206 	event->pmu_ctx = NULL; /* _free_event() */  in perf_event_create_kernel_counter()
13208 	mutex_unlock(&ctx->mutex);  in perf_event_create_kernel_counter()
13227 		put_pmu_ctx(event->pmu_ctx);  in __perf_pmu_remove()
13228 		list_add(&event->migrate_entry, events);  in __perf_pmu_remove()
13232 			put_pmu_ctx(sibling->pmu_ctx);  in __perf_pmu_remove()
13233 			list_add(&sibling->migrate_entry, events);  in __perf_pmu_remove()
13243 	struct perf_event_context *old_ctx = event->ctx;  in __perf_pmu_install_event()
13247 	event->cpu = cpu;  in __perf_pmu_install_event()
13249 	event->pmu_ctx = epc;  in __perf_pmu_install_event()
13251 	if (event->state >= PERF_EVENT_STATE_OFF)  in __perf_pmu_install_event()
13252 		event->state = PERF_EVENT_STATE_INACTIVE;  in __perf_pmu_install_event()
13256 	 * Now that event->ctx is updated and visible, put the old ctx.  in __perf_pmu_install_event()
13267 	 * Re-instate events in 2 passes.  in __perf_pmu_install()
13275 		if (event->group_leader == event)  in __perf_pmu_install()
13278 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13287 		list_del(&event->migrate_entry);  in __perf_pmu_install()
13298 	 * Since per-cpu context is persistent, no need to grab an extra  in perf_pmu_migrate_context()
13301 	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;  in perf_pmu_migrate_context()
13302 	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;  in perf_pmu_migrate_context()
13308 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);  in perf_pmu_migrate_context()
13310 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events);  in perf_pmu_migrate_context()
13311 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events);  in perf_pmu_migrate_context()
13315 		 * Wait for the events to quiesce before re-instating them.  in perf_pmu_migrate_context()
13322 	mutex_unlock(&dst_ctx->mutex);  in perf_pmu_migrate_context()
13323 	mutex_unlock(&src_ctx->mutex);  in perf_pmu_migrate_context()
13329 	struct perf_event *parent_event = child_event->parent;  in sync_child_event()
13332 	if (child_event->attr.inherit_stat) {  in sync_child_event()
13333 		struct task_struct *task = child_event->ctx->task;  in sync_child_event()
13344 	atomic64_add(child_val, &parent_event->child_count);  in sync_child_event()
13345 	atomic64_add(child_event->total_time_enabled,  in sync_child_event()
13346 		     &parent_event->child_total_time_enabled);  in sync_child_event()
13347 	atomic64_add(child_event->total_time_running,  in sync_child_event()
13348 		     &parent_event->child_total_time_running);  in sync_child_event()
13354 	struct perf_event *parent_event = event->parent;  in perf_event_exit_event()
13371 		mutex_lock(&parent_event->child_mutex);  in perf_event_exit_event()
13376 	raw_spin_lock_irq(&ctx->lock);  in perf_event_exit_event()
13377 	if (event->state > PERF_EVENT_STATE_EXIT)  in perf_event_exit_event()
13379 	raw_spin_unlock_irq(&ctx->lock);  in perf_event_exit_event()
13385 		mutex_unlock(&parent_event->child_mutex);  in perf_event_exit_event()
13413 	 * In order to reduce the amount of tricky in ctx tear-down, we hold  in perf_event_exit_task_context()
13422 	mutex_lock(&child_ctx->mutex);  in perf_event_exit_task_context()
13425 	 * In a single ctx::lock section, de-schedule the events and detach the  in perf_event_exit_task_context()
13429 	raw_spin_lock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13433 	 * Now that the context is inactive, destroy the task <-> ctx relation  in perf_event_exit_task_context()
13436 	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);  in perf_event_exit_task_context()
13438 	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);  in perf_event_exit_task_context()
13442 	raw_spin_unlock_irq(&child_ctx->lock);  in perf_event_exit_task_context()
13454 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)  in perf_event_exit_task_context()
13457 	mutex_unlock(&child_ctx->mutex);  in perf_event_exit_task_context()
13472 	mutex_lock(&child->perf_event_mutex);  in perf_event_exit_task()
13473 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,  in perf_event_exit_task()
13475 		list_del_init(&event->owner_entry);  in perf_event_exit_task()
13480 		 * we need to serialize on the owner->perf_event_mutex.  in perf_event_exit_task()
13482 		smp_store_release(&event->owner, NULL);  in perf_event_exit_task()
13484 	mutex_unlock(&child->perf_event_mutex);  in perf_event_exit_task()
13491 	 * child contexts and sets child->perf_event_ctxp[] to NULL.  in perf_event_exit_task()
13500 	struct perf_event *parent = event->parent;  in perf_free_event()
13505 	mutex_lock(&parent->child_mutex);  in perf_free_event()
13506 	list_del_init(&event->child_list);  in perf_free_event()
13507 	mutex_unlock(&parent->child_mutex);  in perf_free_event()
13511 	raw_spin_lock_irq(&ctx->lock);  in perf_free_event()
13514 	raw_spin_unlock_irq(&ctx->lock);  in perf_free_event()
13530 	ctx = rcu_access_pointer(task->perf_event_ctxp);  in perf_event_free_task()
13534 	mutex_lock(&ctx->mutex);  in perf_event_free_task()
13535 	raw_spin_lock_irq(&ctx->lock);  in perf_event_free_task()
13537 	 * Destroy the task <-> ctx relation and mark the context dead.  in perf_event_free_task()
13542 	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);  in perf_event_free_task()
13543 	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);  in perf_event_free_task()
13545 	raw_spin_unlock_irq(&ctx->lock);  in perf_event_free_task()
13548 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)  in perf_event_free_task()
13551 	mutex_unlock(&ctx->mutex);  in perf_event_free_task()
13556 	 * case we must wait for these events to have been freed (in  in perf_event_free_task()
13562 	 * _free_event()'s put_task_struct(event->hw.target) will be a  in perf_event_free_task()
13563 	 * use-after-free.  in perf_event_free_task()
13565 	 * Wait for all events to drop their context reference.  in perf_event_free_task()
13567 	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);  in perf_event_free_task()
13573 	WARN_ON_ONCE(task->perf_event_ctxp);  in perf_event_delayed_put()
13580 		return ERR_PTR(-EBADF);  in perf_event_get()
13582 	if (file->f_op != &perf_fops) {  in perf_event_get()
13584 		return ERR_PTR(-EBADF);  in perf_event_get()
13592 	if (file->f_op != &perf_fops)  in perf_get_event()
13593 		return ERR_PTR(-EINVAL);  in perf_get_event()
13595 	return file->private_data;  in perf_get_event()
13601 		return ERR_PTR(-EINVAL);  in perf_event_attrs()
13603 	return &event->attr;  in perf_event_attrs()
13609 		return -EACCES;  in perf_allow_kernel()
13619  *  - valid pointer on success
13620  *  - NULL for orphaned events
13621  *  - IS_ERR() on error
13631 	enum perf_event_state parent_state = parent_event->state;  in inherit_event()
13642 	if (parent_event->parent)  in inherit_event()
13643 		parent_event = parent_event->parent;  in inherit_event()
13645 	child_event = perf_event_alloc(&parent_event->attr,  in inherit_event()
13646 					   parent_event->cpu,  in inherit_event()
13649 					   NULL, NULL, -1);  in inherit_event()
13653 	pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event);  in inherit_event()
13658 	child_event->pmu_ctx = pmu_ctx;  in inherit_event()
13661 	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)  in inherit_event()
13666 	mutex_lock(&parent_event->child_mutex);  in inherit_event()
13668 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {  in inherit_event()
13669 		mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13683 		child_event->state = PERF_EVENT_STATE_INACTIVE;  in inherit_event()
13685 		child_event->state = PERF_EVENT_STATE_OFF;  in inherit_event()
13687 	if (parent_event->attr.freq) {  in inherit_event()
13688 		u64 sample_period = parent_event->hw.sample_period;  in inherit_event()
13689 		struct hw_perf_event *hwc = &child_event->hw;  in inherit_event()
13691 		hwc->sample_period = sample_period;  in inherit_event()
13692 		hwc->last_period   = sample_period;  in inherit_event()
13694 		local64_set(&hwc->period_left, sample_period);  in inherit_event()
13697 	child_event->ctx = child_ctx;  in inherit_event()
13698 	child_event->overflow_handler = parent_event->overflow_handler;  in inherit_event()
13699 	child_event->overflow_handler_context  in inherit_event()
13700 		= parent_event->overflow_handler_context;  in inherit_event()
13711 	raw_spin_lock_irqsave(&child_ctx->lock, flags);  in inherit_event()
13713 	child_event->attach_state |= PERF_ATTACH_CHILD;  in inherit_event()
13714 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);  in inherit_event()
13719 	list_add_tail(&child_event->child_list, &parent_event->child_list);  in inherit_event()
13720 	mutex_unlock(&parent_event->child_mutex);  in inherit_event()
13732  *  - 0 on success
13733  *  - <0 on error
13760 		if (sub->aux_event == parent_event && child_ctr &&  in inherit_group()
13762 			return -EINVAL;  in inherit_group()
13765 		leader->group_generation = parent_event->group_generation;  in inherit_group()
13770  * Creates the child task context and tries to inherit the event-group.
13777  *  - 0 on success
13778  *  - <0 on error
13789 	if (!event->attr.inherit ||  in inherit_task_group()
13790 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||  in inherit_task_group()
13792 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {  in inherit_task_group()
13797 	child_ctx = child->perf_event_ctxp;  in inherit_task_group()
13807 			return -ENOMEM;  in inherit_task_group()
13809 		child->perf_event_ctxp = child_ctx;  in inherit_task_group()
13832 	if (likely(!parent->perf_event_ctxp))  in perf_event_init_context()
13845 	 * it non-NULL earlier, the only reason for it to become NULL  in perf_event_init_context()
13851 	 * Lock the parent list. No need to lock the child - not PID  in perf_event_init_context()
13854 	mutex_lock(&parent_ctx->mutex);  in perf_event_init_context()
13857 	 * We dont have to disable NMIs - we are only looking at  in perf_event_init_context()
13860 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {  in perf_event_init_context()
13868 	 * We can't hold ctx->lock when iterating the ->flexible_group list due  in perf_event_init_context()
13872 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13873 	parent_ctx->rotate_disable = 1;  in perf_event_init_context()
13874 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13876 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {  in perf_event_init_context()
13883 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);  in perf_event_init_context()
13884 	parent_ctx->rotate_disable = 0;  in perf_event_init_context()
13886 	child_ctx = child->perf_event_ctxp;  in perf_event_init_context()
13894 		 * parent_ctx->lock avoids it from being uncloned.  in perf_event_init_context()
13896 		cloned_ctx = parent_ctx->parent_ctx;  in perf_event_init_context()
13898 			child_ctx->parent_ctx = cloned_ctx;  in perf_event_init_context()
13899 			child_ctx->parent_gen = parent_ctx->parent_gen;  in perf_event_init_context()
13901 			child_ctx->parent_ctx = parent_ctx;  in perf_event_init_context()
13902 			child_ctx->parent_gen = parent_ctx->generation;  in perf_event_init_context()
13904 		get_ctx(child_ctx->parent_ctx);  in perf_event_init_context()
13907 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);  in perf_event_init_context()
13909 	mutex_unlock(&parent_ctx->mutex);  in perf_event_init_context()
13924 	memset(child->perf_recursion, 0, sizeof(child->perf_recursion));  in perf_event_init_task()
13925 	child->perf_event_ctxp = NULL;  in perf_event_init_task()
13926 	mutex_init(&child->perf_event_mutex);  in perf_event_init_task()
13927 	INIT_LIST_HEAD(&child->perf_event_list);  in perf_event_init_task()
13954 		mutex_init(&swhash->hlist_mutex);  in perf_event_init_all_cpus()
13962 		__perf_event_init_context(&cpuctx->ctx);  in perf_event_init_all_cpus()
13963 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_event_init_all_cpus()
13964 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_event_init_all_cpus()
13965 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);  in perf_event_init_all_cpus()
13966 		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);  in perf_event_init_all_cpus()
13967 		cpuctx->heap = cpuctx->heap_default;  in perf_event_init_all_cpus()
13975 	mutex_lock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
13976 	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {  in perf_swevent_init_cpu()
13981 		rcu_assign_pointer(swhash->swevent_hlist, hlist);  in perf_swevent_init_cpu()
13983 	mutex_unlock(&swhash->hlist_mutex);  in perf_swevent_init_cpu()
13993 	raw_spin_lock(&ctx->lock);  in __perf_event_exit_context()
13995 	list_for_each_entry(event, &ctx->event_list, event_entry)  in __perf_event_exit_context()
13997 	raw_spin_unlock(&ctx->lock);  in __perf_event_exit_context()
14012 		target[scope] = -1;  in perf_event_clear_cpumask()
14025 		if (pmu->scope == PERF_PMU_SCOPE_NONE ||  in perf_event_clear_cpumask()
14026 		    WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE))  in perf_event_clear_cpumask()
14029 		if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids)  in perf_event_clear_cpumask()
14030 			perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]);  in perf_event_clear_cpumask()
14039 	// XXX simplify cpuctx->online  in perf_event_exit_cpu_context()
14047 	ctx = &cpuctx->ctx;  in perf_event_exit_cpu_context()
14049 	mutex_lock(&ctx->mutex);  in perf_event_exit_cpu_context()
14051 	cpuctx->online = 0;  in perf_event_exit_cpu_context()
14052 	mutex_unlock(&ctx->mutex);  in perf_event_exit_cpu_context()
14107 	ctx = &cpuctx->ctx;  in perf_event_init_cpu()
14109 	mutex_lock(&ctx->mutex);  in perf_event_init_cpu()
14110 	cpuctx->online = 1;  in perf_event_init_cpu()
14111 	mutex_unlock(&ctx->mutex);  in perf_event_init_cpu()
14152 	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);  in perf_event_init()
14153 	perf_pmu_register(&perf_task_clock, "task_clock", -1);  in perf_event_init()
14177 	if (pmu_attr->event_str)  in perf_event_sysfs_show()
14178 		return sprintf(page, "%s\n", pmu_attr->event_str);  in perf_event_sysfs_show()
14196 		if (pmu->dev)  in perf_event_sysfs_init()
14200 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);  in perf_event_sysfs_init()
14220 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14222 	jc->info = alloc_percpu(struct perf_cgroup_info);  in perf_cgroup_css_alloc()
14223 	if (!jc->info) {  in perf_cgroup_css_alloc()
14225 		return ERR_PTR(-ENOMEM);  in perf_cgroup_css_alloc()
14228 	return &jc->css;  in perf_cgroup_css_alloc()
14235 	free_percpu(jc->info);  in perf_cgroup_css_free()
14241 	perf_event_cgroup(css->cgroup);  in perf_cgroup_css_online()