Lines Matching +full:bias +full:- +full:bus +full:- +full:hold
1 // SPDX-License-Identifier: GPL-2.0
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
58 #include <linux/percpu-rwsem.h>
76 struct task_struct *p = tfc->p; in remote_function()
79 /* -EAGAIN */ in remote_function()
88 tfc->ret = -ESRCH; /* No such (running) process */ in remote_function()
93 tfc->ret = tfc->func(tfc->info); in remote_function()
97 * task_function_call - call a function on the cpu on which a task runs
107 * returns @func return value or -ESRCH or -ENXIO when the process isn't running
116 .ret = -EAGAIN, in task_function_call()
126 if (ret != -EAGAIN) in task_function_call()
136 * cpu_function_call - call a function on the cpu
143 * returns: @func return value or -ENXIO when the cpu is offline
151 .ret = -ENXIO, /* No such CPU */ in cpu_function_call()
175 raw_spin_lock(&ctx->lock); in __perf_ctx_lock()
176 WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN); in __perf_ctx_lock()
182 __perf_ctx_lock(&cpuctx->ctx); in perf_ctx_lock()
193 if (ctx->is_active & EVENT_FROZEN) { in __perf_ctx_unlock()
194 if (!(ctx->is_active & EVENT_ALL)) in __perf_ctx_unlock()
195 ctx->is_active = 0; in __perf_ctx_unlock()
197 ctx->is_active &= ~EVENT_FROZEN; in __perf_ctx_unlock()
199 raw_spin_unlock(&ctx->lock); in __perf_ctx_unlock()
207 __perf_ctx_unlock(&cpuctx->ctx); in perf_ctx_unlock()
216 { perf_ctx_unlock(_T->cpuctx, _T->ctx); } in class_perf_ctx_lock_destructor()
223 #define TASK_TOMBSTONE ((void *)-1L)
227 return READ_ONCE(event->owner) == TASK_TOMBSTONE; in is_kernel_event()
235 return this_cpu_ptr(&perf_cpu_context)->task_ctx; in perf_cpu_task_ctx()
241 * When !ctx->nr_events a task context will not be scheduled. This means
247 * - removing the last event from a task ctx; this is relatively straight
250 * - adding the first event to a task ctx; this is tricky because we cannot
251 * rely on ctx->is_active and therefore cannot use event_function_call().
254 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
269 struct perf_event *event = efs->event; in event_function()
270 struct perf_event_context *ctx = event->ctx; in event_function()
272 struct perf_event_context *task_ctx = cpuctx->task_ctx; in event_function()
279 * Since we do the IPI call without holding ctx->lock things can have in event_function()
282 if (ctx->task) { in event_function()
283 if (ctx->task != current) { in event_function()
284 ret = -ESRCH; in event_function()
292 * above ctx->task != current test), therefore we must have in event_function()
293 * ctx->is_active here. in event_function()
295 WARN_ON_ONCE(!ctx->is_active); in event_function()
297 * And since we have ctx->is_active, cpuctx->task_ctx must in event_function()
302 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function()
305 efs->func(event, cpuctx, ctx, efs->data); in event_function()
314 struct perf_event_context *ctx = event->ctx; in event_function_call()
315 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ in event_function_call()
323 if (!event->parent) { in event_function_call()
325 * If this is a !child event, we must hold ctx::mutex to in event_function_call()
326 * stabilize the event->ctx relation. See in event_function_call()
329 lockdep_assert_held(&ctx->mutex); in event_function_call()
333 cpu_function_call(event->cpu, event_function, &efs); in event_function_call()
351 task = ctx->task; in event_function_call()
354 if (ctx->is_active) { in event_function_call()
371 struct perf_event_context *ctx = event->ctx; in event_function_local()
373 struct task_struct *task = READ_ONCE(ctx->task); in event_function_local()
387 task = ctx->task; in event_function_local()
397 if (ctx->is_active) { in event_function_local()
401 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) in event_function_local()
405 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function_local()
462 * -1 - not paranoid at all
463 * 0 - disallow raw tracepoint access for unpriv
464 * 1 - disallow cpu events for unpriv
465 * 2 - disallow kernel profiling for unpriv
511 return -EINVAL; in perf_event_max_sample_rate_handler()
546 * User-space relies on this file as a feature check for
625 running_len -= running_len/NR_ACCUMULATED_SAMPLES; in perf_sample_event_took()
679 return event->clock(); in perf_event_clock()
685 * The basic idea is to use event->state to determine which (if any) time
707 struct perf_event *leader = event->group_leader; in __perf_effective_state()
709 if (leader->state <= PERF_EVENT_STATE_OFF) in __perf_effective_state()
710 return leader->state; in __perf_effective_state()
712 return event->state; in __perf_effective_state()
719 u64 delta = now - event->tstamp; in __perf_update_times()
721 *enabled = event->total_time_enabled; in __perf_update_times()
725 *running = event->total_time_running; in __perf_update_times()
734 __perf_update_times(event, now, &event->total_time_enabled, in perf_event_update_time()
735 &event->total_time_running); in perf_event_update_time()
736 event->tstamp = now; in perf_event_update_time()
750 if (event->state == state) in perf_event_set_state()
758 if ((event->state < 0) ^ (state < 0)) in perf_event_set_state()
761 WRITE_ONCE(event->state, state); in perf_event_set_state()
765 * UP store-release, load-acquire
782 list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
783 if (_cgroup && !_epc->nr_cgroups) \
785 else if (_pmu && _epc->pmu != _pmu) \
794 perf_pmu_disable(pmu_ctx->pmu); in perf_ctx_disable()
802 perf_pmu_enable(pmu_ctx->pmu); in perf_ctx_enable()
816 if (!event->cgrp) in perf_cgroup_match()
820 if (!cpuctx->cgrp) in perf_cgroup_match()
829 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, in perf_cgroup_match()
830 event->cgrp->css.cgroup); in perf_cgroup_match()
835 css_put(&event->cgrp->css); in perf_detach_cgroup()
836 event->cgrp = NULL; in perf_detach_cgroup()
841 return event->cgrp != NULL; in is_cgroup_event()
848 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time()
849 return t->time; in perf_cgroup_event_time()
856 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time_now()
857 if (!__load_acquire(&t->active)) in perf_cgroup_event_time_now()
858 return t->time; in perf_cgroup_event_time_now()
859 now += READ_ONCE(t->timeoffset); in perf_cgroup_event_time_now()
866 info->time += now - info->timestamp; in __update_cgrp_time()
867 info->timestamp = now; in __update_cgrp_time()
871 WRITE_ONCE(info->timeoffset, info->time - info->timestamp); in __update_cgrp_time()
876 struct perf_cgroup *cgrp = cpuctx->cgrp; in update_cgrp_time_from_cpuctx()
883 for (css = &cgrp->css; css; css = css->parent) { in update_cgrp_time_from_cpuctx()
885 info = this_cpu_ptr(cgrp->info); in update_cgrp_time_from_cpuctx()
889 __store_release(&info->active, 0); in update_cgrp_time_from_cpuctx()
905 info = this_cpu_ptr(event->cgrp->info); in update_cgrp_time_from_event()
909 if (info->active) in update_cgrp_time_from_event()
916 struct perf_event_context *ctx = &cpuctx->ctx; in perf_cgroup_set_timestamp()
917 struct perf_cgroup *cgrp = cpuctx->cgrp; in perf_cgroup_set_timestamp()
922 * ctx->lock held by caller in perf_cgroup_set_timestamp()
929 WARN_ON_ONCE(!ctx->nr_cgroups); in perf_cgroup_set_timestamp()
931 for (css = &cgrp->css; css; css = css->parent) { in perf_cgroup_set_timestamp()
933 info = this_cpu_ptr(cgrp->info); in perf_cgroup_set_timestamp()
934 __update_cgrp_time(info, ctx->timestamp, false); in perf_cgroup_set_timestamp()
935 __store_release(&info->active, 1); in perf_cgroup_set_timestamp()
948 * cpuctx->cgrp is set when the first cgroup event enabled, in perf_cgroup_switch()
951 if (READ_ONCE(cpuctx->cgrp) == NULL) in perf_cgroup_switch()
955 if (READ_ONCE(cpuctx->cgrp) == cgrp) in perf_cgroup_switch()
958 guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
960 * Re-check, could've raced vs perf_remove_from_context(). in perf_cgroup_switch()
962 if (READ_ONCE(cpuctx->cgrp) == NULL) in perf_cgroup_switch()
965 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); in perf_cgroup_switch()
967 perf_ctx_disable(&cpuctx->ctx, true); in perf_cgroup_switch()
969 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
975 cpuctx->cgrp = cgrp; in perf_cgroup_switch()
981 ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
983 perf_ctx_enable(&cpuctx->ctx, true); in perf_cgroup_switch()
997 for (heap_size = 1; css; css = css->parent) in perf_cgroup_ensure_storage()
1002 if (heap_size <= cpuctx->heap_size) in perf_cgroup_ensure_storage()
1008 ret = -ENOMEM; in perf_cgroup_ensure_storage()
1012 raw_spin_lock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
1013 if (cpuctx->heap_size < heap_size) { in perf_cgroup_ensure_storage()
1014 swap(cpuctx->heap, storage); in perf_cgroup_ensure_storage()
1015 if (storage == cpuctx->heap_default) in perf_cgroup_ensure_storage()
1017 cpuctx->heap_size = heap_size; in perf_cgroup_ensure_storage()
1019 raw_spin_unlock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
1037 return -EBADF; in perf_cgroup_connect()
1039 css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry, in perf_cgroup_connect()
1049 event->cgrp = cgrp; in perf_cgroup_connect()
1056 if (group_leader && group_leader->cgrp != cgrp) { in perf_cgroup_connect()
1058 ret = -EINVAL; in perf_cgroup_connect()
1071 event->pmu_ctx->nr_cgroups++; in perf_cgroup_event_enable()
1074 * Because cgroup events are always per-cpu events, in perf_cgroup_event_enable()
1075 * @ctx == &cpuctx->ctx. in perf_cgroup_event_enable()
1079 if (ctx->nr_cgroups++) in perf_cgroup_event_enable()
1082 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); in perf_cgroup_event_enable()
1093 event->pmu_ctx->nr_cgroups--; in perf_cgroup_event_disable()
1096 * Because cgroup events are always per-cpu events, in perf_cgroup_event_disable()
1097 * @ctx == &cpuctx->ctx. in perf_cgroup_event_disable()
1101 if (--ctx->nr_cgroups) in perf_cgroup_event_disable()
1104 cpuctx->cgrp = NULL; in perf_cgroup_event_disable()
1136 return -EINVAL; in perf_cgroup_connect()
1187 raw_spin_lock(&cpc->hrtimer_lock); in perf_mux_hrtimer_handler()
1189 hrtimer_forward_now(hr, cpc->hrtimer_interval); in perf_mux_hrtimer_handler()
1191 cpc->hrtimer_active = 0; in perf_mux_hrtimer_handler()
1192 raw_spin_unlock(&cpc->hrtimer_lock); in perf_mux_hrtimer_handler()
1199 struct hrtimer *timer = &cpc->hrtimer; in __perf_mux_hrtimer_init()
1200 struct pmu *pmu = cpc->epc.pmu; in __perf_mux_hrtimer_init()
1207 interval = pmu->hrtimer_interval_ms; in __perf_mux_hrtimer_init()
1209 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; in __perf_mux_hrtimer_init()
1211 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); in __perf_mux_hrtimer_init()
1213 raw_spin_lock_init(&cpc->hrtimer_lock); in __perf_mux_hrtimer_init()
1220 struct hrtimer *timer = &cpc->hrtimer; in perf_mux_hrtimer_restart()
1223 raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1224 if (!cpc->hrtimer_active) { in perf_mux_hrtimer_restart()
1225 cpc->hrtimer_active = 1; in perf_mux_hrtimer_restart()
1226 hrtimer_forward_now(timer, cpc->hrtimer_interval); in perf_mux_hrtimer_restart()
1229 raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1241 return *this_cpu_ptr(pmu->cpu_pmu_context); in this_cpc()
1246 int *count = &this_cpc(pmu)->pmu_disable_count; in perf_pmu_disable()
1248 pmu->pmu_disable(pmu); in perf_pmu_disable()
1253 int *count = &this_cpc(pmu)->pmu_disable_count; in perf_pmu_enable()
1254 if (!--(*count)) in perf_pmu_enable()
1255 pmu->pmu_enable(pmu); in perf_pmu_enable()
1260 int *count = &this_cpc(pmu)->pmu_disable_count; in perf_assert_pmu_disabled()
1266 if (event->state == PERF_EVENT_STATE_ACTIVE) in perf_pmu_read()
1267 event->pmu->read(event); in perf_pmu_read()
1272 refcount_inc(&ctx->refcount); in get_ctx()
1285 if (refcount_dec_and_test(&ctx->refcount)) { in put_ctx()
1286 if (ctx->parent_ctx) in put_ctx()
1287 put_ctx(ctx->parent_ctx); in put_ctx()
1288 if (ctx->task && ctx->task != TASK_TOMBSTONE) in put_ctx()
1289 put_task_struct(ctx->task); in put_ctx()
1290 call_rcu(&ctx->rcu_head, free_ctx); in put_ctx()
1293 if (ctx->task == TASK_TOMBSTONE) in put_ctx()
1294 wake_up_var(&ctx->refcount); in put_ctx()
1302 * Those places that change perf_event::ctx will hold both
1308 * - perf_event_exit_task_context() [ child , 0 ]
1312 * - perf_event_init_context() [ parent, 0 ]
1320 * While it appears there is an obvious deadlock here -- the parent and child
1322 * life-time rules separate them. That is an exiting task cannot fork, and a
1325 * But remember that these are parent<->child context relations, and
1331 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1346 * However; because event->ctx can change while we're waiting to acquire
1347 * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1363 * cpuctx->mutex / perf_event_context::mutex
1372 ctx = READ_ONCE(event->ctx); in perf_event_ctx_lock_nested()
1373 if (!refcount_inc_not_zero(&ctx->refcount)) { in perf_event_ctx_lock_nested()
1379 mutex_lock_nested(&ctx->mutex, nesting); in perf_event_ctx_lock_nested()
1380 if (event->ctx != ctx) { in perf_event_ctx_lock_nested()
1381 mutex_unlock(&ctx->mutex); in perf_event_ctx_lock_nested()
1398 mutex_unlock(&ctx->mutex); in perf_event_ctx_unlock()
1403 * This must be done under the ctx->lock, such as to serialize against
1405 * calling scheduler related locks and ctx->lock nests inside those.
1410 struct perf_event_context *parent_ctx = ctx->parent_ctx; in unclone_ctx()
1412 lockdep_assert_held(&ctx->lock); in unclone_ctx()
1415 ctx->parent_ctx = NULL; in unclone_ctx()
1416 ctx->generation++; in unclone_ctx()
1428 if (event->parent) in perf_event_pid_type()
1429 event = event->parent; in perf_event_pid_type()
1431 nr = __task_pid_nr_ns(p, type, event->ns); in perf_event_pid_type()
1432 /* avoid -1 if it is idle thread or runs in another ns */ in perf_event_pid_type()
1434 nr = -1; in perf_event_pid_type()
1454 u64 id = event->id; in primary_event_id()
1456 if (event->parent) in primary_event_id()
1457 id = event->parent->id; in primary_event_id()
1477 * part of the read side critical section was irqs-enabled -- see in perf_lock_task_context()
1480 * Since ctx->lock nests under rq->lock we must ensure the entire read in perf_lock_task_context()
1485 ctx = rcu_dereference(task->perf_event_ctxp); in perf_lock_task_context()
1497 raw_spin_lock(&ctx->lock); in perf_lock_task_context()
1498 if (ctx != rcu_dereference(task->perf_event_ctxp)) { in perf_lock_task_context()
1499 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1505 if (ctx->task == TASK_TOMBSTONE || in perf_lock_task_context()
1506 !refcount_inc_not_zero(&ctx->refcount)) { in perf_lock_task_context()
1507 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1510 WARN_ON_ONCE(ctx->task != task); in perf_lock_task_context()
1532 ++ctx->pin_count; in perf_pin_task_context()
1533 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_pin_task_context()
1542 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_unpin_context()
1543 --ctx->pin_count; in perf_unpin_context()
1544 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_unpin_context()
1554 lockdep_assert_held(&ctx->lock); in __update_context_time()
1557 ctx->time += now - ctx->timestamp; in __update_context_time()
1558 ctx->timestamp = now; in __update_context_time()
1561 * The above: time' = time + (now - timestamp), can be re-arranged in __update_context_time()
1562 * into: time` = now + (time - timestamp), which gives a single value in __update_context_time()
1566 * it's (obviously) not possible to acquire ctx->lock in order to read in __update_context_time()
1569 WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp); in __update_context_time()
1579 struct perf_event_context *ctx = event->ctx; in perf_event_time()
1587 return ctx->time; in perf_event_time()
1592 struct perf_event_context *ctx = event->ctx; in perf_event_time_now()
1600 if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) in perf_event_time_now()
1601 return ctx->time; in perf_event_time_now()
1603 now += READ_ONCE(ctx->timeoffset); in perf_event_time_now()
1609 struct perf_event_context *ctx = event->ctx; in get_event_type()
1612 lockdep_assert_held(&ctx->lock); in get_event_type()
1618 if (event->group_leader != event) in get_event_type()
1619 event = event->group_leader; in get_event_type()
1621 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; in get_event_type()
1622 if (!ctx->task) in get_event_type()
1633 RB_CLEAR_NODE(&event->group_node); in init_event_group()
1634 event->group_index = 0; in init_event_group()
1644 if (event->attr.pinned) in get_event_groups()
1645 return &ctx->pinned_groups; in get_event_groups()
1647 return &ctx->flexible_groups; in get_event_groups()
1655 groups->tree = RB_ROOT; in perf_event_groups_init()
1656 groups->index = 0; in perf_event_groups_init()
1664 if (event->cgrp) in event_cgroup()
1665 cgroup = event->cgrp->css.cgroup; in event_cgroup()
1682 if (left_cpu < right->cpu) in perf_event_groups_cmp()
1683 return -1; in perf_event_groups_cmp()
1684 if (left_cpu > right->cpu) in perf_event_groups_cmp()
1688 if (left_pmu < right->pmu_ctx->pmu) in perf_event_groups_cmp()
1689 return -1; in perf_event_groups_cmp()
1690 if (left_pmu > right->pmu_ctx->pmu) in perf_event_groups_cmp()
1704 return -1; in perf_event_groups_cmp()
1715 return -1; in perf_event_groups_cmp()
1722 if (left_group_index < right->group_index) in perf_event_groups_cmp()
1723 return -1; in perf_event_groups_cmp()
1724 if (left_group_index > right->group_index) in perf_event_groups_cmp()
1736 return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e), in __group_less()
1737 e->group_index, __node_2_pe(b)) < 0; in __group_less()
1752 return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b); in __group_cmp()
1762 return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b), in __group_cmp_ignore_cgroup()
1763 b->group_index, b); in __group_cmp_ignore_cgroup()
1768 * {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1775 event->group_index = ++groups->index; in perf_event_groups_insert()
1777 rb_add(&event->group_node, &groups->tree, __group_less); in perf_event_groups_insert()
1799 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || in perf_event_groups_delete()
1800 RB_EMPTY_ROOT(&groups->tree)); in perf_event_groups_delete()
1802 rb_erase(&event->group_node, &groups->tree); in perf_event_groups_delete()
1832 node = rb_find_first(&key, &groups->tree, __group_cmp); in perf_event_groups_first()
1843 .cpu = event->cpu, in perf_event_groups_next()
1849 next = rb_next_match(&key, &event->group_node, __group_cmp); in perf_event_groups_next()
1864 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1866 event = rb_entry_safe(rb_next(&event->group_node), \
1874 return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ); in has_inherit_and_sample_read()
1879 * Must be called with ctx->mutex and ctx->lock held.
1884 lockdep_assert_held(&ctx->lock); in list_add_event()
1886 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); in list_add_event()
1887 event->attach_state |= PERF_ATTACH_CONTEXT; in list_add_event()
1889 event->tstamp = perf_event_time(event); in list_add_event()
1896 if (event->group_leader == event) { in list_add_event()
1897 event->group_caps = event->event_caps; in list_add_event()
1901 list_add_rcu(&event->event_entry, &ctx->event_list); in list_add_event()
1902 ctx->nr_events++; in list_add_event()
1903 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_add_event()
1904 ctx->nr_user++; in list_add_event()
1905 if (event->attr.inherit_stat) in list_add_event()
1906 ctx->nr_stat++; in list_add_event()
1907 if (has_inherit_and_sample_read(&event->attr)) in list_add_event()
1908 local_inc(&ctx->nr_no_switch_fast); in list_add_event()
1910 if (event->state > PERF_EVENT_STATE_OFF) in list_add_event()
1913 ctx->generation++; in list_add_event()
1914 event->pmu_ctx->nr_events++; in list_add_event()
1922 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : in perf_event__state_init()
1962 size += sizeof(data->ip); in __perf_event_header_size()
1965 size += sizeof(data->addr); in __perf_event_header_size()
1968 size += sizeof(data->period); in __perf_event_header_size()
1971 size += sizeof(data->weight.full); in __perf_event_header_size()
1974 size += event->read_size; in __perf_event_header_size()
1977 size += sizeof(data->data_src.val); in __perf_event_header_size()
1980 size += sizeof(data->txn); in __perf_event_header_size()
1983 size += sizeof(data->phys_addr); in __perf_event_header_size()
1986 size += sizeof(data->cgroup); in __perf_event_header_size()
1989 size += sizeof(data->data_page_size); in __perf_event_header_size()
1992 size += sizeof(data->code_page_size); in __perf_event_header_size()
1994 event->header_size = size; in __perf_event_header_size()
2003 event->read_size = in perf_event__header_size()
2004 __perf_event_read_size(event->attr.read_format, in perf_event__header_size()
2005 event->group_leader->nr_siblings); in perf_event__header_size()
2006 __perf_event_header_size(event, event->attr.sample_type); in perf_event__header_size()
2012 u64 sample_type = event->attr.sample_type; in perf_event__id_header_size()
2016 size += sizeof(data->tid_entry); in perf_event__id_header_size()
2019 size += sizeof(data->time); in perf_event__id_header_size()
2022 size += sizeof(data->id); in perf_event__id_header_size()
2025 size += sizeof(data->id); in perf_event__id_header_size()
2028 size += sizeof(data->stream_id); in perf_event__id_header_size()
2031 size += sizeof(data->cpu_entry); in perf_event__id_header_size()
2033 event->id_header_size = size; in perf_event__id_header_size()
2042 * depends on per-event read_format, also (re)check the existing events.
2049 struct perf_event *sibling, *group_leader = event->group_leader; in perf_event_validate_size()
2051 if (__perf_event_read_size(event->attr.read_format, in perf_event_validate_size()
2052 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
2055 if (__perf_event_read_size(group_leader->attr.read_format, in perf_event_validate_size()
2056 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
2060 * When creating a new group leader, group_leader->ctx is initialized in perf_event_validate_size()
2062 * for_each_sibling_event() until group_leader->ctx is set. A new group in perf_event_validate_size()
2064 * the non-existent siblings. in perf_event_validate_size()
2070 if (__perf_event_read_size(sibling->attr.read_format, in perf_event_validate_size()
2071 group_leader->nr_siblings + 1) > 16*1024) in perf_event_validate_size()
2080 struct perf_event *group_leader = event->group_leader, *pos; in perf_group_attach()
2082 lockdep_assert_held(&event->ctx->lock); in perf_group_attach()
2088 if (event->attach_state & PERF_ATTACH_GROUP) in perf_group_attach()
2091 event->attach_state |= PERF_ATTACH_GROUP; in perf_group_attach()
2096 WARN_ON_ONCE(group_leader->ctx != event->ctx); in perf_group_attach()
2098 group_leader->group_caps &= event->event_caps; in perf_group_attach()
2100 list_add_tail(&event->sibling_list, &group_leader->sibling_list); in perf_group_attach()
2101 group_leader->nr_siblings++; in perf_group_attach()
2102 group_leader->group_generation++; in perf_group_attach()
2112 * Must be called with ctx->mutex and ctx->lock held.
2117 WARN_ON_ONCE(event->ctx != ctx); in list_del_event()
2118 lockdep_assert_held(&ctx->lock); in list_del_event()
2121 * We can have double detach due to exit/hot-unplug + close. in list_del_event()
2123 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) in list_del_event()
2126 event->attach_state &= ~PERF_ATTACH_CONTEXT; in list_del_event()
2128 ctx->nr_events--; in list_del_event()
2129 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_del_event()
2130 ctx->nr_user--; in list_del_event()
2131 if (event->attr.inherit_stat) in list_del_event()
2132 ctx->nr_stat--; in list_del_event()
2133 if (has_inherit_and_sample_read(&event->attr)) in list_del_event()
2134 local_dec(&ctx->nr_no_switch_fast); in list_del_event()
2136 list_del_rcu(&event->event_entry); in list_del_event()
2138 if (event->group_leader == event) in list_del_event()
2141 ctx->generation++; in list_del_event()
2142 event->pmu_ctx->nr_events--; in list_del_event()
2151 if (!event->pmu->aux_output_match) in perf_aux_output_match()
2154 return event->pmu->aux_output_match(aux_event); in perf_aux_output_match()
2164 struct perf_event_context *ctx = event->ctx; in perf_put_aux_event()
2170 if (event->aux_event) { in perf_put_aux_event()
2171 iter = event->aux_event; in perf_put_aux_event()
2172 event->aux_event = NULL; in perf_put_aux_event()
2182 if (iter->aux_event != event) in perf_put_aux_event()
2185 iter->aux_event = NULL; in perf_put_aux_event()
2199 return event->attr.aux_output || has_aux_action(event); in perf_need_aux_event()
2217 if (event->attr.aux_output && event->attr.aux_sample_size) in perf_get_aux_event()
2220 if (event->attr.aux_output && in perf_get_aux_event()
2224 if ((event->attr.aux_pause || event->attr.aux_resume) && in perf_get_aux_event()
2225 !(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) in perf_get_aux_event()
2228 if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux) in perf_get_aux_event()
2231 if (!atomic_long_inc_not_zero(&group_leader->refcount)) in perf_get_aux_event()
2240 event->aux_event = group_leader; in perf_get_aux_event()
2247 return event->attr.pinned ? &event->pmu_ctx->pinned_active : in get_event_list()
2248 &event->pmu_ctx->flexible_active; in get_event_list()
2253 struct perf_event *leader = event->group_leader; in perf_group_detach()
2255 struct perf_event_context *ctx = event->ctx; in perf_group_detach()
2257 lockdep_assert_held(&ctx->lock); in perf_group_detach()
2260 * We can have double detach due to exit/hot-unplug + close. in perf_group_detach()
2262 if (!(event->attach_state & PERF_ATTACH_GROUP)) in perf_group_detach()
2265 event->attach_state &= ~PERF_ATTACH_GROUP; in perf_group_detach()
2273 list_del_init(&event->sibling_list); in perf_group_detach()
2274 event->group_leader->nr_siblings--; in perf_group_detach()
2275 event->group_leader->group_generation++; in perf_group_detach()
2284 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { in perf_group_detach()
2293 if (sibling->event_caps & PERF_EV_CAP_SIBLING) in perf_group_detach()
2296 sibling->group_leader = sibling; in perf_group_detach()
2297 list_del_init(&sibling->sibling_list); in perf_group_detach()
2300 sibling->group_caps = event->group_caps; in perf_group_detach()
2302 if (sibling->attach_state & PERF_ATTACH_CONTEXT) { in perf_group_detach()
2303 add_event_to_groups(sibling, event->ctx); in perf_group_detach()
2305 if (sibling->state == PERF_EVENT_STATE_ACTIVE) in perf_group_detach()
2306 list_add_tail(&sibling->active_list, get_event_list(sibling)); in perf_group_detach()
2309 WARN_ON_ONCE(sibling->ctx != event->ctx); in perf_group_detach()
2323 struct perf_event *parent_event = event->parent; in perf_child_detach()
2325 if (!(event->attach_state & PERF_ATTACH_CHILD)) in perf_child_detach()
2328 event->attach_state &= ~PERF_ATTACH_CHILD; in perf_child_detach()
2336 lockdep_assert_held(&parent_event->child_mutex); in perf_child_detach()
2340 list_del_init(&event->child_list); in perf_child_detach()
2345 return event->state == PERF_EVENT_STATE_DEAD; in is_orphaned_event()
2351 return (event->cpu == -1 || event->cpu == smp_processor_id()) && in event_filter_match()
2357 return event->attr.freq && event->attr.sample_freq; in is_event_in_freq_mode()
2363 struct perf_event_pmu_context *epc = event->pmu_ctx; in event_sched_out()
2364 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); in event_sched_out()
2367 // XXX cpc serialization, probably per-cpu IRQ disabled in event_sched_out()
2369 WARN_ON_ONCE(event->ctx != ctx); in event_sched_out()
2370 lockdep_assert_held(&ctx->lock); in event_sched_out()
2372 if (event->state != PERF_EVENT_STATE_ACTIVE) in event_sched_out()
2380 list_del_init(&event->active_list); in event_sched_out()
2382 perf_pmu_disable(event->pmu); in event_sched_out()
2384 event->pmu->del(event, 0); in event_sched_out()
2385 event->oncpu = -1; in event_sched_out()
2387 if (event->pending_disable) { in event_sched_out()
2388 event->pending_disable = 0; in event_sched_out()
2396 cpc->active_oncpu--; in event_sched_out()
2398 ctx->nr_freq--; in event_sched_out()
2399 epc->nr_freq--; in event_sched_out()
2401 if (event->attr.exclusive || !cpc->active_oncpu) in event_sched_out()
2402 cpc->exclusive = 0; in event_sched_out()
2404 perf_pmu_enable(event->pmu); in event_sched_out()
2412 if (group_event->state != PERF_EVENT_STATE_ACTIVE) in group_sched_out()
2415 perf_assert_pmu_disabled(group_event->pmu_ctx->pmu); in group_sched_out()
2429 if (ctx->is_active & EVENT_TIME) { in __ctx_time_update()
2430 if (ctx->is_active & EVENT_FROZEN) in __ctx_time_update()
2450 if (ctx->is_active & EVENT_TIME) in ctx_time_freeze()
2451 ctx->is_active |= EVENT_FROZEN; in ctx_time_freeze()
2457 if (ctx->is_active & EVENT_TIME) { in ctx_time_update_event()
2458 if (ctx->is_active & EVENT_FROZEN) in ctx_time_update_event()
2483 struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx; in __perf_remove_from_context()
2502 if (event->state > PERF_EVENT_STATE_OFF) in __perf_remove_from_context()
2505 perf_event_set_state(event, min(event->state, state)); in __perf_remove_from_context()
2513 if (!pmu_ctx->nr_events) { in __perf_remove_from_context()
2514 pmu_ctx->rotate_necessary = 0; in __perf_remove_from_context()
2516 if (ctx->task && ctx->is_active) { in __perf_remove_from_context()
2517 struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu); in __perf_remove_from_context()
2519 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __perf_remove_from_context()
2520 cpc->task_epc = NULL; in __perf_remove_from_context()
2524 if (!ctx->nr_events && ctx->is_active) { in __perf_remove_from_context()
2525 if (ctx == &cpuctx->ctx) in __perf_remove_from_context()
2528 ctx->is_active = 0; in __perf_remove_from_context()
2529 if (ctx->task) { in __perf_remove_from_context()
2530 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in __perf_remove_from_context()
2531 cpuctx->task_ctx = NULL; in __perf_remove_from_context()
2539 * If event->ctx is a cloned context, callers must make sure that
2540 * every task struct that event->ctx->task could possibly point to
2542 * that only calls us on the top-level context, which can't be a clone.
2548 struct perf_event_context *ctx = event->ctx; in perf_remove_from_context()
2550 lockdep_assert_held(&ctx->mutex); in perf_remove_from_context()
2557 raw_spin_lock_irq(&ctx->lock); in perf_remove_from_context()
2558 if (!ctx->is_active) { in perf_remove_from_context()
2561 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2564 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2586 if (event->state < PERF_EVENT_STATE_INACTIVE) in __perf_event_disable()
2589 perf_pmu_disable(event->pmu_ctx->pmu); in __perf_event_disable()
2596 if (event == event->group_leader) in __perf_event_disable()
2605 perf_pmu_enable(event->pmu_ctx->pmu); in __perf_event_disable()
2611 * If event->ctx is a cloned context, callers must make sure that
2612 * every task struct that event->ctx->task could possibly point to
2615 * hold the top-level event's child_mutex, so any descendant that
2618 * When called from perf_pending_disable it's OK because event->ctx
2624 struct perf_event_context *ctx = event->ctx; in _perf_event_disable()
2626 raw_spin_lock_irq(&ctx->lock); in _perf_event_disable()
2627 if (event->state <= PERF_EVENT_STATE_OFF) { in _perf_event_disable()
2628 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2631 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2657 event->pending_disable = 1; in perf_event_disable_inatomic()
2658 irq_work_queue(&event->pending_disable_irq); in perf_event_disable_inatomic()
2668 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_event_unthrottle()
2671 event->hw.interrupts = 0; in perf_event_unthrottle()
2673 event->pmu->start(event, 0); in perf_event_unthrottle()
2674 if (event == event->group_leader) in perf_event_unthrottle()
2680 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_event_throttle()
2683 event->hw.interrupts = MAX_INTERRUPTS; in perf_event_throttle()
2684 event->pmu->stop(event, 0); in perf_event_throttle()
2685 if (event == event->group_leader) in perf_event_throttle()
2691 struct perf_event *sibling, *leader = event->group_leader; in perf_event_unthrottle_group()
2700 struct perf_event *sibling, *leader = event->group_leader; in perf_event_throttle_group()
2710 struct perf_event_pmu_context *epc = event->pmu_ctx; in event_sched_in()
2711 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); in event_sched_in()
2714 WARN_ON_ONCE(event->ctx != ctx); in event_sched_in()
2716 lockdep_assert_held(&ctx->lock); in event_sched_in()
2718 if (event->state <= PERF_EVENT_STATE_OFF) in event_sched_in()
2721 WRITE_ONCE(event->oncpu, smp_processor_id()); in event_sched_in()
2725 * ->oncpu if it sees ACTIVE. in event_sched_in()
2735 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) in event_sched_in()
2738 perf_pmu_disable(event->pmu); in event_sched_in()
2742 if (event->pmu->add(event, PERF_EF_START)) { in event_sched_in()
2744 event->oncpu = -1; in event_sched_in()
2745 ret = -EAGAIN; in event_sched_in()
2750 cpc->active_oncpu++; in event_sched_in()
2752 ctx->nr_freq++; in event_sched_in()
2753 epc->nr_freq++; in event_sched_in()
2755 if (event->attr.exclusive) in event_sched_in()
2756 cpc->exclusive = 1; in event_sched_in()
2759 perf_pmu_enable(event->pmu); in event_sched_in()
2768 struct pmu *pmu = group_event->pmu_ctx->pmu; in group_sched_in()
2770 if (group_event->state == PERF_EVENT_STATE_OFF) in group_sched_in()
2773 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); in group_sched_in()
2788 if (!pmu->commit_txn(pmu)) in group_sched_in()
2806 pmu->cancel_txn(pmu); in group_sched_in()
2807 return -EAGAIN; in group_sched_in()
2815 struct perf_event_pmu_context *epc = event->pmu_ctx; in group_can_go_on()
2816 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); in group_can_go_on()
2821 if (event->group_caps & PERF_EV_CAP_SOFTWARE) in group_can_go_on()
2827 if (cpc->exclusive) in group_can_go_on()
2833 if (event->attr.exclusive && !list_empty(get_event_list(event))) in group_can_go_on()
2855 if (!cpuctx->task_ctx) in task_ctx_sched_out()
2858 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) in task_ctx_sched_out()
2868 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED); in perf_event_sched_in()
2871 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); in perf_event_sched_in()
2878 * - CPU pinned (EVENT_CPU | EVENT_PINNED)
2879 * - task pinned (EVENT_PINNED)
2880 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2881 * - task flexible (EVENT_FLEXIBLE).
2907 for_each_epc(epc, &cpuctx->ctx, pmu, false) in ctx_resched()
2908 perf_pmu_disable(epc->pmu); in ctx_resched()
2912 perf_pmu_disable(epc->pmu); in ctx_resched()
2920 * - EVENT_CPU: schedule out corresponding groups; in ctx_resched()
2921 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; in ctx_resched()
2922 * - otherwise, do nothing more. in ctx_resched()
2925 ctx_sched_out(&cpuctx->ctx, pmu, event_type); in ctx_resched()
2927 ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); in ctx_resched()
2931 for_each_epc(epc, &cpuctx->ctx, pmu, false) in ctx_resched()
2932 perf_pmu_enable(epc->pmu); in ctx_resched()
2936 perf_pmu_enable(epc->pmu); in ctx_resched()
2943 struct perf_event_context *task_ctx = cpuctx->task_ctx; in perf_pmu_resched()
2954 * things like ctx->is_active and cpuctx->task_ctx are set.
2959 struct perf_event_context *ctx = event->ctx; in __perf_install_in_context()
2961 struct perf_event_context *task_ctx = cpuctx->task_ctx; in __perf_install_in_context()
2965 raw_spin_lock(&cpuctx->ctx.lock); in __perf_install_in_context()
2966 if (ctx->task) { in __perf_install_in_context()
2967 raw_spin_lock(&ctx->lock); in __perf_install_in_context()
2970 reprogram = (ctx->task == current); in __perf_install_in_context()
2976 * If its not running, we don't care, ctx->lock will in __perf_install_in_context()
2979 if (task_curr(ctx->task) && !reprogram) { in __perf_install_in_context()
2980 ret = -ESRCH; in __perf_install_in_context()
2984 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); in __perf_install_in_context()
2986 raw_spin_lock(&task_ctx->lock); in __perf_install_in_context()
2990 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) { in __perf_install_in_context()
2996 reprogram = cgroup_is_descendant(cgrp->css.cgroup, in __perf_install_in_context()
2997 event->cgrp->css.cgroup); in __perf_install_in_context()
3004 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, in __perf_install_in_context()
3029 struct task_struct *task = READ_ONCE(ctx->task); in perf_install_in_context()
3031 lockdep_assert_held(&ctx->mutex); in perf_install_in_context()
3035 if (event->cpu != -1) in perf_install_in_context()
3036 WARN_ON_ONCE(event->cpu != cpu); in perf_install_in_context()
3039 * Ensures that if we can observe event->ctx, both the event and ctx in perf_install_in_context()
3042 smp_store_release(&event->ctx, ctx); in perf_install_in_context()
3047 * that case we need the magic of the IPI to set ctx->is_active. in perf_install_in_context()
3053 ctx->nr_events && !is_cgroup_event(event)) { in perf_install_in_context()
3054 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
3055 if (ctx->task == TASK_TOMBSTONE) { in perf_install_in_context()
3056 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
3060 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
3076 * Installing events is tricky because we cannot rely on ctx->is_active in perf_install_in_context()
3077 * to be set in case this is the nr_events 0 -> 1 transition. in perf_install_in_context()
3091 * our task->perf_event_ctxp[] store, such that it will in fact take in perf_install_in_context()
3100 * This smp_mb() orders the task->perf_event_ctxp[] store with the in perf_install_in_context()
3110 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
3111 task = ctx->task; in perf_install_in_context()
3115 * cannot happen), and we hold ctx->mutex, which serializes us in perf_install_in_context()
3118 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
3122 * If the task is not running, ctx->lock will avoid it becoming so, in perf_install_in_context()
3126 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
3130 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
3141 struct perf_event *leader = event->group_leader; in __perf_event_enable()
3144 if (event->state >= PERF_EVENT_STATE_INACTIVE || in __perf_event_enable()
3145 event->state <= PERF_EVENT_STATE_ERROR) in __perf_event_enable()
3153 if (!ctx->is_active) in __perf_event_enable()
3163 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) in __perf_event_enable()
3166 task_ctx = cpuctx->task_ctx; in __perf_event_enable()
3167 if (ctx->task) in __perf_event_enable()
3170 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event)); in __perf_event_enable()
3176 * If event->ctx is a cloned context, callers must make sure that
3177 * every task struct that event->ctx->task could possibly point to
3184 struct perf_event_context *ctx = event->ctx; in _perf_event_enable()
3186 raw_spin_lock_irq(&ctx->lock); in _perf_event_enable()
3187 if (event->state >= PERF_EVENT_STATE_INACTIVE || in _perf_event_enable()
3188 event->state < PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
3190 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3199 * been scheduled away before the cross-call arrived. in _perf_event_enable()
3201 if (event->state == PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
3205 if (event->event_caps & PERF_EV_CAP_SIBLING && in _perf_event_enable()
3206 event->group_leader == event) in _perf_event_enable()
3209 event->state = PERF_EVENT_STATE_OFF; in _perf_event_enable()
3211 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3237 struct perf_event *event = sd->event; in __perf_event_stop()
3240 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in __perf_event_stop()
3250 if (READ_ONCE(event->oncpu) != smp_processor_id()) in __perf_event_stop()
3251 return -EAGAIN; in __perf_event_stop()
3253 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_stop()
3261 * Since this is happening on an event-local CPU, no trace is lost in __perf_event_stop()
3264 if (sd->restart) in __perf_event_stop()
3265 event->pmu->start(event, 0); in __perf_event_stop()
3279 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in perf_event_stop()
3287 * inactive here (event->oncpu==-1), there's nothing more to do; in perf_event_stop()
3288 * fall through with ret==-ENXIO. in perf_event_stop()
3290 ret = cpu_function_call(READ_ONCE(event->oncpu), in perf_event_stop()
3292 } while (ret == -EAGAIN); in perf_event_stop()
3311 * pre-existing mappings, called once when new filters arrive via SET_FILTER
3326 raw_spin_lock(&ifh->lock); in perf_event_addr_filters_sync()
3327 if (event->addr_filters_gen != event->hw.addr_filters_gen) { in perf_event_addr_filters_sync()
3328 event->pmu->addr_filters_sync(event); in perf_event_addr_filters_sync()
3329 event->hw.addr_filters_gen = event->addr_filters_gen; in perf_event_addr_filters_sync()
3331 raw_spin_unlock(&ifh->lock); in perf_event_addr_filters_sync()
3340 if (event->attr.inherit || !is_sampling_event(event)) in _perf_event_refresh()
3341 return -EINVAL; in _perf_event_refresh()
3343 atomic_add(refresh, &event->event_limit); in _perf_event_refresh()
3374 if (!bp->attr.disabled) in perf_event_modify_breakpoint()
3381 * Copy event-type-independent attributes that may be modified.
3386 to->sig_data = from->sig_data; in perf_event_modify_copy_attr()
3396 if (event->attr.type != attr->type) in perf_event_modify_attr()
3397 return -EINVAL; in perf_event_modify_attr()
3399 switch (event->attr.type) { in perf_event_modify_attr()
3405 return -EOPNOTSUPP; in perf_event_modify_attr()
3408 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_modify_attr()
3410 mutex_lock(&event->child_mutex); in perf_event_modify_attr()
3412 * Event-type-independent attributes must be copied before event-type in perf_event_modify_attr()
3416 perf_event_modify_copy_attr(&event->attr, attr); in perf_event_modify_attr()
3420 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_modify_attr()
3421 perf_event_modify_copy_attr(&child->attr, attr); in perf_event_modify_attr()
3427 mutex_unlock(&event->child_mutex); in perf_event_modify_attr()
3434 struct perf_event_context *ctx = pmu_ctx->ctx; in __pmu_ctx_sched_out()
3436 struct pmu *pmu = pmu_ctx->pmu; in __pmu_ctx_sched_out()
3438 if (ctx->task && !(ctx->is_active & EVENT_ALL)) { in __pmu_ctx_sched_out()
3441 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __pmu_ctx_sched_out()
3442 cpc->task_epc = NULL; in __pmu_ctx_sched_out()
3451 &pmu_ctx->pinned_active, in __pmu_ctx_sched_out()
3458 &pmu_ctx->flexible_active, in __pmu_ctx_sched_out()
3466 pmu_ctx->rotate_necessary = 0; in __pmu_ctx_sched_out()
3477 * state, eg. ctx->is_active cleared even though most EPCs would still actually
3485 int is_active = ctx->is_active; in ctx_sched_out()
3490 lockdep_assert_held(&ctx->lock); in ctx_sched_out()
3492 if (likely(!ctx->nr_events)) { in ctx_sched_out()
3496 WARN_ON_ONCE(ctx->is_active); in ctx_sched_out()
3497 if (ctx->task) in ctx_sched_out()
3498 WARN_ON_ONCE(cpuctx->task_ctx); in ctx_sched_out()
3512 __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx); in ctx_sched_out()
3515 * CPU-release for the below ->is_active store, in ctx_sched_out()
3519 ctx->is_active &= ~event_type; in ctx_sched_out()
3521 if (!(ctx->is_active & EVENT_ALL)) { in ctx_sched_out()
3526 if (ctx->is_active & EVENT_FROZEN) in ctx_sched_out()
3527 ctx->is_active &= EVENT_TIME_FROZEN; in ctx_sched_out()
3529 ctx->is_active = 0; in ctx_sched_out()
3532 if (ctx->task) { in ctx_sched_out()
3533 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_out()
3534 if (!(ctx->is_active & EVENT_ALL)) in ctx_sched_out()
3535 cpuctx->task_ctx = NULL; in ctx_sched_out()
3538 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_out()
3555 lockdep_assert_held(&ctx1->lock); in context_equiv()
3556 lockdep_assert_held(&ctx2->lock); in context_equiv()
3559 if (ctx1->pin_count || ctx2->pin_count) in context_equiv()
3563 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) in context_equiv()
3567 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) in context_equiv()
3574 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && in context_equiv()
3575 ctx1->parent_gen == ctx2->parent_gen) in context_equiv()
3587 if (!event->attr.inherit_stat) in __perf_event_sync_stat()
3602 * In order to keep per-task stats reliable we need to flip the event in __perf_event_sync_stat()
3605 value = local64_read(&next_event->count); in __perf_event_sync_stat()
3606 value = local64_xchg(&event->count, value); in __perf_event_sync_stat()
3607 local64_set(&next_event->count, value); in __perf_event_sync_stat()
3609 swap(event->total_time_enabled, next_event->total_time_enabled); in __perf_event_sync_stat()
3610 swap(event->total_time_running, next_event->total_time_running); in __perf_event_sync_stat()
3624 if (!ctx->nr_stat) in perf_event_sync_stat()
3629 event = list_first_entry(&ctx->event_list, in perf_event_sync_stat()
3632 next_event = list_first_entry(&next_ctx->event_list, in perf_event_sync_stat()
3635 while (&event->event_entry != &ctx->event_list && in perf_event_sync_stat()
3636 &next_event->event_entry != &next_ctx->event_list) { in perf_event_sync_stat()
3651 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in perf_ctx_sched_task_cb()
3652 cpc = this_cpc(pmu_ctx->pmu); in perf_ctx_sched_task_cb()
3654 if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) in perf_ctx_sched_task_cb()
3655 pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); in perf_ctx_sched_task_cb()
3662 struct perf_event_context *ctx = task->perf_event_ctxp; in perf_event_context_sched_out()
3671 next_ctx = rcu_dereference(next->perf_event_ctxp); in perf_event_context_sched_out()
3675 parent = rcu_dereference(ctx->parent_ctx); in perf_event_context_sched_out()
3676 next_parent = rcu_dereference(next_ctx->parent_ctx); in perf_event_context_sched_out()
3687 * lock (including re-checking that neither has been in perf_event_context_sched_out()
3692 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3693 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); in perf_event_context_sched_out()
3698 /* PMIs are disabled; ctx->nr_no_switch_fast is stable. */ in perf_event_context_sched_out()
3699 if (local_read(&ctx->nr_no_switch_fast) || in perf_event_context_sched_out()
3700 local_read(&next_ctx->nr_no_switch_fast)) { in perf_event_context_sched_out()
3703 * events that rely on the ctx->task relation. in perf_event_context_sched_out()
3710 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3715 WRITE_ONCE(ctx->task, next); in perf_event_context_sched_out()
3716 WRITE_ONCE(next_ctx->task, task); in perf_event_context_sched_out()
3725 * ctx->task is immaterial since this value is in perf_event_context_sched_out()
3726 * always verified under ctx->lock which we're now in perf_event_context_sched_out()
3729 RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx); in perf_event_context_sched_out()
3730 RCU_INIT_POINTER(next->perf_event_ctxp, ctx); in perf_event_context_sched_out()
3736 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3737 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3743 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3751 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3765 if (!--cpc->sched_cb_usage) in perf_sched_cb_dec()
3766 list_del(&cpc->sched_cb_entry); in perf_sched_cb_dec()
3774 if (!cpc->sched_cb_usage++) in perf_sched_cb_inc()
3775 list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); in perf_sched_cb_inc()
3785 * This callback is relevant even to per-cpu events; for example multi event
3795 pmu = cpc->epc.pmu; in __perf_pmu_sched_task()
3798 if (WARN_ON_ONCE(!pmu->sched_task)) in __perf_pmu_sched_task()
3801 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3804 pmu->sched_task(cpc->task_epc, task, sched_in); in __perf_pmu_sched_task()
3807 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3817 /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ in perf_pmu_sched_task()
3818 if (prev == next || cpuctx->task_ctx) in perf_pmu_sched_task()
3832 * We stop each event and update the event value in event->count.
3853 * cgroup event are system-wide mode only in __perf_event_task_sched_out()
3863 return le->group_index < re->group_index; in perf_less_group_idx()
3875 struct perf_event **itrs = heap->data; in __heap_add()
3878 itrs[heap->nr] = event; in __heap_add()
3879 heap->nr++; in __heap_add()
3887 if (!pmu_ctx->ctx->task) in __link_epc()
3890 cpc = this_cpc(pmu_ctx->pmu); in __link_epc()
3891 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); in __link_epc()
3892 cpc->task_epc = pmu_ctx; in __link_epc()
3911 if (pmu->filter && pmu->filter(pmu, cpu)) in visit_groups_merge()
3914 if (!ctx->task) { in visit_groups_merge()
3917 .data = cpuctx->heap, in visit_groups_merge()
3919 .size = cpuctx->heap_size, in visit_groups_merge()
3922 lockdep_assert_held(&cpuctx->ctx.lock); in visit_groups_merge()
3925 if (cpuctx->cgrp) in visit_groups_merge()
3926 css = &cpuctx->cgrp->css; in visit_groups_merge()
3935 __heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL)); in visit_groups_merge()
3942 for (; css; css = css->parent) in visit_groups_merge()
3943 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup)); in visit_groups_merge()
3947 __link_epc((*evt)->pmu_ctx); in visit_groups_merge()
3948 perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu); in visit_groups_merge()
3969 * Because the userpage is strictly per-event (there is no concept of context,
3971 * when context time starts :-(
3977 if (likely(!refcount_read(&event->mmap_count))) in event_update_userpage()
3999 struct perf_event_context *ctx = event->ctx; in merge_sched_in()
4002 if (event->state <= PERF_EVENT_STATE_OFF) in merge_sched_in()
4010 list_add_tail(&event->active_list, get_event_list(event)); in merge_sched_in()
4013 if (event->state == PERF_EVENT_STATE_INACTIVE) { in merge_sched_in()
4015 if (event->attr.pinned) { in merge_sched_in()
4020 event->pending_kill = POLL_ERR; in merge_sched_in()
4024 struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu); in merge_sched_in()
4026 event->pmu_ctx->rotate_necessary = 1; in merge_sched_in()
4047 struct perf_event_context *ctx = pmu_ctx->ctx; in __pmu_ctx_sched_in()
4050 pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu); in __pmu_ctx_sched_in()
4052 pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu); in __pmu_ctx_sched_in()
4060 int is_active = ctx->is_active; in ctx_sched_in()
4065 lockdep_assert_held(&ctx->lock); in ctx_sched_in()
4067 if (likely(!ctx->nr_events)) in ctx_sched_in()
4075 * CPU-release for the below ->is_active store, in ctx_sched_in()
4081 ctx->is_active |= (event_type | EVENT_TIME); in ctx_sched_in()
4082 if (ctx->task) { in ctx_sched_in()
4084 cpuctx->task_ctx = ctx; in ctx_sched_in()
4086 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_in()
4089 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_in()
4113 ctx = rcu_dereference(task->perf_event_ctxp); in perf_event_context_sched_in()
4117 if (cpuctx->task_ctx == ctx) { in perf_event_context_sched_in()
4130 * We must check ctx->nr_events while holding ctx->lock, such in perf_event_context_sched_in()
4133 if (!ctx->nr_events) in perf_event_context_sched_in()
4145 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) { in perf_event_context_sched_in()
4146 perf_ctx_disable(&cpuctx->ctx, false); in perf_event_context_sched_in()
4147 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE); in perf_event_context_sched_in()
4152 perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true); in perf_event_context_sched_in()
4154 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) in perf_event_context_sched_in()
4155 perf_ctx_enable(&cpuctx->ctx, false); in perf_event_context_sched_in()
4190 u64 frequency = event->attr.sample_freq; in perf_calculate_period()
4206 * period = ------------------- in perf_calculate_period()
4219 a##_fls--; \ in perf_calculate_period()
4222 b##_fls--; \ in perf_calculate_period()
4266 struct hw_perf_event *hwc = &event->hw; in perf_adjust_period()
4272 delta = (s64)(period - hwc->sample_period); in perf_adjust_period()
4276 delta -= 7; in perf_adjust_period()
4279 sample_period = hwc->sample_period + delta; in perf_adjust_period()
4284 hwc->sample_period = sample_period; in perf_adjust_period()
4286 if (local64_read(&hwc->period_left) > 8*sample_period) { in perf_adjust_period()
4288 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_period()
4290 local64_set(&hwc->period_left, 0); in perf_adjust_period()
4293 event->pmu->start(event, PERF_EF_RELOAD); in perf_adjust_period()
4305 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_adjust_freq_unthr_events()
4308 // XXX use visit thingy to avoid the -1,cpu match in perf_adjust_freq_unthr_events()
4312 hwc = &event->hw; in perf_adjust_freq_unthr_events()
4314 if (hwc->interrupts == MAX_INTERRUPTS) in perf_adjust_freq_unthr_events()
4321 * stop the event and update event->count in perf_adjust_freq_unthr_events()
4323 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_freq_unthr_events()
4325 now = local64_read(&event->count); in perf_adjust_freq_unthr_events()
4326 delta = now - hwc->freq_count_stamp; in perf_adjust_freq_unthr_events()
4327 hwc->freq_count_stamp = now; in perf_adjust_freq_unthr_events()
4339 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); in perf_adjust_freq_unthr_events()
4346 * the rate of unthrottling as that would introduce bias.
4355 * - context have events in frequency mode (needs freq adjust) in perf_adjust_freq_unthr_context()
4356 * - there are events to unthrottle on this cpu in perf_adjust_freq_unthr_context()
4358 if (!(ctx->nr_freq || unthrottle)) in perf_adjust_freq_unthr_context()
4361 raw_spin_lock(&ctx->lock); in perf_adjust_freq_unthr_context()
4363 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { in perf_adjust_freq_unthr_context()
4364 if (!(pmu_ctx->nr_freq || unthrottle)) in perf_adjust_freq_unthr_context()
4368 if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) in perf_adjust_freq_unthr_context()
4371 perf_pmu_disable(pmu_ctx->pmu); in perf_adjust_freq_unthr_context()
4372 perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active); in perf_adjust_freq_unthr_context()
4373 perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active); in perf_adjust_freq_unthr_context()
4374 perf_pmu_enable(pmu_ctx->pmu); in perf_adjust_freq_unthr_context()
4377 raw_spin_unlock(&ctx->lock); in perf_adjust_freq_unthr_context()
4386 * Rotate the first entry last of non-pinned groups. Rotation might be in rotate_ctx()
4389 if (ctx->rotate_disable) in rotate_ctx()
4392 perf_event_groups_delete(&ctx->flexible_groups, event); in rotate_ctx()
4393 perf_event_groups_insert(&ctx->flexible_groups, event); in rotate_ctx()
4404 .pmu = pmu_ctx->pmu, in ctx_event_to_rotate()
4408 event = list_first_entry_or_null(&pmu_ctx->flexible_active, in ctx_event_to_rotate()
4414 tree = &pmu_ctx->ctx->flexible_groups.tree; in ctx_event_to_rotate()
4416 if (!pmu_ctx->ctx->task) { in ctx_event_to_rotate()
4425 key.cpu = -1; in ctx_event_to_rotate()
4442 pmu_ctx->rotate_necessary = 0; in ctx_event_to_rotate()
4460 cpu_epc = &cpc->epc; in perf_rotate_context()
4461 pmu = cpu_epc->pmu; in perf_rotate_context()
4462 task_epc = cpc->task_epc; in perf_rotate_context()
4464 cpu_rotate = cpu_epc->rotate_necessary; in perf_rotate_context()
4465 task_rotate = task_epc ? task_epc->rotate_necessary : 0; in perf_rotate_context()
4470 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4483 update_context_time(task_epc->ctx); in perf_rotate_context()
4488 update_context_time(&cpuctx->ctx); in perf_rotate_context()
4490 rotate_ctx(&cpuctx->ctx, cpu_event); in perf_rotate_context()
4495 rotate_ctx(task_epc->ctx, task_event); in perf_rotate_context()
4501 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4518 perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled); in perf_event_task_tick()
4521 ctx = rcu_dereference(current->perf_event_ctxp); in perf_event_task_tick()
4530 if (!event->attr.enable_on_exec) in event_enable_on_exec()
4533 event->attr.enable_on_exec = 0; in event_enable_on_exec()
4534 if (event->state >= PERF_EVENT_STATE_INACTIVE) in event_enable_on_exec()
4543 * Enable all of a task's events that have been marked enable-on-exec.
4556 if (WARN_ON_ONCE(current->perf_event_ctxp != ctx)) in perf_event_enable_on_exec()
4559 if (!ctx->nr_events) in perf_event_enable_on_exec()
4566 list_for_each_entry(event, &ctx->event_list, event_entry) { in perf_event_enable_on_exec()
4594 * remove-on-exec, and feeds their values back to parent events.
4603 mutex_lock(&ctx->mutex); in perf_event_remove_on_exec()
4605 if (WARN_ON_ONCE(ctx->task != current)) in perf_event_remove_on_exec()
4608 list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { in perf_event_remove_on_exec()
4609 if (!event->attr.remove_on_exec) in perf_event_remove_on_exec()
4620 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_remove_on_exec()
4623 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_remove_on_exec()
4626 mutex_unlock(&ctx->mutex); in perf_event_remove_on_exec()
4648 if (event->group_caps & PERF_EV_CAP_READ_SCOPE) { in __perf_event_read_cpu()
4649 const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu); in __perf_event_read_cpu()
4655 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { in __perf_event_read_cpu()
4672 struct perf_event *sub, *event = data->event; in __perf_event_read()
4673 struct perf_event_context *ctx = event->ctx; in __perf_event_read()
4675 struct pmu *pmu = event->pmu; in __perf_event_read()
4681 * event->count would have been updated to a recent sample in __perf_event_read()
4684 if (ctx->task && cpuctx->task_ctx != ctx) in __perf_event_read()
4687 raw_spin_lock(&ctx->lock); in __perf_event_read()
4691 if (data->group) in __perf_event_read()
4694 if (event->state != PERF_EVENT_STATE_ACTIVE) in __perf_event_read()
4697 if (!data->group) { in __perf_event_read()
4698 pmu->read(event); in __perf_event_read()
4699 data->ret = 0; in __perf_event_read()
4703 pmu->start_txn(pmu, PERF_PMU_TXN_READ); in __perf_event_read()
4705 pmu->read(event); in __perf_event_read()
4710 data->ret = pmu->commit_txn(pmu); in __perf_event_read()
4713 raw_spin_unlock(&ctx->lock); in __perf_event_read()
4719 return local64_read(&event->count); in perf_event_count()
4721 return local64_read(&event->count) + atomic64_read(&event->child_count); in perf_event_count()
4737 * NMI-safe method to read a local event, that is an event that
4739 * - either for the current task, or for this CPU
4740 * - does not have inherit set, for inherited task events
4742 * - must not have a pmu::count method
4762 if (event->attr.inherit) { in perf_event_read_local()
4763 ret = -EOPNOTSUPP; in perf_event_read_local()
4767 /* If this is a per-task event, it must be for current */ in perf_event_read_local()
4768 if ((event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4769 event->hw.target != current) { in perf_event_read_local()
4770 ret = -EINVAL; in perf_event_read_local()
4776 * a per-package event that can be read locally in perf_event_read_local()
4778 event_oncpu = __perf_event_read_cpu(event, event->oncpu); in perf_event_read_local()
4779 event_cpu = __perf_event_read_cpu(event, event->cpu); in perf_event_read_local()
4781 /* If this is a per-CPU event, it must be for this CPU */ in perf_event_read_local()
4782 if (!(event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4784 ret = -EINVAL; in perf_event_read_local()
4789 if (event->attr.pinned && event_oncpu != smp_processor_id()) { in perf_event_read_local()
4790 ret = -EBUSY; in perf_event_read_local()
4795 * If the event is currently on this CPU, its either a per-task event, in perf_event_read_local()
4797 * oncpu == -1). in perf_event_read_local()
4800 event->pmu->read(event); in perf_event_read_local()
4802 *value = local64_read(&event->count); in perf_event_read_local()
4820 enum perf_event_state state = READ_ONCE(event->state); in perf_event_read()
4832 * Orders the ->state and ->oncpu loads such that if we see in perf_event_read()
4833 * ACTIVE we must also see the right ->oncpu. in perf_event_read()
4839 event_cpu = READ_ONCE(event->oncpu); in perf_event_read()
4859 * Therefore, either way, we'll have an up-to-date event count in perf_event_read()
4867 struct perf_event_context *ctx = event->ctx; in perf_event_read()
4870 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_read()
4871 state = event->state; in perf_event_read()
4873 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4886 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4897 raw_spin_lock_init(&ctx->lock); in __perf_event_init_context()
4898 mutex_init(&ctx->mutex); in __perf_event_init_context()
4899 INIT_LIST_HEAD(&ctx->pmu_ctx_list); in __perf_event_init_context()
4900 perf_event_groups_init(&ctx->pinned_groups); in __perf_event_init_context()
4901 perf_event_groups_init(&ctx->flexible_groups); in __perf_event_init_context()
4902 INIT_LIST_HEAD(&ctx->event_list); in __perf_event_init_context()
4903 refcount_set(&ctx->refcount, 1); in __perf_event_init_context()
4909 epc->pmu = pmu; in __perf_init_event_pmu_context()
4910 INIT_LIST_HEAD(&epc->pmu_ctx_entry); in __perf_init_event_pmu_context()
4911 INIT_LIST_HEAD(&epc->pinned_active); in __perf_init_event_pmu_context()
4912 INIT_LIST_HEAD(&epc->flexible_active); in __perf_init_event_pmu_context()
4913 atomic_set(&epc->refcount, 1); in __perf_init_event_pmu_context()
4927 ctx->task = get_task_struct(task); in alloc_perf_context()
4947 return ERR_PTR(-ESRCH); in find_lively_task_by_vpid()
4969 cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in find_get_context()
4970 ctx = &cpuctx->ctx; in find_get_context()
4972 raw_spin_lock_irqsave(&ctx->lock, flags); in find_get_context()
4973 ++ctx->pin_count; in find_get_context()
4974 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4979 err = -EINVAL; in find_get_context()
4984 ++ctx->pin_count; in find_get_context()
4986 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4992 err = -ENOMEM; in find_get_context()
4997 mutex_lock(&task->perf_event_mutex); in find_get_context()
5002 if (task->flags & PF_EXITING) in find_get_context()
5003 err = -ESRCH; in find_get_context()
5004 else if (task->perf_event_ctxp) in find_get_context()
5005 err = -EAGAIN; in find_get_context()
5008 ++ctx->pin_count; in find_get_context()
5009 rcu_assign_pointer(task->perf_event_ctxp, ctx); in find_get_context()
5011 mutex_unlock(&task->perf_event_mutex); in find_get_context()
5016 if (err == -EAGAIN) in find_get_context()
5034 if (!ctx->task) { in find_get_pmu_context()
5042 cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu); in find_get_pmu_context()
5043 epc = &cpc->epc; in find_get_pmu_context()
5044 raw_spin_lock_irq(&ctx->lock); in find_get_pmu_context()
5045 if (!epc->ctx) { in find_get_pmu_context()
5049 atomic_set(&epc->refcount, 2); in find_get_pmu_context()
5050 epc->embedded = 1; in find_get_pmu_context()
5051 list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); in find_get_pmu_context()
5052 epc->ctx = ctx; in find_get_pmu_context()
5054 WARN_ON_ONCE(epc->ctx != ctx); in find_get_pmu_context()
5055 atomic_inc(&epc->refcount); in find_get_pmu_context()
5057 raw_spin_unlock_irq(&ctx->lock); in find_get_pmu_context()
5063 return ERR_PTR(-ENOMEM); in find_get_pmu_context()
5070 * lockdep_assert_held(&ctx->mutex); in find_get_pmu_context()
5072 * can't because perf_event_init_task() doesn't actually hold the in find_get_pmu_context()
5073 * child_ctx->mutex. in find_get_pmu_context()
5076 raw_spin_lock_irq(&ctx->lock); in find_get_pmu_context()
5077 list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) { in find_get_pmu_context()
5078 if (epc->pmu == pmu) { in find_get_pmu_context()
5079 WARN_ON_ONCE(epc->ctx != ctx); in find_get_pmu_context()
5080 atomic_inc(&epc->refcount); in find_get_pmu_context()
5084 if (!pos && epc->pmu->type > pmu->type) in find_get_pmu_context()
5092 list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); in find_get_pmu_context()
5094 list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev); in find_get_pmu_context()
5096 epc->ctx = ctx; in find_get_pmu_context()
5099 raw_spin_unlock_irq(&ctx->lock); in find_get_pmu_context()
5107 WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount)); in get_pmu_ctx()
5127 struct perf_event_context *ctx = epc->ctx; in put_pmu_ctx()
5133 * lockdep_assert_held(&ctx->mutex); in put_pmu_ctx()
5135 * can't because of the call-site in _free_event()/put_event() in put_pmu_ctx()
5136 * which isn't always called under ctx->mutex. in put_pmu_ctx()
5138 if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags)) in put_pmu_ctx()
5141 WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry)); in put_pmu_ctx()
5143 list_del_init(&epc->pmu_ctx_entry); in put_pmu_ctx()
5144 epc->ctx = NULL; in put_pmu_ctx()
5146 WARN_ON_ONCE(!list_empty(&epc->pinned_active)); in put_pmu_ctx()
5147 WARN_ON_ONCE(!list_empty(&epc->flexible_active)); in put_pmu_ctx()
5149 raw_spin_unlock_irqrestore(&ctx->lock, flags); in put_pmu_ctx()
5151 if (epc->embedded) { in put_pmu_ctx()
5152 call_rcu(&epc->rcu_head, free_cpc_rcu); in put_pmu_ctx()
5156 call_rcu(&epc->rcu_head, free_epc_rcu); in put_pmu_ctx()
5165 if (event->ns) in free_event_rcu()
5166 put_pid_ns(event->ns); in free_event_rcu()
5176 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in detach_sb_event()
5178 raw_spin_lock(&pel->lock); in detach_sb_event()
5179 list_del_rcu(&event->sb_list); in detach_sb_event()
5180 raw_spin_unlock(&pel->lock); in detach_sb_event()
5185 struct perf_event_attr *attr = &event->attr; in is_sb_event()
5187 if (event->parent) in is_sb_event()
5190 if (event->attach_state & PERF_ATTACH_TASK) in is_sb_event()
5193 if (attr->mmap || attr->mmap_data || attr->mmap2 || in is_sb_event()
5194 attr->comm || attr->comm_exec || in is_sb_event()
5195 attr->task || attr->ksymbol || in is_sb_event()
5196 attr->context_switch || attr->text_poke || in is_sb_event()
5197 attr->bpf_event) in is_sb_event()
5241 cd->data = kmem_cache_zalloc(ctx_cache, GFP_KERNEL); in alloc_perf_ctx_data()
5242 if (!cd->data) { in alloc_perf_ctx_data()
5247 cd->global = global; in alloc_perf_ctx_data()
5248 cd->ctx_cache = ctx_cache; in alloc_perf_ctx_data()
5249 refcount_set(&cd->refcount, 1); in alloc_perf_ctx_data()
5256 kmem_cache_free(cd->ctx_cache, cd->data); in free_perf_ctx_data()
5270 call_rcu(&cd->rcu_head, __free_perf_ctx_data_rcu); in perf_free_ctx_data_rcu()
5281 return -ENOMEM; in attach_task_ctx_data()
5284 if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) { in attach_task_ctx_data()
5298 if (refcount_inc_not_zero(&old->refcount)) { in attach_task_ctx_data()
5332 cd = rcu_dereference(p->perf_ctx_data); in attach_global_ctx_data()
5333 if (cd && !cd->global) { in attach_global_ctx_data()
5334 cd->global = 1; in attach_global_ctx_data()
5335 if (!refcount_inc_not_zero(&cd->refcount)) in attach_global_ctx_data()
5361 struct task_struct *task = event->hw.target; in attach_perf_ctx_data()
5362 struct kmem_cache *ctx_cache = event->pmu->task_ctx_cache; in attach_perf_ctx_data()
5366 return -ENOMEM; in attach_perf_ctx_data()
5375 event->attach_state |= PERF_ATTACH_GLOBAL_DATA; in attach_perf_ctx_data()
5385 cd = rcu_dereference(p->perf_ctx_data); in detach_task_ctx_data()
5386 if (!cd || !refcount_dec_and_test(&cd->refcount)) in detach_task_ctx_data()
5395 if (try_cmpxchg((struct perf_ctx_data **)&p->perf_ctx_data, &cd, NULL)) in detach_task_ctx_data()
5407 cd = rcu_dereference(p->perf_ctx_data); in __detach_global_ctx_data()
5408 if (!cd || !cd->global) in __detach_global_ctx_data()
5410 cd->global = 0; in __detach_global_ctx_data()
5437 struct task_struct *task = event->hw.target; in detach_perf_ctx_data()
5439 event->attach_state &= ~PERF_ATTACH_TASK_DATA; in detach_perf_ctx_data()
5444 if (event->attach_state & PERF_ATTACH_GLOBAL_DATA) { in detach_perf_ctx_data()
5446 event->attach_state &= ~PERF_ATTACH_GLOBAL_DATA; in detach_perf_ctx_data()
5454 if (event->parent) in unaccount_event()
5457 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in unaccount_event()
5459 if (event->attr.mmap || event->attr.mmap_data) in unaccount_event()
5461 if (event->attr.build_id) in unaccount_event()
5463 if (event->attr.comm) in unaccount_event()
5465 if (event->attr.namespaces) in unaccount_event()
5467 if (event->attr.cgroup) in unaccount_event()
5469 if (event->attr.task) in unaccount_event()
5471 if (event->attr.freq) in unaccount_event()
5473 if (event->attr.context_switch) { in unaccount_event()
5481 if (event->attr.ksymbol) in unaccount_event()
5483 if (event->attr.bpf_event) in unaccount_event()
5485 if (event->attr.text_poke) in unaccount_event()
5489 if (!atomic_add_unless(&perf_sched_count, -1, 1)) in unaccount_event()
5509 * 1) cpu-wide events in the presence of per-task events,
5510 * 2) per-task events in the presence of cpu-wide events,
5514 * _free_event()), the latter -- before the first perf_install_in_context().
5518 struct pmu *pmu = event->pmu; in exclusive_event_init()
5524 * Prevent co-existence of per-task and cpu-wide events on the in exclusive_event_init()
5527 * Negative pmu::exclusive_cnt means there are cpu-wide in exclusive_event_init()
5529 * per-task events. in exclusive_event_init()
5533 * to mean "per-task event", because unlike other attach states it in exclusive_event_init()
5536 if (event->attach_state & PERF_ATTACH_TASK) { in exclusive_event_init()
5537 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) in exclusive_event_init()
5538 return -EBUSY; in exclusive_event_init()
5540 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) in exclusive_event_init()
5541 return -EBUSY; in exclusive_event_init()
5544 event->attach_state |= PERF_ATTACH_EXCLUSIVE; in exclusive_event_init()
5551 struct pmu *pmu = event->pmu; in exclusive_event_destroy()
5554 if (event->attach_state & PERF_ATTACH_TASK) in exclusive_event_destroy()
5555 atomic_dec(&pmu->exclusive_cnt); in exclusive_event_destroy()
5557 atomic_inc(&pmu->exclusive_cnt); in exclusive_event_destroy()
5559 event->attach_state &= ~PERF_ATTACH_EXCLUSIVE; in exclusive_event_destroy()
5564 if ((e1->pmu == e2->pmu) && in exclusive_event_match()
5565 (e1->cpu == e2->cpu || in exclusive_event_match()
5566 e1->cpu == -1 || in exclusive_event_match()
5567 e2->cpu == -1)) in exclusive_event_match()
5576 struct pmu *pmu = event->pmu; in exclusive_event_installable()
5578 lockdep_assert_held(&ctx->mutex); in exclusive_event_installable()
5583 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { in exclusive_event_installable()
5596 struct pmu *pmu = event->pmu; in __free_event()
5598 if (event->attach_state & PERF_ATTACH_CALLCHAIN) in __free_event()
5601 kfree(event->addr_filter_ranges); in __free_event()
5603 if (event->attach_state & PERF_ATTACH_EXCLUSIVE) in __free_event()
5609 if (event->attach_state & PERF_ATTACH_TASK_DATA) in __free_event()
5612 if (event->destroy) in __free_event()
5613 event->destroy(event); in __free_event()
5616 * Must be after ->destroy(), due to uprobe_perf_close() using in __free_event()
5619 if (event->hw.target) in __free_event()
5620 put_task_struct(event->hw.target); in __free_event()
5622 if (event->pmu_ctx) { in __free_event()
5624 * put_pmu_ctx() needs an event->ctx reference, because of in __free_event()
5625 * epc->ctx. in __free_event()
5628 WARN_ON_ONCE(!event->ctx); in __free_event()
5629 WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx); in __free_event()
5630 put_pmu_ctx(event->pmu_ctx); in __free_event()
5637 if (event->ctx) in __free_event()
5638 put_ctx(event->ctx); in __free_event()
5641 module_put(pmu->module); in __free_event()
5642 scoped_guard (spinlock, &pmu->events_lock) { in __free_event()
5643 list_del(&event->pmu_list); in __free_event()
5648 call_rcu(&event->rcu_head, free_event_rcu); in __free_event()
5656 irq_work_sync(&event->pending_irq); in DEFINE_FREE()
5657 irq_work_sync(&event->pending_disable_irq); in DEFINE_FREE()
5663 if (event->rb) { in DEFINE_FREE()
5665 * Can happen when we close an event with re-directed output. in DEFINE_FREE()
5670 mutex_lock(&event->mmap_mutex); in DEFINE_FREE()
5672 mutex_unlock(&event->mmap_mutex); in DEFINE_FREE()
5687 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, in free_event()
5689 atomic_long_read(&event->refcount), event)) { in free_event()
5690 /* leak to avoid use-after-free */ in free_event()
5709 * owner->perf_event_mutex. in perf_remove_from_owner()
5711 owner = READ_ONCE(event->owner); in perf_remove_from_owner()
5725 * holding ctx->mutex which would be an inversion wrt. the in perf_remove_from_owner()
5729 * ctx->mutex. in perf_remove_from_owner()
5731 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); in perf_remove_from_owner()
5734 * We have to re-check the event->owner field, if it is cleared in perf_remove_from_owner()
5739 if (event->owner) { in perf_remove_from_owner()
5740 list_del_init(&event->owner_entry); in perf_remove_from_owner()
5741 smp_store_release(&event->owner, NULL); in perf_remove_from_owner()
5743 mutex_unlock(&owner->perf_event_mutex); in perf_remove_from_owner()
5752 if (!atomic_long_dec_and_test(&event->refcount)) in put_event()
5755 parent = event->parent; in put_event()
5770 struct perf_event_context *ctx = event->ctx; in perf_event_release_kernel()
5778 WARN_ON_ONCE(event->attach_state & in perf_event_release_kernel()
5787 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_release_kernel()
5793 * Anybody acquiring event->child_mutex after the below loop _must_ in perf_event_release_kernel()
5800 if (event->state > PERF_EVENT_STATE_REVOKED) { in perf_event_release_kernel()
5803 event->state = PERF_EVENT_STATE_DEAD; in perf_event_release_kernel()
5809 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5810 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_release_kernel()
5815 ctx = READ_ONCE(child->ctx); in perf_event_release_kernel()
5820 * Since the event cannot get freed while we hold the in perf_event_release_kernel()
5829 * can re-acquire child_mutex. in perf_event_release_kernel()
5831 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5832 mutex_lock(&ctx->mutex); in perf_event_release_kernel()
5833 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5836 * Now that we hold ctx::mutex and child_mutex, revalidate our in perf_event_release_kernel()
5840 tmp = list_first_entry_or_null(&event->child_list, in perf_event_release_kernel()
5848 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5849 mutex_unlock(&ctx->mutex); in perf_event_release_kernel()
5852 /* Last reference unless ->pending_task work is pending */ in perf_event_release_kernel()
5859 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5863 * Last reference unless ->pending_task work is pending on this event in perf_event_release_kernel()
5876 perf_event_release_kernel(file->private_data); in perf_release()
5888 mutex_lock(&event->child_mutex); in __perf_event_read_value()
5893 *enabled += event->total_time_enabled + in __perf_event_read_value()
5894 atomic64_read(&event->child_total_time_enabled); in __perf_event_read_value()
5895 *running += event->total_time_running + in __perf_event_read_value()
5896 atomic64_read(&event->child_total_time_running); in __perf_event_read_value()
5898 list_for_each_entry(child, &event->child_list, child_list) { in __perf_event_read_value()
5901 *enabled += child->total_time_enabled; in __perf_event_read_value()
5902 *running += child->total_time_running; in __perf_event_read_value()
5904 mutex_unlock(&event->child_mutex); in __perf_event_read_value()
5925 struct perf_event_context *ctx = leader->ctx; in __perf_read_group_add()
5935 raw_spin_lock_irqsave(&ctx->lock, flags); in __perf_read_group_add()
5941 * - leader->ctx->lock pins leader->sibling_list in __perf_read_group_add()
5942 * - parent->child_mutex pins parent->child_list in __perf_read_group_add()
5943 * - parent->ctx->mutex pins parent->sibling_list in __perf_read_group_add()
5945 * Because parent->ctx != leader->ctx (and child_list nests inside in __perf_read_group_add()
5946 * ctx->mutex), group destruction is not atomic between children, also in __perf_read_group_add()
5956 parent = leader->parent; in __perf_read_group_add()
5958 (parent->group_generation != leader->group_generation || in __perf_read_group_add()
5959 parent->nr_siblings != leader->nr_siblings)) { in __perf_read_group_add()
5960 ret = -ECHILD; in __perf_read_group_add()
5965 * Since we co-schedule groups, {enabled,running} times of siblings in __perf_read_group_add()
5970 values[n++] += leader->total_time_enabled + in __perf_read_group_add()
5971 atomic64_read(&leader->child_total_time_enabled); in __perf_read_group_add()
5975 values[n++] += leader->total_time_running + in __perf_read_group_add()
5976 atomic64_read(&leader->child_total_time_running); in __perf_read_group_add()
5986 values[n++] = atomic64_read(&leader->lost_samples); in __perf_read_group_add()
5993 values[n++] = atomic64_read(&sub->lost_samples); in __perf_read_group_add()
5997 raw_spin_unlock_irqrestore(&ctx->lock, flags); in __perf_read_group_add()
6004 struct perf_event *leader = event->group_leader, *child; in perf_read_group()
6005 struct perf_event_context *ctx = leader->ctx; in perf_read_group()
6009 lockdep_assert_held(&ctx->mutex); in perf_read_group()
6011 values = kzalloc(event->read_size, GFP_KERNEL); in perf_read_group()
6013 return -ENOMEM; in perf_read_group()
6015 values[0] = 1 + leader->nr_siblings; in perf_read_group()
6017 mutex_lock(&leader->child_mutex); in perf_read_group()
6023 list_for_each_entry(child, &leader->child_list, child_list) { in perf_read_group()
6029 mutex_unlock(&leader->child_mutex); in perf_read_group()
6031 ret = event->read_size; in perf_read_group()
6032 if (copy_to_user(buf, values, event->read_size)) in perf_read_group()
6033 ret = -EFAULT; in perf_read_group()
6037 mutex_unlock(&leader->child_mutex); in perf_read_group()
6058 values[n++] = atomic64_read(&event->lost_samples); in perf_read_one()
6061 return -EFAULT; in perf_read_one()
6070 if (event->state > PERF_EVENT_STATE_EXIT) in is_event_hup()
6073 mutex_lock(&event->child_mutex); in is_event_hup()
6074 no_children = list_empty(&event->child_list); in is_event_hup()
6075 mutex_unlock(&event->child_mutex); in is_event_hup()
6080 * Read the performance event - simple non blocking version for now
6085 u64 read_format = event->attr.read_format; in __perf_read()
6089 * Return end-of-file for a read on an event that is in in __perf_read()
6093 if (event->state == PERF_EVENT_STATE_ERROR) in __perf_read()
6096 if (count < event->read_size) in __perf_read()
6097 return -ENOSPC; in __perf_read()
6099 WARN_ON_ONCE(event->ctx->parent_ctx); in __perf_read()
6111 struct perf_event *event = file->private_data; in perf_read()
6128 struct perf_event *event = file->private_data; in perf_poll()
6132 if (event->state <= PERF_EVENT_STATE_REVOKED) in perf_poll()
6135 poll_wait(file, &event->waitq, wait); in perf_poll()
6137 if (event->state <= PERF_EVENT_STATE_REVOKED) in perf_poll()
6143 if (unlikely(READ_ONCE(event->state) == PERF_EVENT_STATE_ERROR && in perf_poll()
6144 event->attr.pinned)) in perf_poll()
6148 * Pin the event->rb by taking event->mmap_mutex; otherwise in perf_poll()
6151 mutex_lock(&event->mmap_mutex); in perf_poll()
6152 rb = event->rb; in perf_poll()
6154 events = atomic_xchg(&rb->poll, 0); in perf_poll()
6155 mutex_unlock(&event->mmap_mutex); in perf_poll()
6162 local64_set(&event->count, 0); in _perf_event_reset()
6173 WARN_ON_ONCE(event->attr.inherit); in perf_event_pause()
6175 count = local64_read(&event->count); in perf_event_pause()
6177 local64_set(&event->count, 0); in perf_event_pause()
6185 * Holding the top-level event's child_mutex means that any
6195 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_for_each_child()
6197 mutex_lock(&event->child_mutex); in perf_event_for_each_child()
6199 list_for_each_entry(child, &event->child_list, child_list) in perf_event_for_each_child()
6201 mutex_unlock(&event->child_mutex); in perf_event_for_each_child()
6207 struct perf_event_context *ctx = event->ctx; in perf_event_for_each()
6210 lockdep_assert_held(&ctx->mutex); in perf_event_for_each()
6212 event = event->group_leader; in perf_event_for_each()
6227 if (event->attr.freq) { in __perf_event_period()
6228 event->attr.sample_freq = value; in __perf_event_period()
6230 event->attr.sample_period = value; in __perf_event_period()
6231 event->hw.sample_period = value; in __perf_event_period()
6234 active = (event->state == PERF_EVENT_STATE_ACTIVE); in __perf_event_period()
6236 perf_pmu_disable(event->pmu); in __perf_event_period()
6237 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_period()
6240 local64_set(&event->hw.period_left, 0); in __perf_event_period()
6243 event->pmu->start(event, PERF_EF_RELOAD); in __perf_event_period()
6245 * Once the period is force-reset, the event starts immediately. in __perf_event_period()
6248 * while we already re-started the event/group. in __perf_event_period()
6250 if (event->hw.interrupts == MAX_INTERRUPTS) in __perf_event_period()
6252 perf_pmu_enable(event->pmu); in __perf_event_period()
6258 return event->pmu->check_period(event, value); in perf_event_check_period()
6264 return -EINVAL; in _perf_event_period()
6267 return -EINVAL; in _perf_event_period()
6269 if (event->attr.freq) { in _perf_event_period()
6271 return -EINVAL; in _perf_event_period()
6274 return -EINVAL; in _perf_event_period()
6276 return -EINVAL; in _perf_event_period()
6301 return !fd_empty(f) && fd_file(f)->f_op == &perf_fops; in is_perf_file()
6318 if (event->state <= PERF_EVENT_STATE_REVOKED) in _perf_ioctl()
6319 return -ENODEV; in _perf_ioctl()
6340 return -EFAULT; in _perf_ioctl()
6349 return -EFAULT; in _perf_ioctl()
6355 CLASS(fd, output)(arg); // arg == -1 => empty in _perf_ioctl()
6357 if (arg != -1) { in _perf_ioctl()
6359 return -EBADF; in _perf_ioctl()
6360 output_event = fd_file(output)->private_data; in _perf_ioctl()
6390 rb = rcu_dereference(event->rb); in _perf_ioctl()
6391 if (!rb || !rb->nr_pages) { in _perf_ioctl()
6393 return -EINVAL; in _perf_ioctl()
6414 return -ENOTTY; in _perf_ioctl()
6427 struct perf_event *event = file->private_data; in perf_ioctl()
6452 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ in perf_compat_ioctl()
6470 mutex_lock(¤t->perf_event_mutex); in perf_event_task_enable()
6471 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_enable()
6476 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_enable()
6486 mutex_lock(¤t->perf_event_mutex); in perf_event_task_disable()
6487 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_disable()
6492 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_disable()
6499 if (event->hw.state & PERF_HES_STOPPED) in perf_event_index()
6502 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_event_index()
6505 return event->pmu->event_idx(event); in perf_event_index()
6514 rb = rcu_dereference(event->rb); in perf_event_init_userpage()
6518 userpg = rb->user_page; in perf_event_init_userpage()
6521 userpg->cap_bit0_is_deprecated = 1; in perf_event_init_userpage()
6522 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); in perf_event_init_userpage()
6523 userpg->data_offset = PAGE_SIZE; in perf_event_init_userpage()
6524 userpg->data_size = perf_data_size(rb); in perf_event_init_userpage()
6547 rb = rcu_dereference(event->rb); in perf_event_update_userpage()
6562 userpg = rb->user_page; in perf_event_update_userpage()
6568 ++userpg->lock; in perf_event_update_userpage()
6570 userpg->index = perf_event_index(event); in perf_event_update_userpage()
6571 userpg->offset = perf_event_count(event, false); in perf_event_update_userpage()
6572 if (userpg->index) in perf_event_update_userpage()
6573 userpg->offset -= local64_read(&event->hw.prev_count); in perf_event_update_userpage()
6575 userpg->time_enabled = enabled + in perf_event_update_userpage()
6576 atomic64_read(&event->child_total_time_enabled); in perf_event_update_userpage()
6578 userpg->time_running = running + in perf_event_update_userpage()
6579 atomic64_read(&event->child_total_time_running); in perf_event_update_userpage()
6584 ++userpg->lock; in perf_event_update_userpage()
6597 WARN_ON_ONCE(event->parent); in ring_buffer_attach()
6599 if (event->rb) { in ring_buffer_attach()
6602 * event->rb_entry and wait/clear when adding event->rb_entry. in ring_buffer_attach()
6604 WARN_ON_ONCE(event->rcu_pending); in ring_buffer_attach()
6606 old_rb = event->rb; in ring_buffer_attach()
6607 spin_lock_irqsave(&old_rb->event_lock, flags); in ring_buffer_attach()
6608 list_del_rcu(&event->rb_entry); in ring_buffer_attach()
6609 spin_unlock_irqrestore(&old_rb->event_lock, flags); in ring_buffer_attach()
6611 event->rcu_batches = get_state_synchronize_rcu(); in ring_buffer_attach()
6612 event->rcu_pending = 1; in ring_buffer_attach()
6616 if (event->rcu_pending) { in ring_buffer_attach()
6617 cond_synchronize_rcu(event->rcu_batches); in ring_buffer_attach()
6618 event->rcu_pending = 0; in ring_buffer_attach()
6621 spin_lock_irqsave(&rb->event_lock, flags); in ring_buffer_attach()
6622 list_add_rcu(&event->rb_entry, &rb->event_list); in ring_buffer_attach()
6623 spin_unlock_irqrestore(&rb->event_lock, flags); in ring_buffer_attach()
6633 * mid-air, but then again, whoever does it like this is in ring_buffer_attach()
6639 rcu_assign_pointer(event->rb, rb); in ring_buffer_attach()
6648 wake_up_all(&event->waitq); in ring_buffer_attach()
6656 if (event->parent) in ring_buffer_wakeup()
6657 event = event->parent; in ring_buffer_wakeup()
6660 rb = rcu_dereference(event->rb); in ring_buffer_wakeup()
6662 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) in ring_buffer_wakeup()
6663 wake_up_all(&event->waitq); in ring_buffer_wakeup()
6672 if (event->parent) in ring_buffer_get()
6673 event = event->parent; in ring_buffer_get()
6676 rb = rcu_dereference(event->rb); in ring_buffer_get()
6678 if (!refcount_inc_not_zero(&rb->refcount)) in ring_buffer_get()
6688 if (!refcount_dec_and_test(&rb->refcount)) in ring_buffer_put()
6691 WARN_ON_ONCE(!list_empty(&rb->event_list)); in ring_buffer_put()
6693 call_rcu(&rb->rcu_head, rb_free_rcu); in ring_buffer_put()
6702 pmu = READ_ONCE(event->pmu); \
6704 f = pmu->func; \
6710 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_open()
6713 refcount_inc(&event->mmap_count); in perf_mmap_open()
6714 refcount_inc(&event->rb->mmap_count); in perf_mmap_open()
6716 if (vma->vm_pgoff) in perf_mmap_open()
6717 refcount_inc(&event->rb->aux_mmap_count); in perf_mmap_open()
6720 mapped(event, vma->vm_mm); in perf_mmap_open()
6735 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_close()
6738 struct user_struct *mmap_user = rb->mmap_user; in perf_mmap_close()
6739 int mmap_locked = rb->mmap_locked; in perf_mmap_close()
6745 unmapped(event, vma->vm_mm); in perf_mmap_close()
6748 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex in perf_mmap_close()
6751 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && in perf_mmap_close()
6752 refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { in perf_mmap_close()
6762 atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm); in perf_mmap_close()
6763 atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6767 WARN_ON_ONCE(refcount_read(&rb->aux_refcount)); in perf_mmap_close()
6769 mutex_unlock(&rb->aux_mutex); in perf_mmap_close()
6772 if (refcount_dec_and_test(&rb->mmap_count)) in perf_mmap_close()
6775 if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) in perf_mmap_close()
6779 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6792 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { in perf_mmap_close()
6793 if (!atomic_long_inc_not_zero(&event->refcount)) { in perf_mmap_close()
6795 * This event is en-route to free_event() which will in perf_mmap_close()
6802 mutex_lock(&event->mmap_mutex); in perf_mmap_close()
6813 if (event->rb == rb) in perf_mmap_close()
6816 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6828 * It could be there's still a few 0-ref events on the list; they'll in perf_mmap_close()
6829 * get cleaned up by free_event() -- they'll also still have their in perf_mmap_close()
6836 atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked, in perf_mmap_close()
6837 &mmap_user->locked_vm); in perf_mmap_close()
6838 atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6847 /* The first page is the user control page, others are read-only. */ in perf_mmap_pfn_mkwrite()
6848 return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; in perf_mmap_pfn_mkwrite()
6855 * the resulting non-matching offsets and sizes. See open()/close(). in perf_mmap_may_split()
6857 return -EINVAL; in perf_mmap_may_split()
6877 * referencing memory-mapped I/O ranges or non-system RAM i.e. for which in map_range()
6880 * We are mapping kernel-allocated memory (memory we manage ourselves) in map_range()
6886 * 1. It uses vma->vm_page_prot, but this field has not been completely in map_range()
6887 * setup at the point of the f_op->mmp() hook, so we are unable to in map_range()
6894 * vm_ops->page_mkwrite() will be invoked rather than in map_range()
6895 * vm_ops->pfn_mkwrite(), and this means we have to set page->mapping in map_range()
6912 unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; in map_range()
6913 struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); in map_range()
6916 err = -EINVAL; in map_range()
6922 vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); in map_range()
6930 zap_page_range_single(vma, vma->vm_start, nr_pages * PAGE_SIZE, NULL); in map_range()
6941 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); in perf_mmap_calc_limits()
6945 user_locked = atomic_long_read(&user->locked_vm); in perf_mmap_calc_limits()
6949 * user->locked_vm > user_lock_limit in perf_mmap_calc_limits()
6960 *extra = user_locked - user_lock_limit; in perf_mmap_calc_limits()
6961 *user_extra -= *extra; in perf_mmap_calc_limits()
6966 locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra; in perf_mmap_calc_limits()
6975 atomic_long_add(user_extra, &user->locked_vm); in perf_mmap_account()
6976 atomic64_add(extra, &vma->vm_mm->pinned_vm); in perf_mmap_account()
6986 nr_pages -= 1; in perf_mmap_rb()
6989 * If we have rb pages ensure they're a power-of-two number, so we in perf_mmap_rb()
6993 return -EINVAL; in perf_mmap_rb()
6995 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_mmap_rb()
6997 if (event->rb) { in perf_mmap_rb()
6998 if (data_page_nr(event->rb) != nr_pages) in perf_mmap_rb()
6999 return -EINVAL; in perf_mmap_rb()
7001 if (refcount_inc_not_zero(&event->rb->mmap_count)) { in perf_mmap_rb()
7003 * Success -- managed to mmap() the same buffer in perf_mmap_rb()
7007 refcount_inc(&event->mmap_count); in perf_mmap_rb()
7014 * event and continue as if !event->rb in perf_mmap_rb()
7020 return -EPERM; in perf_mmap_rb()
7022 if (vma->vm_flags & VM_WRITE) in perf_mmap_rb()
7026 event->attr.watermark ? event->attr.wakeup_watermark : 0, in perf_mmap_rb()
7027 event->cpu, rb_flags); in perf_mmap_rb()
7030 return -ENOMEM; in perf_mmap_rb()
7032 refcount_set(&rb->mmap_count, 1); in perf_mmap_rb()
7033 rb->mmap_user = get_current_user(); in perf_mmap_rb()
7034 rb->mmap_locked = extra; in perf_mmap_rb()
7043 refcount_set(&event->mmap_count, 1); in perf_mmap_rb()
7056 rb = event->rb; in perf_mmap_aux()
7058 return -EINVAL; in perf_mmap_aux()
7060 guard(mutex)(&rb->aux_mutex); in perf_mmap_aux()
7063 * AUX area mapping: if rb->aux_nr_pages != 0, it's already in perf_mmap_aux()
7067 aux_offset = READ_ONCE(rb->user_page->aux_offset); in perf_mmap_aux()
7068 aux_size = READ_ONCE(rb->user_page->aux_size); in perf_mmap_aux()
7071 return -EINVAL; in perf_mmap_aux()
7073 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) in perf_mmap_aux()
7074 return -EINVAL; in perf_mmap_aux()
7077 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) in perf_mmap_aux()
7078 return -EINVAL; in perf_mmap_aux()
7081 return -EINVAL; in perf_mmap_aux()
7084 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) in perf_mmap_aux()
7085 return -EINVAL; in perf_mmap_aux()
7088 return -EINVAL; in perf_mmap_aux()
7090 if (!refcount_inc_not_zero(&rb->mmap_count)) in perf_mmap_aux()
7091 return -EINVAL; in perf_mmap_aux()
7094 refcount_inc(&rb->aux_mmap_count); in perf_mmap_aux()
7098 refcount_dec(&rb->mmap_count); in perf_mmap_aux()
7099 return -EPERM; in perf_mmap_aux()
7102 WARN_ON(!rb && event->rb); in perf_mmap_aux()
7104 if (vma->vm_flags & VM_WRITE) in perf_mmap_aux()
7107 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, in perf_mmap_aux()
7108 event->attr.aux_watermark, rb_flags); in perf_mmap_aux()
7110 refcount_dec(&rb->mmap_count); in perf_mmap_aux()
7114 refcount_set(&rb->aux_mmap_count, 1); in perf_mmap_aux()
7115 rb->aux_mmap_locked = extra; in perf_mmap_aux()
7119 refcount_inc(&event->mmap_count); in perf_mmap_aux()
7126 struct perf_event *event = file->private_data; in perf_mmap()
7132 * Don't allow mmap() of inherited per-task counters. This would in perf_mmap()
7136 if (event->cpu == -1 && event->attr.inherit) in perf_mmap()
7137 return -EINVAL; in perf_mmap()
7139 if (!(vma->vm_flags & VM_SHARED)) in perf_mmap()
7140 return -EINVAL; in perf_mmap()
7146 vma_size = vma->vm_end - vma->vm_start; in perf_mmap()
7150 return -ENOMEM; in perf_mmap()
7153 return -EINVAL; in perf_mmap()
7155 scoped_guard (mutex, &event->mmap_mutex) { in perf_mmap()
7161 if (event->state <= PERF_EVENT_STATE_REVOKED) in perf_mmap()
7162 return -ENODEV; in perf_mmap()
7164 if (vma->vm_pgoff == 0) in perf_mmap()
7177 vma->vm_ops = &perf_mmap_vmops; in perf_mmap()
7181 mapped(event, vma->vm_mm); in perf_mmap()
7189 ret = map_range(event->rb, vma); in perf_mmap()
7199 struct perf_event *event = filp->private_data; in perf_fasync()
7202 if (event->state <= PERF_EVENT_STATE_REVOKED) in perf_fasync()
7203 return -ENODEV; in perf_fasync()
7206 retval = fasync_helper(fd, filp, on, &event->fasync); in perf_fasync()
7229 * to user-space before waking everybody up.
7236 if (event->pending_kill) { in perf_event_wakeup()
7237 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); in perf_event_wakeup()
7238 event->pending_kill = 0; in perf_event_wakeup()
7248 if (current->flags & PF_EXITING) in perf_sigtrap()
7253 * ctx->task or current has changed in the meantime. This can be the in perf_sigtrap()
7256 if (WARN_ON_ONCE(event->ctx->task != current)) in perf_sigtrap()
7259 send_sig_perf((void __user *)event->pending_addr, in perf_sigtrap()
7260 event->orig_type, event->attr.sig_data); in perf_sigtrap()
7264 * Deliver the pending work in-event-context or follow the context.
7268 int cpu = READ_ONCE(event->oncpu); in __perf_pending_disable()
7281 if (event->pending_disable) { in __perf_pending_disable()
7282 event->pending_disable = 0; in __perf_pending_disable()
7289 * CPU-A CPU-B in __perf_pending_disable()
7295 * sched-out in __perf_pending_disable()
7298 * sched-in in __perf_pending_disable()
7306 * But the event runs on CPU-B and wants disabling there. in __perf_pending_disable()
7308 irq_work_queue_on(&event->pending_disable_irq, cpu); in __perf_pending_disable()
7338 * The wakeup isn't bound to the context of the event -- it can happen in perf_pending_irq()
7341 if (event->pending_wakeup) { in perf_pending_irq()
7342 event->pending_wakeup = 0; in perf_pending_irq()
7361 if (event->pending_work) { in perf_pending_task()
7362 event->pending_work = 0; in perf_pending_task()
7364 local_dec(&event->ctx->nr_no_switch_fast); in perf_pending_task()
7375 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
7376 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
7377 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
7385 static_call_update(__perf_guest_state, cbs->state); in perf_register_guest_info_callbacks()
7386 static_call_update(__perf_guest_get_ip, cbs->get_ip); in perf_register_guest_info_callbacks()
7388 /* Implementing ->handle_intel_pt_intr is optional. */ in perf_register_guest_info_callbacks()
7389 if (cbs->handle_intel_pt_intr) in perf_register_guest_info_callbacks()
7391 cbs->handle_intel_pt_intr); in perf_register_guest_info_callbacks()
7412 return !event->attr.exclude_guest && perf_guest_state(); in should_sample_guest()
7453 regs_user->abi = perf_reg_abi(current); in perf_sample_regs_user()
7454 regs_user->regs = regs; in perf_sample_regs_user()
7455 } else if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { in perf_sample_regs_user()
7458 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; in perf_sample_regs_user()
7459 regs_user->regs = NULL; in perf_sample_regs_user()
7466 regs_intr->regs = regs; in perf_sample_regs_intr()
7467 regs_intr->abi = perf_reg_abi(current); in perf_sample_regs_intr()
7485 return TASK_SIZE - addr; in perf_ustack_task_size()
7499 if (!current->mm) in perf_sample_ustack_size()
7504 * - TASK_SIZE in perf_sample_ustack_size()
7507 * - remaining sample size in perf_sample_ustack_size()
7524 stack_size = USHRT_MAX - header_size - sizeof(u64); in perf_sample_ustack_size()
7547 * - the size requested by user or the best one we can fit in perf_output_sample_ustack()
7550 * - user stack dump data in perf_output_sample_ustack()
7552 * - the actual dumped size in perf_output_sample_ustack()
7561 dyn_size = dump_size - rem; in perf_output_sample_ustack()
7574 struct perf_event *sampler = event->aux_event; in perf_prepare_sample_aux()
7577 data->aux_size = 0; in perf_prepare_sample_aux()
7582 if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE)) in perf_prepare_sample_aux()
7585 if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id())) in perf_prepare_sample_aux()
7596 if (READ_ONCE(rb->aux_in_sampling)) { in perf_prepare_sample_aux()
7597 data->aux_size = 0; in perf_prepare_sample_aux()
7600 data->aux_size = ALIGN(size, sizeof(u64)); in perf_prepare_sample_aux()
7605 return data->aux_size; in perf_prepare_sample_aux()
7617 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler in perf_pmu_snapshot_aux()
7619 * the IRQ ones, that is, for example, re-starting an event that's just in perf_pmu_snapshot_aux()
7630 WRITE_ONCE(rb->aux_in_sampling, 1); in perf_pmu_snapshot_aux()
7633 ret = event->pmu->snapshot_aux(event, handle, size); in perf_pmu_snapshot_aux()
7636 WRITE_ONCE(rb->aux_in_sampling, 0); in perf_pmu_snapshot_aux()
7646 struct perf_event *sampler = event->aux_event; in perf_aux_sample_output()
7651 if (WARN_ON_ONCE(!sampler || !data->aux_size)) in perf_aux_sample_output()
7658 size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size); in perf_aux_sample_output()
7662 * non-zero surplus that it didn't copy), which in its current in perf_aux_sample_output()
7670 * The pad comes from ALIGN()ing data->aux_size up to u64 in in perf_aux_sample_output()
7673 pad = data->aux_size - size; in perf_aux_sample_output()
7687 * A set of common sample data types saved even for non-sample records
7688 * when event->attr.sample_id_all is set.
7698 data->type = event->attr.sample_type; in __perf_event_header__init_id()
7699 data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL; in __perf_event_header__init_id()
7703 data->tid_entry.pid = perf_event_pid(event, current); in __perf_event_header__init_id()
7704 data->tid_entry.tid = perf_event_tid(event, current); in __perf_event_header__init_id()
7708 data->time = perf_event_clock(event); in __perf_event_header__init_id()
7711 data->id = primary_event_id(event); in __perf_event_header__init_id()
7714 data->stream_id = event->id; in __perf_event_header__init_id()
7717 data->cpu_entry.cpu = raw_smp_processor_id(); in __perf_event_header__init_id()
7718 data->cpu_entry.reserved = 0; in __perf_event_header__init_id()
7726 if (event->attr.sample_id_all) { in perf_event_header__init_id()
7727 header->size += event->id_header_size; in perf_event_header__init_id()
7728 __perf_event_header__init_id(data, event, event->attr.sample_type); in perf_event_header__init_id()
7735 u64 sample_type = data->type; in __perf_event__output_id_sample()
7738 perf_output_put(handle, data->tid_entry); in __perf_event__output_id_sample()
7741 perf_output_put(handle, data->time); in __perf_event__output_id_sample()
7744 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
7747 perf_output_put(handle, data->stream_id); in __perf_event__output_id_sample()
7750 perf_output_put(handle, data->cpu_entry); in __perf_event__output_id_sample()
7753 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
7760 if (event->attr.sample_id_all) in perf_event__output_id_sample()
7768 u64 read_format = event->attr.read_format; in perf_output_read_one()
7772 values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr)); in perf_output_read_one()
7775 atomic64_read(&event->child_total_time_enabled); in perf_output_read_one()
7779 atomic64_read(&event->child_total_time_running); in perf_output_read_one()
7784 values[n++] = atomic64_read(&event->lost_samples); in perf_output_read_one()
7793 struct perf_event *leader = event->group_leader, *sub; in perf_output_read_group()
7794 u64 read_format = event->attr.read_format; in perf_output_read_group()
7798 bool self = has_inherit_and_sample_read(&event->attr); in perf_output_read_group()
7806 values[n++] = 1 + leader->nr_siblings; in perf_output_read_group()
7814 if ((leader != event) && !handle->skip_read) in perf_output_read_group()
7821 values[n++] = atomic64_read(&leader->lost_samples); in perf_output_read_group()
7828 if ((sub != event) && !handle->skip_read) in perf_output_read_group()
7835 values[n++] = atomic64_read(&sub->lost_samples); in perf_output_read_group()
7853 * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread
7861 u64 read_format = event->attr.read_format; in perf_output_read()
7875 if (event->attr.read_format & PERF_FORMAT_GROUP) in perf_output_read()
7886 u64 sample_type = data->type; in perf_output_sample()
7888 if (data->sample_flags & PERF_SAMPLE_READ) in perf_output_sample()
7889 handle->skip_read = 1; in perf_output_sample()
7894 perf_output_put(handle, data->id); in perf_output_sample()
7897 perf_output_put(handle, data->ip); in perf_output_sample()
7900 perf_output_put(handle, data->tid_entry); in perf_output_sample()
7903 perf_output_put(handle, data->time); in perf_output_sample()
7906 perf_output_put(handle, data->addr); in perf_output_sample()
7909 perf_output_put(handle, data->id); in perf_output_sample()
7912 perf_output_put(handle, data->stream_id); in perf_output_sample()
7915 perf_output_put(handle, data->cpu_entry); in perf_output_sample()
7918 perf_output_put(handle, data->period); in perf_output_sample()
7926 size += data->callchain->nr; in perf_output_sample()
7928 __output_copy(handle, data->callchain, size); in perf_output_sample()
7932 struct perf_raw_record *raw = data->raw; in perf_output_sample()
7935 struct perf_raw_frag *frag = &raw->frag; in perf_output_sample()
7937 perf_output_put(handle, raw->size); in perf_output_sample()
7939 if (frag->copy) { in perf_output_sample()
7940 __output_custom(handle, frag->copy, in perf_output_sample()
7941 frag->data, frag->size); in perf_output_sample()
7943 __output_copy(handle, frag->data, in perf_output_sample()
7944 frag->size); in perf_output_sample()
7948 frag = frag->next; in perf_output_sample()
7950 if (frag->pad) in perf_output_sample()
7951 __output_skip(handle, NULL, frag->pad); in perf_output_sample()
7965 if (data->br_stack) { in perf_output_sample()
7968 size = data->br_stack->nr in perf_output_sample()
7971 perf_output_put(handle, data->br_stack->nr); in perf_output_sample()
7973 perf_output_put(handle, data->br_stack->hw_idx); in perf_output_sample()
7974 perf_output_copy(handle, data->br_stack->entries, size); in perf_output_sample()
7979 if (data->br_stack_cntr) { in perf_output_sample()
7980 size = data->br_stack->nr * sizeof(u64); in perf_output_sample()
7981 perf_output_copy(handle, data->br_stack_cntr, size); in perf_output_sample()
7993 u64 abi = data->regs_user.abi; in perf_output_sample()
8002 u64 mask = event->attr.sample_regs_user; in perf_output_sample()
8004 data->regs_user.regs, in perf_output_sample()
8011 data->stack_user_size, in perf_output_sample()
8012 data->regs_user.regs); in perf_output_sample()
8016 perf_output_put(handle, data->weight.full); in perf_output_sample()
8019 perf_output_put(handle, data->data_src.val); in perf_output_sample()
8022 perf_output_put(handle, data->txn); in perf_output_sample()
8025 u64 abi = data->regs_intr.abi; in perf_output_sample()
8033 u64 mask = event->attr.sample_regs_intr; in perf_output_sample()
8036 data->regs_intr.regs, in perf_output_sample()
8042 perf_output_put(handle, data->phys_addr); in perf_output_sample()
8045 perf_output_put(handle, data->cgroup); in perf_output_sample()
8048 perf_output_put(handle, data->data_page_size); in perf_output_sample()
8051 perf_output_put(handle, data->code_page_size); in perf_output_sample()
8054 perf_output_put(handle, data->aux_size); in perf_output_sample()
8056 if (data->aux_size) in perf_output_sample()
8060 if (!event->attr.watermark) { in perf_output_sample()
8061 int wakeup_events = event->attr.wakeup_events; in perf_output_sample()
8064 struct perf_buffer *rb = handle->rb; in perf_output_sample()
8065 int events = local_inc_return(&rb->events); in perf_output_sample()
8068 local_sub(wakeup_events, &rb->events); in perf_output_sample()
8069 local_inc(&rb->wakeup); in perf_output_sample()
8092 * Try IRQ-safe get_user_page_fast_only first. in perf_virt_to_phys()
8095 if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { in perf_virt_to_phys()
8180 * Software page-table walkers must disable IRQs, in perf_get_page_size()
8185 mm = current->mm; in perf_get_page_size()
8206 bool kernel = !event->attr.exclude_callchain_kernel; in perf_callchain()
8207 bool user = !event->attr.exclude_callchain_user && in perf_callchain()
8208 !(current->flags & (PF_KTHREAD | PF_USER_WORKER)); in perf_callchain()
8209 /* Disallow cross-task user callchains. */ in perf_callchain()
8210 bool crosstask = event->ctx->task && event->ctx->task != current; in perf_callchain()
8211 const u32 max_stack = event->attr.sample_max_stack; in perf_callchain()
8214 if (!current->mm) in perf_callchain()
8234 u64 sample_type = event->attr.sample_type; in perf_prepare_sample()
8248 filtered_sample_type &= ~data->sample_flags; in perf_prepare_sample()
8251 /* Make sure it has the correct data->type for output */ in perf_prepare_sample()
8252 data->type = event->attr.sample_type; in perf_prepare_sample()
8259 data->ip = perf_instruction_pointer(event, regs); in perf_prepare_sample()
8260 data->sample_flags |= PERF_SAMPLE_IP; in perf_prepare_sample()
8267 data->raw = NULL; in perf_prepare_sample()
8268 data->dyn_size += sizeof(u64); in perf_prepare_sample()
8269 data->sample_flags |= PERF_SAMPLE_RAW; in perf_prepare_sample()
8273 data->br_stack = NULL; in perf_prepare_sample()
8274 data->dyn_size += sizeof(u64); in perf_prepare_sample()
8275 data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; in perf_prepare_sample()
8279 perf_sample_regs_user(&data->regs_user, regs); in perf_prepare_sample()
8286 if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) { in perf_prepare_sample()
8290 if (data->regs_user.regs) { in perf_prepare_sample()
8291 u64 mask = event->attr.sample_regs_user; in perf_prepare_sample()
8295 data->dyn_size += size; in perf_prepare_sample()
8296 data->sample_flags |= PERF_SAMPLE_REGS_USER; in perf_prepare_sample()
8306 u16 stack_size = event->attr.sample_stack_user; in perf_prepare_sample()
8311 data->regs_user.regs); in perf_prepare_sample()
8321 data->stack_user_size = stack_size; in perf_prepare_sample()
8322 data->dyn_size += size; in perf_prepare_sample()
8323 data->sample_flags |= PERF_SAMPLE_STACK_USER; in perf_prepare_sample()
8327 data->weight.full = 0; in perf_prepare_sample()
8328 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; in perf_prepare_sample()
8332 data->data_src.val = PERF_MEM_NA; in perf_prepare_sample()
8333 data->sample_flags |= PERF_SAMPLE_DATA_SRC; in perf_prepare_sample()
8337 data->txn = 0; in perf_prepare_sample()
8338 data->sample_flags |= PERF_SAMPLE_TRANSACTION; in perf_prepare_sample()
8342 data->addr = 0; in perf_prepare_sample()
8343 data->sample_flags |= PERF_SAMPLE_ADDR; in perf_prepare_sample()
8350 perf_sample_regs_intr(&data->regs_intr, regs); in perf_prepare_sample()
8352 if (data->regs_intr.regs) { in perf_prepare_sample()
8353 u64 mask = event->attr.sample_regs_intr; in perf_prepare_sample()
8358 data->dyn_size += size; in perf_prepare_sample()
8359 data->sample_flags |= PERF_SAMPLE_REGS_INTR; in perf_prepare_sample()
8363 data->phys_addr = perf_virt_to_phys(data->addr); in perf_prepare_sample()
8364 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; in perf_prepare_sample()
8372 cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup; in perf_prepare_sample()
8373 data->cgroup = cgroup_id(cgrp); in perf_prepare_sample()
8374 data->sample_flags |= PERF_SAMPLE_CGROUP; in perf_prepare_sample()
8380 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, in perf_prepare_sample()
8384 data->data_page_size = perf_get_page_size(data->addr); in perf_prepare_sample()
8385 data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE; in perf_prepare_sample()
8389 data->code_page_size = perf_get_page_size(data->ip); in perf_prepare_sample()
8390 data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; in perf_prepare_sample()
8405 size = min_t(size_t, U16_MAX - header_size, in perf_prepare_sample()
8406 event->attr.aux_sample_size); in perf_prepare_sample()
8411 data->dyn_size += size + sizeof(u64); /* size above */ in perf_prepare_sample()
8412 data->sample_flags |= PERF_SAMPLE_AUX; in perf_prepare_sample()
8421 header->type = PERF_RECORD_SAMPLE; in perf_prepare_header()
8422 header->size = perf_sample_data_size(data, event); in perf_prepare_header()
8423 header->misc = perf_misc_flags(event, regs); in perf_prepare_header()
8433 WARN_ON_ONCE(header->size & 7); in perf_prepare_header()
8439 if (!event->hw.aux_paused) { in __perf_event_aux_pause()
8440 event->hw.aux_paused = 1; in __perf_event_aux_pause()
8441 event->pmu->stop(event, PERF_EF_PAUSE); in __perf_event_aux_pause()
8444 if (event->hw.aux_paused) { in __perf_event_aux_pause()
8445 event->hw.aux_paused = 0; in __perf_event_aux_pause()
8446 event->pmu->start(event, PERF_EF_RESUME); in __perf_event_aux_pause()
8464 * Guard against self-recursion here. Another event could trip in perf_event_aux_pause()
8467 if (READ_ONCE(rb->aux_in_pause_resume)) in perf_event_aux_pause()
8470 WRITE_ONCE(rb->aux_in_pause_resume, 1); in perf_event_aux_pause()
8474 WRITE_ONCE(rb->aux_in_pause_resume, 0); in perf_event_aux_pause()
8556 .size = sizeof(read_event) + event->read_size, in perf_event_read_event()
8584 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_iterate_ctx()
8586 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_ctx()
8601 list_for_each_entry_rcu(event, &pel->list, sb_list) { in perf_iterate_sb_cpu()
8604 * if we observe event->ctx, both event and ctx will be in perf_iterate_sb_cpu()
8607 if (!smp_load_acquire(&event->ctx)) in perf_iterate_sb_cpu()
8610 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_sb_cpu()
8619 * Iterate all events that need to receive side-band events.
8645 ctx = rcu_dereference(current->perf_event_ctxp); in perf_iterate_sb()
8654 * Clear all file-based filters at exec, they'll have to be
8655 * re-instated when/if these objects are mmapped again.
8667 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_exec()
8668 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_exec()
8669 if (filter->path.dentry) { in perf_event_addr_filters_exec()
8670 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_exec()
8671 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_exec()
8679 event->addr_filters_gen++; in perf_event_addr_filters_exec()
8680 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_exec()
8710 struct perf_event *parent = event->parent; in __perf_event_output_stop()
8712 struct perf_buffer *rb = ro->rb; in __perf_event_output_stop()
8725 * ring-buffer, but it will be the child that's actually using it. in __perf_event_output_stop()
8730 * So ring_buffer_attach() has to stop an aux event before re-assigning in __perf_event_output_stop()
8733 if (rcu_dereference(parent->rb) == rb) in __perf_event_output_stop()
8734 ro->err = __perf_event_stop(&sd); in __perf_event_output_stop()
8742 .rb = event->rb, in __perf_pmu_output_stop()
8746 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); in __perf_pmu_output_stop()
8747 if (cpuctx->task_ctx) in __perf_pmu_output_stop()
8748 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, in __perf_pmu_output_stop()
8762 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { in perf_pmu_output_stop()
8764 * For per-CPU events, we need to make sure that neither they in perf_pmu_output_stop()
8765 * nor their children are running; for cpu==-1 events it's in perf_pmu_output_stop()
8769 cpu = iter->cpu; in perf_pmu_output_stop()
8770 if (cpu == -1) in perf_pmu_output_stop()
8771 cpu = READ_ONCE(iter->oncpu); in perf_pmu_output_stop()
8773 if (cpu == -1) in perf_pmu_output_stop()
8777 if (err == -EAGAIN) { in perf_pmu_output_stop()
8786 * task tracking -- fork/exit
8808 return event->attr.comm || event->attr.mmap || in perf_event_task_match()
8809 event->attr.mmap2 || event->attr.mmap_data || in perf_event_task_match()
8810 event->attr.task; in perf_event_task_match()
8819 struct task_struct *task = task_event->task; in perf_event_task_output()
8820 int ret, size = task_event->event_id.header.size; in perf_event_task_output()
8825 perf_event_header__init_id(&task_event->event_id.header, &sample, event); in perf_event_task_output()
8828 task_event->event_id.header.size); in perf_event_task_output()
8832 task_event->event_id.pid = perf_event_pid(event, task); in perf_event_task_output()
8833 task_event->event_id.tid = perf_event_tid(event, task); in perf_event_task_output()
8835 if (task_event->event_id.header.type == PERF_RECORD_EXIT) { in perf_event_task_output()
8836 task_event->event_id.ppid = perf_event_pid(event, in perf_event_task_output()
8837 task->real_parent); in perf_event_task_output()
8838 task_event->event_id.ptid = perf_event_pid(event, in perf_event_task_output()
8839 task->real_parent); in perf_event_task_output()
8841 task_event->event_id.ppid = perf_event_pid(event, current); in perf_event_task_output()
8842 task_event->event_id.ptid = perf_event_tid(event, current); in perf_event_task_output()
8845 task_event->event_id.time = perf_event_clock(event); in perf_event_task_output()
8847 perf_output_put(&handle, task_event->event_id); in perf_event_task_output()
8853 task_event->event_id.header.size = size; in perf_event_task_output()
8890 * Allocate data for a new task when profiling system-wide
8904 cd = rcu_dereference(parent->perf_ctx_data); in perf_event_alloc_task_data()
8906 ctx_cache = cd->ctx_cache; in perf_event_alloc_task_data()
8914 cd = rcu_dereference(child->perf_ctx_data); in perf_event_alloc_task_data()
8917 * A system-wide event may be unaccount, in perf_event_alloc_task_data()
8925 if (!cd->global) { in perf_event_alloc_task_data()
8926 cd->global = 1; in perf_event_alloc_task_data()
8927 refcount_inc(&cd->refcount); in perf_event_alloc_task_data()
8962 return event->attr.comm; in perf_event_comm_match()
8971 int size = comm_event->event_id.header.size; in perf_event_comm_output()
8977 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); in perf_event_comm_output()
8979 comm_event->event_id.header.size); in perf_event_comm_output()
8984 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); in perf_event_comm_output()
8985 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); in perf_event_comm_output()
8987 perf_output_put(&handle, comm_event->event_id); in perf_event_comm_output()
8988 __output_copy(&handle, comm_event->comm, in perf_event_comm_output()
8989 comm_event->comm_size); in perf_event_comm_output()
8995 comm_event->event_id.header.size = size; in perf_event_comm_output()
9004 strscpy(comm, comm_event->task->comm); in perf_event_comm_event()
9007 comm_event->comm = comm; in perf_event_comm_event()
9008 comm_event->comm_size = size; in perf_event_comm_event()
9010 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; in perf_event_comm_event()
9061 return event->attr.namespaces; in perf_event_namespaces_match()
9070 u16 header_size = namespaces_event->event_id.header.size; in perf_event_namespaces_output()
9076 perf_event_header__init_id(&namespaces_event->event_id.header, in perf_event_namespaces_output()
9079 namespaces_event->event_id.header.size); in perf_event_namespaces_output()
9083 namespaces_event->event_id.pid = perf_event_pid(event, in perf_event_namespaces_output()
9084 namespaces_event->task); in perf_event_namespaces_output()
9085 namespaces_event->event_id.tid = perf_event_tid(event, in perf_event_namespaces_output()
9086 namespaces_event->task); in perf_event_namespaces_output()
9088 perf_output_put(&handle, namespaces_event->event_id); in perf_event_namespaces_output()
9094 namespaces_event->event_id.header.size = header_size; in perf_event_namespaces_output()
9107 ns_inode = ns_path.dentry->d_inode; in perf_fill_ns_link_info()
9108 ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev); in perf_fill_ns_link_info()
9109 ns_link_info->ino = ns_inode->i_ino; in perf_fill_ns_link_info()
9189 return event->attr.cgroup; in perf_event_cgroup_match()
9197 u16 header_size = cgroup_event->event_id.header.size; in perf_event_cgroup_output()
9203 perf_event_header__init_id(&cgroup_event->event_id.header, in perf_event_cgroup_output()
9206 cgroup_event->event_id.header.size); in perf_event_cgroup_output()
9210 perf_output_put(&handle, cgroup_event->event_id); in perf_event_cgroup_output()
9211 __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); in perf_event_cgroup_output()
9217 cgroup_event->event_id.header.size = header_size; in perf_event_cgroup_output()
9246 cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64)); in perf_event_cgroup()
9302 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_match()
9303 int executable = vma->vm_flags & VM_EXEC; in perf_event_mmap_match()
9305 return (!executable && event->attr.mmap_data) || in perf_event_mmap_match()
9306 (executable && (event->attr.mmap || event->attr.mmap2)); in perf_event_mmap_match()
9315 int size = mmap_event->event_id.header.size; in perf_event_mmap_output()
9316 u32 type = mmap_event->event_id.header.type; in perf_event_mmap_output()
9323 if (event->attr.mmap2) { in perf_event_mmap_output()
9324 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; in perf_event_mmap_output()
9325 mmap_event->event_id.header.size += sizeof(mmap_event->maj); in perf_event_mmap_output()
9326 mmap_event->event_id.header.size += sizeof(mmap_event->min); in perf_event_mmap_output()
9327 mmap_event->event_id.header.size += sizeof(mmap_event->ino); in perf_event_mmap_output()
9328 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); in perf_event_mmap_output()
9329 mmap_event->event_id.header.size += sizeof(mmap_event->prot); in perf_event_mmap_output()
9330 mmap_event->event_id.header.size += sizeof(mmap_event->flags); in perf_event_mmap_output()
9333 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); in perf_event_mmap_output()
9335 mmap_event->event_id.header.size); in perf_event_mmap_output()
9339 mmap_event->event_id.pid = perf_event_pid(event, current); in perf_event_mmap_output()
9340 mmap_event->event_id.tid = perf_event_tid(event, current); in perf_event_mmap_output()
9342 use_build_id = event->attr.build_id && mmap_event->build_id_size; in perf_event_mmap_output()
9344 if (event->attr.mmap2 && use_build_id) in perf_event_mmap_output()
9345 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; in perf_event_mmap_output()
9347 perf_output_put(&handle, mmap_event->event_id); in perf_event_mmap_output()
9349 if (event->attr.mmap2) { in perf_event_mmap_output()
9351 u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 }; in perf_event_mmap_output()
9354 __output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX); in perf_event_mmap_output()
9356 perf_output_put(&handle, mmap_event->maj); in perf_event_mmap_output()
9357 perf_output_put(&handle, mmap_event->min); in perf_event_mmap_output()
9358 perf_output_put(&handle, mmap_event->ino); in perf_event_mmap_output()
9359 perf_output_put(&handle, mmap_event->ino_generation); in perf_event_mmap_output()
9361 perf_output_put(&handle, mmap_event->prot); in perf_event_mmap_output()
9362 perf_output_put(&handle, mmap_event->flags); in perf_event_mmap_output()
9365 __output_copy(&handle, mmap_event->file_name, in perf_event_mmap_output()
9366 mmap_event->file_size); in perf_event_mmap_output()
9372 mmap_event->event_id.header.size = size; in perf_event_mmap_output()
9373 mmap_event->event_id.header.type = type; in perf_event_mmap_output()
9378 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_event()
9379 struct file *file = vma->vm_file; in perf_event_mmap_event()
9388 if (vma->vm_flags & VM_READ) in perf_event_mmap_event()
9390 if (vma->vm_flags & VM_WRITE) in perf_event_mmap_event()
9392 if (vma->vm_flags & VM_EXEC) in perf_event_mmap_event()
9395 if (vma->vm_flags & VM_MAYSHARE) in perf_event_mmap_event()
9400 if (vma->vm_flags & VM_LOCKED) in perf_event_mmap_event()
9419 name = d_path(file_user_path(file), buf, PATH_MAX - sizeof(u64)); in perf_event_mmap_event()
9424 inode = file_user_inode(vma->vm_file); in perf_event_mmap_event()
9425 dev = inode->i_sb->s_dev; in perf_event_mmap_event()
9426 ino = inode->i_ino; in perf_event_mmap_event()
9427 gen = inode->i_generation; in perf_event_mmap_event()
9433 if (vma->vm_ops && vma->vm_ops->name) in perf_event_mmap_event()
9434 name = (char *) vma->vm_ops->name(vma); in perf_event_mmap_event()
9460 mmap_event->file_name = name; in perf_event_mmap_event()
9461 mmap_event->file_size = size; in perf_event_mmap_event()
9462 mmap_event->maj = maj; in perf_event_mmap_event()
9463 mmap_event->min = min; in perf_event_mmap_event()
9464 mmap_event->ino = ino; in perf_event_mmap_event()
9465 mmap_event->ino_generation = gen; in perf_event_mmap_event()
9466 mmap_event->prot = prot; in perf_event_mmap_event()
9467 mmap_event->flags = flags; in perf_event_mmap_event()
9469 if (!(vma->vm_flags & VM_EXEC)) in perf_event_mmap_event()
9470 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; in perf_event_mmap_event()
9472 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; in perf_event_mmap_event()
9475 build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); in perf_event_mmap_event()
9491 /* d_inode(NULL) won't be equal to any mapped user-space file */ in perf_addr_filter_match()
9492 if (!filter->path.dentry) in perf_addr_filter_match()
9495 if (d_inode(filter->path.dentry) != file_user_inode(file)) in perf_addr_filter_match()
9498 if (filter->offset > offset + size) in perf_addr_filter_match()
9501 if (filter->offset + filter->size < offset) in perf_addr_filter_match()
9511 unsigned long vma_size = vma->vm_end - vma->vm_start; in perf_addr_filter_vma_adjust()
9512 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; in perf_addr_filter_vma_adjust()
9513 struct file *file = vma->vm_file; in perf_addr_filter_vma_adjust()
9518 if (filter->offset < off) { in perf_addr_filter_vma_adjust()
9519 fr->start = vma->vm_start; in perf_addr_filter_vma_adjust()
9520 fr->size = min(vma_size, filter->size - (off - filter->offset)); in perf_addr_filter_vma_adjust()
9522 fr->start = vma->vm_start + filter->offset - off; in perf_addr_filter_vma_adjust()
9523 fr->size = min(vma->vm_end - fr->start, filter->size); in perf_addr_filter_vma_adjust()
9540 if (!vma->vm_file) in __perf_addr_filters_adjust()
9543 raw_spin_lock_irqsave(&ifh->lock, flags); in __perf_addr_filters_adjust()
9544 list_for_each_entry(filter, &ifh->list, entry) { in __perf_addr_filters_adjust()
9546 &event->addr_filter_ranges[count])) in __perf_addr_filters_adjust()
9553 event->addr_filters_gen++; in __perf_addr_filters_adjust()
9554 raw_spin_unlock_irqrestore(&ifh->lock, flags); in __perf_addr_filters_adjust()
9571 if (!(vma->vm_flags & VM_EXEC)) in perf_addr_filters_adjust()
9575 ctx = rcu_dereference(current->perf_event_ctxp); in perf_addr_filters_adjust()
9600 .start = vma->vm_start, in perf_event_mmap()
9601 .len = vma->vm_end - vma->vm_start, in perf_event_mmap()
9602 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, in perf_event_mmap()
9700 return event->attr.context_switch; in perf_event_switch_match()
9713 /* Only CPU-wide events are allowed to see next/prev pid/tid */ in perf_event_switch_output()
9714 if (event->ctx->task) { in perf_event_switch_output()
9715 se->event_id.header.type = PERF_RECORD_SWITCH; in perf_event_switch_output()
9716 se->event_id.header.size = sizeof(se->event_id.header); in perf_event_switch_output()
9718 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; in perf_event_switch_output()
9719 se->event_id.header.size = sizeof(se->event_id); in perf_event_switch_output()
9720 se->event_id.next_prev_pid = in perf_event_switch_output()
9721 perf_event_pid(event, se->next_prev); in perf_event_switch_output()
9722 se->event_id.next_prev_tid = in perf_event_switch_output()
9723 perf_event_tid(event, se->next_prev); in perf_event_switch_output()
9726 perf_event_header__init_id(&se->event_id.header, &sample, event); in perf_event_switch_output()
9728 ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); in perf_event_switch_output()
9732 if (event->ctx->task) in perf_event_switch_output()
9733 perf_output_put(&handle, se->event_id.header); in perf_event_switch_output()
9735 perf_output_put(&handle, se->event_id); in perf_event_switch_output()
9794 .stream_id = event->id, in perf_log_throttle()
9830 return event->attr.ksymbol; in perf_event_ksymbol_match()
9843 perf_event_header__init_id(&ksymbol_event->event_id.header, in perf_event_ksymbol_output()
9846 ksymbol_event->event_id.header.size); in perf_event_ksymbol_output()
9850 perf_output_put(&handle, ksymbol_event->event_id); in perf_event_ksymbol_output()
9851 __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len); in perf_event_ksymbol_output()
9920 return event->attr.bpf_event; in perf_event_bpf_match()
9933 perf_event_header__init_id(&bpf_event->event_id.header, in perf_event_bpf_output()
9936 bpf_event->event_id.header.size); in perf_event_bpf_output()
9940 perf_output_put(&handle, bpf_event->event_id); in perf_event_bpf_output()
9953 (u64)(unsigned long)prog->bpf_func, in perf_event_bpf_emit_ksymbols()
9954 prog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9955 prog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9957 for (i = 1; i < prog->aux->func_cnt; i++) { in perf_event_bpf_emit_ksymbols()
9958 struct bpf_prog *subprog = prog->aux->func[i]; in perf_event_bpf_emit_ksymbols()
9962 (u64)(unsigned long)subprog->bpf_func, in perf_event_bpf_emit_ksymbols()
9963 subprog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9964 subprog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9996 .id = prog->aux->id, in perf_event_bpf_event()
10002 memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE); in perf_event_bpf_event()
10022 return event->attr.text_poke; in perf_event_text_poke_match()
10036 perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); in perf_event_text_poke_output()
10039 text_poke_event->event_id.header.size); in perf_event_text_poke_output()
10043 perf_output_put(&handle, text_poke_event->event_id); in perf_event_text_poke_output()
10044 perf_output_put(&handle, text_poke_event->old_len); in perf_event_text_poke_output()
10045 perf_output_put(&handle, text_poke_event->new_len); in perf_event_text_poke_output()
10047 __output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len); in perf_event_text_poke_output()
10048 __output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len); in perf_event_text_poke_output()
10050 if (text_poke_event->pad) in perf_event_text_poke_output()
10051 __output_copy(&handle, &padding, text_poke_event->pad); in perf_event_text_poke_output()
10069 pad = ALIGN(tot, sizeof(u64)) - tot; in perf_event_text_poke()
10092 WRITE_ONCE(event->attach_state, event->attach_state | PERF_ATTACH_ITRACE); in perf_event_itrace_started()
10106 if (event->parent) in perf_log_itrace_start()
10107 event = event->parent; in perf_log_itrace_start()
10109 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || in perf_log_itrace_start()
10110 event->attach_state & PERF_ATTACH_ITRACE) in perf_log_itrace_start()
10141 if (event->parent) in perf_report_aux_output_id()
10142 event = event->parent; in perf_report_aux_output_id()
10165 struct hw_perf_event *hwc = &event->hw; in __perf_event_account_interrupt()
10170 if (seq != hwc->interrupts_seq) { in __perf_event_account_interrupt()
10171 hwc->interrupts_seq = seq; in __perf_event_account_interrupt()
10172 hwc->interrupts = 1; in __perf_event_account_interrupt()
10174 hwc->interrupts++; in __perf_event_account_interrupt()
10177 if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) { in __perf_event_account_interrupt()
10184 if (event->attr.freq) { in __perf_event_account_interrupt()
10186 s64 delta = now - hwc->freq_time_stamp; in __perf_event_account_interrupt()
10188 hwc->freq_time_stamp = now; in __perf_event_account_interrupt()
10191 perf_adjust_period(event, delta, hwc->last_period, true); in __perf_event_account_interrupt()
10209 if (event->attr.exclude_kernel && !user_mode(regs)) in sample_is_allowed()
10231 prog = READ_ONCE(event->prog); in bpf_overflow_handler()
10247 if (event->overflow_handler_context) in perf_event_set_bpf_handler()
10249 return -EINVAL; in perf_event_set_bpf_handler()
10251 if (event->prog) in perf_event_set_bpf_handler()
10252 return -EEXIST; in perf_event_set_bpf_handler()
10254 if (prog->type != BPF_PROG_TYPE_PERF_EVENT) in perf_event_set_bpf_handler()
10255 return -EINVAL; in perf_event_set_bpf_handler()
10257 if (event->attr.precise_ip && in perf_event_set_bpf_handler()
10258 prog->call_get_stack && in perf_event_set_bpf_handler()
10259 (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) || in perf_event_set_bpf_handler()
10260 event->attr.exclude_callchain_kernel || in perf_event_set_bpf_handler()
10261 event->attr.exclude_callchain_user)) { in perf_event_set_bpf_handler()
10271 return -EPROTO; in perf_event_set_bpf_handler()
10274 event->prog = prog; in perf_event_set_bpf_handler()
10275 event->bpf_cookie = bpf_cookie; in perf_event_set_bpf_handler()
10281 struct bpf_prog *prog = event->prog; in perf_event_free_bpf_handler()
10286 event->prog = NULL; in perf_event_free_bpf_handler()
10301 return -EOPNOTSUPP; in perf_event_set_bpf_handler()
10317 int events = atomic_read(&event->event_limit); in __perf_event_overflow()
10321 * Non-sampling counters might still use the PMI to fold short in __perf_event_overflow()
10329 if (event->attr.aux_pause) in __perf_event_overflow()
10330 perf_event_aux_pause(event->aux_event, true); in __perf_event_overflow()
10332 if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT && in __perf_event_overflow()
10341 event->pending_kill = POLL_IN; in __perf_event_overflow()
10342 if (events && atomic_dec_and_test(&event->event_limit)) { in __perf_event_overflow()
10344 event->pending_kill = POLL_HUP; in __perf_event_overflow()
10346 event->pmu->stop(event, 0); in __perf_event_overflow()
10349 if (event->attr.sigtrap) { in __perf_event_overflow()
10365 if (!event->pending_work && in __perf_event_overflow()
10366 !task_work_add(current, &event->pending_task, notify_mode)) { in __perf_event_overflow()
10367 event->pending_work = pending_id; in __perf_event_overflow()
10368 local_inc(&event->ctx->nr_no_switch_fast); in __perf_event_overflow()
10369 WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount)); in __perf_event_overflow()
10371 event->pending_addr = 0; in __perf_event_overflow()
10372 if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR)) in __perf_event_overflow()
10373 event->pending_addr = data->addr; in __perf_event_overflow()
10375 } else if (event->attr.exclude_kernel && valid_sample) { in __perf_event_overflow()
10383 * 2. Events that can overflow again before the IRQ- in __perf_event_overflow()
10386 * check 32-bit hash of the current IP. in __perf_event_overflow()
10388 WARN_ON_ONCE(event->pending_work != pending_id); in __perf_event_overflow()
10392 READ_ONCE(event->overflow_handler)(event, data, regs); in __perf_event_overflow()
10394 if (*perf_event_fasync(event) && event->pending_kill) { in __perf_event_overflow()
10395 event->pending_wakeup = 1; in __perf_event_overflow()
10396 irq_work_queue(&event->pending_irq); in __perf_event_overflow()
10399 if (event->attr.aux_resume) in __perf_event_overflow()
10400 perf_event_aux_pause(event->aux_event, false); in __perf_event_overflow()
10424 * We directly increment event->count and keep a second value in
10425 * event->hw.period_left to count intervals. This period event
10426 * is kept in the range [-sample_period, 0] so that we can use the
10432 struct hw_perf_event *hwc = &event->hw; in perf_swevent_set_period()
10433 u64 period = hwc->last_period; in perf_swevent_set_period()
10437 hwc->last_period = hwc->sample_period; in perf_swevent_set_period()
10439 old = local64_read(&hwc->period_left); in perf_swevent_set_period()
10447 val -= offset; in perf_swevent_set_period()
10448 } while (!local64_try_cmpxchg(&hwc->period_left, &old, val)); in perf_swevent_set_period()
10457 struct hw_perf_event *hwc = &event->hw; in perf_swevent_overflow()
10463 if (hwc->interrupts == MAX_INTERRUPTS) in perf_swevent_overflow()
10466 for (; overflow; overflow--) { in perf_swevent_overflow()
10471 * hwc->interrupts == MAX_INTERRUPTS. in perf_swevent_overflow()
10483 struct hw_perf_event *hwc = &event->hw; in perf_swevent_event()
10485 local64_add(nr, &event->count); in perf_swevent_event()
10493 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { in perf_swevent_event()
10494 data->period = nr; in perf_swevent_event()
10497 data->period = event->hw.last_period; in perf_swevent_event()
10499 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) in perf_swevent_event()
10502 if (local64_add_negative(nr, &hwc->period_left)) in perf_swevent_event()
10510 if (event->hw.state & PERF_HES_STOPPED) in perf_exclude_event()
10514 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
10517 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
10530 if (event->attr.type != type) in perf_swevent_match()
10533 if (event->attr.config != event_id) in perf_swevent_match()
10554 return &hlist->heads[hash]; in __find_swevent_head()
10563 hlist = rcu_dereference(swhash->swevent_hlist); in find_swevent_head_rcu()
10575 u32 event_id = event->attr.config; in find_swevent_head()
10576 u64 type = event->attr.type; in find_swevent_head()
10583 hlist = rcu_dereference_protected(swhash->swevent_hlist, in find_swevent_head()
10584 lockdep_is_held(&event->ctx->lock)); in find_swevent_head()
10617 return get_recursion_context(current->perf_recursion); in perf_swevent_get_recursion_context()
10623 put_recursion_context(current->perf_recursion, rctx); in perf_swevent_put_recursion_context()
10660 struct hw_perf_event *hwc = &event->hw; in perf_swevent_add()
10664 hwc->last_period = hwc->sample_period; in perf_swevent_add()
10668 hwc->state = !(flags & PERF_EF_START); in perf_swevent_add()
10672 return -EINVAL; in perf_swevent_add()
10674 hlist_add_head_rcu(&event->hlist_entry, head); in perf_swevent_add()
10682 hlist_del_rcu(&event->hlist_entry); in perf_swevent_del()
10687 event->hw.state = 0; in perf_swevent_start()
10692 event->hw.state = PERF_HES_STOPPED; in perf_swevent_stop()
10699 return rcu_dereference_protected(swhash->swevent_hlist, in swevent_hlist_deref()
10700 lockdep_is_held(&swhash->hlist_mutex)); in swevent_hlist_deref()
10710 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); in swevent_hlist_release()
10718 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
10720 if (!--swhash->hlist_refcount) in swevent_hlist_put_cpu()
10723 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
10739 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
10746 err = -ENOMEM; in swevent_hlist_get_cpu()
10749 rcu_assign_pointer(swhash->swevent_hlist, hlist); in swevent_hlist_get_cpu()
10751 swhash->hlist_refcount++; in swevent_hlist_get_cpu()
10753 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
10786 u64 event_id = event->attr.config; in sw_perf_event_destroy()
10788 WARN_ON(event->parent); in sw_perf_event_destroy()
10799 u64 event_id = event->attr.config; in perf_swevent_init()
10801 if (event->attr.type != PERF_TYPE_SOFTWARE) in perf_swevent_init()
10802 return -ENOENT; in perf_swevent_init()
10808 return -EOPNOTSUPP; in perf_swevent_init()
10812 event->attr.type = perf_cpu_clock.type; in perf_swevent_init()
10813 return -ENOENT; in perf_swevent_init()
10815 event->attr.type = perf_task_clock.type; in perf_swevent_init()
10816 return -ENOENT; in perf_swevent_init()
10823 return -ENOENT; in perf_swevent_init()
10825 if (!event->parent) { in perf_swevent_init()
10833 event->destroy = sw_perf_event_destroy; in perf_swevent_init()
10863 if (event->attr.type != PERF_TYPE_TRACEPOINT) in perf_tp_event_init()
10864 return -ENOENT; in perf_tp_event_init()
10870 return -EOPNOTSUPP; in perf_tp_event_init()
10876 event->destroy = tp_perf_event_destroy; in perf_tp_event_init()
10895 void *record = raw->frag.data; in perf_tp_filter_match()
10898 if (event->parent) in perf_tp_filter_match()
10899 event = event->parent; in perf_tp_filter_match()
10901 if (likely(!event->filter) || filter_match_preds(event->filter, record)) in perf_tp_filter_match()
10910 if (event->hw.state & PERF_HES_STOPPED) in perf_tp_event_match()
10913 * If exclude_kernel, only trace user-space tracepoints (uprobes) in perf_tp_event_match()
10915 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_tp_event_match()
10936 perf_tp_event(call->event.type, count, raw_data, size, regs, head, in perf_trace_run_bpf_submit()
10949 if (event->attr.config != entry->type) in __perf_tp_event_target_task()
10952 if (event->attr.sigtrap) in __perf_tp_event_target_task()
10971 perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) { in perf_tp_event_target_task()
10977 perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) { in perf_tp_event_target_task()
11003 * Here use the same on-stack perf_sample_data, in perf_tp_event()
11004 * some members in data are event-specific and in perf_tp_event()
11005 * need to be re-computed for different sweveents. in perf_tp_event()
11006 * Re-initialize data->sample_flags safely to avoid in perf_tp_event()
11008 * because data->sample_flags is set. in perf_tp_event()
11024 ctx = rcu_dereference(task->perf_event_ctxp); in perf_tp_event()
11028 raw_spin_lock(&ctx->lock); in perf_tp_event()
11030 raw_spin_unlock(&ctx->lock); in perf_tp_event()
11057 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
11096 if (event->attr.type != perf_kprobe.type) in perf_kprobe_event_init()
11097 return -ENOENT; in perf_kprobe_event_init()
11100 return -EACCES; in perf_kprobe_event_init()
11106 return -EOPNOTSUPP; in perf_kprobe_event_init()
11108 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_kprobe_event_init()
11113 event->destroy = perf_kprobe_destroy; in perf_kprobe_event_init()
11120 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
11156 if (event->attr.type != perf_uprobe.type) in perf_uprobe_event_init()
11157 return -ENOENT; in perf_uprobe_event_init()
11160 return -EACCES; in perf_uprobe_event_init()
11166 return -EOPNOTSUPP; in perf_uprobe_event_init()
11168 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_uprobe_event_init()
11169 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT; in perf_uprobe_event_init()
11174 event->destroy = perf_uprobe_destroy; in perf_uprobe_event_init()
11184 perf_pmu_register(&perf_kprobe, "kprobe", -1); in perf_tp_register()
11187 perf_pmu_register(&perf_uprobe, "uprobe", -1); in perf_tp_register()
11202 if (event->pmu == &perf_tracepoint) in perf_event_is_tracing()
11205 if (event->pmu == &perf_kprobe) in perf_event_is_tracing()
11209 if (event->pmu == &perf_uprobe) in perf_event_is_tracing()
11221 if (event->state <= PERF_EVENT_STATE_REVOKED) in __perf_event_set_bpf_prog()
11222 return -ENODEV; in __perf_event_set_bpf_prog()
11227 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE; in __perf_event_set_bpf_prog()
11228 is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE; in __perf_event_set_bpf_prog()
11229 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; in __perf_event_set_bpf_prog()
11230 is_syscall_tp = is_syscall_trace_event(event->tp_event); in __perf_event_set_bpf_prog()
11233 return -EINVAL; in __perf_event_set_bpf_prog()
11235 if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) || in __perf_event_set_bpf_prog()
11236 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) || in __perf_event_set_bpf_prog()
11237 (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) in __perf_event_set_bpf_prog()
11238 return -EINVAL; in __perf_event_set_bpf_prog()
11240 if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe) in __perf_event_set_bpf_prog()
11242 return -EINVAL; in __perf_event_set_bpf_prog()
11245 if (prog->kprobe_override && !is_kprobe) in __perf_event_set_bpf_prog()
11246 return -EINVAL; in __perf_event_set_bpf_prog()
11249 if (prog->aux->kprobe_write_ctx && !is_uprobe) in __perf_event_set_bpf_prog()
11250 return -EINVAL; in __perf_event_set_bpf_prog()
11253 int off = trace_event_get_offsets(event->tp_event); in __perf_event_set_bpf_prog()
11255 if (prog->aux->max_ctx_offset > off) in __perf_event_set_bpf_prog()
11256 return -EACCES; in __perf_event_set_bpf_prog()
11278 if (!event->prog) in perf_event_free_bpf_prog()
11302 return -ENOENT; in __perf_event_set_bpf_prog()
11309 return -ENOENT; in perf_event_set_bpf_prog()
11323 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); in perf_bp_event()
11325 if (!bp->hw.state && !perf_exclude_event(bp, regs)) in perf_bp_event()
11336 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); in perf_addr_filter_new()
11343 INIT_LIST_HEAD(&filter->entry); in perf_addr_filter_new()
11344 list_add_tail(&filter->entry, filters); in perf_addr_filter_new()
11354 path_put(&filter->path); in free_filters_list()
11355 list_del(&filter->entry); in free_filters_list()
11373 if (event->parent) in perf_addr_filters_splice()
11376 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
11378 list_splice_init(&event->addr_filters.list, &list); in perf_addr_filters_splice()
11380 list_splice(head, &event->addr_filters.list); in perf_addr_filters_splice()
11382 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
11392 if (list_empty(&event->addr_filters.list)) in perf_free_addr_filters()
11411 if (!vma->vm_file) in perf_addr_filter_apply()
11426 struct task_struct *task = READ_ONCE(event->ctx->task); in perf_event_addr_filters_apply()
11433 * We may observe TASK_TOMBSTONE, which means that the event tear-down in perf_event_addr_filters_apply()
11439 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
11447 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_apply()
11448 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_apply()
11449 if (filter->path.dentry) { in perf_event_addr_filters_apply()
11454 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_apply()
11455 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_apply()
11457 perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]); in perf_event_addr_filters_apply()
11459 event->addr_filter_ranges[count].start = filter->offset; in perf_event_addr_filters_apply()
11460 event->addr_filter_ranges[count].size = filter->size; in perf_event_addr_filters_apply()
11466 event->addr_filters_gen++; in perf_event_addr_filters_apply()
11467 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_apply()
11469 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
11499 IF_ACT_NONE = -1,
11538 int ret = -EINVAL; in perf_event_parse_addr_filter()
11542 return -ENOMEM; in perf_event_parse_addr_filter()
11550 ret = -EINVAL; in perf_event_parse_addr_filter()
11570 filter->action = actions[token]; in perf_event_parse_addr_filter()
11585 ret = kstrtoul(args[0].from, 0, &filter->offset); in perf_event_parse_addr_filter()
11591 ret = kstrtoul(args[1].from, 0, &filter->size); in perf_event_parse_addr_filter()
11602 ret = -ENOMEM; in perf_event_parse_addr_filter()
11620 ret = -EINVAL; in perf_event_parse_addr_filter()
11623 * ACTION "filter" must have a non-zero length region in perf_event_parse_addr_filter()
11626 if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER && in perf_event_parse_addr_filter()
11627 !filter->size) in perf_event_parse_addr_filter()
11635 * For now, we only support file-based filters in perf_event_parse_addr_filter()
11636 * in per-task events; doing so for CPU-wide in perf_event_parse_addr_filter()
11642 ret = -EOPNOTSUPP; in perf_event_parse_addr_filter()
11643 if (!event->ctx->task) in perf_event_parse_addr_filter()
11648 &filter->path); in perf_event_parse_addr_filter()
11652 ret = -EINVAL; in perf_event_parse_addr_filter()
11653 if (!filter->path.dentry || in perf_event_parse_addr_filter()
11654 !S_ISREG(d_inode(filter->path.dentry) in perf_event_parse_addr_filter()
11655 ->i_mode)) in perf_event_parse_addr_filter()
11658 event->addr_filters.nr_file_filters++; in perf_event_parse_addr_filter()
11696 lockdep_assert_held(&event->ctx->mutex); in perf_event_set_addr_filter()
11698 if (WARN_ON_ONCE(event->parent)) in perf_event_set_addr_filter()
11699 return -EINVAL; in perf_event_set_addr_filter()
11705 ret = event->pmu->addr_filters_validate(&filters); in perf_event_set_addr_filter()
11721 event->addr_filters.nr_file_filters = 0; in perf_event_set_addr_filter()
11728 int ret = -EINVAL; in perf_event_set_filter()
11737 struct perf_event_context *ctx = event->ctx; in perf_event_set_filter()
11742 * the tracepoint muck will deadlock against ctx->mutex, but in perf_event_set_filter()
11744 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we in perf_event_set_filter()
11750 mutex_unlock(&ctx->mutex); in perf_event_set_filter()
11751 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); in perf_event_set_filter()
11752 mutex_lock(&ctx->mutex); in perf_event_set_filter()
11776 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_swevent_hrtimer()
11779 event->pmu->read(event); in perf_swevent_hrtimer()
11781 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_swevent_hrtimer()
11785 if (!(event->attr.exclude_idle && is_idle_task(current))) in perf_swevent_hrtimer()
11790 period = max_t(u64, 10000, event->hw.sample_period); in perf_swevent_hrtimer()
11798 struct hw_perf_event *hwc = &event->hw; in perf_swevent_start_hrtimer()
11804 period = local64_read(&hwc->period_left); in perf_swevent_start_hrtimer()
11809 local64_set(&hwc->period_left, 0); in perf_swevent_start_hrtimer()
11811 period = max_t(u64, 10000, hwc->sample_period); in perf_swevent_start_hrtimer()
11813 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), in perf_swevent_start_hrtimer()
11819 struct hw_perf_event *hwc = &event->hw; in perf_swevent_cancel_hrtimer()
11826 if (is_sampling_event(event) && (hwc->interrupts != MAX_INTERRUPTS)) { in perf_swevent_cancel_hrtimer()
11827 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
11828 local64_set(&hwc->period_left, ktime_to_ns(remaining)); in perf_swevent_cancel_hrtimer()
11830 hrtimer_cancel(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
11836 struct hw_perf_event *hwc = &event->hw; in perf_swevent_init_hrtimer()
11841 hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in perf_swevent_init_hrtimer()
11844 * Since hrtimers have a fixed rate, we can do a static freq->period in perf_swevent_init_hrtimer()
11847 if (event->attr.freq) { in perf_swevent_init_hrtimer()
11848 long freq = event->attr.sample_freq; in perf_swevent_init_hrtimer()
11850 event->attr.sample_period = NSEC_PER_SEC / freq; in perf_swevent_init_hrtimer()
11851 hwc->sample_period = event->attr.sample_period; in perf_swevent_init_hrtimer()
11852 local64_set(&hwc->period_left, hwc->sample_period); in perf_swevent_init_hrtimer()
11853 hwc->last_period = hwc->sample_period; in perf_swevent_init_hrtimer()
11854 event->attr.freq = 0; in perf_swevent_init_hrtimer()
11868 prev = local64_xchg(&event->hw.prev_count, now); in cpu_clock_event_update()
11869 local64_add(now - prev, &event->count); in cpu_clock_event_update()
11874 local64_set(&event->hw.prev_count, local_clock()); in cpu_clock_event_start()
11906 if (event->attr.type != perf_cpu_clock.type) in cpu_clock_event_init()
11907 return -ENOENT; in cpu_clock_event_init()
11909 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) in cpu_clock_event_init()
11910 return -ENOENT; in cpu_clock_event_init()
11916 return -EOPNOTSUPP; in cpu_clock_event_init()
11946 prev = local64_xchg(&event->hw.prev_count, now); in task_clock_event_update()
11947 delta = now - prev; in task_clock_event_update()
11948 local64_add(delta, &event->count); in task_clock_event_update()
11953 local64_set(&event->hw.prev_count, event->ctx->time); in task_clock_event_start()
11961 task_clock_event_update(event, event->ctx->time); in task_clock_event_stop()
11981 u64 delta = now - event->ctx->timestamp; in task_clock_event_read()
11982 u64 time = event->ctx->time + delta; in task_clock_event_read()
11989 if (event->attr.type != perf_task_clock.type) in task_clock_event_init()
11990 return -ENOENT; in task_clock_event_init()
11992 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) in task_clock_event_init()
11993 return -ENOENT; in task_clock_event_init()
11999 return -EOPNOTSUPP; in task_clock_event_init()
12089 return sysfs_emit(page, "%d\n", pmu->nr_addr_filters); in nr_addr_filters_show()
12100 return sysfs_emit(page, "%d\n", pmu->type); in type_show()
12111 return sysfs_emit(page, "%d\n", pmu->hrtimer_interval_ms); in perf_event_mux_interval_ms_show()
12129 return -EINVAL; in perf_event_mux_interval_ms_store()
12132 if (timer == pmu->hrtimer_interval_ms) in perf_event_mux_interval_ms_store()
12136 pmu->hrtimer_interval_ms = timer; in perf_event_mux_interval_ms_store()
12142 cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu); in perf_event_mux_interval_ms_store()
12143 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); in perf_event_mux_interval_ms_store()
12194 struct cpumask *mask = perf_scope_cpumask(pmu->scope); in cpumask_show()
12216 if (n == 2 && !pmu->nr_addr_filters) in pmu_dev_is_visible()
12220 if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE) in pmu_dev_is_visible()
12223 return a->mode; in pmu_dev_is_visible()
12249 int ret = -ENOMEM; in pmu_dev_alloc()
12251 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); in pmu_dev_alloc()
12252 if (!pmu->dev) in pmu_dev_alloc()
12255 pmu->dev->groups = pmu->attr_groups; in pmu_dev_alloc()
12256 device_initialize(pmu->dev); in pmu_dev_alloc()
12258 dev_set_drvdata(pmu->dev, pmu); in pmu_dev_alloc()
12259 pmu->dev->bus = &pmu_bus; in pmu_dev_alloc()
12260 pmu->dev->parent = pmu->parent; in pmu_dev_alloc()
12261 pmu->dev->release = pmu_dev_release; in pmu_dev_alloc()
12263 ret = dev_set_name(pmu->dev, "%s", pmu->name); in pmu_dev_alloc()
12267 ret = device_add(pmu->dev); in pmu_dev_alloc()
12271 if (pmu->attr_update) { in pmu_dev_alloc()
12272 ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update); in pmu_dev_alloc()
12281 device_del(pmu->dev); in pmu_dev_alloc()
12284 put_device(pmu->dev); in pmu_dev_alloc()
12285 pmu->dev = NULL; in pmu_dev_alloc()
12309 if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) { in perf_pmu_free()
12310 if (pmu->nr_addr_filters) in perf_pmu_free()
12311 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); in perf_pmu_free()
12312 device_del(pmu->dev); in perf_pmu_free()
12313 put_device(pmu->dev); in perf_pmu_free()
12316 if (pmu->cpu_pmu_context) { in perf_pmu_free()
12322 cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu); in perf_pmu_free()
12325 if (cpc->epc.embedded) { in perf_pmu_free()
12327 put_pmu_ctx(&cpc->epc); in perf_pmu_free()
12332 free_percpu(pmu->cpu_pmu_context); in perf_pmu_free()
12346 return -EINVAL; in DEFINE_FREE()
12348 if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE, in DEFINE_FREE()
12350 return -EINVAL; in DEFINE_FREE()
12352 pmu->name = name; in DEFINE_FREE()
12363 pmu->type = pmu_type.id; in DEFINE_FREE()
12364 atomic_set(&pmu->exclusive_cnt, 0); in DEFINE_FREE()
12366 if (pmu_bus_running && !pmu->dev) { in DEFINE_FREE()
12372 pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context *); in DEFINE_FREE()
12373 if (!pmu->cpu_pmu_context) in DEFINE_FREE()
12374 return -ENOMEM; in DEFINE_FREE()
12383 return -ENOMEM; in DEFINE_FREE()
12385 *per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc; in DEFINE_FREE()
12386 __perf_init_event_pmu_context(&cpc->epc, pmu); in DEFINE_FREE()
12390 if (!pmu->start_txn) { in DEFINE_FREE()
12391 if (pmu->pmu_enable) { in DEFINE_FREE()
12397 pmu->start_txn = perf_pmu_start_txn; in DEFINE_FREE()
12398 pmu->commit_txn = perf_pmu_commit_txn; in DEFINE_FREE()
12399 pmu->cancel_txn = perf_pmu_cancel_txn; in DEFINE_FREE()
12401 pmu->start_txn = perf_pmu_nop_txn; in DEFINE_FREE()
12402 pmu->commit_txn = perf_pmu_nop_int; in DEFINE_FREE()
12403 pmu->cancel_txn = perf_pmu_nop_void; in DEFINE_FREE()
12407 if (!pmu->pmu_enable) { in DEFINE_FREE()
12408 pmu->pmu_enable = perf_pmu_nop_void; in DEFINE_FREE()
12409 pmu->pmu_disable = perf_pmu_nop_void; in DEFINE_FREE()
12412 if (!pmu->check_period) in DEFINE_FREE()
12413 pmu->check_period = perf_event_nop_int; in DEFINE_FREE()
12415 if (!pmu->event_idx) in DEFINE_FREE()
12416 pmu->event_idx = perf_event_idx_default; in DEFINE_FREE()
12418 INIT_LIST_HEAD(&pmu->events); in DEFINE_FREE()
12419 spin_lock_init(&pmu->events_lock); in DEFINE_FREE()
12424 if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu)) in DEFINE_FREE()
12425 return -EINVAL; in DEFINE_FREE()
12426 list_add_rcu(&pmu->entry, &pmus); in DEFINE_FREE()
12438 * De-schedule the event and mark it REVOKED. in __pmu_detach_event()
12443 * All _free_event() bits that rely on event->pmu: in __pmu_detach_event()
12447 scoped_guard (mutex, &event->mmap_mutex) { in __pmu_detach_event()
12448 WARN_ON_ONCE(pmu->event_unmapped); in __pmu_detach_event()
12458 if (event->destroy) { in __pmu_detach_event()
12459 event->destroy(event); in __pmu_detach_event()
12460 event->destroy = NULL; in __pmu_detach_event()
12463 if (event->pmu_ctx) { in __pmu_detach_event()
12464 put_pmu_ctx(event->pmu_ctx); in __pmu_detach_event()
12465 event->pmu_ctx = NULL; in __pmu_detach_event()
12469 module_put(pmu->module); in __pmu_detach_event()
12471 event->pmu = NULL; /* force fault instead of UAF */ in __pmu_detach_event()
12482 scoped_guard (spinlock, &pmu->events_lock) in pmu_detach_event()
12483 list_del(&event->pmu_list); in pmu_detach_event()
12490 guard(spinlock)(&pmu->events_lock); in pmu_get_event()
12491 list_for_each_entry(event, &pmu->events, pmu_list) { in pmu_get_event()
12492 if (atomic_long_inc_not_zero(&event->refcount)) in pmu_get_event()
12501 guard(spinlock)(&pmu->events_lock); in pmu_empty()
12502 return list_empty(&pmu->events); in pmu_empty()
12527 if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL)) in perf_pmu_unregister()
12528 return -EINVAL; in perf_pmu_unregister()
12530 list_del_rcu(&pmu->entry); in perf_pmu_unregister()
12546 if (pmu->event_unmapped && !pmu_empty(pmu)) { in perf_pmu_unregister()
12552 idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu); in perf_pmu_unregister()
12553 list_add_rcu(&pmu->entry, &pmus); in perf_pmu_unregister()
12554 return -EBUSY; in perf_pmu_unregister()
12558 idr_remove(&pmu_idr, pmu->type); in perf_pmu_unregister()
12576 return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || in has_extended_regs()
12577 (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); in has_extended_regs()
12585 if (!try_module_get(pmu->module)) in perf_try_init_event()
12586 return -ENODEV; in perf_try_init_event()
12589 * A number of pmu->event_init() methods iterate the sibling_list to, in perf_try_init_event()
12591 * if this is a sibling event, acquire the ctx->mutex to protect in perf_try_init_event()
12594 if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) { in perf_try_init_event()
12596 * This ctx->mutex can nest when we're called through in perf_try_init_event()
12599 ctx = perf_event_ctx_lock_nested(event->group_leader, in perf_try_init_event()
12604 event->pmu = pmu; in perf_try_init_event()
12605 ret = pmu->event_init(event); in perf_try_init_event()
12608 perf_event_ctx_unlock(event->group_leader, ctx); in perf_try_init_event()
12613 if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && in perf_try_init_event()
12615 ret = -EOPNOTSUPP; in perf_try_init_event()
12619 if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE && in perf_try_init_event()
12621 ret = -EINVAL; in perf_try_init_event()
12625 if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) { in perf_try_init_event()
12630 cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu); in perf_try_init_event()
12631 pmu_cpumask = perf_scope_cpumask(pmu->scope); in perf_try_init_event()
12633 ret = -ENODEV; in perf_try_init_event()
12641 event->event_caps |= PERF_EV_CAP_READ_SCOPE; in perf_try_init_event()
12647 if (event->destroy) { in perf_try_init_event()
12648 event->destroy(event); in perf_try_init_event()
12649 event->destroy = NULL; in perf_try_init_event()
12653 event->pmu = NULL; in perf_try_init_event()
12654 module_put(pmu->module); in perf_try_init_event()
12667 * Save original type before calling pmu->event_init() since certain in perf_init_event()
12668 * pmus overwrites event->attr.type to forward event to another pmu. in perf_init_event()
12670 event->orig_type = event->attr.type; in perf_init_event()
12673 if (event->parent && event->parent->pmu) { in perf_init_event()
12674 pmu = event->parent->pmu; in perf_init_event()
12684 type = event->attr.type; in perf_init_event()
12686 type = event->attr.config >> PERF_PMU_TYPE_SHIFT; in perf_init_event()
12691 event->attr.config &= PERF_HW_EVENT_MASK; in perf_init_event()
12699 if (event->attr.type != type && type != PERF_TYPE_RAW && in perf_init_event()
12700 !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)) in perf_init_event()
12701 return ERR_PTR(-ENOENT); in perf_init_event()
12704 if (ret == -ENOENT && event->attr.type != type && !extended_type) { in perf_init_event()
12705 type = event->attr.type; in perf_init_event()
12720 if (ret != -ENOENT) in perf_init_event()
12724 return ERR_PTR(-ENOENT); in perf_init_event()
12729 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in attach_sb_event()
12731 raw_spin_lock(&pel->lock); in attach_sb_event()
12732 list_add_rcu(&event->sb_list, &pel->list); in attach_sb_event()
12733 raw_spin_unlock(&pel->lock); in attach_sb_event()
12737 * We keep a list of all !task (and therefore per-cpu) events
12738 * that need to receive side-band records.
12740 * This avoids having to scan all the various PMU per-cpu contexts
12774 if (event->parent) in account_event()
12777 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in account_event()
12779 if (event->attr.mmap || event->attr.mmap_data) in account_event()
12781 if (event->attr.build_id) in account_event()
12783 if (event->attr.comm) in account_event()
12785 if (event->attr.namespaces) in account_event()
12787 if (event->attr.cgroup) in account_event()
12789 if (event->attr.task) in account_event()
12791 if (event->attr.freq) in account_event()
12793 if (event->attr.context_switch) { in account_event()
12801 if (event->attr.ksymbol) in account_event()
12803 if (event->attr.bpf_event) in account_event()
12805 if (event->attr.text_poke) in account_event()
12829 * increments to by-pass the mutex. in account_event()
12852 long err = -EINVAL; in perf_event_alloc()
12856 if (!task || cpu != -1) in perf_event_alloc()
12857 return ERR_PTR(-EINVAL); in perf_event_alloc()
12859 if (attr->sigtrap && !task) { in perf_event_alloc()
12861 return ERR_PTR(-EINVAL); in perf_event_alloc()
12864 node = (cpu >= 0) ? cpu_to_node(cpu) : -1; in perf_event_alloc()
12868 return ERR_PTR(-ENOMEM); in perf_event_alloc()
12877 mutex_init(&event->child_mutex); in perf_event_alloc()
12878 INIT_LIST_HEAD(&event->child_list); in perf_event_alloc()
12880 INIT_LIST_HEAD(&event->event_entry); in perf_event_alloc()
12881 INIT_LIST_HEAD(&event->sibling_list); in perf_event_alloc()
12882 INIT_LIST_HEAD(&event->active_list); in perf_event_alloc()
12884 INIT_LIST_HEAD(&event->rb_entry); in perf_event_alloc()
12885 INIT_LIST_HEAD(&event->active_entry); in perf_event_alloc()
12886 INIT_LIST_HEAD(&event->addr_filters.list); in perf_event_alloc()
12887 INIT_HLIST_NODE(&event->hlist_entry); in perf_event_alloc()
12888 INIT_LIST_HEAD(&event->pmu_list); in perf_event_alloc()
12891 init_waitqueue_head(&event->waitq); in perf_event_alloc()
12892 init_irq_work(&event->pending_irq, perf_pending_irq); in perf_event_alloc()
12893 event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable); in perf_event_alloc()
12894 init_task_work(&event->pending_task, perf_pending_task); in perf_event_alloc()
12896 mutex_init(&event->mmap_mutex); in perf_event_alloc()
12897 raw_spin_lock_init(&event->addr_filters.lock); in perf_event_alloc()
12899 atomic_long_set(&event->refcount, 1); in perf_event_alloc()
12900 event->cpu = cpu; in perf_event_alloc()
12901 event->attr = *attr; in perf_event_alloc()
12902 event->group_leader = group_leader; in perf_event_alloc()
12903 event->pmu = NULL; in perf_event_alloc()
12904 event->oncpu = -1; in perf_event_alloc()
12906 event->parent = parent_event; in perf_event_alloc()
12908 event->ns = get_pid_ns(task_active_pid_ns(current)); in perf_event_alloc()
12909 event->id = atomic64_inc_return(&perf_event_id); in perf_event_alloc()
12911 event->state = PERF_EVENT_STATE_INACTIVE; in perf_event_alloc()
12914 event->event_caps = parent_event->event_caps; in perf_event_alloc()
12917 event->attach_state = PERF_ATTACH_TASK; in perf_event_alloc()
12923 event->hw.target = get_task_struct(task); in perf_event_alloc()
12926 event->clock = &local_clock; in perf_event_alloc()
12928 event->clock = parent_event->clock; in perf_event_alloc()
12931 overflow_handler = parent_event->overflow_handler; in perf_event_alloc()
12932 context = parent_event->overflow_handler_context; in perf_event_alloc()
12934 if (parent_event->prog) { in perf_event_alloc()
12935 struct bpf_prog *prog = parent_event->prog; in perf_event_alloc()
12938 event->prog = prog; in perf_event_alloc()
12944 event->overflow_handler = overflow_handler; in perf_event_alloc()
12945 event->overflow_handler_context = context; in perf_event_alloc()
12947 event->overflow_handler = perf_event_output_backward; in perf_event_alloc()
12948 event->overflow_handler_context = NULL; in perf_event_alloc()
12950 event->overflow_handler = perf_event_output_forward; in perf_event_alloc()
12951 event->overflow_handler_context = NULL; in perf_event_alloc()
12958 hwc = &event->hw; in perf_event_alloc()
12959 hwc->sample_period = attr->sample_period; in perf_event_alloc()
12961 hwc->sample_period = 1; in perf_event_alloc()
12962 hwc->last_period = hwc->sample_period; in perf_event_alloc()
12964 local64_set(&hwc->period_left, hwc->sample_period); in perf_event_alloc()
12969 * collect per-thread samples. in perf_event_alloc()
12972 if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID)) in perf_event_alloc()
12973 return ERR_PTR(-EINVAL); in perf_event_alloc()
12976 event->attr.branch_sample_type = 0; in perf_event_alloc()
12983 * The PERF_ATTACH_TASK_DATA is set in the event_init()->hw_config(). in perf_event_alloc()
12985 * Otherwise, the __free_event() would mistakenly detach the non-exist in perf_event_alloc()
12988 if (event->attach_state & PERF_ATTACH_TASK_DATA) { in perf_event_alloc()
12995 * Disallow uncore-task events. Similarly, disallow uncore-cgroup in perf_event_alloc()
12999 if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) in perf_event_alloc()
13000 return ERR_PTR(-EINVAL); in perf_event_alloc()
13002 if (event->attr.aux_output && in perf_event_alloc()
13003 (!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) || in perf_event_alloc()
13004 event->attr.aux_pause || event->attr.aux_resume)) in perf_event_alloc()
13005 return ERR_PTR(-EOPNOTSUPP); in perf_event_alloc()
13007 if (event->attr.aux_pause && event->attr.aux_resume) in perf_event_alloc()
13008 return ERR_PTR(-EINVAL); in perf_event_alloc()
13010 if (event->attr.aux_start_paused) { in perf_event_alloc()
13011 if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) in perf_event_alloc()
13012 return ERR_PTR(-EOPNOTSUPP); in perf_event_alloc()
13013 event->hw.aux_paused = 1; in perf_event_alloc()
13016 if (cgroup_fd != -1) { in perf_event_alloc()
13027 event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters, in perf_event_alloc()
13030 if (!event->addr_filter_ranges) in perf_event_alloc()
13031 return ERR_PTR(-ENOMEM); in perf_event_alloc()
13037 if (event->parent) { in perf_event_alloc()
13040 raw_spin_lock_irq(&ifh->lock); in perf_event_alloc()
13041 memcpy(event->addr_filter_ranges, in perf_event_alloc()
13042 event->parent->addr_filter_ranges, in perf_event_alloc()
13043 pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range)); in perf_event_alloc()
13044 raw_spin_unlock_irq(&ifh->lock); in perf_event_alloc()
13048 event->addr_filters_gen = 1; in perf_event_alloc()
13051 if (!event->parent) { in perf_event_alloc()
13052 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { in perf_event_alloc()
13053 err = get_callchain_buffers(attr->sample_max_stack); in perf_event_alloc()
13056 event->attach_state |= PERF_ATTACH_CALLCHAIN; in perf_event_alloc()
13071 scoped_guard (spinlock, &pmu->events_lock) in perf_event_alloc()
13072 list_add(&event->pmu_list, &pmu->events); in perf_event_alloc()
13086 ret = get_user(size, &uattr->size); in perf_copy_attr()
13098 if (ret == -E2BIG) in perf_copy_attr()
13103 attr->size = size; in perf_copy_attr()
13105 if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) in perf_copy_attr()
13106 return -EINVAL; in perf_copy_attr()
13108 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) in perf_copy_attr()
13109 return -EINVAL; in perf_copy_attr()
13111 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) in perf_copy_attr()
13112 return -EINVAL; in perf_copy_attr()
13114 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { in perf_copy_attr()
13115 u64 mask = attr->branch_sample_type; in perf_copy_attr()
13118 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) in perf_copy_attr()
13119 return -EINVAL; in perf_copy_attr()
13123 return -EINVAL; in perf_copy_attr()
13129 if (!attr->exclude_kernel) in perf_copy_attr()
13132 if (!attr->exclude_user) in perf_copy_attr()
13135 if (!attr->exclude_hv) in perf_copy_attr()
13140 attr->branch_sample_type = mask; in perf_copy_attr()
13150 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { in perf_copy_attr()
13151 ret = perf_reg_validate(attr->sample_regs_user); in perf_copy_attr()
13156 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { in perf_copy_attr()
13158 return -ENOSYS; in perf_copy_attr()
13165 if (attr->sample_stack_user >= USHRT_MAX) in perf_copy_attr()
13166 return -EINVAL; in perf_copy_attr()
13167 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) in perf_copy_attr()
13168 return -EINVAL; in perf_copy_attr()
13171 if (!attr->sample_max_stack) in perf_copy_attr()
13172 attr->sample_max_stack = sysctl_perf_event_max_stack; in perf_copy_attr()
13174 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) in perf_copy_attr()
13175 ret = perf_reg_validate(attr->sample_regs_intr); in perf_copy_attr()
13178 if (attr->sample_type & PERF_SAMPLE_CGROUP) in perf_copy_attr()
13179 return -EINVAL; in perf_copy_attr()
13181 if ((attr->sample_type & PERF_SAMPLE_WEIGHT) && in perf_copy_attr()
13182 (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) in perf_copy_attr()
13183 return -EINVAL; in perf_copy_attr()
13185 if (!attr->inherit && attr->inherit_thread) in perf_copy_attr()
13186 return -EINVAL; in perf_copy_attr()
13188 if (attr->remove_on_exec && attr->enable_on_exec) in perf_copy_attr()
13189 return -EINVAL; in perf_copy_attr()
13191 if (attr->sigtrap && !attr->remove_on_exec) in perf_copy_attr()
13192 return -EINVAL; in perf_copy_attr()
13198 put_user(sizeof(*attr), &uattr->size); in perf_copy_attr()
13199 ret = -E2BIG; in perf_copy_attr()
13216 int ret = -EINVAL; in perf_event_set_output()
13219 mutex_lock(&event->mmap_mutex); in perf_event_set_output()
13228 * Don't allow cross-cpu buffers in perf_event_set_output()
13230 if (output_event->cpu != event->cpu) in perf_event_set_output()
13234 * If its not a per-cpu rb, it must be the same task. in perf_event_set_output()
13236 if (output_event->cpu == -1 && output_event->hw.target != event->hw.target) in perf_event_set_output()
13242 if (output_event->clock != event->clock) in perf_event_set_output()
13256 event->pmu != output_event->pmu) in perf_event_set_output()
13260 * Hold both mmap_mutex to serialize against perf_mmap_close(). Since in perf_event_set_output()
13261 * output_event is already on rb->event_list, and the list iteration in perf_event_set_output()
13264 * observe !rb->mmap_count. in perf_event_set_output()
13266 mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex); in perf_event_set_output()
13269 if (refcount_read(&event->mmap_count)) in perf_event_set_output()
13273 if (output_event->state <= PERF_EVENT_STATE_REVOKED) in perf_event_set_output()
13282 if (!refcount_read(&rb->mmap_count)) { in perf_event_set_output()
13292 mutex_unlock(&event->mmap_mutex); in perf_event_set_output()
13294 mutex_unlock(&output_event->mmap_mutex); in perf_event_set_output()
13306 event->clock = &ktime_get_mono_fast_ns; in perf_event_set_clock()
13311 event->clock = &ktime_get_raw_fast_ns; in perf_event_set_clock()
13316 event->clock = &ktime_get_real_ns; in perf_event_set_clock()
13320 event->clock = &ktime_get_boottime_ns; in perf_event_set_clock()
13324 event->clock = &ktime_get_clocktai_ns; in perf_event_set_clock()
13328 return -EINVAL; in perf_event_set_clock()
13331 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) in perf_event_set_clock()
13332 return -EINVAL; in perf_event_set_clock()
13343 if (attr->sigtrap) { in perf_check_permission()
13349 is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL); in perf_check_permission()
13369 * sys_perf_event_open - open a performance event, associate it to a task/cpu
13393 int cgroup_fd = -1; in SYSCALL_DEFINE5()
13397 return -EINVAL; in SYSCALL_DEFINE5()
13416 return -EACCES; in SYSCALL_DEFINE5()
13421 return -EINVAL; in SYSCALL_DEFINE5()
13424 return -EINVAL; in SYSCALL_DEFINE5()
13447 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) in SYSCALL_DEFINE5()
13448 return -EINVAL; in SYSCALL_DEFINE5()
13462 CLASS(fd, group)(group_fd); // group_fd == -1 => empty in SYSCALL_DEFINE5()
13463 if (group_fd != -1) { in SYSCALL_DEFINE5()
13465 err = -EBADF; in SYSCALL_DEFINE5()
13468 group_leader = fd_file(group)->private_data; in SYSCALL_DEFINE5()
13469 if (group_leader->state <= PERF_EVENT_STATE_REVOKED) { in SYSCALL_DEFINE5()
13470 err = -ENODEV; in SYSCALL_DEFINE5()
13479 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { in SYSCALL_DEFINE5()
13488 group_leader->attr.inherit != attr.inherit) { in SYSCALL_DEFINE5()
13489 err = -EINVAL; in SYSCALL_DEFINE5()
13504 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { in SYSCALL_DEFINE5()
13505 err = -EOPNOTSUPP; in SYSCALL_DEFINE5()
13514 pmu = event->pmu; in SYSCALL_DEFINE5()
13522 if (pmu->task_ctx_nr == perf_sw_context) in SYSCALL_DEFINE5()
13523 event->event_caps |= PERF_EV_CAP_SOFTWARE; in SYSCALL_DEFINE5()
13526 err = down_read_interruptible(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
13531 * We must hold exec_update_lock across this and any potential in SYSCALL_DEFINE5()
13536 err = -EACCES; in SYSCALL_DEFINE5()
13550 mutex_lock(&ctx->mutex); in SYSCALL_DEFINE5()
13552 if (ctx->task == TASK_TOMBSTONE) { in SYSCALL_DEFINE5()
13553 err = -ESRCH; in SYSCALL_DEFINE5()
13564 struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in SYSCALL_DEFINE5()
13566 if (!cpuctx->online) { in SYSCALL_DEFINE5()
13567 err = -ENODEV; in SYSCALL_DEFINE5()
13573 err = -EINVAL; in SYSCALL_DEFINE5()
13577 * becoming part of another group-sibling): in SYSCALL_DEFINE5()
13579 if (group_leader->group_leader != group_leader) in SYSCALL_DEFINE5()
13583 if (group_leader->clock != event->clock) in SYSCALL_DEFINE5()
13591 if (group_leader->cpu != event->cpu) in SYSCALL_DEFINE5()
13597 if (group_leader->ctx != ctx) in SYSCALL_DEFINE5()
13619 pmu = group_leader->pmu_ctx->pmu; in SYSCALL_DEFINE5()
13622 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { in SYSCALL_DEFINE5()
13633 group_leader->pmu_ctx->pmu != pmu) in SYSCALL_DEFINE5()
13646 event->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
13655 err = -E2BIG; in SYSCALL_DEFINE5()
13660 err = -EINVAL; in SYSCALL_DEFINE5()
13669 err = -EBUSY; in SYSCALL_DEFINE5()
13673 WARN_ON_ONCE(ctx->parent_ctx); in SYSCALL_DEFINE5()
13689 put_pmu_ctx(group_leader->pmu_ctx); in SYSCALL_DEFINE5()
13693 put_pmu_ctx(sibling->pmu_ctx); in SYSCALL_DEFINE5()
13700 * (through the sibling list, which is still in-tact), we can in SYSCALL_DEFINE5()
13703 * By installing siblings first we NO-OP because they're not in SYSCALL_DEFINE5()
13707 sibling->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
13710 perf_install_in_context(ctx, sibling, sibling->cpu); in SYSCALL_DEFINE5()
13718 group_leader->pmu_ctx = pmu_ctx; in SYSCALL_DEFINE5()
13721 perf_install_in_context(ctx, group_leader, group_leader->cpu); in SYSCALL_DEFINE5()
13733 event->owner = current; in SYSCALL_DEFINE5()
13735 perf_install_in_context(ctx, event, event->cpu); in SYSCALL_DEFINE5()
13738 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
13741 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
13745 mutex_lock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
13746 list_add_tail(&event->owner_entry, ¤t->perf_event_list); in SYSCALL_DEFINE5()
13747 mutex_unlock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
13759 put_pmu_ctx(event->pmu_ctx); in SYSCALL_DEFINE5()
13760 event->pmu_ctx = NULL; /* _free_event() */ in SYSCALL_DEFINE5()
13762 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
13767 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
13803 if (attr->aux_output || attr->aux_action) in perf_event_create_kernel_counter()
13804 return ERR_PTR(-EINVAL); in perf_event_create_kernel_counter()
13812 overflow_handler, context, -1); in perf_event_create_kernel_counter()
13819 event->owner = TASK_TOMBSTONE; in perf_event_create_kernel_counter()
13820 pmu = event->pmu; in perf_event_create_kernel_counter()
13822 if (pmu->task_ctx_nr == perf_sw_context) in perf_event_create_kernel_counter()
13823 event->event_caps |= PERF_EV_CAP_SOFTWARE; in perf_event_create_kernel_counter()
13834 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_create_kernel_counter()
13835 mutex_lock(&ctx->mutex); in perf_event_create_kernel_counter()
13836 if (ctx->task == TASK_TOMBSTONE) { in perf_event_create_kernel_counter()
13837 err = -ESRCH; in perf_event_create_kernel_counter()
13846 event->pmu_ctx = pmu_ctx; in perf_event_create_kernel_counter()
13857 if (!cpuctx->online) { in perf_event_create_kernel_counter()
13858 err = -ENODEV; in perf_event_create_kernel_counter()
13864 err = -EBUSY; in perf_event_create_kernel_counter()
13868 perf_install_in_context(ctx, event, event->cpu); in perf_event_create_kernel_counter()
13870 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
13876 event->pmu_ctx = NULL; /* _free_event() */ in perf_event_create_kernel_counter()
13878 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
13897 put_pmu_ctx(event->pmu_ctx); in __perf_pmu_remove()
13898 list_add(&event->migrate_entry, events); in __perf_pmu_remove()
13902 put_pmu_ctx(sibling->pmu_ctx); in __perf_pmu_remove()
13903 list_add(&sibling->migrate_entry, events); in __perf_pmu_remove()
13913 struct perf_event_context *old_ctx = event->ctx; in __perf_pmu_install_event()
13917 event->cpu = cpu; in __perf_pmu_install_event()
13919 event->pmu_ctx = epc; in __perf_pmu_install_event()
13921 if (event->state >= PERF_EVENT_STATE_OFF) in __perf_pmu_install_event()
13922 event->state = PERF_EVENT_STATE_INACTIVE; in __perf_pmu_install_event()
13926 * Now that event->ctx is updated and visible, put the old ctx. in __perf_pmu_install_event()
13937 * Re-instate events in 2 passes. in __perf_pmu_install()
13945 if (event->group_leader == event) in __perf_pmu_install()
13948 list_del(&event->migrate_entry); in __perf_pmu_install()
13957 list_del(&event->migrate_entry); in __perf_pmu_install()
13968 * Since per-cpu context is persistent, no need to grab an extra in perf_pmu_migrate_context()
13971 src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx; in perf_pmu_migrate_context()
13972 dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx; in perf_pmu_migrate_context()
13978 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); in perf_pmu_migrate_context()
13980 __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events); in perf_pmu_migrate_context()
13981 __perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events); in perf_pmu_migrate_context()
13985 * Wait for the events to quiesce before re-instating them. in perf_pmu_migrate_context()
13992 mutex_unlock(&dst_ctx->mutex); in perf_pmu_migrate_context()
13993 mutex_unlock(&src_ctx->mutex); in perf_pmu_migrate_context()
13999 struct perf_event *parent_event = child_event->parent; in sync_child_event()
14002 if (child_event->attr.inherit_stat) { in sync_child_event()
14003 struct task_struct *task = child_event->ctx->task; in sync_child_event()
14014 atomic64_add(child_val, &parent_event->child_count); in sync_child_event()
14015 atomic64_add(child_event->total_time_enabled, in sync_child_event()
14016 &parent_event->child_total_time_enabled); in sync_child_event()
14017 atomic64_add(child_event->total_time_running, in sync_child_event()
14018 &parent_event->child_total_time_running); in sync_child_event()
14025 struct perf_event *parent_event = event->parent; in perf_event_exit_event()
14043 mutex_lock(&parent_event->child_mutex); in perf_event_exit_event()
14045 attach_state = READ_ONCE(event->attach_state); in perf_event_exit_event()
14056 mutex_unlock(&parent_event->child_mutex); in perf_event_exit_event()
14093 * In order to reduce the amount of tricky in ctx tear-down, we hold in perf_event_exit_task_context()
14102 mutex_lock(&ctx->mutex); in perf_event_exit_task_context()
14105 * In a single ctx::lock section, de-schedule the events and detach the in perf_event_exit_task_context()
14109 raw_spin_lock_irq(&ctx->lock); in perf_event_exit_task_context()
14114 * Now that the context is inactive, destroy the task <-> ctx relation in perf_event_exit_task_context()
14117 RCU_INIT_POINTER(task->perf_event_ctxp, NULL); in perf_event_exit_task_context()
14119 WRITE_ONCE(ctx->task, TASK_TOMBSTONE); in perf_event_exit_task_context()
14123 raw_spin_unlock_irq(&ctx->lock); in perf_event_exit_task_context()
14136 list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) in perf_event_exit_task_context()
14139 mutex_unlock(&ctx->mutex); in perf_event_exit_task_context()
14150 * _free_event()'s put_task_struct(event->hw.target) will be a in perf_event_exit_task_context()
14151 * use-after-free. in perf_event_exit_task_context()
14155 wait_var_event(&ctx->refcount, in perf_event_exit_task_context()
14156 refcount_read(&ctx->refcount) == 1); in perf_event_exit_task_context()
14173 mutex_lock(&task->perf_event_mutex); in perf_event_exit_task()
14174 list_for_each_entry_safe(event, tmp, &task->perf_event_list, in perf_event_exit_task()
14176 list_del_init(&event->owner_entry); in perf_event_exit_task()
14181 * we need to serialize on the owner->perf_event_mutex. in perf_event_exit_task()
14183 smp_store_release(&event->owner, NULL); in perf_event_exit_task()
14185 mutex_unlock(&task->perf_event_mutex); in perf_event_exit_task()
14192 * task contexts and sets task->perf_event_ctxp[] to NULL. in perf_event_exit_task()
14198 * Detach the perf_ctx_data for the system-wide event. in perf_event_exit_task()
14218 WARN_ON_ONCE(task->perf_event_ctxp); in perf_event_delayed_put()
14225 return ERR_PTR(-EBADF); in perf_event_get()
14227 if (file->f_op != &perf_fops) { in perf_event_get()
14229 return ERR_PTR(-EBADF); in perf_event_get()
14237 if (file->f_op != &perf_fops) in perf_get_event()
14238 return ERR_PTR(-EINVAL); in perf_get_event()
14240 return file->private_data; in perf_get_event()
14246 return ERR_PTR(-EINVAL); in perf_event_attrs()
14248 return &event->attr; in perf_event_attrs()
14254 return -EACCES; in perf_allow_kernel()
14264 * - valid pointer on success
14265 * - NULL for orphaned events
14266 * - IS_ERR() on error
14276 enum perf_event_state parent_state = parent_event->state; in inherit_event()
14287 if (parent_event->parent) in inherit_event()
14288 parent_event = parent_event->parent; in inherit_event()
14290 if (parent_event->state <= PERF_EVENT_STATE_REVOKED) in inherit_event()
14298 child_event = perf_event_alloc(&parent_event->attr, in inherit_event()
14299 parent_event->cpu, in inherit_event()
14302 NULL, NULL, -1); in inherit_event()
14307 child_event->ctx = child_ctx; in inherit_event()
14309 pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event); in inherit_event()
14314 child_event->pmu_ctx = pmu_ctx; in inherit_event()
14317 * is_orphaned_event() and list_add_tail(&parent_event->child_list) in inherit_event()
14322 mutex_lock(&parent_event->child_mutex); in inherit_event()
14324 !atomic_long_inc_not_zero(&parent_event->refcount)) { in inherit_event()
14325 mutex_unlock(&parent_event->child_mutex); in inherit_event()
14332 * not its attr.disabled bit. We hold the parent's mutex, in inherit_event()
14336 child_event->state = PERF_EVENT_STATE_INACTIVE; in inherit_event()
14338 child_event->state = PERF_EVENT_STATE_OFF; in inherit_event()
14340 if (parent_event->attr.freq) { in inherit_event()
14341 u64 sample_period = parent_event->hw.sample_period; in inherit_event()
14342 struct hw_perf_event *hwc = &child_event->hw; in inherit_event()
14344 hwc->sample_period = sample_period; in inherit_event()
14345 hwc->last_period = sample_period; in inherit_event()
14347 local64_set(&hwc->period_left, sample_period); in inherit_event()
14350 child_event->overflow_handler = parent_event->overflow_handler; in inherit_event()
14351 child_event->overflow_handler_context in inherit_event()
14352 = parent_event->overflow_handler_context; in inherit_event()
14363 raw_spin_lock_irqsave(&child_ctx->lock, flags); in inherit_event()
14365 child_event->attach_state |= PERF_ATTACH_CHILD; in inherit_event()
14366 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); in inherit_event()
14371 list_add_tail(&child_event->child_list, &parent_event->child_list); in inherit_event()
14372 mutex_unlock(&parent_event->child_mutex); in inherit_event()
14384 * - 0 on success
14385 * - <0 on error
14412 if (sub->aux_event == parent_event && child_ctr && in inherit_group()
14414 return -EINVAL; in inherit_group()
14417 leader->group_generation = parent_event->group_generation; in inherit_group()
14422 * Creates the child task context and tries to inherit the event-group.
14429 * - 0 on success
14430 * - <0 on error
14441 if (!event->attr.inherit || in inherit_task_group()
14442 (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) || in inherit_task_group()
14444 (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) { in inherit_task_group()
14449 child_ctx = child->perf_event_ctxp; in inherit_task_group()
14459 return -ENOMEM; in inherit_task_group()
14461 child->perf_event_ctxp = child_ctx; in inherit_task_group()
14484 if (likely(!parent->perf_event_ctxp)) in perf_event_init_context()
14497 * it non-NULL earlier, the only reason for it to become NULL in perf_event_init_context()
14503 * Lock the parent list. No need to lock the child - not PID in perf_event_init_context()
14506 mutex_lock(&parent_ctx->mutex); in perf_event_init_context()
14509 * We dont have to disable NMIs - we are only looking at in perf_event_init_context()
14512 perf_event_groups_for_each(event, &parent_ctx->pinned_groups) { in perf_event_init_context()
14520 * We can't hold ctx->lock when iterating the ->flexible_group list due in perf_event_init_context()
14524 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
14525 parent_ctx->rotate_disable = 1; in perf_event_init_context()
14526 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
14528 perf_event_groups_for_each(event, &parent_ctx->flexible_groups) { in perf_event_init_context()
14535 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
14536 parent_ctx->rotate_disable = 0; in perf_event_init_context()
14538 child_ctx = child->perf_event_ctxp; in perf_event_init_context()
14546 * parent_ctx->lock avoids it from being uncloned. in perf_event_init_context()
14548 cloned_ctx = parent_ctx->parent_ctx; in perf_event_init_context()
14550 child_ctx->parent_ctx = cloned_ctx; in perf_event_init_context()
14551 child_ctx->parent_gen = parent_ctx->parent_gen; in perf_event_init_context()
14553 child_ctx->parent_ctx = parent_ctx; in perf_event_init_context()
14554 child_ctx->parent_gen = parent_ctx->generation; in perf_event_init_context()
14556 get_ctx(child_ctx->parent_ctx); in perf_event_init_context()
14559 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
14561 mutex_unlock(&parent_ctx->mutex); in perf_event_init_context()
14576 memset(child->perf_recursion, 0, sizeof(child->perf_recursion)); in perf_event_init_task()
14577 child->perf_event_ctxp = NULL; in perf_event_init_task()
14578 mutex_init(&child->perf_event_mutex); in perf_event_init_task()
14579 INIT_LIST_HEAD(&child->perf_event_list); in perf_event_init_task()
14580 child->perf_ctx_data = NULL; in perf_event_init_task()
14607 mutex_init(&swhash->hlist_mutex); in perf_event_init_all_cpus()
14615 __perf_event_init_context(&cpuctx->ctx); in perf_event_init_all_cpus()
14616 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); in perf_event_init_all_cpus()
14617 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); in perf_event_init_all_cpus()
14618 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); in perf_event_init_all_cpus()
14619 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); in perf_event_init_all_cpus()
14620 cpuctx->heap = cpuctx->heap_default; in perf_event_init_all_cpus()
14628 mutex_lock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
14629 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { in perf_swevent_init_cpu()
14634 rcu_assign_pointer(swhash->swevent_hlist, hlist); in perf_swevent_init_cpu()
14636 mutex_unlock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
14646 raw_spin_lock(&ctx->lock); in __perf_event_exit_context()
14648 list_for_each_entry(event, &ctx->event_list, event_entry) in __perf_event_exit_context()
14650 raw_spin_unlock(&ctx->lock); in __perf_event_exit_context()
14665 target[scope] = -1; in perf_event_clear_cpumask()
14678 if (pmu->scope == PERF_PMU_SCOPE_NONE || in perf_event_clear_cpumask()
14679 WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE)) in perf_event_clear_cpumask()
14682 if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids) in perf_event_clear_cpumask()
14683 perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]); in perf_event_clear_cpumask()
14692 // XXX simplify cpuctx->online in perf_event_exit_cpu_context()
14700 ctx = &cpuctx->ctx; in perf_event_exit_cpu_context()
14702 mutex_lock(&ctx->mutex); in perf_event_exit_cpu_context()
14704 cpuctx->online = 0; in perf_event_exit_cpu_context()
14705 mutex_unlock(&ctx->mutex); in perf_event_exit_cpu_context()
14760 ctx = &cpuctx->ctx; in perf_event_init_cpu()
14762 mutex_lock(&ctx->mutex); in perf_event_init_cpu()
14763 cpuctx->online = 1; in perf_event_init_cpu()
14764 mutex_unlock(&ctx->mutex); in perf_event_init_cpu()
14805 perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1); in perf_event_init()
14806 perf_pmu_register(&perf_task_clock, "task_clock", -1); in perf_event_init()
14830 if (pmu_attr->event_str) in perf_event_sysfs_show()
14831 return sprintf(page, "%s\n", pmu_attr->event_str); in perf_event_sysfs_show()
14849 if (pmu->dev) in perf_event_sysfs_init()
14853 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); in perf_event_sysfs_init()
14873 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
14875 jc->info = alloc_percpu(struct perf_cgroup_info); in perf_cgroup_css_alloc()
14876 if (!jc->info) { in perf_cgroup_css_alloc()
14878 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
14881 return &jc->css; in perf_cgroup_css_alloc()
14888 free_percpu(jc->info); in perf_cgroup_css_free()
14894 perf_event_cgroup(css->cgroup); in perf_cgroup_css_online()