1bcea3f96SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0
297d5a220SFrederic Weisbecker /*
397d5a220SFrederic Weisbecker * trace event based perf event profiling/tracing
497d5a220SFrederic Weisbecker *
590eec103SPeter Zijlstra * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra
697d5a220SFrederic Weisbecker * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
797d5a220SFrederic Weisbecker */
897d5a220SFrederic Weisbecker
997d5a220SFrederic Weisbecker #include <linux/module.h>
1097d5a220SFrederic Weisbecker #include <linux/kprobes.h>
11da97e184SJoel Fernandes (Google) #include <linux/security.h>
1297d5a220SFrederic Weisbecker #include "trace.h"
13e12f03d7SSong Liu #include "trace_probe.h"
1497d5a220SFrederic Weisbecker
156016ee13SNamhyung Kim static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS];
1697d5a220SFrederic Weisbecker
17eb1e7961SFrederic Weisbecker /*
18eb1e7961SFrederic Weisbecker * Force it to be aligned to unsigned long to avoid misaligned accesses
19f2cc020dSIngo Molnar * surprises
20eb1e7961SFrederic Weisbecker */
21eb1e7961SFrederic Weisbecker typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
22eb1e7961SFrederic Weisbecker perf_trace_t;
2397d5a220SFrederic Weisbecker
2497d5a220SFrederic Weisbecker /* Count the events in use (per event id, not per instance) */
2597d5a220SFrederic Weisbecker static int total_ref_count;
2697d5a220SFrederic Weisbecker
perf_trace_event_perm(struct trace_event_call * tp_event,struct perf_event * p_event)272425bcb9SSteven Rostedt (Red Hat) static int perf_trace_event_perm(struct trace_event_call *tp_event,
2861c32659SFrederic Weisbecker struct perf_event *p_event)
2961c32659SFrederic Weisbecker {
30da97e184SJoel Fernandes (Google) int ret;
31da97e184SJoel Fernandes (Google)
32d5b5f391SPeter Zijlstra if (tp_event->perf_perm) {
33da97e184SJoel Fernandes (Google) ret = tp_event->perf_perm(tp_event, p_event);
34d5b5f391SPeter Zijlstra if (ret)
35d5b5f391SPeter Zijlstra return ret;
36d5b5f391SPeter Zijlstra }
37d5b5f391SPeter Zijlstra
38f4be073dSJiri Olsa /*
39f4be073dSJiri Olsa * We checked and allowed to create parent,
40f4be073dSJiri Olsa * allow children without checking.
41f4be073dSJiri Olsa */
42f4be073dSJiri Olsa if (p_event->parent)
43f4be073dSJiri Olsa return 0;
44f4be073dSJiri Olsa
45f4be073dSJiri Olsa /*
46f4be073dSJiri Olsa * It's ok to check current process (owner) permissions in here,
47f4be073dSJiri Olsa * because code below is called only via perf_event_open syscall.
48f4be073dSJiri Olsa */
49f4be073dSJiri Olsa
50ced39002SJiri Olsa /* The ftrace function trace is allowed only for root. */
51cfa77bc4SJiri Olsa if (ftrace_event_is_function(tp_event)) {
52da97e184SJoel Fernandes (Google) ret = perf_allow_tracepoint(&p_event->attr);
53da97e184SJoel Fernandes (Google) if (ret)
54da97e184SJoel Fernandes (Google) return ret;
55ced39002SJiri Olsa
560a74c5b3SJiri Olsa if (!is_sampling_event(p_event))
570a74c5b3SJiri Olsa return 0;
580a74c5b3SJiri Olsa
59cfa77bc4SJiri Olsa /*
60cfa77bc4SJiri Olsa * We don't allow user space callchains for function trace
61cfa77bc4SJiri Olsa * event, due to issues with page faults while tracing page
62cfa77bc4SJiri Olsa * fault handler and its overall trickiness nature.
63cfa77bc4SJiri Olsa */
64cfa77bc4SJiri Olsa if (!p_event->attr.exclude_callchain_user)
65cfa77bc4SJiri Olsa return -EINVAL;
6663c45f4bSJiri Olsa
6763c45f4bSJiri Olsa /*
6863c45f4bSJiri Olsa * Same reason to disable user stack dump as for user space
6963c45f4bSJiri Olsa * callchains above.
7063c45f4bSJiri Olsa */
7163c45f4bSJiri Olsa if (p_event->attr.sample_type & PERF_SAMPLE_STACK_USER)
7263c45f4bSJiri Olsa return -EINVAL;
73cfa77bc4SJiri Olsa }
74cfa77bc4SJiri Olsa
7561c32659SFrederic Weisbecker /* No tracing, just counting, so no obvious leak */
7661c32659SFrederic Weisbecker if (!(p_event->attr.sample_type & PERF_SAMPLE_RAW))
7761c32659SFrederic Weisbecker return 0;
7861c32659SFrederic Weisbecker
7961c32659SFrederic Weisbecker /* Some events are ok to be traced by non-root users... */
8061c32659SFrederic Weisbecker if (p_event->attach_state == PERF_ATTACH_TASK) {
8161c32659SFrederic Weisbecker if (tp_event->flags & TRACE_EVENT_FL_CAP_ANY)
8261c32659SFrederic Weisbecker return 0;
8361c32659SFrederic Weisbecker }
8461c32659SFrederic Weisbecker
8561c32659SFrederic Weisbecker /*
8661c32659SFrederic Weisbecker * ...otherwise raw tracepoint data can be a severe data leak,
8761c32659SFrederic Weisbecker * only allow root to have these.
8861c32659SFrederic Weisbecker */
89da97e184SJoel Fernandes (Google) ret = perf_allow_tracepoint(&p_event->attr);
90da97e184SJoel Fernandes (Google) if (ret)
91da97e184SJoel Fernandes (Google) return ret;
9261c32659SFrederic Weisbecker
9361c32659SFrederic Weisbecker return 0;
9461c32659SFrederic Weisbecker }
9561c32659SFrederic Weisbecker
perf_trace_event_reg(struct trace_event_call * tp_event,struct perf_event * p_event)962425bcb9SSteven Rostedt (Red Hat) static int perf_trace_event_reg(struct trace_event_call *tp_event,
971c024ecaSPeter Zijlstra struct perf_event *p_event)
9897d5a220SFrederic Weisbecker {
996016ee13SNamhyung Kim struct hlist_head __percpu *list;
100ceec0b6fSJiri Olsa int ret = -ENOMEM;
1011c024ecaSPeter Zijlstra int cpu;
10297d5a220SFrederic Weisbecker
1031c024ecaSPeter Zijlstra p_event->tp_event = tp_event;
1041c024ecaSPeter Zijlstra if (tp_event->perf_refcount++ > 0)
10597d5a220SFrederic Weisbecker return 0;
10697d5a220SFrederic Weisbecker
1071c024ecaSPeter Zijlstra list = alloc_percpu(struct hlist_head);
1081c024ecaSPeter Zijlstra if (!list)
1091c024ecaSPeter Zijlstra goto fail;
1101c024ecaSPeter Zijlstra
1111c024ecaSPeter Zijlstra for_each_possible_cpu(cpu)
1121c024ecaSPeter Zijlstra INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
1131c024ecaSPeter Zijlstra
1141c024ecaSPeter Zijlstra tp_event->perf_events = list;
11597d5a220SFrederic Weisbecker
11697d5a220SFrederic Weisbecker if (!total_ref_count) {
1176016ee13SNamhyung Kim char __percpu *buf;
118b7e2ecefSPeter Zijlstra int i;
119b7e2ecefSPeter Zijlstra
1207ae07ea3SFrederic Weisbecker for (i = 0; i < PERF_NR_CONTEXTS; i++) {
1216016ee13SNamhyung Kim buf = (char __percpu *)alloc_percpu(perf_trace_t);
12297d5a220SFrederic Weisbecker if (!buf)
1231c024ecaSPeter Zijlstra goto fail;
12497d5a220SFrederic Weisbecker
1251c024ecaSPeter Zijlstra perf_trace_buf[i] = buf;
126b7e2ecefSPeter Zijlstra }
12797d5a220SFrederic Weisbecker }
12897d5a220SFrederic Weisbecker
129ceec0b6fSJiri Olsa ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER, NULL);
1301c024ecaSPeter Zijlstra if (ret)
1311c024ecaSPeter Zijlstra goto fail;
1321c024ecaSPeter Zijlstra
13397d5a220SFrederic Weisbecker total_ref_count++;
13497d5a220SFrederic Weisbecker return 0;
13597d5a220SFrederic Weisbecker
1361c024ecaSPeter Zijlstra fail:
137b7e2ecefSPeter Zijlstra if (!total_ref_count) {
138b7e2ecefSPeter Zijlstra int i;
139b7e2ecefSPeter Zijlstra
1407ae07ea3SFrederic Weisbecker for (i = 0; i < PERF_NR_CONTEXTS; i++) {
141b7e2ecefSPeter Zijlstra free_percpu(perf_trace_buf[i]);
142b7e2ecefSPeter Zijlstra perf_trace_buf[i] = NULL;
143b7e2ecefSPeter Zijlstra }
14497d5a220SFrederic Weisbecker }
14597d5a220SFrederic Weisbecker
1461c024ecaSPeter Zijlstra if (!--tp_event->perf_refcount) {
1471c024ecaSPeter Zijlstra free_percpu(tp_event->perf_events);
1481c024ecaSPeter Zijlstra tp_event->perf_events = NULL;
14997d5a220SFrederic Weisbecker }
15097d5a220SFrederic Weisbecker
15197d5a220SFrederic Weisbecker return ret;
15297d5a220SFrederic Weisbecker }
15397d5a220SFrederic Weisbecker
perf_trace_event_unreg(struct perf_event * p_event)154ceec0b6fSJiri Olsa static void perf_trace_event_unreg(struct perf_event *p_event)
155ceec0b6fSJiri Olsa {
1562425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event = p_event->tp_event;
157ceec0b6fSJiri Olsa int i;
158ceec0b6fSJiri Olsa
159ceec0b6fSJiri Olsa if (--tp_event->perf_refcount > 0)
1607249921dSSteven Rostedt (Google) return;
161ceec0b6fSJiri Olsa
162ceec0b6fSJiri Olsa tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER, NULL);
163ceec0b6fSJiri Olsa
164ceec0b6fSJiri Olsa /*
165ceec0b6fSJiri Olsa * Ensure our callback won't be called anymore. The buffers
166ceec0b6fSJiri Olsa * will be freed after that.
167ceec0b6fSJiri Olsa */
168ceec0b6fSJiri Olsa tracepoint_synchronize_unregister();
169ceec0b6fSJiri Olsa
170ceec0b6fSJiri Olsa free_percpu(tp_event->perf_events);
171ceec0b6fSJiri Olsa tp_event->perf_events = NULL;
172ceec0b6fSJiri Olsa
173ceec0b6fSJiri Olsa if (!--total_ref_count) {
174ceec0b6fSJiri Olsa for (i = 0; i < PERF_NR_CONTEXTS; i++) {
175ceec0b6fSJiri Olsa free_percpu(perf_trace_buf[i]);
176ceec0b6fSJiri Olsa perf_trace_buf[i] = NULL;
177ceec0b6fSJiri Olsa }
178ceec0b6fSJiri Olsa }
179ceec0b6fSJiri Olsa }
180ceec0b6fSJiri Olsa
perf_trace_event_open(struct perf_event * p_event)181ceec0b6fSJiri Olsa static int perf_trace_event_open(struct perf_event *p_event)
182ceec0b6fSJiri Olsa {
1832425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event = p_event->tp_event;
184ceec0b6fSJiri Olsa return tp_event->class->reg(tp_event, TRACE_REG_PERF_OPEN, p_event);
185ceec0b6fSJiri Olsa }
186ceec0b6fSJiri Olsa
perf_trace_event_close(struct perf_event * p_event)187ceec0b6fSJiri Olsa static void perf_trace_event_close(struct perf_event *p_event)
188ceec0b6fSJiri Olsa {
1892425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event = p_event->tp_event;
190ceec0b6fSJiri Olsa tp_event->class->reg(tp_event, TRACE_REG_PERF_CLOSE, p_event);
191ceec0b6fSJiri Olsa }
192ceec0b6fSJiri Olsa
perf_trace_event_init(struct trace_event_call * tp_event,struct perf_event * p_event)1932425bcb9SSteven Rostedt (Red Hat) static int perf_trace_event_init(struct trace_event_call *tp_event,
194ceec0b6fSJiri Olsa struct perf_event *p_event)
195ceec0b6fSJiri Olsa {
196ceec0b6fSJiri Olsa int ret;
197ceec0b6fSJiri Olsa
198ceec0b6fSJiri Olsa ret = perf_trace_event_perm(tp_event, p_event);
199ceec0b6fSJiri Olsa if (ret)
200ceec0b6fSJiri Olsa return ret;
201ceec0b6fSJiri Olsa
202ceec0b6fSJiri Olsa ret = perf_trace_event_reg(tp_event, p_event);
203ceec0b6fSJiri Olsa if (ret)
204ceec0b6fSJiri Olsa return ret;
205ceec0b6fSJiri Olsa
206ceec0b6fSJiri Olsa ret = perf_trace_event_open(p_event);
207ceec0b6fSJiri Olsa if (ret) {
208ceec0b6fSJiri Olsa perf_trace_event_unreg(p_event);
209ceec0b6fSJiri Olsa return ret;
210ceec0b6fSJiri Olsa }
211ceec0b6fSJiri Olsa
212ceec0b6fSJiri Olsa return 0;
213ceec0b6fSJiri Olsa }
214ceec0b6fSJiri Olsa
perf_trace_init(struct perf_event * p_event)2151c024ecaSPeter Zijlstra int perf_trace_init(struct perf_event *p_event)
21697d5a220SFrederic Weisbecker {
2172425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event;
2180022ceddSVince Weaver u64 event_id = p_event->attr.config;
21997d5a220SFrederic Weisbecker int ret = -EINVAL;
22097d5a220SFrederic Weisbecker
22197d5a220SFrederic Weisbecker mutex_lock(&event_mutex);
2221c024ecaSPeter Zijlstra list_for_each_entry(tp_event, &ftrace_events, list) {
223ff5f149bSSteven Rostedt if (tp_event->event.type == event_id &&
224a1d0ce82SSteven Rostedt tp_event->class && tp_event->class->reg &&
2251d18538eSSteven Rostedt (VMware) trace_event_try_get_ref(tp_event)) {
2261c024ecaSPeter Zijlstra ret = perf_trace_event_init(tp_event, p_event);
2279cb627d5SLi Zefan if (ret)
2281d18538eSSteven Rostedt (VMware) trace_event_put_ref(tp_event);
22997d5a220SFrederic Weisbecker break;
23097d5a220SFrederic Weisbecker }
23197d5a220SFrederic Weisbecker }
23297d5a220SFrederic Weisbecker mutex_unlock(&event_mutex);
23397d5a220SFrederic Weisbecker
23497d5a220SFrederic Weisbecker return ret;
23597d5a220SFrederic Weisbecker }
23697d5a220SFrederic Weisbecker
perf_trace_destroy(struct perf_event * p_event)237ceec0b6fSJiri Olsa void perf_trace_destroy(struct perf_event *p_event)
238ceec0b6fSJiri Olsa {
239ceec0b6fSJiri Olsa mutex_lock(&event_mutex);
240ceec0b6fSJiri Olsa perf_trace_event_close(p_event);
241ceec0b6fSJiri Olsa perf_trace_event_unreg(p_event);
2427249921dSSteven Rostedt (Google) trace_event_put_ref(p_event->tp_event);
243ceec0b6fSJiri Olsa mutex_unlock(&event_mutex);
244ceec0b6fSJiri Olsa }
245ceec0b6fSJiri Olsa
246e12f03d7SSong Liu #ifdef CONFIG_KPROBE_EVENTS
perf_kprobe_init(struct perf_event * p_event,bool is_retprobe)247e12f03d7SSong Liu int perf_kprobe_init(struct perf_event *p_event, bool is_retprobe)
248e12f03d7SSong Liu {
249e12f03d7SSong Liu int ret;
250e12f03d7SSong Liu char *func = NULL;
251e12f03d7SSong Liu struct trace_event_call *tp_event;
252e12f03d7SSong Liu
253e12f03d7SSong Liu if (p_event->attr.kprobe_func) {
254*9430cd62SChuang Wang func = strndup_user(u64_to_user_ptr(p_event->attr.kprobe_func),
255e12f03d7SSong Liu KSYM_NAME_LEN);
256*9430cd62SChuang Wang if (IS_ERR(func)) {
257*9430cd62SChuang Wang ret = PTR_ERR(func);
258*9430cd62SChuang Wang return (ret == -EINVAL) ? -E2BIG : ret;
259*9430cd62SChuang Wang }
260e12f03d7SSong Liu
261e12f03d7SSong Liu if (func[0] == '\0') {
262e12f03d7SSong Liu kfree(func);
263e12f03d7SSong Liu func = NULL;
264e12f03d7SSong Liu }
265e12f03d7SSong Liu }
266e12f03d7SSong Liu
267e12f03d7SSong Liu tp_event = create_local_trace_kprobe(
268e12f03d7SSong Liu func, (void *)(unsigned long)(p_event->attr.kprobe_addr),
269e12f03d7SSong Liu p_event->attr.probe_offset, is_retprobe);
270e12f03d7SSong Liu if (IS_ERR(tp_event)) {
271e12f03d7SSong Liu ret = PTR_ERR(tp_event);
272e12f03d7SSong Liu goto out;
273e12f03d7SSong Liu }
274e12f03d7SSong Liu
2756b1340ccSPrateek Sood mutex_lock(&event_mutex);
276e12f03d7SSong Liu ret = perf_trace_event_init(tp_event, p_event);
277e12f03d7SSong Liu if (ret)
278e12f03d7SSong Liu destroy_local_trace_kprobe(tp_event);
2796b1340ccSPrateek Sood mutex_unlock(&event_mutex);
280e12f03d7SSong Liu out:
281e12f03d7SSong Liu kfree(func);
282e12f03d7SSong Liu return ret;
283e12f03d7SSong Liu }
284e12f03d7SSong Liu
perf_kprobe_destroy(struct perf_event * p_event)285e12f03d7SSong Liu void perf_kprobe_destroy(struct perf_event *p_event)
286e12f03d7SSong Liu {
2876b1340ccSPrateek Sood mutex_lock(&event_mutex);
288e12f03d7SSong Liu perf_trace_event_close(p_event);
289e12f03d7SSong Liu perf_trace_event_unreg(p_event);
2907249921dSSteven Rostedt (Google) trace_event_put_ref(p_event->tp_event);
2916b1340ccSPrateek Sood mutex_unlock(&event_mutex);
292e12f03d7SSong Liu
293e12f03d7SSong Liu destroy_local_trace_kprobe(p_event->tp_event);
294e12f03d7SSong Liu }
295e12f03d7SSong Liu #endif /* CONFIG_KPROBE_EVENTS */
296e12f03d7SSong Liu
29733ea4b24SSong Liu #ifdef CONFIG_UPROBE_EVENTS
perf_uprobe_init(struct perf_event * p_event,unsigned long ref_ctr_offset,bool is_retprobe)298a6ca88b2SSong Liu int perf_uprobe_init(struct perf_event *p_event,
299a6ca88b2SSong Liu unsigned long ref_ctr_offset, bool is_retprobe)
30033ea4b24SSong Liu {
30133ea4b24SSong Liu int ret;
30233ea4b24SSong Liu char *path = NULL;
30333ea4b24SSong Liu struct trace_event_call *tp_event;
30433ea4b24SSong Liu
30533ea4b24SSong Liu if (!p_event->attr.uprobe_path)
30633ea4b24SSong Liu return -EINVAL;
30783540fbcSJann Horn
30883540fbcSJann Horn path = strndup_user(u64_to_user_ptr(p_event->attr.uprobe_path),
30983540fbcSJann Horn PATH_MAX);
31083540fbcSJann Horn if (IS_ERR(path)) {
31183540fbcSJann Horn ret = PTR_ERR(path);
31283540fbcSJann Horn return (ret == -EINVAL) ? -E2BIG : ret;
31383540fbcSJann Horn }
31433ea4b24SSong Liu if (path[0] == '\0') {
31533ea4b24SSong Liu ret = -EINVAL;
31633ea4b24SSong Liu goto out;
31733ea4b24SSong Liu }
31833ea4b24SSong Liu
319a6ca88b2SSong Liu tp_event = create_local_trace_uprobe(path, p_event->attr.probe_offset,
320a6ca88b2SSong Liu ref_ctr_offset, is_retprobe);
32133ea4b24SSong Liu if (IS_ERR(tp_event)) {
32233ea4b24SSong Liu ret = PTR_ERR(tp_event);
32333ea4b24SSong Liu goto out;
32433ea4b24SSong Liu }
32533ea4b24SSong Liu
32633ea4b24SSong Liu /*
32733ea4b24SSong Liu * local trace_uprobe need to hold event_mutex to call
32833ea4b24SSong Liu * uprobe_buffer_enable() and uprobe_buffer_disable().
32933ea4b24SSong Liu * event_mutex is not required for local trace_kprobes.
33033ea4b24SSong Liu */
33133ea4b24SSong Liu mutex_lock(&event_mutex);
33233ea4b24SSong Liu ret = perf_trace_event_init(tp_event, p_event);
33333ea4b24SSong Liu if (ret)
33433ea4b24SSong Liu destroy_local_trace_uprobe(tp_event);
33533ea4b24SSong Liu mutex_unlock(&event_mutex);
33633ea4b24SSong Liu out:
33733ea4b24SSong Liu kfree(path);
33833ea4b24SSong Liu return ret;
33933ea4b24SSong Liu }
34033ea4b24SSong Liu
perf_uprobe_destroy(struct perf_event * p_event)34133ea4b24SSong Liu void perf_uprobe_destroy(struct perf_event *p_event)
34233ea4b24SSong Liu {
34333ea4b24SSong Liu mutex_lock(&event_mutex);
34433ea4b24SSong Liu perf_trace_event_close(p_event);
34533ea4b24SSong Liu perf_trace_event_unreg(p_event);
3467249921dSSteven Rostedt (Google) trace_event_put_ref(p_event->tp_event);
34733ea4b24SSong Liu mutex_unlock(&event_mutex);
34833ea4b24SSong Liu destroy_local_trace_uprobe(p_event->tp_event);
34933ea4b24SSong Liu }
35033ea4b24SSong Liu #endif /* CONFIG_UPROBE_EVENTS */
35133ea4b24SSong Liu
perf_trace_add(struct perf_event * p_event,int flags)352a4eaf7f1SPeter Zijlstra int perf_trace_add(struct perf_event *p_event, int flags)
35397d5a220SFrederic Weisbecker {
3542425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event = p_event->tp_event;
355466c81c4SPeter Zijlstra
356466c81c4SPeter Zijlstra if (!(flags & PERF_EF_START))
357466c81c4SPeter Zijlstra p_event->hw.state = PERF_HES_STOPPED;
358466c81c4SPeter Zijlstra
359466c81c4SPeter Zijlstra /*
360466c81c4SPeter Zijlstra * If TRACE_REG_PERF_ADD returns false; no custom action was performed
361466c81c4SPeter Zijlstra * and we need to take the default action of enqueueing our event on
362466c81c4SPeter Zijlstra * the right per-cpu hlist.
363466c81c4SPeter Zijlstra */
364466c81c4SPeter Zijlstra if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_ADD, p_event)) {
3656016ee13SNamhyung Kim struct hlist_head __percpu *pcpu_list;
3661c024ecaSPeter Zijlstra struct hlist_head *list;
36797d5a220SFrederic Weisbecker
3686016ee13SNamhyung Kim pcpu_list = tp_event->perf_events;
3696016ee13SNamhyung Kim if (WARN_ON_ONCE(!pcpu_list))
3701c024ecaSPeter Zijlstra return -EINVAL;
37197d5a220SFrederic Weisbecker
3726016ee13SNamhyung Kim list = this_cpu_ptr(pcpu_list);
3731c024ecaSPeter Zijlstra hlist_add_head_rcu(&p_event->hlist_entry, list);
374466c81c4SPeter Zijlstra }
3751c024ecaSPeter Zijlstra
376466c81c4SPeter Zijlstra return 0;
3771c024ecaSPeter Zijlstra }
3781c024ecaSPeter Zijlstra
perf_trace_del(struct perf_event * p_event,int flags)379a4eaf7f1SPeter Zijlstra void perf_trace_del(struct perf_event *p_event, int flags)
3801c024ecaSPeter Zijlstra {
3812425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *tp_event = p_event->tp_event;
382466c81c4SPeter Zijlstra
383466c81c4SPeter Zijlstra /*
384466c81c4SPeter Zijlstra * If TRACE_REG_PERF_DEL returns false; no custom action was performed
385466c81c4SPeter Zijlstra * and we need to take the default action of dequeueing our event from
386466c81c4SPeter Zijlstra * the right per-cpu hlist.
387466c81c4SPeter Zijlstra */
388466c81c4SPeter Zijlstra if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event))
3891c024ecaSPeter Zijlstra hlist_del_rcu(&p_event->hlist_entry);
3901c024ecaSPeter Zijlstra }
3911c024ecaSPeter Zijlstra
perf_trace_buf_alloc(int size,struct pt_regs ** regs,int * rctxp)3921e1dcd93SAlexei Starovoitov void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
39397d5a220SFrederic Weisbecker {
3941c024ecaSPeter Zijlstra char *raw_data;
3951e1dcd93SAlexei Starovoitov int rctx;
39697d5a220SFrederic Weisbecker
397eb1e7961SFrederic Weisbecker BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
398eb1e7961SFrederic Weisbecker
399cd92bf61SOleg Nesterov if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
400a90afe8dSRobin H. Johnson "perf buffer not large enough, wanted %d, have %d",
401a90afe8dSRobin H. Johnson size, PERF_MAX_TRACE_SIZE))
402cd92bf61SOleg Nesterov return NULL;
403cd92bf61SOleg Nesterov
4041e1dcd93SAlexei Starovoitov *rctxp = rctx = perf_swevent_get_recursion_context();
4051e1dcd93SAlexei Starovoitov if (rctx < 0)
4061c024ecaSPeter Zijlstra return NULL;
40797d5a220SFrederic Weisbecker
40886038c5eSPeter Zijlstra (Intel) if (regs)
4091e1dcd93SAlexei Starovoitov *regs = this_cpu_ptr(&__perf_regs[rctx]);
4101e1dcd93SAlexei Starovoitov raw_data = this_cpu_ptr(perf_trace_buf[rctx]);
41197d5a220SFrederic Weisbecker
41297d5a220SFrederic Weisbecker /* zero the dead bytes from align to not leak stack to user */
413eb1e7961SFrederic Weisbecker memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
4141e1dcd93SAlexei Starovoitov return raw_data;
4151e1dcd93SAlexei Starovoitov }
4161e1dcd93SAlexei Starovoitov EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
4171e1dcd93SAlexei Starovoitov NOKPROBE_SYMBOL(perf_trace_buf_alloc);
41897d5a220SFrederic Weisbecker
perf_trace_buf_update(void * record,u16 type)4191e1dcd93SAlexei Starovoitov void perf_trace_buf_update(void *record, u16 type)
4201e1dcd93SAlexei Starovoitov {
4211e1dcd93SAlexei Starovoitov struct trace_entry *entry = record;
4221e1dcd93SAlexei Starovoitov
42336590c50SSebastian Andrzej Siewior tracing_generic_entry_update(entry, type, tracing_gen_ctx());
42497d5a220SFrederic Weisbecker }
4251e1dcd93SAlexei Starovoitov NOKPROBE_SYMBOL(perf_trace_buf_update);
426ced39002SJiri Olsa
427ced39002SJiri Olsa #ifdef CONFIG_FUNCTION_TRACER
428ced39002SJiri Olsa static void
perf_ftrace_function_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * ops,struct ftrace_regs * fregs)4292f5f6ad9SSteven Rostedt perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
430d19ad077SSteven Rostedt (VMware) struct ftrace_ops *ops, struct ftrace_regs *fregs)
431ced39002SJiri Olsa {
432ced39002SJiri Olsa struct ftrace_entry *entry;
433466c81c4SPeter Zijlstra struct perf_event *event;
434466c81c4SPeter Zijlstra struct hlist_head head;
435ced39002SJiri Olsa struct pt_regs regs;
436ced39002SJiri Olsa int rctx;
4375d15a624SSteven Rostedt (VMware) int bit;
438ced39002SJiri Olsa
4395d029b03SSteven Rostedt (VMware) if (!rcu_is_watching())
4405d029b03SSteven Rostedt (VMware) return;
4415d029b03SSteven Rostedt (VMware)
442773c1670SSteven Rostedt (VMware) bit = ftrace_test_recursion_trylock(ip, parent_ip);
4435d15a624SSteven Rostedt (VMware) if (bit < 0)
4445d15a624SSteven Rostedt (VMware) return;
4455d15a624SSteven Rostedt (VMware)
446d33cc657S王贇 if ((unsigned long)ops->private != smp_processor_id())
447d33cc657S王贇 goto out;
448d33cc657S王贇
449466c81c4SPeter Zijlstra event = container_of(ops, struct perf_event, ftrace_ops);
450466c81c4SPeter Zijlstra
451466c81c4SPeter Zijlstra /*
452466c81c4SPeter Zijlstra * @event->hlist entry is NULL (per INIT_HLIST_NODE), and all
453466c81c4SPeter Zijlstra * the perf code does is hlist_for_each_entry_rcu(), so we can
454466c81c4SPeter Zijlstra * get away with simply setting the @head.first pointer in order
455466c81c4SPeter Zijlstra * to create a singular list.
456466c81c4SPeter Zijlstra */
457466c81c4SPeter Zijlstra head.first = &event->hlist_entry;
458466c81c4SPeter Zijlstra
459ced39002SJiri Olsa #define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
460ced39002SJiri Olsa sizeof(u64)) - sizeof(u32))
461ced39002SJiri Olsa
462ced39002SJiri Olsa BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);
463ced39002SJiri Olsa
464ec5e099dSAlexei Starovoitov memset(®s, 0, sizeof(regs));
465ced39002SJiri Olsa perf_fetch_caller_regs(®s);
466ced39002SJiri Olsa
4671e1dcd93SAlexei Starovoitov entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
468ced39002SJiri Olsa if (!entry)
4695d15a624SSteven Rostedt (VMware) goto out;
470ced39002SJiri Olsa
471ced39002SJiri Olsa entry->ip = ip;
472ced39002SJiri Olsa entry->parent_ip = parent_ip;
4731e1dcd93SAlexei Starovoitov perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
474466c81c4SPeter Zijlstra 1, ®s, &head, NULL);
475ced39002SJiri Olsa
4765d15a624SSteven Rostedt (VMware) out:
4775d15a624SSteven Rostedt (VMware) ftrace_test_recursion_unlock(bit);
478ced39002SJiri Olsa #undef ENTRY_SIZE
479ced39002SJiri Olsa }
480ced39002SJiri Olsa
perf_ftrace_function_register(struct perf_event * event)481ced39002SJiri Olsa static int perf_ftrace_function_register(struct perf_event *event)
482ced39002SJiri Olsa {
483ced39002SJiri Olsa struct ftrace_ops *ops = &event->ftrace_ops;
484ced39002SJiri Olsa
485ced39002SJiri Olsa ops->func = perf_ftrace_function_call;
486466c81c4SPeter Zijlstra ops->private = (void *)(unsigned long)nr_cpu_ids;
487466c81c4SPeter Zijlstra
488ced39002SJiri Olsa return register_ftrace_function(ops);
489ced39002SJiri Olsa }
490ced39002SJiri Olsa
perf_ftrace_function_unregister(struct perf_event * event)491ced39002SJiri Olsa static int perf_ftrace_function_unregister(struct perf_event *event)
492ced39002SJiri Olsa {
493ced39002SJiri Olsa struct ftrace_ops *ops = &event->ftrace_ops;
4945500fa51SJiri Olsa int ret = unregister_ftrace_function(ops);
4955500fa51SJiri Olsa ftrace_free_filter(ops);
4965500fa51SJiri Olsa return ret;
497ced39002SJiri Olsa }
498ced39002SJiri Olsa
perf_ftrace_event_register(struct trace_event_call * call,enum trace_reg type,void * data)4992425bcb9SSteven Rostedt (Red Hat) int perf_ftrace_event_register(struct trace_event_call *call,
500ced39002SJiri Olsa enum trace_reg type, void *data)
501ced39002SJiri Olsa {
502466c81c4SPeter Zijlstra struct perf_event *event = data;
503466c81c4SPeter Zijlstra
504ced39002SJiri Olsa switch (type) {
505ced39002SJiri Olsa case TRACE_REG_REGISTER:
506ced39002SJiri Olsa case TRACE_REG_UNREGISTER:
507ced39002SJiri Olsa break;
508ced39002SJiri Olsa case TRACE_REG_PERF_REGISTER:
509ced39002SJiri Olsa case TRACE_REG_PERF_UNREGISTER:
510ced39002SJiri Olsa return 0;
511ced39002SJiri Olsa case TRACE_REG_PERF_OPEN:
512ced39002SJiri Olsa return perf_ftrace_function_register(data);
513ced39002SJiri Olsa case TRACE_REG_PERF_CLOSE:
514ced39002SJiri Olsa return perf_ftrace_function_unregister(data);
515ced39002SJiri Olsa case TRACE_REG_PERF_ADD:
516466c81c4SPeter Zijlstra event->ftrace_ops.private = (void *)(unsigned long)smp_processor_id();
517466c81c4SPeter Zijlstra return 1;
518ced39002SJiri Olsa case TRACE_REG_PERF_DEL:
519466c81c4SPeter Zijlstra event->ftrace_ops.private = (void *)(unsigned long)nr_cpu_ids;
520466c81c4SPeter Zijlstra return 1;
521ced39002SJiri Olsa }
522ced39002SJiri Olsa
523ced39002SJiri Olsa return -EINVAL;
524ced39002SJiri Olsa }
525ced39002SJiri Olsa #endif /* CONFIG_FUNCTION_TRACER */
526