197d5a220SFrederic Weisbecker /* 297d5a220SFrederic Weisbecker * trace event based perf event profiling/tracing 397d5a220SFrederic Weisbecker * 497d5a220SFrederic Weisbecker * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com> 597d5a220SFrederic Weisbecker * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com> 697d5a220SFrederic Weisbecker */ 797d5a220SFrederic Weisbecker 897d5a220SFrederic Weisbecker #include <linux/module.h> 997d5a220SFrederic Weisbecker #include <linux/kprobes.h> 1097d5a220SFrederic Weisbecker #include "trace.h" 1197d5a220SFrederic Weisbecker 1297d5a220SFrederic Weisbecker DEFINE_PER_CPU(struct pt_regs, perf_trace_regs); 13639fe4b1SXiao Guangrong EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs); 1497d5a220SFrederic Weisbecker 15dcd5c166SFrederic Weisbecker EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); 16dcd5c166SFrederic Weisbecker 1797d5a220SFrederic Weisbecker static char *perf_trace_buf; 1897d5a220SFrederic Weisbecker static char *perf_trace_buf_nmi; 1997d5a220SFrederic Weisbecker 20*eb1e7961SFrederic Weisbecker /* 21*eb1e7961SFrederic Weisbecker * Force it to be aligned to unsigned long to avoid misaligned accesses 22*eb1e7961SFrederic Weisbecker * suprises 23*eb1e7961SFrederic Weisbecker */ 24*eb1e7961SFrederic Weisbecker typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)]) 25*eb1e7961SFrederic Weisbecker perf_trace_t; 2697d5a220SFrederic Weisbecker 2797d5a220SFrederic Weisbecker /* Count the events in use (per event id, not per instance) */ 2897d5a220SFrederic Weisbecker static int total_ref_count; 2997d5a220SFrederic Weisbecker 3097d5a220SFrederic Weisbecker static int perf_trace_event_enable(struct ftrace_event_call *event) 3197d5a220SFrederic Weisbecker { 3297d5a220SFrederic Weisbecker char *buf; 3397d5a220SFrederic Weisbecker int ret = -ENOMEM; 3497d5a220SFrederic Weisbecker 3597d5a220SFrederic Weisbecker if (event->perf_refcount++ > 0) 3697d5a220SFrederic Weisbecker return 0; 3797d5a220SFrederic Weisbecker 3897d5a220SFrederic Weisbecker if (!total_ref_count) { 3997d5a220SFrederic Weisbecker buf = (char *)alloc_percpu(perf_trace_t); 4097d5a220SFrederic Weisbecker if (!buf) 4197d5a220SFrederic Weisbecker goto fail_buf; 4297d5a220SFrederic Weisbecker 4397d5a220SFrederic Weisbecker rcu_assign_pointer(perf_trace_buf, buf); 4497d5a220SFrederic Weisbecker 4597d5a220SFrederic Weisbecker buf = (char *)alloc_percpu(perf_trace_t); 4697d5a220SFrederic Weisbecker if (!buf) 4797d5a220SFrederic Weisbecker goto fail_buf_nmi; 4897d5a220SFrederic Weisbecker 4997d5a220SFrederic Weisbecker rcu_assign_pointer(perf_trace_buf_nmi, buf); 5097d5a220SFrederic Weisbecker } 5197d5a220SFrederic Weisbecker 5297d5a220SFrederic Weisbecker ret = event->perf_event_enable(event); 5397d5a220SFrederic Weisbecker if (!ret) { 5497d5a220SFrederic Weisbecker total_ref_count++; 5597d5a220SFrederic Weisbecker return 0; 5697d5a220SFrederic Weisbecker } 5797d5a220SFrederic Weisbecker 5897d5a220SFrederic Weisbecker fail_buf_nmi: 5997d5a220SFrederic Weisbecker if (!total_ref_count) { 6097d5a220SFrederic Weisbecker free_percpu(perf_trace_buf_nmi); 6197d5a220SFrederic Weisbecker free_percpu(perf_trace_buf); 6297d5a220SFrederic Weisbecker perf_trace_buf_nmi = NULL; 6397d5a220SFrederic Weisbecker perf_trace_buf = NULL; 6497d5a220SFrederic Weisbecker } 6597d5a220SFrederic Weisbecker fail_buf: 6697d5a220SFrederic Weisbecker event->perf_refcount--; 6797d5a220SFrederic Weisbecker 6897d5a220SFrederic Weisbecker return ret; 6997d5a220SFrederic Weisbecker } 7097d5a220SFrederic Weisbecker 7197d5a220SFrederic Weisbecker int perf_trace_enable(int event_id) 7297d5a220SFrederic Weisbecker { 7397d5a220SFrederic Weisbecker struct ftrace_event_call *event; 7497d5a220SFrederic Weisbecker int ret = -EINVAL; 7597d5a220SFrederic Weisbecker 7697d5a220SFrederic Weisbecker mutex_lock(&event_mutex); 7797d5a220SFrederic Weisbecker list_for_each_entry(event, &ftrace_events, list) { 7897d5a220SFrederic Weisbecker if (event->id == event_id && event->perf_event_enable && 7997d5a220SFrederic Weisbecker try_module_get(event->mod)) { 8097d5a220SFrederic Weisbecker ret = perf_trace_event_enable(event); 8197d5a220SFrederic Weisbecker break; 8297d5a220SFrederic Weisbecker } 8397d5a220SFrederic Weisbecker } 8497d5a220SFrederic Weisbecker mutex_unlock(&event_mutex); 8597d5a220SFrederic Weisbecker 8697d5a220SFrederic Weisbecker return ret; 8797d5a220SFrederic Weisbecker } 8897d5a220SFrederic Weisbecker 8997d5a220SFrederic Weisbecker static void perf_trace_event_disable(struct ftrace_event_call *event) 9097d5a220SFrederic Weisbecker { 9197d5a220SFrederic Weisbecker char *buf, *nmi_buf; 9297d5a220SFrederic Weisbecker 9397d5a220SFrederic Weisbecker if (--event->perf_refcount > 0) 9497d5a220SFrederic Weisbecker return; 9597d5a220SFrederic Weisbecker 9697d5a220SFrederic Weisbecker event->perf_event_disable(event); 9797d5a220SFrederic Weisbecker 9897d5a220SFrederic Weisbecker if (!--total_ref_count) { 9997d5a220SFrederic Weisbecker buf = perf_trace_buf; 10097d5a220SFrederic Weisbecker rcu_assign_pointer(perf_trace_buf, NULL); 10197d5a220SFrederic Weisbecker 10297d5a220SFrederic Weisbecker nmi_buf = perf_trace_buf_nmi; 10397d5a220SFrederic Weisbecker rcu_assign_pointer(perf_trace_buf_nmi, NULL); 10497d5a220SFrederic Weisbecker 10597d5a220SFrederic Weisbecker /* 10697d5a220SFrederic Weisbecker * Ensure every events in profiling have finished before 10797d5a220SFrederic Weisbecker * releasing the buffers 10897d5a220SFrederic Weisbecker */ 10997d5a220SFrederic Weisbecker synchronize_sched(); 11097d5a220SFrederic Weisbecker 11197d5a220SFrederic Weisbecker free_percpu(buf); 11297d5a220SFrederic Weisbecker free_percpu(nmi_buf); 11397d5a220SFrederic Weisbecker } 11497d5a220SFrederic Weisbecker } 11597d5a220SFrederic Weisbecker 11697d5a220SFrederic Weisbecker void perf_trace_disable(int event_id) 11797d5a220SFrederic Weisbecker { 11897d5a220SFrederic Weisbecker struct ftrace_event_call *event; 11997d5a220SFrederic Weisbecker 12097d5a220SFrederic Weisbecker mutex_lock(&event_mutex); 12197d5a220SFrederic Weisbecker list_for_each_entry(event, &ftrace_events, list) { 12297d5a220SFrederic Weisbecker if (event->id == event_id) { 12397d5a220SFrederic Weisbecker perf_trace_event_disable(event); 12497d5a220SFrederic Weisbecker module_put(event->mod); 12597d5a220SFrederic Weisbecker break; 12697d5a220SFrederic Weisbecker } 12797d5a220SFrederic Weisbecker } 12897d5a220SFrederic Weisbecker mutex_unlock(&event_mutex); 12997d5a220SFrederic Weisbecker } 13097d5a220SFrederic Weisbecker 13197d5a220SFrederic Weisbecker __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, 13297d5a220SFrederic Weisbecker int *rctxp, unsigned long *irq_flags) 13397d5a220SFrederic Weisbecker { 13497d5a220SFrederic Weisbecker struct trace_entry *entry; 13597d5a220SFrederic Weisbecker char *trace_buf, *raw_data; 13697d5a220SFrederic Weisbecker int pc, cpu; 13797d5a220SFrederic Weisbecker 138*eb1e7961SFrederic Weisbecker BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long)); 139*eb1e7961SFrederic Weisbecker 14097d5a220SFrederic Weisbecker pc = preempt_count(); 14197d5a220SFrederic Weisbecker 14297d5a220SFrederic Weisbecker /* Protect the per cpu buffer, begin the rcu read side */ 14397d5a220SFrederic Weisbecker local_irq_save(*irq_flags); 14497d5a220SFrederic Weisbecker 14597d5a220SFrederic Weisbecker *rctxp = perf_swevent_get_recursion_context(); 14697d5a220SFrederic Weisbecker if (*rctxp < 0) 14797d5a220SFrederic Weisbecker goto err_recursion; 14897d5a220SFrederic Weisbecker 14997d5a220SFrederic Weisbecker cpu = smp_processor_id(); 15097d5a220SFrederic Weisbecker 15197d5a220SFrederic Weisbecker if (in_nmi()) 152f82c37e7SLinus Torvalds trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); 15397d5a220SFrederic Weisbecker else 154f82c37e7SLinus Torvalds trace_buf = rcu_dereference_sched(perf_trace_buf); 15597d5a220SFrederic Weisbecker 15697d5a220SFrederic Weisbecker if (!trace_buf) 15797d5a220SFrederic Weisbecker goto err; 15897d5a220SFrederic Weisbecker 15997d5a220SFrederic Weisbecker raw_data = per_cpu_ptr(trace_buf, cpu); 16097d5a220SFrederic Weisbecker 16197d5a220SFrederic Weisbecker /* zero the dead bytes from align to not leak stack to user */ 162*eb1e7961SFrederic Weisbecker memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64)); 16397d5a220SFrederic Weisbecker 16497d5a220SFrederic Weisbecker entry = (struct trace_entry *)raw_data; 16597d5a220SFrederic Weisbecker tracing_generic_entry_update(entry, *irq_flags, pc); 16697d5a220SFrederic Weisbecker entry->type = type; 16797d5a220SFrederic Weisbecker 16897d5a220SFrederic Weisbecker return raw_data; 16997d5a220SFrederic Weisbecker err: 17097d5a220SFrederic Weisbecker perf_swevent_put_recursion_context(*rctxp); 17197d5a220SFrederic Weisbecker err_recursion: 17297d5a220SFrederic Weisbecker local_irq_restore(*irq_flags); 17397d5a220SFrederic Weisbecker return NULL; 17497d5a220SFrederic Weisbecker } 17597d5a220SFrederic Weisbecker EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); 176