1 /* 2 * Performance events callchain code, extracted from core.c: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 7 * Copyright � 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/perf_event.h> 13 #include <linux/slab.h> 14 #include "internal.h" 15 16 struct callchain_cpus_entries { 17 struct rcu_head rcu_head; 18 struct perf_callchain_entry *cpu_entries[0]; 19 }; 20 21 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); 22 static atomic_t nr_callchain_events; 23 static DEFINE_MUTEX(callchain_mutex); 24 static struct callchain_cpus_entries *callchain_cpus_entries; 25 26 27 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry, 28 struct pt_regs *regs) 29 { 30 } 31 32 __weak void perf_callchain_user(struct perf_callchain_entry *entry, 33 struct pt_regs *regs) 34 { 35 } 36 37 static void release_callchain_buffers_rcu(struct rcu_head *head) 38 { 39 struct callchain_cpus_entries *entries; 40 int cpu; 41 42 entries = container_of(head, struct callchain_cpus_entries, rcu_head); 43 44 for_each_possible_cpu(cpu) 45 kfree(entries->cpu_entries[cpu]); 46 47 kfree(entries); 48 } 49 50 static void release_callchain_buffers(void) 51 { 52 struct callchain_cpus_entries *entries; 53 54 entries = callchain_cpus_entries; 55 rcu_assign_pointer(callchain_cpus_entries, NULL); 56 call_rcu(&entries->rcu_head, release_callchain_buffers_rcu); 57 } 58 59 static int alloc_callchain_buffers(void) 60 { 61 int cpu; 62 int size; 63 struct callchain_cpus_entries *entries; 64 65 /* 66 * We can't use the percpu allocation API for data that can be 67 * accessed from NMI. Use a temporary manual per cpu allocation 68 * until that gets sorted out. 69 */ 70 size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]); 71 72 entries = kzalloc(size, GFP_KERNEL); 73 if (!entries) 74 return -ENOMEM; 75 76 size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS; 77 78 for_each_possible_cpu(cpu) { 79 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL, 80 cpu_to_node(cpu)); 81 if (!entries->cpu_entries[cpu]) 82 goto fail; 83 } 84 85 rcu_assign_pointer(callchain_cpus_entries, entries); 86 87 return 0; 88 89 fail: 90 for_each_possible_cpu(cpu) 91 kfree(entries->cpu_entries[cpu]); 92 kfree(entries); 93 94 return -ENOMEM; 95 } 96 97 int get_callchain_buffers(void) 98 { 99 int err = 0; 100 int count; 101 102 mutex_lock(&callchain_mutex); 103 104 count = atomic_inc_return(&nr_callchain_events); 105 if (WARN_ON_ONCE(count < 1)) { 106 err = -EINVAL; 107 goto exit; 108 } 109 110 if (count > 1) { 111 /* If the allocation failed, give up */ 112 if (!callchain_cpus_entries) 113 err = -ENOMEM; 114 goto exit; 115 } 116 117 err = alloc_callchain_buffers(); 118 if (err) 119 release_callchain_buffers(); 120 exit: 121 mutex_unlock(&callchain_mutex); 122 123 return err; 124 } 125 126 void put_callchain_buffers(void) 127 { 128 if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) { 129 release_callchain_buffers(); 130 mutex_unlock(&callchain_mutex); 131 } 132 } 133 134 static struct perf_callchain_entry *get_callchain_entry(int *rctx) 135 { 136 int cpu; 137 struct callchain_cpus_entries *entries; 138 139 *rctx = get_recursion_context(__get_cpu_var(callchain_recursion)); 140 if (*rctx == -1) 141 return NULL; 142 143 entries = rcu_dereference(callchain_cpus_entries); 144 if (!entries) 145 return NULL; 146 147 cpu = smp_processor_id(); 148 149 return &entries->cpu_entries[cpu][*rctx]; 150 } 151 152 static void 153 put_callchain_entry(int rctx) 154 { 155 put_recursion_context(__get_cpu_var(callchain_recursion), rctx); 156 } 157 158 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) 159 { 160 int rctx; 161 struct perf_callchain_entry *entry; 162 163 164 entry = get_callchain_entry(&rctx); 165 if (rctx == -1) 166 return NULL; 167 168 if (!entry) 169 goto exit_put; 170 171 entry->nr = 0; 172 173 if (!user_mode(regs)) { 174 perf_callchain_store(entry, PERF_CONTEXT_KERNEL); 175 perf_callchain_kernel(entry, regs); 176 if (current->mm) 177 regs = task_pt_regs(current); 178 else 179 regs = NULL; 180 } 181 182 if (regs) { 183 perf_callchain_store(entry, PERF_CONTEXT_USER); 184 perf_callchain_user(entry, regs); 185 } 186 187 exit_put: 188 put_callchain_entry(rctx); 189 190 return entry; 191 } 192