1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * unlikely profiler 4 * 5 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> 6 */ 7 #include <linux/kallsyms.h> 8 #include <linux/seq_file.h> 9 #include <linux/spinlock.h> 10 #include <linux/irqflags.h> 11 #include <linux/uaccess.h> 12 #include <linux/module.h> 13 #include <linux/ftrace.h> 14 #include <linux/hash.h> 15 #include <linux/fs.h> 16 #include <asm/local.h> 17 18 #include "trace.h" 19 #include "trace_stat.h" 20 #include "trace_output.h" 21 22 #ifdef CONFIG_BRANCH_TRACER 23 24 static struct tracer branch_trace; 25 static int branch_tracing_enabled __read_mostly; 26 static DEFINE_MUTEX(branch_tracing_mutex); 27 28 static struct trace_array *branch_tracer; 29 30 static void 31 probe_likely_condition(struct ftrace_likely_data *f, int val, int expect) 32 { 33 struct trace_array *tr = branch_tracer; 34 struct trace_buffer *buffer; 35 struct ring_buffer_event *event; 36 struct trace_branch *entry; 37 unsigned long flags; 38 unsigned int trace_ctx; 39 const char *p; 40 41 if (current->trace_recursion & TRACE_BRANCH_BIT) 42 return; 43 44 /* 45 * I would love to save just the ftrace_likely_data pointer, but 46 * this code can also be used by modules. Ugly things can happen 47 * if the module is unloaded, and then we go and read the 48 * pointer. This is slower, but much safer. 49 */ 50 51 if (unlikely(!tr)) 52 return; 53 54 raw_local_irq_save(flags); 55 current->trace_recursion |= TRACE_BRANCH_BIT; 56 if (!tracer_tracing_is_on_cpu(tr, raw_smp_processor_id())) 57 goto out; 58 59 trace_ctx = tracing_gen_ctx_flags(flags); 60 buffer = tr->array_buffer.buffer; 61 event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH, 62 sizeof(*entry), trace_ctx); 63 if (!event) 64 goto out; 65 66 entry = ring_buffer_event_data(event); 67 68 /* Strip off the path, only save the file */ 69 p = f->data.file + strlen(f->data.file); 70 while (p >= f->data.file && *p != '/') 71 p--; 72 p++; 73 74 strscpy(entry->func, f->data.func); 75 strscpy(entry->file, p); 76 entry->constant = f->constant; 77 entry->line = f->data.line; 78 entry->correct = val == expect; 79 80 trace_buffer_unlock_commit_nostack(buffer, event); 81 82 out: 83 current->trace_recursion &= ~TRACE_BRANCH_BIT; 84 raw_local_irq_restore(flags); 85 } 86 87 static inline 88 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect) 89 { 90 if (!branch_tracing_enabled) 91 return; 92 93 probe_likely_condition(f, val, expect); 94 } 95 96 int enable_branch_tracing(struct trace_array *tr) 97 { 98 mutex_lock(&branch_tracing_mutex); 99 branch_tracer = tr; 100 /* 101 * Must be seen before enabling. The reader is a condition 102 * where we do not need a matching rmb() 103 */ 104 smp_wmb(); 105 branch_tracing_enabled++; 106 mutex_unlock(&branch_tracing_mutex); 107 108 return 0; 109 } 110 111 void disable_branch_tracing(void) 112 { 113 mutex_lock(&branch_tracing_mutex); 114 115 if (!branch_tracing_enabled) 116 goto out_unlock; 117 118 branch_tracing_enabled--; 119 120 out_unlock: 121 mutex_unlock(&branch_tracing_mutex); 122 } 123 124 static int branch_trace_init(struct trace_array *tr) 125 { 126 return enable_branch_tracing(tr); 127 } 128 129 static void branch_trace_reset(struct trace_array *tr) 130 { 131 disable_branch_tracing(); 132 } 133 134 static enum print_line_t trace_branch_print(struct trace_iterator *iter, 135 int flags, struct trace_event *event) 136 { 137 struct trace_branch *field; 138 139 trace_assign_type(field, iter->ent); 140 141 trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n", 142 field->correct ? " ok " : " MISS ", 143 field->func, 144 field->file, 145 field->line); 146 147 return trace_handle_return(&iter->seq); 148 } 149 150 static void branch_print_header(struct seq_file *s) 151 { 152 seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT" 153 " FUNC:FILE:LINE\n" 154 "# | | | | | " 155 " |\n"); 156 } 157 158 static struct trace_event_functions trace_branch_funcs = { 159 .trace = trace_branch_print, 160 }; 161 162 static struct trace_event trace_branch_event = { 163 .type = TRACE_BRANCH, 164 .funcs = &trace_branch_funcs, 165 }; 166 167 static struct tracer branch_trace __read_mostly = 168 { 169 .name = "branch", 170 .init = branch_trace_init, 171 .reset = branch_trace_reset, 172 #ifdef CONFIG_FTRACE_SELFTEST 173 .selftest = trace_selftest_startup_branch, 174 #endif /* CONFIG_FTRACE_SELFTEST */ 175 .print_header = branch_print_header, 176 }; 177 178 __init static int init_branch_tracer(void) 179 { 180 int ret; 181 182 ret = register_trace_event(&trace_branch_event); 183 if (!ret) { 184 pr_warn("Warning: could not register branch events\n"); 185 return 1; 186 } 187 return register_tracer(&branch_trace); 188 } 189 core_initcall(init_branch_tracer); 190 191 #else 192 static inline 193 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect) 194 { 195 } 196 #endif /* CONFIG_BRANCH_TRACER */ 197 198 void ftrace_likely_update(struct ftrace_likely_data *f, int val, 199 int expect, int is_constant) 200 { 201 unsigned long flags = user_access_save(); 202 203 /* A constant is always correct */ 204 if (is_constant) { 205 f->constant++; 206 val = expect; 207 } 208 /* 209 * I would love to have a trace point here instead, but the 210 * trace point code is so inundated with unlikely and likely 211 * conditions that the recursive nightmare that exists is too 212 * much to try to get working. At least for now. 213 */ 214 trace_likely_condition(f, val, expect); 215 216 /* FIXME: Make this atomic! */ 217 if (val == expect) 218 f->data.correct++; 219 else 220 f->data.incorrect++; 221 222 user_access_restore(flags); 223 } 224 EXPORT_SYMBOL(ftrace_likely_update); 225 226 extern unsigned long __start_annotated_branch_profile[]; 227 extern unsigned long __stop_annotated_branch_profile[]; 228 229 static int annotated_branch_stat_headers(struct seq_file *m) 230 { 231 seq_puts(m, " correct incorrect % " 232 " Function " 233 " File Line\n" 234 " ------- --------- - " 235 " -------- " 236 " ---- ----\n"); 237 return 0; 238 } 239 240 static inline long get_incorrect_percent(const struct ftrace_branch_data *p) 241 { 242 long percent; 243 244 if (p->correct) { 245 percent = p->incorrect * 100; 246 percent /= p->correct + p->incorrect; 247 } else 248 percent = p->incorrect ? 100 : -1; 249 250 return percent; 251 } 252 253 static const char *branch_stat_process_file(struct ftrace_branch_data *p) 254 { 255 const char *f; 256 257 /* Only print the file, not the path */ 258 f = p->file + strlen(p->file); 259 while (f >= p->file && *f != '/') 260 f--; 261 return ++f; 262 } 263 264 static void branch_stat_show(struct seq_file *m, 265 struct ftrace_branch_data *p, const char *f) 266 { 267 long percent; 268 269 /* 270 * The miss is overlayed on correct, and hit on incorrect. 271 */ 272 percent = get_incorrect_percent(p); 273 274 if (percent < 0) 275 seq_puts(m, " X "); 276 else 277 seq_printf(m, "%3ld ", percent); 278 279 seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line); 280 } 281 282 static int branch_stat_show_normal(struct seq_file *m, 283 struct ftrace_branch_data *p, const char *f) 284 { 285 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 286 branch_stat_show(m, p, f); 287 return 0; 288 } 289 290 static int annotate_branch_stat_show(struct seq_file *m, void *v) 291 { 292 struct ftrace_likely_data *p = v; 293 const char *f; 294 int l; 295 296 f = branch_stat_process_file(&p->data); 297 298 if (!p->constant) 299 return branch_stat_show_normal(m, &p->data, f); 300 301 l = snprintf(NULL, 0, "/%lu", p->constant); 302 l = l > 8 ? 0 : 8 - l; 303 304 seq_printf(m, "%8lu/%lu %*lu ", 305 p->data.correct, p->constant, l, p->data.incorrect); 306 branch_stat_show(m, &p->data, f); 307 return 0; 308 } 309 310 static void *annotated_branch_stat_start(struct tracer_stat *trace) 311 { 312 return __start_annotated_branch_profile; 313 } 314 315 static void * 316 annotated_branch_stat_next(void *v, int idx) 317 { 318 struct ftrace_likely_data *p = v; 319 320 ++p; 321 322 if ((void *)p >= (void *)__stop_annotated_branch_profile) 323 return NULL; 324 325 return p; 326 } 327 328 static int annotated_branch_stat_cmp(const void *p1, const void *p2) 329 { 330 const struct ftrace_branch_data *a = p1; 331 const struct ftrace_branch_data *b = p2; 332 333 long percent_a, percent_b; 334 335 percent_a = get_incorrect_percent(a); 336 percent_b = get_incorrect_percent(b); 337 338 if (percent_a < percent_b) 339 return -1; 340 if (percent_a > percent_b) 341 return 1; 342 343 if (a->incorrect < b->incorrect) 344 return -1; 345 if (a->incorrect > b->incorrect) 346 return 1; 347 348 /* 349 * Since the above shows worse (incorrect) cases 350 * first, we continue that by showing best (correct) 351 * cases last. 352 */ 353 if (a->correct > b->correct) 354 return -1; 355 if (a->correct < b->correct) 356 return 1; 357 358 return 0; 359 } 360 361 static struct tracer_stat annotated_branch_stats = { 362 .name = "branch_annotated", 363 .stat_start = annotated_branch_stat_start, 364 .stat_next = annotated_branch_stat_next, 365 .stat_cmp = annotated_branch_stat_cmp, 366 .stat_headers = annotated_branch_stat_headers, 367 .stat_show = annotate_branch_stat_show 368 }; 369 370 __init static int init_annotated_branch_stats(void) 371 { 372 int ret; 373 374 ret = register_stat_tracer(&annotated_branch_stats); 375 if (ret) { 376 pr_warn("Warning: could not register annotated branches stats\n"); 377 return ret; 378 } 379 return 0; 380 } 381 fs_initcall(init_annotated_branch_stats); 382 383 #ifdef CONFIG_PROFILE_ALL_BRANCHES 384 385 extern unsigned long __start_branch_profile[]; 386 extern unsigned long __stop_branch_profile[]; 387 388 static int all_branch_stat_headers(struct seq_file *m) 389 { 390 seq_puts(m, " miss hit % " 391 " Function " 392 " File Line\n" 393 " ------- --------- - " 394 " -------- " 395 " ---- ----\n"); 396 return 0; 397 } 398 399 static void *all_branch_stat_start(struct tracer_stat *trace) 400 { 401 return __start_branch_profile; 402 } 403 404 static void * 405 all_branch_stat_next(void *v, int idx) 406 { 407 struct ftrace_branch_data *p = v; 408 409 ++p; 410 411 if ((void *)p >= (void *)__stop_branch_profile) 412 return NULL; 413 414 return p; 415 } 416 417 static int all_branch_stat_show(struct seq_file *m, void *v) 418 { 419 struct ftrace_branch_data *p = v; 420 const char *f; 421 422 f = branch_stat_process_file(p); 423 return branch_stat_show_normal(m, p, f); 424 } 425 426 static struct tracer_stat all_branch_stats = { 427 .name = "branch_all", 428 .stat_start = all_branch_stat_start, 429 .stat_next = all_branch_stat_next, 430 .stat_headers = all_branch_stat_headers, 431 .stat_show = all_branch_stat_show 432 }; 433 434 __init static int all_annotated_branch_stats(void) 435 { 436 int ret; 437 438 ret = register_stat_tracer(&all_branch_stats); 439 if (ret) { 440 pr_warn("Warning: could not register all branches stats\n"); 441 return ret; 442 } 443 return 0; 444 } 445 fs_initcall(all_annotated_branch_stats); 446 #endif /* CONFIG_PROFILE_ALL_BRANCHES */ 447