1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Ftrace header. For implementation details beyond the random comments
4 * scattered below, see: Documentation/trace/ftrace-design.rst
5 */
6
7 #ifndef _LINUX_FTRACE_H
8 #define _LINUX_FTRACE_H
9
10 #include <linux/trace_recursion.h>
11 #include <linux/trace_clock.h>
12 #include <linux/jump_label.h>
13 #include <linux/kallsyms.h>
14 #include <linux/linkage.h>
15 #include <linux/bitops.h>
16 #include <linux/ptrace.h>
17 #include <linux/ktime.h>
18 #include <linux/sched.h>
19 #include <linux/types.h>
20 #include <linux/init.h>
21 #include <linux/fs.h>
22
23 #include <asm/ftrace.h>
24
25 /*
26 * If the arch supports passing the variable contents of
27 * function_trace_op as the third parameter back from the
28 * mcount call, then the arch should define this as 1.
29 */
30 #ifndef ARCH_SUPPORTS_FTRACE_OPS
31 #define ARCH_SUPPORTS_FTRACE_OPS 0
32 #endif
33
34 #ifdef CONFIG_TRACER_SNAPSHOT
35 extern void ftrace_boot_snapshot(void);
36 #else
ftrace_boot_snapshot(void)37 static inline void ftrace_boot_snapshot(void) { }
38 #endif
39
40 struct ftrace_ops;
41 struct ftrace_regs;
42 struct dyn_ftrace;
43
44 char *arch_ftrace_match_adjust(char *str, const char *search);
45
46 #ifdef CONFIG_HAVE_FUNCTION_GRAPH_FREGS
47 unsigned long ftrace_return_to_handler(struct ftrace_regs *fregs);
48 #else
49 unsigned long ftrace_return_to_handler(unsigned long frame_pointer);
50 #endif
51
52 #ifdef CONFIG_FUNCTION_TRACER
53 /*
54 * If the arch's mcount caller does not support all of ftrace's
55 * features, then it must call an indirect function that
56 * does. Or at least does enough to prevent any unwelcome side effects.
57 *
58 * Also define the function prototype that these architectures use
59 * to call the ftrace_ops_list_func().
60 */
61 #if !ARCH_SUPPORTS_FTRACE_OPS
62 # define FTRACE_FORCE_LIST_FUNC 1
63 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
64 #else
65 # define FTRACE_FORCE_LIST_FUNC 0
66 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
67 struct ftrace_ops *op, struct ftrace_regs *fregs);
68 #endif
69 extern const struct ftrace_ops ftrace_nop_ops;
70 extern const struct ftrace_ops ftrace_list_ops;
71 struct ftrace_ops *ftrace_find_unique_ops(struct dyn_ftrace *rec);
72 #endif /* CONFIG_FUNCTION_TRACER */
73
74 /* Main tracing buffer and events set up */
75 #ifdef CONFIG_TRACING
76 void trace_init(void);
77 void early_trace_init(void);
78 #else
trace_init(void)79 static inline void trace_init(void) { }
early_trace_init(void)80 static inline void early_trace_init(void) { }
81 #endif
82
83 struct module;
84 struct ftrace_hash;
85 struct ftrace_func_entry;
86
87 #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_MODULES) && \
88 defined(CONFIG_DYNAMIC_FTRACE)
89 int
90 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
91 unsigned long *off, char **modname,
92 const unsigned char **modbuildid, char *sym);
93 #else
94 static inline int
ftrace_mod_address_lookup(unsigned long addr,unsigned long * size,unsigned long * off,char ** modname,const unsigned char ** modbuildid,char * sym)95 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
96 unsigned long *off, char **modname,
97 const unsigned char **modbuildid, char *sym)
98 {
99 return 0;
100 }
101 #endif
102
103 #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
104 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
105 char *type, char *name,
106 char *module_name, int *exported);
107 #else
ftrace_mod_get_kallsym(unsigned int symnum,unsigned long * value,char * type,char * name,char * module_name,int * exported)108 static inline int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
109 char *type, char *name,
110 char *module_name, int *exported)
111 {
112 return -1;
113 }
114 #endif
115
116 #ifdef CONFIG_FUNCTION_TRACER
117
118 #include <linux/ftrace_regs.h>
119
120 extern int ftrace_enabled;
121
122 /**
123 * ftrace_regs - ftrace partial/optimal register set
124 *
125 * ftrace_regs represents a group of registers which is used at the
126 * function entry and exit. There are three types of registers.
127 *
128 * - Registers for passing the parameters to callee, including the stack
129 * pointer. (e.g. rcx, rdx, rdi, rsi, r8, r9 and rsp on x86_64)
130 * - Registers for passing the return values to caller.
131 * (e.g. rax and rdx on x86_64)
132 * - Registers for hooking the function call and return including the
133 * frame pointer (the frame pointer is architecture/config dependent)
134 * (e.g. rip, rbp and rsp for x86_64)
135 *
136 * Also, architecture dependent fields can be used for internal process.
137 * (e.g. orig_ax on x86_64)
138 *
139 * Basically, ftrace_regs stores the registers related to the context.
140 * On function entry, registers for function parameters and hooking the
141 * function call are stored, and on function exit, registers for function
142 * return value and frame pointers are stored.
143 *
144 * And also, it dpends on the context that which registers are restored
145 * from the ftrace_regs.
146 * On the function entry, those registers will be restored except for
147 * the stack pointer, so that user can change the function parameters
148 * and instruction pointer (e.g. live patching.)
149 * On the function exit, only registers which is used for return values
150 * are restored.
151 *
152 * NOTE: user *must not* access regs directly, only do it via APIs, because
153 * the member can be changed according to the architecture.
154 * This is why the structure is empty here, so that nothing accesses
155 * the ftrace_regs directly.
156 */
157 struct ftrace_regs {
158 /* Nothing to see here, use the accessor functions! */
159 };
160
161 #define ftrace_regs_size() sizeof(struct __arch_ftrace_regs)
162
163 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
164 /*
165 * Architectures that define HAVE_DYNAMIC_FTRACE_WITH_ARGS must define their own
166 * arch_ftrace_get_regs() where it only returns pt_regs *if* it is fully
167 * populated. It should return NULL otherwise.
168 */
arch_ftrace_get_regs(struct ftrace_regs * fregs)169 static inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *fregs)
170 {
171 return &arch_ftrace_regs(fregs)->regs;
172 }
173
174 /*
175 * ftrace_regs_set_instruction_pointer() is to be defined by the architecture
176 * if to allow setting of the instruction pointer from the ftrace_regs when
177 * HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports live kernel patching.
178 */
179 #define ftrace_regs_set_instruction_pointer(fregs, ip) do { } while (0)
180 #endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
181
182 #ifdef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS
183
184 static_assert(sizeof(struct pt_regs) == ftrace_regs_size());
185
186 #endif /* CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */
187
ftrace_get_regs(struct ftrace_regs * fregs)188 static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
189 {
190 if (!fregs)
191 return NULL;
192
193 return arch_ftrace_get_regs(fregs);
194 }
195
196 #if !defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) || \
197 defined(CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS)
198
199 #ifndef arch_ftrace_partial_regs
200 #define arch_ftrace_partial_regs(regs) do {} while (0)
201 #endif
202
203 static __always_inline struct pt_regs *
ftrace_partial_regs(struct ftrace_regs * fregs,struct pt_regs * regs)204 ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
205 {
206 /*
207 * If CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS=y, ftrace_regs memory
208 * layout is including pt_regs. So always returns that address.
209 * Since arch_ftrace_get_regs() will check some members and may return
210 * NULL, we can not use it.
211 */
212 regs = &arch_ftrace_regs(fregs)->regs;
213
214 /* Allow arch specific updates to regs. */
215 arch_ftrace_partial_regs(regs);
216 return regs;
217 }
218
219 #endif /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS || CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */
220
221 #ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
222
223 /*
224 * Please define arch dependent pt_regs which compatible to the
225 * perf_arch_fetch_caller_regs() but based on ftrace_regs.
226 * This requires
227 * - user_mode(_regs) returns false (always kernel mode).
228 * - able to use the _regs for stack trace.
229 */
230 #ifndef arch_ftrace_fill_perf_regs
231 /* As same as perf_arch_fetch_caller_regs(), do nothing by default */
232 #define arch_ftrace_fill_perf_regs(fregs, _regs) do {} while (0)
233 #endif
234
235 static __always_inline struct pt_regs *
ftrace_fill_perf_regs(struct ftrace_regs * fregs,struct pt_regs * regs)236 ftrace_fill_perf_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
237 {
238 arch_ftrace_fill_perf_regs(fregs, regs);
239 return regs;
240 }
241
242 #else /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
243
244 static __always_inline struct pt_regs *
ftrace_fill_perf_regs(struct ftrace_regs * fregs,struct pt_regs * regs)245 ftrace_fill_perf_regs(struct ftrace_regs *fregs, struct pt_regs *regs)
246 {
247 return &arch_ftrace_regs(fregs)->regs;
248 }
249
250 #endif
251
252 /*
253 * When true, the ftrace_regs_{get,set}_*() functions may be used on fregs.
254 * Note: this can be true even when ftrace_get_regs() cannot provide a pt_regs.
255 */
ftrace_regs_has_args(struct ftrace_regs * fregs)256 static __always_inline bool ftrace_regs_has_args(struct ftrace_regs *fregs)
257 {
258 if (IS_ENABLED(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS))
259 return true;
260
261 return ftrace_get_regs(fregs) != NULL;
262 }
263
264 #ifdef CONFIG_HAVE_REGS_AND_STACK_ACCESS_API
265 static __always_inline unsigned long
ftrace_regs_get_kernel_stack_nth(struct ftrace_regs * fregs,unsigned int nth)266 ftrace_regs_get_kernel_stack_nth(struct ftrace_regs *fregs, unsigned int nth)
267 {
268 unsigned long *stackp;
269
270 stackp = (unsigned long *)ftrace_regs_get_stack_pointer(fregs);
271 if (((unsigned long)(stackp + nth) & ~(THREAD_SIZE - 1)) ==
272 ((unsigned long)stackp & ~(THREAD_SIZE - 1)))
273 return *(stackp + nth);
274
275 return 0;
276 }
277 #else /* !CONFIG_HAVE_REGS_AND_STACK_ACCESS_API */
278 #define ftrace_regs_get_kernel_stack_nth(fregs, nth) (0L)
279 #endif /* CONFIG_HAVE_REGS_AND_STACK_ACCESS_API */
280
281 typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
282 struct ftrace_ops *op, struct ftrace_regs *fregs);
283
284 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops);
285
286 /*
287 * FTRACE_OPS_FL_* bits denote the state of ftrace_ops struct and are
288 * set in the flags member.
289 * CONTROL, SAVE_REGS, SAVE_REGS_IF_SUPPORTED, RECURSION, STUB and
290 * IPMODIFY are a kind of attribute flags which can be set only before
291 * registering the ftrace_ops, and can not be modified while registered.
292 * Changing those attribute flags after registering ftrace_ops will
293 * cause unexpected results.
294 *
295 * ENABLED - set/unset when ftrace_ops is registered/unregistered
296 * DYNAMIC - set when ftrace_ops is registered to denote dynamically
297 * allocated ftrace_ops which need special care
298 * SAVE_REGS - The ftrace_ops wants regs saved at each function called
299 * and passed to the callback. If this flag is set, but the
300 * architecture does not support passing regs
301 * (CONFIG_DYNAMIC_FTRACE_WITH_REGS is not defined), then the
302 * ftrace_ops will fail to register, unless the next flag
303 * is set.
304 * SAVE_REGS_IF_SUPPORTED - This is the same as SAVE_REGS, but if the
305 * handler can handle an arch that does not save regs
306 * (the handler tests if regs == NULL), then it can set
307 * this flag instead. It will not fail registering the ftrace_ops
308 * but, the regs field will be NULL if the arch does not support
309 * passing regs to the handler.
310 * Note, if this flag is set, the SAVE_REGS flag will automatically
311 * get set upon registering the ftrace_ops, if the arch supports it.
312 * RECURSION - The ftrace_ops can set this to tell the ftrace infrastructure
313 * that the call back needs recursion protection. If it does
314 * not set this, then the ftrace infrastructure will assume
315 * that the callback can handle recursion on its own.
316 * STUB - The ftrace_ops is just a place holder.
317 * INITIALIZED - The ftrace_ops has already been initialized (first use time
318 * register_ftrace_function() is called, it will initialized the ops)
319 * DELETED - The ops are being deleted, do not let them be registered again.
320 * ADDING - The ops is in the process of being added.
321 * REMOVING - The ops is in the process of being removed.
322 * MODIFYING - The ops is in the process of changing its filter functions.
323 * ALLOC_TRAMP - A dynamic trampoline was allocated by the core code.
324 * The arch specific code sets this flag when it allocated a
325 * trampoline. This lets the arch know that it can update the
326 * trampoline in case the callback function changes.
327 * The ftrace_ops trampoline can be set by the ftrace users, and
328 * in such cases the arch must not modify it. Only the arch ftrace
329 * core code should set this flag.
330 * IPMODIFY - The ops can modify the IP register. This can only be set with
331 * SAVE_REGS. If another ops with this flag set is already registered
332 * for any of the functions that this ops will be registered for, then
333 * this ops will fail to register or set_filter_ip.
334 * PID - Is affected by set_ftrace_pid (allows filtering on those pids)
335 * RCU - Set when the ops can only be called when RCU is watching.
336 * TRACE_ARRAY - The ops->private points to a trace_array descriptor.
337 * PERMANENT - Set when the ops is permanent and should not be affected by
338 * ftrace_enabled.
339 * DIRECT - Used by the direct ftrace_ops helper for direct functions
340 * (internal ftrace only, should not be used by others)
341 * SUBOP - Is controlled by another op in field managed.
342 * GRAPH - Is a component of the fgraph_ops structure
343 */
344 enum {
345 FTRACE_OPS_FL_ENABLED = BIT(0),
346 FTRACE_OPS_FL_DYNAMIC = BIT(1),
347 FTRACE_OPS_FL_SAVE_REGS = BIT(2),
348 FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED = BIT(3),
349 FTRACE_OPS_FL_RECURSION = BIT(4),
350 FTRACE_OPS_FL_STUB = BIT(5),
351 FTRACE_OPS_FL_INITIALIZED = BIT(6),
352 FTRACE_OPS_FL_DELETED = BIT(7),
353 FTRACE_OPS_FL_ADDING = BIT(8),
354 FTRACE_OPS_FL_REMOVING = BIT(9),
355 FTRACE_OPS_FL_MODIFYING = BIT(10),
356 FTRACE_OPS_FL_ALLOC_TRAMP = BIT(11),
357 FTRACE_OPS_FL_IPMODIFY = BIT(12),
358 FTRACE_OPS_FL_PID = BIT(13),
359 FTRACE_OPS_FL_RCU = BIT(14),
360 FTRACE_OPS_FL_TRACE_ARRAY = BIT(15),
361 FTRACE_OPS_FL_PERMANENT = BIT(16),
362 FTRACE_OPS_FL_DIRECT = BIT(17),
363 FTRACE_OPS_FL_SUBOP = BIT(18),
364 FTRACE_OPS_FL_GRAPH = BIT(19),
365 };
366
367 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
368 #define FTRACE_OPS_FL_SAVE_ARGS FTRACE_OPS_FL_SAVE_REGS
369 #else
370 #define FTRACE_OPS_FL_SAVE_ARGS 0
371 #endif
372
373 /*
374 * FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes
375 * to a ftrace_ops. Note, the requests may fail.
376 *
377 * ENABLE_SHARE_IPMODIFY_SELF - enable a DIRECT ops to work on the same
378 * function as an ops with IPMODIFY. Called
379 * when the DIRECT ops is being registered.
380 * This is called with both direct_mutex and
381 * ftrace_lock are locked.
382 *
383 * ENABLE_SHARE_IPMODIFY_PEER - enable a DIRECT ops to work on the same
384 * function as an ops with IPMODIFY. Called
385 * when the other ops (the one with IPMODIFY)
386 * is being registered.
387 * This is called with direct_mutex locked.
388 *
389 * DISABLE_SHARE_IPMODIFY_PEER - disable a DIRECT ops to work on the same
390 * function as an ops with IPMODIFY. Called
391 * when the other ops (the one with IPMODIFY)
392 * is being unregistered.
393 * This is called with direct_mutex locked.
394 */
395 enum ftrace_ops_cmd {
396 FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF,
397 FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER,
398 FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER,
399 };
400
401 /*
402 * For most ftrace_ops_cmd,
403 * Returns:
404 * 0 - Success.
405 * Negative on failure. The return value is dependent on the
406 * callback.
407 */
408 typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, unsigned long ip, enum ftrace_ops_cmd cmd);
409
410 #ifdef CONFIG_DYNAMIC_FTRACE
411
412 #define FTRACE_HASH_DEFAULT_BITS 10
413
414 struct ftrace_hash *alloc_ftrace_hash(int size_bits);
415 void free_ftrace_hash(struct ftrace_hash *hash);
416 struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash,
417 unsigned long ip, unsigned long direct);
418 void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry);
419 void ftrace_hash_remove(struct ftrace_hash *hash);
420
421 /* The hash used to know what functions callbacks trace */
422 struct ftrace_ops_hash {
423 struct ftrace_hash __rcu *notrace_hash;
424 struct ftrace_hash __rcu *filter_hash;
425 struct mutex regex_lock;
426 };
427
428 void ftrace_free_init_mem(void);
429 void ftrace_free_mem(struct module *mod, void *start, void *end);
430 #else
ftrace_free_init_mem(void)431 static inline void ftrace_free_init_mem(void)
432 {
433 ftrace_boot_snapshot();
434 }
ftrace_free_mem(struct module * mod,void * start,void * end)435 static inline void ftrace_free_mem(struct module *mod, void *start, void *end) { }
436 #endif
437
438 /*
439 * Note, ftrace_ops can be referenced outside of RCU protection, unless
440 * the RCU flag is set. If ftrace_ops is allocated and not part of kernel
441 * core data, the unregistering of it will perform a scheduling on all CPUs
442 * to make sure that there are no more users. Depending on the load of the
443 * system that may take a bit of time.
444 *
445 * Any private data added must also take care not to be freed and if private
446 * data is added to a ftrace_ops that is in core code, the user of the
447 * ftrace_ops must perform a schedule_on_each_cpu() before freeing it.
448 */
449 struct ftrace_ops {
450 ftrace_func_t func;
451 struct ftrace_ops __rcu *next;
452 unsigned long flags;
453 void *private;
454 ftrace_func_t saved_func;
455 #ifdef CONFIG_DYNAMIC_FTRACE
456 struct ftrace_ops_hash local_hash;
457 struct ftrace_ops_hash *func_hash;
458 struct ftrace_ops_hash old_hash;
459 unsigned long trampoline;
460 unsigned long trampoline_size;
461 struct list_head list;
462 struct list_head subop_list;
463 ftrace_ops_func_t ops_func;
464 struct ftrace_ops *managed;
465 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
466 unsigned long direct_call;
467 #endif
468 #endif
469 };
470
471 extern struct ftrace_ops __rcu *ftrace_ops_list;
472 extern struct ftrace_ops ftrace_list_end;
473
474 /*
475 * Traverse the ftrace_ops_list, invoking all entries. The reason that we
476 * can use rcu_dereference_raw_check() is that elements removed from this list
477 * are simply leaked, so there is no need to interact with a grace-period
478 * mechanism. The rcu_dereference_raw_check() calls are needed to handle
479 * concurrent insertions into the ftrace_ops_list.
480 *
481 * Silly Alpha and silly pointer-speculation compiler optimizations!
482 */
483 #define do_for_each_ftrace_op(op, list) \
484 op = rcu_dereference_raw_check(list); \
485 do
486
487 /*
488 * Optimized for just a single item in the list (as that is the normal case).
489 */
490 #define while_for_each_ftrace_op(op) \
491 while (likely(op = rcu_dereference_raw_check((op)->next)) && \
492 unlikely((op) != &ftrace_list_end))
493
494 /*
495 * Type of the current tracing.
496 */
497 enum ftrace_tracing_type_t {
498 FTRACE_TYPE_ENTER = 0, /* Hook the call of the function */
499 FTRACE_TYPE_RETURN, /* Hook the return of the function */
500 };
501
502 /* Current tracing type, default is FTRACE_TYPE_ENTER */
503 extern enum ftrace_tracing_type_t ftrace_tracing_type;
504
505 /*
506 * The ftrace_ops must be a static and should also
507 * be read_mostly. These functions do modify read_mostly variables
508 * so use them sparely. Never free an ftrace_op or modify the
509 * next pointer after it has been registered. Even after unregistering
510 * it, the next pointer may still be used internally.
511 */
512 int register_ftrace_function(struct ftrace_ops *ops);
513 int unregister_ftrace_function(struct ftrace_ops *ops);
514
515 extern void ftrace_stub(unsigned long a0, unsigned long a1,
516 struct ftrace_ops *op, struct ftrace_regs *fregs);
517
518
519 int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs);
520 #else /* !CONFIG_FUNCTION_TRACER */
521 /*
522 * (un)register_ftrace_function must be a macro since the ops parameter
523 * must not be evaluated.
524 */
525 #define register_ftrace_function(ops) ({ 0; })
526 #define unregister_ftrace_function(ops) ({ 0; })
ftrace_kill(void)527 static inline void ftrace_kill(void) { }
ftrace_free_init_mem(void)528 static inline void ftrace_free_init_mem(void) { }
ftrace_free_mem(struct module * mod,void * start,void * end)529 static inline void ftrace_free_mem(struct module *mod, void *start, void *end) { }
ftrace_lookup_symbols(const char ** sorted_syms,size_t cnt,unsigned long * addrs)530 static inline int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs)
531 {
532 return -EOPNOTSUPP;
533 }
534 #endif /* CONFIG_FUNCTION_TRACER */
535
536 struct ftrace_func_entry {
537 struct hlist_node hlist;
538 unsigned long ip;
539 unsigned long direct; /* for direct lookup only */
540 };
541
542 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
543 unsigned long ftrace_find_rec_direct(unsigned long ip);
544 int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
545 int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
546 bool free_filters);
547 int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
548 int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr);
549
550 int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash);
551 int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash);
552 int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock);
553
554 void ftrace_stub_direct_tramp(void);
555
556 unsigned long ftrace_hash_count(struct ftrace_hash *hash);
557
558 #else
559 struct ftrace_ops;
ftrace_find_rec_direct(unsigned long ip)560 static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
561 {
562 return 0;
563 }
register_ftrace_direct(struct ftrace_ops * ops,unsigned long addr)564 static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
565 {
566 return -ENODEV;
567 }
unregister_ftrace_direct(struct ftrace_ops * ops,unsigned long addr,bool free_filters)568 static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
569 bool free_filters)
570 {
571 return -ENODEV;
572 }
modify_ftrace_direct(struct ftrace_ops * ops,unsigned long addr)573 static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
574 {
575 return -ENODEV;
576 }
modify_ftrace_direct_nolock(struct ftrace_ops * ops,unsigned long addr)577 static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
578 {
579 return -ENODEV;
580 }
581
update_ftrace_direct_add(struct ftrace_ops * ops,struct ftrace_hash * hash)582 static inline int update_ftrace_direct_add(struct ftrace_ops *ops, struct ftrace_hash *hash)
583 {
584 return -ENODEV;
585 }
586
update_ftrace_direct_del(struct ftrace_ops * ops,struct ftrace_hash * hash)587 static inline int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
588 {
589 return -ENODEV;
590 }
591
update_ftrace_direct_mod(struct ftrace_ops * ops,struct ftrace_hash * hash,bool do_direct_lock)592 static inline int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, bool do_direct_lock)
593 {
594 return -ENODEV;
595 }
596
ftrace_hash_count(struct ftrace_hash * hash)597 static inline unsigned long ftrace_hash_count(struct ftrace_hash *hash)
598 {
599 return 0;
600 }
601
602 /*
603 * This must be implemented by the architecture.
604 * It is the way the ftrace direct_ops helper, when called
605 * via ftrace (because there's other callbacks besides the
606 * direct call), can inform the architecture's trampoline that this
607 * routine has a direct caller, and what the caller is.
608 *
609 * For example, in x86, it returns the direct caller
610 * callback function via the regs->orig_ax parameter.
611 * Then in the ftrace trampoline, if this is set, it makes
612 * the return from the trampoline jump to the direct caller
613 * instead of going back to the function it just traced.
614 */
arch_ftrace_set_direct_caller(struct ftrace_regs * fregs,unsigned long addr)615 static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
616 unsigned long addr) { }
617 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
618
619 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
ftrace_is_jmp(unsigned long addr)620 static inline bool ftrace_is_jmp(unsigned long addr)
621 {
622 return addr & 1;
623 }
624
ftrace_jmp_set(unsigned long addr)625 static inline unsigned long ftrace_jmp_set(unsigned long addr)
626 {
627 return addr | 1UL;
628 }
629
ftrace_jmp_get(unsigned long addr)630 static inline unsigned long ftrace_jmp_get(unsigned long addr)
631 {
632 return addr & ~1UL;
633 }
634 #else
ftrace_is_jmp(unsigned long addr)635 static inline bool ftrace_is_jmp(unsigned long addr)
636 {
637 return false;
638 }
639
ftrace_jmp_set(unsigned long addr)640 static inline unsigned long ftrace_jmp_set(unsigned long addr)
641 {
642 return addr;
643 }
644
ftrace_jmp_get(unsigned long addr)645 static inline unsigned long ftrace_jmp_get(unsigned long addr)
646 {
647 return addr;
648 }
649 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_JMP */
650
651 #ifdef CONFIG_STACK_TRACER
652
653 int stack_trace_sysctl(const struct ctl_table *table, int write, void *buffer,
654 size_t *lenp, loff_t *ppos);
655
656 /* DO NOT MODIFY THIS VARIABLE DIRECTLY! */
657 DECLARE_PER_CPU(int, disable_stack_tracer);
658
659 /**
660 * stack_tracer_disable - temporarily disable the stack tracer
661 *
662 * There's a few locations (namely in RCU) where stack tracing
663 * cannot be executed. This function is used to disable stack
664 * tracing during those critical sections.
665 *
666 * This function must be called with preemption or interrupts
667 * disabled and stack_tracer_enable() must be called shortly after
668 * while preemption or interrupts are still disabled.
669 */
stack_tracer_disable(void)670 static inline void stack_tracer_disable(void)
671 {
672 /* Preemption or interrupts must be disabled */
673 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
674 WARN_ON_ONCE(!preempt_count() || !irqs_disabled());
675 this_cpu_inc(disable_stack_tracer);
676 }
677
678 /**
679 * stack_tracer_enable - re-enable the stack tracer
680 *
681 * After stack_tracer_disable() is called, stack_tracer_enable()
682 * must be called shortly afterward.
683 */
stack_tracer_enable(void)684 static inline void stack_tracer_enable(void)
685 {
686 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
687 WARN_ON_ONCE(!preempt_count() || !irqs_disabled());
688 this_cpu_dec(disable_stack_tracer);
689 }
690 #else
stack_tracer_disable(void)691 static inline void stack_tracer_disable(void) { }
stack_tracer_enable(void)692 static inline void stack_tracer_enable(void) { }
693 #endif
694
695 enum {
696 FTRACE_UPDATE_CALLS = (1 << 0),
697 FTRACE_DISABLE_CALLS = (1 << 1),
698 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
699 FTRACE_START_FUNC_RET = (1 << 3),
700 FTRACE_STOP_FUNC_RET = (1 << 4),
701 FTRACE_MAY_SLEEP = (1 << 5),
702 };
703
704 /* Arches can override ftrace_get_symaddr() to convert fentry_ip to symaddr. */
705 #ifndef ftrace_get_symaddr
706 /**
707 * ftrace_get_symaddr - return the symbol address from fentry_ip
708 * @fentry_ip: the address of ftrace location
709 *
710 * Get the symbol address from @fentry_ip (fast path). If there is no fast
711 * search path, this returns 0.
712 * User may need to use kallsyms API to find the symbol address.
713 */
714 #define ftrace_get_symaddr(fentry_ip) (0)
715 #endif
716
717 void ftrace_sync_ipi(void *data);
718
719 #ifdef CONFIG_DYNAMIC_FTRACE
720
721 void ftrace_arch_code_modify_prepare(void);
722 void ftrace_arch_code_modify_post_process(void);
723
724 enum ftrace_bug_type {
725 FTRACE_BUG_UNKNOWN,
726 FTRACE_BUG_INIT,
727 FTRACE_BUG_NOP,
728 FTRACE_BUG_CALL,
729 FTRACE_BUG_UPDATE,
730 };
731 extern enum ftrace_bug_type ftrace_bug_type;
732
733 /*
734 * Archs can set this to point to a variable that holds the value that was
735 * expected at the call site before calling ftrace_bug().
736 */
737 extern const void *ftrace_expected;
738
739 void ftrace_bug(int err, struct dyn_ftrace *rec);
740
741 struct seq_file;
742
743 extern int ftrace_text_reserved(const void *start, const void *end);
744
745 struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr);
746
747 bool is_ftrace_trampoline(unsigned long addr);
748
749 /*
750 * The dyn_ftrace record's flags field is split into two parts.
751 * the first part which is '0-FTRACE_REF_MAX' is a counter of
752 * the number of callbacks that have registered the function that
753 * the dyn_ftrace descriptor represents.
754 *
755 * The second part is a mask:
756 * ENABLED - the function is being traced
757 * REGS - the record wants the function to save regs
758 * REGS_EN - the function is set up to save regs.
759 * IPMODIFY - the record allows for the IP address to be changed.
760 * DISABLED - the record is not ready to be touched yet
761 * DIRECT - there is a direct function to call
762 * CALL_OPS - the record can use callsite-specific ops
763 * CALL_OPS_EN - the function is set up to use callsite-specific ops
764 * TOUCHED - A callback was added since boot up
765 * MODIFIED - The function had IPMODIFY or DIRECT attached to it
766 *
767 * When a new ftrace_ops is registered and wants a function to save
768 * pt_regs, the rec->flags REGS is set. When the function has been
769 * set up to save regs, the REG_EN flag is set. Once a function
770 * starts saving regs it will do so until all ftrace_ops are removed
771 * from tracing that function.
772 */
773 enum {
774 FTRACE_FL_ENABLED = (1UL << 31),
775 FTRACE_FL_REGS = (1UL << 30),
776 FTRACE_FL_REGS_EN = (1UL << 29),
777 FTRACE_FL_TRAMP = (1UL << 28),
778 FTRACE_FL_TRAMP_EN = (1UL << 27),
779 FTRACE_FL_IPMODIFY = (1UL << 26),
780 FTRACE_FL_DISABLED = (1UL << 25),
781 FTRACE_FL_DIRECT = (1UL << 24),
782 FTRACE_FL_DIRECT_EN = (1UL << 23),
783 FTRACE_FL_CALL_OPS = (1UL << 22),
784 FTRACE_FL_CALL_OPS_EN = (1UL << 21),
785 FTRACE_FL_TOUCHED = (1UL << 20),
786 FTRACE_FL_MODIFIED = (1UL << 19),
787 };
788
789 #define FTRACE_REF_MAX_SHIFT 19
790 #define FTRACE_REF_MAX ((1UL << FTRACE_REF_MAX_SHIFT) - 1)
791
792 #define ftrace_rec_count(rec) ((rec)->flags & FTRACE_REF_MAX)
793
794 struct dyn_ftrace {
795 unsigned long ip; /* address of mcount call-site */
796 unsigned long flags;
797 struct dyn_arch_ftrace arch;
798 };
799
800 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
801 int remove, int reset);
802 int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips,
803 unsigned int cnt, int remove, int reset);
804 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
805 int len, int reset);
806 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
807 int len, int reset);
808 void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
809 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
810 void ftrace_free_filter(struct ftrace_ops *ops);
811 void ftrace_ops_set_global_filter(struct ftrace_ops *ops);
812
813 /*
814 * The FTRACE_UPDATE_* enum is used to pass information back
815 * from the ftrace_update_record() and ftrace_test_record()
816 * functions. These are called by the code update routines
817 * to find out what is to be done for a given function.
818 *
819 * IGNORE - The function is already what we want it to be
820 * MAKE_CALL - Start tracing the function
821 * MODIFY_CALL - Stop saving regs for the function
822 * MAKE_NOP - Stop tracing the function
823 */
824 enum {
825 FTRACE_UPDATE_IGNORE,
826 FTRACE_UPDATE_MAKE_CALL,
827 FTRACE_UPDATE_MODIFY_CALL,
828 FTRACE_UPDATE_MAKE_NOP,
829 };
830
831 enum {
832 FTRACE_ITER_FILTER = (1 << 0),
833 FTRACE_ITER_NOTRACE = (1 << 1),
834 FTRACE_ITER_PRINTALL = (1 << 2),
835 FTRACE_ITER_DO_PROBES = (1 << 3),
836 FTRACE_ITER_PROBE = (1 << 4),
837 FTRACE_ITER_MOD = (1 << 5),
838 FTRACE_ITER_ENABLED = (1 << 6),
839 FTRACE_ITER_TOUCHED = (1 << 7),
840 FTRACE_ITER_ADDRS = (1 << 8),
841 };
842
843 void arch_ftrace_update_code(int command);
844 void arch_ftrace_update_trampoline(struct ftrace_ops *ops);
845 void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec);
846 void arch_ftrace_trampoline_free(struct ftrace_ops *ops);
847
848 struct ftrace_rec_iter;
849
850 struct ftrace_rec_iter *ftrace_rec_iter_start(void);
851 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
852 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
853
854 #define for_ftrace_rec_iter(iter) \
855 for (iter = ftrace_rec_iter_start(); \
856 iter; \
857 iter = ftrace_rec_iter_next(iter))
858
859
860 int ftrace_update_record(struct dyn_ftrace *rec, bool enable);
861 int ftrace_test_record(struct dyn_ftrace *rec, bool enable);
862 void ftrace_run_stop_machine(int command);
863 unsigned long ftrace_location(unsigned long ip);
864 unsigned long ftrace_location_range(unsigned long start, unsigned long end);
865 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec);
866 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec);
867
868 extern ftrace_func_t ftrace_trace_function;
869 struct trace_array;
870
871 int ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag,
872 struct inode *inode, struct file *file);
873 ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
874 size_t cnt, loff_t *ppos);
875 ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
876 size_t cnt, loff_t *ppos);
877 int ftrace_regex_release(struct inode *inode, struct file *file);
878
879 void __init
880 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
881
882 /* defined in arch */
883 extern int ftrace_dyn_arch_init(void);
884 extern void ftrace_replace_code(int enable);
885 extern int ftrace_update_ftrace_func(ftrace_func_t func);
886 extern void ftrace_caller(void);
887 extern void ftrace_regs_caller(void);
888 extern void ftrace_call(void);
889 extern void ftrace_regs_call(void);
890 extern void mcount_call(void);
891
892 void ftrace_modify_all_code(int command);
893
894 #ifndef FTRACE_ADDR
895 #define FTRACE_ADDR ((unsigned long)ftrace_caller)
896 #endif
897
898 #ifndef FTRACE_GRAPH_ADDR
899 #define FTRACE_GRAPH_ADDR ((unsigned long)ftrace_graph_caller)
900 #endif
901
902 #ifndef FTRACE_REGS_ADDR
903 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
904 # define FTRACE_REGS_ADDR ((unsigned long)ftrace_regs_caller)
905 #else
906 # define FTRACE_REGS_ADDR FTRACE_ADDR
907 #endif
908 #endif
909
910 /*
911 * If an arch would like functions that are only traced
912 * by the function graph tracer to jump directly to its own
913 * trampoline, then they can define FTRACE_GRAPH_TRAMP_ADDR
914 * to be that address to jump to.
915 */
916 #ifndef FTRACE_GRAPH_TRAMP_ADDR
917 #define FTRACE_GRAPH_TRAMP_ADDR ((unsigned long) 0)
918 #endif
919
920 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
921 extern void ftrace_graph_caller(void);
922 extern int ftrace_enable_ftrace_graph_caller(void);
923 extern int ftrace_disable_ftrace_graph_caller(void);
924 #else
ftrace_enable_ftrace_graph_caller(void)925 static inline int ftrace_enable_ftrace_graph_caller(void) { return 0; }
ftrace_disable_ftrace_graph_caller(void)926 static inline int ftrace_disable_ftrace_graph_caller(void) { return 0; }
927 #endif
928
929 /**
930 * ftrace_make_nop - convert code into nop
931 * @mod: module structure if called by module load initialization
932 * @rec: the call site record (e.g. mcount/fentry)
933 * @addr: the address that the call site should be calling
934 *
935 * This is a very sensitive operation and great care needs
936 * to be taken by the arch. The operation should carefully
937 * read the location, check to see if what is read is indeed
938 * what we expect it to be, and then on success of the compare,
939 * it should write to the location.
940 *
941 * The code segment at @rec->ip should be a caller to @addr
942 *
943 * Return must be:
944 * 0 on success
945 * -EFAULT on error reading the location
946 * -EINVAL on a failed compare of the contents
947 * -EPERM on error writing to the location
948 * Any other value will be considered a failure.
949 */
950 extern int ftrace_make_nop(struct module *mod,
951 struct dyn_ftrace *rec, unsigned long addr);
952
953 /**
954 * ftrace_need_init_nop - return whether nop call sites should be initialized
955 *
956 * Normally the compiler's -mnop-mcount generates suitable nops, so we don't
957 * need to call ftrace_init_nop() if the code is built with that flag.
958 * Architectures where this is not always the case may define their own
959 * condition.
960 *
961 * Return must be:
962 * 0 if ftrace_init_nop() should be called
963 * Nonzero if ftrace_init_nop() should not be called
964 */
965
966 #ifndef ftrace_need_init_nop
967 #define ftrace_need_init_nop() (!__is_defined(CC_USING_NOP_MCOUNT))
968 #endif
969
970 /**
971 * ftrace_init_nop - initialize a nop call site
972 * @mod: module structure if called by module load initialization
973 * @rec: the call site record (e.g. mcount/fentry)
974 *
975 * This is a very sensitive operation and great care needs
976 * to be taken by the arch. The operation should carefully
977 * read the location, check to see if what is read is indeed
978 * what we expect it to be, and then on success of the compare,
979 * it should write to the location.
980 *
981 * The code segment at @rec->ip should contain the contents created by
982 * the compiler
983 *
984 * Return must be:
985 * 0 on success
986 * -EFAULT on error reading the location
987 * -EINVAL on a failed compare of the contents
988 * -EPERM on error writing to the location
989 * Any other value will be considered a failure.
990 */
991 #ifndef ftrace_init_nop
ftrace_init_nop(struct module * mod,struct dyn_ftrace * rec)992 static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
993 {
994 return ftrace_make_nop(mod, rec, MCOUNT_ADDR);
995 }
996 #endif
997
998 /**
999 * ftrace_make_call - convert a nop call site into a call to addr
1000 * @rec: the call site record (e.g. mcount/fentry)
1001 * @addr: the address that the call site should call
1002 *
1003 * This is a very sensitive operation and great care needs
1004 * to be taken by the arch. The operation should carefully
1005 * read the location, check to see if what is read is indeed
1006 * what we expect it to be, and then on success of the compare,
1007 * it should write to the location.
1008 *
1009 * The code segment at @rec->ip should be a nop
1010 *
1011 * Return must be:
1012 * 0 on success
1013 * -EFAULT on error reading the location
1014 * -EINVAL on a failed compare of the contents
1015 * -EPERM on error writing to the location
1016 * Any other value will be considered a failure.
1017 */
1018 extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
1019
1020 #if defined(CONFIG_DYNAMIC_FTRACE_WITH_REGS) || \
1021 defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \
1022 defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)
1023 /**
1024 * ftrace_modify_call - convert from one addr to another (no nop)
1025 * @rec: the call site record (e.g. mcount/fentry)
1026 * @old_addr: the address expected to be currently called to
1027 * @addr: the address to change to
1028 *
1029 * This is a very sensitive operation and great care needs
1030 * to be taken by the arch. The operation should carefully
1031 * read the location, check to see if what is read is indeed
1032 * what we expect it to be, and then on success of the compare,
1033 * it should write to the location.
1034 *
1035 * When using call ops, this is called when the associated ops change, even
1036 * when (addr == old_addr).
1037 *
1038 * The code segment at @rec->ip should be a caller to @old_addr
1039 *
1040 * Return must be:
1041 * 0 on success
1042 * -EFAULT on error reading the location
1043 * -EINVAL on a failed compare of the contents
1044 * -EPERM on error writing to the location
1045 * Any other value will be considered a failure.
1046 */
1047 extern int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
1048 unsigned long addr);
1049 #else
1050 /* Should never be called */
ftrace_modify_call(struct dyn_ftrace * rec,unsigned long old_addr,unsigned long addr)1051 static inline int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
1052 unsigned long addr)
1053 {
1054 return -EINVAL;
1055 }
1056 #endif
1057
1058 extern int skip_trace(unsigned long ip);
1059 extern void ftrace_module_init(struct module *mod);
1060 extern void ftrace_module_enable(struct module *mod);
1061 extern void ftrace_release_mod(struct module *mod);
1062 #else /* CONFIG_DYNAMIC_FTRACE */
skip_trace(unsigned long ip)1063 static inline int skip_trace(unsigned long ip) { return 0; }
ftrace_module_init(struct module * mod)1064 static inline void ftrace_module_init(struct module *mod) { }
ftrace_module_enable(struct module * mod)1065 static inline void ftrace_module_enable(struct module *mod) { }
ftrace_release_mod(struct module * mod)1066 static inline void ftrace_release_mod(struct module *mod) { }
ftrace_text_reserved(const void * start,const void * end)1067 static inline int ftrace_text_reserved(const void *start, const void *end)
1068 {
1069 return 0;
1070 }
ftrace_location(unsigned long ip)1071 static inline unsigned long ftrace_location(unsigned long ip)
1072 {
1073 return 0;
1074 }
1075
1076 /*
1077 * Again users of functions that have ftrace_ops may not
1078 * have them defined when ftrace is not enabled, but these
1079 * functions may still be called. Use a macro instead of inline.
1080 */
1081 #define ftrace_regex_open(tr, ops, flag, inode, file) ({ -ENODEV; })
1082 #define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
1083 #define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; })
1084 #define ftrace_set_filter_ips(ops, ips, cnt, remove, reset) ({ -ENODEV; })
1085 #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; })
1086 #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; })
1087 #define ftrace_free_filter(ops) do { } while (0)
1088 #define ftrace_ops_set_global_filter(ops) do { } while (0)
1089
ftrace_filter_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)1090 static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
1091 size_t cnt, loff_t *ppos) { return -ENODEV; }
ftrace_notrace_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)1092 static inline ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
1093 size_t cnt, loff_t *ppos) { return -ENODEV; }
1094 static inline int
ftrace_regex_release(struct inode * inode,struct file * file)1095 ftrace_regex_release(struct inode *inode, struct file *file) { return -ENODEV; }
1096
is_ftrace_trampoline(unsigned long addr)1097 static inline bool is_ftrace_trampoline(unsigned long addr)
1098 {
1099 return false;
1100 }
1101 #endif /* CONFIG_DYNAMIC_FTRACE */
1102
1103 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1104 #ifndef ftrace_graph_func
1105 # define ftrace_graph_func ftrace_stub
1106 # define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB
1107 /*
1108 * The function graph is called every time the function tracer is called.
1109 * It must always test the ops hash and cannot just directly call
1110 * the handler.
1111 */
1112 # define FGRAPH_NO_DIRECT 1
1113 #else
1114 # define FTRACE_OPS_GRAPH_STUB 0
1115 # define FGRAPH_NO_DIRECT 0
1116 #endif
1117 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1118
1119 /* totally disable ftrace - can not re-enable after this */
1120 void ftrace_kill(void);
1121
tracer_disable(void)1122 static inline void tracer_disable(void)
1123 {
1124 #ifdef CONFIG_FUNCTION_TRACER
1125 ftrace_enabled = 0;
1126 #endif
1127 }
1128
1129 /*
1130 * Ftrace disable/restore without lock. Some synchronization mechanism
1131 * must be used to prevent ftrace_enabled to be changed between
1132 * disable/restore.
1133 */
__ftrace_enabled_save(void)1134 static inline int __ftrace_enabled_save(void)
1135 {
1136 #ifdef CONFIG_FUNCTION_TRACER
1137 int saved_ftrace_enabled = ftrace_enabled;
1138 ftrace_enabled = 0;
1139 return saved_ftrace_enabled;
1140 #else
1141 return 0;
1142 #endif
1143 }
1144
__ftrace_enabled_restore(int enabled)1145 static inline void __ftrace_enabled_restore(int enabled)
1146 {
1147 #ifdef CONFIG_FUNCTION_TRACER
1148 ftrace_enabled = enabled;
1149 #endif
1150 }
1151
1152 /* All archs should have this, but we define it for consistency */
1153 #ifndef ftrace_return_address0
1154 # define ftrace_return_address0 __builtin_return_address(0)
1155 #endif
1156
1157 /* Archs may use other ways for ADDR1 and beyond */
1158 #ifndef ftrace_return_address
1159 # ifdef CONFIG_FRAME_POINTER
1160 # define ftrace_return_address(n) __builtin_return_address(n)
1161 # else
1162 # define ftrace_return_address(n) 0UL
1163 # endif
1164 #endif
1165
1166 #define CALLER_ADDR0 ((unsigned long)ftrace_return_address0)
1167 #define CALLER_ADDR1 ((unsigned long)ftrace_return_address(1))
1168 #define CALLER_ADDR2 ((unsigned long)ftrace_return_address(2))
1169 #define CALLER_ADDR3 ((unsigned long)ftrace_return_address(3))
1170 #define CALLER_ADDR4 ((unsigned long)ftrace_return_address(4))
1171 #define CALLER_ADDR5 ((unsigned long)ftrace_return_address(5))
1172 #define CALLER_ADDR6 ((unsigned long)ftrace_return_address(6))
1173
get_lock_parent_ip(void)1174 static __always_inline unsigned long get_lock_parent_ip(void)
1175 {
1176 unsigned long addr = CALLER_ADDR0;
1177
1178 if (!in_lock_functions(addr))
1179 return addr;
1180 addr = CALLER_ADDR1;
1181 if (!in_lock_functions(addr))
1182 return addr;
1183 return CALLER_ADDR2;
1184 }
1185
1186 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
1187 extern void trace_preempt_on(unsigned long a0, unsigned long a1);
1188 extern void trace_preempt_off(unsigned long a0, unsigned long a1);
1189 #else
1190 /*
1191 * Use defines instead of static inlines because some arches will make code out
1192 * of the CALLER_ADDR, when we really want these to be a real nop.
1193 */
1194 # define trace_preempt_on(a0, a1) do { } while (0)
1195 # define trace_preempt_off(a0, a1) do { } while (0)
1196 #endif
1197
1198 #ifdef CONFIG_DYNAMIC_FTRACE
1199 extern void ftrace_init(void);
1200 #ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY
1201 #define FTRACE_CALLSITE_SECTION "__patchable_function_entries"
1202 #else
1203 #define FTRACE_CALLSITE_SECTION "__mcount_loc"
1204 #endif
1205 #else
ftrace_init(void)1206 static inline void ftrace_init(void) { }
1207 #endif
1208
1209 /*
1210 * Structure that defines an entry function trace.
1211 * It's already packed but the attribute "packed" is needed
1212 * to remove extra padding at the end.
1213 */
1214 struct ftrace_graph_ent {
1215 unsigned long func; /* Current function */
1216 long depth; /* signed to check for less than zero */
1217 } __packed;
1218
1219 /*
1220 * Structure that defines an entry function trace with retaddr.
1221 */
1222 struct fgraph_retaddr_ent {
1223 struct ftrace_graph_ent ent;
1224 unsigned long retaddr; /* Return address */
1225 } __packed;
1226
1227 /*
1228 * Structure that defines a return function trace.
1229 * It's already packed but the attribute "packed" is needed
1230 * to remove extra padding at the end.
1231 */
1232 struct ftrace_graph_ret {
1233 unsigned long func; /* Current function */
1234 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL
1235 unsigned long retval;
1236 #endif
1237 int depth;
1238 /* Number of functions that overran the depth limit for current task */
1239 unsigned int overrun;
1240 } __packed;
1241
1242 struct fgraph_ops;
1243
1244 /* Type of the callback handlers for tracing function graph*/
1245 typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *,
1246 struct fgraph_ops *,
1247 struct ftrace_regs *); /* return */
1248 typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *,
1249 struct fgraph_ops *,
1250 struct ftrace_regs *); /* entry */
1251
1252 extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace,
1253 struct fgraph_ops *gops,
1254 struct ftrace_regs *fregs);
1255 bool ftrace_pids_enabled(struct ftrace_ops *ops);
1256
1257 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1258
1259 struct fgraph_ops {
1260 trace_func_graph_ent_t entryfunc;
1261 trace_func_graph_ret_t retfunc;
1262 struct ftrace_ops ops; /* for the hash lists */
1263 void *private;
1264 trace_func_graph_ent_t saved_func;
1265 int idx;
1266 };
1267
1268 void *fgraph_reserve_data(int idx, int size_bytes);
1269 void *fgraph_retrieve_data(int idx, int *size_bytes);
1270 void *fgraph_retrieve_parent_data(int idx, int *size_bytes, int depth);
1271
1272 /*
1273 * Stack of return addresses for functions
1274 * of a thread.
1275 * Used in struct thread_info
1276 */
1277 struct ftrace_ret_stack {
1278 unsigned long ret;
1279 unsigned long func;
1280 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
1281 unsigned long fp;
1282 #endif
1283 unsigned long *retp;
1284 };
1285
1286 /*
1287 * Primary handler of a function return.
1288 * It relays on ftrace_return_to_handler.
1289 * Defined in entry_32/64.S
1290 */
1291 extern void return_to_handler(void);
1292
1293 extern int
1294 function_graph_enter_regs(unsigned long ret, unsigned long func,
1295 unsigned long frame_pointer, unsigned long *retp,
1296 struct ftrace_regs *fregs);
1297
function_graph_enter(unsigned long ret,unsigned long func,unsigned long fp,unsigned long * retp)1298 static inline int function_graph_enter(unsigned long ret, unsigned long func,
1299 unsigned long fp, unsigned long *retp)
1300 {
1301 return function_graph_enter_regs(ret, func, fp, retp, NULL);
1302 }
1303
1304 struct ftrace_ret_stack *
1305 ftrace_graph_get_ret_stack(struct task_struct *task, int skip);
1306 unsigned long ftrace_graph_top_ret_addr(struct task_struct *task);
1307
1308 unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
1309 unsigned long ret, unsigned long *retp);
1310 unsigned long *fgraph_get_task_var(struct fgraph_ops *gops);
1311
1312 /*
1313 * Sometimes we don't want to trace a function with the function
1314 * graph tracer but we want them to keep traced by the usual function
1315 * tracer if the function graph tracer is not configured.
1316 */
1317 #define __notrace_funcgraph notrace
1318
1319 #define FTRACE_RETFUNC_DEPTH 50
1320 #define FTRACE_RETSTACK_ALLOC_SIZE 32
1321
1322 extern int register_ftrace_graph(struct fgraph_ops *ops);
1323 extern void unregister_ftrace_graph(struct fgraph_ops *ops);
1324
1325 /**
1326 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
1327 *
1328 * ftrace_graph_stop() is called when a severe error is detected in
1329 * the function graph tracing. This function is called by the critical
1330 * paths of function graph to keep those paths from doing any more harm.
1331 */
1332 DECLARE_STATIC_KEY_FALSE(kill_ftrace_graph);
1333
ftrace_graph_is_dead(void)1334 static inline bool ftrace_graph_is_dead(void)
1335 {
1336 return static_branch_unlikely(&kill_ftrace_graph);
1337 }
1338
1339 extern void ftrace_graph_stop(void);
1340
1341 /* The current handlers in use */
1342 extern trace_func_graph_ret_t ftrace_graph_return;
1343 extern trace_func_graph_ent_t ftrace_graph_entry;
1344
1345 extern void ftrace_graph_init_task(struct task_struct *t);
1346 extern void ftrace_graph_exit_task(struct task_struct *t);
1347 extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
1348
1349 /* Used by assembly, but to quiet sparse warnings */
1350 extern struct ftrace_ops *function_trace_op;
1351
pause_graph_tracing(void)1352 static inline void pause_graph_tracing(void)
1353 {
1354 atomic_inc(¤t->tracing_graph_pause);
1355 }
1356
unpause_graph_tracing(void)1357 static inline void unpause_graph_tracing(void)
1358 {
1359 atomic_dec(¤t->tracing_graph_pause);
1360 }
1361 #else /* !CONFIG_FUNCTION_GRAPH_TRACER */
1362
1363 #define __notrace_funcgraph
1364
ftrace_graph_init_task(struct task_struct * t)1365 static inline void ftrace_graph_init_task(struct task_struct *t) { }
ftrace_graph_exit_task(struct task_struct * t)1366 static inline void ftrace_graph_exit_task(struct task_struct *t) { }
ftrace_graph_init_idle_task(struct task_struct * t,int cpu)1367 static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { }
1368
1369 /* Define as macros as fgraph_ops may not be defined */
1370 #define register_ftrace_graph(ops) ({ -1; })
1371 #define unregister_ftrace_graph(ops) do { } while (0)
1372
1373 static inline unsigned long
ftrace_graph_ret_addr(struct task_struct * task,int * idx,unsigned long ret,unsigned long * retp)1374 ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret,
1375 unsigned long *retp)
1376 {
1377 return ret;
1378 }
1379
pause_graph_tracing(void)1380 static inline void pause_graph_tracing(void) { }
unpause_graph_tracing(void)1381 static inline void unpause_graph_tracing(void) { }
1382 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1383
1384 #ifdef CONFIG_TRACING
1385 enum ftrace_dump_mode;
1386
1387 extern int ftrace_dump_on_oops_enabled(void);
1388
1389 extern void disable_trace_on_warning(void);
1390
1391 #else /* CONFIG_TRACING */
disable_trace_on_warning(void)1392 static inline void disable_trace_on_warning(void) { }
1393 #endif /* CONFIG_TRACING */
1394
1395 #ifdef CONFIG_FTRACE_SYSCALLS
1396
1397 unsigned long arch_syscall_addr(int nr);
1398
1399 #endif /* CONFIG_FTRACE_SYSCALLS */
1400
1401 #endif /* _LINUX_FTRACE_H */
1402