xref: /linux/include/linux/trace_events.h (revision 146430a0c26eb7b515abb04664e1a516078ec5c2)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 
3 #ifndef _LINUX_TRACE_EVENT_H
4 #define _LINUX_TRACE_EVENT_H
5 
6 #include <linux/ring_buffer.h>
7 #include <linux/trace_seq.h>
8 #include <linux/percpu.h>
9 #include <linux/hardirq.h>
10 #include <linux/perf_event.h>
11 #include <linux/tracepoint.h>
12 
13 struct trace_array;
14 struct array_buffer;
15 struct tracer;
16 struct dentry;
17 struct bpf_prog;
18 union bpf_attr;
19 
20 /* Used for event string fields when they are NULL */
21 #define EVENT_NULL_STR		"(null)"
22 
23 const char *trace_print_flags_seq(struct trace_seq *p, const char *delim,
24 				  unsigned long flags,
25 				  const struct trace_print_flags *flag_array);
26 
27 const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
28 				    const struct trace_print_flags *symbol_array);
29 
30 #if BITS_PER_LONG == 32
31 const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
32 		      unsigned long long flags,
33 		      const struct trace_print_flags_u64 *flag_array);
34 
35 const char *trace_print_symbols_seq_u64(struct trace_seq *p,
36 					unsigned long long val,
37 					const struct trace_print_flags_u64
38 								 *symbol_array);
39 #endif
40 
41 const char *trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
42 				    unsigned int bitmask_size);
43 
44 const char *trace_print_hex_seq(struct trace_seq *p,
45 				const unsigned char *buf, int len,
46 				bool concatenate);
47 
48 const char *trace_print_array_seq(struct trace_seq *p,
49 				   const void *buf, int count,
50 				   size_t el_size);
51 
52 const char *
53 trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str,
54 			 int prefix_type, int rowsize, int groupsize,
55 			 const void *buf, size_t len, bool ascii);
56 
57 struct trace_iterator;
58 struct trace_event;
59 
60 int trace_raw_output_prep(struct trace_iterator *iter,
61 			  struct trace_event *event);
62 extern __printf(2, 3)
63 void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...);
64 
65 /* Used to find the offset and length of dynamic fields in trace events */
66 struct trace_dynamic_info {
67 #ifdef CONFIG_CPU_BIG_ENDIAN
68 	u16	len;
69 	u16	offset;
70 #else
71 	u16	offset;
72 	u16	len;
73 #endif
74 } __packed;
75 
76 /*
77  * The trace entry - the most basic unit of tracing. This is what
78  * is printed in the end as a single line in the trace output, such as:
79  *
80  *     bash-15816 [01]   235.197585: idle_cpu <- irq_enter
81  */
82 struct trace_entry {
83 	unsigned short		type;
84 	unsigned char		flags;
85 	unsigned char		preempt_count;
86 	int			pid;
87 };
88 
89 #define TRACE_EVENT_TYPE_MAX						\
90 	((1 << (sizeof(((struct trace_entry *)0)->type) * 8)) - 1)
91 
92 /*
93  * Trace iterator - used by printout routines who present trace
94  * results to users and which routines might sleep, etc:
95  */
96 struct trace_iterator {
97 	struct trace_array	*tr;
98 	struct tracer		*trace;
99 	struct array_buffer	*array_buffer;
100 	void			*private;
101 	int			cpu_file;
102 	struct mutex		mutex;
103 	struct ring_buffer_iter	**buffer_iter;
104 	unsigned long		iter_flags;
105 	void			*temp;	/* temp holder */
106 	unsigned int		temp_size;
107 	char			*fmt;	/* modified format holder */
108 	unsigned int		fmt_size;
109 	atomic_t		wait_index;
110 
111 	/* trace_seq for __print_flags() and __print_symbolic() etc. */
112 	struct trace_seq	tmp_seq;
113 
114 	cpumask_var_t		started;
115 
116 	/* Set when the file is closed to prevent new waiters */
117 	bool			closed;
118 
119 	/* it's true when current open file is snapshot */
120 	bool			snapshot;
121 
122 	/* The below is zeroed out in pipe_read */
123 	struct trace_seq	seq;
124 	struct trace_entry	*ent;
125 	unsigned long		lost_events;
126 	int			leftover;
127 	int			ent_size;
128 	int			cpu;
129 	u64			ts;
130 
131 	loff_t			pos;
132 	long			idx;
133 
134 	/* All new field here will be zeroed out in pipe_read */
135 };
136 
137 enum trace_iter_flags {
138 	TRACE_FILE_LAT_FMT	= 1,
139 	TRACE_FILE_ANNOTATE	= 2,
140 	TRACE_FILE_TIME_IN_NS	= 4,
141 };
142 
143 
144 typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
145 				      int flags, struct trace_event *event);
146 
147 struct trace_event_functions {
148 	trace_print_func	trace;
149 	trace_print_func	raw;
150 	trace_print_func	hex;
151 	trace_print_func	binary;
152 };
153 
154 struct trace_event {
155 	struct hlist_node		node;
156 	int				type;
157 	struct trace_event_functions	*funcs;
158 };
159 
160 extern int register_trace_event(struct trace_event *event);
161 extern int unregister_trace_event(struct trace_event *event);
162 
163 /* Return values for print_line callback */
164 enum print_line_t {
165 	TRACE_TYPE_PARTIAL_LINE	= 0,	/* Retry after flushing the seq */
166 	TRACE_TYPE_HANDLED	= 1,
167 	TRACE_TYPE_UNHANDLED	= 2,	/* Relay to other output functions */
168 	TRACE_TYPE_NO_CONSUME	= 3	/* Handled but ask to not consume */
169 };
170 
171 enum print_line_t trace_handle_return(struct trace_seq *s);
172 
tracing_generic_entry_update(struct trace_entry * entry,unsigned short type,unsigned int trace_ctx)173 static inline void tracing_generic_entry_update(struct trace_entry *entry,
174 						unsigned short type,
175 						unsigned int trace_ctx)
176 {
177 	entry->preempt_count		= trace_ctx & 0xff;
178 	entry->pid			= current->pid;
179 	entry->type			= type;
180 	entry->flags =			trace_ctx >> 16;
181 }
182 
183 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status);
184 
185 enum trace_flag_type {
186 	TRACE_FLAG_IRQS_OFF		= 0x01,
187 	TRACE_FLAG_IRQS_NOSUPPORT	= 0x02,
188 	TRACE_FLAG_NEED_RESCHED		= 0x04,
189 	TRACE_FLAG_HARDIRQ		= 0x08,
190 	TRACE_FLAG_SOFTIRQ		= 0x10,
191 	TRACE_FLAG_PREEMPT_RESCHED	= 0x20,
192 	TRACE_FLAG_NMI			= 0x40,
193 	TRACE_FLAG_BH_OFF		= 0x80,
194 };
195 
196 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
tracing_gen_ctx_flags(unsigned long irqflags)197 static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
198 {
199 	unsigned int irq_status = irqs_disabled_flags(irqflags) ?
200 		TRACE_FLAG_IRQS_OFF : 0;
201 	return tracing_gen_ctx_irq_test(irq_status);
202 }
tracing_gen_ctx(void)203 static inline unsigned int tracing_gen_ctx(void)
204 {
205 	unsigned long irqflags;
206 
207 	local_save_flags(irqflags);
208 	return tracing_gen_ctx_flags(irqflags);
209 }
210 #else
211 
tracing_gen_ctx_flags(unsigned long irqflags)212 static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
213 {
214 	return tracing_gen_ctx_irq_test(TRACE_FLAG_IRQS_NOSUPPORT);
215 }
tracing_gen_ctx(void)216 static inline unsigned int tracing_gen_ctx(void)
217 {
218 	return tracing_gen_ctx_irq_test(TRACE_FLAG_IRQS_NOSUPPORT);
219 }
220 #endif
221 
tracing_gen_ctx_dec(void)222 static inline unsigned int tracing_gen_ctx_dec(void)
223 {
224 	unsigned int trace_ctx;
225 
226 	trace_ctx = tracing_gen_ctx();
227 	/*
228 	 * Subtract one from the preemption counter if preemption is enabled,
229 	 * see trace_event_buffer_reserve()for details.
230 	 */
231 	if (IS_ENABLED(CONFIG_PREEMPTION))
232 		trace_ctx--;
233 	return trace_ctx;
234 }
235 
236 struct trace_event_file;
237 
238 struct ring_buffer_event *
239 trace_event_buffer_lock_reserve(struct trace_buffer **current_buffer,
240 				struct trace_event_file *trace_file,
241 				int type, unsigned long len,
242 				unsigned int trace_ctx);
243 
244 #define TRACE_RECORD_CMDLINE	BIT(0)
245 #define TRACE_RECORD_TGID	BIT(1)
246 
247 void tracing_record_taskinfo(struct task_struct *task, int flags);
248 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
249 					  struct task_struct *next, int flags);
250 
251 void tracing_record_cmdline(struct task_struct *task);
252 void tracing_record_tgid(struct task_struct *task);
253 
254 int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
255 	 __printf(3, 4);
256 
257 struct event_filter;
258 
259 enum trace_reg {
260 	TRACE_REG_REGISTER,
261 	TRACE_REG_UNREGISTER,
262 #ifdef CONFIG_PERF_EVENTS
263 	TRACE_REG_PERF_REGISTER,
264 	TRACE_REG_PERF_UNREGISTER,
265 	TRACE_REG_PERF_OPEN,
266 	TRACE_REG_PERF_CLOSE,
267 	/*
268 	 * These (ADD/DEL) use a 'boolean' return value, where 1 (true) means a
269 	 * custom action was taken and the default action is not to be
270 	 * performed.
271 	 */
272 	TRACE_REG_PERF_ADD,
273 	TRACE_REG_PERF_DEL,
274 #endif
275 };
276 
277 struct trace_event_call;
278 
279 #define TRACE_FUNCTION_TYPE ((const char *)~0UL)
280 
281 struct trace_event_fields {
282 	const char *type;
283 	union {
284 		struct {
285 			const char *name;
286 			const int  size;
287 			const int  align;
288 			const int  is_signed;
289 			const int  filter_type;
290 			const int  len;
291 		};
292 		int (*define_fields)(struct trace_event_call *);
293 	};
294 };
295 
296 struct trace_event_class {
297 	const char		*system;
298 	void			*probe;
299 #ifdef CONFIG_PERF_EVENTS
300 	void			*perf_probe;
301 #endif
302 	int			(*reg)(struct trace_event_call *event,
303 				       enum trace_reg type, void *data);
304 	struct trace_event_fields *fields_array;
305 	struct list_head	*(*get_fields)(struct trace_event_call *);
306 	struct list_head	fields;
307 	int			(*raw_init)(struct trace_event_call *);
308 };
309 
310 extern int trace_event_reg(struct trace_event_call *event,
311 			    enum trace_reg type, void *data);
312 
313 struct trace_event_buffer {
314 	struct trace_buffer		*buffer;
315 	struct ring_buffer_event	*event;
316 	struct trace_event_file		*trace_file;
317 	void				*entry;
318 	unsigned int			trace_ctx;
319 	struct pt_regs			*regs;
320 };
321 
322 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
323 				  struct trace_event_file *trace_file,
324 				  unsigned long len);
325 
326 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer);
327 
328 enum {
329 	TRACE_EVENT_FL_FILTERED_BIT,
330 	TRACE_EVENT_FL_CAP_ANY_BIT,
331 	TRACE_EVENT_FL_NO_SET_FILTER_BIT,
332 	TRACE_EVENT_FL_IGNORE_ENABLE_BIT,
333 	TRACE_EVENT_FL_TRACEPOINT_BIT,
334 	TRACE_EVENT_FL_DYNAMIC_BIT,
335 	TRACE_EVENT_FL_KPROBE_BIT,
336 	TRACE_EVENT_FL_UPROBE_BIT,
337 	TRACE_EVENT_FL_EPROBE_BIT,
338 	TRACE_EVENT_FL_FPROBE_BIT,
339 	TRACE_EVENT_FL_CUSTOM_BIT,
340 };
341 
342 /*
343  * Event flags:
344  *  FILTERED	  - The event has a filter attached
345  *  CAP_ANY	  - Any user can enable for perf
346  *  NO_SET_FILTER - Set when filter has error and is to be ignored
347  *  IGNORE_ENABLE - For trace internal events, do not enable with debugfs file
348  *  TRACEPOINT    - Event is a tracepoint
349  *  DYNAMIC       - Event is a dynamic event (created at run time)
350  *  KPROBE        - Event is a kprobe
351  *  UPROBE        - Event is a uprobe
352  *  EPROBE        - Event is an event probe
353  *  FPROBE        - Event is an function probe
354  *  CUSTOM        - Event is a custom event (to be attached to an exsiting tracepoint)
355  *                   This is set when the custom event has not been attached
356  *                   to a tracepoint yet, then it is cleared when it is.
357  */
358 enum {
359 	TRACE_EVENT_FL_FILTERED		= (1 << TRACE_EVENT_FL_FILTERED_BIT),
360 	TRACE_EVENT_FL_CAP_ANY		= (1 << TRACE_EVENT_FL_CAP_ANY_BIT),
361 	TRACE_EVENT_FL_NO_SET_FILTER	= (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT),
362 	TRACE_EVENT_FL_IGNORE_ENABLE	= (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT),
363 	TRACE_EVENT_FL_TRACEPOINT	= (1 << TRACE_EVENT_FL_TRACEPOINT_BIT),
364 	TRACE_EVENT_FL_DYNAMIC		= (1 << TRACE_EVENT_FL_DYNAMIC_BIT),
365 	TRACE_EVENT_FL_KPROBE		= (1 << TRACE_EVENT_FL_KPROBE_BIT),
366 	TRACE_EVENT_FL_UPROBE		= (1 << TRACE_EVENT_FL_UPROBE_BIT),
367 	TRACE_EVENT_FL_EPROBE		= (1 << TRACE_EVENT_FL_EPROBE_BIT),
368 	TRACE_EVENT_FL_FPROBE		= (1 << TRACE_EVENT_FL_FPROBE_BIT),
369 	TRACE_EVENT_FL_CUSTOM		= (1 << TRACE_EVENT_FL_CUSTOM_BIT),
370 };
371 
372 #define TRACE_EVENT_FL_UKPROBE (TRACE_EVENT_FL_KPROBE | TRACE_EVENT_FL_UPROBE)
373 
374 struct trace_event_call {
375 	struct list_head	list;
376 	struct trace_event_class *class;
377 	union {
378 		char			*name;
379 		/* Set TRACE_EVENT_FL_TRACEPOINT flag when using "tp" */
380 		struct tracepoint	*tp;
381 	};
382 	struct trace_event	event;
383 	char			*print_fmt;
384 	struct event_filter	*filter;
385 	/*
386 	 * Static events can disappear with modules,
387 	 * where as dynamic ones need their own ref count.
388 	 */
389 	union {
390 		void				*module;
391 		atomic_t			refcnt;
392 	};
393 	void			*data;
394 
395 	/* See the TRACE_EVENT_FL_* flags above */
396 	int			flags; /* static flags of different events */
397 
398 #ifdef CONFIG_PERF_EVENTS
399 	int				perf_refcount;
400 	struct hlist_head __percpu	*perf_events;
401 	struct bpf_prog_array __rcu	*prog_array;
402 
403 	int	(*perf_perm)(struct trace_event_call *,
404 			     struct perf_event *);
405 #endif
406 };
407 
408 #ifdef CONFIG_DYNAMIC_EVENTS
409 bool trace_event_dyn_try_get_ref(struct trace_event_call *call);
410 void trace_event_dyn_put_ref(struct trace_event_call *call);
411 bool trace_event_dyn_busy(struct trace_event_call *call);
412 #else
trace_event_dyn_try_get_ref(struct trace_event_call * call)413 static inline bool trace_event_dyn_try_get_ref(struct trace_event_call *call)
414 {
415 	/* Without DYNAMIC_EVENTS configured, nothing should be calling this */
416 	return false;
417 }
trace_event_dyn_put_ref(struct trace_event_call * call)418 static inline void trace_event_dyn_put_ref(struct trace_event_call *call)
419 {
420 }
trace_event_dyn_busy(struct trace_event_call * call)421 static inline bool trace_event_dyn_busy(struct trace_event_call *call)
422 {
423 	/* Nothing should call this without DYNAIMIC_EVENTS configured. */
424 	return true;
425 }
426 #endif
427 
trace_event_try_get_ref(struct trace_event_call * call)428 static inline bool trace_event_try_get_ref(struct trace_event_call *call)
429 {
430 	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
431 		return trace_event_dyn_try_get_ref(call);
432 	else
433 		return try_module_get(call->module);
434 }
435 
trace_event_put_ref(struct trace_event_call * call)436 static inline void trace_event_put_ref(struct trace_event_call *call)
437 {
438 	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
439 		trace_event_dyn_put_ref(call);
440 	else
441 		module_put(call->module);
442 }
443 
444 #ifdef CONFIG_PERF_EVENTS
bpf_prog_array_valid(struct trace_event_call * call)445 static inline bool bpf_prog_array_valid(struct trace_event_call *call)
446 {
447 	/*
448 	 * This inline function checks whether call->prog_array
449 	 * is valid or not. The function is called in various places,
450 	 * outside rcu_read_lock/unlock, as a heuristic to speed up execution.
451 	 *
452 	 * If this function returns true, and later call->prog_array
453 	 * becomes false inside rcu_read_lock/unlock region,
454 	 * we bail out then. If this function return false,
455 	 * there is a risk that we might miss a few events if the checking
456 	 * were delayed until inside rcu_read_lock/unlock region and
457 	 * call->prog_array happened to become non-NULL then.
458 	 *
459 	 * Here, READ_ONCE() is used instead of rcu_access_pointer().
460 	 * rcu_access_pointer() requires the actual definition of
461 	 * "struct bpf_prog_array" while READ_ONCE() only needs
462 	 * a declaration of the same type.
463 	 */
464 	return !!READ_ONCE(call->prog_array);
465 }
466 #endif
467 
468 static inline const char *
trace_event_name(struct trace_event_call * call)469 trace_event_name(struct trace_event_call *call)
470 {
471 	if (call->flags & TRACE_EVENT_FL_CUSTOM)
472 		return call->name;
473 	else if (call->flags & TRACE_EVENT_FL_TRACEPOINT)
474 		return call->tp ? call->tp->name : NULL;
475 	else
476 		return call->name;
477 }
478 
479 static inline struct list_head *
trace_get_fields(struct trace_event_call * event_call)480 trace_get_fields(struct trace_event_call *event_call)
481 {
482 	if (!event_call->class->get_fields)
483 		return &event_call->class->fields;
484 	return event_call->class->get_fields(event_call);
485 }
486 
487 struct trace_subsystem_dir;
488 
489 enum {
490 	EVENT_FILE_FL_ENABLED_BIT,
491 	EVENT_FILE_FL_RECORDED_CMD_BIT,
492 	EVENT_FILE_FL_RECORDED_TGID_BIT,
493 	EVENT_FILE_FL_FILTERED_BIT,
494 	EVENT_FILE_FL_NO_SET_FILTER_BIT,
495 	EVENT_FILE_FL_SOFT_MODE_BIT,
496 	EVENT_FILE_FL_SOFT_DISABLED_BIT,
497 	EVENT_FILE_FL_TRIGGER_MODE_BIT,
498 	EVENT_FILE_FL_TRIGGER_COND_BIT,
499 	EVENT_FILE_FL_PID_FILTER_BIT,
500 	EVENT_FILE_FL_WAS_ENABLED_BIT,
501 	EVENT_FILE_FL_FREED_BIT,
502 };
503 
504 extern struct trace_event_file *trace_get_event_file(const char *instance,
505 						     const char *system,
506 						     const char *event);
507 extern void trace_put_event_file(struct trace_event_file *file);
508 
509 #define MAX_DYNEVENT_CMD_LEN	(2048)
510 
511 enum dynevent_type {
512 	DYNEVENT_TYPE_SYNTH = 1,
513 	DYNEVENT_TYPE_KPROBE,
514 	DYNEVENT_TYPE_NONE,
515 };
516 
517 struct dynevent_cmd;
518 
519 typedef int (*dynevent_create_fn_t)(struct dynevent_cmd *cmd);
520 
521 struct dynevent_cmd {
522 	struct seq_buf		seq;
523 	const char		*event_name;
524 	unsigned int		n_fields;
525 	enum dynevent_type	type;
526 	dynevent_create_fn_t	run_command;
527 	void			*private_data;
528 };
529 
530 extern int dynevent_create(struct dynevent_cmd *cmd);
531 
532 extern int synth_event_delete(const char *name);
533 
534 extern void synth_event_cmd_init(struct dynevent_cmd *cmd,
535 				 char *buf, int maxlen);
536 
537 extern int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd,
538 				       const char *name,
539 				       struct module *mod, ...);
540 
541 #define synth_event_gen_cmd_start(cmd, name, mod, ...)	\
542 	__synth_event_gen_cmd_start(cmd, name, mod, ## __VA_ARGS__, NULL)
543 
544 struct synth_field_desc {
545 	const char *type;
546 	const char *name;
547 };
548 
549 extern int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd,
550 					   const char *name,
551 					   struct module *mod,
552 					   struct synth_field_desc *fields,
553 					   unsigned int n_fields);
554 extern int synth_event_create(const char *name,
555 			      struct synth_field_desc *fields,
556 			      unsigned int n_fields, struct module *mod);
557 
558 extern int synth_event_add_field(struct dynevent_cmd *cmd,
559 				 const char *type,
560 				 const char *name);
561 extern int synth_event_add_field_str(struct dynevent_cmd *cmd,
562 				     const char *type_name);
563 extern int synth_event_add_fields(struct dynevent_cmd *cmd,
564 				  struct synth_field_desc *fields,
565 				  unsigned int n_fields);
566 
567 #define synth_event_gen_cmd_end(cmd)	\
568 	dynevent_create(cmd)
569 
570 struct synth_event;
571 
572 struct synth_event_trace_state {
573 	struct trace_event_buffer fbuffer;
574 	struct synth_trace_event *entry;
575 	struct trace_buffer *buffer;
576 	struct synth_event *event;
577 	unsigned int cur_field;
578 	unsigned int n_u64;
579 	bool disabled;
580 	bool add_next;
581 	bool add_name;
582 };
583 
584 extern int synth_event_trace(struct trace_event_file *file,
585 			     unsigned int n_vals, ...);
586 extern int synth_event_trace_array(struct trace_event_file *file, u64 *vals,
587 				   unsigned int n_vals);
588 extern int synth_event_trace_start(struct trace_event_file *file,
589 				   struct synth_event_trace_state *trace_state);
590 extern int synth_event_add_next_val(u64 val,
591 				    struct synth_event_trace_state *trace_state);
592 extern int synth_event_add_val(const char *field_name, u64 val,
593 			       struct synth_event_trace_state *trace_state);
594 extern int synth_event_trace_end(struct synth_event_trace_state *trace_state);
595 
596 extern int kprobe_event_delete(const char *name);
597 
598 extern void kprobe_event_cmd_init(struct dynevent_cmd *cmd,
599 				  char *buf, int maxlen);
600 
601 #define kprobe_event_gen_cmd_start(cmd, name, loc, ...)			\
602 	__kprobe_event_gen_cmd_start(cmd, false, name, loc, ## __VA_ARGS__, NULL)
603 
604 #define kretprobe_event_gen_cmd_start(cmd, name, loc, ...)		\
605 	__kprobe_event_gen_cmd_start(cmd, true, name, loc, ## __VA_ARGS__, NULL)
606 
607 extern int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd,
608 					bool kretprobe,
609 					const char *name,
610 					const char *loc, ...);
611 
612 #define kprobe_event_add_fields(cmd, ...)	\
613 	__kprobe_event_add_fields(cmd, ## __VA_ARGS__, NULL)
614 
615 #define kprobe_event_add_field(cmd, field)	\
616 	__kprobe_event_add_fields(cmd, field, NULL)
617 
618 extern int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...);
619 
620 #define kprobe_event_gen_cmd_end(cmd)		\
621 	dynevent_create(cmd)
622 
623 #define kretprobe_event_gen_cmd_end(cmd)	\
624 	dynevent_create(cmd)
625 
626 /*
627  * Event file flags:
628  *  ENABLED	  - The event is enabled
629  *  RECORDED_CMD  - The comms should be recorded at sched_switch
630  *  RECORDED_TGID - The tgids should be recorded at sched_switch
631  *  FILTERED	  - The event has a filter attached
632  *  NO_SET_FILTER - Set when filter has error and is to be ignored
633  *  SOFT_MODE     - The event is enabled/disabled by SOFT_DISABLED
634  *  SOFT_DISABLED - When set, do not trace the event (even though its
635  *                   tracepoint may be enabled)
636  *  TRIGGER_MODE  - When set, invoke the triggers associated with the event
637  *  TRIGGER_COND  - When set, one or more triggers has an associated filter
638  *  PID_FILTER    - When set, the event is filtered based on pid
639  *  WAS_ENABLED   - Set when enabled to know to clear trace on module removal
640  *  FREED         - File descriptor is freed, all fields should be considered invalid
641  */
642 enum {
643 	EVENT_FILE_FL_ENABLED		= (1 << EVENT_FILE_FL_ENABLED_BIT),
644 	EVENT_FILE_FL_RECORDED_CMD	= (1 << EVENT_FILE_FL_RECORDED_CMD_BIT),
645 	EVENT_FILE_FL_RECORDED_TGID	= (1 << EVENT_FILE_FL_RECORDED_TGID_BIT),
646 	EVENT_FILE_FL_FILTERED		= (1 << EVENT_FILE_FL_FILTERED_BIT),
647 	EVENT_FILE_FL_NO_SET_FILTER	= (1 << EVENT_FILE_FL_NO_SET_FILTER_BIT),
648 	EVENT_FILE_FL_SOFT_MODE		= (1 << EVENT_FILE_FL_SOFT_MODE_BIT),
649 	EVENT_FILE_FL_SOFT_DISABLED	= (1 << EVENT_FILE_FL_SOFT_DISABLED_BIT),
650 	EVENT_FILE_FL_TRIGGER_MODE	= (1 << EVENT_FILE_FL_TRIGGER_MODE_BIT),
651 	EVENT_FILE_FL_TRIGGER_COND	= (1 << EVENT_FILE_FL_TRIGGER_COND_BIT),
652 	EVENT_FILE_FL_PID_FILTER	= (1 << EVENT_FILE_FL_PID_FILTER_BIT),
653 	EVENT_FILE_FL_WAS_ENABLED	= (1 << EVENT_FILE_FL_WAS_ENABLED_BIT),
654 	EVENT_FILE_FL_FREED		= (1 << EVENT_FILE_FL_FREED_BIT),
655 };
656 
657 struct trace_event_file {
658 	struct list_head		list;
659 	struct trace_event_call		*event_call;
660 	struct event_filter __rcu	*filter;
661 	struct eventfs_inode		*ei;
662 	struct trace_array		*tr;
663 	struct trace_subsystem_dir	*system;
664 	struct list_head		triggers;
665 
666 	/*
667 	 * 32 bit flags:
668 	 *   bit 0:		enabled
669 	 *   bit 1:		enabled cmd record
670 	 *   bit 2:		enable/disable with the soft disable bit
671 	 *   bit 3:		soft disabled
672 	 *   bit 4:		trigger enabled
673 	 *
674 	 * Note: The bits must be set atomically to prevent races
675 	 * from other writers. Reads of flags do not need to be in
676 	 * sync as they occur in critical sections. But the way flags
677 	 * is currently used, these changes do not affect the code
678 	 * except that when a change is made, it may have a slight
679 	 * delay in propagating the changes to other CPUs due to
680 	 * caching and such. Which is mostly OK ;-)
681 	 */
682 	unsigned long		flags;
683 	refcount_t		ref;	/* ref count for opened files */
684 	atomic_t		sm_ref;	/* soft-mode reference counter */
685 	atomic_t		tm_ref;	/* trigger-mode reference counter */
686 };
687 
688 #define __TRACE_EVENT_FLAGS(name, value)				\
689 	static int __init trace_init_flags_##name(void)			\
690 	{								\
691 		event_##name.flags |= value;				\
692 		return 0;						\
693 	}								\
694 	early_initcall(trace_init_flags_##name);
695 
696 #define __TRACE_EVENT_PERF_PERM(name, expr...)				\
697 	static int perf_perm_##name(struct trace_event_call *tp_event, \
698 				    struct perf_event *p_event)		\
699 	{								\
700 		return ({ expr; });					\
701 	}								\
702 	static int __init trace_init_perf_perm_##name(void)		\
703 	{								\
704 		event_##name.perf_perm = &perf_perm_##name;		\
705 		return 0;						\
706 	}								\
707 	early_initcall(trace_init_perf_perm_##name);
708 
709 #define PERF_MAX_TRACE_SIZE	8192
710 
711 #define MAX_FILTER_STR_VAL	256U	/* Should handle KSYM_SYMBOL_LEN */
712 
713 enum event_trigger_type {
714 	ETT_NONE		= (0),
715 	ETT_TRACE_ONOFF		= (1 << 0),
716 	ETT_SNAPSHOT		= (1 << 1),
717 	ETT_STACKTRACE		= (1 << 2),
718 	ETT_EVENT_ENABLE	= (1 << 3),
719 	ETT_EVENT_HIST		= (1 << 4),
720 	ETT_HIST_ENABLE		= (1 << 5),
721 	ETT_EVENT_EPROBE	= (1 << 6),
722 };
723 
724 extern int filter_match_preds(struct event_filter *filter, void *rec);
725 
726 extern enum event_trigger_type
727 event_triggers_call(struct trace_event_file *file,
728 		    struct trace_buffer *buffer, void *rec,
729 		    struct ring_buffer_event *event);
730 extern void
731 event_triggers_post_call(struct trace_event_file *file,
732 			 enum event_trigger_type tt);
733 
734 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file);
735 
736 bool __trace_trigger_soft_disabled(struct trace_event_file *file);
737 
738 /**
739  * trace_trigger_soft_disabled - do triggers and test if soft disabled
740  * @file: The file pointer of the event to test
741  *
742  * If any triggers without filters are attached to this event, they
743  * will be called here. If the event is soft disabled and has no
744  * triggers that require testing the fields, it will return true,
745  * otherwise false.
746  */
747 static __always_inline bool
trace_trigger_soft_disabled(struct trace_event_file * file)748 trace_trigger_soft_disabled(struct trace_event_file *file)
749 {
750 	unsigned long eflags = file->flags;
751 
752 	if (likely(!(eflags & (EVENT_FILE_FL_TRIGGER_MODE |
753 			       EVENT_FILE_FL_SOFT_DISABLED |
754 			       EVENT_FILE_FL_PID_FILTER))))
755 		return false;
756 
757 	if (likely(eflags & EVENT_FILE_FL_TRIGGER_COND))
758 		return false;
759 
760 	return __trace_trigger_soft_disabled(file);
761 }
762 
763 #ifdef CONFIG_BPF_EVENTS
764 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
765 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
766 void perf_event_detach_bpf_prog(struct perf_event *event);
767 int perf_event_query_prog_array(struct perf_event *event, void __user *info);
768 
769 struct bpf_raw_tp_link;
770 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
771 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
772 
773 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name);
774 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
775 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
776 			    u32 *fd_type, const char **buf,
777 			    u64 *probe_offset, u64 *probe_addr,
778 			    unsigned long *missed);
779 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
780 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
781 #else
trace_call_bpf(struct trace_event_call * call,void * ctx)782 static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
783 {
784 	return 1;
785 }
786 
787 static inline int
perf_event_attach_bpf_prog(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)788 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
789 {
790 	return -EOPNOTSUPP;
791 }
792 
perf_event_detach_bpf_prog(struct perf_event * event)793 static inline void perf_event_detach_bpf_prog(struct perf_event *event) { }
794 
795 static inline int
perf_event_query_prog_array(struct perf_event * event,void __user * info)796 perf_event_query_prog_array(struct perf_event *event, void __user *info)
797 {
798 	return -EOPNOTSUPP;
799 }
800 struct bpf_raw_tp_link;
bpf_probe_register(struct bpf_raw_event_map * btp,struct bpf_raw_tp_link * link)801 static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
802 {
803 	return -EOPNOTSUPP;
804 }
bpf_probe_unregister(struct bpf_raw_event_map * btp,struct bpf_raw_tp_link * link)805 static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
806 {
807 	return -EOPNOTSUPP;
808 }
bpf_get_raw_tracepoint(const char * name)809 static inline struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
810 {
811 	return NULL;
812 }
bpf_put_raw_tracepoint(struct bpf_raw_event_map * btp)813 static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
814 {
815 }
bpf_get_perf_event_info(const struct perf_event * event,u32 * prog_id,u32 * fd_type,const char ** buf,u64 * probe_offset,u64 * probe_addr,unsigned long * missed)816 static inline int bpf_get_perf_event_info(const struct perf_event *event,
817 					  u32 *prog_id, u32 *fd_type,
818 					  const char **buf, u64 *probe_offset,
819 					  u64 *probe_addr, unsigned long *missed)
820 {
821 	return -EOPNOTSUPP;
822 }
823 static inline int
bpf_kprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)824 bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
825 {
826 	return -EOPNOTSUPP;
827 }
828 static inline int
bpf_uprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)829 bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
830 {
831 	return -EOPNOTSUPP;
832 }
833 #endif
834 
835 enum {
836 	FILTER_OTHER = 0,
837 	FILTER_STATIC_STRING,
838 	FILTER_DYN_STRING,
839 	FILTER_RDYN_STRING,
840 	FILTER_PTR_STRING,
841 	FILTER_TRACE_FN,
842 	FILTER_CPUMASK,
843 	FILTER_COMM,
844 	FILTER_CPU,
845 	FILTER_STACKTRACE,
846 };
847 
848 extern int trace_event_raw_init(struct trace_event_call *call);
849 extern int trace_define_field(struct trace_event_call *call, const char *type,
850 			      const char *name, int offset, int size,
851 			      int is_signed, int filter_type);
852 extern int trace_add_event_call(struct trace_event_call *call);
853 extern int trace_remove_event_call(struct trace_event_call *call);
854 extern int trace_event_get_offsets(struct trace_event_call *call);
855 
856 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
857 int trace_set_clr_event(const char *system, const char *event, int set);
858 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
859 		const char *event, bool enable);
860 /*
861  * The double __builtin_constant_p is because gcc will give us an error
862  * if we try to allocate the static variable to fmt if it is not a
863  * constant. Even with the outer if statement optimizing out.
864  */
865 #define event_trace_printk(ip, fmt, args...)				\
866 do {									\
867 	__trace_printk_check_format(fmt, ##args);			\
868 	tracing_record_cmdline(current);				\
869 	if (__builtin_constant_p(fmt)) {				\
870 		static const char *trace_printk_fmt			\
871 		  __section("__trace_printk_fmt") =			\
872 			__builtin_constant_p(fmt) ? fmt : NULL;		\
873 									\
874 		__trace_bprintk(ip, trace_printk_fmt, ##args);		\
875 	} else								\
876 		__trace_printk(ip, fmt, ##args);			\
877 } while (0)
878 
879 #ifdef CONFIG_PERF_EVENTS
880 struct perf_event;
881 
882 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
883 
884 extern int  perf_trace_init(struct perf_event *event);
885 extern void perf_trace_destroy(struct perf_event *event);
886 extern int  perf_trace_add(struct perf_event *event, int flags);
887 extern void perf_trace_del(struct perf_event *event, int flags);
888 #ifdef CONFIG_KPROBE_EVENTS
889 extern int  perf_kprobe_init(struct perf_event *event, bool is_retprobe);
890 extern void perf_kprobe_destroy(struct perf_event *event);
891 extern int bpf_get_kprobe_info(const struct perf_event *event,
892 			       u32 *fd_type, const char **symbol,
893 			       u64 *probe_offset, u64 *probe_addr,
894 			       unsigned long *missed,
895 			       bool perf_type_tracepoint);
896 #endif
897 #ifdef CONFIG_UPROBE_EVENTS
898 extern int  perf_uprobe_init(struct perf_event *event,
899 			     unsigned long ref_ctr_offset, bool is_retprobe);
900 extern void perf_uprobe_destroy(struct perf_event *event);
901 extern int bpf_get_uprobe_info(const struct perf_event *event,
902 			       u32 *fd_type, const char **filename,
903 			       u64 *probe_offset, u64 *probe_addr,
904 			       bool perf_type_tracepoint);
905 #endif
906 extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,
907 				     char *filter_str);
908 extern void ftrace_profile_free_filter(struct perf_event *event);
909 void perf_trace_buf_update(void *record, u16 type);
910 void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
911 
912 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
913 void perf_event_free_bpf_prog(struct perf_event *event);
914 
915 void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1);
916 void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2);
917 void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
918 		    u64 arg3);
919 void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
920 		    u64 arg3, u64 arg4);
921 void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
922 		    u64 arg3, u64 arg4, u64 arg5);
923 void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
924 		    u64 arg3, u64 arg4, u64 arg5, u64 arg6);
925 void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
926 		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);
927 void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
928 		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
929 		    u64 arg8);
930 void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
931 		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
932 		    u64 arg8, u64 arg9);
933 void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
934 		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
935 		     u64 arg8, u64 arg9, u64 arg10);
936 void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
937 		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
938 		     u64 arg8, u64 arg9, u64 arg10, u64 arg11);
939 void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
940 		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
941 		     u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);
942 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
943 			       struct trace_event_call *call, u64 count,
944 			       struct pt_regs *regs, struct hlist_head *head,
945 			       struct task_struct *task);
946 
947 static inline void
perf_trace_buf_submit(void * raw_data,int size,int rctx,u16 type,u64 count,struct pt_regs * regs,void * head,struct task_struct * task)948 perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type,
949 		       u64 count, struct pt_regs *regs, void *head,
950 		       struct task_struct *task)
951 {
952 	perf_tp_event(type, count, raw_data, size, regs, head, rctx, task);
953 }
954 
955 #endif
956 
957 #define TRACE_EVENT_STR_MAX	512
958 
959 /*
960  * gcc warns that you can not use a va_list in an inlined
961  * function. But lets me make it into a macro :-/
962  */
963 #define __trace_event_vstr_len(fmt, va)			\
964 ({							\
965 	va_list __ap;					\
966 	int __ret;					\
967 							\
968 	va_copy(__ap, *(va));				\
969 	__ret = vsnprintf(NULL, 0, fmt, __ap) + 1;	\
970 	va_end(__ap);					\
971 							\
972 	min(__ret, TRACE_EVENT_STR_MAX);		\
973 })
974 
975 #endif /* _LINUX_TRACE_EVENT_H */
976 
977 /*
978  * Note: we keep the TRACE_CUSTOM_EVENT outside the include file ifdef protection.
979  *  This is due to the way trace custom events work. If a file includes two
980  *  trace event headers under one "CREATE_CUSTOM_TRACE_EVENTS" the first include
981  *  will override the TRACE_CUSTOM_EVENT and break the second include.
982  */
983 
984 #ifndef TRACE_CUSTOM_EVENT
985 
986 #define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print)
987 #define DEFINE_CUSTOM_EVENT(template, name, proto, args)
988 #define TRACE_CUSTOM_EVENT(name, proto, args, struct, assign, print)
989 
990 #endif /* ifdef TRACE_CUSTOM_EVENT (see note above) */
991