xref: /linux/kernel/trace/trace_osnoise.c (revision 90ab2117f4cdde06ac1fd4906bfa46149ddd4323)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4  * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5  *
6  * Based on "hwlat_detector" tracer by:
7  *   Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8  *   Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9  *   With feedback from Clark Williams <williams@redhat.com>
10  *
11  * And also based on the rtsl tracer presented on:
12  *  DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13  *  scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14  *  (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15  *
16  * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
17  */
18 
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
27 #include "trace.h"
28 
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
34 
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
37 
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
40 
41 /*
42  * Default values.
43  */
44 #define BANNER			"osnoise: "
45 #define DEFAULT_SAMPLE_PERIOD	1000000			/* 1s */
46 #define DEFAULT_SAMPLE_RUNTIME	1000000			/* 1s */
47 
48 #define DEFAULT_TIMERLAT_PERIOD	1000			/* 1ms */
49 #define DEFAULT_TIMERLAT_PRIO	95			/* FIFO 95 */
50 
51 /*
52  * osnoise/options entries.
53  */
54 enum osnoise_options_index {
55 	OSN_DEFAULTS = 0,
56 	OSN_WORKLOAD,
57 	OSN_PANIC_ON_STOP,
58 	OSN_PREEMPT_DISABLE,
59 	OSN_IRQ_DISABLE,
60 	OSN_MAX
61 };
62 
63 static const char * const osnoise_options_str[OSN_MAX] = {
64 							"DEFAULTS",
65 							"OSNOISE_WORKLOAD",
66 							"PANIC_ON_STOP",
67 							"OSNOISE_PREEMPT_DISABLE",
68 							"OSNOISE_IRQ_DISABLE" };
69 
70 #define OSN_DEFAULT_OPTIONS		0x2
71 static unsigned long osnoise_options	= OSN_DEFAULT_OPTIONS;
72 
73 /*
74  * trace_array of the enabled osnoise/timerlat instances.
75  */
76 struct osnoise_instance {
77 	struct list_head	list;
78 	struct trace_array	*tr;
79 };
80 
81 static struct list_head osnoise_instances;
82 
osnoise_has_registered_instances(void)83 static bool osnoise_has_registered_instances(void)
84 {
85 	return !!list_first_or_null_rcu(&osnoise_instances,
86 					struct osnoise_instance,
87 					list);
88 }
89 
90 /*
91  * osnoise_instance_registered - check if a tr is already registered
92  */
osnoise_instance_registered(struct trace_array * tr)93 static int osnoise_instance_registered(struct trace_array *tr)
94 {
95 	struct osnoise_instance *inst;
96 	int found = 0;
97 
98 	rcu_read_lock();
99 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
100 		if (inst->tr == tr)
101 			found = 1;
102 	}
103 	rcu_read_unlock();
104 
105 	return found;
106 }
107 
108 /*
109  * osnoise_register_instance - register a new trace instance
110  *
111  * Register a trace_array *tr in the list of instances running
112  * osnoise/timerlat tracers.
113  */
osnoise_register_instance(struct trace_array * tr)114 static int osnoise_register_instance(struct trace_array *tr)
115 {
116 	struct osnoise_instance *inst;
117 
118 	/*
119 	 * register/unregister serialization is provided by trace's
120 	 * trace_types_lock.
121 	 */
122 	lockdep_assert_held(&trace_types_lock);
123 
124 	inst = kmalloc(sizeof(*inst), GFP_KERNEL);
125 	if (!inst)
126 		return -ENOMEM;
127 
128 	INIT_LIST_HEAD_RCU(&inst->list);
129 	inst->tr = tr;
130 	list_add_tail_rcu(&inst->list, &osnoise_instances);
131 
132 	return 0;
133 }
134 
135 /*
136  *  osnoise_unregister_instance - unregister a registered trace instance
137  *
138  * Remove the trace_array *tr from the list of instances running
139  * osnoise/timerlat tracers.
140  */
osnoise_unregister_instance(struct trace_array * tr)141 static void osnoise_unregister_instance(struct trace_array *tr)
142 {
143 	struct osnoise_instance *inst;
144 	int found = 0;
145 
146 	/*
147 	 * register/unregister serialization is provided by trace's
148 	 * trace_types_lock.
149 	 */
150 	list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 				lockdep_is_held(&trace_types_lock)) {
152 		if (inst->tr == tr) {
153 			list_del_rcu(&inst->list);
154 			found = 1;
155 			break;
156 		}
157 	}
158 
159 	if (!found)
160 		return;
161 
162 	kvfree_rcu_mightsleep(inst);
163 }
164 
165 /*
166  * NMI runtime info.
167  */
168 struct osn_nmi {
169 	u64	count;
170 	u64	delta_start;
171 };
172 
173 /*
174  * IRQ runtime info.
175  */
176 struct osn_irq {
177 	u64	count;
178 	u64	arrival_time;
179 	u64	delta_start;
180 };
181 
182 #define IRQ_CONTEXT	0
183 #define THREAD_CONTEXT	1
184 #define THREAD_URET	2
185 /*
186  * sofirq runtime info.
187  */
188 struct osn_softirq {
189 	u64	count;
190 	u64	arrival_time;
191 	u64	delta_start;
192 };
193 
194 /*
195  * thread runtime info.
196  */
197 struct osn_thread {
198 	u64	count;
199 	u64	arrival_time;
200 	u64	delta_start;
201 };
202 
203 /*
204  * Runtime information: this structure saves the runtime information used by
205  * one sampling thread.
206  */
207 struct osnoise_variables {
208 	struct task_struct	*kthread;
209 	bool			sampling;
210 	pid_t			pid;
211 	struct osn_nmi		nmi;
212 	struct osn_irq		irq;
213 	struct osn_softirq	softirq;
214 	struct osn_thread	thread;
215 	local_t			int_counter;
216 };
217 
218 /*
219  * Per-cpu runtime information.
220  */
221 static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
222 
223 /*
224  * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
225  */
this_cpu_osn_var(void)226 static inline struct osnoise_variables *this_cpu_osn_var(void)
227 {
228 	return this_cpu_ptr(&per_cpu_osnoise_var);
229 }
230 
231 /*
232  * Protect the interface.
233  */
234 static struct mutex interface_lock;
235 
236 #ifdef CONFIG_TIMERLAT_TRACER
237 /*
238  * Runtime information for the timer mode.
239  */
240 struct timerlat_variables {
241 	struct task_struct	*kthread;
242 	struct hrtimer		timer;
243 	u64			rel_period;
244 	u64			abs_period;
245 	bool			tracing_thread;
246 	u64			count;
247 	bool			uthread_migrate;
248 };
249 
250 static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
251 
252 /*
253  * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
254  */
this_cpu_tmr_var(void)255 static inline struct timerlat_variables *this_cpu_tmr_var(void)
256 {
257 	return this_cpu_ptr(&per_cpu_timerlat_var);
258 }
259 
260 /*
261  * tlat_var_reset - Reset the values of the given timerlat_variables
262  */
tlat_var_reset(void)263 static inline void tlat_var_reset(void)
264 {
265 	struct timerlat_variables *tlat_var;
266 	int cpu;
267 
268 	/* Synchronize with the timerlat interfaces */
269 	mutex_lock(&interface_lock);
270 	/*
271 	 * So far, all the values are initialized as 0, so
272 	 * zeroing the structure is perfect.
273 	 */
274 	for_each_cpu(cpu, cpu_online_mask) {
275 		tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
276 		if (tlat_var->kthread)
277 			hrtimer_cancel(&tlat_var->timer);
278 		memset(tlat_var, 0, sizeof(*tlat_var));
279 	}
280 	mutex_unlock(&interface_lock);
281 }
282 #else /* CONFIG_TIMERLAT_TRACER */
283 #define tlat_var_reset()	do {} while (0)
284 #endif /* CONFIG_TIMERLAT_TRACER */
285 
286 /*
287  * osn_var_reset - Reset the values of the given osnoise_variables
288  */
osn_var_reset(void)289 static inline void osn_var_reset(void)
290 {
291 	struct osnoise_variables *osn_var;
292 	int cpu;
293 
294 	/*
295 	 * So far, all the values are initialized as 0, so
296 	 * zeroing the structure is perfect.
297 	 */
298 	for_each_cpu(cpu, cpu_online_mask) {
299 		osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
300 		memset(osn_var, 0, sizeof(*osn_var));
301 	}
302 }
303 
304 /*
305  * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
306  */
osn_var_reset_all(void)307 static inline void osn_var_reset_all(void)
308 {
309 	osn_var_reset();
310 	tlat_var_reset();
311 }
312 
313 /*
314  * Tells NMIs to call back to the osnoise tracer to record timestamps.
315  */
316 bool trace_osnoise_callback_enabled;
317 
318 /*
319  * osnoise sample structure definition. Used to store the statistics of a
320  * sample run.
321  */
322 struct osnoise_sample {
323 	u64			runtime;	/* runtime */
324 	u64			noise;		/* noise */
325 	u64			max_sample;	/* max single noise sample */
326 	int			hw_count;	/* # HW (incl. hypervisor) interference */
327 	int			nmi_count;	/* # NMIs during this sample */
328 	int			irq_count;	/* # IRQs during this sample */
329 	int			softirq_count;	/* # softirqs during this sample */
330 	int			thread_count;	/* # threads during this sample */
331 };
332 
333 #ifdef CONFIG_TIMERLAT_TRACER
334 /*
335  * timerlat sample structure definition. Used to store the statistics of
336  * a sample run.
337  */
338 struct timerlat_sample {
339 	u64			timer_latency;	/* timer_latency */
340 	unsigned int		seqnum;		/* unique sequence */
341 	int			context;	/* timer context */
342 };
343 #endif
344 
345 /*
346  * Tracer data.
347  */
348 static struct osnoise_data {
349 	u64	sample_period;		/* total sampling period */
350 	u64	sample_runtime;		/* active sampling portion of period */
351 	u64	stop_tracing;		/* stop trace in the internal operation (loop/irq) */
352 	u64	stop_tracing_total;	/* stop trace in the final operation (report/thread) */
353 #ifdef CONFIG_TIMERLAT_TRACER
354 	u64	timerlat_period;	/* timerlat period */
355 	u64	print_stack;		/* print IRQ stack if total > */
356 	int	timerlat_tracer;	/* timerlat tracer */
357 #endif
358 	bool	tainted;		/* infor users and developers about a problem */
359 } osnoise_data = {
360 	.sample_period			= DEFAULT_SAMPLE_PERIOD,
361 	.sample_runtime			= DEFAULT_SAMPLE_RUNTIME,
362 	.stop_tracing			= 0,
363 	.stop_tracing_total		= 0,
364 #ifdef CONFIG_TIMERLAT_TRACER
365 	.print_stack			= 0,
366 	.timerlat_period		= DEFAULT_TIMERLAT_PERIOD,
367 	.timerlat_tracer		= 0,
368 #endif
369 };
370 
371 #ifdef CONFIG_TIMERLAT_TRACER
timerlat_enabled(void)372 static inline bool timerlat_enabled(void)
373 {
374 	return osnoise_data.timerlat_tracer;
375 }
376 
timerlat_softirq_exit(struct osnoise_variables * osn_var)377 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
378 {
379 	struct timerlat_variables *tlat_var = this_cpu_tmr_var();
380 	/*
381 	 * If the timerlat is enabled, but the irq handler did
382 	 * not run yet enabling timerlat_tracer, do not trace.
383 	 */
384 	if (!tlat_var->tracing_thread) {
385 		osn_var->softirq.arrival_time = 0;
386 		osn_var->softirq.delta_start = 0;
387 		return 0;
388 	}
389 	return 1;
390 }
391 
timerlat_thread_exit(struct osnoise_variables * osn_var)392 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
393 {
394 	struct timerlat_variables *tlat_var = this_cpu_tmr_var();
395 	/*
396 	 * If the timerlat is enabled, but the irq handler did
397 	 * not run yet enabling timerlat_tracer, do not trace.
398 	 */
399 	if (!tlat_var->tracing_thread) {
400 		osn_var->thread.delta_start = 0;
401 		osn_var->thread.arrival_time = 0;
402 		return 0;
403 	}
404 	return 1;
405 }
406 #else /* CONFIG_TIMERLAT_TRACER */
timerlat_enabled(void)407 static inline bool timerlat_enabled(void)
408 {
409 	return false;
410 }
411 
timerlat_softirq_exit(struct osnoise_variables * osn_var)412 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
413 {
414 	return 1;
415 }
timerlat_thread_exit(struct osnoise_variables * osn_var)416 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
417 {
418 	return 1;
419 }
420 #endif
421 
422 #ifdef CONFIG_PREEMPT_RT
423 /*
424  * Print the osnoise header info.
425  */
print_osnoise_headers(struct seq_file * s)426 static void print_osnoise_headers(struct seq_file *s)
427 {
428 	if (osnoise_data.tainted)
429 		seq_puts(s, "# osnoise is tainted!\n");
430 
431 	seq_puts(s, "#                                _-------=> irqs-off\n");
432 	seq_puts(s, "#                               / _------=> need-resched\n");
433 	seq_puts(s, "#                              | / _-----=> need-resched-lazy\n");
434 	seq_puts(s, "#                              || / _----=> hardirq/softirq\n");
435 	seq_puts(s, "#                              ||| / _---=> preempt-depth\n");
436 	seq_puts(s, "#                              |||| / _--=> preempt-lazy-depth\n");
437 	seq_puts(s, "#                              ||||| / _-=> migrate-disable\n");
438 
439 	seq_puts(s, "#                              |||||| /          ");
440 	seq_puts(s, "                                     MAX\n");
441 
442 	seq_puts(s, "#                              ||||| /                         ");
443 	seq_puts(s, "                    SINGLE      Interference counters:\n");
444 
445 	seq_puts(s, "#                              |||||||               RUNTIME   ");
446 	seq_puts(s, "   NOISE  %% OF CPU  NOISE    +-----------------------------+\n");
447 
448 	seq_puts(s, "#           TASK-PID      CPU# |||||||   TIMESTAMP    IN US    ");
449 	seq_puts(s, "   IN US  AVAILABLE  IN US     HW    NMI    IRQ   SIRQ THREAD\n");
450 
451 	seq_puts(s, "#              | |         |   |||||||      |           |      ");
452 	seq_puts(s, "       |    |            |      |      |      |      |      |\n");
453 }
454 #else /* CONFIG_PREEMPT_RT */
print_osnoise_headers(struct seq_file * s)455 static void print_osnoise_headers(struct seq_file *s)
456 {
457 	if (osnoise_data.tainted)
458 		seq_puts(s, "# osnoise is tainted!\n");
459 
460 	seq_puts(s, "#                                _-----=> irqs-off\n");
461 	seq_puts(s, "#                               / _----=> need-resched\n");
462 	seq_puts(s, "#                              | / _---=> hardirq/softirq\n");
463 	seq_puts(s, "#                              || / _--=> preempt-depth\n");
464 	seq_puts(s, "#                              ||| / _-=> migrate-disable     ");
465 	seq_puts(s, "                    MAX\n");
466 	seq_puts(s, "#                              |||| /     delay               ");
467 	seq_puts(s, "                    SINGLE      Interference counters:\n");
468 
469 	seq_puts(s, "#                              |||||               RUNTIME   ");
470 	seq_puts(s, "   NOISE  %% OF CPU  NOISE    +-----------------------------+\n");
471 
472 	seq_puts(s, "#           TASK-PID      CPU# |||||   TIMESTAMP    IN US    ");
473 	seq_puts(s, "   IN US  AVAILABLE  IN US     HW    NMI    IRQ   SIRQ THREAD\n");
474 
475 	seq_puts(s, "#              | |         |   |||||      |           |      ");
476 	seq_puts(s, "       |    |            |      |      |      |      |      |\n");
477 }
478 #endif /* CONFIG_PREEMPT_RT */
479 
480 /*
481  * osnoise_taint - report an osnoise error.
482  */
483 #define osnoise_taint(msg) ({							\
484 	struct osnoise_instance *inst;						\
485 	struct trace_buffer *buffer;						\
486 										\
487 	rcu_read_lock();							\
488 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {		\
489 		buffer = inst->tr->array_buffer.buffer;				\
490 		trace_array_printk_buf(buffer, _THIS_IP_, msg);			\
491 	}									\
492 	rcu_read_unlock();							\
493 	osnoise_data.tainted = true;						\
494 })
495 
496 /*
497  * Record an osnoise_sample into the tracer buffer.
498  */
499 static void
__trace_osnoise_sample(struct osnoise_sample * sample,struct trace_buffer * buffer)500 __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
501 {
502 	struct ring_buffer_event *event;
503 	struct osnoise_entry *entry;
504 
505 	event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
506 					  tracing_gen_ctx());
507 	if (!event)
508 		return;
509 	entry	= ring_buffer_event_data(event);
510 	entry->runtime		= sample->runtime;
511 	entry->noise		= sample->noise;
512 	entry->max_sample	= sample->max_sample;
513 	entry->hw_count		= sample->hw_count;
514 	entry->nmi_count	= sample->nmi_count;
515 	entry->irq_count	= sample->irq_count;
516 	entry->softirq_count	= sample->softirq_count;
517 	entry->thread_count	= sample->thread_count;
518 
519 	trace_buffer_unlock_commit_nostack(buffer, event);
520 }
521 
522 /*
523  * Record an osnoise_sample on all osnoise instances.
524  */
trace_osnoise_sample(struct osnoise_sample * sample)525 static void trace_osnoise_sample(struct osnoise_sample *sample)
526 {
527 	struct osnoise_instance *inst;
528 	struct trace_buffer *buffer;
529 
530 	rcu_read_lock();
531 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
532 		buffer = inst->tr->array_buffer.buffer;
533 		__trace_osnoise_sample(sample, buffer);
534 	}
535 	rcu_read_unlock();
536 }
537 
538 #ifdef CONFIG_TIMERLAT_TRACER
539 /*
540  * Print the timerlat header info.
541  */
542 #ifdef CONFIG_PREEMPT_RT
print_timerlat_headers(struct seq_file * s)543 static void print_timerlat_headers(struct seq_file *s)
544 {
545 	seq_puts(s, "#                                _-------=> irqs-off\n");
546 	seq_puts(s, "#                               / _------=> need-resched\n");
547 	seq_puts(s, "#                              | / _-----=> need-resched-lazy\n");
548 	seq_puts(s, "#                              || / _----=> hardirq/softirq\n");
549 	seq_puts(s, "#                              ||| / _---=> preempt-depth\n");
550 	seq_puts(s, "#                              |||| / _--=> preempt-lazy-depth\n");
551 	seq_puts(s, "#                              ||||| / _-=> migrate-disable\n");
552 	seq_puts(s, "#                              |||||| /\n");
553 	seq_puts(s, "#                              |||||||             ACTIVATION\n");
554 	seq_puts(s, "#           TASK-PID      CPU# |||||||   TIMESTAMP    ID     ");
555 	seq_puts(s, "       CONTEXT                LATENCY\n");
556 	seq_puts(s, "#              | |         |   |||||||      |         |      ");
557 	seq_puts(s, "            |                       |\n");
558 }
559 #else /* CONFIG_PREEMPT_RT */
print_timerlat_headers(struct seq_file * s)560 static void print_timerlat_headers(struct seq_file *s)
561 {
562 	seq_puts(s, "#                                _-----=> irqs-off\n");
563 	seq_puts(s, "#                               / _----=> need-resched\n");
564 	seq_puts(s, "#                              | / _---=> hardirq/softirq\n");
565 	seq_puts(s, "#                              || / _--=> preempt-depth\n");
566 	seq_puts(s, "#                              ||| / _-=> migrate-disable\n");
567 	seq_puts(s, "#                              |||| /     delay\n");
568 	seq_puts(s, "#                              |||||            ACTIVATION\n");
569 	seq_puts(s, "#           TASK-PID      CPU# |||||   TIMESTAMP   ID      ");
570 	seq_puts(s, "      CONTEXT                 LATENCY\n");
571 	seq_puts(s, "#              | |         |   |||||      |         |      ");
572 	seq_puts(s, "            |                       |\n");
573 }
574 #endif /* CONFIG_PREEMPT_RT */
575 
576 static void
__trace_timerlat_sample(struct timerlat_sample * sample,struct trace_buffer * buffer)577 __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
578 {
579 	struct ring_buffer_event *event;
580 	struct timerlat_entry *entry;
581 
582 	event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
583 					  tracing_gen_ctx());
584 	if (!event)
585 		return;
586 	entry	= ring_buffer_event_data(event);
587 	entry->seqnum			= sample->seqnum;
588 	entry->context			= sample->context;
589 	entry->timer_latency		= sample->timer_latency;
590 
591 	trace_buffer_unlock_commit_nostack(buffer, event);
592 }
593 
594 /*
595  * Record an timerlat_sample into the tracer buffer.
596  */
trace_timerlat_sample(struct timerlat_sample * sample)597 static void trace_timerlat_sample(struct timerlat_sample *sample)
598 {
599 	struct osnoise_instance *inst;
600 	struct trace_buffer *buffer;
601 
602 	rcu_read_lock();
603 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
604 		buffer = inst->tr->array_buffer.buffer;
605 		__trace_timerlat_sample(sample, buffer);
606 	}
607 	rcu_read_unlock();
608 }
609 
610 #ifdef CONFIG_STACKTRACE
611 
612 #define	MAX_CALLS	256
613 
614 /*
615  * Stack trace will take place only at IRQ level, so, no need
616  * to control nesting here.
617  */
618 struct trace_stack {
619 	int		stack_size;
620 	int		nr_entries;
621 	unsigned long	calls[MAX_CALLS];
622 };
623 
624 static DEFINE_PER_CPU(struct trace_stack, trace_stack);
625 
626 /*
627  * timerlat_save_stack - save a stack trace without printing
628  *
629  * Save the current stack trace without printing. The
630  * stack will be printed later, after the end of the measurement.
631  */
timerlat_save_stack(int skip)632 static void timerlat_save_stack(int skip)
633 {
634 	unsigned int size, nr_entries;
635 	struct trace_stack *fstack;
636 
637 	fstack = this_cpu_ptr(&trace_stack);
638 
639 	size = ARRAY_SIZE(fstack->calls);
640 
641 	nr_entries = stack_trace_save(fstack->calls, size, skip);
642 
643 	fstack->stack_size = nr_entries * sizeof(unsigned long);
644 	fstack->nr_entries = nr_entries;
645 
646 	return;
647 
648 }
649 
650 static void
__timerlat_dump_stack(struct trace_buffer * buffer,struct trace_stack * fstack,unsigned int size)651 __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
652 {
653 	struct ring_buffer_event *event;
654 	struct stack_entry *entry;
655 
656 	event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
657 					  tracing_gen_ctx());
658 	if (!event)
659 		return;
660 
661 	entry = ring_buffer_event_data(event);
662 
663 	memcpy(&entry->caller, fstack->calls, size);
664 	entry->size = fstack->nr_entries;
665 
666 	trace_buffer_unlock_commit_nostack(buffer, event);
667 }
668 
669 /*
670  * timerlat_dump_stack - dump a stack trace previously saved
671  */
timerlat_dump_stack(u64 latency)672 static void timerlat_dump_stack(u64 latency)
673 {
674 	struct osnoise_instance *inst;
675 	struct trace_buffer *buffer;
676 	struct trace_stack *fstack;
677 	unsigned int size;
678 
679 	/*
680 	 * trace only if latency > print_stack config, if enabled.
681 	 */
682 	if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
683 		return;
684 
685 	preempt_disable_notrace();
686 	fstack = this_cpu_ptr(&trace_stack);
687 	size = fstack->stack_size;
688 
689 	rcu_read_lock();
690 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
691 		buffer = inst->tr->array_buffer.buffer;
692 		__timerlat_dump_stack(buffer, fstack, size);
693 
694 	}
695 	rcu_read_unlock();
696 	preempt_enable_notrace();
697 }
698 #else /* CONFIG_STACKTRACE */
699 #define timerlat_dump_stack(u64 latency) do {} while (0)
700 #define timerlat_save_stack(a) do {} while (0)
701 #endif /* CONFIG_STACKTRACE */
702 #endif /* CONFIG_TIMERLAT_TRACER */
703 
704 /*
705  * Macros to encapsulate the time capturing infrastructure.
706  */
707 #define time_get()	trace_clock_local()
708 #define time_to_us(x)	div_u64(x, 1000)
709 #define time_sub(a, b)	((a) - (b))
710 
711 /*
712  * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
713  *
714  * If an IRQ is preempted by an NMI, its delta_start is pushed forward
715  * to discount the NMI interference.
716  *
717  * See get_int_safe_duration().
718  */
719 static inline void
cond_move_irq_delta_start(struct osnoise_variables * osn_var,u64 duration)720 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
721 {
722 	if (osn_var->irq.delta_start)
723 		osn_var->irq.delta_start += duration;
724 }
725 
726 #ifndef CONFIG_PREEMPT_RT
727 /*
728  * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
729  *
730  * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
731  * forward to discount the interference.
732  *
733  * See get_int_safe_duration().
734  */
735 static inline void
cond_move_softirq_delta_start(struct osnoise_variables * osn_var,u64 duration)736 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
737 {
738 	if (osn_var->softirq.delta_start)
739 		osn_var->softirq.delta_start += duration;
740 }
741 #else /* CONFIG_PREEMPT_RT */
742 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
743 #endif
744 
745 /*
746  * cond_move_thread_delta_start - Forward the delta_start of a running thread
747  *
748  * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
749  * is pushed forward to discount the interference.
750  *
751  * See get_int_safe_duration().
752  */
753 static inline void
cond_move_thread_delta_start(struct osnoise_variables * osn_var,u64 duration)754 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
755 {
756 	if (osn_var->thread.delta_start)
757 		osn_var->thread.delta_start += duration;
758 }
759 
760 /*
761  * get_int_safe_duration - Get the duration of a window
762  *
763  * The irq, softirq and thread varaibles need to have its duration without
764  * the interference from higher priority interrupts. Instead of keeping a
765  * variable to discount the interrupt interference from these variables, the
766  * starting time of these variables are pushed forward with the interrupt's
767  * duration. In this way, a single variable is used to:
768  *
769  *   - Know if a given window is being measured.
770  *   - Account its duration.
771  *   - Discount the interference.
772  *
773  * To avoid getting inconsistent values, e.g.,:
774  *
775  *	now = time_get()
776  *		--->	interrupt!
777  *			delta_start -= int duration;
778  *		<---
779  *	duration = now - delta_start;
780  *
781  *	result: negative duration if the variable duration before the
782  *	interrupt was smaller than the interrupt execution.
783  *
784  * A counter of interrupts is used. If the counter increased, try
785  * to capture an interference safe duration.
786  */
787 static inline s64
get_int_safe_duration(struct osnoise_variables * osn_var,u64 * delta_start)788 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
789 {
790 	u64 int_counter, now;
791 	s64 duration;
792 
793 	do {
794 		int_counter = local_read(&osn_var->int_counter);
795 		/* synchronize with interrupts */
796 		barrier();
797 
798 		now = time_get();
799 		duration = (now - *delta_start);
800 
801 		/* synchronize with interrupts */
802 		barrier();
803 	} while (int_counter != local_read(&osn_var->int_counter));
804 
805 	/*
806 	 * This is an evidence of race conditions that cause
807 	 * a value to be "discounted" too much.
808 	 */
809 	if (duration < 0)
810 		osnoise_taint("Negative duration!\n");
811 
812 	*delta_start = 0;
813 
814 	return duration;
815 }
816 
817 /*
818  *
819  * set_int_safe_time - Save the current time on *time, aware of interference
820  *
821  * Get the time, taking into consideration a possible interference from
822  * higher priority interrupts.
823  *
824  * See get_int_safe_duration() for an explanation.
825  */
826 static u64
set_int_safe_time(struct osnoise_variables * osn_var,u64 * time)827 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
828 {
829 	u64 int_counter;
830 
831 	do {
832 		int_counter = local_read(&osn_var->int_counter);
833 		/* synchronize with interrupts */
834 		barrier();
835 
836 		*time = time_get();
837 
838 		/* synchronize with interrupts */
839 		barrier();
840 	} while (int_counter != local_read(&osn_var->int_counter));
841 
842 	return int_counter;
843 }
844 
845 #ifdef CONFIG_TIMERLAT_TRACER
846 /*
847  * copy_int_safe_time - Copy *src into *desc aware of interference
848  */
849 static u64
copy_int_safe_time(struct osnoise_variables * osn_var,u64 * dst,u64 * src)850 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
851 {
852 	u64 int_counter;
853 
854 	do {
855 		int_counter = local_read(&osn_var->int_counter);
856 		/* synchronize with interrupts */
857 		barrier();
858 
859 		*dst = *src;
860 
861 		/* synchronize with interrupts */
862 		barrier();
863 	} while (int_counter != local_read(&osn_var->int_counter));
864 
865 	return int_counter;
866 }
867 #endif /* CONFIG_TIMERLAT_TRACER */
868 
869 /*
870  * trace_osnoise_callback - NMI entry/exit callback
871  *
872  * This function is called at the entry and exit NMI code. The bool enter
873  * distinguishes between either case. This function is used to note a NMI
874  * occurrence, compute the noise caused by the NMI, and to remove the noise
875  * it is potentially causing on other interference variables.
876  */
trace_osnoise_callback(bool enter)877 void trace_osnoise_callback(bool enter)
878 {
879 	struct osnoise_variables *osn_var = this_cpu_osn_var();
880 	u64 duration;
881 
882 	if (!osn_var->sampling)
883 		return;
884 
885 	/*
886 	 * Currently trace_clock_local() calls sched_clock() and the
887 	 * generic version is not NMI safe.
888 	 */
889 	if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
890 		if (enter) {
891 			osn_var->nmi.delta_start = time_get();
892 			local_inc(&osn_var->int_counter);
893 		} else {
894 			duration = time_get() - osn_var->nmi.delta_start;
895 
896 			trace_nmi_noise(osn_var->nmi.delta_start, duration);
897 
898 			cond_move_irq_delta_start(osn_var, duration);
899 			cond_move_softirq_delta_start(osn_var, duration);
900 			cond_move_thread_delta_start(osn_var, duration);
901 		}
902 	}
903 
904 	if (enter)
905 		osn_var->nmi.count++;
906 }
907 
908 /*
909  * osnoise_trace_irq_entry - Note the starting of an IRQ
910  *
911  * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
912  * it is safe to use a single variable (ons_var->irq) to save the statistics.
913  * The arrival_time is used to report... the arrival time. The delta_start
914  * is used to compute the duration at the IRQ exit handler. See
915  * cond_move_irq_delta_start().
916  */
osnoise_trace_irq_entry(int id)917 void osnoise_trace_irq_entry(int id)
918 {
919 	struct osnoise_variables *osn_var = this_cpu_osn_var();
920 
921 	if (!osn_var->sampling)
922 		return;
923 	/*
924 	 * This value will be used in the report, but not to compute
925 	 * the execution time, so it is safe to get it unsafe.
926 	 */
927 	osn_var->irq.arrival_time = time_get();
928 	set_int_safe_time(osn_var, &osn_var->irq.delta_start);
929 	osn_var->irq.count++;
930 
931 	local_inc(&osn_var->int_counter);
932 }
933 
934 /*
935  * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
936  *
937  * Computes the duration of the IRQ noise, and trace it. Also discounts the
938  * interference from other sources of noise could be currently being accounted.
939  */
osnoise_trace_irq_exit(int id,const char * desc)940 void osnoise_trace_irq_exit(int id, const char *desc)
941 {
942 	struct osnoise_variables *osn_var = this_cpu_osn_var();
943 	s64 duration;
944 
945 	if (!osn_var->sampling)
946 		return;
947 
948 	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
949 	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
950 	osn_var->irq.arrival_time = 0;
951 	cond_move_softirq_delta_start(osn_var, duration);
952 	cond_move_thread_delta_start(osn_var, duration);
953 }
954 
955 /*
956  * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
957  *
958  * Used to note the starting of an IRQ occurece.
959  */
trace_irqentry_callback(void * data,int irq,struct irqaction * action)960 static void trace_irqentry_callback(void *data, int irq,
961 				    struct irqaction *action)
962 {
963 	osnoise_trace_irq_entry(irq);
964 }
965 
966 /*
967  * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
968  *
969  * Used to note the end of an IRQ occurece.
970  */
trace_irqexit_callback(void * data,int irq,struct irqaction * action,int ret)971 static void trace_irqexit_callback(void *data, int irq,
972 				   struct irqaction *action, int ret)
973 {
974 	osnoise_trace_irq_exit(irq, action->name);
975 }
976 
977 /*
978  * arch specific register function.
979  */
osnoise_arch_register(void)980 int __weak osnoise_arch_register(void)
981 {
982 	return 0;
983 }
984 
985 /*
986  * arch specific unregister function.
987  */
osnoise_arch_unregister(void)988 void __weak osnoise_arch_unregister(void)
989 {
990 	return;
991 }
992 
993 /*
994  * hook_irq_events - Hook IRQ handling events
995  *
996  * This function hooks the IRQ related callbacks to the respective trace
997  * events.
998  */
hook_irq_events(void)999 static int hook_irq_events(void)
1000 {
1001 	int ret;
1002 
1003 	ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1004 	if (ret)
1005 		goto out_err;
1006 
1007 	ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1008 	if (ret)
1009 		goto out_unregister_entry;
1010 
1011 	ret = osnoise_arch_register();
1012 	if (ret)
1013 		goto out_irq_exit;
1014 
1015 	return 0;
1016 
1017 out_irq_exit:
1018 	unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1019 out_unregister_entry:
1020 	unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1021 out_err:
1022 	return -EINVAL;
1023 }
1024 
1025 /*
1026  * unhook_irq_events - Unhook IRQ handling events
1027  *
1028  * This function unhooks the IRQ related callbacks to the respective trace
1029  * events.
1030  */
unhook_irq_events(void)1031 static void unhook_irq_events(void)
1032 {
1033 	osnoise_arch_unregister();
1034 	unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1035 	unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1036 }
1037 
1038 #ifndef CONFIG_PREEMPT_RT
1039 /*
1040  * trace_softirq_entry_callback - Note the starting of a softirq
1041  *
1042  * Save the starting time of a softirq. As softirqs are non-preemptive to
1043  * other softirqs, it is safe to use a single variable (ons_var->softirq)
1044  * to save the statistics. The arrival_time is used to report... the
1045  * arrival time. The delta_start is used to compute the duration at the
1046  * softirq exit handler. See cond_move_softirq_delta_start().
1047  */
trace_softirq_entry_callback(void * data,unsigned int vec_nr)1048 static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1049 {
1050 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1051 
1052 	if (!osn_var->sampling)
1053 		return;
1054 	/*
1055 	 * This value will be used in the report, but not to compute
1056 	 * the execution time, so it is safe to get it unsafe.
1057 	 */
1058 	osn_var->softirq.arrival_time = time_get();
1059 	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1060 	osn_var->softirq.count++;
1061 
1062 	local_inc(&osn_var->int_counter);
1063 }
1064 
1065 /*
1066  * trace_softirq_exit_callback - Note the end of an softirq
1067  *
1068  * Computes the duration of the softirq noise, and trace it. Also discounts the
1069  * interference from other sources of noise could be currently being accounted.
1070  */
trace_softirq_exit_callback(void * data,unsigned int vec_nr)1071 static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1072 {
1073 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1074 	s64 duration;
1075 
1076 	if (!osn_var->sampling)
1077 		return;
1078 
1079 	if (unlikely(timerlat_enabled()))
1080 		if (!timerlat_softirq_exit(osn_var))
1081 			return;
1082 
1083 	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1084 	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1085 	cond_move_thread_delta_start(osn_var, duration);
1086 	osn_var->softirq.arrival_time = 0;
1087 }
1088 
1089 /*
1090  * hook_softirq_events - Hook softirq handling events
1091  *
1092  * This function hooks the softirq related callbacks to the respective trace
1093  * events.
1094  */
hook_softirq_events(void)1095 static int hook_softirq_events(void)
1096 {
1097 	int ret;
1098 
1099 	ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1100 	if (ret)
1101 		goto out_err;
1102 
1103 	ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1104 	if (ret)
1105 		goto out_unreg_entry;
1106 
1107 	return 0;
1108 
1109 out_unreg_entry:
1110 	unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1111 out_err:
1112 	return -EINVAL;
1113 }
1114 
1115 /*
1116  * unhook_softirq_events - Unhook softirq handling events
1117  *
1118  * This function hooks the softirq related callbacks to the respective trace
1119  * events.
1120  */
unhook_softirq_events(void)1121 static void unhook_softirq_events(void)
1122 {
1123 	unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1124 	unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1125 }
1126 #else /* CONFIG_PREEMPT_RT */
1127 /*
1128  * softirq are threads on the PREEMPT_RT mode.
1129  */
hook_softirq_events(void)1130 static int hook_softirq_events(void)
1131 {
1132 	return 0;
1133 }
unhook_softirq_events(void)1134 static void unhook_softirq_events(void)
1135 {
1136 }
1137 #endif
1138 
1139 /*
1140  * thread_entry - Record the starting of a thread noise window
1141  *
1142  * It saves the context switch time for a noisy thread, and increments
1143  * the interference counters.
1144  */
1145 static void
thread_entry(struct osnoise_variables * osn_var,struct task_struct * t)1146 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1147 {
1148 	if (!osn_var->sampling)
1149 		return;
1150 	/*
1151 	 * The arrival time will be used in the report, but not to compute
1152 	 * the execution time, so it is safe to get it unsafe.
1153 	 */
1154 	osn_var->thread.arrival_time = time_get();
1155 
1156 	set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1157 
1158 	osn_var->thread.count++;
1159 	local_inc(&osn_var->int_counter);
1160 }
1161 
1162 /*
1163  * thread_exit - Report the end of a thread noise window
1164  *
1165  * It computes the total noise from a thread, tracing if needed.
1166  */
1167 static void
thread_exit(struct osnoise_variables * osn_var,struct task_struct * t)1168 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1169 {
1170 	s64 duration;
1171 
1172 	if (!osn_var->sampling)
1173 		return;
1174 
1175 	if (unlikely(timerlat_enabled()))
1176 		if (!timerlat_thread_exit(osn_var))
1177 			return;
1178 
1179 	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1180 
1181 	trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1182 
1183 	osn_var->thread.arrival_time = 0;
1184 }
1185 
1186 #ifdef CONFIG_TIMERLAT_TRACER
1187 /*
1188  * osnoise_stop_exception - Stop tracing and the tracer.
1189  */
osnoise_stop_exception(char * msg,int cpu)1190 static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1191 {
1192 	struct osnoise_instance *inst;
1193 	struct trace_array *tr;
1194 
1195 	rcu_read_lock();
1196 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1197 		tr = inst->tr;
1198 		trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1199 				       "stop tracing hit on cpu %d due to exception: %s\n",
1200 				       smp_processor_id(),
1201 				       msg);
1202 
1203 		if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1204 			panic("tracer hit on cpu %d due to exception: %s\n",
1205 			      smp_processor_id(),
1206 			      msg);
1207 
1208 		tracer_tracing_off(tr);
1209 	}
1210 	rcu_read_unlock();
1211 }
1212 
1213 /*
1214  * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1215  *
1216  * his function is hooked to the sched:sched_migrate_task trace event, and monitors
1217  * timerlat user-space thread migration.
1218  */
trace_sched_migrate_callback(void * data,struct task_struct * p,int dest_cpu)1219 static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu)
1220 {
1221 	struct osnoise_variables *osn_var;
1222 	long cpu = task_cpu(p);
1223 
1224 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1225 	if (osn_var->pid == p->pid && dest_cpu != cpu) {
1226 		per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1227 		osnoise_taint("timerlat user-thread migrated\n");
1228 		osnoise_stop_exception("timerlat user-thread migrated", cpu);
1229 	}
1230 }
1231 
1232 static bool monitor_enabled;
1233 
register_migration_monitor(void)1234 static int register_migration_monitor(void)
1235 {
1236 	int ret = 0;
1237 
1238 	/*
1239 	 * Timerlat thread migration check is only required when running timerlat in user-space.
1240 	 * Thus, enable callback only if timerlat is set with no workload.
1241 	 */
1242 	if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options)) {
1243 		if (WARN_ON_ONCE(monitor_enabled))
1244 			return 0;
1245 
1246 		ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1247 		if (!ret)
1248 			monitor_enabled = true;
1249 	}
1250 
1251 	return ret;
1252 }
1253 
unregister_migration_monitor(void)1254 static void unregister_migration_monitor(void)
1255 {
1256 	if (!monitor_enabled)
1257 		return;
1258 
1259 	unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1260 	monitor_enabled = false;
1261 }
1262 #else
register_migration_monitor(void)1263 static int register_migration_monitor(void)
1264 {
1265 	return 0;
1266 }
unregister_migration_monitor(void)1267 static void unregister_migration_monitor(void) {}
1268 #endif
1269 /*
1270  * trace_sched_switch - sched:sched_switch trace event handler
1271  *
1272  * This function is hooked to the sched:sched_switch trace event, and it is
1273  * used to record the beginning and to report the end of a thread noise window.
1274  */
1275 static void
trace_sched_switch_callback(void * data,bool preempt,struct task_struct * p,struct task_struct * n,unsigned int prev_state)1276 trace_sched_switch_callback(void *data, bool preempt,
1277 			    struct task_struct *p,
1278 			    struct task_struct *n,
1279 			    unsigned int prev_state)
1280 {
1281 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1282 	int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1283 
1284 	if ((p->pid != osn_var->pid) || !workload)
1285 		thread_exit(osn_var, p);
1286 
1287 	if ((n->pid != osn_var->pid) || !workload)
1288 		thread_entry(osn_var, n);
1289 }
1290 
1291 /*
1292  * hook_thread_events - Hook the instrumentation for thread noise
1293  *
1294  * Hook the osnoise tracer callbacks to handle the noise from other
1295  * threads on the necessary kernel events.
1296  */
hook_thread_events(void)1297 static int hook_thread_events(void)
1298 {
1299 	int ret;
1300 
1301 	ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1302 	if (ret)
1303 		return -EINVAL;
1304 
1305 	ret = register_migration_monitor();
1306 	if (ret)
1307 		goto out_unreg;
1308 
1309 	return 0;
1310 
1311 out_unreg:
1312 	unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1313 	return -EINVAL;
1314 }
1315 
1316 /*
1317  * unhook_thread_events - unhook the instrumentation for thread noise
1318  *
1319  * Unook the osnoise tracer callbacks to handle the noise from other
1320  * threads on the necessary kernel events.
1321  */
unhook_thread_events(void)1322 static void unhook_thread_events(void)
1323 {
1324 	unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1325 	unregister_migration_monitor();
1326 }
1327 
1328 /*
1329  * save_osn_sample_stats - Save the osnoise_sample statistics
1330  *
1331  * Save the osnoise_sample statistics before the sampling phase. These
1332  * values will be used later to compute the diff betwneen the statistics
1333  * before and after the osnoise sampling.
1334  */
1335 static void
save_osn_sample_stats(struct osnoise_variables * osn_var,struct osnoise_sample * s)1336 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1337 {
1338 	s->nmi_count = osn_var->nmi.count;
1339 	s->irq_count = osn_var->irq.count;
1340 	s->softirq_count = osn_var->softirq.count;
1341 	s->thread_count = osn_var->thread.count;
1342 }
1343 
1344 /*
1345  * diff_osn_sample_stats - Compute the osnoise_sample statistics
1346  *
1347  * After a sample period, compute the difference on the osnoise_sample
1348  * statistics. The struct osnoise_sample *s contains the statistics saved via
1349  * save_osn_sample_stats() before the osnoise sampling.
1350  */
1351 static void
diff_osn_sample_stats(struct osnoise_variables * osn_var,struct osnoise_sample * s)1352 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1353 {
1354 	s->nmi_count = osn_var->nmi.count - s->nmi_count;
1355 	s->irq_count = osn_var->irq.count - s->irq_count;
1356 	s->softirq_count = osn_var->softirq.count - s->softirq_count;
1357 	s->thread_count = osn_var->thread.count - s->thread_count;
1358 }
1359 
1360 /*
1361  * osnoise_stop_tracing - Stop tracing and the tracer.
1362  */
osnoise_stop_tracing(void)1363 static __always_inline void osnoise_stop_tracing(void)
1364 {
1365 	struct osnoise_instance *inst;
1366 	struct trace_array *tr;
1367 
1368 	rcu_read_lock();
1369 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1370 		tr = inst->tr;
1371 		trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1372 				"stop tracing hit on cpu %d\n", smp_processor_id());
1373 
1374 		if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1375 			panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1376 
1377 		tracer_tracing_off(tr);
1378 	}
1379 	rcu_read_unlock();
1380 }
1381 
1382 /*
1383  * osnoise_has_tracing_on - Check if there is at least one instance on
1384  */
osnoise_has_tracing_on(void)1385 static __always_inline int osnoise_has_tracing_on(void)
1386 {
1387 	struct osnoise_instance *inst;
1388 	int trace_is_on = 0;
1389 
1390 	rcu_read_lock();
1391 	list_for_each_entry_rcu(inst, &osnoise_instances, list)
1392 		trace_is_on += tracer_tracing_is_on(inst->tr);
1393 	rcu_read_unlock();
1394 
1395 	return trace_is_on;
1396 }
1397 
1398 /*
1399  * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1400  */
notify_new_max_latency(u64 latency)1401 static void notify_new_max_latency(u64 latency)
1402 {
1403 	struct osnoise_instance *inst;
1404 	struct trace_array *tr;
1405 
1406 	rcu_read_lock();
1407 	list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1408 		tr = inst->tr;
1409 		if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1410 			tr->max_latency = latency;
1411 			latency_fsnotify(tr);
1412 		}
1413 	}
1414 	rcu_read_unlock();
1415 }
1416 
1417 /*
1418  * run_osnoise - Sample the time and look for osnoise
1419  *
1420  * Used to capture the time, looking for potential osnoise latency repeatedly.
1421  * Different from hwlat_detector, it is called with preemption and interrupts
1422  * enabled. This allows irqs, softirqs and threads to run, interfering on the
1423  * osnoise sampling thread, as they would do with a regular thread.
1424  */
run_osnoise(void)1425 static int run_osnoise(void)
1426 {
1427 	bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1428 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1429 	u64 start, sample, last_sample;
1430 	u64 last_int_count, int_count;
1431 	s64 noise = 0, max_noise = 0;
1432 	s64 total, last_total = 0;
1433 	struct osnoise_sample s;
1434 	bool disable_preemption;
1435 	unsigned int threshold;
1436 	u64 runtime, stop_in;
1437 	u64 sum_noise = 0;
1438 	int hw_count = 0;
1439 	int ret = -1;
1440 
1441 	/*
1442 	 * Disabling preemption is only required if IRQs are enabled,
1443 	 * and the options is set on.
1444 	 */
1445 	disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1446 
1447 	/*
1448 	 * Considers the current thread as the workload.
1449 	 */
1450 	osn_var->pid = current->pid;
1451 
1452 	/*
1453 	 * Save the current stats for the diff
1454 	 */
1455 	save_osn_sample_stats(osn_var, &s);
1456 
1457 	/*
1458 	 * if threshold is 0, use the default value of 1 us.
1459 	 */
1460 	threshold = tracing_thresh ? : 1000;
1461 
1462 	/*
1463 	 * Apply PREEMPT and IRQ disabled options.
1464 	 */
1465 	if (disable_irq)
1466 		local_irq_disable();
1467 
1468 	if (disable_preemption)
1469 		preempt_disable();
1470 
1471 	/*
1472 	 * Make sure NMIs see sampling first
1473 	 */
1474 	osn_var->sampling = true;
1475 	barrier();
1476 
1477 	/*
1478 	 * Transform the *_us config to nanoseconds to avoid the
1479 	 * division on the main loop.
1480 	 */
1481 	runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1482 	stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1483 
1484 	/*
1485 	 * Start timestemp
1486 	 */
1487 	start = time_get();
1488 
1489 	/*
1490 	 * "previous" loop.
1491 	 */
1492 	last_int_count = set_int_safe_time(osn_var, &last_sample);
1493 
1494 	do {
1495 		/*
1496 		 * Get sample!
1497 		 */
1498 		int_count = set_int_safe_time(osn_var, &sample);
1499 
1500 		noise = time_sub(sample, last_sample);
1501 
1502 		/*
1503 		 * This shouldn't happen.
1504 		 */
1505 		if (noise < 0) {
1506 			osnoise_taint("negative noise!");
1507 			goto out;
1508 		}
1509 
1510 		/*
1511 		 * Sample runtime.
1512 		 */
1513 		total = time_sub(sample, start);
1514 
1515 		/*
1516 		 * Check for possible overflows.
1517 		 */
1518 		if (total < last_total) {
1519 			osnoise_taint("total overflow!");
1520 			break;
1521 		}
1522 
1523 		last_total = total;
1524 
1525 		if (noise >= threshold) {
1526 			int interference = int_count - last_int_count;
1527 
1528 			if (noise > max_noise)
1529 				max_noise = noise;
1530 
1531 			if (!interference)
1532 				hw_count++;
1533 
1534 			sum_noise += noise;
1535 
1536 			trace_sample_threshold(last_sample, noise, interference);
1537 
1538 			if (osnoise_data.stop_tracing)
1539 				if (noise > stop_in)
1540 					osnoise_stop_tracing();
1541 		}
1542 
1543 		/*
1544 		 * In some cases, notably when running on a nohz_full CPU with
1545 		 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1546 		 * This will eventually cause unwarranted noise as PREEMPT_RCU
1547 		 * will force preemption as the means of ending the current
1548 		 * grace period. We avoid this problem by calling
1549 		 * rcu_momentary_eqs(), which performs a zero duration
1550 		 * EQS allowing PREEMPT_RCU to end the current grace period.
1551 		 * This call shouldn't be wrapped inside an RCU critical
1552 		 * section.
1553 		 *
1554 		 * Note that in non PREEMPT_RCU kernels QSs are handled through
1555 		 * cond_resched()
1556 		 */
1557 		if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1558 			if (!disable_irq)
1559 				local_irq_disable();
1560 
1561 			rcu_momentary_eqs();
1562 
1563 			if (!disable_irq)
1564 				local_irq_enable();
1565 		}
1566 
1567 		/*
1568 		 * For the non-preemptive kernel config: let threads runs, if
1569 		 * they so wish, unless set not do to so.
1570 		 */
1571 		if (!disable_irq && !disable_preemption)
1572 			cond_resched();
1573 
1574 		last_sample = sample;
1575 		last_int_count = int_count;
1576 
1577 	} while (total < runtime && !kthread_should_stop());
1578 
1579 	/*
1580 	 * Finish the above in the view for interrupts.
1581 	 */
1582 	barrier();
1583 
1584 	osn_var->sampling = false;
1585 
1586 	/*
1587 	 * Make sure sampling data is no longer updated.
1588 	 */
1589 	barrier();
1590 
1591 	/*
1592 	 * Return to the preemptive state.
1593 	 */
1594 	if (disable_preemption)
1595 		preempt_enable();
1596 
1597 	if (disable_irq)
1598 		local_irq_enable();
1599 
1600 	/*
1601 	 * Save noise info.
1602 	 */
1603 	s.noise = time_to_us(sum_noise);
1604 	s.runtime = time_to_us(total);
1605 	s.max_sample = time_to_us(max_noise);
1606 	s.hw_count = hw_count;
1607 
1608 	/* Save interference stats info */
1609 	diff_osn_sample_stats(osn_var, &s);
1610 
1611 	trace_osnoise_sample(&s);
1612 
1613 	notify_new_max_latency(max_noise);
1614 
1615 	if (osnoise_data.stop_tracing_total)
1616 		if (s.noise > osnoise_data.stop_tracing_total)
1617 			osnoise_stop_tracing();
1618 
1619 	return 0;
1620 out:
1621 	return ret;
1622 }
1623 
1624 static struct cpumask osnoise_cpumask;
1625 static struct cpumask save_cpumask;
1626 static struct cpumask kthread_cpumask;
1627 
1628 /*
1629  * osnoise_sleep - sleep until the next period
1630  */
osnoise_sleep(bool skip_period)1631 static void osnoise_sleep(bool skip_period)
1632 {
1633 	u64 interval;
1634 	ktime_t wake_time;
1635 
1636 	mutex_lock(&interface_lock);
1637 	if (skip_period)
1638 		interval = osnoise_data.sample_period;
1639 	else
1640 		interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1641 	mutex_unlock(&interface_lock);
1642 
1643 	/*
1644 	 * differently from hwlat_detector, the osnoise tracer can run
1645 	 * without a pause because preemption is on.
1646 	 */
1647 	if (!interval) {
1648 		/* Let synchronize_rcu_tasks() make progress */
1649 		cond_resched_tasks_rcu_qs();
1650 		return;
1651 	}
1652 
1653 	wake_time = ktime_add_us(ktime_get(), interval);
1654 	__set_current_state(TASK_INTERRUPTIBLE);
1655 
1656 	while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1657 		if (kthread_should_stop())
1658 			break;
1659 	}
1660 }
1661 
1662 /*
1663  * osnoise_migration_pending - checks if the task needs to migrate
1664  *
1665  * osnoise/timerlat threads are per-cpu. If there is a pending request to
1666  * migrate the thread away from the current CPU, something bad has happened.
1667  * Play the good citizen and leave.
1668  *
1669  * Returns 0 if it is safe to continue, 1 otherwise.
1670  */
osnoise_migration_pending(void)1671 static inline int osnoise_migration_pending(void)
1672 {
1673 	if (!current->migration_pending)
1674 		return 0;
1675 
1676 	/*
1677 	 * If migration is pending, there is a task waiting for the
1678 	 * tracer to enable migration. The tracer does not allow migration,
1679 	 * thus: taint and leave to unblock the blocked thread.
1680 	 */
1681 	osnoise_taint("migration requested to osnoise threads, leaving.");
1682 
1683 	/*
1684 	 * Unset this thread from the threads managed by the interface.
1685 	 * The tracers are responsible for cleaning their env before
1686 	 * exiting.
1687 	 */
1688 	mutex_lock(&interface_lock);
1689 	this_cpu_osn_var()->kthread = NULL;
1690 	cpumask_clear_cpu(smp_processor_id(), &kthread_cpumask);
1691 	mutex_unlock(&interface_lock);
1692 
1693 	return 1;
1694 }
1695 
1696 /*
1697  * osnoise_main - The osnoise detection kernel thread
1698  *
1699  * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1700  * every period.
1701  */
osnoise_main(void * data)1702 static int osnoise_main(void *data)
1703 {
1704 	unsigned long flags;
1705 
1706 	/*
1707 	 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1708 	 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1709 	 *
1710 	 * To work around this limitation, disable migration and remove the
1711 	 * flag.
1712 	 */
1713 	migrate_disable();
1714 	raw_spin_lock_irqsave(&current->pi_lock, flags);
1715 	current->flags &= ~(PF_NO_SETAFFINITY);
1716 	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1717 
1718 	while (!kthread_should_stop()) {
1719 		if (osnoise_migration_pending())
1720 			break;
1721 
1722 		/* skip a period if tracing is off on all instances */
1723 		if (!osnoise_has_tracing_on()) {
1724 			osnoise_sleep(true);
1725 			continue;
1726 		}
1727 
1728 		run_osnoise();
1729 		osnoise_sleep(false);
1730 	}
1731 
1732 	migrate_enable();
1733 	return 0;
1734 }
1735 
1736 #ifdef CONFIG_TIMERLAT_TRACER
1737 /*
1738  * timerlat_irq - hrtimer handler for timerlat.
1739  */
timerlat_irq(struct hrtimer * timer)1740 static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1741 {
1742 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1743 	struct timerlat_variables *tlat;
1744 	struct timerlat_sample s;
1745 	u64 now;
1746 	u64 diff;
1747 
1748 	/*
1749 	 * I am not sure if the timer was armed for this CPU. So, get
1750 	 * the timerlat struct from the timer itself, not from this
1751 	 * CPU.
1752 	 */
1753 	tlat = container_of(timer, struct timerlat_variables, timer);
1754 
1755 	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1756 
1757 	/*
1758 	 * Enable the osnoise: events for thread an softirq.
1759 	 */
1760 	tlat->tracing_thread = true;
1761 
1762 	osn_var->thread.arrival_time = time_get();
1763 
1764 	/*
1765 	 * A hardirq is running: the timer IRQ. It is for sure preempting
1766 	 * a thread, and potentially preempting a softirq.
1767 	 *
1768 	 * At this point, it is not interesting to know the duration of the
1769 	 * preempted thread (and maybe softirq), but how much time they will
1770 	 * delay the beginning of the execution of the timer thread.
1771 	 *
1772 	 * To get the correct (net) delay added by the softirq, its delta_start
1773 	 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1774 	 * start of the sofitrq will be zeroed, accounting then only the time
1775 	 * after that.
1776 	 *
1777 	 * The thread follows the same principle. However, if a softirq is
1778 	 * running, the thread needs to receive the softirq delta_start. The
1779 	 * reason being is that the softirq will be the last to be unfolded,
1780 	 * resseting the thread delay to zero.
1781 	 *
1782 	 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1783 	 * on RT, moving the thread is enough.
1784 	 */
1785 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1786 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1787 				   &osn_var->softirq.delta_start);
1788 
1789 		copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1790 				    &osn_var->irq.delta_start);
1791 	} else {
1792 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1793 				    &osn_var->irq.delta_start);
1794 	}
1795 
1796 	/*
1797 	 * Compute the current time with the expected time.
1798 	 */
1799 	diff = now - tlat->abs_period;
1800 
1801 	tlat->count++;
1802 	s.seqnum = tlat->count;
1803 	s.timer_latency = diff;
1804 	s.context = IRQ_CONTEXT;
1805 
1806 	trace_timerlat_sample(&s);
1807 
1808 	if (osnoise_data.stop_tracing) {
1809 		if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1810 
1811 			/*
1812 			 * At this point, if stop_tracing is set and <= print_stack,
1813 			 * print_stack is set and would be printed in the thread handler.
1814 			 *
1815 			 * Thus, print the stack trace as it is helpful to define the
1816 			 * root cause of an IRQ latency.
1817 			 */
1818 			if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1819 				timerlat_save_stack(0);
1820 				timerlat_dump_stack(time_to_us(diff));
1821 			}
1822 
1823 			osnoise_stop_tracing();
1824 			notify_new_max_latency(diff);
1825 
1826 			wake_up_process(tlat->kthread);
1827 
1828 			return HRTIMER_NORESTART;
1829 		}
1830 	}
1831 
1832 	wake_up_process(tlat->kthread);
1833 
1834 	if (osnoise_data.print_stack)
1835 		timerlat_save_stack(0);
1836 
1837 	return HRTIMER_NORESTART;
1838 }
1839 
1840 /*
1841  * wait_next_period - Wait for the next period for timerlat
1842  */
wait_next_period(struct timerlat_variables * tlat)1843 static int wait_next_period(struct timerlat_variables *tlat)
1844 {
1845 	ktime_t next_abs_period, now;
1846 	u64 rel_period = osnoise_data.timerlat_period * 1000;
1847 
1848 	now = hrtimer_cb_get_time(&tlat->timer);
1849 	next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1850 
1851 	/*
1852 	 * Save the next abs_period.
1853 	 */
1854 	tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1855 
1856 	/*
1857 	 * If the new abs_period is in the past, skip the activation.
1858 	 */
1859 	while (ktime_compare(now, next_abs_period) > 0) {
1860 		next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1861 		tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1862 	}
1863 
1864 	set_current_state(TASK_INTERRUPTIBLE);
1865 
1866 	hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1867 	schedule();
1868 	return 1;
1869 }
1870 
1871 /*
1872  * timerlat_main- Timerlat main
1873  */
timerlat_main(void * data)1874 static int timerlat_main(void *data)
1875 {
1876 	struct osnoise_variables *osn_var = this_cpu_osn_var();
1877 	struct timerlat_variables *tlat = this_cpu_tmr_var();
1878 	struct timerlat_sample s;
1879 	struct sched_param sp;
1880 	unsigned long flags;
1881 	u64 now, diff;
1882 
1883 	/*
1884 	 * Make the thread RT, that is how cyclictest is usually used.
1885 	 */
1886 	sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1887 	sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1888 
1889 	/*
1890 	 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1891 	 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1892 	 *
1893 	 * To work around this limitation, disable migration and remove the
1894 	 * flag.
1895 	 */
1896 	migrate_disable();
1897 	raw_spin_lock_irqsave(&current->pi_lock, flags);
1898 	current->flags &= ~(PF_NO_SETAFFINITY);
1899 	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1900 
1901 	tlat->count = 0;
1902 	tlat->tracing_thread = false;
1903 
1904 	hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1905 	tlat->timer.function = timerlat_irq;
1906 	tlat->kthread = current;
1907 	osn_var->pid = current->pid;
1908 	/*
1909 	 * Anotate the arrival time.
1910 	 */
1911 	tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1912 
1913 	wait_next_period(tlat);
1914 
1915 	osn_var->sampling = 1;
1916 
1917 	while (!kthread_should_stop()) {
1918 
1919 		now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1920 		diff = now - tlat->abs_period;
1921 
1922 		s.seqnum = tlat->count;
1923 		s.timer_latency = diff;
1924 		s.context = THREAD_CONTEXT;
1925 
1926 		trace_timerlat_sample(&s);
1927 
1928 		notify_new_max_latency(diff);
1929 
1930 		timerlat_dump_stack(time_to_us(diff));
1931 
1932 		tlat->tracing_thread = false;
1933 		if (osnoise_data.stop_tracing_total)
1934 			if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1935 				osnoise_stop_tracing();
1936 
1937 		if (osnoise_migration_pending())
1938 			break;
1939 
1940 		wait_next_period(tlat);
1941 	}
1942 
1943 	hrtimer_cancel(&tlat->timer);
1944 	migrate_enable();
1945 	return 0;
1946 }
1947 #else /* CONFIG_TIMERLAT_TRACER */
timerlat_main(void * data)1948 static int timerlat_main(void *data)
1949 {
1950 	return 0;
1951 }
1952 #endif /* CONFIG_TIMERLAT_TRACER */
1953 
1954 /*
1955  * stop_kthread - stop a workload thread
1956  */
stop_kthread(unsigned int cpu)1957 static void stop_kthread(unsigned int cpu)
1958 {
1959 	struct task_struct *kthread;
1960 
1961 	kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
1962 	if (kthread) {
1963 		if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask) &&
1964 		    !WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options))) {
1965 			kthread_stop(kthread);
1966 		} else if (!WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options))) {
1967 			/*
1968 			 * This is a user thread waiting on the timerlat_fd. We need
1969 			 * to close all users, and the best way to guarantee this is
1970 			 * by killing the thread. NOTE: this is a purpose specific file.
1971 			 */
1972 			kill_pid(kthread->thread_pid, SIGKILL, 1);
1973 			put_task_struct(kthread);
1974 		}
1975 	} else {
1976 		/* if no workload, just return */
1977 		if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1978 			/*
1979 			 * This is set in the osnoise tracer case.
1980 			 */
1981 			per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1982 			barrier();
1983 		}
1984 	}
1985 }
1986 
1987 /*
1988  * stop_per_cpu_kthread - Stop per-cpu threads
1989  *
1990  * Stop the osnoise sampling htread. Use this on unload and at system
1991  * shutdown.
1992  */
stop_per_cpu_kthreads(void)1993 static void stop_per_cpu_kthreads(void)
1994 {
1995 	int cpu;
1996 
1997 	cpus_read_lock();
1998 
1999 	for_each_online_cpu(cpu)
2000 		stop_kthread(cpu);
2001 
2002 	cpus_read_unlock();
2003 }
2004 
2005 /*
2006  * start_kthread - Start a workload tread
2007  */
start_kthread(unsigned int cpu)2008 static int start_kthread(unsigned int cpu)
2009 {
2010 	struct task_struct *kthread;
2011 	void *main = osnoise_main;
2012 	char comm[24];
2013 
2014 	/* Do not start a new thread if it is already running */
2015 	if (per_cpu(per_cpu_osnoise_var, cpu).kthread)
2016 		return 0;
2017 
2018 	if (timerlat_enabled()) {
2019 		snprintf(comm, 24, "timerlat/%d", cpu);
2020 		main = timerlat_main;
2021 	} else {
2022 		/* if no workload, just return */
2023 		if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2024 			per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
2025 			barrier();
2026 			return 0;
2027 		}
2028 		snprintf(comm, 24, "osnoise/%d", cpu);
2029 	}
2030 
2031 	kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2032 
2033 	if (IS_ERR(kthread)) {
2034 		pr_err(BANNER "could not start sampling thread\n");
2035 		stop_per_cpu_kthreads();
2036 		return -ENOMEM;
2037 	}
2038 
2039 	per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2040 	cpumask_set_cpu(cpu, &kthread_cpumask);
2041 
2042 	return 0;
2043 }
2044 
2045 /*
2046  * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2047  *
2048  * This starts the kernel thread that will look for osnoise on many
2049  * cpus.
2050  */
start_per_cpu_kthreads(void)2051 static int start_per_cpu_kthreads(void)
2052 {
2053 	struct cpumask *current_mask = &save_cpumask;
2054 	int retval = 0;
2055 	int cpu;
2056 
2057 	if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2058 		if (timerlat_enabled())
2059 			return 0;
2060 	}
2061 
2062 	cpus_read_lock();
2063 	/*
2064 	 * Run only on online CPUs in which osnoise is allowed to run.
2065 	 */
2066 	cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2067 
2068 	for_each_possible_cpu(cpu) {
2069 		if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) {
2070 			struct task_struct *kthread;
2071 
2072 			kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL);
2073 			if (!WARN_ON(!kthread))
2074 				kthread_stop(kthread);
2075 		}
2076 	}
2077 
2078 	for_each_cpu(cpu, current_mask) {
2079 		retval = start_kthread(cpu);
2080 		if (retval) {
2081 			cpus_read_unlock();
2082 			stop_per_cpu_kthreads();
2083 			return retval;
2084 		}
2085 	}
2086 
2087 	cpus_read_unlock();
2088 
2089 	return retval;
2090 }
2091 
2092 #ifdef CONFIG_HOTPLUG_CPU
osnoise_hotplug_workfn(struct work_struct * dummy)2093 static void osnoise_hotplug_workfn(struct work_struct *dummy)
2094 {
2095 	unsigned int cpu = smp_processor_id();
2096 
2097 	guard(mutex)(&trace_types_lock);
2098 
2099 	if (!osnoise_has_registered_instances())
2100 		return;
2101 
2102 	guard(mutex)(&interface_lock);
2103 	guard(cpus_read_lock)();
2104 
2105 	if (!cpu_online(cpu))
2106 		return;
2107 
2108 	if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2109 		return;
2110 
2111 	start_kthread(cpu);
2112 }
2113 
2114 static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2115 
2116 /*
2117  * osnoise_cpu_init - CPU hotplug online callback function
2118  */
osnoise_cpu_init(unsigned int cpu)2119 static int osnoise_cpu_init(unsigned int cpu)
2120 {
2121 	schedule_work_on(cpu, &osnoise_hotplug_work);
2122 	return 0;
2123 }
2124 
2125 /*
2126  * osnoise_cpu_die - CPU hotplug offline callback function
2127  */
osnoise_cpu_die(unsigned int cpu)2128 static int osnoise_cpu_die(unsigned int cpu)
2129 {
2130 	stop_kthread(cpu);
2131 	return 0;
2132 }
2133 
osnoise_init_hotplug_support(void)2134 static void osnoise_init_hotplug_support(void)
2135 {
2136 	int ret;
2137 
2138 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2139 				osnoise_cpu_init, osnoise_cpu_die);
2140 	if (ret < 0)
2141 		pr_warn(BANNER "Error to init cpu hotplug support\n");
2142 
2143 	return;
2144 }
2145 #else /* CONFIG_HOTPLUG_CPU */
osnoise_init_hotplug_support(void)2146 static void osnoise_init_hotplug_support(void)
2147 {
2148 	return;
2149 }
2150 #endif /* CONFIG_HOTPLUG_CPU */
2151 
2152 /*
2153  * seq file functions for the osnoise/options file.
2154  */
s_options_start(struct seq_file * s,loff_t * pos)2155 static void *s_options_start(struct seq_file *s, loff_t *pos)
2156 {
2157 	int option = *pos;
2158 
2159 	mutex_lock(&interface_lock);
2160 
2161 	if (option >= OSN_MAX)
2162 		return NULL;
2163 
2164 	return pos;
2165 }
2166 
s_options_next(struct seq_file * s,void * v,loff_t * pos)2167 static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
2168 {
2169 	int option = ++(*pos);
2170 
2171 	if (option >= OSN_MAX)
2172 		return NULL;
2173 
2174 	return pos;
2175 }
2176 
s_options_show(struct seq_file * s,void * v)2177 static int s_options_show(struct seq_file *s, void *v)
2178 {
2179 	loff_t *pos = v;
2180 	int option = *pos;
2181 
2182 	if (option == OSN_DEFAULTS) {
2183 		if (osnoise_options == OSN_DEFAULT_OPTIONS)
2184 			seq_printf(s, "%s", osnoise_options_str[option]);
2185 		else
2186 			seq_printf(s, "NO_%s", osnoise_options_str[option]);
2187 		goto out;
2188 	}
2189 
2190 	if (test_bit(option, &osnoise_options))
2191 		seq_printf(s, "%s", osnoise_options_str[option]);
2192 	else
2193 		seq_printf(s, "NO_%s", osnoise_options_str[option]);
2194 
2195 out:
2196 	if (option != OSN_MAX)
2197 		seq_puts(s, " ");
2198 
2199 	return 0;
2200 }
2201 
s_options_stop(struct seq_file * s,void * v)2202 static void s_options_stop(struct seq_file *s, void *v)
2203 {
2204 	seq_puts(s, "\n");
2205 	mutex_unlock(&interface_lock);
2206 }
2207 
2208 static const struct seq_operations osnoise_options_seq_ops = {
2209 	.start		= s_options_start,
2210 	.next		= s_options_next,
2211 	.show		= s_options_show,
2212 	.stop		= s_options_stop
2213 };
2214 
osnoise_options_open(struct inode * inode,struct file * file)2215 static int osnoise_options_open(struct inode *inode, struct file *file)
2216 {
2217 	return seq_open(file, &osnoise_options_seq_ops);
2218 };
2219 
2220 /**
2221  * osnoise_options_write - Write function for "options" entry
2222  * @filp: The active open file structure
2223  * @ubuf: The user buffer that contains the value to write
2224  * @cnt: The maximum number of bytes to write to "file"
2225  * @ppos: The current position in @file
2226  *
2227  * Writing the option name sets the option, writing the "NO_"
2228  * prefix in front of the option name disables it.
2229  *
2230  * Writing "DEFAULTS" resets the option values to the default ones.
2231  */
osnoise_options_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2232 static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2233 				     size_t cnt, loff_t *ppos)
2234 {
2235 	int running, option, enable, retval;
2236 	char buf[256], *option_str;
2237 
2238 	if (cnt >= 256)
2239 		return -EINVAL;
2240 
2241 	if (copy_from_user(buf, ubuf, cnt))
2242 		return -EFAULT;
2243 
2244 	buf[cnt] = 0;
2245 
2246 	if (strncmp(buf, "NO_", 3)) {
2247 		option_str = strstrip(buf);
2248 		enable = true;
2249 	} else {
2250 		option_str = strstrip(&buf[3]);
2251 		enable = false;
2252 	}
2253 
2254 	option = match_string(osnoise_options_str, OSN_MAX, option_str);
2255 	if (option < 0)
2256 		return -EINVAL;
2257 
2258 	/*
2259 	 * trace_types_lock is taken to avoid concurrency on start/stop.
2260 	 */
2261 	mutex_lock(&trace_types_lock);
2262 	running = osnoise_has_registered_instances();
2263 	if (running)
2264 		stop_per_cpu_kthreads();
2265 
2266 	mutex_lock(&interface_lock);
2267 	/*
2268 	 * avoid CPU hotplug operations that might read options.
2269 	 */
2270 	cpus_read_lock();
2271 
2272 	retval = cnt;
2273 
2274 	if (enable) {
2275 		if (option == OSN_DEFAULTS)
2276 			osnoise_options = OSN_DEFAULT_OPTIONS;
2277 		else
2278 			set_bit(option, &osnoise_options);
2279 	} else {
2280 		if (option == OSN_DEFAULTS)
2281 			retval = -EINVAL;
2282 		else
2283 			clear_bit(option, &osnoise_options);
2284 	}
2285 
2286 	cpus_read_unlock();
2287 	mutex_unlock(&interface_lock);
2288 
2289 	if (running)
2290 		start_per_cpu_kthreads();
2291 	mutex_unlock(&trace_types_lock);
2292 
2293 	return retval;
2294 }
2295 
2296 /*
2297  * osnoise_cpus_read - Read function for reading the "cpus" file
2298  * @filp: The active open file structure
2299  * @ubuf: The userspace provided buffer to read value into
2300  * @cnt: The maximum number of bytes to read
2301  * @ppos: The current "file" position
2302  *
2303  * Prints the "cpus" output into the user-provided buffer.
2304  */
2305 static ssize_t
osnoise_cpus_read(struct file * filp,char __user * ubuf,size_t count,loff_t * ppos)2306 osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2307 		  loff_t *ppos)
2308 {
2309 	char *mask_str __free(kfree) = NULL;
2310 	int len;
2311 
2312 	guard(mutex)(&interface_lock);
2313 
2314 	len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2315 	mask_str = kmalloc(len, GFP_KERNEL);
2316 	if (!mask_str)
2317 		return -ENOMEM;
2318 
2319 	len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2320 	if (len >= count)
2321 		return -EINVAL;
2322 
2323 	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2324 
2325 	return count;
2326 }
2327 
2328 /*
2329  * osnoise_cpus_write - Write function for "cpus" entry
2330  * @filp: The active open file structure
2331  * @ubuf: The user buffer that contains the value to write
2332  * @cnt: The maximum number of bytes to write to "file"
2333  * @ppos: The current position in @file
2334  *
2335  * This function provides a write implementation for the "cpus"
2336  * interface to the osnoise trace. By default, it lists all  CPUs,
2337  * in this way, allowing osnoise threads to run on any online CPU
2338  * of the system. It serves to restrict the execution of osnoise to the
2339  * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2340  * Because the user might be interested in tracing what is running on
2341  * other CPUs. For instance, one might run osnoise in one HT CPU
2342  * while observing what is running on the sibling HT CPU.
2343  */
2344 static ssize_t
osnoise_cpus_write(struct file * filp,const char __user * ubuf,size_t count,loff_t * ppos)2345 osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2346 		   loff_t *ppos)
2347 {
2348 	cpumask_var_t osnoise_cpumask_new;
2349 	int running, err;
2350 	char buf[256];
2351 
2352 	if (count >= 256)
2353 		return -EINVAL;
2354 
2355 	if (copy_from_user(buf, ubuf, count))
2356 		return -EFAULT;
2357 
2358 	if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2359 		return -ENOMEM;
2360 
2361 	err = cpulist_parse(buf, osnoise_cpumask_new);
2362 	if (err)
2363 		goto err_free;
2364 
2365 	/*
2366 	 * trace_types_lock is taken to avoid concurrency on start/stop.
2367 	 */
2368 	mutex_lock(&trace_types_lock);
2369 	running = osnoise_has_registered_instances();
2370 	if (running)
2371 		stop_per_cpu_kthreads();
2372 
2373 	mutex_lock(&interface_lock);
2374 	/*
2375 	 * osnoise_cpumask is read by CPU hotplug operations.
2376 	 */
2377 	cpus_read_lock();
2378 
2379 	cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2380 
2381 	cpus_read_unlock();
2382 	mutex_unlock(&interface_lock);
2383 
2384 	if (running)
2385 		start_per_cpu_kthreads();
2386 	mutex_unlock(&trace_types_lock);
2387 
2388 	free_cpumask_var(osnoise_cpumask_new);
2389 	return count;
2390 
2391 err_free:
2392 	free_cpumask_var(osnoise_cpumask_new);
2393 
2394 	return err;
2395 }
2396 
2397 #ifdef CONFIG_TIMERLAT_TRACER
timerlat_fd_open(struct inode * inode,struct file * file)2398 static int timerlat_fd_open(struct inode *inode, struct file *file)
2399 {
2400 	struct osnoise_variables *osn_var;
2401 	struct timerlat_variables *tlat;
2402 	long cpu = (long) inode->i_cdev;
2403 
2404 	mutex_lock(&interface_lock);
2405 
2406 	/*
2407 	 * This file is accessible only if timerlat is enabled, and
2408 	 * NO_OSNOISE_WORKLOAD is set.
2409 	 */
2410 	if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) {
2411 		mutex_unlock(&interface_lock);
2412 		return -EINVAL;
2413 	}
2414 
2415 	migrate_disable();
2416 
2417 	osn_var = this_cpu_osn_var();
2418 
2419 	/*
2420 	 * The osn_var->pid holds the single access to this file.
2421 	 */
2422 	if (osn_var->pid) {
2423 		mutex_unlock(&interface_lock);
2424 		migrate_enable();
2425 		return -EBUSY;
2426 	}
2427 
2428 	/*
2429 	 * timerlat tracer is a per-cpu tracer. Check if the user-space too
2430 	 * is pinned to a single CPU. The tracer laters monitor if the task
2431 	 * migrates and then disables tracer if it does. However, it is
2432 	 * worth doing this basic acceptance test to avoid obviusly wrong
2433 	 * setup.
2434 	 */
2435 	if (current->nr_cpus_allowed > 1 ||  cpu != smp_processor_id()) {
2436 		mutex_unlock(&interface_lock);
2437 		migrate_enable();
2438 		return -EPERM;
2439 	}
2440 
2441 	/*
2442 	 * From now on, it is good to go.
2443 	 */
2444 	file->private_data = inode->i_cdev;
2445 
2446 	get_task_struct(current);
2447 
2448 	osn_var->kthread = current;
2449 	osn_var->pid = current->pid;
2450 
2451 	/*
2452 	 * Setup is done.
2453 	 */
2454 	mutex_unlock(&interface_lock);
2455 
2456 	tlat = this_cpu_tmr_var();
2457 	tlat->count = 0;
2458 
2459 	hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2460 	tlat->timer.function = timerlat_irq;
2461 
2462 	migrate_enable();
2463 	return 0;
2464 };
2465 
2466 /*
2467  * timerlat_fd_read - Read function for "timerlat_fd" file
2468  * @file: The active open file structure
2469  * @ubuf: The userspace provided buffer to read value into
2470  * @cnt: The maximum number of bytes to read
2471  * @ppos: The current "file" position
2472  *
2473  * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2474  */
2475 static ssize_t
timerlat_fd_read(struct file * file,char __user * ubuf,size_t count,loff_t * ppos)2476 timerlat_fd_read(struct file *file, char __user *ubuf, size_t count,
2477 		  loff_t *ppos)
2478 {
2479 	long cpu = (long) file->private_data;
2480 	struct osnoise_variables *osn_var;
2481 	struct timerlat_variables *tlat;
2482 	struct timerlat_sample s;
2483 	s64 diff;
2484 	u64 now;
2485 
2486 	migrate_disable();
2487 
2488 	tlat = this_cpu_tmr_var();
2489 
2490 	/*
2491 	 * While in user-space, the thread is migratable. There is nothing
2492 	 * we can do about it.
2493 	 * So, if the thread is running on another CPU, stop the machinery.
2494 	 */
2495 	if (cpu == smp_processor_id()) {
2496 		if (tlat->uthread_migrate) {
2497 			migrate_enable();
2498 			return -EINVAL;
2499 		}
2500 	} else {
2501 		per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2502 		osnoise_taint("timerlat user thread migrate\n");
2503 		osnoise_stop_tracing();
2504 		migrate_enable();
2505 		return -EINVAL;
2506 	}
2507 
2508 	osn_var = this_cpu_osn_var();
2509 
2510 	/*
2511 	 * The timerlat in user-space runs in a different order:
2512 	 * the read() starts from the execution of the previous occurrence,
2513 	 * sleeping for the next occurrence.
2514 	 *
2515 	 * So, skip if we are entering on read() before the first wakeup
2516 	 * from timerlat IRQ:
2517 	 */
2518 	if (likely(osn_var->sampling)) {
2519 		now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2520 		diff = now - tlat->abs_period;
2521 
2522 		/*
2523 		 * it was not a timer firing, but some other signal?
2524 		 */
2525 		if (diff < 0)
2526 			goto out;
2527 
2528 		s.seqnum = tlat->count;
2529 		s.timer_latency = diff;
2530 		s.context = THREAD_URET;
2531 
2532 		trace_timerlat_sample(&s);
2533 
2534 		notify_new_max_latency(diff);
2535 
2536 		tlat->tracing_thread = false;
2537 		if (osnoise_data.stop_tracing_total)
2538 			if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
2539 				osnoise_stop_tracing();
2540 	} else {
2541 		tlat->tracing_thread = false;
2542 		tlat->kthread = current;
2543 
2544 		/* Annotate now to drift new period */
2545 		tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2546 
2547 		osn_var->sampling = 1;
2548 	}
2549 
2550 	/* wait for the next period */
2551 	wait_next_period(tlat);
2552 
2553 	/* This is the wakeup from this cycle */
2554 	now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2555 	diff = now - tlat->abs_period;
2556 
2557 	/*
2558 	 * it was not a timer firing, but some other signal?
2559 	 */
2560 	if (diff < 0)
2561 		goto out;
2562 
2563 	s.seqnum = tlat->count;
2564 	s.timer_latency = diff;
2565 	s.context = THREAD_CONTEXT;
2566 
2567 	trace_timerlat_sample(&s);
2568 
2569 	if (osnoise_data.stop_tracing_total) {
2570 		if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2571 			timerlat_dump_stack(time_to_us(diff));
2572 			notify_new_max_latency(diff);
2573 			osnoise_stop_tracing();
2574 		}
2575 	}
2576 
2577 out:
2578 	migrate_enable();
2579 	return 0;
2580 }
2581 
timerlat_fd_release(struct inode * inode,struct file * file)2582 static int timerlat_fd_release(struct inode *inode, struct file *file)
2583 {
2584 	struct osnoise_variables *osn_var;
2585 	struct timerlat_variables *tlat_var;
2586 	long cpu = (long) file->private_data;
2587 
2588 	migrate_disable();
2589 	mutex_lock(&interface_lock);
2590 
2591 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2592 	tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2593 
2594 	if (tlat_var->kthread)
2595 		hrtimer_cancel(&tlat_var->timer);
2596 	memset(tlat_var, 0, sizeof(*tlat_var));
2597 
2598 	osn_var->sampling = 0;
2599 	osn_var->pid = 0;
2600 
2601 	/*
2602 	 * We are leaving, not being stopped... see stop_kthread();
2603 	 */
2604 	if (osn_var->kthread) {
2605 		put_task_struct(osn_var->kthread);
2606 		osn_var->kthread = NULL;
2607 	}
2608 
2609 	mutex_unlock(&interface_lock);
2610 	migrate_enable();
2611 	return 0;
2612 }
2613 #endif
2614 
2615 /*
2616  * osnoise/runtime_us: cannot be greater than the period.
2617  */
2618 static struct trace_min_max_param osnoise_runtime = {
2619 	.lock	= &interface_lock,
2620 	.val	= &osnoise_data.sample_runtime,
2621 	.max	= &osnoise_data.sample_period,
2622 	.min	= NULL,
2623 };
2624 
2625 /*
2626  * osnoise/period_us: cannot be smaller than the runtime.
2627  */
2628 static struct trace_min_max_param osnoise_period = {
2629 	.lock	= &interface_lock,
2630 	.val	= &osnoise_data.sample_period,
2631 	.max	= NULL,
2632 	.min	= &osnoise_data.sample_runtime,
2633 };
2634 
2635 /*
2636  * osnoise/stop_tracing_us: no limit.
2637  */
2638 static struct trace_min_max_param osnoise_stop_tracing_in = {
2639 	.lock	= &interface_lock,
2640 	.val	= &osnoise_data.stop_tracing,
2641 	.max	= NULL,
2642 	.min	= NULL,
2643 };
2644 
2645 /*
2646  * osnoise/stop_tracing_total_us: no limit.
2647  */
2648 static struct trace_min_max_param osnoise_stop_tracing_total = {
2649 	.lock	= &interface_lock,
2650 	.val	= &osnoise_data.stop_tracing_total,
2651 	.max	= NULL,
2652 	.min	= NULL,
2653 };
2654 
2655 #ifdef CONFIG_TIMERLAT_TRACER
2656 /*
2657  * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2658  * latency is higher than val.
2659  */
2660 static struct trace_min_max_param osnoise_print_stack = {
2661 	.lock	= &interface_lock,
2662 	.val	= &osnoise_data.print_stack,
2663 	.max	= NULL,
2664 	.min	= NULL,
2665 };
2666 
2667 /*
2668  * osnoise/timerlat_period: min 100 us, max 1 s
2669  */
2670 static u64 timerlat_min_period = 100;
2671 static u64 timerlat_max_period = 1000000;
2672 static struct trace_min_max_param timerlat_period = {
2673 	.lock	= &interface_lock,
2674 	.val	= &osnoise_data.timerlat_period,
2675 	.max	= &timerlat_max_period,
2676 	.min	= &timerlat_min_period,
2677 };
2678 
2679 static const struct file_operations timerlat_fd_fops = {
2680 	.open		= timerlat_fd_open,
2681 	.read		= timerlat_fd_read,
2682 	.release	= timerlat_fd_release,
2683 	.llseek		= generic_file_llseek,
2684 };
2685 #endif
2686 
2687 static const struct file_operations cpus_fops = {
2688 	.open		= tracing_open_generic,
2689 	.read		= osnoise_cpus_read,
2690 	.write		= osnoise_cpus_write,
2691 	.llseek		= generic_file_llseek,
2692 };
2693 
2694 static const struct file_operations osnoise_options_fops = {
2695 	.open		= osnoise_options_open,
2696 	.read		= seq_read,
2697 	.llseek		= seq_lseek,
2698 	.release	= seq_release,
2699 	.write		= osnoise_options_write
2700 };
2701 
2702 #ifdef CONFIG_TIMERLAT_TRACER
2703 #ifdef CONFIG_STACKTRACE
init_timerlat_stack_tracefs(struct dentry * top_dir)2704 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2705 {
2706 	struct dentry *tmp;
2707 
2708 	tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2709 				  &osnoise_print_stack, &trace_min_max_fops);
2710 	if (!tmp)
2711 		return -ENOMEM;
2712 
2713 	return 0;
2714 }
2715 #else /* CONFIG_STACKTRACE */
init_timerlat_stack_tracefs(struct dentry * top_dir)2716 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2717 {
2718 	return 0;
2719 }
2720 #endif /* CONFIG_STACKTRACE */
2721 
osnoise_create_cpu_timerlat_fd(struct dentry * top_dir)2722 static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2723 {
2724 	struct dentry *timerlat_fd;
2725 	struct dentry *per_cpu;
2726 	struct dentry *cpu_dir;
2727 	char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2728 	long cpu;
2729 
2730 	/*
2731 	 * Why not using tracing instance per_cpu/ dir?
2732 	 *
2733 	 * Because osnoise/timerlat have a single workload, having
2734 	 * multiple files like these are wast of memory.
2735 	 */
2736 	per_cpu = tracefs_create_dir("per_cpu", top_dir);
2737 	if (!per_cpu)
2738 		return -ENOMEM;
2739 
2740 	for_each_possible_cpu(cpu) {
2741 		snprintf(cpu_str, 30, "cpu%ld", cpu);
2742 		cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2743 		if (!cpu_dir)
2744 			goto out_clean;
2745 
2746 		timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2747 						cpu_dir, NULL, &timerlat_fd_fops);
2748 		if (!timerlat_fd)
2749 			goto out_clean;
2750 
2751 		/* Record the CPU */
2752 		d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2753 	}
2754 
2755 	return 0;
2756 
2757 out_clean:
2758 	tracefs_remove(per_cpu);
2759 	return -ENOMEM;
2760 }
2761 
2762 /*
2763  * init_timerlat_tracefs - A function to initialize the timerlat interface files
2764  */
init_timerlat_tracefs(struct dentry * top_dir)2765 static int init_timerlat_tracefs(struct dentry *top_dir)
2766 {
2767 	struct dentry *tmp;
2768 	int retval;
2769 
2770 	tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2771 				  &timerlat_period, &trace_min_max_fops);
2772 	if (!tmp)
2773 		return -ENOMEM;
2774 
2775 	retval = osnoise_create_cpu_timerlat_fd(top_dir);
2776 	if (retval)
2777 		return retval;
2778 
2779 	return init_timerlat_stack_tracefs(top_dir);
2780 }
2781 #else /* CONFIG_TIMERLAT_TRACER */
init_timerlat_tracefs(struct dentry * top_dir)2782 static int init_timerlat_tracefs(struct dentry *top_dir)
2783 {
2784 	return 0;
2785 }
2786 #endif /* CONFIG_TIMERLAT_TRACER */
2787 
2788 /*
2789  * init_tracefs - A function to initialize the tracefs interface files
2790  *
2791  * This function creates entries in tracefs for "osnoise" and "timerlat".
2792  * It creates these directories in the tracing directory, and within that
2793  * directory the use can change and view the configs.
2794  */
init_tracefs(void)2795 static int init_tracefs(void)
2796 {
2797 	struct dentry *top_dir;
2798 	struct dentry *tmp;
2799 	int ret;
2800 
2801 	ret = tracing_init_dentry();
2802 	if (ret)
2803 		return -ENOMEM;
2804 
2805 	top_dir = tracefs_create_dir("osnoise", NULL);
2806 	if (!top_dir)
2807 		return 0;
2808 
2809 	tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2810 				  &osnoise_period, &trace_min_max_fops);
2811 	if (!tmp)
2812 		goto err;
2813 
2814 	tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2815 				  &osnoise_runtime, &trace_min_max_fops);
2816 	if (!tmp)
2817 		goto err;
2818 
2819 	tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2820 				  &osnoise_stop_tracing_in, &trace_min_max_fops);
2821 	if (!tmp)
2822 		goto err;
2823 
2824 	tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2825 				  &osnoise_stop_tracing_total, &trace_min_max_fops);
2826 	if (!tmp)
2827 		goto err;
2828 
2829 	tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2830 	if (!tmp)
2831 		goto err;
2832 
2833 	tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2834 				&osnoise_options_fops);
2835 	if (!tmp)
2836 		goto err;
2837 
2838 	ret = init_timerlat_tracefs(top_dir);
2839 	if (ret)
2840 		goto err;
2841 
2842 	return 0;
2843 
2844 err:
2845 	tracefs_remove(top_dir);
2846 	return -ENOMEM;
2847 }
2848 
osnoise_hook_events(void)2849 static int osnoise_hook_events(void)
2850 {
2851 	int retval;
2852 
2853 	/*
2854 	 * Trace is already hooked, we are re-enabling from
2855 	 * a stop_tracing_*.
2856 	 */
2857 	if (trace_osnoise_callback_enabled)
2858 		return 0;
2859 
2860 	retval = hook_irq_events();
2861 	if (retval)
2862 		return -EINVAL;
2863 
2864 	retval = hook_softirq_events();
2865 	if (retval)
2866 		goto out_unhook_irq;
2867 
2868 	retval = hook_thread_events();
2869 	/*
2870 	 * All fine!
2871 	 */
2872 	if (!retval)
2873 		return 0;
2874 
2875 	unhook_softirq_events();
2876 out_unhook_irq:
2877 	unhook_irq_events();
2878 	return -EINVAL;
2879 }
2880 
osnoise_unhook_events(void)2881 static void osnoise_unhook_events(void)
2882 {
2883 	unhook_thread_events();
2884 	unhook_softirq_events();
2885 	unhook_irq_events();
2886 }
2887 
2888 /*
2889  * osnoise_workload_start - start the workload and hook to events
2890  */
osnoise_workload_start(void)2891 static int osnoise_workload_start(void)
2892 {
2893 	int retval;
2894 
2895 	/*
2896 	 * Instances need to be registered after calling workload
2897 	 * start. Hence, if there is already an instance, the
2898 	 * workload was already registered. Otherwise, this
2899 	 * code is on the way to register the first instance,
2900 	 * and the workload will start.
2901 	 */
2902 	if (osnoise_has_registered_instances())
2903 		return 0;
2904 
2905 	osn_var_reset_all();
2906 
2907 	retval = osnoise_hook_events();
2908 	if (retval)
2909 		return retval;
2910 
2911 	/*
2912 	 * Make sure that ftrace_nmi_enter/exit() see reset values
2913 	 * before enabling trace_osnoise_callback_enabled.
2914 	 */
2915 	barrier();
2916 	trace_osnoise_callback_enabled = true;
2917 
2918 	retval = start_per_cpu_kthreads();
2919 	if (retval) {
2920 		trace_osnoise_callback_enabled = false;
2921 		/*
2922 		 * Make sure that ftrace_nmi_enter/exit() see
2923 		 * trace_osnoise_callback_enabled as false before continuing.
2924 		 */
2925 		barrier();
2926 
2927 		osnoise_unhook_events();
2928 		return retval;
2929 	}
2930 
2931 	return 0;
2932 }
2933 
2934 /*
2935  * osnoise_workload_stop - stop the workload and unhook the events
2936  */
osnoise_workload_stop(void)2937 static void osnoise_workload_stop(void)
2938 {
2939 	/*
2940 	 * Instances need to be unregistered before calling
2941 	 * stop. Hence, if there is a registered instance, more
2942 	 * than one instance is running, and the workload will not
2943 	 * yet stop. Otherwise, this code is on the way to disable
2944 	 * the last instance, and the workload can stop.
2945 	 */
2946 	if (osnoise_has_registered_instances())
2947 		return;
2948 
2949 	/*
2950 	 * If callbacks were already disabled in a previous stop
2951 	 * call, there is no need to disable then again.
2952 	 *
2953 	 * For instance, this happens when tracing is stopped via:
2954 	 * echo 0 > tracing_on
2955 	 * echo nop > current_tracer.
2956 	 */
2957 	if (!trace_osnoise_callback_enabled)
2958 		return;
2959 
2960 	trace_osnoise_callback_enabled = false;
2961 	/*
2962 	 * Make sure that ftrace_nmi_enter/exit() see
2963 	 * trace_osnoise_callback_enabled as false before continuing.
2964 	 */
2965 	barrier();
2966 
2967 	stop_per_cpu_kthreads();
2968 
2969 	osnoise_unhook_events();
2970 }
2971 
osnoise_tracer_start(struct trace_array * tr)2972 static void osnoise_tracer_start(struct trace_array *tr)
2973 {
2974 	int retval;
2975 
2976 	/*
2977 	 * If the instance is already registered, there is no need to
2978 	 * register it again.
2979 	 */
2980 	if (osnoise_instance_registered(tr))
2981 		return;
2982 
2983 	retval = osnoise_workload_start();
2984 	if (retval)
2985 		pr_err(BANNER "Error starting osnoise tracer\n");
2986 
2987 	osnoise_register_instance(tr);
2988 }
2989 
osnoise_tracer_stop(struct trace_array * tr)2990 static void osnoise_tracer_stop(struct trace_array *tr)
2991 {
2992 	osnoise_unregister_instance(tr);
2993 	osnoise_workload_stop();
2994 }
2995 
osnoise_tracer_init(struct trace_array * tr)2996 static int osnoise_tracer_init(struct trace_array *tr)
2997 {
2998 	/*
2999 	 * Only allow osnoise tracer if timerlat tracer is not running
3000 	 * already.
3001 	 */
3002 	if (timerlat_enabled())
3003 		return -EBUSY;
3004 
3005 	tr->max_latency = 0;
3006 
3007 	osnoise_tracer_start(tr);
3008 	return 0;
3009 }
3010 
osnoise_tracer_reset(struct trace_array * tr)3011 static void osnoise_tracer_reset(struct trace_array *tr)
3012 {
3013 	osnoise_tracer_stop(tr);
3014 }
3015 
3016 static struct tracer osnoise_tracer __read_mostly = {
3017 	.name		= "osnoise",
3018 	.init		= osnoise_tracer_init,
3019 	.reset		= osnoise_tracer_reset,
3020 	.start		= osnoise_tracer_start,
3021 	.stop		= osnoise_tracer_stop,
3022 	.print_header	= print_osnoise_headers,
3023 	.allow_instances = true,
3024 };
3025 
3026 #ifdef CONFIG_TIMERLAT_TRACER
timerlat_tracer_start(struct trace_array * tr)3027 static void timerlat_tracer_start(struct trace_array *tr)
3028 {
3029 	int retval;
3030 
3031 	/*
3032 	 * If the instance is already registered, there is no need to
3033 	 * register it again.
3034 	 */
3035 	if (osnoise_instance_registered(tr))
3036 		return;
3037 
3038 	retval = osnoise_workload_start();
3039 	if (retval)
3040 		pr_err(BANNER "Error starting timerlat tracer\n");
3041 
3042 	osnoise_register_instance(tr);
3043 
3044 	return;
3045 }
3046 
timerlat_tracer_stop(struct trace_array * tr)3047 static void timerlat_tracer_stop(struct trace_array *tr)
3048 {
3049 	int cpu;
3050 
3051 	osnoise_unregister_instance(tr);
3052 
3053 	/*
3054 	 * Instruct the threads to stop only if this is the last instance.
3055 	 */
3056 	if (!osnoise_has_registered_instances()) {
3057 		for_each_online_cpu(cpu)
3058 			per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
3059 	}
3060 
3061 	osnoise_workload_stop();
3062 }
3063 
timerlat_tracer_init(struct trace_array * tr)3064 static int timerlat_tracer_init(struct trace_array *tr)
3065 {
3066 	/*
3067 	 * Only allow timerlat tracer if osnoise tracer is not running already.
3068 	 */
3069 	if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3070 		return -EBUSY;
3071 
3072 	/*
3073 	 * If this is the first instance, set timerlat_tracer to block
3074 	 * osnoise tracer start.
3075 	 */
3076 	if (!osnoise_has_registered_instances())
3077 		osnoise_data.timerlat_tracer = 1;
3078 
3079 	tr->max_latency = 0;
3080 	timerlat_tracer_start(tr);
3081 
3082 	return 0;
3083 }
3084 
timerlat_tracer_reset(struct trace_array * tr)3085 static void timerlat_tracer_reset(struct trace_array *tr)
3086 {
3087 	timerlat_tracer_stop(tr);
3088 
3089 	/*
3090 	 * If this is the last instance, reset timerlat_tracer allowing
3091 	 * osnoise to be started.
3092 	 */
3093 	if (!osnoise_has_registered_instances())
3094 		osnoise_data.timerlat_tracer = 0;
3095 }
3096 
3097 static struct tracer timerlat_tracer __read_mostly = {
3098 	.name		= "timerlat",
3099 	.init		= timerlat_tracer_init,
3100 	.reset		= timerlat_tracer_reset,
3101 	.start		= timerlat_tracer_start,
3102 	.stop		= timerlat_tracer_stop,
3103 	.print_header	= print_timerlat_headers,
3104 	.allow_instances = true,
3105 };
3106 
init_timerlat_tracer(void)3107 __init static int init_timerlat_tracer(void)
3108 {
3109 	return register_tracer(&timerlat_tracer);
3110 }
3111 #else /* CONFIG_TIMERLAT_TRACER */
init_timerlat_tracer(void)3112 __init static int init_timerlat_tracer(void)
3113 {
3114 	return 0;
3115 }
3116 #endif /* CONFIG_TIMERLAT_TRACER */
3117 
init_osnoise_tracer(void)3118 __init static int init_osnoise_tracer(void)
3119 {
3120 	int ret;
3121 
3122 	mutex_init(&interface_lock);
3123 
3124 	cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3125 
3126 	ret = register_tracer(&osnoise_tracer);
3127 	if (ret) {
3128 		pr_err(BANNER "Error registering osnoise!\n");
3129 		return ret;
3130 	}
3131 
3132 	ret = init_timerlat_tracer();
3133 	if (ret) {
3134 		pr_err(BANNER "Error registering timerlat!\n");
3135 		return ret;
3136 	}
3137 
3138 	osnoise_init_hotplug_support();
3139 
3140 	INIT_LIST_HEAD_RCU(&osnoise_instances);
3141 
3142 	init_tracefs();
3143 
3144 	return 0;
3145 }
3146 late_initcall(init_osnoise_tracer);
3147