306 	struct osnoise_variables *osn_var;  in osn_var_reset()  local
314 		osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in osn_var_reset()
315 		memset(osn_var, 0, sizeof(*osn_var));  in osn_var_reset()
367 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
375 		osn_var->softirq.arrival_time = 0;  in timerlat_softirq_exit()
376 		osn_var->softirq.delta_start = 0;  in timerlat_softirq_exit()
382 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
390 		osn_var->thread.delta_start = 0;  in timerlat_thread_exit()
391 		osn_var->thread.arrival_time = 0;  in timerlat_thread_exit()
402 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
406 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
714 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_irq_delta_start()  argument
716 	if (osn_var->irq.delta_start)  in cond_move_irq_delta_start()
717 		osn_var->irq.delta_start += duration;  in cond_move_irq_delta_start()
730 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_softirq_delta_start()  argument
732 	if (osn_var->softirq.delta_start)  in cond_move_softirq_delta_start()
733 		osn_var->softirq.delta_start += duration;  in cond_move_softirq_delta_start()
736 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)  argument
748 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_thread_delta_start()  argument
750 	if (osn_var->thread.delta_start)  in cond_move_thread_delta_start()
751 		osn_var->thread.delta_start += duration;  in cond_move_thread_delta_start()
782 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)  in get_int_safe_duration()  argument
788 		int_counter = local_read(&osn_var->int_counter);  in get_int_safe_duration()
797 	} while (int_counter != local_read(&osn_var->int_counter));  in get_int_safe_duration()
821 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)  in set_int_safe_time()  argument
826 		int_counter = local_read(&osn_var->int_counter);  in set_int_safe_time()
834 	} while (int_counter != local_read(&osn_var->int_counter));  in set_int_safe_time()
844 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)  in copy_int_safe_time()  argument
849 		int_counter = local_read(&osn_var->int_counter);  in copy_int_safe_time()
857 	} while (int_counter != local_read(&osn_var->int_counter));  in copy_int_safe_time()
873 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_osnoise_callback()  local
876 	if (!osn_var->sampling)  in trace_osnoise_callback()
885 			osn_var->nmi.delta_start = time_get();  in trace_osnoise_callback()
886 			local_inc(&osn_var->int_counter);  in trace_osnoise_callback()
888 			duration = time_get() - osn_var->nmi.delta_start;  in trace_osnoise_callback()
890 			trace_nmi_noise(osn_var->nmi.delta_start, duration);  in trace_osnoise_callback()
892 			cond_move_irq_delta_start(osn_var, duration);  in trace_osnoise_callback()
893 			cond_move_softirq_delta_start(osn_var, duration);  in trace_osnoise_callback()
894 			cond_move_thread_delta_start(osn_var, duration);  in trace_osnoise_callback()
899 		osn_var->nmi.count++;  in trace_osnoise_callback()
913 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_entry()  local
915 	if (!osn_var->sampling)  in osnoise_trace_irq_entry()
921 	osn_var->irq.arrival_time = time_get();  in osnoise_trace_irq_entry()
922 	set_int_safe_time(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_entry()
923 	osn_var->irq.count++;  in osnoise_trace_irq_entry()
925 	local_inc(&osn_var->int_counter);  in osnoise_trace_irq_entry()
936 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_exit()  local
939 	if (!osn_var->sampling)  in osnoise_trace_irq_exit()
942 	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_exit()
943 	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);  in osnoise_trace_irq_exit()
944 	osn_var->irq.arrival_time = 0;  in osnoise_trace_irq_exit()
945 	cond_move_softirq_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
946 	cond_move_thread_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
1044 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_entry_callback()  local
1046 	if (!osn_var->sampling)  in trace_softirq_entry_callback()
1052 	osn_var->softirq.arrival_time = time_get();  in trace_softirq_entry_callback()
1053 	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_entry_callback()
1054 	osn_var->softirq.count++;  in trace_softirq_entry_callback()
1056 	local_inc(&osn_var->int_counter);  in trace_softirq_entry_callback()
1067 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_exit_callback()  local
1070 	if (!osn_var->sampling)  in trace_softirq_exit_callback()
1074 		if (!timerlat_softirq_exit(osn_var))  in trace_softirq_exit_callback()
1077 	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_exit_callback()
1078 	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);  in trace_softirq_exit_callback()
1079 	cond_move_thread_delta_start(osn_var, duration);  in trace_softirq_exit_callback()
1080 	osn_var->softirq.arrival_time = 0;  in trace_softirq_exit_callback()
1140 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_entry()  argument
1142 	if (!osn_var->sampling)  in thread_entry()
1148 	osn_var->thread.arrival_time = time_get();  in thread_entry()
1150 	set_int_safe_time(osn_var, &osn_var->thread.delta_start);  in thread_entry()
1152 	osn_var->thread.count++;  in thread_entry()
1153 	local_inc(&osn_var->int_counter);  in thread_entry()
1162 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_exit()  argument
1166 	if (!osn_var->sampling)  in thread_exit()
1170 		if (!timerlat_thread_exit(osn_var))  in thread_exit()
1173 	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);  in thread_exit()
1175 	trace_thread_noise(t, osn_var->thread.arrival_time, duration);  in thread_exit()
1177 	osn_var->thread.arrival_time = 0;  in thread_exit()
1215 	struct osnoise_variables *osn_var;  in trace_sched_migrate_callback()  local
1218 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in trace_sched_migrate_callback()
1219 	if (osn_var->pid == p->pid && dest_cpu != cpu) {  in trace_sched_migrate_callback()
1275 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_sched_switch_callback()  local
1278 	if ((p->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1279 		thread_exit(osn_var, p);  in trace_sched_switch_callback()
1281 	if ((n->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1282 		thread_entry(osn_var, n);  in trace_sched_switch_callback()
1330 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in save_osn_sample_stats()  argument
1332 	s->nmi_count = osn_var->nmi.count;  in save_osn_sample_stats()
1333 	s->irq_count = osn_var->irq.count;  in save_osn_sample_stats()
1334 	s->softirq_count = osn_var->softirq.count;  in save_osn_sample_stats()
1335 	s->thread_count = osn_var->thread.count;  in save_osn_sample_stats()
1346 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in diff_osn_sample_stats()  argument
1348 	s->nmi_count = osn_var->nmi.count - s->nmi_count;  in diff_osn_sample_stats()
1349 	s->irq_count = osn_var->irq.count - s->irq_count;  in diff_osn_sample_stats()
1350 	s->softirq_count = osn_var->softirq.count - s->softirq_count;  in diff_osn_sample_stats()
1351 	s->thread_count = osn_var->thread.count - s->thread_count;  in diff_osn_sample_stats()
1422 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in run_osnoise()  local
1444 	osn_var->pid = current->pid;  in run_osnoise()
1449 	save_osn_sample_stats(osn_var, &s);  in run_osnoise()
1468 	osn_var->sampling = true;  in run_osnoise()
1486 	last_int_count = set_int_safe_time(osn_var, &last_sample);  in run_osnoise()
1492 		int_count = set_int_safe_time(osn_var, &sample);  in run_osnoise()
1576 	osn_var->sampling = false;  in run_osnoise()
1601 	diff_osn_sample_stats(osn_var, &s);  in run_osnoise()
1734 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_irq()  local
1754 	osn_var->thread.arrival_time = time_get();  in timerlat_irq()
1777 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {  in timerlat_irq()
1778 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1779 				   &osn_var->softirq.delta_start);  in timerlat_irq()
1781 		copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,  in timerlat_irq()
1782 				    &osn_var->irq.delta_start);  in timerlat_irq()
1784 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1785 				    &osn_var->irq.delta_start);  in timerlat_irq()
1888 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_main()  local
1918 	osn_var->pid = current->pid;  in timerlat_main()
1926 	osn_var->sampling = 1;  in timerlat_main()
2411 	struct osnoise_variables *osn_var;  in timerlat_fd_open()  local
2428 	osn_var = this_cpu_osn_var();  in timerlat_fd_open()
2433 	if (osn_var->pid) {  in timerlat_fd_open()
2459 	osn_var->kthread = current;  in timerlat_fd_open()
2460 	osn_var->pid = current->pid;  in timerlat_fd_open()
2490 	struct osnoise_variables *osn_var;  in timerlat_fd_read()  local
2518 	osn_var = this_cpu_osn_var();  in timerlat_fd_read()
2528 	if (likely(osn_var->sampling)) {  in timerlat_fd_read()
2557 		osn_var->sampling = 1;  in timerlat_fd_read()
2594 	struct osnoise_variables *osn_var;  in timerlat_fd_release()  local
2601 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in timerlat_fd_release()
2608 	osn_var->sampling = 0;  in timerlat_fd_release()
2609 	osn_var->pid = 0;  in timerlat_fd_release()
2614 	if (osn_var->kthread) {  in timerlat_fd_release()
2615 		put_task_struct(osn_var->kthread);  in timerlat_fd_release()
2616 		osn_var->kthread = NULL;  in timerlat_fd_release()