1 // SPDX-License-Identifier: GPL-2.0
3  * trace_hwlat.c - A simple Hardware Latency detector.
20  * Although certain hardware-inducing latencies are necessary (for example,
22  * and remote management) they can wreak havoc upon any OS-level performance
23  * guarantees toward low-latency, especially when the OS is not even made
27  * sampling the built-in CPU timer, looking for discontiguous readings.
31  *          environment requiring any kind of low-latency performance
34  * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
35  * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
67 static char *thread_mode_str[] = { "none", "round-robin", "per-cpu" };
99 	int			count;		/* # of iterations over thresh */
109 	u64	sample_window;		/* total sampling window (on+off) */
110 	u64	sample_width;		/* active sampling portion of window */
133 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_hwlat_sample()
142 	entry->seqnum			= sample->seqnum;  in trace_hwlat_sample()
143 	entry->duration			= sample->duration;  in trace_hwlat_sample()
144 	entry->outer_duration		= sample->outer_duration;  in trace_hwlat_sample()
145 	entry->timestamp		= sample->timestamp;  in trace_hwlat_sample()
146 	entry->nmi_total_ts		= sample->nmi_total_ts;  in trace_hwlat_sample()
147 	entry->nmi_count		= sample->nmi_count;  in trace_hwlat_sample()
148 	entry->count			= sample->count;  in trace_hwlat_sample()
157 #define time_sub(a, b)	((a) - (b))
165 	if (!kdata->kthread)  in trace_hwlat_callback()
174 			kdata->nmi_ts_start = time_get();  in trace_hwlat_callback()
176 			kdata->nmi_total_ts += time_get() - kdata->nmi_ts_start;  in trace_hwlat_callback()
180 		kdata->nmi_count++;  in trace_hwlat_callback()
184  * hwlat_err - report a hwlat error.
189 	trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg);	\
193  * get_sample - sample the CPU TSC and look for likely hardware latencies
196  * hardware-induced latency. Called with interrupts disabled and with
209 	int ret = -1;  in get_sample()
214 	kdata->nmi_total_ts = 0;  in get_sample()
215 	kdata->nmi_count = 0;  in get_sample()
285 		if (kdata->nmi_total_ts)  in get_sample()
286 			do_div(kdata->nmi_total_ts, NSEC_PER_USEC);  in get_sample()
292 		s.nmi_total_ts = kdata->nmi_total_ts;  in get_sample()
293 		s.nmi_count = kdata->nmi_count;  in get_sample()
300 		if (latency > tr->max_latency) {  in get_sample()
301 			tr->max_latency = latency;  in get_sample()
323 	if (!cpumask_equal(current_mask, current->cpus_ptr))  in move_to_next_cpu()
327 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in move_to_next_cpu()
345 	pr_info(BANNER "cpumask changed while in round-robin mode, switching to mode none\n");  in move_to_next_cpu()
349  * kthread_fn - The CPU time sampling/hardware latency detection kernel thread
372 		interval = hwlat_data.sample_window - hwlat_data.sample_width;  in kthread_fn()
389  * stop_stop_kthread - Inform the hardware latency sampling/detector kthread to stop
391  * This kicks the running hardware latency sampling/detector kernel thread and
392  * tells it to stop sampling now. Use this on unload and at system shutdown.
400 	kthread = kdata->kthread;  in stop_single_kthread()
406 	kdata->kthread = NULL;  in stop_single_kthread()
414  * start_single_kthread - Kick off the hardware latency sampling/detector kthread
427 	if (kdata->kthread)  in start_single_kthread()
432 		pr_err(BANNER "could not start sampling thread\n");  in start_single_kthread()
434 		return -ENOMEM;  in start_single_kthread()
438 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_single_kthread()
449 	kdata->kthread = kthread;  in start_single_kthread()
458  * stop_cpu_kthread - Stop a hwlat cpu kthread
471  * stop_per_cpu_kthreads - Inform the hardware latency sampling/detector kthread to stop
473  * This kicks the running hardware latency sampling/detector kernel threads and
474  * tells it to stop sampling now. Use this on unload and at system shutdown.
487  * start_cpu_kthread - Start a hwlat cpu kthread
499 		pr_err(BANNER "could not start sampling thread\n");  in start_cpu_kthread()
500 		return -ENOMEM;  in start_cpu_kthread()
523 	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask))  in hwlat_hotplug_workfn()
537  * hwlat_cpu_init - CPU hotplug online callback function
546  * hwlat_cpu_die - CPU hotplug offline callback function
573  * start_per_cpu_kthreads - Kick off the hardware latency sampling/detector kthreads
589 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_per_cpu_kthreads()
638 	if (mode < MODE_MAX - 1) /* if mode is any but last */  in s_mode_show()
666  * hwlat_mode_write - Write function for "mode" entry
676  * the "round-robin" one, in which a single hwlatd thread runs, migrating
677  * among the allowed CPUs in a round-robin fashion. The "per-cpu" mode
689 		return -EINVAL;  in hwlat_mode_write()
692 		return -EFAULT;  in hwlat_mode_write()
698 	ret = -EINVAL;  in hwlat_mode_write()
762  * init_tracefs - A function to initialize the tracefs interface files
776 		return -ENOMEM;  in init_tracefs()
780 		return -ENOMEM;  in init_tracefs()
807 	return -ENOMEM;  in init_tracefs()
834 		return -EBUSY;  in hwlat_tracer_init()
839 	tr->max_latency = 0;  in hwlat_tracer_init()