1 // SPDX-License-Identifier: GPL-2.0+
42 #define SCALE_FLAG "-ref-scale: "
78 // Number of seconds to extend warm-up and cool-down for multiple guest OSes
80 	      "Number of seconds to extend warm-up/cool-down for multiple guest OSes.");
83 	      "Holdoff time before test start (s)");
88 // Number of readers, with -1 defaulting to about 75% of the CPUs.
89 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
93 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
155 	for (i = nloops; i >= 0; i--) {  in ref_rcu_read_section()
165 	for (i = nloops; i >= 0; i--) {  in ref_rcu_delay_section()
193 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_read_section()
204 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_delay_section()
223 	for (i = nloops; i >= 0; i--) {  in srcu_lite_ref_scale_read_section()
234 	for (i = nloops; i >= 0; i--) {  in srcu_lite_ref_scale_delay_section()
245 	.name		= "srcu-lite"
256 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_read_section()
264 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_delay_section()
272 	.name		= "rcu-tasks"
290 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_read_section()
300 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_delay_section()
311 	.name		= "rcu-trace"
329 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_section()
339 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_delay_section()
366 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_section()
376 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_delay_section()
403 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_section()
413 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_delay_section()
435 	for (i = nloops; i >= 0; i--) {  in ref_lock_section()
447 	for (i = nloops; i >= 0; i--) {  in ref_lock_delay_section()
461 // Definitions for global irq-save spinlock
469 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_section()
482 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_delay_section()
493 	.name		= "lock-irq"
496 // Definitions acquire-release.
505 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_section()
518 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_delay_section()
540 	for (i = nloops; i >= 0; i--)  in ref_clock_section()
552 	for (i = nloops; i >= 0; i--) {  in ref_clock_delay_section()
572 	for (i = nloops; i >= 0; i--)  in ref_jiffies_section()
584 	for (i = nloops; i >= 0; i--) {  in ref_jiffies_delay_section()
619 // Conditionally acquire an explicit in-structure reference count.
622 	return atomic_inc_not_zero(&rtsp->rts_refctr);  in typesafe_ref_acquire()
625 // Unconditionally release an explicit in-structure reference count.
628 	if (!atomic_dec_return(&rtsp->rts_refctr)) {  in typesafe_ref_release()
629 		WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_ref_release()
635 // Unconditionally acquire an explicit in-structure spinlock.
638 	spin_lock(&rtsp->rts_lock);  in typesafe_lock_acquire()
642 // Unconditionally release an explicit in-structure spinlock.
645 	spin_unlock(&rtsp->rts_lock);  in typesafe_lock_release()
649 // Unconditionally acquire an explicit in-structure sequence lock.
652 	*start = read_seqbegin(&rtsp->rts_seqlock);  in typesafe_seqlock_acquire()
656 // Conditionally release an explicit in-structure sequence lock.  Return
660 	return !read_seqretry(&rtsp->rts_seqlock, start);  in typesafe_seqlock_release()
663 // Do a read-side critical section with the specified delay in
675 	for (i = nloops; i >= 0; i--) {  in typesafe_delay_section()
682 		a = READ_ONCE(rtsp->a);  in typesafe_delay_section()
687 		if (a != READ_ONCE(rtsp->a)) {  in typesafe_delay_section()
693 		b = READ_ONCE(rtsp->a);  in typesafe_delay_section()
694 		// Remember, seqlock read-side release can fail.  in typesafe_delay_section()
699 		WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);  in typesafe_delay_section()
700 		b = rtsp->b;  in typesafe_delay_section()
706 // Because the acquisition and release methods are expensive, there
723 	atomic_set(&rtsp->rts_refctr, 1);  in typesafe_alloc_one()
724 	WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_alloc_one()
725 	WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);  in typesafe_alloc_one()
729 // Slab-allocator constructor for refscale_typesafe structures created
735 	spin_lock_init(&rtsp->rts_lock);  in refscale_typesafe_ctor()
736 	seqlock_init(&rtsp->rts_seqlock);  in refscale_typesafe_ctor()
738 	rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));  in refscale_typesafe_ctor()
758 		si = -si * nr_cpu_ids;  in typesafe_init()
832 		cur_ops->readsection(loops);  in rcu_scale_one_reader()
834 		cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);  in rcu_scale_one_reader()
841 	unsigned long jdone = jiffies + (guest_os_delay > 0 ? guest_os_delay * HZ : -1);  in rcu_scale_warm_cool()
849 // Reader kthread.  Repeatedly does empty RCU read-side
850 // critical section, minimizing update-side interference.
870 	wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||  in ref_scale_reader()
879 	WRITE_ONCE(rt->start_reader, 0);  in ref_scale_reader()
887 	// To reduce noise, do an initial cache-warming invocation, check  in ref_scale_reader()
900 	duration = ktime_get_mono_fast_ns() - start;  in ref_scale_reader()
903 	rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;  in ref_scale_reader()
904 	// To reduce runtime-skew noise, do maintain-load invocations until  in ref_scale_reader()
931 		rt->last_duration_ns = 0;  in reset_readers()
949 	seq_buf_printf(&s, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",  in process_durations()
963 		seq_buf_printf(&s, "%d: %llu\t", i, rt->last_duration_ns);  in process_durations()
965 		sum += rt->last_duration_ns;  in process_durations()
1041 	pr_alert("Runs\tTime(ns)\n");  in main_func()
1079 …"--- %s:  verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nrea…  in ref_scale_print_module_parms()
1106 	// Do scale-type-specific cleanup operations.  in ref_scale_cleanup()
1107 	if (cur_ops->cleanup != NULL)  in ref_scale_cleanup()
1108 		cur_ops->cleanup();  in ref_scale_cleanup()
1123 	return -EINVAL;  in ref_scale_shutdown()
1139 		return -EBUSY;  in ref_scale_init()
1143 		if (strcmp(scale_type, cur_ops->name) == 0)  in ref_scale_init()
1147 		pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);  in ref_scale_init()
1148 		pr_alert("rcu-scale types:");  in ref_scale_init()
1150 			pr_cont(" %s", scale_ops[i]->name);  in ref_scale_init()
1152 		firsterr = -EINVAL;  in ref_scale_init()
1156 	if (cur_ops->init)  in ref_scale_init()
1157 		if (!cur_ops->init()) {  in ref_scale_init()
1158 			firsterr = -EUCLEAN;  in ref_scale_init()
1187 		firsterr = -ENOMEM;  in ref_scale_init()