1 // SPDX-License-Identifier: GPL-2.0+
3  * Read-Copy Update module-based scalability-test facility
34 #include <linux/delay.h>
46 MODULE_DESCRIPTION("Read-Copy Update module-based scalability-test facility");
50 #define SCALE_FLAG "-scale:"
68  *	value specified by nr_cpus for an update-only test.
72  *	value specified by nr_cpus for a read-only test.
78  * pessimistic due to the per-critical-section interrupt disabling,
79  * test-end checks, and the pair of calls through pointers.
91 torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
92 torture_param(int, minruntime, 0, "Minimum run time (s)");
93 torture_param(int, nreaders, -1, "Number of RCU reader threads");
94 torture_param(int, nwriters, -1, "Number of RCU updater threads");
106 MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)");
108 // Structure definitions for custom fixed-per-task allocator.
303  * Definitions for RCU-tasks scalability testing.
347  * Definitions for RCU-tasks-rude scalability testing.
375 	.name		= "tasks-rude"
389  * Definitions for RCU-tasks-trace scalability testing.
421 	.name		= "tasks-tracing"
434 	if (!cur_ops->gp_diff)  in rcuscale_seq_diff()
435 		return new - old;  in rcuscale_seq_diff()
436 	return cur_ops->gp_diff(new, old);  in rcuscale_seq_diff()
452  * RCU scalability reader kthread.  Repeatedly does empty RCU read-side
453  * critical section, minimizing update-side interference.  However, the
455  * to serve as a test load for update-side scalability testing.
471 		idx = cur_ops->readlock();  in rcu_scale_reader()
472 		cur_ops->readunlock(idx);  in rcu_scale_reader()
493 	if (llist_empty(&wflp->ws_lhp)) {  in rcu_scale_alloc()
494 		// ->ws_lhp is private to its rcu_scale_writer task.  in rcu_scale_alloc()
495 		wmbp = container_of(llist_del_all(&wflp->ws_lhg), struct writer_mblock, wmb_node);  in rcu_scale_alloc()
496 		wflp->ws_lhp.first = &wmbp->wmb_node;  in rcu_scale_alloc()
498 	llnp = llist_del_first(&wflp->ws_lhp);  in rcu_scale_alloc()
513 	wflp = wmbp->wmb_wfl;  in rcu_scale_free()
514 	llist_add(&wmbp->wmb_node, &wflp->ws_lhg);  in rcu_scale_free()
523 	struct writer_freelist *wflp = wmbp->wmb_wfl;  in rcu_scale_async_cb()
525 	atomic_dec(&wflp->ws_inflight);  in rcu_scale_async_cb()
551 	current->flags |= PF_NO_SETAFFINITY;  in rcu_scale_writer()
570 				cur_ops->exp_completed() / 2;  in rcu_scale_writer()
572 			b_rcu_gp_test_started = cur_ops->get_gp_seq();  in rcu_scale_writer()
586 		if (gp_async && !WARN_ON_ONCE(!cur_ops->async)) {  in rcu_scale_writer()
589 			if (wmbp && atomic_read(&wflp->ws_inflight) < gp_async_max) {  in rcu_scale_writer()
590 				atomic_inc(&wflp->ws_inflight);  in rcu_scale_writer()
591 				cur_ops->async(&wmbp->wmb_rh, rcu_scale_async_cb);  in rcu_scale_writer()
595 				cur_ops->gp_barrier();  in rcu_scale_writer()
601 			cur_ops->exp_sync();  in rcu_scale_writer()
604 			cur_ops->sync();  in rcu_scale_writer()
608 		*wdp = t - *wdp;  in rcu_scale_writer()
627 						cur_ops->exp_completed() / 2;  in rcu_scale_writer()
630 						cur_ops->get_gp_seq();  in rcu_scale_writer()
652 				if (cur_ops->stats)  in rcu_scale_writer()
653 					cur_ops->stats();  in rcu_scale_writer()
657 			pr_info("%s: Writer %ld self-report: started %d done %d/%d->%d i %d jdone %lu.\n",  in rcu_scale_writer()
658 …me, started, done, writer_done[me], atomic_read(&n_rcu_scale_writer_finished), i, jiffies - jdone);  in rcu_scale_writer()
661 		if (gp_succeeded && started && !alldone && i < MAX_MEAS - 1)  in rcu_scale_writer()
665 	if (gp_async && cur_ops->async) {  in rcu_scale_writer()
667 		cur_ops->gp_barrier();  in rcu_scale_writer()
678 …"--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d wr…  in rcu_scale_print_module_parms()
683  * Return the number if non-negative.  If -1, the number of CPUs.
684  * If less than -1, that much less than the number of CPUs, but
703  * of iterations and measure total time and number of GP for all iterations to complete.
706 torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu().");
709 torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale te…
710 torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale te…
724 /* Used if doing RCU-kfree'ing via call_rcu(). */
752 			b_rcu_gp_test_started = cur_ops->exp_completed() / 2;  in kfree_scale_thread()
754 			b_rcu_gp_test_started = cur_ops->get_gp_seq();  in kfree_scale_thread()
767 				return -ENOMEM;  in kfree_scale_thread()
770 				call_rcu(&(alloc_ptr->rh), kfree_call_rcu);  in kfree_scale_thread()
792 			b_rcu_gp_test_finished = cur_ops->exp_completed() / 2;  in kfree_scale_thread()
794 			b_rcu_gp_test_finished = cur_ops->get_gp_seq();  in kfree_scale_thread()
796 …pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %…  in kfree_scale_thread()
797 		       (unsigned long long)(end_time - start_time), kfree_loops,  in kfree_scale_thread()
799 		       PAGES_TO_MB(mem_begin - mem_during));  in kfree_scale_thread()
843 	return -EINVAL;  in kfree_scale_shutdown()
846 // Used if doing RCU-kfree'ing via call_rcu().
865 …"--- kfree_rcu_test: kfree_mult=%d kfree_by_call_rcu=%d kfree_nthreads=%d kfree_alloc_num=%d kfree…  in kfree_scale_init()
868 	// Also, do a quick self-test to ensure laziness is as much as  in kfree_scale_init()
890 		if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start < 2 * HZ)) {  in kfree_scale_init()
892 			firsterr = -1;  in kfree_scale_init()
896 		if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start > 3 * HZ)) {  in kfree_scale_init()
898 			firsterr = -1;  in kfree_scale_init()
921 		firsterr = -ENOMEM;  in kfree_scale_init()
955 	 * during the mid-boot phase, so have to wait till the end.  in rcu_scale_cleanup()
958 		SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");  in rcu_scale_cleanup()
960 		SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");  in rcu_scale_cleanup()
964 	// If built-in, just report all of the GP kthread's CPU time.  in rcu_scale_cleanup()
965 	if (IS_BUILTIN(CONFIG_RCU_SCALE_TEST) && !kthread_tp && cur_ops->rso_gp_kthread)  in rcu_scale_cleanup()
966 		kthread_tp = cur_ops->rso_gp_kthread();  in rcu_scale_cleanup()
971 		kthread_stime = kthread_tp->stime - kthread_stime;  in rcu_scale_cleanup()
973 		pr_info("rcu_scale: Grace-period kthread CPU time: %llu.%03u us\n", us, ns);  in rcu_scale_cleanup()
1010 			 t_rcu_scale_writer_finished -  in rcu_scale_cleanup()
1025 				pr_alert("%s%s %4d writer-duration: %5d %llu\n",  in rcu_scale_cleanup()
1037 				if (wflp->ws_mblocks) {  in rcu_scale_cleanup()
1038 					llist_for_each(llnp, wflp->ws_lhg.first)  in rcu_scale_cleanup()
1040 					llist_for_each(llnp, wflp->ws_lhp.first)  in rcu_scale_cleanup()
1045 					kfree(wflp->ws_mblocks);  in rcu_scale_cleanup()
1061 	/* Do torture-type-specific cleanup operations.  */  in rcu_scale_cleanup()
1062 	if (cur_ops->cleanup != NULL)  in rcu_scale_cleanup()
1063 		cur_ops->cleanup();  in rcu_scale_cleanup()
1079 	return -EINVAL;  in rcu_scale_shutdown()
1093 		return -EBUSY;  in rcu_scale_init()
1098 		if (strcmp(scale_type, cur_ops->name) == 0)  in rcu_scale_init()
1102 		pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);  in rcu_scale_init()
1103 		pr_alert("rcu-scale types:");  in rcu_scale_init()
1105 			pr_cont(" %s", scale_ops[i]->name);  in rcu_scale_init()
1107 		firsterr = -EINVAL;  in rcu_scale_init()
1111 	if (cur_ops->init)  in rcu_scale_init()
1112 		cur_ops->init();  in rcu_scale_init()
1114 	if (cur_ops->rso_gp_kthread) {  in rcu_scale_init()
1115 		kthread_tp = cur_ops->rso_gp_kthread();  in rcu_scale_init()
1117 			kthread_stime = kthread_tp->stime;  in rcu_scale_init()
1143 		firsterr = -ENOMEM;  in rcu_scale_init()
1163 			firsterr = -EINVAL;  in rcu_scale_init()
1171 		firsterr = -ENOMEM;  in rcu_scale_init()
1179 			firsterr = -ENOMEM;  in rcu_scale_init()
1185 			init_llist_head(&wflp->ws_lhg);  in rcu_scale_init()
1186 			init_llist_head(&wflp->ws_lhp);  in rcu_scale_init()
1187 			wflp->ws_mblocks = kcalloc(gp_async_max, sizeof(wflp->ws_mblocks[0]),  in rcu_scale_init()
1189 			if (!wflp->ws_mblocks) {  in rcu_scale_init()
1190 				firsterr = -ENOMEM;  in rcu_scale_init()
1194 				struct writer_mblock *wmbp = &wflp->ws_mblocks[j];  in rcu_scale_init()
1196 				wmbp->wmb_wfl = wflp;  in rcu_scale_init()
1197 				llist_add(&wmbp->wmb_node, &wflp->ws_lhp);  in rcu_scale_init()