1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based scalability-test facility 4 * 5 * Copyright (C) IBM Corporation, 2015 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 */ 9 10 #define pr_fmt(fmt) fmt 11 12 #include <linux/types.h> 13 #include <linux/kernel.h> 14 #include <linux/init.h> 15 #include <linux/mm.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/rcupdate.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/atomic.h> 26 #include <linux/bitops.h> 27 #include <linux/completion.h> 28 #include <linux/moduleparam.h> 29 #include <linux/percpu.h> 30 #include <linux/notifier.h> 31 #include <linux/reboot.h> 32 #include <linux/freezer.h> 33 #include <linux/cpu.h> 34 #include <linux/delay.h> 35 #include <linux/stat.h> 36 #include <linux/srcu.h> 37 #include <linux/slab.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include <linux/vmalloc.h> 41 #include <linux/rcupdate_trace.h> 42 #include <linux/sched/debug.h> 43 44 #include "rcu.h" 45 46 MODULE_DESCRIPTION("Read-Copy Update module-based scalability-test facility"); 47 MODULE_LICENSE("GPL"); 48 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 49 50 #define SCALE_FLAG "-scale:" 51 #define SCALEOUT_STRING(s) \ 52 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s) 53 #define VERBOSE_SCALEOUT_STRING(s) \ 54 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0) 55 #define SCALEOUT_ERRSTRING(s) \ 56 pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s) 57 58 /* 59 * The intended use cases for the nreaders and nwriters module parameters 60 * are as follows: 61 * 62 * 1. Specify only the nr_cpus kernel boot parameter. This will 63 * set both nreaders and nwriters to the value specified by 64 * nr_cpus for a mixed reader/writer test. 65 * 66 * 2. Specify the nr_cpus kernel boot parameter, but set 67 * rcuscale.nreaders to zero. This will set nwriters to the 68 * value specified by nr_cpus for an update-only test. 69 * 70 * 3. Specify the nr_cpus kernel boot parameter, but set 71 * rcuscale.nwriters to zero. This will set nreaders to the 72 * value specified by nr_cpus for a read-only test. 73 * 74 * Various other use cases may of course be specified. 75 * 76 * Note that this test's readers are intended only as a test load for 77 * the writers. The reader scalability statistics will be overly 78 * pessimistic due to the per-critical-section interrupt disabling, 79 * test-end checks, and the pair of calls through pointers. 80 */ 81 82 torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); 83 torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer"); 84 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 85 torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); 86 torture_param(int, minruntime, 0, "Minimum run time (s)"); 87 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 88 torture_param(int, nwriters, -1, "Number of RCU updater threads"); 89 torture_param(int, shutdown_secs, !IS_MODULE(CONFIG_RCU_SCALE_TEST) * 300, 90 "Shutdown at end of scalability tests or at specified timeout (s)."); 91 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 92 torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); 93 torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); 94 torture_param(int, nexp, 0, "Number of expedited GP threads to run concurrently"); 95 torture_param(int, exp_interval, 0, "Interval (us) between expedited GPs, zero to disable"); 96 torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); 97 torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); 98 torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); 99 100 static char *scale_type = "rcu"; 101 module_param(scale_type, charp, 0444); 102 MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)"); 103 104 // Structure definitions for custom fixed-per-task allocator. 105 struct writer_mblock { 106 struct rcu_head wmb_rh; 107 struct llist_node wmb_node; 108 struct writer_freelist *wmb_wfl; 109 }; 110 111 struct writer_freelist { 112 struct llist_head ws_lhg; 113 atomic_t ws_inflight; 114 struct llist_head ____cacheline_internodealigned_in_smp ws_lhp; 115 struct writer_mblock *ws_mblocks; 116 }; 117 118 static int nrealreaders; 119 static int nrealwriters; 120 static int nrealexp; 121 static struct task_struct **writer_tasks; 122 static struct task_struct **reader_tasks; 123 static struct task_struct **exp_tasks; 124 125 static u64 **writer_durations; 126 static bool *writer_done; 127 static struct writer_freelist *writer_freelists; 128 static int *writer_n_durations; 129 static atomic_t n_rcu_scale_reader_started; 130 static atomic_t n_rcu_scale_writer_started; 131 static atomic_t n_rcu_scale_writer_finished; 132 static u64 t_rcu_scale_writer_started; 133 static u64 t_rcu_scale_writer_finished; 134 static unsigned long b_rcu_gp_test_started; 135 static unsigned long b_rcu_gp_test_finished; 136 137 #define MAX_MEAS 10000 138 #define MIN_MEAS 100 139 140 /* 141 * Operations vector for selecting different types of tests. 142 */ 143 144 struct rcu_scale_ops { 145 int ptype; 146 void (*init)(void); 147 void (*cleanup)(void); 148 int (*readlock)(void); 149 void (*readunlock)(int idx); 150 unsigned long (*get_gp_seq)(void); 151 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 152 unsigned long (*exp_completed)(void); 153 void (*async)(struct rcu_head *head, rcu_callback_t func); 154 void (*gp_barrier)(void); 155 void (*sync)(void); 156 void (*exp_sync)(void); 157 struct task_struct *(*rso_gp_kthread)(void); 158 void (*stats)(void); 159 const char *name; 160 }; 161 162 static struct rcu_scale_ops *cur_ops; 163 164 /* 165 * Definitions for rcu scalability testing. 166 */ 167 168 static int rcu_scale_read_lock(void) __acquires(RCU) 169 { 170 rcu_read_lock(); 171 return 0; 172 } 173 174 static void rcu_scale_read_unlock(int idx) __releases(RCU) 175 { 176 rcu_read_unlock(); 177 } 178 179 static unsigned long __maybe_unused rcu_no_completed(void) 180 { 181 return 0; 182 } 183 184 static void rcu_sync_scale_init(void) 185 { 186 } 187 188 static struct rcu_scale_ops rcu_ops = { 189 .ptype = RCU_FLAVOR, 190 .init = rcu_sync_scale_init, 191 .readlock = rcu_scale_read_lock, 192 .readunlock = rcu_scale_read_unlock, 193 .get_gp_seq = rcu_get_gp_seq, 194 .gp_diff = rcu_seq_diff, 195 .exp_completed = rcu_exp_batches_completed, 196 .async = call_rcu_hurry, 197 .gp_barrier = rcu_barrier, 198 .sync = synchronize_rcu, 199 .exp_sync = synchronize_rcu_expedited, 200 .name = "rcu" 201 }; 202 203 /* 204 * Definitions for srcu scalability testing. 205 */ 206 207 DEFINE_STATIC_SRCU(srcu_ctl_scale); 208 static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale; 209 210 static int srcu_scale_read_lock(void) __acquires(srcu_ctlp) 211 { 212 return srcu_read_lock(srcu_ctlp); 213 } 214 215 static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp) 216 { 217 srcu_read_unlock(srcu_ctlp, idx); 218 } 219 220 static unsigned long srcu_scale_completed(void) 221 { 222 return srcu_batches_completed(srcu_ctlp); 223 } 224 225 static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func) 226 { 227 call_srcu(srcu_ctlp, head, func); 228 } 229 230 static void srcu_rcu_barrier(void) 231 { 232 srcu_barrier(srcu_ctlp); 233 } 234 235 static void srcu_scale_synchronize(void) 236 { 237 synchronize_srcu(srcu_ctlp); 238 } 239 240 static void srcu_scale_stats(void) 241 { 242 srcu_torture_stats_print(srcu_ctlp, scale_type, SCALE_FLAG); 243 } 244 245 static void srcu_scale_synchronize_expedited(void) 246 { 247 synchronize_srcu_expedited(srcu_ctlp); 248 } 249 250 static struct rcu_scale_ops srcu_ops = { 251 .ptype = SRCU_FLAVOR, 252 .init = rcu_sync_scale_init, 253 .readlock = srcu_scale_read_lock, 254 .readunlock = srcu_scale_read_unlock, 255 .get_gp_seq = srcu_scale_completed, 256 .gp_diff = rcu_seq_diff, 257 .exp_completed = srcu_scale_completed, 258 .async = srcu_call_rcu, 259 .gp_barrier = srcu_rcu_barrier, 260 .sync = srcu_scale_synchronize, 261 .exp_sync = srcu_scale_synchronize_expedited, 262 .stats = srcu_scale_stats, 263 .name = "srcu" 264 }; 265 266 static struct srcu_struct srcud; 267 268 static void srcu_sync_scale_init(void) 269 { 270 srcu_ctlp = &srcud; 271 init_srcu_struct(srcu_ctlp); 272 } 273 274 static void srcu_sync_scale_cleanup(void) 275 { 276 cleanup_srcu_struct(srcu_ctlp); 277 } 278 279 static struct rcu_scale_ops srcud_ops = { 280 .ptype = SRCU_FLAVOR, 281 .init = srcu_sync_scale_init, 282 .cleanup = srcu_sync_scale_cleanup, 283 .readlock = srcu_scale_read_lock, 284 .readunlock = srcu_scale_read_unlock, 285 .get_gp_seq = srcu_scale_completed, 286 .gp_diff = rcu_seq_diff, 287 .exp_completed = srcu_scale_completed, 288 .async = srcu_call_rcu, 289 .gp_barrier = srcu_rcu_barrier, 290 .sync = srcu_scale_synchronize, 291 .exp_sync = srcu_scale_synchronize_expedited, 292 .stats = srcu_scale_stats, 293 .name = "srcud" 294 }; 295 296 #ifdef CONFIG_TASKS_RCU 297 298 /* 299 * Definitions for RCU-tasks scalability testing. 300 */ 301 302 static int tasks_scale_read_lock(void) 303 { 304 return 0; 305 } 306 307 static void tasks_scale_read_unlock(int idx) 308 { 309 } 310 311 static void rcu_tasks_scale_stats(void) 312 { 313 rcu_tasks_torture_stats_print(scale_type, SCALE_FLAG); 314 } 315 316 static struct rcu_scale_ops tasks_ops = { 317 .ptype = RCU_TASKS_FLAVOR, 318 .init = rcu_sync_scale_init, 319 .readlock = tasks_scale_read_lock, 320 .readunlock = tasks_scale_read_unlock, 321 .get_gp_seq = rcu_no_completed, 322 .gp_diff = rcu_seq_diff, 323 .async = call_rcu_tasks, 324 .gp_barrier = rcu_barrier_tasks, 325 .sync = synchronize_rcu_tasks, 326 .exp_sync = synchronize_rcu_tasks, 327 .rso_gp_kthread = get_rcu_tasks_gp_kthread, 328 .stats = IS_ENABLED(CONFIG_TINY_RCU) ? NULL : rcu_tasks_scale_stats, 329 .name = "tasks" 330 }; 331 332 #define TASKS_OPS &tasks_ops, 333 334 #else // #ifdef CONFIG_TASKS_RCU 335 336 #define TASKS_OPS 337 338 #endif // #else // #ifdef CONFIG_TASKS_RCU 339 340 #ifdef CONFIG_TASKS_RUDE_RCU 341 342 /* 343 * Definitions for RCU-tasks-rude scalability testing. 344 */ 345 346 static int tasks_rude_scale_read_lock(void) 347 { 348 return 0; 349 } 350 351 static void tasks_rude_scale_read_unlock(int idx) 352 { 353 } 354 355 static void rcu_tasks_rude_scale_stats(void) 356 { 357 rcu_tasks_rude_torture_stats_print(scale_type, SCALE_FLAG); 358 } 359 360 static struct rcu_scale_ops tasks_rude_ops = { 361 .ptype = RCU_TASKS_RUDE_FLAVOR, 362 .init = rcu_sync_scale_init, 363 .readlock = tasks_rude_scale_read_lock, 364 .readunlock = tasks_rude_scale_read_unlock, 365 .get_gp_seq = rcu_no_completed, 366 .gp_diff = rcu_seq_diff, 367 .sync = synchronize_rcu_tasks_rude, 368 .exp_sync = synchronize_rcu_tasks_rude, 369 .rso_gp_kthread = get_rcu_tasks_rude_gp_kthread, 370 .stats = IS_ENABLED(CONFIG_TINY_RCU) ? NULL : rcu_tasks_rude_scale_stats, 371 .name = "tasks-rude" 372 }; 373 374 #define TASKS_RUDE_OPS &tasks_rude_ops, 375 376 #else // #ifdef CONFIG_TASKS_RUDE_RCU 377 378 #define TASKS_RUDE_OPS 379 380 #endif // #else // #ifdef CONFIG_TASKS_RUDE_RCU 381 382 #ifdef CONFIG_TASKS_TRACE_RCU 383 384 /* 385 * Definitions for RCU-tasks-trace scalability testing. 386 */ 387 388 static int tasks_trace_scale_read_lock(void) 389 { 390 rcu_read_lock_trace(); 391 return 0; 392 } 393 394 static void tasks_trace_scale_read_unlock(int idx) 395 { 396 rcu_read_unlock_trace(); 397 } 398 399 static struct rcu_scale_ops tasks_tracing_ops = { 400 .ptype = RCU_TASKS_FLAVOR, 401 .init = rcu_sync_scale_init, 402 .readlock = tasks_trace_scale_read_lock, 403 .readunlock = tasks_trace_scale_read_unlock, 404 .get_gp_seq = rcu_no_completed, 405 .gp_diff = rcu_seq_diff, 406 .async = call_rcu_tasks_trace, 407 .gp_barrier = rcu_barrier_tasks_trace, 408 .sync = synchronize_rcu_tasks_trace, 409 .exp_sync = synchronize_rcu_tasks_trace, 410 .name = "tasks-tracing" 411 }; 412 413 #define TASKS_TRACING_OPS &tasks_tracing_ops, 414 415 #else // #ifdef CONFIG_TASKS_TRACE_RCU 416 417 #define TASKS_TRACING_OPS 418 419 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU 420 421 static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old) 422 { 423 if (!cur_ops->gp_diff) 424 return new - old; 425 return cur_ops->gp_diff(new, old); 426 } 427 428 /* 429 * If scalability tests complete, wait for shutdown to commence. 430 */ 431 static void rcu_scale_wait_shutdown(void) 432 { 433 cond_resched_tasks_rcu_qs(); 434 if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters) 435 return; 436 while (!torture_must_stop()) 437 schedule_timeout_uninterruptible(1); 438 } 439 440 /* 441 * RCU scalability reader kthread. Repeatedly does empty RCU read-side 442 * critical section, minimizing update-side interference. However, the 443 * point of this test is not to evaluate reader scalability, but instead 444 * to serve as a test load for update-side scalability testing. 445 */ 446 static int 447 rcu_scale_reader(void *arg) 448 { 449 unsigned long flags; 450 int idx; 451 long me = (long)arg; 452 453 VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started"); 454 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 455 set_user_nice(current, MAX_NICE); 456 atomic_inc(&n_rcu_scale_reader_started); 457 458 do { 459 local_irq_save(flags); 460 idx = cur_ops->readlock(); 461 cur_ops->readunlock(idx); 462 local_irq_restore(flags); 463 rcu_scale_wait_shutdown(); 464 } while (!torture_must_stop()); 465 torture_kthread_stopping("rcu_scale_reader"); 466 return 0; 467 } 468 469 /* 470 * RCU expedited GP kthread. Repeatedly invokes expedited grace periods 471 * to generate concurrent expedited GP load while the normal-GP writers 472 * are being measured. This allows measuring the benefit of callbacks 473 * that can piggyback on expedited grace periods. 474 */ 475 static int 476 rcu_scale_exp(void *arg) 477 { 478 long me = (long)arg; 479 480 VERBOSE_SCALEOUT_STRING("rcu_scale_exp task started"); 481 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 482 set_user_nice(current, MIN_NICE); 483 484 if (holdoff) 485 schedule_timeout_idle(holdoff * HZ); 486 487 do { 488 if (exp_interval) 489 udelay(exp_interval); 490 cur_ops->exp_sync(); 491 rcu_scale_wait_shutdown(); 492 } while (!torture_must_stop()); 493 torture_kthread_stopping("rcu_scale_exp"); 494 return 0; 495 } 496 497 /* 498 * Allocate a writer_mblock structure for the specified rcu_scale_writer 499 * task. 500 */ 501 static struct writer_mblock *rcu_scale_alloc(long me) 502 { 503 struct llist_node *llnp; 504 struct writer_freelist *wflp; 505 struct writer_mblock *wmbp; 506 507 if (WARN_ON_ONCE(!writer_freelists)) 508 return NULL; 509 wflp = &writer_freelists[me]; 510 if (llist_empty(&wflp->ws_lhp)) { 511 // ->ws_lhp is private to its rcu_scale_writer task. 512 wmbp = container_of(llist_del_all(&wflp->ws_lhg), struct writer_mblock, wmb_node); 513 wflp->ws_lhp.first = &wmbp->wmb_node; 514 } 515 llnp = llist_del_first(&wflp->ws_lhp); 516 if (!llnp) 517 return NULL; 518 return container_of(llnp, struct writer_mblock, wmb_node); 519 } 520 521 /* 522 * Free a writer_mblock structure to its rcu_scale_writer task. 523 */ 524 static void rcu_scale_free(struct writer_mblock *wmbp) 525 { 526 struct writer_freelist *wflp; 527 528 if (!wmbp) 529 return; 530 wflp = wmbp->wmb_wfl; 531 llist_add(&wmbp->wmb_node, &wflp->ws_lhg); 532 } 533 534 /* 535 * Callback function for asynchronous grace periods from rcu_scale_writer(). 536 */ 537 static void rcu_scale_async_cb(struct rcu_head *rhp) 538 { 539 struct writer_mblock *wmbp = container_of(rhp, struct writer_mblock, wmb_rh); 540 struct writer_freelist *wflp = wmbp->wmb_wfl; 541 542 atomic_dec(&wflp->ws_inflight); 543 rcu_scale_free(wmbp); 544 } 545 546 static void rcu_scale_cleanup(void); 547 548 /* 549 * RCU scale writer kthread. Repeatedly does a grace period. 550 */ 551 static int 552 rcu_scale_writer(void *arg) 553 { 554 int i = 0; 555 int i_max; 556 unsigned long jdone; 557 long me = (long)arg; 558 bool selfreport = false; 559 bool started = false, done = false, alldone = false; 560 u64 t; 561 DEFINE_TORTURE_RANDOM(tr); 562 u64 *wdp; 563 u64 *wdpp = writer_durations[me]; 564 struct writer_freelist *wflp = &writer_freelists[me]; 565 struct writer_mblock *wmbp = NULL; 566 567 VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started"); 568 WARN_ON(!wdpp); 569 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 570 current->flags |= PF_NO_SETAFFINITY; 571 sched_set_fifo_low(current); 572 573 if (holdoff) 574 schedule_timeout_idle(holdoff * HZ); 575 576 /* 577 * Wait until rcu_end_inkernel_boot() is called for normal GP tests 578 * so that RCU is not always expedited for normal GP tests. 579 * The system_state test is approximate, but works well in practice. 580 */ 581 while (!gp_exp && system_state != SYSTEM_RUNNING) 582 schedule_timeout_uninterruptible(1); 583 584 t = ktime_get_mono_fast_ns(); 585 if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) { 586 t_rcu_scale_writer_started = t; 587 if (gp_exp) { 588 b_rcu_gp_test_started = 589 cur_ops->exp_completed() / 2; 590 } else { 591 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 592 } 593 } 594 595 jdone = jiffies + minruntime * HZ; 596 do { 597 bool gp_succeeded = false; 598 599 if (writer_holdoff) 600 udelay(writer_holdoff); 601 if (writer_holdoff_jiffies) 602 schedule_timeout_idle(torture_random(&tr) % writer_holdoff_jiffies + 1); 603 wdp = &wdpp[i]; 604 *wdp = ktime_get_mono_fast_ns(); 605 if (gp_async && !WARN_ON_ONCE(!cur_ops->async)) { 606 if (!wmbp) 607 wmbp = rcu_scale_alloc(me); 608 if (wmbp && atomic_read(&wflp->ws_inflight) < gp_async_max) { 609 atomic_inc(&wflp->ws_inflight); 610 cur_ops->async(&wmbp->wmb_rh, rcu_scale_async_cb); 611 wmbp = NULL; 612 gp_succeeded = true; 613 } else if (!kthread_should_stop()) { 614 cur_ops->gp_barrier(); 615 } else { 616 rcu_scale_free(wmbp); /* Because we are stopping. */ 617 wmbp = NULL; 618 } 619 } else if (gp_exp) { 620 cur_ops->exp_sync(); 621 gp_succeeded = true; 622 } else { 623 cur_ops->sync(); 624 gp_succeeded = true; 625 } 626 t = ktime_get_mono_fast_ns(); 627 *wdp = t - *wdp; 628 i_max = i; 629 if (!started && 630 atomic_read(&n_rcu_scale_writer_started) >= nrealwriters) 631 started = true; 632 if (!done && i >= MIN_MEAS && time_after(jiffies, jdone)) { 633 done = true; 634 WRITE_ONCE(writer_done[me], true); 635 sched_set_normal(current, 0); 636 pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n", 637 scale_type, SCALE_FLAG, me, MIN_MEAS); 638 if (atomic_inc_return(&n_rcu_scale_writer_finished) >= 639 nrealwriters) { 640 schedule_timeout_interruptible(10); 641 rcu_ftrace_dump(DUMP_ALL); 642 SCALEOUT_STRING("Test complete"); 643 t_rcu_scale_writer_finished = t; 644 if (gp_exp) { 645 b_rcu_gp_test_finished = 646 cur_ops->exp_completed() / 2; 647 } else { 648 b_rcu_gp_test_finished = 649 cur_ops->get_gp_seq(); 650 } 651 if (shutdown_secs) { 652 writer_tasks[me] = NULL; 653 smp_mb(); /* Assign before wake. */ 654 rcu_scale_cleanup(); 655 kernel_power_off(); 656 } 657 } 658 } 659 if (done && !alldone && 660 atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters) 661 alldone = true; 662 if (done && !alldone && time_after(jiffies, jdone + HZ * 60)) { 663 static atomic_t dumped; 664 int i; 665 666 if (!atomic_xchg(&dumped, 1)) { 667 for (i = 0; i < nrealwriters; i++) { 668 if (writer_done[i]) 669 continue; 670 pr_info("%s: Task %ld flags writer %d:\n", __func__, me, i); 671 sched_show_task(writer_tasks[i]); 672 } 673 if (cur_ops->stats) 674 cur_ops->stats(); 675 } 676 } 677 if (!selfreport && time_after(jiffies, jdone + HZ * (70 + me))) { 678 pr_info("%s: Writer %ld self-report: started %d done %d/%d->%d i %d jdone %lu.\n", 679 __func__, me, started, done, writer_done[me], atomic_read(&n_rcu_scale_writer_finished), i, jiffies - jdone); 680 selfreport = true; 681 } 682 if (gp_succeeded && started && !alldone && i < MAX_MEAS - 1) 683 i++; 684 rcu_scale_wait_shutdown(); 685 } while (!torture_must_stop()); 686 if (gp_async && cur_ops->async) { 687 rcu_scale_free(wmbp); 688 cur_ops->gp_barrier(); 689 } 690 writer_n_durations[me] = i_max + 1; 691 torture_kthread_stopping("rcu_scale_writer"); 692 return 0; 693 } 694 695 static void 696 rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) 697 { 698 pr_alert("%s" SCALE_FLAG 699 "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d nexp=%d exp_interval=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n", 700 scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, 701 minruntime, nrealreaders, nrealwriters, nrealexp, exp_interval, 702 writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); 703 } 704 705 /* 706 * Return the number if non-negative. If -1, the number of CPUs. 707 * If less than -1, that much less than the number of CPUs, but 708 * at least one. 709 */ 710 static int compute_real(int n) 711 { 712 int nr; 713 714 if (n >= 0) { 715 nr = n; 716 } else { 717 nr = num_online_cpus() + 1 + n; 718 if (nr <= 0) 719 nr = 1; 720 } 721 return nr; 722 } 723 724 /* 725 * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number 726 * of iterations and measure total time and number of GP for all iterations to complete. 727 */ 728 729 torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu()."); 730 torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration."); 731 torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees."); 732 torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?"); 733 torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?"); 734 735 static struct task_struct **kfree_reader_tasks; 736 static int kfree_nrealthreads; 737 static atomic_t n_kfree_scale_thread_started; 738 static atomic_t n_kfree_scale_thread_ended; 739 static struct task_struct *kthread_tp; 740 static u64 kthread_stime; 741 742 struct kfree_obj { 743 char kfree_obj[8]; 744 struct rcu_head rh; 745 }; 746 747 /* Used if doing RCU-kfree'ing via call_rcu(). */ 748 static void kfree_call_rcu(struct rcu_head *rh) 749 { 750 struct kfree_obj *obj = container_of(rh, struct kfree_obj, rh); 751 752 kfree(obj); 753 } 754 755 static void kfree_scale_cleanup(void); 756 757 static int 758 kfree_scale_thread(void *arg) 759 { 760 int i, loop = 0; 761 long me = (long)arg; 762 struct kfree_obj *alloc_ptr; 763 u64 start_time, end_time; 764 long long mem_begin, mem_during = 0; 765 bool kfree_rcu_test_both; 766 DEFINE_TORTURE_RANDOM(tr); 767 768 VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started"); 769 set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); 770 set_user_nice(current, MAX_NICE); 771 kfree_rcu_test_both = (kfree_rcu_test_single == kfree_rcu_test_double); 772 773 start_time = ktime_get_mono_fast_ns(); 774 775 if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) { 776 if (gp_exp) 777 b_rcu_gp_test_started = cur_ops->exp_completed() / 2; 778 else 779 b_rcu_gp_test_started = cur_ops->get_gp_seq(); 780 } 781 782 do { 783 if (!mem_during) { 784 mem_during = mem_begin = si_mem_available(); 785 } else if (loop % (kfree_loops / 4) == 0) { 786 mem_during = (mem_during + si_mem_available()) / 2; 787 } 788 789 for (i = 0; i < kfree_alloc_num; i++) { 790 alloc_ptr = kzalloc_objs(struct kfree_obj, kfree_mult); 791 if (!alloc_ptr) 792 return -ENOMEM; 793 794 if (kfree_by_call_rcu) { 795 call_rcu(&(alloc_ptr->rh), kfree_call_rcu); 796 continue; 797 } 798 799 // By default kfree_rcu_test_single and kfree_rcu_test_double are 800 // initialized to false. If both have the same value (false or true) 801 // both are randomly tested, otherwise only the one with value true 802 // is tested. 803 if ((kfree_rcu_test_single && !kfree_rcu_test_double) || 804 (kfree_rcu_test_both && torture_random(&tr) & 0x800)) 805 kfree_rcu_mightsleep(alloc_ptr); 806 else 807 kfree_rcu(alloc_ptr, rh); 808 } 809 810 cond_resched(); 811 } while (!torture_must_stop() && ++loop < kfree_loops); 812 813 if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) { 814 end_time = ktime_get_mono_fast_ns(); 815 816 if (gp_exp) 817 b_rcu_gp_test_finished = cur_ops->exp_completed() / 2; 818 else 819 b_rcu_gp_test_finished = cur_ops->get_gp_seq(); 820 821 pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n", 822 (unsigned long long)(end_time - start_time), kfree_loops, 823 rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started), 824 PAGES_TO_MB(mem_begin - mem_during)); 825 826 if (shutdown_secs) { 827 kfree_reader_tasks[me] = NULL; 828 smp_mb(); /* Assign before wake. */ 829 kfree_scale_cleanup(); 830 kernel_power_off(); 831 } 832 } 833 834 torture_kthread_stopping("kfree_scale_thread"); 835 return 0; 836 } 837 838 static void 839 kfree_scale_cleanup(void) 840 { 841 int i; 842 843 if (torture_cleanup_begin()) 844 return; 845 846 if (exp_tasks) { 847 for (i = 0; i < nrealexp; i++) 848 torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); 849 kfree(exp_tasks); 850 exp_tasks = NULL; 851 } 852 853 if (kfree_reader_tasks) { 854 for (i = 0; i < kfree_nrealthreads; i++) 855 torture_stop_kthread(kfree_scale_thread, 856 kfree_reader_tasks[i]); 857 kfree(kfree_reader_tasks); 858 kfree_reader_tasks = NULL; 859 } 860 861 torture_cleanup_end(); 862 } 863 864 // Used if doing RCU-kfree'ing via call_rcu(). 865 static unsigned long jiffies_at_lazy_cb; 866 static struct rcu_head lazy_test1_rh; 867 static int rcu_lazy_test1_cb_called; 868 static void call_rcu_lazy_test1(struct rcu_head *rh) 869 { 870 jiffies_at_lazy_cb = jiffies; 871 WRITE_ONCE(rcu_lazy_test1_cb_called, 1); 872 } 873 874 static int __init 875 kfree_scale_init(void) 876 { 877 int firsterr = 0; 878 long i; 879 unsigned long jif_start; 880 unsigned long orig_jif; 881 882 pr_alert("%s" SCALE_FLAG 883 "--- kfree_rcu_test: kfree_mult=%d kfree_by_call_rcu=%d kfree_nthreads=%d kfree_alloc_num=%d kfree_loops=%d kfree_rcu_test_double=%d kfree_rcu_test_single=%d\n", 884 scale_type, kfree_mult, kfree_by_call_rcu, kfree_nthreads, kfree_alloc_num, kfree_loops, kfree_rcu_test_double, kfree_rcu_test_single); 885 886 // Also, do a quick self-test to ensure laziness is as much as 887 // expected. 888 if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) { 889 pr_alert("CONFIG_RCU_LAZY is disabled, falling back to kfree_rcu() for delayed RCU kfree'ing\n"); 890 kfree_by_call_rcu = 0; 891 } 892 893 if (kfree_by_call_rcu) { 894 /* do a test to check the timeout. */ 895 orig_jif = rcu_get_jiffies_lazy_flush(); 896 897 rcu_set_jiffies_lazy_flush(2 * HZ); 898 rcu_barrier(); 899 900 jif_start = jiffies; 901 jiffies_at_lazy_cb = 0; 902 call_rcu(&lazy_test1_rh, call_rcu_lazy_test1); 903 904 smp_cond_load_relaxed(&rcu_lazy_test1_cb_called, VAL == 1); 905 906 rcu_set_jiffies_lazy_flush(orig_jif); 907 908 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start < 2 * HZ)) { 909 pr_alert("ERROR: call_rcu() CBs are not being lazy as expected!\n"); 910 firsterr = -1; 911 goto unwind; 912 } 913 914 if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start > 3 * HZ)) { 915 pr_alert("ERROR: call_rcu() CBs are being too lazy!\n"); 916 firsterr = -1; 917 goto unwind; 918 } 919 } 920 921 kfree_nrealthreads = compute_real(kfree_nthreads); 922 /* Start up the kthreads. */ 923 if (shutdown_secs) { 924 firsterr = torture_shutdown_init(shutdown_secs, kfree_scale_cleanup); 925 if (torture_init_error(firsterr)) 926 goto unwind; 927 } 928 929 pr_alert("kfree object size=%zu, kfree_by_call_rcu=%d\n", 930 kfree_mult * sizeof(struct kfree_obj), 931 kfree_by_call_rcu); 932 933 kfree_reader_tasks = kzalloc_objs(kfree_reader_tasks[0], 934 kfree_nrealthreads); 935 if (kfree_reader_tasks == NULL) { 936 firsterr = -ENOMEM; 937 goto unwind; 938 } 939 940 for (i = 0; i < kfree_nrealthreads; i++) { 941 firsterr = torture_create_kthread(kfree_scale_thread, (void *)i, 942 kfree_reader_tasks[i]); 943 if (torture_init_error(firsterr)) 944 goto unwind; 945 } 946 947 if (nrealexp > 0 && cur_ops->exp_sync) { 948 exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); 949 if (!exp_tasks) { 950 SCALEOUT_ERRSTRING("out of memory"); 951 firsterr = -ENOMEM; 952 goto unwind; 953 } 954 for (i = 0; i < nrealexp; i++) { 955 firsterr = torture_create_kthread(rcu_scale_exp, 956 (void *)i, 957 exp_tasks[i]); 958 if (torture_init_error(firsterr)) 959 goto unwind; 960 } 961 } 962 963 while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads) 964 schedule_timeout_uninterruptible(1); 965 966 torture_init_end(); 967 return 0; 968 969 unwind: 970 torture_init_end(); 971 kfree_scale_cleanup(); 972 return firsterr; 973 } 974 975 static void 976 rcu_scale_cleanup(void) 977 { 978 int i; 979 int j; 980 int ngps = 0; 981 u64 *wdp; 982 u64 *wdpp; 983 984 /* 985 * Would like warning at start, but everything is expedited 986 * during the mid-boot phase, so have to wait till the end. 987 */ 988 if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp) 989 SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!"); 990 if (rcu_gp_is_normal() && gp_exp) 991 SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!"); 992 if (gp_exp && gp_async) 993 SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); 994 995 // If built-in, just report all of the GP kthread's CPU time. 996 if (IS_BUILTIN(CONFIG_RCU_SCALE_TEST) && !kthread_tp && cur_ops->rso_gp_kthread) 997 kthread_tp = cur_ops->rso_gp_kthread(); 998 if (kthread_tp) { 999 u32 ns; 1000 u64 us; 1001 1002 kthread_stime = kthread_tp->stime - kthread_stime; 1003 us = div_u64_rem(kthread_stime, 1000, &ns); 1004 pr_info("rcu_scale: Grace-period kthread CPU time: %llu.%03u us\n", us, ns); 1005 show_rcu_gp_kthreads(); 1006 } 1007 if (kfree_rcu_test) { 1008 kfree_scale_cleanup(); 1009 return; 1010 } 1011 1012 if (torture_cleanup_begin()) 1013 return; 1014 if (!cur_ops) { 1015 torture_cleanup_end(); 1016 return; 1017 } 1018 1019 if (exp_tasks) { 1020 for (i = 0; i < nrealexp; i++) 1021 torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); 1022 kfree(exp_tasks); 1023 exp_tasks = NULL; 1024 } 1025 1026 if (reader_tasks) { 1027 for (i = 0; i < nrealreaders; i++) 1028 torture_stop_kthread(rcu_scale_reader, 1029 reader_tasks[i]); 1030 kfree(reader_tasks); 1031 reader_tasks = NULL; 1032 } 1033 1034 if (writer_tasks) { 1035 for (i = 0; i < nrealwriters; i++) { 1036 torture_stop_kthread(rcu_scale_writer, 1037 writer_tasks[i]); 1038 if (!writer_n_durations) 1039 continue; 1040 j = writer_n_durations[i]; 1041 pr_alert("%s%s writer %d gps: %d\n", 1042 scale_type, SCALE_FLAG, i, j); 1043 ngps += j; 1044 } 1045 pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n", 1046 scale_type, SCALE_FLAG, 1047 t_rcu_scale_writer_started, t_rcu_scale_writer_finished, 1048 t_rcu_scale_writer_finished - 1049 t_rcu_scale_writer_started, 1050 ngps, 1051 rcuscale_seq_diff(b_rcu_gp_test_finished, 1052 b_rcu_gp_test_started)); 1053 for (i = 0; i < nrealwriters; i++) { 1054 if (!writer_durations) 1055 break; 1056 if (!writer_n_durations) 1057 continue; 1058 wdpp = writer_durations[i]; 1059 if (!wdpp) 1060 continue; 1061 for (j = 0; j < writer_n_durations[i]; j++) { 1062 wdp = &wdpp[j]; 1063 pr_alert("%s%s %4d writer-duration: %5d %llu\n", 1064 scale_type, SCALE_FLAG, 1065 i, j, *wdp); 1066 if (j % 100 == 0) 1067 schedule_timeout_uninterruptible(1); 1068 } 1069 kfree(writer_durations[i]); 1070 if (writer_freelists) { 1071 int ctr = 0; 1072 struct llist_node *llnp; 1073 struct writer_freelist *wflp = &writer_freelists[i]; 1074 1075 if (wflp->ws_mblocks) { 1076 llist_for_each(llnp, wflp->ws_lhg.first) 1077 ctr++; 1078 llist_for_each(llnp, wflp->ws_lhp.first) 1079 ctr++; 1080 WARN_ONCE(ctr != gp_async_max, 1081 "%s: ctr = %d gp_async_max = %d\n", 1082 __func__, ctr, gp_async_max); 1083 kfree(wflp->ws_mblocks); 1084 } 1085 } 1086 } 1087 kfree(writer_tasks); 1088 writer_tasks = NULL; 1089 kfree(writer_durations); 1090 writer_durations = NULL; 1091 kfree(writer_n_durations); 1092 writer_n_durations = NULL; 1093 kfree(writer_done); 1094 writer_done = NULL; 1095 kfree(writer_freelists); 1096 writer_freelists = NULL; 1097 } 1098 1099 /* Do torture-type-specific cleanup operations. */ 1100 if (cur_ops->cleanup != NULL) 1101 cur_ops->cleanup(); 1102 1103 torture_cleanup_end(); 1104 } 1105 1106 static int __init 1107 rcu_scale_init(void) 1108 { 1109 int firsterr = 0; 1110 long i; 1111 long j; 1112 static struct rcu_scale_ops *scale_ops[] = { 1113 &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 1114 }; 1115 1116 if (!torture_init_begin(scale_type, verbose)) 1117 return -EBUSY; 1118 1119 /* Process args and announce that the scalability'er is on the job. */ 1120 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 1121 cur_ops = scale_ops[i]; 1122 if (strcmp(scale_type, cur_ops->name) == 0) 1123 break; 1124 } 1125 if (i == ARRAY_SIZE(scale_ops)) { 1126 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 1127 pr_alert("rcu-scale types:"); 1128 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 1129 pr_cont(" %s", scale_ops[i]->name); 1130 pr_cont("\n"); 1131 firsterr = -EINVAL; 1132 cur_ops = NULL; 1133 goto unwind; 1134 } 1135 if (cur_ops->init) 1136 cur_ops->init(); 1137 1138 if (cur_ops->rso_gp_kthread) { 1139 kthread_tp = cur_ops->rso_gp_kthread(); 1140 if (kthread_tp) 1141 kthread_stime = kthread_tp->stime; 1142 } 1143 nrealexp = nexp; 1144 if (kfree_rcu_test) 1145 return kfree_scale_init(); 1146 1147 nrealwriters = compute_real(nwriters); 1148 nrealreaders = compute_real(nreaders); 1149 atomic_set(&n_rcu_scale_reader_started, 0); 1150 atomic_set(&n_rcu_scale_writer_started, 0); 1151 atomic_set(&n_rcu_scale_writer_finished, 0); 1152 rcu_scale_print_module_parms(cur_ops, "Start of test"); 1153 1154 /* Start up the kthreads. */ 1155 1156 if (shutdown_secs) { 1157 firsterr = torture_shutdown_init(shutdown_secs, rcu_scale_cleanup); 1158 if (torture_init_error(firsterr)) 1159 goto unwind; 1160 } 1161 reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); 1162 if (reader_tasks == NULL) { 1163 SCALEOUT_ERRSTRING("out of memory"); 1164 firsterr = -ENOMEM; 1165 goto unwind; 1166 } 1167 for (i = 0; i < nrealreaders; i++) { 1168 firsterr = torture_create_kthread(rcu_scale_reader, (void *)i, 1169 reader_tasks[i]); 1170 if (torture_init_error(firsterr)) 1171 goto unwind; 1172 } 1173 while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) 1174 schedule_timeout_uninterruptible(1); 1175 if (nrealexp > 0 && cur_ops->exp_sync) { 1176 exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); 1177 if (!exp_tasks) { 1178 SCALEOUT_ERRSTRING("out of memory"); 1179 firsterr = -ENOMEM; 1180 goto unwind; 1181 } 1182 for (i = 0; i < nrealexp; i++) { 1183 firsterr = torture_create_kthread(rcu_scale_exp, 1184 (void *)i, 1185 exp_tasks[i]); 1186 if (torture_init_error(firsterr)) 1187 goto unwind; 1188 } 1189 } 1190 writer_tasks = kzalloc_objs(writer_tasks[0], nrealwriters); 1191 writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations), GFP_KERNEL); 1192 writer_n_durations = kzalloc_objs(*writer_n_durations, nrealwriters); 1193 writer_done = kzalloc_objs(writer_done[0], nrealwriters); 1194 if (gp_async) { 1195 if (gp_async_max <= 0) { 1196 pr_warn("%s: gp_async_max = %d must be greater than zero.\n", 1197 __func__, gp_async_max); 1198 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 1199 firsterr = -EINVAL; 1200 goto unwind; 1201 } 1202 writer_freelists = kzalloc_objs(writer_freelists[0], 1203 nrealwriters); 1204 } 1205 if (!writer_tasks || !writer_durations || !writer_n_durations || !writer_done || 1206 (gp_async && !writer_freelists)) { 1207 SCALEOUT_ERRSTRING("out of memory"); 1208 firsterr = -ENOMEM; 1209 goto unwind; 1210 } 1211 for (i = 0; i < nrealwriters; i++) { 1212 writer_durations[i] = 1213 kcalloc(MAX_MEAS, sizeof(*writer_durations[i]), 1214 GFP_KERNEL); 1215 if (!writer_durations[i]) { 1216 firsterr = -ENOMEM; 1217 goto unwind; 1218 } 1219 if (writer_freelists) { 1220 struct writer_freelist *wflp = &writer_freelists[i]; 1221 1222 init_llist_head(&wflp->ws_lhg); 1223 init_llist_head(&wflp->ws_lhp); 1224 wflp->ws_mblocks = kzalloc_objs(wflp->ws_mblocks[0], 1225 gp_async_max); 1226 if (!wflp->ws_mblocks) { 1227 firsterr = -ENOMEM; 1228 goto unwind; 1229 } 1230 for (j = 0; j < gp_async_max; j++) { 1231 struct writer_mblock *wmbp = &wflp->ws_mblocks[j]; 1232 1233 wmbp->wmb_wfl = wflp; 1234 llist_add(&wmbp->wmb_node, &wflp->ws_lhp); 1235 } 1236 } 1237 firsterr = torture_create_kthread(rcu_scale_writer, (void *)i, 1238 writer_tasks[i]); 1239 if (torture_init_error(firsterr)) 1240 goto unwind; 1241 } 1242 torture_init_end(); 1243 return 0; 1244 1245 unwind: 1246 torture_init_end(); 1247 rcu_scale_cleanup(); 1248 if (shutdown_secs) { 1249 WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST)); 1250 kernel_power_off(); 1251 } 1252 return firsterr; 1253 } 1254 1255 module_init(rcu_scale_init); 1256 module_exit(rcu_scale_cleanup); 1257