1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched/signal.h> 36 #include <uapi/linux/sched/types.h> 37 #include <linux/atomic.h> 38 #include <linux/bitops.h> 39 #include <linux/completion.h> 40 #include <linux/moduleparam.h> 41 #include <linux/percpu.h> 42 #include <linux/notifier.h> 43 #include <linux/reboot.h> 44 #include <linux/freezer.h> 45 #include <linux/cpu.h> 46 #include <linux/delay.h> 47 #include <linux/stat.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/trace_clock.h> 51 #include <asm/byteorder.h> 52 #include <linux/torture.h> 53 #include <linux/vmalloc.h> 54 #include <linux/sched/debug.h> 55 56 #include "rcu.h" 57 58 MODULE_LICENSE("GPL"); 59 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 60 61 62 torture_param(int, cbflood_inter_holdoff, HZ, 63 "Holdoff between floods (jiffies)"); 64 torture_param(int, cbflood_intra_holdoff, 1, 65 "Holdoff between bursts (jiffies)"); 66 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 67 torture_param(int, cbflood_n_per_burst, 20000, 68 "# callbacks per burst in flood"); 69 torture_param(int, fqs_duration, 0, 70 "Duration of fqs bursts (us), 0 to disable"); 71 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 72 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 73 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 74 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 75 torture_param(bool, gp_normal, false, 76 "Use normal (non-expedited) GP wait primitives"); 77 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 78 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 79 torture_param(int, n_barrier_cbs, 0, 80 "# of callbacks/kthreads for barrier testing"); 81 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 82 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 83 torture_param(int, object_debug, 0, 84 "Enable debug-object double call_rcu() testing"); 85 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 86 torture_param(int, onoff_interval, 0, 87 "Time between CPU hotplugs (s), 0=disable"); 88 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 89 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 90 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 91 torture_param(int, stall_cpu_holdoff, 10, 92 "Time to wait before starting stall (s)."); 93 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 94 torture_param(int, stat_interval, 60, 95 "Number of seconds between stats printk()s"); 96 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 97 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 98 torture_param(int, test_boost_duration, 4, 99 "Duration of each boost test, seconds."); 100 torture_param(int, test_boost_interval, 7, 101 "Interval between boost tests, seconds."); 102 torture_param(bool, test_no_idle_hz, true, 103 "Test support for tickless idle CPUs"); 104 torture_param(bool, verbose, true, 105 "Enable verbose debugging printk()s"); 106 107 static char *torture_type = "rcu"; 108 module_param(torture_type, charp, 0444); 109 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 110 111 static int nrealreaders; 112 static int ncbflooders; 113 static struct task_struct *writer_task; 114 static struct task_struct **fakewriter_tasks; 115 static struct task_struct **reader_tasks; 116 static struct task_struct *stats_task; 117 static struct task_struct **cbflood_task; 118 static struct task_struct *fqs_task; 119 static struct task_struct *boost_tasks[NR_CPUS]; 120 static struct task_struct *stall_task; 121 static struct task_struct **barrier_cbs_tasks; 122 static struct task_struct *barrier_task; 123 124 #define RCU_TORTURE_PIPE_LEN 10 125 126 struct rcu_torture { 127 struct rcu_head rtort_rcu; 128 int rtort_pipe_count; 129 struct list_head rtort_free; 130 int rtort_mbtest; 131 }; 132 133 static LIST_HEAD(rcu_torture_freelist); 134 static struct rcu_torture __rcu *rcu_torture_current; 135 static unsigned long rcu_torture_current_version; 136 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 137 static DEFINE_SPINLOCK(rcu_torture_lock); 138 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 139 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 140 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 141 static atomic_t n_rcu_torture_alloc; 142 static atomic_t n_rcu_torture_alloc_fail; 143 static atomic_t n_rcu_torture_free; 144 static atomic_t n_rcu_torture_mberror; 145 static atomic_t n_rcu_torture_error; 146 static long n_rcu_torture_barrier_error; 147 static long n_rcu_torture_boost_ktrerror; 148 static long n_rcu_torture_boost_rterror; 149 static long n_rcu_torture_boost_failure; 150 static long n_rcu_torture_boosts; 151 static long n_rcu_torture_timers; 152 static long n_barrier_attempts; 153 static long n_barrier_successes; 154 static atomic_long_t n_cbfloods; 155 static struct list_head rcu_torture_removed; 156 157 static int rcu_torture_writer_state; 158 #define RTWS_FIXED_DELAY 0 159 #define RTWS_DELAY 1 160 #define RTWS_REPLACE 2 161 #define RTWS_DEF_FREE 3 162 #define RTWS_EXP_SYNC 4 163 #define RTWS_COND_GET 5 164 #define RTWS_COND_SYNC 6 165 #define RTWS_SYNC 7 166 #define RTWS_STUTTER 8 167 #define RTWS_STOPPING 9 168 static const char * const rcu_torture_writer_state_names[] = { 169 "RTWS_FIXED_DELAY", 170 "RTWS_DELAY", 171 "RTWS_REPLACE", 172 "RTWS_DEF_FREE", 173 "RTWS_EXP_SYNC", 174 "RTWS_COND_GET", 175 "RTWS_COND_SYNC", 176 "RTWS_SYNC", 177 "RTWS_STUTTER", 178 "RTWS_STOPPING", 179 }; 180 181 static const char *rcu_torture_writer_state_getname(void) 182 { 183 unsigned int i = READ_ONCE(rcu_torture_writer_state); 184 185 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 186 return "???"; 187 return rcu_torture_writer_state_names[i]; 188 } 189 190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 191 #define rcu_can_boost() 1 192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 193 #define rcu_can_boost() 0 194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 195 196 #ifdef CONFIG_RCU_TRACE 197 static u64 notrace rcu_trace_clock_local(void) 198 { 199 u64 ts = trace_clock_local(); 200 201 (void)do_div(ts, NSEC_PER_USEC); 202 return ts; 203 } 204 #else /* #ifdef CONFIG_RCU_TRACE */ 205 static u64 notrace rcu_trace_clock_local(void) 206 { 207 return 0ULL; 208 } 209 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 210 211 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 212 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 213 /* and boost task create/destroy. */ 214 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 215 static bool barrier_phase; /* Test phase. */ 216 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 217 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 218 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 219 220 /* 221 * Allocate an element from the rcu_tortures pool. 222 */ 223 static struct rcu_torture * 224 rcu_torture_alloc(void) 225 { 226 struct list_head *p; 227 228 spin_lock_bh(&rcu_torture_lock); 229 if (list_empty(&rcu_torture_freelist)) { 230 atomic_inc(&n_rcu_torture_alloc_fail); 231 spin_unlock_bh(&rcu_torture_lock); 232 return NULL; 233 } 234 atomic_inc(&n_rcu_torture_alloc); 235 p = rcu_torture_freelist.next; 236 list_del_init(p); 237 spin_unlock_bh(&rcu_torture_lock); 238 return container_of(p, struct rcu_torture, rtort_free); 239 } 240 241 /* 242 * Free an element to the rcu_tortures pool. 243 */ 244 static void 245 rcu_torture_free(struct rcu_torture *p) 246 { 247 atomic_inc(&n_rcu_torture_free); 248 spin_lock_bh(&rcu_torture_lock); 249 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 250 spin_unlock_bh(&rcu_torture_lock); 251 } 252 253 /* 254 * Operations vector for selecting different types of tests. 255 */ 256 257 struct rcu_torture_ops { 258 int ttype; 259 void (*init)(void); 260 void (*cleanup)(void); 261 int (*readlock)(void); 262 void (*read_delay)(struct torture_random_state *rrsp); 263 void (*readunlock)(int idx); 264 unsigned long (*started)(void); 265 unsigned long (*completed)(void); 266 void (*deferred_free)(struct rcu_torture *p); 267 void (*sync)(void); 268 void (*exp_sync)(void); 269 unsigned long (*get_state)(void); 270 void (*cond_sync)(unsigned long oldstate); 271 call_rcu_func_t call; 272 void (*cb_barrier)(void); 273 void (*fqs)(void); 274 void (*stats)(void); 275 int irq_capable; 276 int can_boost; 277 const char *name; 278 }; 279 280 static struct rcu_torture_ops *cur_ops; 281 282 /* 283 * Definitions for rcu torture testing. 284 */ 285 286 static int rcu_torture_read_lock(void) __acquires(RCU) 287 { 288 rcu_read_lock(); 289 return 0; 290 } 291 292 static void rcu_read_delay(struct torture_random_state *rrsp) 293 { 294 unsigned long started; 295 unsigned long completed; 296 const unsigned long shortdelay_us = 200; 297 const unsigned long longdelay_ms = 50; 298 unsigned long long ts; 299 300 /* We want a short delay sometimes to make a reader delay the grace 301 * period, and we want a long delay occasionally to trigger 302 * force_quiescent_state. */ 303 304 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 305 started = cur_ops->completed(); 306 ts = rcu_trace_clock_local(); 307 mdelay(longdelay_ms); 308 completed = cur_ops->completed(); 309 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 310 started, completed); 311 } 312 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 313 udelay(shortdelay_us); 314 if (!preempt_count() && 315 !(torture_random(rrsp) % (nrealreaders * 500))) 316 torture_preempt_schedule(); /* QS only if preemptible. */ 317 } 318 319 static void rcu_torture_read_unlock(int idx) __releases(RCU) 320 { 321 rcu_read_unlock(); 322 } 323 324 /* 325 * Update callback in the pipe. This should be invoked after a grace period. 326 */ 327 static bool 328 rcu_torture_pipe_update_one(struct rcu_torture *rp) 329 { 330 int i; 331 332 i = rp->rtort_pipe_count; 333 if (i > RCU_TORTURE_PIPE_LEN) 334 i = RCU_TORTURE_PIPE_LEN; 335 atomic_inc(&rcu_torture_wcount[i]); 336 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 337 rp->rtort_mbtest = 0; 338 return true; 339 } 340 return false; 341 } 342 343 /* 344 * Update all callbacks in the pipe. Suitable for synchronous grace-period 345 * primitives. 346 */ 347 static void 348 rcu_torture_pipe_update(struct rcu_torture *old_rp) 349 { 350 struct rcu_torture *rp; 351 struct rcu_torture *rp1; 352 353 if (old_rp) 354 list_add(&old_rp->rtort_free, &rcu_torture_removed); 355 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 356 if (rcu_torture_pipe_update_one(rp)) { 357 list_del(&rp->rtort_free); 358 rcu_torture_free(rp); 359 } 360 } 361 } 362 363 static void 364 rcu_torture_cb(struct rcu_head *p) 365 { 366 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 367 368 if (torture_must_stop_irq()) { 369 /* Test is ending, just drop callbacks on the floor. */ 370 /* The next initialization will pick up the pieces. */ 371 return; 372 } 373 if (rcu_torture_pipe_update_one(rp)) 374 rcu_torture_free(rp); 375 else 376 cur_ops->deferred_free(rp); 377 } 378 379 static unsigned long rcu_no_completed(void) 380 { 381 return 0; 382 } 383 384 static void rcu_torture_deferred_free(struct rcu_torture *p) 385 { 386 call_rcu(&p->rtort_rcu, rcu_torture_cb); 387 } 388 389 static void rcu_sync_torture_init(void) 390 { 391 INIT_LIST_HEAD(&rcu_torture_removed); 392 } 393 394 static struct rcu_torture_ops rcu_ops = { 395 .ttype = RCU_FLAVOR, 396 .init = rcu_sync_torture_init, 397 .readlock = rcu_torture_read_lock, 398 .read_delay = rcu_read_delay, 399 .readunlock = rcu_torture_read_unlock, 400 .started = rcu_batches_started, 401 .completed = rcu_batches_completed, 402 .deferred_free = rcu_torture_deferred_free, 403 .sync = synchronize_rcu, 404 .exp_sync = synchronize_rcu_expedited, 405 .get_state = get_state_synchronize_rcu, 406 .cond_sync = cond_synchronize_rcu, 407 .call = call_rcu, 408 .cb_barrier = rcu_barrier, 409 .fqs = rcu_force_quiescent_state, 410 .stats = NULL, 411 .irq_capable = 1, 412 .can_boost = rcu_can_boost(), 413 .name = "rcu" 414 }; 415 416 /* 417 * Definitions for rcu_bh torture testing. 418 */ 419 420 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 421 { 422 rcu_read_lock_bh(); 423 return 0; 424 } 425 426 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 427 { 428 rcu_read_unlock_bh(); 429 } 430 431 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static struct rcu_torture_ops rcu_bh_ops = { 437 .ttype = RCU_BH_FLAVOR, 438 .init = rcu_sync_torture_init, 439 .readlock = rcu_bh_torture_read_lock, 440 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 441 .readunlock = rcu_bh_torture_read_unlock, 442 .started = rcu_batches_started_bh, 443 .completed = rcu_batches_completed_bh, 444 .deferred_free = rcu_bh_torture_deferred_free, 445 .sync = synchronize_rcu_bh, 446 .exp_sync = synchronize_rcu_bh_expedited, 447 .call = call_rcu_bh, 448 .cb_barrier = rcu_barrier_bh, 449 .fqs = rcu_bh_force_quiescent_state, 450 .stats = NULL, 451 .irq_capable = 1, 452 .name = "rcu_bh" 453 }; 454 455 /* 456 * Don't even think about trying any of these in real life!!! 457 * The names includes "busted", and they really means it! 458 * The only purpose of these functions is to provide a buggy RCU 459 * implementation to make sure that rcutorture correctly emits 460 * buggy-RCU error messages. 461 */ 462 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 463 { 464 /* This is a deliberate bug for testing purposes only! */ 465 rcu_torture_cb(&p->rtort_rcu); 466 } 467 468 static void synchronize_rcu_busted(void) 469 { 470 /* This is a deliberate bug for testing purposes only! */ 471 } 472 473 static void 474 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 475 { 476 /* This is a deliberate bug for testing purposes only! */ 477 func(head); 478 } 479 480 static struct rcu_torture_ops rcu_busted_ops = { 481 .ttype = INVALID_RCU_FLAVOR, 482 .init = rcu_sync_torture_init, 483 .readlock = rcu_torture_read_lock, 484 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 485 .readunlock = rcu_torture_read_unlock, 486 .started = rcu_no_completed, 487 .completed = rcu_no_completed, 488 .deferred_free = rcu_busted_torture_deferred_free, 489 .sync = synchronize_rcu_busted, 490 .exp_sync = synchronize_rcu_busted, 491 .call = call_rcu_busted, 492 .cb_barrier = NULL, 493 .fqs = NULL, 494 .stats = NULL, 495 .irq_capable = 1, 496 .name = "busted" 497 }; 498 499 /* 500 * Definitions for srcu torture testing. 501 */ 502 503 DEFINE_STATIC_SRCU(srcu_ctl); 504 static struct srcu_struct srcu_ctld; 505 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 506 507 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 508 { 509 return srcu_read_lock(srcu_ctlp); 510 } 511 512 static void srcu_read_delay(struct torture_random_state *rrsp) 513 { 514 long delay; 515 const long uspertick = 1000000 / HZ; 516 const long longdelay = 10; 517 518 /* We want there to be long-running readers, but not all the time. */ 519 520 delay = torture_random(rrsp) % 521 (nrealreaders * 2 * longdelay * uspertick); 522 if (!delay && in_task()) 523 schedule_timeout_interruptible(longdelay); 524 else 525 rcu_read_delay(rrsp); 526 } 527 528 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 529 { 530 srcu_read_unlock(srcu_ctlp, idx); 531 } 532 533 static unsigned long srcu_torture_completed(void) 534 { 535 return srcu_batches_completed(srcu_ctlp); 536 } 537 538 static void srcu_torture_deferred_free(struct rcu_torture *rp) 539 { 540 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 541 } 542 543 static void srcu_torture_synchronize(void) 544 { 545 synchronize_srcu(srcu_ctlp); 546 } 547 548 static void srcu_torture_call(struct rcu_head *head, 549 rcu_callback_t func) 550 { 551 call_srcu(srcu_ctlp, head, func); 552 } 553 554 static void srcu_torture_barrier(void) 555 { 556 srcu_barrier(srcu_ctlp); 557 } 558 559 static void srcu_torture_stats(void) 560 { 561 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 562 } 563 564 static void srcu_torture_synchronize_expedited(void) 565 { 566 synchronize_srcu_expedited(srcu_ctlp); 567 } 568 569 static struct rcu_torture_ops srcu_ops = { 570 .ttype = SRCU_FLAVOR, 571 .init = rcu_sync_torture_init, 572 .readlock = srcu_torture_read_lock, 573 .read_delay = srcu_read_delay, 574 .readunlock = srcu_torture_read_unlock, 575 .started = NULL, 576 .completed = srcu_torture_completed, 577 .deferred_free = srcu_torture_deferred_free, 578 .sync = srcu_torture_synchronize, 579 .exp_sync = srcu_torture_synchronize_expedited, 580 .call = srcu_torture_call, 581 .cb_barrier = srcu_torture_barrier, 582 .stats = srcu_torture_stats, 583 .irq_capable = 1, 584 .name = "srcu" 585 }; 586 587 static void srcu_torture_init(void) 588 { 589 rcu_sync_torture_init(); 590 WARN_ON(init_srcu_struct(&srcu_ctld)); 591 srcu_ctlp = &srcu_ctld; 592 } 593 594 static void srcu_torture_cleanup(void) 595 { 596 cleanup_srcu_struct(&srcu_ctld); 597 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 598 } 599 600 /* As above, but dynamically allocated. */ 601 static struct rcu_torture_ops srcud_ops = { 602 .ttype = SRCU_FLAVOR, 603 .init = srcu_torture_init, 604 .cleanup = srcu_torture_cleanup, 605 .readlock = srcu_torture_read_lock, 606 .read_delay = srcu_read_delay, 607 .readunlock = srcu_torture_read_unlock, 608 .started = NULL, 609 .completed = srcu_torture_completed, 610 .deferred_free = srcu_torture_deferred_free, 611 .sync = srcu_torture_synchronize, 612 .exp_sync = srcu_torture_synchronize_expedited, 613 .call = srcu_torture_call, 614 .cb_barrier = srcu_torture_barrier, 615 .stats = srcu_torture_stats, 616 .irq_capable = 1, 617 .name = "srcud" 618 }; 619 620 /* 621 * Definitions for sched torture testing. 622 */ 623 624 static int sched_torture_read_lock(void) 625 { 626 preempt_disable(); 627 return 0; 628 } 629 630 static void sched_torture_read_unlock(int idx) 631 { 632 preempt_enable(); 633 } 634 635 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 636 { 637 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 638 } 639 640 static struct rcu_torture_ops sched_ops = { 641 .ttype = RCU_SCHED_FLAVOR, 642 .init = rcu_sync_torture_init, 643 .readlock = sched_torture_read_lock, 644 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 645 .readunlock = sched_torture_read_unlock, 646 .started = rcu_batches_started_sched, 647 .completed = rcu_batches_completed_sched, 648 .deferred_free = rcu_sched_torture_deferred_free, 649 .sync = synchronize_sched, 650 .exp_sync = synchronize_sched_expedited, 651 .get_state = get_state_synchronize_sched, 652 .cond_sync = cond_synchronize_sched, 653 .call = call_rcu_sched, 654 .cb_barrier = rcu_barrier_sched, 655 .fqs = rcu_sched_force_quiescent_state, 656 .stats = NULL, 657 .irq_capable = 1, 658 .name = "sched" 659 }; 660 661 /* 662 * Definitions for RCU-tasks torture testing. 663 */ 664 665 static int tasks_torture_read_lock(void) 666 { 667 return 0; 668 } 669 670 static void tasks_torture_read_unlock(int idx) 671 { 672 } 673 674 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 675 { 676 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 677 } 678 679 static struct rcu_torture_ops tasks_ops = { 680 .ttype = RCU_TASKS_FLAVOR, 681 .init = rcu_sync_torture_init, 682 .readlock = tasks_torture_read_lock, 683 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 684 .readunlock = tasks_torture_read_unlock, 685 .started = rcu_no_completed, 686 .completed = rcu_no_completed, 687 .deferred_free = rcu_tasks_torture_deferred_free, 688 .sync = synchronize_rcu_tasks, 689 .exp_sync = synchronize_rcu_tasks, 690 .call = call_rcu_tasks, 691 .cb_barrier = rcu_barrier_tasks, 692 .fqs = NULL, 693 .stats = NULL, 694 .irq_capable = 1, 695 .name = "tasks" 696 }; 697 698 static bool __maybe_unused torturing_tasks(void) 699 { 700 return cur_ops == &tasks_ops; 701 } 702 703 /* 704 * RCU torture priority-boost testing. Runs one real-time thread per 705 * CPU for moderate bursts, repeatedly registering RCU callbacks and 706 * spinning waiting for them to be invoked. If a given callback takes 707 * too long to be invoked, we assume that priority inversion has occurred. 708 */ 709 710 struct rcu_boost_inflight { 711 struct rcu_head rcu; 712 int inflight; 713 }; 714 715 static void rcu_torture_boost_cb(struct rcu_head *head) 716 { 717 struct rcu_boost_inflight *rbip = 718 container_of(head, struct rcu_boost_inflight, rcu); 719 720 /* Ensure RCU-core accesses precede clearing ->inflight */ 721 smp_store_release(&rbip->inflight, 0); 722 } 723 724 static int rcu_torture_boost(void *arg) 725 { 726 unsigned long call_rcu_time; 727 unsigned long endtime; 728 unsigned long oldstarttime; 729 struct rcu_boost_inflight rbi = { .inflight = 0 }; 730 struct sched_param sp; 731 732 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 733 734 /* Set real-time priority. */ 735 sp.sched_priority = 1; 736 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 737 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 738 n_rcu_torture_boost_rterror++; 739 } 740 741 init_rcu_head_on_stack(&rbi.rcu); 742 /* Each pass through the following loop does one boost-test cycle. */ 743 do { 744 /* Wait for the next test interval. */ 745 oldstarttime = boost_starttime; 746 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 747 schedule_timeout_interruptible(oldstarttime - jiffies); 748 stutter_wait("rcu_torture_boost"); 749 if (torture_must_stop()) 750 goto checkwait; 751 } 752 753 /* Do one boost-test interval. */ 754 endtime = oldstarttime + test_boost_duration * HZ; 755 call_rcu_time = jiffies; 756 while (ULONG_CMP_LT(jiffies, endtime)) { 757 /* If we don't have a callback in flight, post one. */ 758 if (!smp_load_acquire(&rbi.inflight)) { 759 /* RCU core before ->inflight = 1. */ 760 smp_store_release(&rbi.inflight, 1); 761 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 762 if (jiffies - call_rcu_time > 763 test_boost_duration * HZ - HZ / 2) { 764 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 765 n_rcu_torture_boost_failure++; 766 } 767 call_rcu_time = jiffies; 768 } 769 stutter_wait("rcu_torture_boost"); 770 if (torture_must_stop()) 771 goto checkwait; 772 } 773 774 /* 775 * Set the start time of the next test interval. 776 * Yes, this is vulnerable to long delays, but such 777 * delays simply cause a false negative for the next 778 * interval. Besides, we are running at RT priority, 779 * so delays should be relatively rare. 780 */ 781 while (oldstarttime == boost_starttime && 782 !kthread_should_stop()) { 783 if (mutex_trylock(&boost_mutex)) { 784 boost_starttime = jiffies + 785 test_boost_interval * HZ; 786 n_rcu_torture_boosts++; 787 mutex_unlock(&boost_mutex); 788 break; 789 } 790 schedule_timeout_uninterruptible(1); 791 } 792 793 /* Go do the stutter. */ 794 checkwait: stutter_wait("rcu_torture_boost"); 795 } while (!torture_must_stop()); 796 797 /* Clean up and exit. */ 798 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 799 torture_shutdown_absorb("rcu_torture_boost"); 800 schedule_timeout_uninterruptible(1); 801 } 802 destroy_rcu_head_on_stack(&rbi.rcu); 803 torture_kthread_stopping("rcu_torture_boost"); 804 return 0; 805 } 806 807 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 808 { 809 } 810 811 /* 812 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 813 * to call_rcu() or analogous, increasing the probability of occurrence 814 * of callback-overflow corner cases. 815 */ 816 static int 817 rcu_torture_cbflood(void *arg) 818 { 819 int err = 1; 820 int i; 821 int j; 822 struct rcu_head *rhp; 823 824 if (cbflood_n_per_burst > 0 && 825 cbflood_inter_holdoff > 0 && 826 cbflood_intra_holdoff > 0 && 827 cur_ops->call && 828 cur_ops->cb_barrier) { 829 rhp = vmalloc(sizeof(*rhp) * 830 cbflood_n_burst * cbflood_n_per_burst); 831 err = !rhp; 832 } 833 if (err) { 834 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 835 goto wait_for_stop; 836 } 837 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 838 do { 839 schedule_timeout_interruptible(cbflood_inter_holdoff); 840 atomic_long_inc(&n_cbfloods); 841 WARN_ON(signal_pending(current)); 842 for (i = 0; i < cbflood_n_burst; i++) { 843 for (j = 0; j < cbflood_n_per_burst; j++) { 844 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 845 rcu_torture_cbflood_cb); 846 } 847 schedule_timeout_interruptible(cbflood_intra_holdoff); 848 WARN_ON(signal_pending(current)); 849 } 850 cur_ops->cb_barrier(); 851 stutter_wait("rcu_torture_cbflood"); 852 } while (!torture_must_stop()); 853 vfree(rhp); 854 wait_for_stop: 855 torture_kthread_stopping("rcu_torture_cbflood"); 856 return 0; 857 } 858 859 /* 860 * RCU torture force-quiescent-state kthread. Repeatedly induces 861 * bursts of calls to force_quiescent_state(), increasing the probability 862 * of occurrence of some important types of race conditions. 863 */ 864 static int 865 rcu_torture_fqs(void *arg) 866 { 867 unsigned long fqs_resume_time; 868 int fqs_burst_remaining; 869 870 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 871 do { 872 fqs_resume_time = jiffies + fqs_stutter * HZ; 873 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 874 !kthread_should_stop()) { 875 schedule_timeout_interruptible(1); 876 } 877 fqs_burst_remaining = fqs_duration; 878 while (fqs_burst_remaining > 0 && 879 !kthread_should_stop()) { 880 cur_ops->fqs(); 881 udelay(fqs_holdoff); 882 fqs_burst_remaining -= fqs_holdoff; 883 } 884 stutter_wait("rcu_torture_fqs"); 885 } while (!torture_must_stop()); 886 torture_kthread_stopping("rcu_torture_fqs"); 887 return 0; 888 } 889 890 /* 891 * RCU torture writer kthread. Repeatedly substitutes a new structure 892 * for that pointed to by rcu_torture_current, freeing the old structure 893 * after a series of grace periods (the "pipeline"). 894 */ 895 static int 896 rcu_torture_writer(void *arg) 897 { 898 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 899 int expediting = 0; 900 unsigned long gp_snap; 901 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 902 bool gp_sync1 = gp_sync; 903 int i; 904 struct rcu_torture *rp; 905 struct rcu_torture *old_rp; 906 static DEFINE_TORTURE_RANDOM(rand); 907 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 908 RTWS_COND_GET, RTWS_SYNC }; 909 int nsynctypes = 0; 910 911 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 912 if (!can_expedite) 913 pr_alert("%s" TORTURE_FLAG 914 " GP expediting controlled from boot/sysfs for %s.\n", 915 torture_type, cur_ops->name); 916 917 /* Initialize synctype[] array. If none set, take default. */ 918 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 919 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 920 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) { 921 synctype[nsynctypes++] = RTWS_COND_GET; 922 pr_info("%s: Testing conditional GPs.\n", __func__); 923 } else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) { 924 pr_alert("%s: gp_cond without primitives.\n", __func__); 925 } 926 if (gp_exp1 && cur_ops->exp_sync) { 927 synctype[nsynctypes++] = RTWS_EXP_SYNC; 928 pr_info("%s: Testing expedited GPs.\n", __func__); 929 } else if (gp_exp && !cur_ops->exp_sync) { 930 pr_alert("%s: gp_exp without primitives.\n", __func__); 931 } 932 if (gp_normal1 && cur_ops->deferred_free) { 933 synctype[nsynctypes++] = RTWS_DEF_FREE; 934 pr_info("%s: Testing asynchronous GPs.\n", __func__); 935 } else if (gp_normal && !cur_ops->deferred_free) { 936 pr_alert("%s: gp_normal without primitives.\n", __func__); 937 } 938 if (gp_sync1 && cur_ops->sync) { 939 synctype[nsynctypes++] = RTWS_SYNC; 940 pr_info("%s: Testing normal GPs.\n", __func__); 941 } else if (gp_sync && !cur_ops->sync) { 942 pr_alert("%s: gp_sync without primitives.\n", __func__); 943 } 944 if (WARN_ONCE(nsynctypes == 0, 945 "rcu_torture_writer: No update-side primitives.\n")) { 946 /* 947 * No updates primitives, so don't try updating. 948 * The resulting test won't be testing much, hence the 949 * above WARN_ONCE(). 950 */ 951 rcu_torture_writer_state = RTWS_STOPPING; 952 torture_kthread_stopping("rcu_torture_writer"); 953 } 954 955 do { 956 rcu_torture_writer_state = RTWS_FIXED_DELAY; 957 schedule_timeout_uninterruptible(1); 958 rp = rcu_torture_alloc(); 959 if (rp == NULL) 960 continue; 961 rp->rtort_pipe_count = 0; 962 rcu_torture_writer_state = RTWS_DELAY; 963 udelay(torture_random(&rand) & 0x3ff); 964 rcu_torture_writer_state = RTWS_REPLACE; 965 old_rp = rcu_dereference_check(rcu_torture_current, 966 current == writer_task); 967 rp->rtort_mbtest = 1; 968 rcu_assign_pointer(rcu_torture_current, rp); 969 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 970 if (old_rp) { 971 i = old_rp->rtort_pipe_count; 972 if (i > RCU_TORTURE_PIPE_LEN) 973 i = RCU_TORTURE_PIPE_LEN; 974 atomic_inc(&rcu_torture_wcount[i]); 975 old_rp->rtort_pipe_count++; 976 switch (synctype[torture_random(&rand) % nsynctypes]) { 977 case RTWS_DEF_FREE: 978 rcu_torture_writer_state = RTWS_DEF_FREE; 979 cur_ops->deferred_free(old_rp); 980 break; 981 case RTWS_EXP_SYNC: 982 rcu_torture_writer_state = RTWS_EXP_SYNC; 983 cur_ops->exp_sync(); 984 rcu_torture_pipe_update(old_rp); 985 break; 986 case RTWS_COND_GET: 987 rcu_torture_writer_state = RTWS_COND_GET; 988 gp_snap = cur_ops->get_state(); 989 i = torture_random(&rand) % 16; 990 if (i != 0) 991 schedule_timeout_interruptible(i); 992 udelay(torture_random(&rand) % 1000); 993 rcu_torture_writer_state = RTWS_COND_SYNC; 994 cur_ops->cond_sync(gp_snap); 995 rcu_torture_pipe_update(old_rp); 996 break; 997 case RTWS_SYNC: 998 rcu_torture_writer_state = RTWS_SYNC; 999 cur_ops->sync(); 1000 rcu_torture_pipe_update(old_rp); 1001 break; 1002 default: 1003 WARN_ON_ONCE(1); 1004 break; 1005 } 1006 } 1007 rcutorture_record_progress(++rcu_torture_current_version); 1008 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1009 if (can_expedite && 1010 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1011 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1012 if (expediting >= 0) 1013 rcu_expedite_gp(); 1014 else 1015 rcu_unexpedite_gp(); 1016 if (++expediting > 3) 1017 expediting = -expediting; 1018 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1019 can_expedite = !rcu_gp_is_expedited() && 1020 !rcu_gp_is_normal(); 1021 } 1022 rcu_torture_writer_state = RTWS_STUTTER; 1023 stutter_wait("rcu_torture_writer"); 1024 } while (!torture_must_stop()); 1025 /* Reset expediting back to unexpedited. */ 1026 if (expediting > 0) 1027 expediting = -expediting; 1028 while (can_expedite && expediting++ < 0) 1029 rcu_unexpedite_gp(); 1030 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1031 if (!can_expedite) 1032 pr_alert("%s" TORTURE_FLAG 1033 " Dynamic grace-period expediting was disabled.\n", 1034 torture_type); 1035 rcu_torture_writer_state = RTWS_STOPPING; 1036 torture_kthread_stopping("rcu_torture_writer"); 1037 return 0; 1038 } 1039 1040 /* 1041 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1042 * delay between calls. 1043 */ 1044 static int 1045 rcu_torture_fakewriter(void *arg) 1046 { 1047 DEFINE_TORTURE_RANDOM(rand); 1048 1049 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1050 set_user_nice(current, MAX_NICE); 1051 1052 do { 1053 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1054 udelay(torture_random(&rand) & 0x3ff); 1055 if (cur_ops->cb_barrier != NULL && 1056 torture_random(&rand) % (nfakewriters * 8) == 0) { 1057 cur_ops->cb_barrier(); 1058 } else if (gp_normal == gp_exp) { 1059 if (cur_ops->sync && torture_random(&rand) & 0x80) 1060 cur_ops->sync(); 1061 else if (cur_ops->exp_sync) 1062 cur_ops->exp_sync(); 1063 } else if (gp_normal && cur_ops->sync) { 1064 cur_ops->sync(); 1065 } else if (cur_ops->exp_sync) { 1066 cur_ops->exp_sync(); 1067 } 1068 stutter_wait("rcu_torture_fakewriter"); 1069 } while (!torture_must_stop()); 1070 1071 torture_kthread_stopping("rcu_torture_fakewriter"); 1072 return 0; 1073 } 1074 1075 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1076 { 1077 kfree(rhp); 1078 } 1079 1080 /* 1081 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1082 * incrementing the corresponding element of the pipeline array. The 1083 * counter in the element should never be greater than 1, otherwise, the 1084 * RCU implementation is broken. 1085 */ 1086 static void rcu_torture_timer(struct timer_list *unused) 1087 { 1088 int idx; 1089 unsigned long started; 1090 unsigned long completed; 1091 static DEFINE_TORTURE_RANDOM(rand); 1092 static DEFINE_SPINLOCK(rand_lock); 1093 struct rcu_torture *p; 1094 int pipe_count; 1095 unsigned long long ts; 1096 1097 idx = cur_ops->readlock(); 1098 if (cur_ops->started) 1099 started = cur_ops->started(); 1100 else 1101 started = cur_ops->completed(); 1102 ts = rcu_trace_clock_local(); 1103 p = rcu_dereference_check(rcu_torture_current, 1104 rcu_read_lock_bh_held() || 1105 rcu_read_lock_sched_held() || 1106 srcu_read_lock_held(srcu_ctlp) || 1107 torturing_tasks()); 1108 if (p == NULL) { 1109 /* Leave because rcu_torture_writer is not yet underway */ 1110 cur_ops->readunlock(idx); 1111 return; 1112 } 1113 if (p->rtort_mbtest == 0) 1114 atomic_inc(&n_rcu_torture_mberror); 1115 spin_lock(&rand_lock); 1116 cur_ops->read_delay(&rand); 1117 n_rcu_torture_timers++; 1118 spin_unlock(&rand_lock); 1119 preempt_disable(); 1120 pipe_count = p->rtort_pipe_count; 1121 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1122 /* Should not happen, but... */ 1123 pipe_count = RCU_TORTURE_PIPE_LEN; 1124 } 1125 completed = cur_ops->completed(); 1126 if (pipe_count > 1) { 1127 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1128 started, completed); 1129 rcu_ftrace_dump(DUMP_ALL); 1130 } 1131 __this_cpu_inc(rcu_torture_count[pipe_count]); 1132 completed = completed - started; 1133 if (cur_ops->started) 1134 completed++; 1135 if (completed > RCU_TORTURE_PIPE_LEN) { 1136 /* Should not happen, but... */ 1137 completed = RCU_TORTURE_PIPE_LEN; 1138 } 1139 __this_cpu_inc(rcu_torture_batch[completed]); 1140 preempt_enable(); 1141 cur_ops->readunlock(idx); 1142 1143 /* Test call_rcu() invocation from interrupt handler. */ 1144 if (cur_ops->call) { 1145 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1146 1147 if (rhp) 1148 cur_ops->call(rhp, rcu_torture_timer_cb); 1149 } 1150 } 1151 1152 /* 1153 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1154 * incrementing the corresponding element of the pipeline array. The 1155 * counter in the element should never be greater than 1, otherwise, the 1156 * RCU implementation is broken. 1157 */ 1158 static int 1159 rcu_torture_reader(void *arg) 1160 { 1161 unsigned long started; 1162 unsigned long completed; 1163 int idx; 1164 DEFINE_TORTURE_RANDOM(rand); 1165 struct rcu_torture *p; 1166 int pipe_count; 1167 struct timer_list t; 1168 unsigned long long ts; 1169 1170 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1171 set_user_nice(current, MAX_NICE); 1172 if (irqreader && cur_ops->irq_capable) 1173 timer_setup_on_stack(&t, rcu_torture_timer, 0); 1174 1175 do { 1176 if (irqreader && cur_ops->irq_capable) { 1177 if (!timer_pending(&t)) 1178 mod_timer(&t, jiffies + 1); 1179 } 1180 idx = cur_ops->readlock(); 1181 if (cur_ops->started) 1182 started = cur_ops->started(); 1183 else 1184 started = cur_ops->completed(); 1185 ts = rcu_trace_clock_local(); 1186 p = rcu_dereference_check(rcu_torture_current, 1187 rcu_read_lock_bh_held() || 1188 rcu_read_lock_sched_held() || 1189 srcu_read_lock_held(srcu_ctlp) || 1190 torturing_tasks()); 1191 if (p == NULL) { 1192 /* Wait for rcu_torture_writer to get underway */ 1193 cur_ops->readunlock(idx); 1194 schedule_timeout_interruptible(HZ); 1195 continue; 1196 } 1197 if (p->rtort_mbtest == 0) 1198 atomic_inc(&n_rcu_torture_mberror); 1199 cur_ops->read_delay(&rand); 1200 preempt_disable(); 1201 pipe_count = p->rtort_pipe_count; 1202 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1203 /* Should not happen, but... */ 1204 pipe_count = RCU_TORTURE_PIPE_LEN; 1205 } 1206 completed = cur_ops->completed(); 1207 if (pipe_count > 1) { 1208 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1209 ts, started, completed); 1210 rcu_ftrace_dump(DUMP_ALL); 1211 } 1212 __this_cpu_inc(rcu_torture_count[pipe_count]); 1213 completed = completed - started; 1214 if (cur_ops->started) 1215 completed++; 1216 if (completed > RCU_TORTURE_PIPE_LEN) { 1217 /* Should not happen, but... */ 1218 completed = RCU_TORTURE_PIPE_LEN; 1219 } 1220 __this_cpu_inc(rcu_torture_batch[completed]); 1221 preempt_enable(); 1222 cur_ops->readunlock(idx); 1223 stutter_wait("rcu_torture_reader"); 1224 } while (!torture_must_stop()); 1225 if (irqreader && cur_ops->irq_capable) { 1226 del_timer_sync(&t); 1227 destroy_timer_on_stack(&t); 1228 } 1229 torture_kthread_stopping("rcu_torture_reader"); 1230 return 0; 1231 } 1232 1233 /* 1234 * Print torture statistics. Caller must ensure that there is only 1235 * one call to this function at a given time!!! This is normally 1236 * accomplished by relying on the module system to only have one copy 1237 * of the module loaded, and then by giving the rcu_torture_stats 1238 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1239 * thread is not running). 1240 */ 1241 static void 1242 rcu_torture_stats_print(void) 1243 { 1244 int cpu; 1245 int i; 1246 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1247 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1248 static unsigned long rtcv_snap = ULONG_MAX; 1249 static bool splatted; 1250 struct task_struct *wtp; 1251 1252 for_each_possible_cpu(cpu) { 1253 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1254 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1255 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1256 } 1257 } 1258 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1259 if (pipesummary[i] != 0) 1260 break; 1261 } 1262 1263 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1264 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1265 rcu_torture_current, 1266 rcu_torture_current_version, 1267 list_empty(&rcu_torture_freelist), 1268 atomic_read(&n_rcu_torture_alloc), 1269 atomic_read(&n_rcu_torture_alloc_fail), 1270 atomic_read(&n_rcu_torture_free)); 1271 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1272 atomic_read(&n_rcu_torture_mberror), 1273 n_rcu_torture_barrier_error, 1274 n_rcu_torture_boost_ktrerror, 1275 n_rcu_torture_boost_rterror); 1276 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1277 n_rcu_torture_boost_failure, 1278 n_rcu_torture_boosts, 1279 n_rcu_torture_timers); 1280 torture_onoff_stats(); 1281 pr_cont("barrier: %ld/%ld:%ld ", 1282 n_barrier_successes, 1283 n_barrier_attempts, 1284 n_rcu_torture_barrier_error); 1285 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1286 1287 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1288 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1289 n_rcu_torture_barrier_error != 0 || 1290 n_rcu_torture_boost_ktrerror != 0 || 1291 n_rcu_torture_boost_rterror != 0 || 1292 n_rcu_torture_boost_failure != 0 || 1293 i > 1) { 1294 pr_cont("%s", "!!! "); 1295 atomic_inc(&n_rcu_torture_error); 1296 WARN_ON_ONCE(1); 1297 } 1298 pr_cont("Reader Pipe: "); 1299 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1300 pr_cont(" %ld", pipesummary[i]); 1301 pr_cont("\n"); 1302 1303 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1304 pr_cont("Reader Batch: "); 1305 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1306 pr_cont(" %ld", batchsummary[i]); 1307 pr_cont("\n"); 1308 1309 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1310 pr_cont("Free-Block Circulation: "); 1311 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1312 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1313 } 1314 pr_cont("\n"); 1315 1316 if (cur_ops->stats) 1317 cur_ops->stats(); 1318 if (rtcv_snap == rcu_torture_current_version && 1319 rcu_torture_current != NULL) { 1320 int __maybe_unused flags = 0; 1321 unsigned long __maybe_unused gpnum = 0; 1322 unsigned long __maybe_unused completed = 0; 1323 1324 rcutorture_get_gp_data(cur_ops->ttype, 1325 &flags, &gpnum, &completed); 1326 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1327 &flags, &gpnum, &completed); 1328 wtp = READ_ONCE(writer_task); 1329 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx cpu %d\n", 1330 rcu_torture_writer_state_getname(), 1331 rcu_torture_writer_state, 1332 gpnum, completed, flags, 1333 wtp == NULL ? ~0UL : wtp->state, 1334 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1335 if (!splatted && wtp) { 1336 sched_show_task(wtp); 1337 splatted = true; 1338 } 1339 show_rcu_gp_kthreads(); 1340 rcu_ftrace_dump(DUMP_ALL); 1341 } 1342 rtcv_snap = rcu_torture_current_version; 1343 } 1344 1345 /* 1346 * Periodically prints torture statistics, if periodic statistics printing 1347 * was specified via the stat_interval module parameter. 1348 */ 1349 static int 1350 rcu_torture_stats(void *arg) 1351 { 1352 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1353 do { 1354 schedule_timeout_interruptible(stat_interval * HZ); 1355 rcu_torture_stats_print(); 1356 torture_shutdown_absorb("rcu_torture_stats"); 1357 } while (!torture_must_stop()); 1358 torture_kthread_stopping("rcu_torture_stats"); 1359 return 0; 1360 } 1361 1362 static inline void 1363 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1364 { 1365 pr_alert("%s" TORTURE_FLAG 1366 "--- %s: nreaders=%d nfakewriters=%d " 1367 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1368 "shuffle_interval=%d stutter=%d irqreader=%d " 1369 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1370 "test_boost=%d/%d test_boost_interval=%d " 1371 "test_boost_duration=%d shutdown_secs=%d " 1372 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 1373 "n_barrier_cbs=%d " 1374 "onoff_interval=%d onoff_holdoff=%d\n", 1375 torture_type, tag, nrealreaders, nfakewriters, 1376 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1377 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1378 test_boost, cur_ops->can_boost, 1379 test_boost_interval, test_boost_duration, shutdown_secs, 1380 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 1381 n_barrier_cbs, 1382 onoff_interval, onoff_holdoff); 1383 } 1384 1385 static int rcutorture_booster_cleanup(unsigned int cpu) 1386 { 1387 struct task_struct *t; 1388 1389 if (boost_tasks[cpu] == NULL) 1390 return 0; 1391 mutex_lock(&boost_mutex); 1392 t = boost_tasks[cpu]; 1393 boost_tasks[cpu] = NULL; 1394 mutex_unlock(&boost_mutex); 1395 1396 /* This must be outside of the mutex, otherwise deadlock! */ 1397 torture_stop_kthread(rcu_torture_boost, t); 1398 return 0; 1399 } 1400 1401 static int rcutorture_booster_init(unsigned int cpu) 1402 { 1403 int retval; 1404 1405 if (boost_tasks[cpu] != NULL) 1406 return 0; /* Already created, nothing more to do. */ 1407 1408 /* Don't allow time recalculation while creating a new task. */ 1409 mutex_lock(&boost_mutex); 1410 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1411 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1412 cpu_to_node(cpu), 1413 "rcu_torture_boost"); 1414 if (IS_ERR(boost_tasks[cpu])) { 1415 retval = PTR_ERR(boost_tasks[cpu]); 1416 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1417 n_rcu_torture_boost_ktrerror++; 1418 boost_tasks[cpu] = NULL; 1419 mutex_unlock(&boost_mutex); 1420 return retval; 1421 } 1422 kthread_bind(boost_tasks[cpu], cpu); 1423 wake_up_process(boost_tasks[cpu]); 1424 mutex_unlock(&boost_mutex); 1425 return 0; 1426 } 1427 1428 /* 1429 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1430 * induces a CPU stall for the time specified by stall_cpu. 1431 */ 1432 static int rcu_torture_stall(void *args) 1433 { 1434 unsigned long stop_at; 1435 1436 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1437 if (stall_cpu_holdoff > 0) { 1438 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1439 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1440 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1441 } 1442 if (!kthread_should_stop()) { 1443 stop_at = get_seconds() + stall_cpu; 1444 /* RCU CPU stall is expected behavior in following code. */ 1445 rcu_read_lock(); 1446 if (stall_cpu_irqsoff) 1447 local_irq_disable(); 1448 else 1449 preempt_disable(); 1450 pr_alert("rcu_torture_stall start on CPU %d.\n", 1451 smp_processor_id()); 1452 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1453 continue; /* Induce RCU CPU stall warning. */ 1454 if (stall_cpu_irqsoff) 1455 local_irq_enable(); 1456 else 1457 preempt_enable(); 1458 rcu_read_unlock(); 1459 pr_alert("rcu_torture_stall end.\n"); 1460 } 1461 torture_shutdown_absorb("rcu_torture_stall"); 1462 while (!kthread_should_stop()) 1463 schedule_timeout_interruptible(10 * HZ); 1464 return 0; 1465 } 1466 1467 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1468 static int __init rcu_torture_stall_init(void) 1469 { 1470 if (stall_cpu <= 0) 1471 return 0; 1472 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1473 } 1474 1475 /* Callback function for RCU barrier testing. */ 1476 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1477 { 1478 atomic_inc(&barrier_cbs_invoked); 1479 } 1480 1481 /* kthread function to register callbacks used to test RCU barriers. */ 1482 static int rcu_torture_barrier_cbs(void *arg) 1483 { 1484 long myid = (long)arg; 1485 bool lastphase = 0; 1486 bool newphase; 1487 struct rcu_head rcu; 1488 1489 init_rcu_head_on_stack(&rcu); 1490 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1491 set_user_nice(current, MAX_NICE); 1492 do { 1493 wait_event(barrier_cbs_wq[myid], 1494 (newphase = 1495 smp_load_acquire(&barrier_phase)) != lastphase || 1496 torture_must_stop()); 1497 lastphase = newphase; 1498 if (torture_must_stop()) 1499 break; 1500 /* 1501 * The above smp_load_acquire() ensures barrier_phase load 1502 * is ordered before the following ->call(). 1503 */ 1504 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1505 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1506 local_irq_enable(); 1507 if (atomic_dec_and_test(&barrier_cbs_count)) 1508 wake_up(&barrier_wq); 1509 } while (!torture_must_stop()); 1510 if (cur_ops->cb_barrier != NULL) 1511 cur_ops->cb_barrier(); 1512 destroy_rcu_head_on_stack(&rcu); 1513 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1514 return 0; 1515 } 1516 1517 /* kthread function to drive and coordinate RCU barrier testing. */ 1518 static int rcu_torture_barrier(void *arg) 1519 { 1520 int i; 1521 1522 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1523 do { 1524 atomic_set(&barrier_cbs_invoked, 0); 1525 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1526 /* Ensure barrier_phase ordered after prior assignments. */ 1527 smp_store_release(&barrier_phase, !barrier_phase); 1528 for (i = 0; i < n_barrier_cbs; i++) 1529 wake_up(&barrier_cbs_wq[i]); 1530 wait_event(barrier_wq, 1531 atomic_read(&barrier_cbs_count) == 0 || 1532 torture_must_stop()); 1533 if (torture_must_stop()) 1534 break; 1535 n_barrier_attempts++; 1536 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1537 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1538 n_rcu_torture_barrier_error++; 1539 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1540 atomic_read(&barrier_cbs_invoked), 1541 n_barrier_cbs); 1542 WARN_ON_ONCE(1); 1543 } 1544 n_barrier_successes++; 1545 schedule_timeout_interruptible(HZ / 10); 1546 } while (!torture_must_stop()); 1547 torture_kthread_stopping("rcu_torture_barrier"); 1548 return 0; 1549 } 1550 1551 /* Initialize RCU barrier testing. */ 1552 static int rcu_torture_barrier_init(void) 1553 { 1554 int i; 1555 int ret; 1556 1557 if (n_barrier_cbs <= 0) 1558 return 0; 1559 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1560 pr_alert("%s" TORTURE_FLAG 1561 " Call or barrier ops missing for %s,\n", 1562 torture_type, cur_ops->name); 1563 pr_alert("%s" TORTURE_FLAG 1564 " RCU barrier testing omitted from run.\n", 1565 torture_type); 1566 return 0; 1567 } 1568 atomic_set(&barrier_cbs_count, 0); 1569 atomic_set(&barrier_cbs_invoked, 0); 1570 barrier_cbs_tasks = 1571 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 1572 GFP_KERNEL); 1573 barrier_cbs_wq = 1574 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 1575 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1576 return -ENOMEM; 1577 for (i = 0; i < n_barrier_cbs; i++) { 1578 init_waitqueue_head(&barrier_cbs_wq[i]); 1579 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1580 (void *)(long)i, 1581 barrier_cbs_tasks[i]); 1582 if (ret) 1583 return ret; 1584 } 1585 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1586 } 1587 1588 /* Clean up after RCU barrier testing. */ 1589 static void rcu_torture_barrier_cleanup(void) 1590 { 1591 int i; 1592 1593 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1594 if (barrier_cbs_tasks != NULL) { 1595 for (i = 0; i < n_barrier_cbs; i++) 1596 torture_stop_kthread(rcu_torture_barrier_cbs, 1597 barrier_cbs_tasks[i]); 1598 kfree(barrier_cbs_tasks); 1599 barrier_cbs_tasks = NULL; 1600 } 1601 if (barrier_cbs_wq != NULL) { 1602 kfree(barrier_cbs_wq); 1603 barrier_cbs_wq = NULL; 1604 } 1605 } 1606 1607 static enum cpuhp_state rcutor_hp; 1608 1609 static void 1610 rcu_torture_cleanup(void) 1611 { 1612 int i; 1613 1614 rcutorture_record_test_transition(); 1615 if (torture_cleanup_begin()) { 1616 if (cur_ops->cb_barrier != NULL) 1617 cur_ops->cb_barrier(); 1618 return; 1619 } 1620 1621 rcu_torture_barrier_cleanup(); 1622 torture_stop_kthread(rcu_torture_stall, stall_task); 1623 torture_stop_kthread(rcu_torture_writer, writer_task); 1624 1625 if (reader_tasks) { 1626 for (i = 0; i < nrealreaders; i++) 1627 torture_stop_kthread(rcu_torture_reader, 1628 reader_tasks[i]); 1629 kfree(reader_tasks); 1630 } 1631 rcu_torture_current = NULL; 1632 1633 if (fakewriter_tasks) { 1634 for (i = 0; i < nfakewriters; i++) { 1635 torture_stop_kthread(rcu_torture_fakewriter, 1636 fakewriter_tasks[i]); 1637 } 1638 kfree(fakewriter_tasks); 1639 fakewriter_tasks = NULL; 1640 } 1641 1642 torture_stop_kthread(rcu_torture_stats, stats_task); 1643 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1644 for (i = 0; i < ncbflooders; i++) 1645 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1646 if ((test_boost == 1 && cur_ops->can_boost) || 1647 test_boost == 2) 1648 cpuhp_remove_state(rcutor_hp); 1649 1650 /* 1651 * Wait for all RCU callbacks to fire, then do flavor-specific 1652 * cleanup operations. 1653 */ 1654 if (cur_ops->cb_barrier != NULL) 1655 cur_ops->cb_barrier(); 1656 if (cur_ops->cleanup != NULL) 1657 cur_ops->cleanup(); 1658 1659 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1660 1661 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1662 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1663 else if (torture_onoff_failures()) 1664 rcu_torture_print_module_parms(cur_ops, 1665 "End of test: RCU_HOTPLUG"); 1666 else 1667 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1668 torture_cleanup_end(); 1669 } 1670 1671 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1672 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1673 { 1674 } 1675 1676 static void rcu_torture_err_cb(struct rcu_head *rhp) 1677 { 1678 /* 1679 * This -might- happen due to race conditions, but is unlikely. 1680 * The scenario that leads to this happening is that the 1681 * first of the pair of duplicate callbacks is queued, 1682 * someone else starts a grace period that includes that 1683 * callback, then the second of the pair must wait for the 1684 * next grace period. Unlikely, but can happen. If it 1685 * does happen, the debug-objects subsystem won't have splatted. 1686 */ 1687 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 1688 } 1689 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1690 1691 /* 1692 * Verify that double-free causes debug-objects to complain, but only 1693 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1694 * cannot be carried out. 1695 */ 1696 static void rcu_test_debug_objects(void) 1697 { 1698 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1699 struct rcu_head rh1; 1700 struct rcu_head rh2; 1701 1702 init_rcu_head_on_stack(&rh1); 1703 init_rcu_head_on_stack(&rh2); 1704 pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME); 1705 1706 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1707 preempt_disable(); /* Prevent preemption from interrupting test. */ 1708 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1709 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1710 local_irq_disable(); /* Make it harder to start a new grace period. */ 1711 call_rcu(&rh2, rcu_torture_leak_cb); 1712 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1713 local_irq_enable(); 1714 rcu_read_unlock(); 1715 preempt_enable(); 1716 1717 /* Wait for them all to get done so we can safely return. */ 1718 rcu_barrier(); 1719 pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); 1720 destroy_rcu_head_on_stack(&rh1); 1721 destroy_rcu_head_on_stack(&rh2); 1722 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1723 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); 1724 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1725 } 1726 1727 static int __init 1728 rcu_torture_init(void) 1729 { 1730 int i; 1731 int cpu; 1732 int firsterr = 0; 1733 static struct rcu_torture_ops *torture_ops[] = { 1734 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1735 &sched_ops, &tasks_ops, 1736 }; 1737 1738 if (!torture_init_begin(torture_type, verbose)) 1739 return -EBUSY; 1740 1741 /* Process args and tell the world that the torturer is on the job. */ 1742 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1743 cur_ops = torture_ops[i]; 1744 if (strcmp(torture_type, cur_ops->name) == 0) 1745 break; 1746 } 1747 if (i == ARRAY_SIZE(torture_ops)) { 1748 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1749 torture_type); 1750 pr_alert("rcu-torture types:"); 1751 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1752 pr_alert(" %s", torture_ops[i]->name); 1753 pr_alert("\n"); 1754 firsterr = -EINVAL; 1755 goto unwind; 1756 } 1757 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1758 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1759 fqs_duration = 0; 1760 } 1761 if (cur_ops->init) 1762 cur_ops->init(); 1763 1764 if (nreaders >= 0) { 1765 nrealreaders = nreaders; 1766 } else { 1767 nrealreaders = num_online_cpus() - 2 - nreaders; 1768 if (nrealreaders <= 0) 1769 nrealreaders = 1; 1770 } 1771 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1772 1773 /* Set up the freelist. */ 1774 1775 INIT_LIST_HEAD(&rcu_torture_freelist); 1776 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1777 rcu_tortures[i].rtort_mbtest = 0; 1778 list_add_tail(&rcu_tortures[i].rtort_free, 1779 &rcu_torture_freelist); 1780 } 1781 1782 /* Initialize the statistics so that each run gets its own numbers. */ 1783 1784 rcu_torture_current = NULL; 1785 rcu_torture_current_version = 0; 1786 atomic_set(&n_rcu_torture_alloc, 0); 1787 atomic_set(&n_rcu_torture_alloc_fail, 0); 1788 atomic_set(&n_rcu_torture_free, 0); 1789 atomic_set(&n_rcu_torture_mberror, 0); 1790 atomic_set(&n_rcu_torture_error, 0); 1791 n_rcu_torture_barrier_error = 0; 1792 n_rcu_torture_boost_ktrerror = 0; 1793 n_rcu_torture_boost_rterror = 0; 1794 n_rcu_torture_boost_failure = 0; 1795 n_rcu_torture_boosts = 0; 1796 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1797 atomic_set(&rcu_torture_wcount[i], 0); 1798 for_each_possible_cpu(cpu) { 1799 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1800 per_cpu(rcu_torture_count, cpu)[i] = 0; 1801 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1802 } 1803 } 1804 1805 /* Start up the kthreads. */ 1806 1807 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1808 writer_task); 1809 if (firsterr) 1810 goto unwind; 1811 if (nfakewriters > 0) { 1812 fakewriter_tasks = kcalloc(nfakewriters, 1813 sizeof(fakewriter_tasks[0]), 1814 GFP_KERNEL); 1815 if (fakewriter_tasks == NULL) { 1816 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1817 firsterr = -ENOMEM; 1818 goto unwind; 1819 } 1820 } 1821 for (i = 0; i < nfakewriters; i++) { 1822 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1823 NULL, fakewriter_tasks[i]); 1824 if (firsterr) 1825 goto unwind; 1826 } 1827 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 1828 GFP_KERNEL); 1829 if (reader_tasks == NULL) { 1830 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1831 firsterr = -ENOMEM; 1832 goto unwind; 1833 } 1834 for (i = 0; i < nrealreaders; i++) { 1835 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1836 reader_tasks[i]); 1837 if (firsterr) 1838 goto unwind; 1839 } 1840 if (stat_interval > 0) { 1841 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1842 stats_task); 1843 if (firsterr) 1844 goto unwind; 1845 } 1846 if (test_no_idle_hz && shuffle_interval > 0) { 1847 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1848 if (firsterr) 1849 goto unwind; 1850 } 1851 if (stutter < 0) 1852 stutter = 0; 1853 if (stutter) { 1854 firsterr = torture_stutter_init(stutter * HZ); 1855 if (firsterr) 1856 goto unwind; 1857 } 1858 if (fqs_duration < 0) 1859 fqs_duration = 0; 1860 if (fqs_duration) { 1861 /* Create the fqs thread */ 1862 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1863 fqs_task); 1864 if (firsterr) 1865 goto unwind; 1866 } 1867 if (test_boost_interval < 1) 1868 test_boost_interval = 1; 1869 if (test_boost_duration < 2) 1870 test_boost_duration = 2; 1871 if ((test_boost == 1 && cur_ops->can_boost) || 1872 test_boost == 2) { 1873 1874 boost_starttime = jiffies + test_boost_interval * HZ; 1875 1876 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1877 rcutorture_booster_init, 1878 rcutorture_booster_cleanup); 1879 if (firsterr < 0) 1880 goto unwind; 1881 rcutor_hp = firsterr; 1882 } 1883 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1884 if (firsterr) 1885 goto unwind; 1886 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1887 if (firsterr) 1888 goto unwind; 1889 firsterr = rcu_torture_stall_init(); 1890 if (firsterr) 1891 goto unwind; 1892 firsterr = rcu_torture_barrier_init(); 1893 if (firsterr) 1894 goto unwind; 1895 if (object_debug) 1896 rcu_test_debug_objects(); 1897 if (cbflood_n_burst > 0) { 1898 /* Create the cbflood threads */ 1899 ncbflooders = (num_online_cpus() + 3) / 4; 1900 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1901 GFP_KERNEL); 1902 if (!cbflood_task) { 1903 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1904 firsterr = -ENOMEM; 1905 goto unwind; 1906 } 1907 for (i = 0; i < ncbflooders; i++) { 1908 firsterr = torture_create_kthread(rcu_torture_cbflood, 1909 NULL, 1910 cbflood_task[i]); 1911 if (firsterr) 1912 goto unwind; 1913 } 1914 } 1915 rcutorture_record_test_transition(); 1916 torture_init_end(); 1917 return 0; 1918 1919 unwind: 1920 torture_init_end(); 1921 rcu_torture_cleanup(); 1922 return firsterr; 1923 } 1924 1925 module_init(rcu_torture_init); 1926 module_exit(rcu_torture_cleanup); 1927