1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 #include <linux/vmalloc.h> 53 54 MODULE_LICENSE("GPL"); 55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 56 57 58 torture_param(int, cbflood_inter_holdoff, HZ, 59 "Holdoff between floods (jiffies)"); 60 torture_param(int, cbflood_intra_holdoff, 1, 61 "Holdoff between bursts (jiffies)"); 62 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 63 torture_param(int, cbflood_n_per_burst, 20000, 64 "# callbacks per burst in flood"); 65 torture_param(int, fqs_duration, 0, 66 "Duration of fqs bursts (us), 0 to disable"); 67 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 68 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 69 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 70 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 71 torture_param(bool, gp_normal, false, 72 "Use normal (non-expedited) GP wait primitives"); 73 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 74 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 75 torture_param(int, n_barrier_cbs, 0, 76 "# of callbacks/kthreads for barrier testing"); 77 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 78 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 79 torture_param(int, object_debug, 0, 80 "Enable debug-object double call_rcu() testing"); 81 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 82 torture_param(int, onoff_interval, 0, 83 "Time between CPU hotplugs (s), 0=disable"); 84 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 85 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 86 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 87 torture_param(int, stall_cpu_holdoff, 10, 88 "Time to wait before starting stall (s)."); 89 torture_param(int, stat_interval, 60, 90 "Number of seconds between stats printk()s"); 91 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 92 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 93 torture_param(int, test_boost_duration, 4, 94 "Duration of each boost test, seconds."); 95 torture_param(int, test_boost_interval, 7, 96 "Interval between boost tests, seconds."); 97 torture_param(bool, test_no_idle_hz, true, 98 "Test support for tickless idle CPUs"); 99 torture_param(bool, verbose, true, 100 "Enable verbose debugging printk()s"); 101 102 static char *torture_type = "rcu"; 103 module_param(torture_type, charp, 0444); 104 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 105 106 static int nrealreaders; 107 static int ncbflooders; 108 static struct task_struct *writer_task; 109 static struct task_struct **fakewriter_tasks; 110 static struct task_struct **reader_tasks; 111 static struct task_struct *stats_task; 112 static struct task_struct **cbflood_task; 113 static struct task_struct *fqs_task; 114 static struct task_struct *boost_tasks[NR_CPUS]; 115 static struct task_struct *stall_task; 116 static struct task_struct **barrier_cbs_tasks; 117 static struct task_struct *barrier_task; 118 119 #define RCU_TORTURE_PIPE_LEN 10 120 121 struct rcu_torture { 122 struct rcu_head rtort_rcu; 123 int rtort_pipe_count; 124 struct list_head rtort_free; 125 int rtort_mbtest; 126 }; 127 128 static LIST_HEAD(rcu_torture_freelist); 129 static struct rcu_torture __rcu *rcu_torture_current; 130 static unsigned long rcu_torture_current_version; 131 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 132 static DEFINE_SPINLOCK(rcu_torture_lock); 133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 134 rcu_torture_count) = { 0 }; 135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 136 rcu_torture_batch) = { 0 }; 137 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 138 static atomic_t n_rcu_torture_alloc; 139 static atomic_t n_rcu_torture_alloc_fail; 140 static atomic_t n_rcu_torture_free; 141 static atomic_t n_rcu_torture_mberror; 142 static atomic_t n_rcu_torture_error; 143 static long n_rcu_torture_barrier_error; 144 static long n_rcu_torture_boost_ktrerror; 145 static long n_rcu_torture_boost_rterror; 146 static long n_rcu_torture_boost_failure; 147 static long n_rcu_torture_boosts; 148 static long n_rcu_torture_timers; 149 static long n_barrier_attempts; 150 static long n_barrier_successes; 151 static atomic_long_t n_cbfloods; 152 static struct list_head rcu_torture_removed; 153 154 static int rcu_torture_writer_state; 155 #define RTWS_FIXED_DELAY 0 156 #define RTWS_DELAY 1 157 #define RTWS_REPLACE 2 158 #define RTWS_DEF_FREE 3 159 #define RTWS_EXP_SYNC 4 160 #define RTWS_COND_GET 5 161 #define RTWS_COND_SYNC 6 162 #define RTWS_SYNC 7 163 #define RTWS_STUTTER 8 164 #define RTWS_STOPPING 9 165 166 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 167 #define RCUTORTURE_RUNNABLE_INIT 1 168 #else 169 #define RCUTORTURE_RUNNABLE_INIT 0 170 #endif 171 static int torture_runnable = RCUTORTURE_RUNNABLE_INIT; 172 module_param(torture_runnable, int, 0444); 173 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 174 175 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 176 #define rcu_can_boost() 1 177 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 178 #define rcu_can_boost() 0 179 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 180 181 #ifdef CONFIG_RCU_TRACE 182 static u64 notrace rcu_trace_clock_local(void) 183 { 184 u64 ts = trace_clock_local(); 185 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 186 return ts; 187 } 188 #else /* #ifdef CONFIG_RCU_TRACE */ 189 static u64 notrace rcu_trace_clock_local(void) 190 { 191 return 0ULL; 192 } 193 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 194 195 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 196 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 197 /* and boost task create/destroy. */ 198 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 199 static bool barrier_phase; /* Test phase. */ 200 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 201 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 202 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 203 204 /* 205 * Allocate an element from the rcu_tortures pool. 206 */ 207 static struct rcu_torture * 208 rcu_torture_alloc(void) 209 { 210 struct list_head *p; 211 212 spin_lock_bh(&rcu_torture_lock); 213 if (list_empty(&rcu_torture_freelist)) { 214 atomic_inc(&n_rcu_torture_alloc_fail); 215 spin_unlock_bh(&rcu_torture_lock); 216 return NULL; 217 } 218 atomic_inc(&n_rcu_torture_alloc); 219 p = rcu_torture_freelist.next; 220 list_del_init(p); 221 spin_unlock_bh(&rcu_torture_lock); 222 return container_of(p, struct rcu_torture, rtort_free); 223 } 224 225 /* 226 * Free an element to the rcu_tortures pool. 227 */ 228 static void 229 rcu_torture_free(struct rcu_torture *p) 230 { 231 atomic_inc(&n_rcu_torture_free); 232 spin_lock_bh(&rcu_torture_lock); 233 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 234 spin_unlock_bh(&rcu_torture_lock); 235 } 236 237 /* 238 * Operations vector for selecting different types of tests. 239 */ 240 241 struct rcu_torture_ops { 242 int ttype; 243 void (*init)(void); 244 void (*cleanup)(void); 245 int (*readlock)(void); 246 void (*read_delay)(struct torture_random_state *rrsp); 247 void (*readunlock)(int idx); 248 unsigned long (*started)(void); 249 unsigned long (*completed)(void); 250 void (*deferred_free)(struct rcu_torture *p); 251 void (*sync)(void); 252 void (*exp_sync)(void); 253 unsigned long (*get_state)(void); 254 void (*cond_sync)(unsigned long oldstate); 255 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 256 void (*cb_barrier)(void); 257 void (*fqs)(void); 258 void (*stats)(void); 259 int irq_capable; 260 int can_boost; 261 const char *name; 262 }; 263 264 static struct rcu_torture_ops *cur_ops; 265 266 /* 267 * Definitions for rcu torture testing. 268 */ 269 270 static int rcu_torture_read_lock(void) __acquires(RCU) 271 { 272 rcu_read_lock(); 273 return 0; 274 } 275 276 static void rcu_read_delay(struct torture_random_state *rrsp) 277 { 278 const unsigned long shortdelay_us = 200; 279 const unsigned long longdelay_ms = 50; 280 281 /* We want a short delay sometimes to make a reader delay the grace 282 * period, and we want a long delay occasionally to trigger 283 * force_quiescent_state. */ 284 285 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 286 mdelay(longdelay_ms); 287 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 288 udelay(shortdelay_us); 289 #ifdef CONFIG_PREEMPT 290 if (!preempt_count() && 291 !(torture_random(rrsp) % (nrealreaders * 20000))) 292 preempt_schedule(); /* No QS if preempt_disable() in effect */ 293 #endif 294 } 295 296 static void rcu_torture_read_unlock(int idx) __releases(RCU) 297 { 298 rcu_read_unlock(); 299 } 300 301 /* 302 * Update callback in the pipe. This should be invoked after a grace period. 303 */ 304 static bool 305 rcu_torture_pipe_update_one(struct rcu_torture *rp) 306 { 307 int i; 308 309 i = rp->rtort_pipe_count; 310 if (i > RCU_TORTURE_PIPE_LEN) 311 i = RCU_TORTURE_PIPE_LEN; 312 atomic_inc(&rcu_torture_wcount[i]); 313 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 314 rp->rtort_mbtest = 0; 315 return true; 316 } 317 return false; 318 } 319 320 /* 321 * Update all callbacks in the pipe. Suitable for synchronous grace-period 322 * primitives. 323 */ 324 static void 325 rcu_torture_pipe_update(struct rcu_torture *old_rp) 326 { 327 struct rcu_torture *rp; 328 struct rcu_torture *rp1; 329 330 if (old_rp) 331 list_add(&old_rp->rtort_free, &rcu_torture_removed); 332 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 333 if (rcu_torture_pipe_update_one(rp)) { 334 list_del(&rp->rtort_free); 335 rcu_torture_free(rp); 336 } 337 } 338 } 339 340 static void 341 rcu_torture_cb(struct rcu_head *p) 342 { 343 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 344 345 if (torture_must_stop_irq()) { 346 /* Test is ending, just drop callbacks on the floor. */ 347 /* The next initialization will pick up the pieces. */ 348 return; 349 } 350 if (rcu_torture_pipe_update_one(rp)) 351 rcu_torture_free(rp); 352 else 353 cur_ops->deferred_free(rp); 354 } 355 356 static unsigned long rcu_no_completed(void) 357 { 358 return 0; 359 } 360 361 static void rcu_torture_deferred_free(struct rcu_torture *p) 362 { 363 call_rcu(&p->rtort_rcu, rcu_torture_cb); 364 } 365 366 static void rcu_sync_torture_init(void) 367 { 368 INIT_LIST_HEAD(&rcu_torture_removed); 369 } 370 371 static struct rcu_torture_ops rcu_ops = { 372 .ttype = RCU_FLAVOR, 373 .init = rcu_sync_torture_init, 374 .readlock = rcu_torture_read_lock, 375 .read_delay = rcu_read_delay, 376 .readunlock = rcu_torture_read_unlock, 377 .started = rcu_batches_started, 378 .completed = rcu_batches_completed, 379 .deferred_free = rcu_torture_deferred_free, 380 .sync = synchronize_rcu, 381 .exp_sync = synchronize_rcu_expedited, 382 .get_state = get_state_synchronize_rcu, 383 .cond_sync = cond_synchronize_rcu, 384 .call = call_rcu, 385 .cb_barrier = rcu_barrier, 386 .fqs = rcu_force_quiescent_state, 387 .stats = NULL, 388 .irq_capable = 1, 389 .can_boost = rcu_can_boost(), 390 .name = "rcu" 391 }; 392 393 /* 394 * Definitions for rcu_bh torture testing. 395 */ 396 397 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 398 { 399 rcu_read_lock_bh(); 400 return 0; 401 } 402 403 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 404 { 405 rcu_read_unlock_bh(); 406 } 407 408 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 409 { 410 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 411 } 412 413 static struct rcu_torture_ops rcu_bh_ops = { 414 .ttype = RCU_BH_FLAVOR, 415 .init = rcu_sync_torture_init, 416 .readlock = rcu_bh_torture_read_lock, 417 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 418 .readunlock = rcu_bh_torture_read_unlock, 419 .started = rcu_batches_started_bh, 420 .completed = rcu_batches_completed_bh, 421 .deferred_free = rcu_bh_torture_deferred_free, 422 .sync = synchronize_rcu_bh, 423 .exp_sync = synchronize_rcu_bh_expedited, 424 .call = call_rcu_bh, 425 .cb_barrier = rcu_barrier_bh, 426 .fqs = rcu_bh_force_quiescent_state, 427 .stats = NULL, 428 .irq_capable = 1, 429 .name = "rcu_bh" 430 }; 431 432 /* 433 * Don't even think about trying any of these in real life!!! 434 * The names includes "busted", and they really means it! 435 * The only purpose of these functions is to provide a buggy RCU 436 * implementation to make sure that rcutorture correctly emits 437 * buggy-RCU error messages. 438 */ 439 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 440 { 441 /* This is a deliberate bug for testing purposes only! */ 442 rcu_torture_cb(&p->rtort_rcu); 443 } 444 445 static void synchronize_rcu_busted(void) 446 { 447 /* This is a deliberate bug for testing purposes only! */ 448 } 449 450 static void 451 call_rcu_busted(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 452 { 453 /* This is a deliberate bug for testing purposes only! */ 454 func(head); 455 } 456 457 static struct rcu_torture_ops rcu_busted_ops = { 458 .ttype = INVALID_RCU_FLAVOR, 459 .init = rcu_sync_torture_init, 460 .readlock = rcu_torture_read_lock, 461 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 462 .readunlock = rcu_torture_read_unlock, 463 .started = rcu_no_completed, 464 .completed = rcu_no_completed, 465 .deferred_free = rcu_busted_torture_deferred_free, 466 .sync = synchronize_rcu_busted, 467 .exp_sync = synchronize_rcu_busted, 468 .call = call_rcu_busted, 469 .cb_barrier = NULL, 470 .fqs = NULL, 471 .stats = NULL, 472 .irq_capable = 1, 473 .name = "rcu_busted" 474 }; 475 476 /* 477 * Definitions for srcu torture testing. 478 */ 479 480 DEFINE_STATIC_SRCU(srcu_ctl); 481 static struct srcu_struct srcu_ctld; 482 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 483 484 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 485 { 486 return srcu_read_lock(srcu_ctlp); 487 } 488 489 static void srcu_read_delay(struct torture_random_state *rrsp) 490 { 491 long delay; 492 const long uspertick = 1000000 / HZ; 493 const long longdelay = 10; 494 495 /* We want there to be long-running readers, but not all the time. */ 496 497 delay = torture_random(rrsp) % 498 (nrealreaders * 2 * longdelay * uspertick); 499 if (!delay) 500 schedule_timeout_interruptible(longdelay); 501 else 502 rcu_read_delay(rrsp); 503 } 504 505 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 506 { 507 srcu_read_unlock(srcu_ctlp, idx); 508 } 509 510 static unsigned long srcu_torture_completed(void) 511 { 512 return srcu_batches_completed(srcu_ctlp); 513 } 514 515 static void srcu_torture_deferred_free(struct rcu_torture *rp) 516 { 517 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 518 } 519 520 static void srcu_torture_synchronize(void) 521 { 522 synchronize_srcu(srcu_ctlp); 523 } 524 525 static void srcu_torture_call(struct rcu_head *head, 526 void (*func)(struct rcu_head *head)) 527 { 528 call_srcu(srcu_ctlp, head, func); 529 } 530 531 static void srcu_torture_barrier(void) 532 { 533 srcu_barrier(srcu_ctlp); 534 } 535 536 static void srcu_torture_stats(void) 537 { 538 int cpu; 539 int idx = srcu_ctlp->completed & 0x1; 540 541 pr_alert("%s%s per-CPU(idx=%d):", 542 torture_type, TORTURE_FLAG, idx); 543 for_each_possible_cpu(cpu) { 544 long c0, c1; 545 546 c0 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[!idx]; 547 c1 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[idx]; 548 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 549 } 550 pr_cont("\n"); 551 } 552 553 static void srcu_torture_synchronize_expedited(void) 554 { 555 synchronize_srcu_expedited(srcu_ctlp); 556 } 557 558 static struct rcu_torture_ops srcu_ops = { 559 .ttype = SRCU_FLAVOR, 560 .init = rcu_sync_torture_init, 561 .readlock = srcu_torture_read_lock, 562 .read_delay = srcu_read_delay, 563 .readunlock = srcu_torture_read_unlock, 564 .started = NULL, 565 .completed = srcu_torture_completed, 566 .deferred_free = srcu_torture_deferred_free, 567 .sync = srcu_torture_synchronize, 568 .exp_sync = srcu_torture_synchronize_expedited, 569 .call = srcu_torture_call, 570 .cb_barrier = srcu_torture_barrier, 571 .stats = srcu_torture_stats, 572 .name = "srcu" 573 }; 574 575 static void srcu_torture_init(void) 576 { 577 rcu_sync_torture_init(); 578 WARN_ON(init_srcu_struct(&srcu_ctld)); 579 srcu_ctlp = &srcu_ctld; 580 } 581 582 static void srcu_torture_cleanup(void) 583 { 584 cleanup_srcu_struct(&srcu_ctld); 585 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 586 } 587 588 /* As above, but dynamically allocated. */ 589 static struct rcu_torture_ops srcud_ops = { 590 .ttype = SRCU_FLAVOR, 591 .init = srcu_torture_init, 592 .cleanup = srcu_torture_cleanup, 593 .readlock = srcu_torture_read_lock, 594 .read_delay = srcu_read_delay, 595 .readunlock = srcu_torture_read_unlock, 596 .started = NULL, 597 .completed = srcu_torture_completed, 598 .deferred_free = srcu_torture_deferred_free, 599 .sync = srcu_torture_synchronize, 600 .exp_sync = srcu_torture_synchronize_expedited, 601 .call = srcu_torture_call, 602 .cb_barrier = srcu_torture_barrier, 603 .stats = srcu_torture_stats, 604 .name = "srcud" 605 }; 606 607 /* 608 * Definitions for sched torture testing. 609 */ 610 611 static int sched_torture_read_lock(void) 612 { 613 preempt_disable(); 614 return 0; 615 } 616 617 static void sched_torture_read_unlock(int idx) 618 { 619 preempt_enable(); 620 } 621 622 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 623 { 624 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 625 } 626 627 static struct rcu_torture_ops sched_ops = { 628 .ttype = RCU_SCHED_FLAVOR, 629 .init = rcu_sync_torture_init, 630 .readlock = sched_torture_read_lock, 631 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 632 .readunlock = sched_torture_read_unlock, 633 .started = rcu_batches_started_sched, 634 .completed = rcu_batches_completed_sched, 635 .deferred_free = rcu_sched_torture_deferred_free, 636 .sync = synchronize_sched, 637 .exp_sync = synchronize_sched_expedited, 638 .get_state = get_state_synchronize_sched, 639 .cond_sync = cond_synchronize_sched, 640 .call = call_rcu_sched, 641 .cb_barrier = rcu_barrier_sched, 642 .fqs = rcu_sched_force_quiescent_state, 643 .stats = NULL, 644 .irq_capable = 1, 645 .name = "sched" 646 }; 647 648 #ifdef CONFIG_TASKS_RCU 649 650 /* 651 * Definitions for RCU-tasks torture testing. 652 */ 653 654 static int tasks_torture_read_lock(void) 655 { 656 return 0; 657 } 658 659 static void tasks_torture_read_unlock(int idx) 660 { 661 } 662 663 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 664 { 665 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 666 } 667 668 static struct rcu_torture_ops tasks_ops = { 669 .ttype = RCU_TASKS_FLAVOR, 670 .init = rcu_sync_torture_init, 671 .readlock = tasks_torture_read_lock, 672 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 673 .readunlock = tasks_torture_read_unlock, 674 .started = rcu_no_completed, 675 .completed = rcu_no_completed, 676 .deferred_free = rcu_tasks_torture_deferred_free, 677 .sync = synchronize_rcu_tasks, 678 .exp_sync = synchronize_rcu_tasks, 679 .call = call_rcu_tasks, 680 .cb_barrier = rcu_barrier_tasks, 681 .fqs = NULL, 682 .stats = NULL, 683 .irq_capable = 1, 684 .name = "tasks" 685 }; 686 687 #define RCUTORTURE_TASKS_OPS &tasks_ops, 688 689 static bool __maybe_unused torturing_tasks(void) 690 { 691 return cur_ops == &tasks_ops; 692 } 693 694 #else /* #ifdef CONFIG_TASKS_RCU */ 695 696 #define RCUTORTURE_TASKS_OPS 697 698 static bool torturing_tasks(void) 699 { 700 return false; 701 } 702 703 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 704 705 /* 706 * RCU torture priority-boost testing. Runs one real-time thread per 707 * CPU for moderate bursts, repeatedly registering RCU callbacks and 708 * spinning waiting for them to be invoked. If a given callback takes 709 * too long to be invoked, we assume that priority inversion has occurred. 710 */ 711 712 struct rcu_boost_inflight { 713 struct rcu_head rcu; 714 int inflight; 715 }; 716 717 static void rcu_torture_boost_cb(struct rcu_head *head) 718 { 719 struct rcu_boost_inflight *rbip = 720 container_of(head, struct rcu_boost_inflight, rcu); 721 722 /* Ensure RCU-core accesses precede clearing ->inflight */ 723 smp_store_release(&rbip->inflight, 0); 724 } 725 726 static int rcu_torture_boost(void *arg) 727 { 728 unsigned long call_rcu_time; 729 unsigned long endtime; 730 unsigned long oldstarttime; 731 struct rcu_boost_inflight rbi = { .inflight = 0 }; 732 struct sched_param sp; 733 734 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 735 736 /* Set real-time priority. */ 737 sp.sched_priority = 1; 738 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 739 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 740 n_rcu_torture_boost_rterror++; 741 } 742 743 init_rcu_head_on_stack(&rbi.rcu); 744 /* Each pass through the following loop does one boost-test cycle. */ 745 do { 746 /* Wait for the next test interval. */ 747 oldstarttime = boost_starttime; 748 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 749 schedule_timeout_interruptible(oldstarttime - jiffies); 750 stutter_wait("rcu_torture_boost"); 751 if (torture_must_stop()) 752 goto checkwait; 753 } 754 755 /* Do one boost-test interval. */ 756 endtime = oldstarttime + test_boost_duration * HZ; 757 call_rcu_time = jiffies; 758 while (ULONG_CMP_LT(jiffies, endtime)) { 759 /* If we don't have a callback in flight, post one. */ 760 if (!smp_load_acquire(&rbi.inflight)) { 761 /* RCU core before ->inflight = 1. */ 762 smp_store_release(&rbi.inflight, 1); 763 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 764 if (jiffies - call_rcu_time > 765 test_boost_duration * HZ - HZ / 2) { 766 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 767 n_rcu_torture_boost_failure++; 768 } 769 call_rcu_time = jiffies; 770 } 771 cond_resched_rcu_qs(); 772 stutter_wait("rcu_torture_boost"); 773 if (torture_must_stop()) 774 goto checkwait; 775 } 776 777 /* 778 * Set the start time of the next test interval. 779 * Yes, this is vulnerable to long delays, but such 780 * delays simply cause a false negative for the next 781 * interval. Besides, we are running at RT priority, 782 * so delays should be relatively rare. 783 */ 784 while (oldstarttime == boost_starttime && 785 !kthread_should_stop()) { 786 if (mutex_trylock(&boost_mutex)) { 787 boost_starttime = jiffies + 788 test_boost_interval * HZ; 789 n_rcu_torture_boosts++; 790 mutex_unlock(&boost_mutex); 791 break; 792 } 793 schedule_timeout_uninterruptible(1); 794 } 795 796 /* Go do the stutter. */ 797 checkwait: stutter_wait("rcu_torture_boost"); 798 } while (!torture_must_stop()); 799 800 /* Clean up and exit. */ 801 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 802 torture_shutdown_absorb("rcu_torture_boost"); 803 schedule_timeout_uninterruptible(1); 804 } 805 destroy_rcu_head_on_stack(&rbi.rcu); 806 torture_kthread_stopping("rcu_torture_boost"); 807 return 0; 808 } 809 810 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 811 { 812 } 813 814 /* 815 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 816 * to call_rcu() or analogous, increasing the probability of occurrence 817 * of callback-overflow corner cases. 818 */ 819 static int 820 rcu_torture_cbflood(void *arg) 821 { 822 int err = 1; 823 int i; 824 int j; 825 struct rcu_head *rhp; 826 827 if (cbflood_n_per_burst > 0 && 828 cbflood_inter_holdoff > 0 && 829 cbflood_intra_holdoff > 0 && 830 cur_ops->call && 831 cur_ops->cb_barrier) { 832 rhp = vmalloc(sizeof(*rhp) * 833 cbflood_n_burst * cbflood_n_per_burst); 834 err = !rhp; 835 } 836 if (err) { 837 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 838 goto wait_for_stop; 839 } 840 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 841 do { 842 schedule_timeout_interruptible(cbflood_inter_holdoff); 843 atomic_long_inc(&n_cbfloods); 844 WARN_ON(signal_pending(current)); 845 for (i = 0; i < cbflood_n_burst; i++) { 846 for (j = 0; j < cbflood_n_per_burst; j++) { 847 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 848 rcu_torture_cbflood_cb); 849 } 850 schedule_timeout_interruptible(cbflood_intra_holdoff); 851 WARN_ON(signal_pending(current)); 852 } 853 cur_ops->cb_barrier(); 854 stutter_wait("rcu_torture_cbflood"); 855 } while (!torture_must_stop()); 856 vfree(rhp); 857 wait_for_stop: 858 torture_kthread_stopping("rcu_torture_cbflood"); 859 return 0; 860 } 861 862 /* 863 * RCU torture force-quiescent-state kthread. Repeatedly induces 864 * bursts of calls to force_quiescent_state(), increasing the probability 865 * of occurrence of some important types of race conditions. 866 */ 867 static int 868 rcu_torture_fqs(void *arg) 869 { 870 unsigned long fqs_resume_time; 871 int fqs_burst_remaining; 872 873 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 874 do { 875 fqs_resume_time = jiffies + fqs_stutter * HZ; 876 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 877 !kthread_should_stop()) { 878 schedule_timeout_interruptible(1); 879 } 880 fqs_burst_remaining = fqs_duration; 881 while (fqs_burst_remaining > 0 && 882 !kthread_should_stop()) { 883 cur_ops->fqs(); 884 udelay(fqs_holdoff); 885 fqs_burst_remaining -= fqs_holdoff; 886 } 887 stutter_wait("rcu_torture_fqs"); 888 } while (!torture_must_stop()); 889 torture_kthread_stopping("rcu_torture_fqs"); 890 return 0; 891 } 892 893 /* 894 * RCU torture writer kthread. Repeatedly substitutes a new structure 895 * for that pointed to by rcu_torture_current, freeing the old structure 896 * after a series of grace periods (the "pipeline"). 897 */ 898 static int 899 rcu_torture_writer(void *arg) 900 { 901 bool can_expedite = !rcu_gp_is_expedited(); 902 int expediting = 0; 903 unsigned long gp_snap; 904 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 905 bool gp_sync1 = gp_sync; 906 int i; 907 struct rcu_torture *rp; 908 struct rcu_torture *old_rp; 909 static DEFINE_TORTURE_RANDOM(rand); 910 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 911 RTWS_COND_GET, RTWS_SYNC }; 912 int nsynctypes = 0; 913 914 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 915 pr_alert("%s" TORTURE_FLAG 916 " Grace periods expedited from boot/sysfs for %s,\n", 917 torture_type, cur_ops->name); 918 pr_alert("%s" TORTURE_FLAG 919 " Testing of dynamic grace-period expediting diabled.\n", 920 torture_type); 921 922 /* Initialize synctype[] array. If none set, take default. */ 923 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 924 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 925 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 926 synctype[nsynctypes++] = RTWS_COND_GET; 927 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 928 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 929 if (gp_exp1 && cur_ops->exp_sync) 930 synctype[nsynctypes++] = RTWS_EXP_SYNC; 931 else if (gp_exp && !cur_ops->exp_sync) 932 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 933 if (gp_normal1 && cur_ops->deferred_free) 934 synctype[nsynctypes++] = RTWS_DEF_FREE; 935 else if (gp_normal && !cur_ops->deferred_free) 936 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 937 if (gp_sync1 && cur_ops->sync) 938 synctype[nsynctypes++] = RTWS_SYNC; 939 else if (gp_sync && !cur_ops->sync) 940 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 941 if (WARN_ONCE(nsynctypes == 0, 942 "rcu_torture_writer: No update-side primitives.\n")) { 943 /* 944 * No updates primitives, so don't try updating. 945 * The resulting test won't be testing much, hence the 946 * above WARN_ONCE(). 947 */ 948 rcu_torture_writer_state = RTWS_STOPPING; 949 torture_kthread_stopping("rcu_torture_writer"); 950 } 951 952 do { 953 rcu_torture_writer_state = RTWS_FIXED_DELAY; 954 schedule_timeout_uninterruptible(1); 955 rp = rcu_torture_alloc(); 956 if (rp == NULL) 957 continue; 958 rp->rtort_pipe_count = 0; 959 rcu_torture_writer_state = RTWS_DELAY; 960 udelay(torture_random(&rand) & 0x3ff); 961 rcu_torture_writer_state = RTWS_REPLACE; 962 old_rp = rcu_dereference_check(rcu_torture_current, 963 current == writer_task); 964 rp->rtort_mbtest = 1; 965 rcu_assign_pointer(rcu_torture_current, rp); 966 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 967 if (old_rp) { 968 i = old_rp->rtort_pipe_count; 969 if (i > RCU_TORTURE_PIPE_LEN) 970 i = RCU_TORTURE_PIPE_LEN; 971 atomic_inc(&rcu_torture_wcount[i]); 972 old_rp->rtort_pipe_count++; 973 switch (synctype[torture_random(&rand) % nsynctypes]) { 974 case RTWS_DEF_FREE: 975 rcu_torture_writer_state = RTWS_DEF_FREE; 976 cur_ops->deferred_free(old_rp); 977 break; 978 case RTWS_EXP_SYNC: 979 rcu_torture_writer_state = RTWS_EXP_SYNC; 980 cur_ops->exp_sync(); 981 rcu_torture_pipe_update(old_rp); 982 break; 983 case RTWS_COND_GET: 984 rcu_torture_writer_state = RTWS_COND_GET; 985 gp_snap = cur_ops->get_state(); 986 i = torture_random(&rand) % 16; 987 if (i != 0) 988 schedule_timeout_interruptible(i); 989 udelay(torture_random(&rand) % 1000); 990 rcu_torture_writer_state = RTWS_COND_SYNC; 991 cur_ops->cond_sync(gp_snap); 992 rcu_torture_pipe_update(old_rp); 993 break; 994 case RTWS_SYNC: 995 rcu_torture_writer_state = RTWS_SYNC; 996 cur_ops->sync(); 997 rcu_torture_pipe_update(old_rp); 998 break; 999 default: 1000 WARN_ON_ONCE(1); 1001 break; 1002 } 1003 } 1004 rcutorture_record_progress(++rcu_torture_current_version); 1005 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1006 if (can_expedite && 1007 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1008 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1009 if (expediting >= 0) 1010 rcu_expedite_gp(); 1011 else 1012 rcu_unexpedite_gp(); 1013 if (++expediting > 3) 1014 expediting = -expediting; 1015 } 1016 rcu_torture_writer_state = RTWS_STUTTER; 1017 stutter_wait("rcu_torture_writer"); 1018 } while (!torture_must_stop()); 1019 /* Reset expediting back to unexpedited. */ 1020 if (expediting > 0) 1021 expediting = -expediting; 1022 while (can_expedite && expediting++ < 0) 1023 rcu_unexpedite_gp(); 1024 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1025 rcu_torture_writer_state = RTWS_STOPPING; 1026 torture_kthread_stopping("rcu_torture_writer"); 1027 return 0; 1028 } 1029 1030 /* 1031 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1032 * delay between calls. 1033 */ 1034 static int 1035 rcu_torture_fakewriter(void *arg) 1036 { 1037 DEFINE_TORTURE_RANDOM(rand); 1038 1039 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1040 set_user_nice(current, MAX_NICE); 1041 1042 do { 1043 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1044 udelay(torture_random(&rand) & 0x3ff); 1045 if (cur_ops->cb_barrier != NULL && 1046 torture_random(&rand) % (nfakewriters * 8) == 0) { 1047 cur_ops->cb_barrier(); 1048 } else if (gp_normal == gp_exp) { 1049 if (torture_random(&rand) & 0x80) 1050 cur_ops->sync(); 1051 else 1052 cur_ops->exp_sync(); 1053 } else if (gp_normal) { 1054 cur_ops->sync(); 1055 } else { 1056 cur_ops->exp_sync(); 1057 } 1058 stutter_wait("rcu_torture_fakewriter"); 1059 } while (!torture_must_stop()); 1060 1061 torture_kthread_stopping("rcu_torture_fakewriter"); 1062 return 0; 1063 } 1064 1065 static void rcutorture_trace_dump(void) 1066 { 1067 static atomic_t beenhere = ATOMIC_INIT(0); 1068 1069 if (atomic_read(&beenhere)) 1070 return; 1071 if (atomic_xchg(&beenhere, 1) != 0) 1072 return; 1073 ftrace_dump(DUMP_ALL); 1074 } 1075 1076 /* 1077 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1078 * incrementing the corresponding element of the pipeline array. The 1079 * counter in the element should never be greater than 1, otherwise, the 1080 * RCU implementation is broken. 1081 */ 1082 static void rcu_torture_timer(unsigned long unused) 1083 { 1084 int idx; 1085 unsigned long started; 1086 unsigned long completed; 1087 static DEFINE_TORTURE_RANDOM(rand); 1088 static DEFINE_SPINLOCK(rand_lock); 1089 struct rcu_torture *p; 1090 int pipe_count; 1091 unsigned long long ts; 1092 1093 idx = cur_ops->readlock(); 1094 if (cur_ops->started) 1095 started = cur_ops->started(); 1096 else 1097 started = cur_ops->completed(); 1098 ts = rcu_trace_clock_local(); 1099 p = rcu_dereference_check(rcu_torture_current, 1100 rcu_read_lock_bh_held() || 1101 rcu_read_lock_sched_held() || 1102 srcu_read_lock_held(srcu_ctlp) || 1103 torturing_tasks()); 1104 if (p == NULL) { 1105 /* Leave because rcu_torture_writer is not yet underway */ 1106 cur_ops->readunlock(idx); 1107 return; 1108 } 1109 if (p->rtort_mbtest == 0) 1110 atomic_inc(&n_rcu_torture_mberror); 1111 spin_lock(&rand_lock); 1112 cur_ops->read_delay(&rand); 1113 n_rcu_torture_timers++; 1114 spin_unlock(&rand_lock); 1115 preempt_disable(); 1116 pipe_count = p->rtort_pipe_count; 1117 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1118 /* Should not happen, but... */ 1119 pipe_count = RCU_TORTURE_PIPE_LEN; 1120 } 1121 completed = cur_ops->completed(); 1122 if (pipe_count > 1) { 1123 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1124 started, completed); 1125 rcutorture_trace_dump(); 1126 } 1127 __this_cpu_inc(rcu_torture_count[pipe_count]); 1128 completed = completed - started; 1129 if (cur_ops->started) 1130 completed++; 1131 if (completed > RCU_TORTURE_PIPE_LEN) { 1132 /* Should not happen, but... */ 1133 completed = RCU_TORTURE_PIPE_LEN; 1134 } 1135 __this_cpu_inc(rcu_torture_batch[completed]); 1136 preempt_enable(); 1137 cur_ops->readunlock(idx); 1138 } 1139 1140 /* 1141 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1142 * incrementing the corresponding element of the pipeline array. The 1143 * counter in the element should never be greater than 1, otherwise, the 1144 * RCU implementation is broken. 1145 */ 1146 static int 1147 rcu_torture_reader(void *arg) 1148 { 1149 unsigned long started; 1150 unsigned long completed; 1151 int idx; 1152 DEFINE_TORTURE_RANDOM(rand); 1153 struct rcu_torture *p; 1154 int pipe_count; 1155 struct timer_list t; 1156 unsigned long long ts; 1157 1158 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1159 set_user_nice(current, MAX_NICE); 1160 if (irqreader && cur_ops->irq_capable) 1161 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1162 1163 do { 1164 if (irqreader && cur_ops->irq_capable) { 1165 if (!timer_pending(&t)) 1166 mod_timer(&t, jiffies + 1); 1167 } 1168 idx = cur_ops->readlock(); 1169 if (cur_ops->started) 1170 started = cur_ops->started(); 1171 else 1172 started = cur_ops->completed(); 1173 ts = rcu_trace_clock_local(); 1174 p = rcu_dereference_check(rcu_torture_current, 1175 rcu_read_lock_bh_held() || 1176 rcu_read_lock_sched_held() || 1177 srcu_read_lock_held(srcu_ctlp) || 1178 torturing_tasks()); 1179 if (p == NULL) { 1180 /* Wait for rcu_torture_writer to get underway */ 1181 cur_ops->readunlock(idx); 1182 schedule_timeout_interruptible(HZ); 1183 continue; 1184 } 1185 if (p->rtort_mbtest == 0) 1186 atomic_inc(&n_rcu_torture_mberror); 1187 cur_ops->read_delay(&rand); 1188 preempt_disable(); 1189 pipe_count = p->rtort_pipe_count; 1190 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1191 /* Should not happen, but... */ 1192 pipe_count = RCU_TORTURE_PIPE_LEN; 1193 } 1194 completed = cur_ops->completed(); 1195 if (pipe_count > 1) { 1196 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1197 ts, started, completed); 1198 rcutorture_trace_dump(); 1199 } 1200 __this_cpu_inc(rcu_torture_count[pipe_count]); 1201 completed = completed - started; 1202 if (cur_ops->started) 1203 completed++; 1204 if (completed > RCU_TORTURE_PIPE_LEN) { 1205 /* Should not happen, but... */ 1206 completed = RCU_TORTURE_PIPE_LEN; 1207 } 1208 __this_cpu_inc(rcu_torture_batch[completed]); 1209 preempt_enable(); 1210 cur_ops->readunlock(idx); 1211 cond_resched_rcu_qs(); 1212 stutter_wait("rcu_torture_reader"); 1213 } while (!torture_must_stop()); 1214 if (irqreader && cur_ops->irq_capable) { 1215 del_timer_sync(&t); 1216 destroy_timer_on_stack(&t); 1217 } 1218 torture_kthread_stopping("rcu_torture_reader"); 1219 return 0; 1220 } 1221 1222 /* 1223 * Print torture statistics. Caller must ensure that there is only 1224 * one call to this function at a given time!!! This is normally 1225 * accomplished by relying on the module system to only have one copy 1226 * of the module loaded, and then by giving the rcu_torture_stats 1227 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1228 * thread is not running). 1229 */ 1230 static void 1231 rcu_torture_stats_print(void) 1232 { 1233 int cpu; 1234 int i; 1235 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1236 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1237 static unsigned long rtcv_snap = ULONG_MAX; 1238 1239 for_each_possible_cpu(cpu) { 1240 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1241 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1242 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1243 } 1244 } 1245 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1246 if (pipesummary[i] != 0) 1247 break; 1248 } 1249 1250 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1251 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1252 rcu_torture_current, 1253 rcu_torture_current_version, 1254 list_empty(&rcu_torture_freelist), 1255 atomic_read(&n_rcu_torture_alloc), 1256 atomic_read(&n_rcu_torture_alloc_fail), 1257 atomic_read(&n_rcu_torture_free)); 1258 pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ", 1259 atomic_read(&n_rcu_torture_mberror), 1260 n_rcu_torture_boost_ktrerror, 1261 n_rcu_torture_boost_rterror); 1262 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1263 n_rcu_torture_boost_failure, 1264 n_rcu_torture_boosts, 1265 n_rcu_torture_timers); 1266 torture_onoff_stats(); 1267 pr_cont("barrier: %ld/%ld:%ld ", 1268 n_barrier_successes, 1269 n_barrier_attempts, 1270 n_rcu_torture_barrier_error); 1271 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1272 1273 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1274 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1275 n_rcu_torture_barrier_error != 0 || 1276 n_rcu_torture_boost_ktrerror != 0 || 1277 n_rcu_torture_boost_rterror != 0 || 1278 n_rcu_torture_boost_failure != 0 || 1279 i > 1) { 1280 pr_cont("%s", "!!! "); 1281 atomic_inc(&n_rcu_torture_error); 1282 WARN_ON_ONCE(1); 1283 } 1284 pr_cont("Reader Pipe: "); 1285 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1286 pr_cont(" %ld", pipesummary[i]); 1287 pr_cont("\n"); 1288 1289 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1290 pr_cont("Reader Batch: "); 1291 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1292 pr_cont(" %ld", batchsummary[i]); 1293 pr_cont("\n"); 1294 1295 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1296 pr_cont("Free-Block Circulation: "); 1297 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1298 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1299 } 1300 pr_cont("\n"); 1301 1302 if (cur_ops->stats) 1303 cur_ops->stats(); 1304 if (rtcv_snap == rcu_torture_current_version && 1305 rcu_torture_current != NULL) { 1306 int __maybe_unused flags; 1307 unsigned long __maybe_unused gpnum; 1308 unsigned long __maybe_unused completed; 1309 1310 rcutorture_get_gp_data(cur_ops->ttype, 1311 &flags, &gpnum, &completed); 1312 pr_alert("??? Writer stall state %d g%lu c%lu f%#x\n", 1313 rcu_torture_writer_state, 1314 gpnum, completed, flags); 1315 show_rcu_gp_kthreads(); 1316 rcutorture_trace_dump(); 1317 } 1318 rtcv_snap = rcu_torture_current_version; 1319 } 1320 1321 /* 1322 * Periodically prints torture statistics, if periodic statistics printing 1323 * was specified via the stat_interval module parameter. 1324 */ 1325 static int 1326 rcu_torture_stats(void *arg) 1327 { 1328 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1329 do { 1330 schedule_timeout_interruptible(stat_interval * HZ); 1331 rcu_torture_stats_print(); 1332 torture_shutdown_absorb("rcu_torture_stats"); 1333 } while (!torture_must_stop()); 1334 torture_kthread_stopping("rcu_torture_stats"); 1335 return 0; 1336 } 1337 1338 static inline void 1339 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1340 { 1341 pr_alert("%s" TORTURE_FLAG 1342 "--- %s: nreaders=%d nfakewriters=%d " 1343 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1344 "shuffle_interval=%d stutter=%d irqreader=%d " 1345 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1346 "test_boost=%d/%d test_boost_interval=%d " 1347 "test_boost_duration=%d shutdown_secs=%d " 1348 "stall_cpu=%d stall_cpu_holdoff=%d " 1349 "n_barrier_cbs=%d " 1350 "onoff_interval=%d onoff_holdoff=%d\n", 1351 torture_type, tag, nrealreaders, nfakewriters, 1352 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1353 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1354 test_boost, cur_ops->can_boost, 1355 test_boost_interval, test_boost_duration, shutdown_secs, 1356 stall_cpu, stall_cpu_holdoff, 1357 n_barrier_cbs, 1358 onoff_interval, onoff_holdoff); 1359 } 1360 1361 static void rcutorture_booster_cleanup(int cpu) 1362 { 1363 struct task_struct *t; 1364 1365 if (boost_tasks[cpu] == NULL) 1366 return; 1367 mutex_lock(&boost_mutex); 1368 t = boost_tasks[cpu]; 1369 boost_tasks[cpu] = NULL; 1370 mutex_unlock(&boost_mutex); 1371 1372 /* This must be outside of the mutex, otherwise deadlock! */ 1373 torture_stop_kthread(rcu_torture_boost, t); 1374 } 1375 1376 static int rcutorture_booster_init(int cpu) 1377 { 1378 int retval; 1379 1380 if (boost_tasks[cpu] != NULL) 1381 return 0; /* Already created, nothing more to do. */ 1382 1383 /* Don't allow time recalculation while creating a new task. */ 1384 mutex_lock(&boost_mutex); 1385 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1386 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1387 cpu_to_node(cpu), 1388 "rcu_torture_boost"); 1389 if (IS_ERR(boost_tasks[cpu])) { 1390 retval = PTR_ERR(boost_tasks[cpu]); 1391 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1392 n_rcu_torture_boost_ktrerror++; 1393 boost_tasks[cpu] = NULL; 1394 mutex_unlock(&boost_mutex); 1395 return retval; 1396 } 1397 kthread_bind(boost_tasks[cpu], cpu); 1398 wake_up_process(boost_tasks[cpu]); 1399 mutex_unlock(&boost_mutex); 1400 return 0; 1401 } 1402 1403 /* 1404 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1405 * induces a CPU stall for the time specified by stall_cpu. 1406 */ 1407 static int rcu_torture_stall(void *args) 1408 { 1409 unsigned long stop_at; 1410 1411 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1412 if (stall_cpu_holdoff > 0) { 1413 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1414 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1415 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1416 } 1417 if (!kthread_should_stop()) { 1418 stop_at = get_seconds() + stall_cpu; 1419 /* RCU CPU stall is expected behavior in following code. */ 1420 pr_alert("rcu_torture_stall start.\n"); 1421 rcu_read_lock(); 1422 preempt_disable(); 1423 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1424 continue; /* Induce RCU CPU stall warning. */ 1425 preempt_enable(); 1426 rcu_read_unlock(); 1427 pr_alert("rcu_torture_stall end.\n"); 1428 } 1429 torture_shutdown_absorb("rcu_torture_stall"); 1430 while (!kthread_should_stop()) 1431 schedule_timeout_interruptible(10 * HZ); 1432 return 0; 1433 } 1434 1435 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1436 static int __init rcu_torture_stall_init(void) 1437 { 1438 if (stall_cpu <= 0) 1439 return 0; 1440 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1441 } 1442 1443 /* Callback function for RCU barrier testing. */ 1444 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1445 { 1446 atomic_inc(&barrier_cbs_invoked); 1447 } 1448 1449 /* kthread function to register callbacks used to test RCU barriers. */ 1450 static int rcu_torture_barrier_cbs(void *arg) 1451 { 1452 long myid = (long)arg; 1453 bool lastphase = 0; 1454 bool newphase; 1455 struct rcu_head rcu; 1456 1457 init_rcu_head_on_stack(&rcu); 1458 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1459 set_user_nice(current, MAX_NICE); 1460 do { 1461 wait_event(barrier_cbs_wq[myid], 1462 (newphase = 1463 smp_load_acquire(&barrier_phase)) != lastphase || 1464 torture_must_stop()); 1465 lastphase = newphase; 1466 if (torture_must_stop()) 1467 break; 1468 /* 1469 * The above smp_load_acquire() ensures barrier_phase load 1470 * is ordered before the folloiwng ->call(). 1471 */ 1472 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1473 if (atomic_dec_and_test(&barrier_cbs_count)) 1474 wake_up(&barrier_wq); 1475 } while (!torture_must_stop()); 1476 if (cur_ops->cb_barrier != NULL) 1477 cur_ops->cb_barrier(); 1478 destroy_rcu_head_on_stack(&rcu); 1479 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1480 return 0; 1481 } 1482 1483 /* kthread function to drive and coordinate RCU barrier testing. */ 1484 static int rcu_torture_barrier(void *arg) 1485 { 1486 int i; 1487 1488 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1489 do { 1490 atomic_set(&barrier_cbs_invoked, 0); 1491 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1492 /* Ensure barrier_phase ordered after prior assignments. */ 1493 smp_store_release(&barrier_phase, !barrier_phase); 1494 for (i = 0; i < n_barrier_cbs; i++) 1495 wake_up(&barrier_cbs_wq[i]); 1496 wait_event(barrier_wq, 1497 atomic_read(&barrier_cbs_count) == 0 || 1498 torture_must_stop()); 1499 if (torture_must_stop()) 1500 break; 1501 n_barrier_attempts++; 1502 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1503 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1504 n_rcu_torture_barrier_error++; 1505 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1506 atomic_read(&barrier_cbs_invoked), 1507 n_barrier_cbs); 1508 WARN_ON_ONCE(1); 1509 } 1510 n_barrier_successes++; 1511 schedule_timeout_interruptible(HZ / 10); 1512 } while (!torture_must_stop()); 1513 torture_kthread_stopping("rcu_torture_barrier"); 1514 return 0; 1515 } 1516 1517 /* Initialize RCU barrier testing. */ 1518 static int rcu_torture_barrier_init(void) 1519 { 1520 int i; 1521 int ret; 1522 1523 if (n_barrier_cbs <= 0) 1524 return 0; 1525 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1526 pr_alert("%s" TORTURE_FLAG 1527 " Call or barrier ops missing for %s,\n", 1528 torture_type, cur_ops->name); 1529 pr_alert("%s" TORTURE_FLAG 1530 " RCU barrier testing omitted from run.\n", 1531 torture_type); 1532 return 0; 1533 } 1534 atomic_set(&barrier_cbs_count, 0); 1535 atomic_set(&barrier_cbs_invoked, 0); 1536 barrier_cbs_tasks = 1537 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1538 GFP_KERNEL); 1539 barrier_cbs_wq = 1540 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1541 GFP_KERNEL); 1542 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1543 return -ENOMEM; 1544 for (i = 0; i < n_barrier_cbs; i++) { 1545 init_waitqueue_head(&barrier_cbs_wq[i]); 1546 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1547 (void *)(long)i, 1548 barrier_cbs_tasks[i]); 1549 if (ret) 1550 return ret; 1551 } 1552 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1553 } 1554 1555 /* Clean up after RCU barrier testing. */ 1556 static void rcu_torture_barrier_cleanup(void) 1557 { 1558 int i; 1559 1560 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1561 if (barrier_cbs_tasks != NULL) { 1562 for (i = 0; i < n_barrier_cbs; i++) 1563 torture_stop_kthread(rcu_torture_barrier_cbs, 1564 barrier_cbs_tasks[i]); 1565 kfree(barrier_cbs_tasks); 1566 barrier_cbs_tasks = NULL; 1567 } 1568 if (barrier_cbs_wq != NULL) { 1569 kfree(barrier_cbs_wq); 1570 barrier_cbs_wq = NULL; 1571 } 1572 } 1573 1574 static int rcutorture_cpu_notify(struct notifier_block *self, 1575 unsigned long action, void *hcpu) 1576 { 1577 long cpu = (long)hcpu; 1578 1579 switch (action) { 1580 case CPU_ONLINE: 1581 case CPU_DOWN_FAILED: 1582 (void)rcutorture_booster_init(cpu); 1583 break; 1584 case CPU_DOWN_PREPARE: 1585 rcutorture_booster_cleanup(cpu); 1586 break; 1587 default: 1588 break; 1589 } 1590 return NOTIFY_OK; 1591 } 1592 1593 static struct notifier_block rcutorture_cpu_nb = { 1594 .notifier_call = rcutorture_cpu_notify, 1595 }; 1596 1597 static void 1598 rcu_torture_cleanup(void) 1599 { 1600 int i; 1601 1602 rcutorture_record_test_transition(); 1603 if (torture_cleanup_begin()) { 1604 if (cur_ops->cb_barrier != NULL) 1605 cur_ops->cb_barrier(); 1606 return; 1607 } 1608 1609 rcu_torture_barrier_cleanup(); 1610 torture_stop_kthread(rcu_torture_stall, stall_task); 1611 torture_stop_kthread(rcu_torture_writer, writer_task); 1612 1613 if (reader_tasks) { 1614 for (i = 0; i < nrealreaders; i++) 1615 torture_stop_kthread(rcu_torture_reader, 1616 reader_tasks[i]); 1617 kfree(reader_tasks); 1618 } 1619 rcu_torture_current = NULL; 1620 1621 if (fakewriter_tasks) { 1622 for (i = 0; i < nfakewriters; i++) { 1623 torture_stop_kthread(rcu_torture_fakewriter, 1624 fakewriter_tasks[i]); 1625 } 1626 kfree(fakewriter_tasks); 1627 fakewriter_tasks = NULL; 1628 } 1629 1630 torture_stop_kthread(rcu_torture_stats, stats_task); 1631 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1632 for (i = 0; i < ncbflooders; i++) 1633 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1634 if ((test_boost == 1 && cur_ops->can_boost) || 1635 test_boost == 2) { 1636 unregister_cpu_notifier(&rcutorture_cpu_nb); 1637 for_each_possible_cpu(i) 1638 rcutorture_booster_cleanup(i); 1639 } 1640 1641 /* 1642 * Wait for all RCU callbacks to fire, then do flavor-specific 1643 * cleanup operations. 1644 */ 1645 if (cur_ops->cb_barrier != NULL) 1646 cur_ops->cb_barrier(); 1647 if (cur_ops->cleanup != NULL) 1648 cur_ops->cleanup(); 1649 1650 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1651 1652 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1653 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1654 else if (torture_onoff_failures()) 1655 rcu_torture_print_module_parms(cur_ops, 1656 "End of test: RCU_HOTPLUG"); 1657 else 1658 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1659 torture_cleanup_end(); 1660 } 1661 1662 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1663 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1664 { 1665 } 1666 1667 static void rcu_torture_err_cb(struct rcu_head *rhp) 1668 { 1669 /* 1670 * This -might- happen due to race conditions, but is unlikely. 1671 * The scenario that leads to this happening is that the 1672 * first of the pair of duplicate callbacks is queued, 1673 * someone else starts a grace period that includes that 1674 * callback, then the second of the pair must wait for the 1675 * next grace period. Unlikely, but can happen. If it 1676 * does happen, the debug-objects subsystem won't have splatted. 1677 */ 1678 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1679 } 1680 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1681 1682 /* 1683 * Verify that double-free causes debug-objects to complain, but only 1684 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1685 * cannot be carried out. 1686 */ 1687 static void rcu_test_debug_objects(void) 1688 { 1689 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1690 struct rcu_head rh1; 1691 struct rcu_head rh2; 1692 1693 init_rcu_head_on_stack(&rh1); 1694 init_rcu_head_on_stack(&rh2); 1695 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1696 1697 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1698 preempt_disable(); /* Prevent preemption from interrupting test. */ 1699 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1700 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1701 local_irq_disable(); /* Make it harder to start a new grace period. */ 1702 call_rcu(&rh2, rcu_torture_leak_cb); 1703 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1704 local_irq_enable(); 1705 rcu_read_unlock(); 1706 preempt_enable(); 1707 1708 /* Wait for them all to get done so we can safely return. */ 1709 rcu_barrier(); 1710 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1711 destroy_rcu_head_on_stack(&rh1); 1712 destroy_rcu_head_on_stack(&rh2); 1713 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1714 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1715 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1716 } 1717 1718 static int __init 1719 rcu_torture_init(void) 1720 { 1721 int i; 1722 int cpu; 1723 int firsterr = 0; 1724 static struct rcu_torture_ops *torture_ops[] = { 1725 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1726 &sched_ops, RCUTORTURE_TASKS_OPS 1727 }; 1728 1729 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1730 return -EBUSY; 1731 1732 /* Process args and tell the world that the torturer is on the job. */ 1733 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1734 cur_ops = torture_ops[i]; 1735 if (strcmp(torture_type, cur_ops->name) == 0) 1736 break; 1737 } 1738 if (i == ARRAY_SIZE(torture_ops)) { 1739 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1740 torture_type); 1741 pr_alert("rcu-torture types:"); 1742 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1743 pr_alert(" %s", torture_ops[i]->name); 1744 pr_alert("\n"); 1745 torture_init_end(); 1746 return -EINVAL; 1747 } 1748 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1749 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1750 fqs_duration = 0; 1751 } 1752 if (cur_ops->init) 1753 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1754 1755 if (nreaders >= 0) { 1756 nrealreaders = nreaders; 1757 } else { 1758 nrealreaders = num_online_cpus() - 2 - nreaders; 1759 if (nrealreaders <= 0) 1760 nrealreaders = 1; 1761 } 1762 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1763 1764 /* Set up the freelist. */ 1765 1766 INIT_LIST_HEAD(&rcu_torture_freelist); 1767 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1768 rcu_tortures[i].rtort_mbtest = 0; 1769 list_add_tail(&rcu_tortures[i].rtort_free, 1770 &rcu_torture_freelist); 1771 } 1772 1773 /* Initialize the statistics so that each run gets its own numbers. */ 1774 1775 rcu_torture_current = NULL; 1776 rcu_torture_current_version = 0; 1777 atomic_set(&n_rcu_torture_alloc, 0); 1778 atomic_set(&n_rcu_torture_alloc_fail, 0); 1779 atomic_set(&n_rcu_torture_free, 0); 1780 atomic_set(&n_rcu_torture_mberror, 0); 1781 atomic_set(&n_rcu_torture_error, 0); 1782 n_rcu_torture_barrier_error = 0; 1783 n_rcu_torture_boost_ktrerror = 0; 1784 n_rcu_torture_boost_rterror = 0; 1785 n_rcu_torture_boost_failure = 0; 1786 n_rcu_torture_boosts = 0; 1787 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1788 atomic_set(&rcu_torture_wcount[i], 0); 1789 for_each_possible_cpu(cpu) { 1790 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1791 per_cpu(rcu_torture_count, cpu)[i] = 0; 1792 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1793 } 1794 } 1795 1796 /* Start up the kthreads. */ 1797 1798 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1799 writer_task); 1800 if (firsterr) 1801 goto unwind; 1802 if (nfakewriters > 0) { 1803 fakewriter_tasks = kzalloc(nfakewriters * 1804 sizeof(fakewriter_tasks[0]), 1805 GFP_KERNEL); 1806 if (fakewriter_tasks == NULL) { 1807 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1808 firsterr = -ENOMEM; 1809 goto unwind; 1810 } 1811 } 1812 for (i = 0; i < nfakewriters; i++) { 1813 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1814 NULL, fakewriter_tasks[i]); 1815 if (firsterr) 1816 goto unwind; 1817 } 1818 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1819 GFP_KERNEL); 1820 if (reader_tasks == NULL) { 1821 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1822 firsterr = -ENOMEM; 1823 goto unwind; 1824 } 1825 for (i = 0; i < nrealreaders; i++) { 1826 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1827 reader_tasks[i]); 1828 if (firsterr) 1829 goto unwind; 1830 } 1831 if (stat_interval > 0) { 1832 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1833 stats_task); 1834 if (firsterr) 1835 goto unwind; 1836 } 1837 if (test_no_idle_hz && shuffle_interval > 0) { 1838 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1839 if (firsterr) 1840 goto unwind; 1841 } 1842 if (stutter < 0) 1843 stutter = 0; 1844 if (stutter) { 1845 firsterr = torture_stutter_init(stutter * HZ); 1846 if (firsterr) 1847 goto unwind; 1848 } 1849 if (fqs_duration < 0) 1850 fqs_duration = 0; 1851 if (fqs_duration) { 1852 /* Create the fqs thread */ 1853 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1854 fqs_task); 1855 if (firsterr) 1856 goto unwind; 1857 } 1858 if (test_boost_interval < 1) 1859 test_boost_interval = 1; 1860 if (test_boost_duration < 2) 1861 test_boost_duration = 2; 1862 if ((test_boost == 1 && cur_ops->can_boost) || 1863 test_boost == 2) { 1864 1865 boost_starttime = jiffies + test_boost_interval * HZ; 1866 register_cpu_notifier(&rcutorture_cpu_nb); 1867 for_each_possible_cpu(i) { 1868 if (cpu_is_offline(i)) 1869 continue; /* Heuristic: CPU can go offline. */ 1870 firsterr = rcutorture_booster_init(i); 1871 if (firsterr) 1872 goto unwind; 1873 } 1874 } 1875 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1876 if (firsterr) 1877 goto unwind; 1878 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1879 if (firsterr) 1880 goto unwind; 1881 firsterr = rcu_torture_stall_init(); 1882 if (firsterr) 1883 goto unwind; 1884 firsterr = rcu_torture_barrier_init(); 1885 if (firsterr) 1886 goto unwind; 1887 if (object_debug) 1888 rcu_test_debug_objects(); 1889 if (cbflood_n_burst > 0) { 1890 /* Create the cbflood threads */ 1891 ncbflooders = (num_online_cpus() + 3) / 4; 1892 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1893 GFP_KERNEL); 1894 if (!cbflood_task) { 1895 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1896 firsterr = -ENOMEM; 1897 goto unwind; 1898 } 1899 for (i = 0; i < ncbflooders; i++) { 1900 firsterr = torture_create_kthread(rcu_torture_cbflood, 1901 NULL, 1902 cbflood_task[i]); 1903 if (firsterr) 1904 goto unwind; 1905 } 1906 } 1907 rcutorture_record_test_transition(); 1908 torture_init_end(); 1909 return 0; 1910 1911 unwind: 1912 torture_init_end(); 1913 rcu_torture_cleanup(); 1914 return firsterr; 1915 } 1916 1917 module_init(rcu_torture_init); 1918 module_exit(rcu_torture_cleanup); 1919