1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 #include <linux/vmalloc.h> 53 54 MODULE_LICENSE("GPL"); 55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 56 57 58 torture_param(int, cbflood_inter_holdoff, HZ, 59 "Holdoff between floods (jiffies)"); 60 torture_param(int, cbflood_intra_holdoff, 1, 61 "Holdoff between bursts (jiffies)"); 62 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 63 torture_param(int, cbflood_n_per_burst, 20000, 64 "# callbacks per burst in flood"); 65 torture_param(int, fqs_duration, 0, 66 "Duration of fqs bursts (us), 0 to disable"); 67 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 68 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 69 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 70 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 71 torture_param(bool, gp_normal, false, 72 "Use normal (non-expedited) GP wait primitives"); 73 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 74 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 75 torture_param(int, n_barrier_cbs, 0, 76 "# of callbacks/kthreads for barrier testing"); 77 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 78 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 79 torture_param(int, object_debug, 0, 80 "Enable debug-object double call_rcu() testing"); 81 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 82 torture_param(int, onoff_interval, 0, 83 "Time between CPU hotplugs (s), 0=disable"); 84 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 85 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 86 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 87 torture_param(int, stall_cpu_holdoff, 10, 88 "Time to wait before starting stall (s)."); 89 torture_param(int, stat_interval, 60, 90 "Number of seconds between stats printk()s"); 91 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 92 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 93 torture_param(int, test_boost_duration, 4, 94 "Duration of each boost test, seconds."); 95 torture_param(int, test_boost_interval, 7, 96 "Interval between boost tests, seconds."); 97 torture_param(bool, test_no_idle_hz, true, 98 "Test support for tickless idle CPUs"); 99 torture_param(bool, verbose, true, 100 "Enable verbose debugging printk()s"); 101 102 static char *torture_type = "rcu"; 103 module_param(torture_type, charp, 0444); 104 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 105 106 static int nrealreaders; 107 static int ncbflooders; 108 static struct task_struct *writer_task; 109 static struct task_struct **fakewriter_tasks; 110 static struct task_struct **reader_tasks; 111 static struct task_struct *stats_task; 112 static struct task_struct **cbflood_task; 113 static struct task_struct *fqs_task; 114 static struct task_struct *boost_tasks[NR_CPUS]; 115 static struct task_struct *stall_task; 116 static struct task_struct **barrier_cbs_tasks; 117 static struct task_struct *barrier_task; 118 119 #define RCU_TORTURE_PIPE_LEN 10 120 121 struct rcu_torture { 122 struct rcu_head rtort_rcu; 123 int rtort_pipe_count; 124 struct list_head rtort_free; 125 int rtort_mbtest; 126 }; 127 128 static LIST_HEAD(rcu_torture_freelist); 129 static struct rcu_torture __rcu *rcu_torture_current; 130 static unsigned long rcu_torture_current_version; 131 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 132 static DEFINE_SPINLOCK(rcu_torture_lock); 133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 134 rcu_torture_count) = { 0 }; 135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 136 rcu_torture_batch) = { 0 }; 137 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 138 static atomic_t n_rcu_torture_alloc; 139 static atomic_t n_rcu_torture_alloc_fail; 140 static atomic_t n_rcu_torture_free; 141 static atomic_t n_rcu_torture_mberror; 142 static atomic_t n_rcu_torture_error; 143 static long n_rcu_torture_barrier_error; 144 static long n_rcu_torture_boost_ktrerror; 145 static long n_rcu_torture_boost_rterror; 146 static long n_rcu_torture_boost_failure; 147 static long n_rcu_torture_boosts; 148 static long n_rcu_torture_timers; 149 static long n_barrier_attempts; 150 static long n_barrier_successes; 151 static atomic_long_t n_cbfloods; 152 static struct list_head rcu_torture_removed; 153 154 static int rcu_torture_writer_state; 155 #define RTWS_FIXED_DELAY 0 156 #define RTWS_DELAY 1 157 #define RTWS_REPLACE 2 158 #define RTWS_DEF_FREE 3 159 #define RTWS_EXP_SYNC 4 160 #define RTWS_COND_GET 5 161 #define RTWS_COND_SYNC 6 162 #define RTWS_SYNC 7 163 #define RTWS_STUTTER 8 164 #define RTWS_STOPPING 9 165 166 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 167 #define RCUTORTURE_RUNNABLE_INIT 1 168 #else 169 #define RCUTORTURE_RUNNABLE_INIT 0 170 #endif 171 static int torture_runnable = RCUTORTURE_RUNNABLE_INIT; 172 module_param(torture_runnable, int, 0444); 173 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 174 175 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 176 #define rcu_can_boost() 1 177 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 178 #define rcu_can_boost() 0 179 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 180 181 #ifdef CONFIG_RCU_TRACE 182 static u64 notrace rcu_trace_clock_local(void) 183 { 184 u64 ts = trace_clock_local(); 185 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 186 return ts; 187 } 188 #else /* #ifdef CONFIG_RCU_TRACE */ 189 static u64 notrace rcu_trace_clock_local(void) 190 { 191 return 0ULL; 192 } 193 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 194 195 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 196 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 197 /* and boost task create/destroy. */ 198 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 199 static bool barrier_phase; /* Test phase. */ 200 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 201 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 202 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 203 204 /* 205 * Allocate an element from the rcu_tortures pool. 206 */ 207 static struct rcu_torture * 208 rcu_torture_alloc(void) 209 { 210 struct list_head *p; 211 212 spin_lock_bh(&rcu_torture_lock); 213 if (list_empty(&rcu_torture_freelist)) { 214 atomic_inc(&n_rcu_torture_alloc_fail); 215 spin_unlock_bh(&rcu_torture_lock); 216 return NULL; 217 } 218 atomic_inc(&n_rcu_torture_alloc); 219 p = rcu_torture_freelist.next; 220 list_del_init(p); 221 spin_unlock_bh(&rcu_torture_lock); 222 return container_of(p, struct rcu_torture, rtort_free); 223 } 224 225 /* 226 * Free an element to the rcu_tortures pool. 227 */ 228 static void 229 rcu_torture_free(struct rcu_torture *p) 230 { 231 atomic_inc(&n_rcu_torture_free); 232 spin_lock_bh(&rcu_torture_lock); 233 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 234 spin_unlock_bh(&rcu_torture_lock); 235 } 236 237 /* 238 * Operations vector for selecting different types of tests. 239 */ 240 241 struct rcu_torture_ops { 242 int ttype; 243 void (*init)(void); 244 int (*readlock)(void); 245 void (*read_delay)(struct torture_random_state *rrsp); 246 void (*readunlock)(int idx); 247 unsigned long (*started)(void); 248 unsigned long (*completed)(void); 249 void (*deferred_free)(struct rcu_torture *p); 250 void (*sync)(void); 251 void (*exp_sync)(void); 252 unsigned long (*get_state)(void); 253 void (*cond_sync)(unsigned long oldstate); 254 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 255 void (*cb_barrier)(void); 256 void (*fqs)(void); 257 void (*stats)(void); 258 int irq_capable; 259 int can_boost; 260 const char *name; 261 }; 262 263 static struct rcu_torture_ops *cur_ops; 264 265 /* 266 * Definitions for rcu torture testing. 267 */ 268 269 static int rcu_torture_read_lock(void) __acquires(RCU) 270 { 271 rcu_read_lock(); 272 return 0; 273 } 274 275 static void rcu_read_delay(struct torture_random_state *rrsp) 276 { 277 const unsigned long shortdelay_us = 200; 278 const unsigned long longdelay_ms = 50; 279 280 /* We want a short delay sometimes to make a reader delay the grace 281 * period, and we want a long delay occasionally to trigger 282 * force_quiescent_state. */ 283 284 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 285 mdelay(longdelay_ms); 286 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 287 udelay(shortdelay_us); 288 #ifdef CONFIG_PREEMPT 289 if (!preempt_count() && 290 !(torture_random(rrsp) % (nrealreaders * 20000))) 291 preempt_schedule(); /* No QS if preempt_disable() in effect */ 292 #endif 293 } 294 295 static void rcu_torture_read_unlock(int idx) __releases(RCU) 296 { 297 rcu_read_unlock(); 298 } 299 300 /* 301 * Update callback in the pipe. This should be invoked after a grace period. 302 */ 303 static bool 304 rcu_torture_pipe_update_one(struct rcu_torture *rp) 305 { 306 int i; 307 308 i = rp->rtort_pipe_count; 309 if (i > RCU_TORTURE_PIPE_LEN) 310 i = RCU_TORTURE_PIPE_LEN; 311 atomic_inc(&rcu_torture_wcount[i]); 312 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 313 rp->rtort_mbtest = 0; 314 return true; 315 } 316 return false; 317 } 318 319 /* 320 * Update all callbacks in the pipe. Suitable for synchronous grace-period 321 * primitives. 322 */ 323 static void 324 rcu_torture_pipe_update(struct rcu_torture *old_rp) 325 { 326 struct rcu_torture *rp; 327 struct rcu_torture *rp1; 328 329 if (old_rp) 330 list_add(&old_rp->rtort_free, &rcu_torture_removed); 331 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 332 if (rcu_torture_pipe_update_one(rp)) { 333 list_del(&rp->rtort_free); 334 rcu_torture_free(rp); 335 } 336 } 337 } 338 339 static void 340 rcu_torture_cb(struct rcu_head *p) 341 { 342 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 343 344 if (torture_must_stop_irq()) { 345 /* Test is ending, just drop callbacks on the floor. */ 346 /* The next initialization will pick up the pieces. */ 347 return; 348 } 349 if (rcu_torture_pipe_update_one(rp)) 350 rcu_torture_free(rp); 351 else 352 cur_ops->deferred_free(rp); 353 } 354 355 static unsigned long rcu_no_completed(void) 356 { 357 return 0; 358 } 359 360 static void rcu_torture_deferred_free(struct rcu_torture *p) 361 { 362 call_rcu(&p->rtort_rcu, rcu_torture_cb); 363 } 364 365 static void rcu_sync_torture_init(void) 366 { 367 INIT_LIST_HEAD(&rcu_torture_removed); 368 } 369 370 static struct rcu_torture_ops rcu_ops = { 371 .ttype = RCU_FLAVOR, 372 .init = rcu_sync_torture_init, 373 .readlock = rcu_torture_read_lock, 374 .read_delay = rcu_read_delay, 375 .readunlock = rcu_torture_read_unlock, 376 .started = rcu_batches_started, 377 .completed = rcu_batches_completed, 378 .deferred_free = rcu_torture_deferred_free, 379 .sync = synchronize_rcu, 380 .exp_sync = synchronize_rcu_expedited, 381 .get_state = get_state_synchronize_rcu, 382 .cond_sync = cond_synchronize_rcu, 383 .call = call_rcu, 384 .cb_barrier = rcu_barrier, 385 .fqs = rcu_force_quiescent_state, 386 .stats = NULL, 387 .irq_capable = 1, 388 .can_boost = rcu_can_boost(), 389 .name = "rcu" 390 }; 391 392 /* 393 * Definitions for rcu_bh torture testing. 394 */ 395 396 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 397 { 398 rcu_read_lock_bh(); 399 return 0; 400 } 401 402 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 403 { 404 rcu_read_unlock_bh(); 405 } 406 407 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 408 { 409 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 410 } 411 412 static struct rcu_torture_ops rcu_bh_ops = { 413 .ttype = RCU_BH_FLAVOR, 414 .init = rcu_sync_torture_init, 415 .readlock = rcu_bh_torture_read_lock, 416 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 417 .readunlock = rcu_bh_torture_read_unlock, 418 .started = rcu_batches_started_bh, 419 .completed = rcu_batches_completed_bh, 420 .deferred_free = rcu_bh_torture_deferred_free, 421 .sync = synchronize_rcu_bh, 422 .exp_sync = synchronize_rcu_bh_expedited, 423 .call = call_rcu_bh, 424 .cb_barrier = rcu_barrier_bh, 425 .fqs = rcu_bh_force_quiescent_state, 426 .stats = NULL, 427 .irq_capable = 1, 428 .name = "rcu_bh" 429 }; 430 431 /* 432 * Don't even think about trying any of these in real life!!! 433 * The names includes "busted", and they really means it! 434 * The only purpose of these functions is to provide a buggy RCU 435 * implementation to make sure that rcutorture correctly emits 436 * buggy-RCU error messages. 437 */ 438 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 439 { 440 /* This is a deliberate bug for testing purposes only! */ 441 rcu_torture_cb(&p->rtort_rcu); 442 } 443 444 static void synchronize_rcu_busted(void) 445 { 446 /* This is a deliberate bug for testing purposes only! */ 447 } 448 449 static void 450 call_rcu_busted(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 451 { 452 /* This is a deliberate bug for testing purposes only! */ 453 func(head); 454 } 455 456 static struct rcu_torture_ops rcu_busted_ops = { 457 .ttype = INVALID_RCU_FLAVOR, 458 .init = rcu_sync_torture_init, 459 .readlock = rcu_torture_read_lock, 460 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 461 .readunlock = rcu_torture_read_unlock, 462 .started = rcu_no_completed, 463 .completed = rcu_no_completed, 464 .deferred_free = rcu_busted_torture_deferred_free, 465 .sync = synchronize_rcu_busted, 466 .exp_sync = synchronize_rcu_busted, 467 .call = call_rcu_busted, 468 .cb_barrier = NULL, 469 .fqs = NULL, 470 .stats = NULL, 471 .irq_capable = 1, 472 .name = "rcu_busted" 473 }; 474 475 /* 476 * Definitions for srcu torture testing. 477 */ 478 479 DEFINE_STATIC_SRCU(srcu_ctl); 480 481 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 482 { 483 return srcu_read_lock(&srcu_ctl); 484 } 485 486 static void srcu_read_delay(struct torture_random_state *rrsp) 487 { 488 long delay; 489 const long uspertick = 1000000 / HZ; 490 const long longdelay = 10; 491 492 /* We want there to be long-running readers, but not all the time. */ 493 494 delay = torture_random(rrsp) % 495 (nrealreaders * 2 * longdelay * uspertick); 496 if (!delay) 497 schedule_timeout_interruptible(longdelay); 498 else 499 rcu_read_delay(rrsp); 500 } 501 502 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 503 { 504 srcu_read_unlock(&srcu_ctl, idx); 505 } 506 507 static unsigned long srcu_torture_completed(void) 508 { 509 return srcu_batches_completed(&srcu_ctl); 510 } 511 512 static void srcu_torture_deferred_free(struct rcu_torture *rp) 513 { 514 call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb); 515 } 516 517 static void srcu_torture_synchronize(void) 518 { 519 synchronize_srcu(&srcu_ctl); 520 } 521 522 static void srcu_torture_call(struct rcu_head *head, 523 void (*func)(struct rcu_head *head)) 524 { 525 call_srcu(&srcu_ctl, head, func); 526 } 527 528 static void srcu_torture_barrier(void) 529 { 530 srcu_barrier(&srcu_ctl); 531 } 532 533 static void srcu_torture_stats(void) 534 { 535 int cpu; 536 int idx = srcu_ctl.completed & 0x1; 537 538 pr_alert("%s%s per-CPU(idx=%d):", 539 torture_type, TORTURE_FLAG, idx); 540 for_each_possible_cpu(cpu) { 541 long c0, c1; 542 543 c0 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx]; 544 c1 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]; 545 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 546 } 547 pr_cont("\n"); 548 } 549 550 static void srcu_torture_synchronize_expedited(void) 551 { 552 synchronize_srcu_expedited(&srcu_ctl); 553 } 554 555 static struct rcu_torture_ops srcu_ops = { 556 .ttype = SRCU_FLAVOR, 557 .init = rcu_sync_torture_init, 558 .readlock = srcu_torture_read_lock, 559 .read_delay = srcu_read_delay, 560 .readunlock = srcu_torture_read_unlock, 561 .started = NULL, 562 .completed = srcu_torture_completed, 563 .deferred_free = srcu_torture_deferred_free, 564 .sync = srcu_torture_synchronize, 565 .exp_sync = srcu_torture_synchronize_expedited, 566 .call = srcu_torture_call, 567 .cb_barrier = srcu_torture_barrier, 568 .stats = srcu_torture_stats, 569 .name = "srcu" 570 }; 571 572 /* 573 * Definitions for sched torture testing. 574 */ 575 576 static int sched_torture_read_lock(void) 577 { 578 preempt_disable(); 579 return 0; 580 } 581 582 static void sched_torture_read_unlock(int idx) 583 { 584 preempt_enable(); 585 } 586 587 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 588 { 589 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 590 } 591 592 static struct rcu_torture_ops sched_ops = { 593 .ttype = RCU_SCHED_FLAVOR, 594 .init = rcu_sync_torture_init, 595 .readlock = sched_torture_read_lock, 596 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 597 .readunlock = sched_torture_read_unlock, 598 .started = rcu_batches_started_sched, 599 .completed = rcu_batches_completed_sched, 600 .deferred_free = rcu_sched_torture_deferred_free, 601 .sync = synchronize_sched, 602 .exp_sync = synchronize_sched_expedited, 603 .call = call_rcu_sched, 604 .cb_barrier = rcu_barrier_sched, 605 .fqs = rcu_sched_force_quiescent_state, 606 .stats = NULL, 607 .irq_capable = 1, 608 .name = "sched" 609 }; 610 611 #ifdef CONFIG_TASKS_RCU 612 613 /* 614 * Definitions for RCU-tasks torture testing. 615 */ 616 617 static int tasks_torture_read_lock(void) 618 { 619 return 0; 620 } 621 622 static void tasks_torture_read_unlock(int idx) 623 { 624 } 625 626 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 627 { 628 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 629 } 630 631 static struct rcu_torture_ops tasks_ops = { 632 .ttype = RCU_TASKS_FLAVOR, 633 .init = rcu_sync_torture_init, 634 .readlock = tasks_torture_read_lock, 635 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 636 .readunlock = tasks_torture_read_unlock, 637 .started = rcu_no_completed, 638 .completed = rcu_no_completed, 639 .deferred_free = rcu_tasks_torture_deferred_free, 640 .sync = synchronize_rcu_tasks, 641 .exp_sync = synchronize_rcu_tasks, 642 .call = call_rcu_tasks, 643 .cb_barrier = rcu_barrier_tasks, 644 .fqs = NULL, 645 .stats = NULL, 646 .irq_capable = 1, 647 .name = "tasks" 648 }; 649 650 #define RCUTORTURE_TASKS_OPS &tasks_ops, 651 652 #else /* #ifdef CONFIG_TASKS_RCU */ 653 654 #define RCUTORTURE_TASKS_OPS 655 656 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 657 658 /* 659 * RCU torture priority-boost testing. Runs one real-time thread per 660 * CPU for moderate bursts, repeatedly registering RCU callbacks and 661 * spinning waiting for them to be invoked. If a given callback takes 662 * too long to be invoked, we assume that priority inversion has occurred. 663 */ 664 665 struct rcu_boost_inflight { 666 struct rcu_head rcu; 667 int inflight; 668 }; 669 670 static void rcu_torture_boost_cb(struct rcu_head *head) 671 { 672 struct rcu_boost_inflight *rbip = 673 container_of(head, struct rcu_boost_inflight, rcu); 674 675 /* Ensure RCU-core accesses precede clearing ->inflight */ 676 smp_store_release(&rbip->inflight, 0); 677 } 678 679 static int rcu_torture_boost(void *arg) 680 { 681 unsigned long call_rcu_time; 682 unsigned long endtime; 683 unsigned long oldstarttime; 684 struct rcu_boost_inflight rbi = { .inflight = 0 }; 685 struct sched_param sp; 686 687 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 688 689 /* Set real-time priority. */ 690 sp.sched_priority = 1; 691 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 692 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 693 n_rcu_torture_boost_rterror++; 694 } 695 696 init_rcu_head_on_stack(&rbi.rcu); 697 /* Each pass through the following loop does one boost-test cycle. */ 698 do { 699 /* Wait for the next test interval. */ 700 oldstarttime = boost_starttime; 701 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 702 schedule_timeout_interruptible(oldstarttime - jiffies); 703 stutter_wait("rcu_torture_boost"); 704 if (torture_must_stop()) 705 goto checkwait; 706 } 707 708 /* Do one boost-test interval. */ 709 endtime = oldstarttime + test_boost_duration * HZ; 710 call_rcu_time = jiffies; 711 while (ULONG_CMP_LT(jiffies, endtime)) { 712 /* If we don't have a callback in flight, post one. */ 713 if (!smp_load_acquire(&rbi.inflight)) { 714 /* RCU core before ->inflight = 1. */ 715 smp_store_release(&rbi.inflight, 1); 716 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 717 if (jiffies - call_rcu_time > 718 test_boost_duration * HZ - HZ / 2) { 719 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 720 n_rcu_torture_boost_failure++; 721 } 722 call_rcu_time = jiffies; 723 } 724 cond_resched_rcu_qs(); 725 stutter_wait("rcu_torture_boost"); 726 if (torture_must_stop()) 727 goto checkwait; 728 } 729 730 /* 731 * Set the start time of the next test interval. 732 * Yes, this is vulnerable to long delays, but such 733 * delays simply cause a false negative for the next 734 * interval. Besides, we are running at RT priority, 735 * so delays should be relatively rare. 736 */ 737 while (oldstarttime == boost_starttime && 738 !kthread_should_stop()) { 739 if (mutex_trylock(&boost_mutex)) { 740 boost_starttime = jiffies + 741 test_boost_interval * HZ; 742 n_rcu_torture_boosts++; 743 mutex_unlock(&boost_mutex); 744 break; 745 } 746 schedule_timeout_uninterruptible(1); 747 } 748 749 /* Go do the stutter. */ 750 checkwait: stutter_wait("rcu_torture_boost"); 751 } while (!torture_must_stop()); 752 753 /* Clean up and exit. */ 754 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 755 torture_shutdown_absorb("rcu_torture_boost"); 756 schedule_timeout_uninterruptible(1); 757 } 758 destroy_rcu_head_on_stack(&rbi.rcu); 759 torture_kthread_stopping("rcu_torture_boost"); 760 return 0; 761 } 762 763 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 764 { 765 } 766 767 /* 768 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 769 * to call_rcu() or analogous, increasing the probability of occurrence 770 * of callback-overflow corner cases. 771 */ 772 static int 773 rcu_torture_cbflood(void *arg) 774 { 775 int err = 1; 776 int i; 777 int j; 778 struct rcu_head *rhp; 779 780 if (cbflood_n_per_burst > 0 && 781 cbflood_inter_holdoff > 0 && 782 cbflood_intra_holdoff > 0 && 783 cur_ops->call && 784 cur_ops->cb_barrier) { 785 rhp = vmalloc(sizeof(*rhp) * 786 cbflood_n_burst * cbflood_n_per_burst); 787 err = !rhp; 788 } 789 if (err) { 790 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 791 while (!torture_must_stop()) 792 schedule_timeout_interruptible(HZ); 793 return 0; 794 } 795 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 796 do { 797 schedule_timeout_interruptible(cbflood_inter_holdoff); 798 atomic_long_inc(&n_cbfloods); 799 WARN_ON(signal_pending(current)); 800 for (i = 0; i < cbflood_n_burst; i++) { 801 for (j = 0; j < cbflood_n_per_burst; j++) { 802 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 803 rcu_torture_cbflood_cb); 804 } 805 schedule_timeout_interruptible(cbflood_intra_holdoff); 806 WARN_ON(signal_pending(current)); 807 } 808 cur_ops->cb_barrier(); 809 stutter_wait("rcu_torture_cbflood"); 810 } while (!torture_must_stop()); 811 vfree(rhp); 812 torture_kthread_stopping("rcu_torture_cbflood"); 813 return 0; 814 } 815 816 /* 817 * RCU torture force-quiescent-state kthread. Repeatedly induces 818 * bursts of calls to force_quiescent_state(), increasing the probability 819 * of occurrence of some important types of race conditions. 820 */ 821 static int 822 rcu_torture_fqs(void *arg) 823 { 824 unsigned long fqs_resume_time; 825 int fqs_burst_remaining; 826 827 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 828 do { 829 fqs_resume_time = jiffies + fqs_stutter * HZ; 830 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 831 !kthread_should_stop()) { 832 schedule_timeout_interruptible(1); 833 } 834 fqs_burst_remaining = fqs_duration; 835 while (fqs_burst_remaining > 0 && 836 !kthread_should_stop()) { 837 cur_ops->fqs(); 838 udelay(fqs_holdoff); 839 fqs_burst_remaining -= fqs_holdoff; 840 } 841 stutter_wait("rcu_torture_fqs"); 842 } while (!torture_must_stop()); 843 torture_kthread_stopping("rcu_torture_fqs"); 844 return 0; 845 } 846 847 /* 848 * RCU torture writer kthread. Repeatedly substitutes a new structure 849 * for that pointed to by rcu_torture_current, freeing the old structure 850 * after a series of grace periods (the "pipeline"). 851 */ 852 static int 853 rcu_torture_writer(void *arg) 854 { 855 bool can_expedite = !rcu_gp_is_expedited(); 856 int expediting = 0; 857 unsigned long gp_snap; 858 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 859 bool gp_sync1 = gp_sync; 860 int i; 861 struct rcu_torture *rp; 862 struct rcu_torture *old_rp; 863 static DEFINE_TORTURE_RANDOM(rand); 864 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 865 RTWS_COND_GET, RTWS_SYNC }; 866 int nsynctypes = 0; 867 868 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 869 pr_alert("%s" TORTURE_FLAG 870 " Grace periods expedited from boot/sysfs for %s,\n", 871 torture_type, cur_ops->name); 872 pr_alert("%s" TORTURE_FLAG 873 " Testing of dynamic grace-period expediting diabled.\n", 874 torture_type); 875 876 /* Initialize synctype[] array. If none set, take default. */ 877 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 878 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 879 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 880 synctype[nsynctypes++] = RTWS_COND_GET; 881 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 882 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 883 if (gp_exp1 && cur_ops->exp_sync) 884 synctype[nsynctypes++] = RTWS_EXP_SYNC; 885 else if (gp_exp && !cur_ops->exp_sync) 886 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 887 if (gp_normal1 && cur_ops->deferred_free) 888 synctype[nsynctypes++] = RTWS_DEF_FREE; 889 else if (gp_normal && !cur_ops->deferred_free) 890 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 891 if (gp_sync1 && cur_ops->sync) 892 synctype[nsynctypes++] = RTWS_SYNC; 893 else if (gp_sync && !cur_ops->sync) 894 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 895 if (WARN_ONCE(nsynctypes == 0, 896 "rcu_torture_writer: No update-side primitives.\n")) { 897 /* 898 * No updates primitives, so don't try updating. 899 * The resulting test won't be testing much, hence the 900 * above WARN_ONCE(). 901 */ 902 rcu_torture_writer_state = RTWS_STOPPING; 903 torture_kthread_stopping("rcu_torture_writer"); 904 } 905 906 do { 907 rcu_torture_writer_state = RTWS_FIXED_DELAY; 908 schedule_timeout_uninterruptible(1); 909 rp = rcu_torture_alloc(); 910 if (rp == NULL) 911 continue; 912 rp->rtort_pipe_count = 0; 913 rcu_torture_writer_state = RTWS_DELAY; 914 udelay(torture_random(&rand) & 0x3ff); 915 rcu_torture_writer_state = RTWS_REPLACE; 916 old_rp = rcu_dereference_check(rcu_torture_current, 917 current == writer_task); 918 rp->rtort_mbtest = 1; 919 rcu_assign_pointer(rcu_torture_current, rp); 920 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 921 if (old_rp) { 922 i = old_rp->rtort_pipe_count; 923 if (i > RCU_TORTURE_PIPE_LEN) 924 i = RCU_TORTURE_PIPE_LEN; 925 atomic_inc(&rcu_torture_wcount[i]); 926 old_rp->rtort_pipe_count++; 927 switch (synctype[torture_random(&rand) % nsynctypes]) { 928 case RTWS_DEF_FREE: 929 rcu_torture_writer_state = RTWS_DEF_FREE; 930 cur_ops->deferred_free(old_rp); 931 break; 932 case RTWS_EXP_SYNC: 933 rcu_torture_writer_state = RTWS_EXP_SYNC; 934 cur_ops->exp_sync(); 935 rcu_torture_pipe_update(old_rp); 936 break; 937 case RTWS_COND_GET: 938 rcu_torture_writer_state = RTWS_COND_GET; 939 gp_snap = cur_ops->get_state(); 940 i = torture_random(&rand) % 16; 941 if (i != 0) 942 schedule_timeout_interruptible(i); 943 udelay(torture_random(&rand) % 1000); 944 rcu_torture_writer_state = RTWS_COND_SYNC; 945 cur_ops->cond_sync(gp_snap); 946 rcu_torture_pipe_update(old_rp); 947 break; 948 case RTWS_SYNC: 949 rcu_torture_writer_state = RTWS_SYNC; 950 cur_ops->sync(); 951 rcu_torture_pipe_update(old_rp); 952 break; 953 default: 954 WARN_ON_ONCE(1); 955 break; 956 } 957 } 958 rcutorture_record_progress(++rcu_torture_current_version); 959 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 960 if (can_expedite && 961 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 962 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 963 if (expediting >= 0) 964 rcu_expedite_gp(); 965 else 966 rcu_unexpedite_gp(); 967 if (++expediting > 3) 968 expediting = -expediting; 969 } 970 rcu_torture_writer_state = RTWS_STUTTER; 971 stutter_wait("rcu_torture_writer"); 972 } while (!torture_must_stop()); 973 /* Reset expediting back to unexpedited. */ 974 if (expediting > 0) 975 expediting = -expediting; 976 while (can_expedite && expediting++ < 0) 977 rcu_unexpedite_gp(); 978 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 979 rcu_torture_writer_state = RTWS_STOPPING; 980 torture_kthread_stopping("rcu_torture_writer"); 981 return 0; 982 } 983 984 /* 985 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 986 * delay between calls. 987 */ 988 static int 989 rcu_torture_fakewriter(void *arg) 990 { 991 DEFINE_TORTURE_RANDOM(rand); 992 993 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 994 set_user_nice(current, MAX_NICE); 995 996 do { 997 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 998 udelay(torture_random(&rand) & 0x3ff); 999 if (cur_ops->cb_barrier != NULL && 1000 torture_random(&rand) % (nfakewriters * 8) == 0) { 1001 cur_ops->cb_barrier(); 1002 } else if (gp_normal == gp_exp) { 1003 if (torture_random(&rand) & 0x80) 1004 cur_ops->sync(); 1005 else 1006 cur_ops->exp_sync(); 1007 } else if (gp_normal) { 1008 cur_ops->sync(); 1009 } else { 1010 cur_ops->exp_sync(); 1011 } 1012 stutter_wait("rcu_torture_fakewriter"); 1013 } while (!torture_must_stop()); 1014 1015 torture_kthread_stopping("rcu_torture_fakewriter"); 1016 return 0; 1017 } 1018 1019 static void rcutorture_trace_dump(void) 1020 { 1021 static atomic_t beenhere = ATOMIC_INIT(0); 1022 1023 if (atomic_read(&beenhere)) 1024 return; 1025 if (atomic_xchg(&beenhere, 1) != 0) 1026 return; 1027 ftrace_dump(DUMP_ALL); 1028 } 1029 1030 /* 1031 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1032 * incrementing the corresponding element of the pipeline array. The 1033 * counter in the element should never be greater than 1, otherwise, the 1034 * RCU implementation is broken. 1035 */ 1036 static void rcu_torture_timer(unsigned long unused) 1037 { 1038 int idx; 1039 unsigned long started; 1040 unsigned long completed; 1041 static DEFINE_TORTURE_RANDOM(rand); 1042 static DEFINE_SPINLOCK(rand_lock); 1043 struct rcu_torture *p; 1044 int pipe_count; 1045 unsigned long long ts; 1046 1047 idx = cur_ops->readlock(); 1048 if (cur_ops->started) 1049 started = cur_ops->started(); 1050 else 1051 started = cur_ops->completed(); 1052 ts = rcu_trace_clock_local(); 1053 p = rcu_dereference_check(rcu_torture_current, 1054 rcu_read_lock_bh_held() || 1055 rcu_read_lock_sched_held() || 1056 srcu_read_lock_held(&srcu_ctl)); 1057 if (p == NULL) { 1058 /* Leave because rcu_torture_writer is not yet underway */ 1059 cur_ops->readunlock(idx); 1060 return; 1061 } 1062 if (p->rtort_mbtest == 0) 1063 atomic_inc(&n_rcu_torture_mberror); 1064 spin_lock(&rand_lock); 1065 cur_ops->read_delay(&rand); 1066 n_rcu_torture_timers++; 1067 spin_unlock(&rand_lock); 1068 preempt_disable(); 1069 pipe_count = p->rtort_pipe_count; 1070 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1071 /* Should not happen, but... */ 1072 pipe_count = RCU_TORTURE_PIPE_LEN; 1073 } 1074 completed = cur_ops->completed(); 1075 if (pipe_count > 1) { 1076 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1077 started, completed); 1078 rcutorture_trace_dump(); 1079 } 1080 __this_cpu_inc(rcu_torture_count[pipe_count]); 1081 completed = completed - started; 1082 if (cur_ops->started) 1083 completed++; 1084 if (completed > RCU_TORTURE_PIPE_LEN) { 1085 /* Should not happen, but... */ 1086 completed = RCU_TORTURE_PIPE_LEN; 1087 } 1088 __this_cpu_inc(rcu_torture_batch[completed]); 1089 preempt_enable(); 1090 cur_ops->readunlock(idx); 1091 } 1092 1093 /* 1094 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1095 * incrementing the corresponding element of the pipeline array. The 1096 * counter in the element should never be greater than 1, otherwise, the 1097 * RCU implementation is broken. 1098 */ 1099 static int 1100 rcu_torture_reader(void *arg) 1101 { 1102 unsigned long started; 1103 unsigned long completed; 1104 int idx; 1105 DEFINE_TORTURE_RANDOM(rand); 1106 struct rcu_torture *p; 1107 int pipe_count; 1108 struct timer_list t; 1109 unsigned long long ts; 1110 1111 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1112 set_user_nice(current, MAX_NICE); 1113 if (irqreader && cur_ops->irq_capable) 1114 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1115 1116 do { 1117 if (irqreader && cur_ops->irq_capable) { 1118 if (!timer_pending(&t)) 1119 mod_timer(&t, jiffies + 1); 1120 } 1121 idx = cur_ops->readlock(); 1122 if (cur_ops->started) 1123 started = cur_ops->started(); 1124 else 1125 started = cur_ops->completed(); 1126 ts = rcu_trace_clock_local(); 1127 p = rcu_dereference_check(rcu_torture_current, 1128 rcu_read_lock_bh_held() || 1129 rcu_read_lock_sched_held() || 1130 srcu_read_lock_held(&srcu_ctl)); 1131 if (p == NULL) { 1132 /* Wait for rcu_torture_writer to get underway */ 1133 cur_ops->readunlock(idx); 1134 schedule_timeout_interruptible(HZ); 1135 continue; 1136 } 1137 if (p->rtort_mbtest == 0) 1138 atomic_inc(&n_rcu_torture_mberror); 1139 cur_ops->read_delay(&rand); 1140 preempt_disable(); 1141 pipe_count = p->rtort_pipe_count; 1142 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1143 /* Should not happen, but... */ 1144 pipe_count = RCU_TORTURE_PIPE_LEN; 1145 } 1146 completed = cur_ops->completed(); 1147 if (pipe_count > 1) { 1148 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1149 ts, started, completed); 1150 rcutorture_trace_dump(); 1151 } 1152 __this_cpu_inc(rcu_torture_count[pipe_count]); 1153 completed = completed - started; 1154 if (cur_ops->started) 1155 completed++; 1156 if (completed > RCU_TORTURE_PIPE_LEN) { 1157 /* Should not happen, but... */ 1158 completed = RCU_TORTURE_PIPE_LEN; 1159 } 1160 __this_cpu_inc(rcu_torture_batch[completed]); 1161 preempt_enable(); 1162 cur_ops->readunlock(idx); 1163 cond_resched_rcu_qs(); 1164 stutter_wait("rcu_torture_reader"); 1165 } while (!torture_must_stop()); 1166 if (irqreader && cur_ops->irq_capable) { 1167 del_timer_sync(&t); 1168 destroy_timer_on_stack(&t); 1169 } 1170 torture_kthread_stopping("rcu_torture_reader"); 1171 return 0; 1172 } 1173 1174 /* 1175 * Print torture statistics. Caller must ensure that there is only 1176 * one call to this function at a given time!!! This is normally 1177 * accomplished by relying on the module system to only have one copy 1178 * of the module loaded, and then by giving the rcu_torture_stats 1179 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1180 * thread is not running). 1181 */ 1182 static void 1183 rcu_torture_stats_print(void) 1184 { 1185 int cpu; 1186 int i; 1187 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1188 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1189 static unsigned long rtcv_snap = ULONG_MAX; 1190 1191 for_each_possible_cpu(cpu) { 1192 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1193 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1194 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1195 } 1196 } 1197 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1198 if (pipesummary[i] != 0) 1199 break; 1200 } 1201 1202 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1203 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1204 rcu_torture_current, 1205 rcu_torture_current_version, 1206 list_empty(&rcu_torture_freelist), 1207 atomic_read(&n_rcu_torture_alloc), 1208 atomic_read(&n_rcu_torture_alloc_fail), 1209 atomic_read(&n_rcu_torture_free)); 1210 pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ", 1211 atomic_read(&n_rcu_torture_mberror), 1212 n_rcu_torture_boost_ktrerror, 1213 n_rcu_torture_boost_rterror); 1214 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1215 n_rcu_torture_boost_failure, 1216 n_rcu_torture_boosts, 1217 n_rcu_torture_timers); 1218 torture_onoff_stats(); 1219 pr_cont("barrier: %ld/%ld:%ld ", 1220 n_barrier_successes, 1221 n_barrier_attempts, 1222 n_rcu_torture_barrier_error); 1223 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1224 1225 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1226 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1227 n_rcu_torture_barrier_error != 0 || 1228 n_rcu_torture_boost_ktrerror != 0 || 1229 n_rcu_torture_boost_rterror != 0 || 1230 n_rcu_torture_boost_failure != 0 || 1231 i > 1) { 1232 pr_cont("%s", "!!! "); 1233 atomic_inc(&n_rcu_torture_error); 1234 WARN_ON_ONCE(1); 1235 } 1236 pr_cont("Reader Pipe: "); 1237 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1238 pr_cont(" %ld", pipesummary[i]); 1239 pr_cont("\n"); 1240 1241 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1242 pr_cont("Reader Batch: "); 1243 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1244 pr_cont(" %ld", batchsummary[i]); 1245 pr_cont("\n"); 1246 1247 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1248 pr_cont("Free-Block Circulation: "); 1249 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1250 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1251 } 1252 pr_cont("\n"); 1253 1254 if (cur_ops->stats) 1255 cur_ops->stats(); 1256 if (rtcv_snap == rcu_torture_current_version && 1257 rcu_torture_current != NULL) { 1258 int __maybe_unused flags; 1259 unsigned long __maybe_unused gpnum; 1260 unsigned long __maybe_unused completed; 1261 1262 rcutorture_get_gp_data(cur_ops->ttype, 1263 &flags, &gpnum, &completed); 1264 pr_alert("??? Writer stall state %d g%lu c%lu f%#x\n", 1265 rcu_torture_writer_state, 1266 gpnum, completed, flags); 1267 show_rcu_gp_kthreads(); 1268 rcutorture_trace_dump(); 1269 } 1270 rtcv_snap = rcu_torture_current_version; 1271 } 1272 1273 /* 1274 * Periodically prints torture statistics, if periodic statistics printing 1275 * was specified via the stat_interval module parameter. 1276 */ 1277 static int 1278 rcu_torture_stats(void *arg) 1279 { 1280 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1281 do { 1282 schedule_timeout_interruptible(stat_interval * HZ); 1283 rcu_torture_stats_print(); 1284 torture_shutdown_absorb("rcu_torture_stats"); 1285 } while (!torture_must_stop()); 1286 torture_kthread_stopping("rcu_torture_stats"); 1287 return 0; 1288 } 1289 1290 static inline void 1291 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1292 { 1293 pr_alert("%s" TORTURE_FLAG 1294 "--- %s: nreaders=%d nfakewriters=%d " 1295 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1296 "shuffle_interval=%d stutter=%d irqreader=%d " 1297 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1298 "test_boost=%d/%d test_boost_interval=%d " 1299 "test_boost_duration=%d shutdown_secs=%d " 1300 "stall_cpu=%d stall_cpu_holdoff=%d " 1301 "n_barrier_cbs=%d " 1302 "onoff_interval=%d onoff_holdoff=%d\n", 1303 torture_type, tag, nrealreaders, nfakewriters, 1304 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1305 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1306 test_boost, cur_ops->can_boost, 1307 test_boost_interval, test_boost_duration, shutdown_secs, 1308 stall_cpu, stall_cpu_holdoff, 1309 n_barrier_cbs, 1310 onoff_interval, onoff_holdoff); 1311 } 1312 1313 static void rcutorture_booster_cleanup(int cpu) 1314 { 1315 struct task_struct *t; 1316 1317 if (boost_tasks[cpu] == NULL) 1318 return; 1319 mutex_lock(&boost_mutex); 1320 t = boost_tasks[cpu]; 1321 boost_tasks[cpu] = NULL; 1322 mutex_unlock(&boost_mutex); 1323 1324 /* This must be outside of the mutex, otherwise deadlock! */ 1325 torture_stop_kthread(rcu_torture_boost, t); 1326 } 1327 1328 static int rcutorture_booster_init(int cpu) 1329 { 1330 int retval; 1331 1332 if (boost_tasks[cpu] != NULL) 1333 return 0; /* Already created, nothing more to do. */ 1334 1335 /* Don't allow time recalculation while creating a new task. */ 1336 mutex_lock(&boost_mutex); 1337 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1338 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1339 cpu_to_node(cpu), 1340 "rcu_torture_boost"); 1341 if (IS_ERR(boost_tasks[cpu])) { 1342 retval = PTR_ERR(boost_tasks[cpu]); 1343 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1344 n_rcu_torture_boost_ktrerror++; 1345 boost_tasks[cpu] = NULL; 1346 mutex_unlock(&boost_mutex); 1347 return retval; 1348 } 1349 kthread_bind(boost_tasks[cpu], cpu); 1350 wake_up_process(boost_tasks[cpu]); 1351 mutex_unlock(&boost_mutex); 1352 return 0; 1353 } 1354 1355 /* 1356 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1357 * induces a CPU stall for the time specified by stall_cpu. 1358 */ 1359 static int rcu_torture_stall(void *args) 1360 { 1361 unsigned long stop_at; 1362 1363 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1364 if (stall_cpu_holdoff > 0) { 1365 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1366 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1367 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1368 } 1369 if (!kthread_should_stop()) { 1370 stop_at = get_seconds() + stall_cpu; 1371 /* RCU CPU stall is expected behavior in following code. */ 1372 pr_alert("rcu_torture_stall start.\n"); 1373 rcu_read_lock(); 1374 preempt_disable(); 1375 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1376 continue; /* Induce RCU CPU stall warning. */ 1377 preempt_enable(); 1378 rcu_read_unlock(); 1379 pr_alert("rcu_torture_stall end.\n"); 1380 } 1381 torture_shutdown_absorb("rcu_torture_stall"); 1382 while (!kthread_should_stop()) 1383 schedule_timeout_interruptible(10 * HZ); 1384 return 0; 1385 } 1386 1387 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1388 static int __init rcu_torture_stall_init(void) 1389 { 1390 if (stall_cpu <= 0) 1391 return 0; 1392 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1393 } 1394 1395 /* Callback function for RCU barrier testing. */ 1396 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1397 { 1398 atomic_inc(&barrier_cbs_invoked); 1399 } 1400 1401 /* kthread function to register callbacks used to test RCU barriers. */ 1402 static int rcu_torture_barrier_cbs(void *arg) 1403 { 1404 long myid = (long)arg; 1405 bool lastphase = 0; 1406 bool newphase; 1407 struct rcu_head rcu; 1408 1409 init_rcu_head_on_stack(&rcu); 1410 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1411 set_user_nice(current, MAX_NICE); 1412 do { 1413 wait_event(barrier_cbs_wq[myid], 1414 (newphase = 1415 smp_load_acquire(&barrier_phase)) != lastphase || 1416 torture_must_stop()); 1417 lastphase = newphase; 1418 if (torture_must_stop()) 1419 break; 1420 /* 1421 * The above smp_load_acquire() ensures barrier_phase load 1422 * is ordered before the folloiwng ->call(). 1423 */ 1424 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1425 if (atomic_dec_and_test(&barrier_cbs_count)) 1426 wake_up(&barrier_wq); 1427 } while (!torture_must_stop()); 1428 if (cur_ops->cb_barrier != NULL) 1429 cur_ops->cb_barrier(); 1430 destroy_rcu_head_on_stack(&rcu); 1431 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1432 return 0; 1433 } 1434 1435 /* kthread function to drive and coordinate RCU barrier testing. */ 1436 static int rcu_torture_barrier(void *arg) 1437 { 1438 int i; 1439 1440 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1441 do { 1442 atomic_set(&barrier_cbs_invoked, 0); 1443 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1444 /* Ensure barrier_phase ordered after prior assignments. */ 1445 smp_store_release(&barrier_phase, !barrier_phase); 1446 for (i = 0; i < n_barrier_cbs; i++) 1447 wake_up(&barrier_cbs_wq[i]); 1448 wait_event(barrier_wq, 1449 atomic_read(&barrier_cbs_count) == 0 || 1450 torture_must_stop()); 1451 if (torture_must_stop()) 1452 break; 1453 n_barrier_attempts++; 1454 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1455 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1456 n_rcu_torture_barrier_error++; 1457 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1458 atomic_read(&barrier_cbs_invoked), 1459 n_barrier_cbs); 1460 WARN_ON_ONCE(1); 1461 } 1462 n_barrier_successes++; 1463 schedule_timeout_interruptible(HZ / 10); 1464 } while (!torture_must_stop()); 1465 torture_kthread_stopping("rcu_torture_barrier"); 1466 return 0; 1467 } 1468 1469 /* Initialize RCU barrier testing. */ 1470 static int rcu_torture_barrier_init(void) 1471 { 1472 int i; 1473 int ret; 1474 1475 if (n_barrier_cbs == 0) 1476 return 0; 1477 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1478 pr_alert("%s" TORTURE_FLAG 1479 " Call or barrier ops missing for %s,\n", 1480 torture_type, cur_ops->name); 1481 pr_alert("%s" TORTURE_FLAG 1482 " RCU barrier testing omitted from run.\n", 1483 torture_type); 1484 return 0; 1485 } 1486 atomic_set(&barrier_cbs_count, 0); 1487 atomic_set(&barrier_cbs_invoked, 0); 1488 barrier_cbs_tasks = 1489 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1490 GFP_KERNEL); 1491 barrier_cbs_wq = 1492 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1493 GFP_KERNEL); 1494 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1495 return -ENOMEM; 1496 for (i = 0; i < n_barrier_cbs; i++) { 1497 init_waitqueue_head(&barrier_cbs_wq[i]); 1498 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1499 (void *)(long)i, 1500 barrier_cbs_tasks[i]); 1501 if (ret) 1502 return ret; 1503 } 1504 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1505 } 1506 1507 /* Clean up after RCU barrier testing. */ 1508 static void rcu_torture_barrier_cleanup(void) 1509 { 1510 int i; 1511 1512 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1513 if (barrier_cbs_tasks != NULL) { 1514 for (i = 0; i < n_barrier_cbs; i++) 1515 torture_stop_kthread(rcu_torture_barrier_cbs, 1516 barrier_cbs_tasks[i]); 1517 kfree(barrier_cbs_tasks); 1518 barrier_cbs_tasks = NULL; 1519 } 1520 if (barrier_cbs_wq != NULL) { 1521 kfree(barrier_cbs_wq); 1522 barrier_cbs_wq = NULL; 1523 } 1524 } 1525 1526 static int rcutorture_cpu_notify(struct notifier_block *self, 1527 unsigned long action, void *hcpu) 1528 { 1529 long cpu = (long)hcpu; 1530 1531 switch (action) { 1532 case CPU_ONLINE: 1533 case CPU_DOWN_FAILED: 1534 (void)rcutorture_booster_init(cpu); 1535 break; 1536 case CPU_DOWN_PREPARE: 1537 rcutorture_booster_cleanup(cpu); 1538 break; 1539 default: 1540 break; 1541 } 1542 return NOTIFY_OK; 1543 } 1544 1545 static struct notifier_block rcutorture_cpu_nb = { 1546 .notifier_call = rcutorture_cpu_notify, 1547 }; 1548 1549 static void 1550 rcu_torture_cleanup(void) 1551 { 1552 int i; 1553 1554 rcutorture_record_test_transition(); 1555 if (torture_cleanup_begin()) { 1556 if (cur_ops->cb_barrier != NULL) 1557 cur_ops->cb_barrier(); 1558 return; 1559 } 1560 1561 rcu_torture_barrier_cleanup(); 1562 torture_stop_kthread(rcu_torture_stall, stall_task); 1563 torture_stop_kthread(rcu_torture_writer, writer_task); 1564 1565 if (reader_tasks) { 1566 for (i = 0; i < nrealreaders; i++) 1567 torture_stop_kthread(rcu_torture_reader, 1568 reader_tasks[i]); 1569 kfree(reader_tasks); 1570 } 1571 rcu_torture_current = NULL; 1572 1573 if (fakewriter_tasks) { 1574 for (i = 0; i < nfakewriters; i++) { 1575 torture_stop_kthread(rcu_torture_fakewriter, 1576 fakewriter_tasks[i]); 1577 } 1578 kfree(fakewriter_tasks); 1579 fakewriter_tasks = NULL; 1580 } 1581 1582 torture_stop_kthread(rcu_torture_stats, stats_task); 1583 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1584 for (i = 0; i < ncbflooders; i++) 1585 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1586 if ((test_boost == 1 && cur_ops->can_boost) || 1587 test_boost == 2) { 1588 unregister_cpu_notifier(&rcutorture_cpu_nb); 1589 for_each_possible_cpu(i) 1590 rcutorture_booster_cleanup(i); 1591 } 1592 1593 /* Wait for all RCU callbacks to fire. */ 1594 1595 if (cur_ops->cb_barrier != NULL) 1596 cur_ops->cb_barrier(); 1597 1598 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1599 1600 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1601 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1602 else if (torture_onoff_failures()) 1603 rcu_torture_print_module_parms(cur_ops, 1604 "End of test: RCU_HOTPLUG"); 1605 else 1606 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1607 torture_cleanup_end(); 1608 } 1609 1610 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1611 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1612 { 1613 } 1614 1615 static void rcu_torture_err_cb(struct rcu_head *rhp) 1616 { 1617 /* 1618 * This -might- happen due to race conditions, but is unlikely. 1619 * The scenario that leads to this happening is that the 1620 * first of the pair of duplicate callbacks is queued, 1621 * someone else starts a grace period that includes that 1622 * callback, then the second of the pair must wait for the 1623 * next grace period. Unlikely, but can happen. If it 1624 * does happen, the debug-objects subsystem won't have splatted. 1625 */ 1626 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1627 } 1628 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1629 1630 /* 1631 * Verify that double-free causes debug-objects to complain, but only 1632 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1633 * cannot be carried out. 1634 */ 1635 static void rcu_test_debug_objects(void) 1636 { 1637 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1638 struct rcu_head rh1; 1639 struct rcu_head rh2; 1640 1641 init_rcu_head_on_stack(&rh1); 1642 init_rcu_head_on_stack(&rh2); 1643 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1644 1645 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1646 preempt_disable(); /* Prevent preemption from interrupting test. */ 1647 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1648 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1649 local_irq_disable(); /* Make it harder to start a new grace period. */ 1650 call_rcu(&rh2, rcu_torture_leak_cb); 1651 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1652 local_irq_enable(); 1653 rcu_read_unlock(); 1654 preempt_enable(); 1655 1656 /* Wait for them all to get done so we can safely return. */ 1657 rcu_barrier(); 1658 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1659 destroy_rcu_head_on_stack(&rh1); 1660 destroy_rcu_head_on_stack(&rh2); 1661 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1662 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1663 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1664 } 1665 1666 static int __init 1667 rcu_torture_init(void) 1668 { 1669 int i; 1670 int cpu; 1671 int firsterr = 0; 1672 static struct rcu_torture_ops *torture_ops[] = { 1673 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &sched_ops, 1674 RCUTORTURE_TASKS_OPS 1675 }; 1676 1677 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1678 return -EBUSY; 1679 1680 /* Process args and tell the world that the torturer is on the job. */ 1681 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1682 cur_ops = torture_ops[i]; 1683 if (strcmp(torture_type, cur_ops->name) == 0) 1684 break; 1685 } 1686 if (i == ARRAY_SIZE(torture_ops)) { 1687 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1688 torture_type); 1689 pr_alert("rcu-torture types:"); 1690 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1691 pr_alert(" %s", torture_ops[i]->name); 1692 pr_alert("\n"); 1693 torture_init_end(); 1694 return -EINVAL; 1695 } 1696 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1697 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1698 fqs_duration = 0; 1699 } 1700 if (cur_ops->init) 1701 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1702 1703 if (nreaders >= 0) { 1704 nrealreaders = nreaders; 1705 } else { 1706 nrealreaders = num_online_cpus() - 2 - nreaders; 1707 if (nrealreaders <= 0) 1708 nrealreaders = 1; 1709 } 1710 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1711 1712 /* Set up the freelist. */ 1713 1714 INIT_LIST_HEAD(&rcu_torture_freelist); 1715 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1716 rcu_tortures[i].rtort_mbtest = 0; 1717 list_add_tail(&rcu_tortures[i].rtort_free, 1718 &rcu_torture_freelist); 1719 } 1720 1721 /* Initialize the statistics so that each run gets its own numbers. */ 1722 1723 rcu_torture_current = NULL; 1724 rcu_torture_current_version = 0; 1725 atomic_set(&n_rcu_torture_alloc, 0); 1726 atomic_set(&n_rcu_torture_alloc_fail, 0); 1727 atomic_set(&n_rcu_torture_free, 0); 1728 atomic_set(&n_rcu_torture_mberror, 0); 1729 atomic_set(&n_rcu_torture_error, 0); 1730 n_rcu_torture_barrier_error = 0; 1731 n_rcu_torture_boost_ktrerror = 0; 1732 n_rcu_torture_boost_rterror = 0; 1733 n_rcu_torture_boost_failure = 0; 1734 n_rcu_torture_boosts = 0; 1735 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1736 atomic_set(&rcu_torture_wcount[i], 0); 1737 for_each_possible_cpu(cpu) { 1738 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1739 per_cpu(rcu_torture_count, cpu)[i] = 0; 1740 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1741 } 1742 } 1743 1744 /* Start up the kthreads. */ 1745 1746 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1747 writer_task); 1748 if (firsterr) 1749 goto unwind; 1750 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 1751 GFP_KERNEL); 1752 if (fakewriter_tasks == NULL) { 1753 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1754 firsterr = -ENOMEM; 1755 goto unwind; 1756 } 1757 for (i = 0; i < nfakewriters; i++) { 1758 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1759 NULL, fakewriter_tasks[i]); 1760 if (firsterr) 1761 goto unwind; 1762 } 1763 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1764 GFP_KERNEL); 1765 if (reader_tasks == NULL) { 1766 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1767 firsterr = -ENOMEM; 1768 goto unwind; 1769 } 1770 for (i = 0; i < nrealreaders; i++) { 1771 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1772 reader_tasks[i]); 1773 if (firsterr) 1774 goto unwind; 1775 } 1776 if (stat_interval > 0) { 1777 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1778 stats_task); 1779 if (firsterr) 1780 goto unwind; 1781 } 1782 if (test_no_idle_hz) { 1783 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1784 if (firsterr) 1785 goto unwind; 1786 } 1787 if (stutter < 0) 1788 stutter = 0; 1789 if (stutter) { 1790 firsterr = torture_stutter_init(stutter * HZ); 1791 if (firsterr) 1792 goto unwind; 1793 } 1794 if (fqs_duration < 0) 1795 fqs_duration = 0; 1796 if (fqs_duration) { 1797 /* Create the fqs thread */ 1798 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1799 fqs_task); 1800 if (firsterr) 1801 goto unwind; 1802 } 1803 if (test_boost_interval < 1) 1804 test_boost_interval = 1; 1805 if (test_boost_duration < 2) 1806 test_boost_duration = 2; 1807 if ((test_boost == 1 && cur_ops->can_boost) || 1808 test_boost == 2) { 1809 1810 boost_starttime = jiffies + test_boost_interval * HZ; 1811 register_cpu_notifier(&rcutorture_cpu_nb); 1812 for_each_possible_cpu(i) { 1813 if (cpu_is_offline(i)) 1814 continue; /* Heuristic: CPU can go offline. */ 1815 firsterr = rcutorture_booster_init(i); 1816 if (firsterr) 1817 goto unwind; 1818 } 1819 } 1820 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1821 if (firsterr) 1822 goto unwind; 1823 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1824 if (firsterr) 1825 goto unwind; 1826 firsterr = rcu_torture_stall_init(); 1827 if (firsterr) 1828 goto unwind; 1829 firsterr = rcu_torture_barrier_init(); 1830 if (firsterr) 1831 goto unwind; 1832 if (object_debug) 1833 rcu_test_debug_objects(); 1834 if (cbflood_n_burst > 0) { 1835 /* Create the cbflood threads */ 1836 ncbflooders = (num_online_cpus() + 3) / 4; 1837 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1838 GFP_KERNEL); 1839 if (!cbflood_task) { 1840 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1841 firsterr = -ENOMEM; 1842 goto unwind; 1843 } 1844 for (i = 0; i < ncbflooders; i++) { 1845 firsterr = torture_create_kthread(rcu_torture_cbflood, 1846 NULL, 1847 cbflood_task[i]); 1848 if (firsterr) 1849 goto unwind; 1850 } 1851 } 1852 rcutorture_record_test_transition(); 1853 torture_init_end(); 1854 return 0; 1855 1856 unwind: 1857 torture_init_end(); 1858 rcu_torture_cleanup(); 1859 return firsterr; 1860 } 1861 1862 module_init(rcu_torture_init); 1863 module_exit(rcu_torture_cleanup); 1864