1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@joshtriplett.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched/signal.h> 36 #include <uapi/linux/sched/types.h> 37 #include <linux/atomic.h> 38 #include <linux/bitops.h> 39 #include <linux/completion.h> 40 #include <linux/moduleparam.h> 41 #include <linux/percpu.h> 42 #include <linux/notifier.h> 43 #include <linux/reboot.h> 44 #include <linux/freezer.h> 45 #include <linux/cpu.h> 46 #include <linux/delay.h> 47 #include <linux/stat.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/trace_clock.h> 51 #include <asm/byteorder.h> 52 #include <linux/torture.h> 53 #include <linux/vmalloc.h> 54 55 MODULE_LICENSE("GPL"); 56 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 57 58 59 torture_param(int, cbflood_inter_holdoff, HZ, 60 "Holdoff between floods (jiffies)"); 61 torture_param(int, cbflood_intra_holdoff, 1, 62 "Holdoff between bursts (jiffies)"); 63 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable"); 64 torture_param(int, cbflood_n_per_burst, 20000, 65 "# callbacks per burst in flood"); 66 torture_param(int, fqs_duration, 0, 67 "Duration of fqs bursts (us), 0 to disable"); 68 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 69 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 70 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 71 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 72 torture_param(bool, gp_normal, false, 73 "Use normal (non-expedited) GP wait primitives"); 74 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 75 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 76 torture_param(int, n_barrier_cbs, 0, 77 "# of callbacks/kthreads for barrier testing"); 78 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 79 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 80 torture_param(int, object_debug, 0, 81 "Enable debug-object double call_rcu() testing"); 82 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 83 torture_param(int, onoff_interval, 0, 84 "Time between CPU hotplugs (s), 0=disable"); 85 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 86 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 87 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 88 torture_param(int, stall_cpu_holdoff, 10, 89 "Time to wait before starting stall (s)."); 90 torture_param(int, stat_interval, 60, 91 "Number of seconds between stats printk()s"); 92 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 93 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 94 torture_param(int, test_boost_duration, 4, 95 "Duration of each boost test, seconds."); 96 torture_param(int, test_boost_interval, 7, 97 "Interval between boost tests, seconds."); 98 torture_param(bool, test_no_idle_hz, true, 99 "Test support for tickless idle CPUs"); 100 torture_param(bool, verbose, true, 101 "Enable verbose debugging printk()s"); 102 103 static char *torture_type = "rcu"; 104 module_param(torture_type, charp, 0444); 105 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 106 107 static int nrealreaders; 108 static int ncbflooders; 109 static struct task_struct *writer_task; 110 static struct task_struct **fakewriter_tasks; 111 static struct task_struct **reader_tasks; 112 static struct task_struct *stats_task; 113 static struct task_struct **cbflood_task; 114 static struct task_struct *fqs_task; 115 static struct task_struct *boost_tasks[NR_CPUS]; 116 static struct task_struct *stall_task; 117 static struct task_struct **barrier_cbs_tasks; 118 static struct task_struct *barrier_task; 119 120 #define RCU_TORTURE_PIPE_LEN 10 121 122 struct rcu_torture { 123 struct rcu_head rtort_rcu; 124 int rtort_pipe_count; 125 struct list_head rtort_free; 126 int rtort_mbtest; 127 }; 128 129 static LIST_HEAD(rcu_torture_freelist); 130 static struct rcu_torture __rcu *rcu_torture_current; 131 static unsigned long rcu_torture_current_version; 132 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 133 static DEFINE_SPINLOCK(rcu_torture_lock); 134 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 136 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 137 static atomic_t n_rcu_torture_alloc; 138 static atomic_t n_rcu_torture_alloc_fail; 139 static atomic_t n_rcu_torture_free; 140 static atomic_t n_rcu_torture_mberror; 141 static atomic_t n_rcu_torture_error; 142 static long n_rcu_torture_barrier_error; 143 static long n_rcu_torture_boost_ktrerror; 144 static long n_rcu_torture_boost_rterror; 145 static long n_rcu_torture_boost_failure; 146 static long n_rcu_torture_boosts; 147 static long n_rcu_torture_timers; 148 static long n_barrier_attempts; 149 static long n_barrier_successes; 150 static atomic_long_t n_cbfloods; 151 static struct list_head rcu_torture_removed; 152 153 static int rcu_torture_writer_state; 154 #define RTWS_FIXED_DELAY 0 155 #define RTWS_DELAY 1 156 #define RTWS_REPLACE 2 157 #define RTWS_DEF_FREE 3 158 #define RTWS_EXP_SYNC 4 159 #define RTWS_COND_GET 5 160 #define RTWS_COND_SYNC 6 161 #define RTWS_SYNC 7 162 #define RTWS_STUTTER 8 163 #define RTWS_STOPPING 9 164 static const char * const rcu_torture_writer_state_names[] = { 165 "RTWS_FIXED_DELAY", 166 "RTWS_DELAY", 167 "RTWS_REPLACE", 168 "RTWS_DEF_FREE", 169 "RTWS_EXP_SYNC", 170 "RTWS_COND_GET", 171 "RTWS_COND_SYNC", 172 "RTWS_SYNC", 173 "RTWS_STUTTER", 174 "RTWS_STOPPING", 175 }; 176 177 static const char *rcu_torture_writer_state_getname(void) 178 { 179 unsigned int i = READ_ONCE(rcu_torture_writer_state); 180 181 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 182 return "???"; 183 return rcu_torture_writer_state_names[i]; 184 } 185 186 static int torture_runnable = IS_ENABLED(MODULE); 187 module_param(torture_runnable, int, 0444); 188 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot"); 189 190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 191 #define rcu_can_boost() 1 192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 193 #define rcu_can_boost() 0 194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 195 196 #ifdef CONFIG_RCU_TRACE 197 static u64 notrace rcu_trace_clock_local(void) 198 { 199 u64 ts = trace_clock_local(); 200 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 201 return ts; 202 } 203 #else /* #ifdef CONFIG_RCU_TRACE */ 204 static u64 notrace rcu_trace_clock_local(void) 205 { 206 return 0ULL; 207 } 208 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 209 210 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 211 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 212 /* and boost task create/destroy. */ 213 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 214 static bool barrier_phase; /* Test phase. */ 215 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 216 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 217 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 218 219 /* 220 * Allocate an element from the rcu_tortures pool. 221 */ 222 static struct rcu_torture * 223 rcu_torture_alloc(void) 224 { 225 struct list_head *p; 226 227 spin_lock_bh(&rcu_torture_lock); 228 if (list_empty(&rcu_torture_freelist)) { 229 atomic_inc(&n_rcu_torture_alloc_fail); 230 spin_unlock_bh(&rcu_torture_lock); 231 return NULL; 232 } 233 atomic_inc(&n_rcu_torture_alloc); 234 p = rcu_torture_freelist.next; 235 list_del_init(p); 236 spin_unlock_bh(&rcu_torture_lock); 237 return container_of(p, struct rcu_torture, rtort_free); 238 } 239 240 /* 241 * Free an element to the rcu_tortures pool. 242 */ 243 static void 244 rcu_torture_free(struct rcu_torture *p) 245 { 246 atomic_inc(&n_rcu_torture_free); 247 spin_lock_bh(&rcu_torture_lock); 248 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 249 spin_unlock_bh(&rcu_torture_lock); 250 } 251 252 /* 253 * Operations vector for selecting different types of tests. 254 */ 255 256 struct rcu_torture_ops { 257 int ttype; 258 void (*init)(void); 259 void (*cleanup)(void); 260 int (*readlock)(void); 261 void (*read_delay)(struct torture_random_state *rrsp); 262 void (*readunlock)(int idx); 263 unsigned long (*started)(void); 264 unsigned long (*completed)(void); 265 void (*deferred_free)(struct rcu_torture *p); 266 void (*sync)(void); 267 void (*exp_sync)(void); 268 unsigned long (*get_state)(void); 269 void (*cond_sync)(unsigned long oldstate); 270 call_rcu_func_t call; 271 void (*cb_barrier)(void); 272 void (*fqs)(void); 273 void (*stats)(void); 274 int irq_capable; 275 int can_boost; 276 const char *name; 277 }; 278 279 static struct rcu_torture_ops *cur_ops; 280 281 /* 282 * Definitions for rcu torture testing. 283 */ 284 285 static int rcu_torture_read_lock(void) __acquires(RCU) 286 { 287 rcu_read_lock(); 288 return 0; 289 } 290 291 static void rcu_read_delay(struct torture_random_state *rrsp) 292 { 293 unsigned long started; 294 unsigned long completed; 295 const unsigned long shortdelay_us = 200; 296 const unsigned long longdelay_ms = 50; 297 unsigned long long ts; 298 299 /* We want a short delay sometimes to make a reader delay the grace 300 * period, and we want a long delay occasionally to trigger 301 * force_quiescent_state. */ 302 303 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 304 started = cur_ops->completed(); 305 ts = rcu_trace_clock_local(); 306 mdelay(longdelay_ms); 307 completed = cur_ops->completed(); 308 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 309 started, completed); 310 } 311 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 312 udelay(shortdelay_us); 313 #ifdef CONFIG_PREEMPT 314 if (!preempt_count() && 315 !(torture_random(rrsp) % (nrealreaders * 20000))) 316 preempt_schedule(); /* No QS if preempt_disable() in effect */ 317 #endif 318 } 319 320 static void rcu_torture_read_unlock(int idx) __releases(RCU) 321 { 322 rcu_read_unlock(); 323 } 324 325 /* 326 * Update callback in the pipe. This should be invoked after a grace period. 327 */ 328 static bool 329 rcu_torture_pipe_update_one(struct rcu_torture *rp) 330 { 331 int i; 332 333 i = rp->rtort_pipe_count; 334 if (i > RCU_TORTURE_PIPE_LEN) 335 i = RCU_TORTURE_PIPE_LEN; 336 atomic_inc(&rcu_torture_wcount[i]); 337 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 338 rp->rtort_mbtest = 0; 339 return true; 340 } 341 return false; 342 } 343 344 /* 345 * Update all callbacks in the pipe. Suitable for synchronous grace-period 346 * primitives. 347 */ 348 static void 349 rcu_torture_pipe_update(struct rcu_torture *old_rp) 350 { 351 struct rcu_torture *rp; 352 struct rcu_torture *rp1; 353 354 if (old_rp) 355 list_add(&old_rp->rtort_free, &rcu_torture_removed); 356 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 357 if (rcu_torture_pipe_update_one(rp)) { 358 list_del(&rp->rtort_free); 359 rcu_torture_free(rp); 360 } 361 } 362 } 363 364 static void 365 rcu_torture_cb(struct rcu_head *p) 366 { 367 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 368 369 if (torture_must_stop_irq()) { 370 /* Test is ending, just drop callbacks on the floor. */ 371 /* The next initialization will pick up the pieces. */ 372 return; 373 } 374 if (rcu_torture_pipe_update_one(rp)) 375 rcu_torture_free(rp); 376 else 377 cur_ops->deferred_free(rp); 378 } 379 380 static unsigned long rcu_no_completed(void) 381 { 382 return 0; 383 } 384 385 static void rcu_torture_deferred_free(struct rcu_torture *p) 386 { 387 call_rcu(&p->rtort_rcu, rcu_torture_cb); 388 } 389 390 static void rcu_sync_torture_init(void) 391 { 392 INIT_LIST_HEAD(&rcu_torture_removed); 393 } 394 395 static struct rcu_torture_ops rcu_ops = { 396 .ttype = RCU_FLAVOR, 397 .init = rcu_sync_torture_init, 398 .readlock = rcu_torture_read_lock, 399 .read_delay = rcu_read_delay, 400 .readunlock = rcu_torture_read_unlock, 401 .started = rcu_batches_started, 402 .completed = rcu_batches_completed, 403 .deferred_free = rcu_torture_deferred_free, 404 .sync = synchronize_rcu, 405 .exp_sync = synchronize_rcu_expedited, 406 .get_state = get_state_synchronize_rcu, 407 .cond_sync = cond_synchronize_rcu, 408 .call = call_rcu, 409 .cb_barrier = rcu_barrier, 410 .fqs = rcu_force_quiescent_state, 411 .stats = NULL, 412 .irq_capable = 1, 413 .can_boost = rcu_can_boost(), 414 .name = "rcu" 415 }; 416 417 /* 418 * Definitions for rcu_bh torture testing. 419 */ 420 421 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 422 { 423 rcu_read_lock_bh(); 424 return 0; 425 } 426 427 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 428 { 429 rcu_read_unlock_bh(); 430 } 431 432 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 433 { 434 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 435 } 436 437 static struct rcu_torture_ops rcu_bh_ops = { 438 .ttype = RCU_BH_FLAVOR, 439 .init = rcu_sync_torture_init, 440 .readlock = rcu_bh_torture_read_lock, 441 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 442 .readunlock = rcu_bh_torture_read_unlock, 443 .started = rcu_batches_started_bh, 444 .completed = rcu_batches_completed_bh, 445 .deferred_free = rcu_bh_torture_deferred_free, 446 .sync = synchronize_rcu_bh, 447 .exp_sync = synchronize_rcu_bh_expedited, 448 .call = call_rcu_bh, 449 .cb_barrier = rcu_barrier_bh, 450 .fqs = rcu_bh_force_quiescent_state, 451 .stats = NULL, 452 .irq_capable = 1, 453 .name = "rcu_bh" 454 }; 455 456 /* 457 * Don't even think about trying any of these in real life!!! 458 * The names includes "busted", and they really means it! 459 * The only purpose of these functions is to provide a buggy RCU 460 * implementation to make sure that rcutorture correctly emits 461 * buggy-RCU error messages. 462 */ 463 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 464 { 465 /* This is a deliberate bug for testing purposes only! */ 466 rcu_torture_cb(&p->rtort_rcu); 467 } 468 469 static void synchronize_rcu_busted(void) 470 { 471 /* This is a deliberate bug for testing purposes only! */ 472 } 473 474 static void 475 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 476 { 477 /* This is a deliberate bug for testing purposes only! */ 478 func(head); 479 } 480 481 static struct rcu_torture_ops rcu_busted_ops = { 482 .ttype = INVALID_RCU_FLAVOR, 483 .init = rcu_sync_torture_init, 484 .readlock = rcu_torture_read_lock, 485 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 486 .readunlock = rcu_torture_read_unlock, 487 .started = rcu_no_completed, 488 .completed = rcu_no_completed, 489 .deferred_free = rcu_busted_torture_deferred_free, 490 .sync = synchronize_rcu_busted, 491 .exp_sync = synchronize_rcu_busted, 492 .call = call_rcu_busted, 493 .cb_barrier = NULL, 494 .fqs = NULL, 495 .stats = NULL, 496 .irq_capable = 1, 497 .name = "rcu_busted" 498 }; 499 500 /* 501 * Definitions for srcu torture testing. 502 */ 503 504 DEFINE_STATIC_SRCU(srcu_ctl); 505 static struct srcu_struct srcu_ctld; 506 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 507 508 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 509 { 510 return srcu_read_lock(srcu_ctlp); 511 } 512 513 static void srcu_read_delay(struct torture_random_state *rrsp) 514 { 515 long delay; 516 const long uspertick = 1000000 / HZ; 517 const long longdelay = 10; 518 519 /* We want there to be long-running readers, but not all the time. */ 520 521 delay = torture_random(rrsp) % 522 (nrealreaders * 2 * longdelay * uspertick); 523 if (!delay) 524 schedule_timeout_interruptible(longdelay); 525 else 526 rcu_read_delay(rrsp); 527 } 528 529 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 530 { 531 srcu_read_unlock(srcu_ctlp, idx); 532 } 533 534 static unsigned long srcu_torture_completed(void) 535 { 536 return srcu_batches_completed(srcu_ctlp); 537 } 538 539 static void srcu_torture_deferred_free(struct rcu_torture *rp) 540 { 541 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 542 } 543 544 static void srcu_torture_synchronize(void) 545 { 546 synchronize_srcu(srcu_ctlp); 547 } 548 549 static void srcu_torture_call(struct rcu_head *head, 550 rcu_callback_t func) 551 { 552 call_srcu(srcu_ctlp, head, func); 553 } 554 555 static void srcu_torture_barrier(void) 556 { 557 srcu_barrier(srcu_ctlp); 558 } 559 560 static void srcu_torture_stats(void) 561 { 562 int __maybe_unused cpu; 563 int idx; 564 565 #if defined(CONFIG_TREE_SRCU) || defined(CONFIG_CLASSIC_SRCU) 566 #ifdef CONFIG_TREE_SRCU 567 idx = srcu_ctlp->srcu_idx & 0x1; 568 #else /* #ifdef CONFIG_TREE_SRCU */ 569 idx = srcu_ctlp->completed & 0x1; 570 #endif /* #else #ifdef CONFIG_TREE_SRCU */ 571 pr_alert("%s%s Tree SRCU per-CPU(idx=%d):", 572 torture_type, TORTURE_FLAG, idx); 573 for_each_possible_cpu(cpu) { 574 unsigned long l0, l1; 575 unsigned long u0, u1; 576 long c0, c1; 577 #ifdef CONFIG_TREE_SRCU 578 struct srcu_data *counts; 579 580 counts = per_cpu_ptr(srcu_ctlp->sda, cpu); 581 u0 = counts->srcu_unlock_count[!idx]; 582 u1 = counts->srcu_unlock_count[idx]; 583 #else /* #ifdef CONFIG_TREE_SRCU */ 584 struct srcu_array *counts; 585 586 counts = per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu); 587 u0 = counts->unlock_count[!idx]; 588 u1 = counts->unlock_count[idx]; 589 #endif /* #else #ifdef CONFIG_TREE_SRCU */ 590 591 /* 592 * Make sure that a lock is always counted if the corresponding 593 * unlock is counted. 594 */ 595 smp_rmb(); 596 597 #ifdef CONFIG_TREE_SRCU 598 l0 = counts->srcu_lock_count[!idx]; 599 l1 = counts->srcu_lock_count[idx]; 600 #else /* #ifdef CONFIG_TREE_SRCU */ 601 l0 = counts->lock_count[!idx]; 602 l1 = counts->lock_count[idx]; 603 #endif /* #else #ifdef CONFIG_TREE_SRCU */ 604 605 c0 = l0 - u0; 606 c1 = l1 - u1; 607 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 608 } 609 pr_cont("\n"); 610 #elif defined(CONFIG_TINY_SRCU) 611 idx = READ_ONCE(srcu_ctlp->srcu_idx) & 0x1; 612 pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%d,%d)\n", 613 torture_type, TORTURE_FLAG, idx, 614 READ_ONCE(srcu_ctlp->srcu_lock_nesting[!idx]), 615 READ_ONCE(srcu_ctlp->srcu_lock_nesting[idx])); 616 #endif 617 } 618 619 static void srcu_torture_synchronize_expedited(void) 620 { 621 synchronize_srcu_expedited(srcu_ctlp); 622 } 623 624 static struct rcu_torture_ops srcu_ops = { 625 .ttype = SRCU_FLAVOR, 626 .init = rcu_sync_torture_init, 627 .readlock = srcu_torture_read_lock, 628 .read_delay = srcu_read_delay, 629 .readunlock = srcu_torture_read_unlock, 630 .started = NULL, 631 .completed = srcu_torture_completed, 632 .deferred_free = srcu_torture_deferred_free, 633 .sync = srcu_torture_synchronize, 634 .exp_sync = srcu_torture_synchronize_expedited, 635 .call = srcu_torture_call, 636 .cb_barrier = srcu_torture_barrier, 637 .stats = srcu_torture_stats, 638 .name = "srcu" 639 }; 640 641 static void srcu_torture_init(void) 642 { 643 rcu_sync_torture_init(); 644 WARN_ON(init_srcu_struct(&srcu_ctld)); 645 srcu_ctlp = &srcu_ctld; 646 } 647 648 static void srcu_torture_cleanup(void) 649 { 650 cleanup_srcu_struct(&srcu_ctld); 651 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 652 } 653 654 /* As above, but dynamically allocated. */ 655 static struct rcu_torture_ops srcud_ops = { 656 .ttype = SRCU_FLAVOR, 657 .init = srcu_torture_init, 658 .cleanup = srcu_torture_cleanup, 659 .readlock = srcu_torture_read_lock, 660 .read_delay = srcu_read_delay, 661 .readunlock = srcu_torture_read_unlock, 662 .started = NULL, 663 .completed = srcu_torture_completed, 664 .deferred_free = srcu_torture_deferred_free, 665 .sync = srcu_torture_synchronize, 666 .exp_sync = srcu_torture_synchronize_expedited, 667 .call = srcu_torture_call, 668 .cb_barrier = srcu_torture_barrier, 669 .stats = srcu_torture_stats, 670 .name = "srcud" 671 }; 672 673 /* 674 * Definitions for sched torture testing. 675 */ 676 677 static int sched_torture_read_lock(void) 678 { 679 preempt_disable(); 680 return 0; 681 } 682 683 static void sched_torture_read_unlock(int idx) 684 { 685 preempt_enable(); 686 } 687 688 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 689 { 690 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 691 } 692 693 static struct rcu_torture_ops sched_ops = { 694 .ttype = RCU_SCHED_FLAVOR, 695 .init = rcu_sync_torture_init, 696 .readlock = sched_torture_read_lock, 697 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 698 .readunlock = sched_torture_read_unlock, 699 .started = rcu_batches_started_sched, 700 .completed = rcu_batches_completed_sched, 701 .deferred_free = rcu_sched_torture_deferred_free, 702 .sync = synchronize_sched, 703 .exp_sync = synchronize_sched_expedited, 704 .get_state = get_state_synchronize_sched, 705 .cond_sync = cond_synchronize_sched, 706 .call = call_rcu_sched, 707 .cb_barrier = rcu_barrier_sched, 708 .fqs = rcu_sched_force_quiescent_state, 709 .stats = NULL, 710 .irq_capable = 1, 711 .name = "sched" 712 }; 713 714 #ifdef CONFIG_TASKS_RCU 715 716 /* 717 * Definitions for RCU-tasks torture testing. 718 */ 719 720 static int tasks_torture_read_lock(void) 721 { 722 return 0; 723 } 724 725 static void tasks_torture_read_unlock(int idx) 726 { 727 } 728 729 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 730 { 731 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 732 } 733 734 static struct rcu_torture_ops tasks_ops = { 735 .ttype = RCU_TASKS_FLAVOR, 736 .init = rcu_sync_torture_init, 737 .readlock = tasks_torture_read_lock, 738 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 739 .readunlock = tasks_torture_read_unlock, 740 .started = rcu_no_completed, 741 .completed = rcu_no_completed, 742 .deferred_free = rcu_tasks_torture_deferred_free, 743 .sync = synchronize_rcu_tasks, 744 .exp_sync = synchronize_rcu_tasks, 745 .call = call_rcu_tasks, 746 .cb_barrier = rcu_barrier_tasks, 747 .fqs = NULL, 748 .stats = NULL, 749 .irq_capable = 1, 750 .name = "tasks" 751 }; 752 753 #define RCUTORTURE_TASKS_OPS &tasks_ops, 754 755 static bool __maybe_unused torturing_tasks(void) 756 { 757 return cur_ops == &tasks_ops; 758 } 759 760 #else /* #ifdef CONFIG_TASKS_RCU */ 761 762 #define RCUTORTURE_TASKS_OPS 763 764 static bool __maybe_unused torturing_tasks(void) 765 { 766 return false; 767 } 768 769 #endif /* #else #ifdef CONFIG_TASKS_RCU */ 770 771 /* 772 * RCU torture priority-boost testing. Runs one real-time thread per 773 * CPU for moderate bursts, repeatedly registering RCU callbacks and 774 * spinning waiting for them to be invoked. If a given callback takes 775 * too long to be invoked, we assume that priority inversion has occurred. 776 */ 777 778 struct rcu_boost_inflight { 779 struct rcu_head rcu; 780 int inflight; 781 }; 782 783 static void rcu_torture_boost_cb(struct rcu_head *head) 784 { 785 struct rcu_boost_inflight *rbip = 786 container_of(head, struct rcu_boost_inflight, rcu); 787 788 /* Ensure RCU-core accesses precede clearing ->inflight */ 789 smp_store_release(&rbip->inflight, 0); 790 } 791 792 static int rcu_torture_boost(void *arg) 793 { 794 unsigned long call_rcu_time; 795 unsigned long endtime; 796 unsigned long oldstarttime; 797 struct rcu_boost_inflight rbi = { .inflight = 0 }; 798 struct sched_param sp; 799 800 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 801 802 /* Set real-time priority. */ 803 sp.sched_priority = 1; 804 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 805 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 806 n_rcu_torture_boost_rterror++; 807 } 808 809 init_rcu_head_on_stack(&rbi.rcu); 810 /* Each pass through the following loop does one boost-test cycle. */ 811 do { 812 /* Wait for the next test interval. */ 813 oldstarttime = boost_starttime; 814 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 815 schedule_timeout_interruptible(oldstarttime - jiffies); 816 stutter_wait("rcu_torture_boost"); 817 if (torture_must_stop()) 818 goto checkwait; 819 } 820 821 /* Do one boost-test interval. */ 822 endtime = oldstarttime + test_boost_duration * HZ; 823 call_rcu_time = jiffies; 824 while (ULONG_CMP_LT(jiffies, endtime)) { 825 /* If we don't have a callback in flight, post one. */ 826 if (!smp_load_acquire(&rbi.inflight)) { 827 /* RCU core before ->inflight = 1. */ 828 smp_store_release(&rbi.inflight, 1); 829 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 830 if (jiffies - call_rcu_time > 831 test_boost_duration * HZ - HZ / 2) { 832 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 833 n_rcu_torture_boost_failure++; 834 } 835 call_rcu_time = jiffies; 836 } 837 stutter_wait("rcu_torture_boost"); 838 if (torture_must_stop()) 839 goto checkwait; 840 } 841 842 /* 843 * Set the start time of the next test interval. 844 * Yes, this is vulnerable to long delays, but such 845 * delays simply cause a false negative for the next 846 * interval. Besides, we are running at RT priority, 847 * so delays should be relatively rare. 848 */ 849 while (oldstarttime == boost_starttime && 850 !kthread_should_stop()) { 851 if (mutex_trylock(&boost_mutex)) { 852 boost_starttime = jiffies + 853 test_boost_interval * HZ; 854 n_rcu_torture_boosts++; 855 mutex_unlock(&boost_mutex); 856 break; 857 } 858 schedule_timeout_uninterruptible(1); 859 } 860 861 /* Go do the stutter. */ 862 checkwait: stutter_wait("rcu_torture_boost"); 863 } while (!torture_must_stop()); 864 865 /* Clean up and exit. */ 866 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 867 torture_shutdown_absorb("rcu_torture_boost"); 868 schedule_timeout_uninterruptible(1); 869 } 870 destroy_rcu_head_on_stack(&rbi.rcu); 871 torture_kthread_stopping("rcu_torture_boost"); 872 return 0; 873 } 874 875 static void rcu_torture_cbflood_cb(struct rcu_head *rhp) 876 { 877 } 878 879 /* 880 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls 881 * to call_rcu() or analogous, increasing the probability of occurrence 882 * of callback-overflow corner cases. 883 */ 884 static int 885 rcu_torture_cbflood(void *arg) 886 { 887 int err = 1; 888 int i; 889 int j; 890 struct rcu_head *rhp; 891 892 if (cbflood_n_per_burst > 0 && 893 cbflood_inter_holdoff > 0 && 894 cbflood_intra_holdoff > 0 && 895 cur_ops->call && 896 cur_ops->cb_barrier) { 897 rhp = vmalloc(sizeof(*rhp) * 898 cbflood_n_burst * cbflood_n_per_burst); 899 err = !rhp; 900 } 901 if (err) { 902 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM"); 903 goto wait_for_stop; 904 } 905 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started"); 906 do { 907 schedule_timeout_interruptible(cbflood_inter_holdoff); 908 atomic_long_inc(&n_cbfloods); 909 WARN_ON(signal_pending(current)); 910 for (i = 0; i < cbflood_n_burst; i++) { 911 for (j = 0; j < cbflood_n_per_burst; j++) { 912 cur_ops->call(&rhp[i * cbflood_n_per_burst + j], 913 rcu_torture_cbflood_cb); 914 } 915 schedule_timeout_interruptible(cbflood_intra_holdoff); 916 WARN_ON(signal_pending(current)); 917 } 918 cur_ops->cb_barrier(); 919 stutter_wait("rcu_torture_cbflood"); 920 } while (!torture_must_stop()); 921 vfree(rhp); 922 wait_for_stop: 923 torture_kthread_stopping("rcu_torture_cbflood"); 924 return 0; 925 } 926 927 /* 928 * RCU torture force-quiescent-state kthread. Repeatedly induces 929 * bursts of calls to force_quiescent_state(), increasing the probability 930 * of occurrence of some important types of race conditions. 931 */ 932 static int 933 rcu_torture_fqs(void *arg) 934 { 935 unsigned long fqs_resume_time; 936 int fqs_burst_remaining; 937 938 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 939 do { 940 fqs_resume_time = jiffies + fqs_stutter * HZ; 941 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 942 !kthread_should_stop()) { 943 schedule_timeout_interruptible(1); 944 } 945 fqs_burst_remaining = fqs_duration; 946 while (fqs_burst_remaining > 0 && 947 !kthread_should_stop()) { 948 cur_ops->fqs(); 949 udelay(fqs_holdoff); 950 fqs_burst_remaining -= fqs_holdoff; 951 } 952 stutter_wait("rcu_torture_fqs"); 953 } while (!torture_must_stop()); 954 torture_kthread_stopping("rcu_torture_fqs"); 955 return 0; 956 } 957 958 /* 959 * RCU torture writer kthread. Repeatedly substitutes a new structure 960 * for that pointed to by rcu_torture_current, freeing the old structure 961 * after a series of grace periods (the "pipeline"). 962 */ 963 static int 964 rcu_torture_writer(void *arg) 965 { 966 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 967 int expediting = 0; 968 unsigned long gp_snap; 969 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 970 bool gp_sync1 = gp_sync; 971 int i; 972 struct rcu_torture *rp; 973 struct rcu_torture *old_rp; 974 static DEFINE_TORTURE_RANDOM(rand); 975 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 976 RTWS_COND_GET, RTWS_SYNC }; 977 int nsynctypes = 0; 978 979 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 980 if (!can_expedite) { 981 pr_alert("%s" TORTURE_FLAG 982 " GP expediting controlled from boot/sysfs for %s,\n", 983 torture_type, cur_ops->name); 984 pr_alert("%s" TORTURE_FLAG 985 " Disabled dynamic grace-period expediting.\n", 986 torture_type); 987 } 988 989 /* Initialize synctype[] array. If none set, take default. */ 990 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 991 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 992 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 993 synctype[nsynctypes++] = RTWS_COND_GET; 994 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 995 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 996 if (gp_exp1 && cur_ops->exp_sync) 997 synctype[nsynctypes++] = RTWS_EXP_SYNC; 998 else if (gp_exp && !cur_ops->exp_sync) 999 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 1000 if (gp_normal1 && cur_ops->deferred_free) 1001 synctype[nsynctypes++] = RTWS_DEF_FREE; 1002 else if (gp_normal && !cur_ops->deferred_free) 1003 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 1004 if (gp_sync1 && cur_ops->sync) 1005 synctype[nsynctypes++] = RTWS_SYNC; 1006 else if (gp_sync && !cur_ops->sync) 1007 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 1008 if (WARN_ONCE(nsynctypes == 0, 1009 "rcu_torture_writer: No update-side primitives.\n")) { 1010 /* 1011 * No updates primitives, so don't try updating. 1012 * The resulting test won't be testing much, hence the 1013 * above WARN_ONCE(). 1014 */ 1015 rcu_torture_writer_state = RTWS_STOPPING; 1016 torture_kthread_stopping("rcu_torture_writer"); 1017 } 1018 1019 do { 1020 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1021 schedule_timeout_uninterruptible(1); 1022 rp = rcu_torture_alloc(); 1023 if (rp == NULL) 1024 continue; 1025 rp->rtort_pipe_count = 0; 1026 rcu_torture_writer_state = RTWS_DELAY; 1027 udelay(torture_random(&rand) & 0x3ff); 1028 rcu_torture_writer_state = RTWS_REPLACE; 1029 old_rp = rcu_dereference_check(rcu_torture_current, 1030 current == writer_task); 1031 rp->rtort_mbtest = 1; 1032 rcu_assign_pointer(rcu_torture_current, rp); 1033 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1034 if (old_rp) { 1035 i = old_rp->rtort_pipe_count; 1036 if (i > RCU_TORTURE_PIPE_LEN) 1037 i = RCU_TORTURE_PIPE_LEN; 1038 atomic_inc(&rcu_torture_wcount[i]); 1039 old_rp->rtort_pipe_count++; 1040 switch (synctype[torture_random(&rand) % nsynctypes]) { 1041 case RTWS_DEF_FREE: 1042 rcu_torture_writer_state = RTWS_DEF_FREE; 1043 cur_ops->deferred_free(old_rp); 1044 break; 1045 case RTWS_EXP_SYNC: 1046 rcu_torture_writer_state = RTWS_EXP_SYNC; 1047 cur_ops->exp_sync(); 1048 rcu_torture_pipe_update(old_rp); 1049 break; 1050 case RTWS_COND_GET: 1051 rcu_torture_writer_state = RTWS_COND_GET; 1052 gp_snap = cur_ops->get_state(); 1053 i = torture_random(&rand) % 16; 1054 if (i != 0) 1055 schedule_timeout_interruptible(i); 1056 udelay(torture_random(&rand) % 1000); 1057 rcu_torture_writer_state = RTWS_COND_SYNC; 1058 cur_ops->cond_sync(gp_snap); 1059 rcu_torture_pipe_update(old_rp); 1060 break; 1061 case RTWS_SYNC: 1062 rcu_torture_writer_state = RTWS_SYNC; 1063 cur_ops->sync(); 1064 rcu_torture_pipe_update(old_rp); 1065 break; 1066 default: 1067 WARN_ON_ONCE(1); 1068 break; 1069 } 1070 } 1071 rcutorture_record_progress(++rcu_torture_current_version); 1072 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1073 if (can_expedite && 1074 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1075 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1076 if (expediting >= 0) 1077 rcu_expedite_gp(); 1078 else 1079 rcu_unexpedite_gp(); 1080 if (++expediting > 3) 1081 expediting = -expediting; 1082 } 1083 rcu_torture_writer_state = RTWS_STUTTER; 1084 stutter_wait("rcu_torture_writer"); 1085 } while (!torture_must_stop()); 1086 /* Reset expediting back to unexpedited. */ 1087 if (expediting > 0) 1088 expediting = -expediting; 1089 while (can_expedite && expediting++ < 0) 1090 rcu_unexpedite_gp(); 1091 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1092 rcu_torture_writer_state = RTWS_STOPPING; 1093 torture_kthread_stopping("rcu_torture_writer"); 1094 return 0; 1095 } 1096 1097 /* 1098 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1099 * delay between calls. 1100 */ 1101 static int 1102 rcu_torture_fakewriter(void *arg) 1103 { 1104 DEFINE_TORTURE_RANDOM(rand); 1105 1106 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1107 set_user_nice(current, MAX_NICE); 1108 1109 do { 1110 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1111 udelay(torture_random(&rand) & 0x3ff); 1112 if (cur_ops->cb_barrier != NULL && 1113 torture_random(&rand) % (nfakewriters * 8) == 0) { 1114 cur_ops->cb_barrier(); 1115 } else if (gp_normal == gp_exp) { 1116 if (torture_random(&rand) & 0x80) 1117 cur_ops->sync(); 1118 else 1119 cur_ops->exp_sync(); 1120 } else if (gp_normal) { 1121 cur_ops->sync(); 1122 } else { 1123 cur_ops->exp_sync(); 1124 } 1125 stutter_wait("rcu_torture_fakewriter"); 1126 } while (!torture_must_stop()); 1127 1128 torture_kthread_stopping("rcu_torture_fakewriter"); 1129 return 0; 1130 } 1131 1132 /* 1133 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1134 * incrementing the corresponding element of the pipeline array. The 1135 * counter in the element should never be greater than 1, otherwise, the 1136 * RCU implementation is broken. 1137 */ 1138 static void rcu_torture_timer(unsigned long unused) 1139 { 1140 int idx; 1141 unsigned long started; 1142 unsigned long completed; 1143 static DEFINE_TORTURE_RANDOM(rand); 1144 static DEFINE_SPINLOCK(rand_lock); 1145 struct rcu_torture *p; 1146 int pipe_count; 1147 unsigned long long ts; 1148 1149 idx = cur_ops->readlock(); 1150 if (cur_ops->started) 1151 started = cur_ops->started(); 1152 else 1153 started = cur_ops->completed(); 1154 ts = rcu_trace_clock_local(); 1155 p = rcu_dereference_check(rcu_torture_current, 1156 rcu_read_lock_bh_held() || 1157 rcu_read_lock_sched_held() || 1158 srcu_read_lock_held(srcu_ctlp) || 1159 torturing_tasks()); 1160 if (p == NULL) { 1161 /* Leave because rcu_torture_writer is not yet underway */ 1162 cur_ops->readunlock(idx); 1163 return; 1164 } 1165 if (p->rtort_mbtest == 0) 1166 atomic_inc(&n_rcu_torture_mberror); 1167 spin_lock(&rand_lock); 1168 cur_ops->read_delay(&rand); 1169 n_rcu_torture_timers++; 1170 spin_unlock(&rand_lock); 1171 preempt_disable(); 1172 pipe_count = p->rtort_pipe_count; 1173 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1174 /* Should not happen, but... */ 1175 pipe_count = RCU_TORTURE_PIPE_LEN; 1176 } 1177 completed = cur_ops->completed(); 1178 if (pipe_count > 1) { 1179 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 1180 started, completed); 1181 rcu_ftrace_dump(DUMP_ALL); 1182 } 1183 __this_cpu_inc(rcu_torture_count[pipe_count]); 1184 completed = completed - started; 1185 if (cur_ops->started) 1186 completed++; 1187 if (completed > RCU_TORTURE_PIPE_LEN) { 1188 /* Should not happen, but... */ 1189 completed = RCU_TORTURE_PIPE_LEN; 1190 } 1191 __this_cpu_inc(rcu_torture_batch[completed]); 1192 preempt_enable(); 1193 cur_ops->readunlock(idx); 1194 } 1195 1196 /* 1197 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1198 * incrementing the corresponding element of the pipeline array. The 1199 * counter in the element should never be greater than 1, otherwise, the 1200 * RCU implementation is broken. 1201 */ 1202 static int 1203 rcu_torture_reader(void *arg) 1204 { 1205 unsigned long started; 1206 unsigned long completed; 1207 int idx; 1208 DEFINE_TORTURE_RANDOM(rand); 1209 struct rcu_torture *p; 1210 int pipe_count; 1211 struct timer_list t; 1212 unsigned long long ts; 1213 1214 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1215 set_user_nice(current, MAX_NICE); 1216 if (irqreader && cur_ops->irq_capable) 1217 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1218 1219 do { 1220 if (irqreader && cur_ops->irq_capable) { 1221 if (!timer_pending(&t)) 1222 mod_timer(&t, jiffies + 1); 1223 } 1224 idx = cur_ops->readlock(); 1225 if (cur_ops->started) 1226 started = cur_ops->started(); 1227 else 1228 started = cur_ops->completed(); 1229 ts = rcu_trace_clock_local(); 1230 p = rcu_dereference_check(rcu_torture_current, 1231 rcu_read_lock_bh_held() || 1232 rcu_read_lock_sched_held() || 1233 srcu_read_lock_held(srcu_ctlp) || 1234 torturing_tasks()); 1235 if (p == NULL) { 1236 /* Wait for rcu_torture_writer to get underway */ 1237 cur_ops->readunlock(idx); 1238 schedule_timeout_interruptible(HZ); 1239 continue; 1240 } 1241 if (p->rtort_mbtest == 0) 1242 atomic_inc(&n_rcu_torture_mberror); 1243 cur_ops->read_delay(&rand); 1244 preempt_disable(); 1245 pipe_count = p->rtort_pipe_count; 1246 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1247 /* Should not happen, but... */ 1248 pipe_count = RCU_TORTURE_PIPE_LEN; 1249 } 1250 completed = cur_ops->completed(); 1251 if (pipe_count > 1) { 1252 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1253 ts, started, completed); 1254 rcu_ftrace_dump(DUMP_ALL); 1255 } 1256 __this_cpu_inc(rcu_torture_count[pipe_count]); 1257 completed = completed - started; 1258 if (cur_ops->started) 1259 completed++; 1260 if (completed > RCU_TORTURE_PIPE_LEN) { 1261 /* Should not happen, but... */ 1262 completed = RCU_TORTURE_PIPE_LEN; 1263 } 1264 __this_cpu_inc(rcu_torture_batch[completed]); 1265 preempt_enable(); 1266 cur_ops->readunlock(idx); 1267 stutter_wait("rcu_torture_reader"); 1268 } while (!torture_must_stop()); 1269 if (irqreader && cur_ops->irq_capable) { 1270 del_timer_sync(&t); 1271 destroy_timer_on_stack(&t); 1272 } 1273 torture_kthread_stopping("rcu_torture_reader"); 1274 return 0; 1275 } 1276 1277 /* 1278 * Print torture statistics. Caller must ensure that there is only 1279 * one call to this function at a given time!!! This is normally 1280 * accomplished by relying on the module system to only have one copy 1281 * of the module loaded, and then by giving the rcu_torture_stats 1282 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1283 * thread is not running). 1284 */ 1285 static void 1286 rcu_torture_stats_print(void) 1287 { 1288 int cpu; 1289 int i; 1290 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1291 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1292 static unsigned long rtcv_snap = ULONG_MAX; 1293 struct task_struct *wtp; 1294 1295 for_each_possible_cpu(cpu) { 1296 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1297 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1298 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1299 } 1300 } 1301 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1302 if (pipesummary[i] != 0) 1303 break; 1304 } 1305 1306 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1307 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1308 rcu_torture_current, 1309 rcu_torture_current_version, 1310 list_empty(&rcu_torture_freelist), 1311 atomic_read(&n_rcu_torture_alloc), 1312 atomic_read(&n_rcu_torture_alloc_fail), 1313 atomic_read(&n_rcu_torture_free)); 1314 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1315 atomic_read(&n_rcu_torture_mberror), 1316 n_rcu_torture_barrier_error, 1317 n_rcu_torture_boost_ktrerror, 1318 n_rcu_torture_boost_rterror); 1319 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1320 n_rcu_torture_boost_failure, 1321 n_rcu_torture_boosts, 1322 n_rcu_torture_timers); 1323 torture_onoff_stats(); 1324 pr_cont("barrier: %ld/%ld:%ld ", 1325 n_barrier_successes, 1326 n_barrier_attempts, 1327 n_rcu_torture_barrier_error); 1328 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods)); 1329 1330 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1331 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1332 n_rcu_torture_barrier_error != 0 || 1333 n_rcu_torture_boost_ktrerror != 0 || 1334 n_rcu_torture_boost_rterror != 0 || 1335 n_rcu_torture_boost_failure != 0 || 1336 i > 1) { 1337 pr_cont("%s", "!!! "); 1338 atomic_inc(&n_rcu_torture_error); 1339 WARN_ON_ONCE(1); 1340 } 1341 pr_cont("Reader Pipe: "); 1342 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1343 pr_cont(" %ld", pipesummary[i]); 1344 pr_cont("\n"); 1345 1346 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1347 pr_cont("Reader Batch: "); 1348 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1349 pr_cont(" %ld", batchsummary[i]); 1350 pr_cont("\n"); 1351 1352 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1353 pr_cont("Free-Block Circulation: "); 1354 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1355 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1356 } 1357 pr_cont("\n"); 1358 1359 if (cur_ops->stats) 1360 cur_ops->stats(); 1361 if (rtcv_snap == rcu_torture_current_version && 1362 rcu_torture_current != NULL) { 1363 int __maybe_unused flags = 0; 1364 unsigned long __maybe_unused gpnum = 0; 1365 unsigned long __maybe_unused completed = 0; 1366 1367 rcutorture_get_gp_data(cur_ops->ttype, 1368 &flags, &gpnum, &completed); 1369 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1370 &flags, &gpnum, &completed); 1371 wtp = READ_ONCE(writer_task); 1372 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n", 1373 rcu_torture_writer_state_getname(), 1374 rcu_torture_writer_state, 1375 gpnum, completed, flags, 1376 wtp == NULL ? ~0UL : wtp->state); 1377 show_rcu_gp_kthreads(); 1378 rcu_ftrace_dump(DUMP_ALL); 1379 } 1380 rtcv_snap = rcu_torture_current_version; 1381 } 1382 1383 /* 1384 * Periodically prints torture statistics, if periodic statistics printing 1385 * was specified via the stat_interval module parameter. 1386 */ 1387 static int 1388 rcu_torture_stats(void *arg) 1389 { 1390 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1391 do { 1392 schedule_timeout_interruptible(stat_interval * HZ); 1393 rcu_torture_stats_print(); 1394 torture_shutdown_absorb("rcu_torture_stats"); 1395 } while (!torture_must_stop()); 1396 torture_kthread_stopping("rcu_torture_stats"); 1397 return 0; 1398 } 1399 1400 static inline void 1401 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1402 { 1403 pr_alert("%s" TORTURE_FLAG 1404 "--- %s: nreaders=%d nfakewriters=%d " 1405 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1406 "shuffle_interval=%d stutter=%d irqreader=%d " 1407 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1408 "test_boost=%d/%d test_boost_interval=%d " 1409 "test_boost_duration=%d shutdown_secs=%d " 1410 "stall_cpu=%d stall_cpu_holdoff=%d " 1411 "n_barrier_cbs=%d " 1412 "onoff_interval=%d onoff_holdoff=%d\n", 1413 torture_type, tag, nrealreaders, nfakewriters, 1414 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1415 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1416 test_boost, cur_ops->can_boost, 1417 test_boost_interval, test_boost_duration, shutdown_secs, 1418 stall_cpu, stall_cpu_holdoff, 1419 n_barrier_cbs, 1420 onoff_interval, onoff_holdoff); 1421 } 1422 1423 static int rcutorture_booster_cleanup(unsigned int cpu) 1424 { 1425 struct task_struct *t; 1426 1427 if (boost_tasks[cpu] == NULL) 1428 return 0; 1429 mutex_lock(&boost_mutex); 1430 t = boost_tasks[cpu]; 1431 boost_tasks[cpu] = NULL; 1432 mutex_unlock(&boost_mutex); 1433 1434 /* This must be outside of the mutex, otherwise deadlock! */ 1435 torture_stop_kthread(rcu_torture_boost, t); 1436 return 0; 1437 } 1438 1439 static int rcutorture_booster_init(unsigned int cpu) 1440 { 1441 int retval; 1442 1443 if (boost_tasks[cpu] != NULL) 1444 return 0; /* Already created, nothing more to do. */ 1445 1446 /* Don't allow time recalculation while creating a new task. */ 1447 mutex_lock(&boost_mutex); 1448 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1449 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1450 cpu_to_node(cpu), 1451 "rcu_torture_boost"); 1452 if (IS_ERR(boost_tasks[cpu])) { 1453 retval = PTR_ERR(boost_tasks[cpu]); 1454 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1455 n_rcu_torture_boost_ktrerror++; 1456 boost_tasks[cpu] = NULL; 1457 mutex_unlock(&boost_mutex); 1458 return retval; 1459 } 1460 kthread_bind(boost_tasks[cpu], cpu); 1461 wake_up_process(boost_tasks[cpu]); 1462 mutex_unlock(&boost_mutex); 1463 return 0; 1464 } 1465 1466 /* 1467 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1468 * induces a CPU stall for the time specified by stall_cpu. 1469 */ 1470 static int rcu_torture_stall(void *args) 1471 { 1472 unsigned long stop_at; 1473 1474 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1475 if (stall_cpu_holdoff > 0) { 1476 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1477 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1478 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1479 } 1480 if (!kthread_should_stop()) { 1481 stop_at = get_seconds() + stall_cpu; 1482 /* RCU CPU stall is expected behavior in following code. */ 1483 pr_alert("rcu_torture_stall start.\n"); 1484 rcu_read_lock(); 1485 preempt_disable(); 1486 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1487 continue; /* Induce RCU CPU stall warning. */ 1488 preempt_enable(); 1489 rcu_read_unlock(); 1490 pr_alert("rcu_torture_stall end.\n"); 1491 } 1492 torture_shutdown_absorb("rcu_torture_stall"); 1493 while (!kthread_should_stop()) 1494 schedule_timeout_interruptible(10 * HZ); 1495 return 0; 1496 } 1497 1498 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1499 static int __init rcu_torture_stall_init(void) 1500 { 1501 if (stall_cpu <= 0) 1502 return 0; 1503 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1504 } 1505 1506 /* Callback function for RCU barrier testing. */ 1507 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1508 { 1509 atomic_inc(&barrier_cbs_invoked); 1510 } 1511 1512 /* kthread function to register callbacks used to test RCU barriers. */ 1513 static int rcu_torture_barrier_cbs(void *arg) 1514 { 1515 long myid = (long)arg; 1516 bool lastphase = 0; 1517 bool newphase; 1518 struct rcu_head rcu; 1519 1520 init_rcu_head_on_stack(&rcu); 1521 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1522 set_user_nice(current, MAX_NICE); 1523 do { 1524 wait_event(barrier_cbs_wq[myid], 1525 (newphase = 1526 smp_load_acquire(&barrier_phase)) != lastphase || 1527 torture_must_stop()); 1528 lastphase = newphase; 1529 if (torture_must_stop()) 1530 break; 1531 /* 1532 * The above smp_load_acquire() ensures barrier_phase load 1533 * is ordered before the following ->call(). 1534 */ 1535 local_irq_disable(); /* Just to test no-irq call_rcu(). */ 1536 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1537 local_irq_enable(); 1538 if (atomic_dec_and_test(&barrier_cbs_count)) 1539 wake_up(&barrier_wq); 1540 } while (!torture_must_stop()); 1541 if (cur_ops->cb_barrier != NULL) 1542 cur_ops->cb_barrier(); 1543 destroy_rcu_head_on_stack(&rcu); 1544 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1545 return 0; 1546 } 1547 1548 /* kthread function to drive and coordinate RCU barrier testing. */ 1549 static int rcu_torture_barrier(void *arg) 1550 { 1551 int i; 1552 1553 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1554 do { 1555 atomic_set(&barrier_cbs_invoked, 0); 1556 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1557 /* Ensure barrier_phase ordered after prior assignments. */ 1558 smp_store_release(&barrier_phase, !barrier_phase); 1559 for (i = 0; i < n_barrier_cbs; i++) 1560 wake_up(&barrier_cbs_wq[i]); 1561 wait_event(barrier_wq, 1562 atomic_read(&barrier_cbs_count) == 0 || 1563 torture_must_stop()); 1564 if (torture_must_stop()) 1565 break; 1566 n_barrier_attempts++; 1567 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1568 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1569 n_rcu_torture_barrier_error++; 1570 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 1571 atomic_read(&barrier_cbs_invoked), 1572 n_barrier_cbs); 1573 WARN_ON_ONCE(1); 1574 } 1575 n_barrier_successes++; 1576 schedule_timeout_interruptible(HZ / 10); 1577 } while (!torture_must_stop()); 1578 torture_kthread_stopping("rcu_torture_barrier"); 1579 return 0; 1580 } 1581 1582 /* Initialize RCU barrier testing. */ 1583 static int rcu_torture_barrier_init(void) 1584 { 1585 int i; 1586 int ret; 1587 1588 if (n_barrier_cbs <= 0) 1589 return 0; 1590 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1591 pr_alert("%s" TORTURE_FLAG 1592 " Call or barrier ops missing for %s,\n", 1593 torture_type, cur_ops->name); 1594 pr_alert("%s" TORTURE_FLAG 1595 " RCU barrier testing omitted from run.\n", 1596 torture_type); 1597 return 0; 1598 } 1599 atomic_set(&barrier_cbs_count, 0); 1600 atomic_set(&barrier_cbs_invoked, 0); 1601 barrier_cbs_tasks = 1602 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1603 GFP_KERNEL); 1604 barrier_cbs_wq = 1605 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1606 GFP_KERNEL); 1607 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1608 return -ENOMEM; 1609 for (i = 0; i < n_barrier_cbs; i++) { 1610 init_waitqueue_head(&barrier_cbs_wq[i]); 1611 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1612 (void *)(long)i, 1613 barrier_cbs_tasks[i]); 1614 if (ret) 1615 return ret; 1616 } 1617 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1618 } 1619 1620 /* Clean up after RCU barrier testing. */ 1621 static void rcu_torture_barrier_cleanup(void) 1622 { 1623 int i; 1624 1625 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1626 if (barrier_cbs_tasks != NULL) { 1627 for (i = 0; i < n_barrier_cbs; i++) 1628 torture_stop_kthread(rcu_torture_barrier_cbs, 1629 barrier_cbs_tasks[i]); 1630 kfree(barrier_cbs_tasks); 1631 barrier_cbs_tasks = NULL; 1632 } 1633 if (barrier_cbs_wq != NULL) { 1634 kfree(barrier_cbs_wq); 1635 barrier_cbs_wq = NULL; 1636 } 1637 } 1638 1639 static enum cpuhp_state rcutor_hp; 1640 1641 static void 1642 rcu_torture_cleanup(void) 1643 { 1644 int i; 1645 1646 rcutorture_record_test_transition(); 1647 if (torture_cleanup_begin()) { 1648 if (cur_ops->cb_barrier != NULL) 1649 cur_ops->cb_barrier(); 1650 return; 1651 } 1652 1653 rcu_torture_barrier_cleanup(); 1654 torture_stop_kthread(rcu_torture_stall, stall_task); 1655 torture_stop_kthread(rcu_torture_writer, writer_task); 1656 1657 if (reader_tasks) { 1658 for (i = 0; i < nrealreaders; i++) 1659 torture_stop_kthread(rcu_torture_reader, 1660 reader_tasks[i]); 1661 kfree(reader_tasks); 1662 } 1663 rcu_torture_current = NULL; 1664 1665 if (fakewriter_tasks) { 1666 for (i = 0; i < nfakewriters; i++) { 1667 torture_stop_kthread(rcu_torture_fakewriter, 1668 fakewriter_tasks[i]); 1669 } 1670 kfree(fakewriter_tasks); 1671 fakewriter_tasks = NULL; 1672 } 1673 1674 torture_stop_kthread(rcu_torture_stats, stats_task); 1675 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1676 for (i = 0; i < ncbflooders; i++) 1677 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]); 1678 if ((test_boost == 1 && cur_ops->can_boost) || 1679 test_boost == 2) 1680 cpuhp_remove_state(rcutor_hp); 1681 1682 /* 1683 * Wait for all RCU callbacks to fire, then do flavor-specific 1684 * cleanup operations. 1685 */ 1686 if (cur_ops->cb_barrier != NULL) 1687 cur_ops->cb_barrier(); 1688 if (cur_ops->cleanup != NULL) 1689 cur_ops->cleanup(); 1690 1691 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1692 1693 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1694 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1695 else if (torture_onoff_failures()) 1696 rcu_torture_print_module_parms(cur_ops, 1697 "End of test: RCU_HOTPLUG"); 1698 else 1699 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1700 torture_cleanup_end(); 1701 } 1702 1703 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1704 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1705 { 1706 } 1707 1708 static void rcu_torture_err_cb(struct rcu_head *rhp) 1709 { 1710 /* 1711 * This -might- happen due to race conditions, but is unlikely. 1712 * The scenario that leads to this happening is that the 1713 * first of the pair of duplicate callbacks is queued, 1714 * someone else starts a grace period that includes that 1715 * callback, then the second of the pair must wait for the 1716 * next grace period. Unlikely, but can happen. If it 1717 * does happen, the debug-objects subsystem won't have splatted. 1718 */ 1719 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1720 } 1721 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1722 1723 /* 1724 * Verify that double-free causes debug-objects to complain, but only 1725 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1726 * cannot be carried out. 1727 */ 1728 static void rcu_test_debug_objects(void) 1729 { 1730 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1731 struct rcu_head rh1; 1732 struct rcu_head rh2; 1733 1734 init_rcu_head_on_stack(&rh1); 1735 init_rcu_head_on_stack(&rh2); 1736 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1737 1738 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1739 preempt_disable(); /* Prevent preemption from interrupting test. */ 1740 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1741 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1742 local_irq_disable(); /* Make it harder to start a new grace period. */ 1743 call_rcu(&rh2, rcu_torture_leak_cb); 1744 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1745 local_irq_enable(); 1746 rcu_read_unlock(); 1747 preempt_enable(); 1748 1749 /* Wait for them all to get done so we can safely return. */ 1750 rcu_barrier(); 1751 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1752 destroy_rcu_head_on_stack(&rh1); 1753 destroy_rcu_head_on_stack(&rh2); 1754 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1755 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1756 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1757 } 1758 1759 static int __init 1760 rcu_torture_init(void) 1761 { 1762 int i; 1763 int cpu; 1764 int firsterr = 0; 1765 static struct rcu_torture_ops *torture_ops[] = { 1766 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 1767 &sched_ops, RCUTORTURE_TASKS_OPS 1768 }; 1769 1770 if (!torture_init_begin(torture_type, verbose, &torture_runnable)) 1771 return -EBUSY; 1772 1773 /* Process args and tell the world that the torturer is on the job. */ 1774 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1775 cur_ops = torture_ops[i]; 1776 if (strcmp(torture_type, cur_ops->name) == 0) 1777 break; 1778 } 1779 if (i == ARRAY_SIZE(torture_ops)) { 1780 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1781 torture_type); 1782 pr_alert("rcu-torture types:"); 1783 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1784 pr_alert(" %s", torture_ops[i]->name); 1785 pr_alert("\n"); 1786 firsterr = -EINVAL; 1787 goto unwind; 1788 } 1789 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1790 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1791 fqs_duration = 0; 1792 } 1793 if (cur_ops->init) 1794 cur_ops->init(); 1795 1796 if (nreaders >= 0) { 1797 nrealreaders = nreaders; 1798 } else { 1799 nrealreaders = num_online_cpus() - 2 - nreaders; 1800 if (nrealreaders <= 0) 1801 nrealreaders = 1; 1802 } 1803 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1804 1805 /* Set up the freelist. */ 1806 1807 INIT_LIST_HEAD(&rcu_torture_freelist); 1808 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1809 rcu_tortures[i].rtort_mbtest = 0; 1810 list_add_tail(&rcu_tortures[i].rtort_free, 1811 &rcu_torture_freelist); 1812 } 1813 1814 /* Initialize the statistics so that each run gets its own numbers. */ 1815 1816 rcu_torture_current = NULL; 1817 rcu_torture_current_version = 0; 1818 atomic_set(&n_rcu_torture_alloc, 0); 1819 atomic_set(&n_rcu_torture_alloc_fail, 0); 1820 atomic_set(&n_rcu_torture_free, 0); 1821 atomic_set(&n_rcu_torture_mberror, 0); 1822 atomic_set(&n_rcu_torture_error, 0); 1823 n_rcu_torture_barrier_error = 0; 1824 n_rcu_torture_boost_ktrerror = 0; 1825 n_rcu_torture_boost_rterror = 0; 1826 n_rcu_torture_boost_failure = 0; 1827 n_rcu_torture_boosts = 0; 1828 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1829 atomic_set(&rcu_torture_wcount[i], 0); 1830 for_each_possible_cpu(cpu) { 1831 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1832 per_cpu(rcu_torture_count, cpu)[i] = 0; 1833 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1834 } 1835 } 1836 1837 /* Start up the kthreads. */ 1838 1839 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1840 writer_task); 1841 if (firsterr) 1842 goto unwind; 1843 if (nfakewriters > 0) { 1844 fakewriter_tasks = kzalloc(nfakewriters * 1845 sizeof(fakewriter_tasks[0]), 1846 GFP_KERNEL); 1847 if (fakewriter_tasks == NULL) { 1848 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1849 firsterr = -ENOMEM; 1850 goto unwind; 1851 } 1852 } 1853 for (i = 0; i < nfakewriters; i++) { 1854 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1855 NULL, fakewriter_tasks[i]); 1856 if (firsterr) 1857 goto unwind; 1858 } 1859 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1860 GFP_KERNEL); 1861 if (reader_tasks == NULL) { 1862 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1863 firsterr = -ENOMEM; 1864 goto unwind; 1865 } 1866 for (i = 0; i < nrealreaders; i++) { 1867 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1868 reader_tasks[i]); 1869 if (firsterr) 1870 goto unwind; 1871 } 1872 if (stat_interval > 0) { 1873 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1874 stats_task); 1875 if (firsterr) 1876 goto unwind; 1877 } 1878 if (test_no_idle_hz && shuffle_interval > 0) { 1879 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1880 if (firsterr) 1881 goto unwind; 1882 } 1883 if (stutter < 0) 1884 stutter = 0; 1885 if (stutter) { 1886 firsterr = torture_stutter_init(stutter * HZ); 1887 if (firsterr) 1888 goto unwind; 1889 } 1890 if (fqs_duration < 0) 1891 fqs_duration = 0; 1892 if (fqs_duration) { 1893 /* Create the fqs thread */ 1894 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1895 fqs_task); 1896 if (firsterr) 1897 goto unwind; 1898 } 1899 if (test_boost_interval < 1) 1900 test_boost_interval = 1; 1901 if (test_boost_duration < 2) 1902 test_boost_duration = 2; 1903 if ((test_boost == 1 && cur_ops->can_boost) || 1904 test_boost == 2) { 1905 1906 boost_starttime = jiffies + test_boost_interval * HZ; 1907 1908 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 1909 rcutorture_booster_init, 1910 rcutorture_booster_cleanup); 1911 if (firsterr < 0) 1912 goto unwind; 1913 rcutor_hp = firsterr; 1914 } 1915 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1916 if (firsterr) 1917 goto unwind; 1918 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1919 if (firsterr) 1920 goto unwind; 1921 firsterr = rcu_torture_stall_init(); 1922 if (firsterr) 1923 goto unwind; 1924 firsterr = rcu_torture_barrier_init(); 1925 if (firsterr) 1926 goto unwind; 1927 if (object_debug) 1928 rcu_test_debug_objects(); 1929 if (cbflood_n_burst > 0) { 1930 /* Create the cbflood threads */ 1931 ncbflooders = (num_online_cpus() + 3) / 4; 1932 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task), 1933 GFP_KERNEL); 1934 if (!cbflood_task) { 1935 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1936 firsterr = -ENOMEM; 1937 goto unwind; 1938 } 1939 for (i = 0; i < ncbflooders; i++) { 1940 firsterr = torture_create_kthread(rcu_torture_cbflood, 1941 NULL, 1942 cbflood_task[i]); 1943 if (firsterr) 1944 goto unwind; 1945 } 1946 } 1947 rcutorture_record_test_transition(); 1948 torture_init_end(); 1949 return 0; 1950 1951 unwind: 1952 torture_init_end(); 1953 rcu_torture_cleanup(); 1954 return firsterr; 1955 } 1956 1957 module_init(rcu_torture_init); 1958 module_exit(rcu_torture_cleanup); 1959