1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, you can access it online at 16 * http://www.gnu.org/licenses/gpl-2.0.html. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@freedesktop.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <linux/trace_clock.h> 50 #include <asm/byteorder.h> 51 #include <linux/torture.h> 52 53 MODULE_LICENSE("GPL"); 54 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>"); 55 56 57 torture_param(int, fqs_duration, 0, 58 "Duration of fqs bursts (us), 0 to disable"); 59 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 60 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 61 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 62 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 63 torture_param(bool, gp_normal, false, 64 "Use normal (non-expedited) GP wait primitives"); 65 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 66 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 67 torture_param(int, n_barrier_cbs, 0, 68 "# of callbacks/kthreads for barrier testing"); 69 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 70 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 71 torture_param(int, object_debug, 0, 72 "Enable debug-object double call_rcu() testing"); 73 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 74 torture_param(int, onoff_interval, 0, 75 "Time between CPU hotplugs (s), 0=disable"); 76 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 77 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 78 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 79 torture_param(int, stall_cpu_holdoff, 10, 80 "Time to wait before starting stall (s)."); 81 torture_param(int, stat_interval, 60, 82 "Number of seconds between stats printk()s"); 83 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 84 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 85 torture_param(int, test_boost_duration, 4, 86 "Duration of each boost test, seconds."); 87 torture_param(int, test_boost_interval, 7, 88 "Interval between boost tests, seconds."); 89 torture_param(bool, test_no_idle_hz, true, 90 "Test support for tickless idle CPUs"); 91 torture_param(bool, verbose, true, 92 "Enable verbose debugging printk()s"); 93 94 static char *torture_type = "rcu"; 95 module_param(torture_type, charp, 0444); 96 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)"); 97 98 static int nrealreaders; 99 static struct task_struct *writer_task; 100 static struct task_struct **fakewriter_tasks; 101 static struct task_struct **reader_tasks; 102 static struct task_struct *stats_task; 103 static struct task_struct *fqs_task; 104 static struct task_struct *boost_tasks[NR_CPUS]; 105 static struct task_struct *stall_task; 106 static struct task_struct **barrier_cbs_tasks; 107 static struct task_struct *barrier_task; 108 109 #define RCU_TORTURE_PIPE_LEN 10 110 111 struct rcu_torture { 112 struct rcu_head rtort_rcu; 113 int rtort_pipe_count; 114 struct list_head rtort_free; 115 int rtort_mbtest; 116 }; 117 118 static LIST_HEAD(rcu_torture_freelist); 119 static struct rcu_torture __rcu *rcu_torture_current; 120 static unsigned long rcu_torture_current_version; 121 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 122 static DEFINE_SPINLOCK(rcu_torture_lock); 123 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 124 rcu_torture_count) = { 0 }; 125 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], 126 rcu_torture_batch) = { 0 }; 127 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 128 static atomic_t n_rcu_torture_alloc; 129 static atomic_t n_rcu_torture_alloc_fail; 130 static atomic_t n_rcu_torture_free; 131 static atomic_t n_rcu_torture_mberror; 132 static atomic_t n_rcu_torture_error; 133 static long n_rcu_torture_barrier_error; 134 static long n_rcu_torture_boost_ktrerror; 135 static long n_rcu_torture_boost_rterror; 136 static long n_rcu_torture_boost_failure; 137 static long n_rcu_torture_boosts; 138 static long n_rcu_torture_timers; 139 static long n_barrier_attempts; 140 static long n_barrier_successes; 141 static struct list_head rcu_torture_removed; 142 143 static int rcu_torture_writer_state; 144 #define RTWS_FIXED_DELAY 0 145 #define RTWS_DELAY 1 146 #define RTWS_REPLACE 2 147 #define RTWS_DEF_FREE 3 148 #define RTWS_EXP_SYNC 4 149 #define RTWS_COND_GET 5 150 #define RTWS_COND_SYNC 6 151 #define RTWS_SYNC 7 152 #define RTWS_STUTTER 8 153 #define RTWS_STOPPING 9 154 155 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 156 #define RCUTORTURE_RUNNABLE_INIT 1 157 #else 158 #define RCUTORTURE_RUNNABLE_INIT 0 159 #endif 160 int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 161 module_param(rcutorture_runnable, int, 0444); 162 MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot"); 163 164 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 165 #define rcu_can_boost() 1 166 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 167 #define rcu_can_boost() 0 168 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 169 170 #ifdef CONFIG_RCU_TRACE 171 static u64 notrace rcu_trace_clock_local(void) 172 { 173 u64 ts = trace_clock_local(); 174 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC); 175 return ts; 176 } 177 #else /* #ifdef CONFIG_RCU_TRACE */ 178 static u64 notrace rcu_trace_clock_local(void) 179 { 180 return 0ULL; 181 } 182 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 183 184 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 185 DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 186 /* and boost task create/destroy. */ 187 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 188 static bool barrier_phase; /* Test phase. */ 189 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 190 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 191 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 192 193 /* 194 * Allocate an element from the rcu_tortures pool. 195 */ 196 static struct rcu_torture * 197 rcu_torture_alloc(void) 198 { 199 struct list_head *p; 200 201 spin_lock_bh(&rcu_torture_lock); 202 if (list_empty(&rcu_torture_freelist)) { 203 atomic_inc(&n_rcu_torture_alloc_fail); 204 spin_unlock_bh(&rcu_torture_lock); 205 return NULL; 206 } 207 atomic_inc(&n_rcu_torture_alloc); 208 p = rcu_torture_freelist.next; 209 list_del_init(p); 210 spin_unlock_bh(&rcu_torture_lock); 211 return container_of(p, struct rcu_torture, rtort_free); 212 } 213 214 /* 215 * Free an element to the rcu_tortures pool. 216 */ 217 static void 218 rcu_torture_free(struct rcu_torture *p) 219 { 220 atomic_inc(&n_rcu_torture_free); 221 spin_lock_bh(&rcu_torture_lock); 222 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 223 spin_unlock_bh(&rcu_torture_lock); 224 } 225 226 /* 227 * Operations vector for selecting different types of tests. 228 */ 229 230 struct rcu_torture_ops { 231 int ttype; 232 void (*init)(void); 233 int (*readlock)(void); 234 void (*read_delay)(struct torture_random_state *rrsp); 235 void (*readunlock)(int idx); 236 int (*completed)(void); 237 void (*deferred_free)(struct rcu_torture *p); 238 void (*sync)(void); 239 void (*exp_sync)(void); 240 unsigned long (*get_state)(void); 241 void (*cond_sync)(unsigned long oldstate); 242 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 243 void (*cb_barrier)(void); 244 void (*fqs)(void); 245 void (*stats)(char *page); 246 int irq_capable; 247 int can_boost; 248 const char *name; 249 }; 250 251 static struct rcu_torture_ops *cur_ops; 252 253 /* 254 * Definitions for rcu torture testing. 255 */ 256 257 static int rcu_torture_read_lock(void) __acquires(RCU) 258 { 259 rcu_read_lock(); 260 return 0; 261 } 262 263 static void rcu_read_delay(struct torture_random_state *rrsp) 264 { 265 const unsigned long shortdelay_us = 200; 266 const unsigned long longdelay_ms = 50; 267 268 /* We want a short delay sometimes to make a reader delay the grace 269 * period, and we want a long delay occasionally to trigger 270 * force_quiescent_state. */ 271 272 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 273 mdelay(longdelay_ms); 274 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 275 udelay(shortdelay_us); 276 #ifdef CONFIG_PREEMPT 277 if (!preempt_count() && 278 !(torture_random(rrsp) % (nrealreaders * 20000))) 279 preempt_schedule(); /* No QS if preempt_disable() in effect */ 280 #endif 281 } 282 283 static void rcu_torture_read_unlock(int idx) __releases(RCU) 284 { 285 rcu_read_unlock(); 286 } 287 288 static int rcu_torture_completed(void) 289 { 290 return rcu_batches_completed(); 291 } 292 293 /* 294 * Update callback in the pipe. This should be invoked after a grace period. 295 */ 296 static bool 297 rcu_torture_pipe_update_one(struct rcu_torture *rp) 298 { 299 int i; 300 301 i = rp->rtort_pipe_count; 302 if (i > RCU_TORTURE_PIPE_LEN) 303 i = RCU_TORTURE_PIPE_LEN; 304 atomic_inc(&rcu_torture_wcount[i]); 305 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 306 rp->rtort_mbtest = 0; 307 return true; 308 } 309 return false; 310 } 311 312 /* 313 * Update all callbacks in the pipe. Suitable for synchronous grace-period 314 * primitives. 315 */ 316 static void 317 rcu_torture_pipe_update(struct rcu_torture *old_rp) 318 { 319 struct rcu_torture *rp; 320 struct rcu_torture *rp1; 321 322 if (old_rp) 323 list_add(&old_rp->rtort_free, &rcu_torture_removed); 324 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 325 if (rcu_torture_pipe_update_one(rp)) { 326 list_del(&rp->rtort_free); 327 rcu_torture_free(rp); 328 } 329 } 330 } 331 332 static void 333 rcu_torture_cb(struct rcu_head *p) 334 { 335 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 336 337 if (torture_must_stop_irq()) { 338 /* Test is ending, just drop callbacks on the floor. */ 339 /* The next initialization will pick up the pieces. */ 340 return; 341 } 342 if (rcu_torture_pipe_update_one(rp)) 343 rcu_torture_free(rp); 344 else 345 cur_ops->deferred_free(rp); 346 } 347 348 static int rcu_no_completed(void) 349 { 350 return 0; 351 } 352 353 static void rcu_torture_deferred_free(struct rcu_torture *p) 354 { 355 call_rcu(&p->rtort_rcu, rcu_torture_cb); 356 } 357 358 static void rcu_sync_torture_init(void) 359 { 360 INIT_LIST_HEAD(&rcu_torture_removed); 361 } 362 363 static struct rcu_torture_ops rcu_ops = { 364 .ttype = RCU_FLAVOR, 365 .init = rcu_sync_torture_init, 366 .readlock = rcu_torture_read_lock, 367 .read_delay = rcu_read_delay, 368 .readunlock = rcu_torture_read_unlock, 369 .completed = rcu_torture_completed, 370 .deferred_free = rcu_torture_deferred_free, 371 .sync = synchronize_rcu, 372 .exp_sync = synchronize_rcu_expedited, 373 .get_state = get_state_synchronize_rcu, 374 .cond_sync = cond_synchronize_rcu, 375 .call = call_rcu, 376 .cb_barrier = rcu_barrier, 377 .fqs = rcu_force_quiescent_state, 378 .stats = NULL, 379 .irq_capable = 1, 380 .can_boost = rcu_can_boost(), 381 .name = "rcu" 382 }; 383 384 /* 385 * Definitions for rcu_bh torture testing. 386 */ 387 388 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 389 { 390 rcu_read_lock_bh(); 391 return 0; 392 } 393 394 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 395 { 396 rcu_read_unlock_bh(); 397 } 398 399 static int rcu_bh_torture_completed(void) 400 { 401 return rcu_batches_completed_bh(); 402 } 403 404 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 405 { 406 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 407 } 408 409 static struct rcu_torture_ops rcu_bh_ops = { 410 .ttype = RCU_BH_FLAVOR, 411 .init = rcu_sync_torture_init, 412 .readlock = rcu_bh_torture_read_lock, 413 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 414 .readunlock = rcu_bh_torture_read_unlock, 415 .completed = rcu_bh_torture_completed, 416 .deferred_free = rcu_bh_torture_deferred_free, 417 .sync = synchronize_rcu_bh, 418 .exp_sync = synchronize_rcu_bh_expedited, 419 .call = call_rcu_bh, 420 .cb_barrier = rcu_barrier_bh, 421 .fqs = rcu_bh_force_quiescent_state, 422 .stats = NULL, 423 .irq_capable = 1, 424 .name = "rcu_bh" 425 }; 426 427 /* 428 * Don't even think about trying any of these in real life!!! 429 * The names includes "busted", and they really means it! 430 * The only purpose of these functions is to provide a buggy RCU 431 * implementation to make sure that rcutorture correctly emits 432 * buggy-RCU error messages. 433 */ 434 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 435 { 436 /* This is a deliberate bug for testing purposes only! */ 437 rcu_torture_cb(&p->rtort_rcu); 438 } 439 440 static void synchronize_rcu_busted(void) 441 { 442 /* This is a deliberate bug for testing purposes only! */ 443 } 444 445 static void 446 call_rcu_busted(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) 447 { 448 /* This is a deliberate bug for testing purposes only! */ 449 func(head); 450 } 451 452 static struct rcu_torture_ops rcu_busted_ops = { 453 .ttype = INVALID_RCU_FLAVOR, 454 .init = rcu_sync_torture_init, 455 .readlock = rcu_torture_read_lock, 456 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 457 .readunlock = rcu_torture_read_unlock, 458 .completed = rcu_no_completed, 459 .deferred_free = rcu_busted_torture_deferred_free, 460 .sync = synchronize_rcu_busted, 461 .exp_sync = synchronize_rcu_busted, 462 .call = call_rcu_busted, 463 .cb_barrier = NULL, 464 .fqs = NULL, 465 .stats = NULL, 466 .irq_capable = 1, 467 .name = "rcu_busted" 468 }; 469 470 /* 471 * Definitions for srcu torture testing. 472 */ 473 474 DEFINE_STATIC_SRCU(srcu_ctl); 475 476 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 477 { 478 return srcu_read_lock(&srcu_ctl); 479 } 480 481 static void srcu_read_delay(struct torture_random_state *rrsp) 482 { 483 long delay; 484 const long uspertick = 1000000 / HZ; 485 const long longdelay = 10; 486 487 /* We want there to be long-running readers, but not all the time. */ 488 489 delay = torture_random(rrsp) % 490 (nrealreaders * 2 * longdelay * uspertick); 491 if (!delay) 492 schedule_timeout_interruptible(longdelay); 493 else 494 rcu_read_delay(rrsp); 495 } 496 497 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 498 { 499 srcu_read_unlock(&srcu_ctl, idx); 500 } 501 502 static int srcu_torture_completed(void) 503 { 504 return srcu_batches_completed(&srcu_ctl); 505 } 506 507 static void srcu_torture_deferred_free(struct rcu_torture *rp) 508 { 509 call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb); 510 } 511 512 static void srcu_torture_synchronize(void) 513 { 514 synchronize_srcu(&srcu_ctl); 515 } 516 517 static void srcu_torture_call(struct rcu_head *head, 518 void (*func)(struct rcu_head *head)) 519 { 520 call_srcu(&srcu_ctl, head, func); 521 } 522 523 static void srcu_torture_barrier(void) 524 { 525 srcu_barrier(&srcu_ctl); 526 } 527 528 static void srcu_torture_stats(char *page) 529 { 530 int cpu; 531 int idx = srcu_ctl.completed & 0x1; 532 533 page += sprintf(page, "%s%s per-CPU(idx=%d):", 534 torture_type, TORTURE_FLAG, idx); 535 for_each_possible_cpu(cpu) { 536 long c0, c1; 537 538 c0 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx]; 539 c1 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]; 540 page += sprintf(page, " %d(%ld,%ld)", cpu, c0, c1); 541 } 542 sprintf(page, "\n"); 543 } 544 545 static void srcu_torture_synchronize_expedited(void) 546 { 547 synchronize_srcu_expedited(&srcu_ctl); 548 } 549 550 static struct rcu_torture_ops srcu_ops = { 551 .ttype = SRCU_FLAVOR, 552 .init = rcu_sync_torture_init, 553 .readlock = srcu_torture_read_lock, 554 .read_delay = srcu_read_delay, 555 .readunlock = srcu_torture_read_unlock, 556 .completed = srcu_torture_completed, 557 .deferred_free = srcu_torture_deferred_free, 558 .sync = srcu_torture_synchronize, 559 .exp_sync = srcu_torture_synchronize_expedited, 560 .call = srcu_torture_call, 561 .cb_barrier = srcu_torture_barrier, 562 .stats = srcu_torture_stats, 563 .name = "srcu" 564 }; 565 566 /* 567 * Definitions for sched torture testing. 568 */ 569 570 static int sched_torture_read_lock(void) 571 { 572 preempt_disable(); 573 return 0; 574 } 575 576 static void sched_torture_read_unlock(int idx) 577 { 578 preempt_enable(); 579 } 580 581 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 582 { 583 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 584 } 585 586 static struct rcu_torture_ops sched_ops = { 587 .ttype = RCU_SCHED_FLAVOR, 588 .init = rcu_sync_torture_init, 589 .readlock = sched_torture_read_lock, 590 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 591 .readunlock = sched_torture_read_unlock, 592 .completed = rcu_no_completed, 593 .deferred_free = rcu_sched_torture_deferred_free, 594 .sync = synchronize_sched, 595 .exp_sync = synchronize_sched_expedited, 596 .call = call_rcu_sched, 597 .cb_barrier = rcu_barrier_sched, 598 .fqs = rcu_sched_force_quiescent_state, 599 .stats = NULL, 600 .irq_capable = 1, 601 .name = "sched" 602 }; 603 604 /* 605 * RCU torture priority-boost testing. Runs one real-time thread per 606 * CPU for moderate bursts, repeatedly registering RCU callbacks and 607 * spinning waiting for them to be invoked. If a given callback takes 608 * too long to be invoked, we assume that priority inversion has occurred. 609 */ 610 611 struct rcu_boost_inflight { 612 struct rcu_head rcu; 613 int inflight; 614 }; 615 616 static void rcu_torture_boost_cb(struct rcu_head *head) 617 { 618 struct rcu_boost_inflight *rbip = 619 container_of(head, struct rcu_boost_inflight, rcu); 620 621 smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */ 622 rbip->inflight = 0; 623 } 624 625 static int rcu_torture_boost(void *arg) 626 { 627 unsigned long call_rcu_time; 628 unsigned long endtime; 629 unsigned long oldstarttime; 630 struct rcu_boost_inflight rbi = { .inflight = 0 }; 631 struct sched_param sp; 632 633 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 634 635 /* Set real-time priority. */ 636 sp.sched_priority = 1; 637 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 638 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 639 n_rcu_torture_boost_rterror++; 640 } 641 642 init_rcu_head_on_stack(&rbi.rcu); 643 /* Each pass through the following loop does one boost-test cycle. */ 644 do { 645 /* Wait for the next test interval. */ 646 oldstarttime = boost_starttime; 647 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 648 schedule_timeout_interruptible(oldstarttime - jiffies); 649 stutter_wait("rcu_torture_boost"); 650 if (torture_must_stop()) 651 goto checkwait; 652 } 653 654 /* Do one boost-test interval. */ 655 endtime = oldstarttime + test_boost_duration * HZ; 656 call_rcu_time = jiffies; 657 while (ULONG_CMP_LT(jiffies, endtime)) { 658 /* If we don't have a callback in flight, post one. */ 659 if (!rbi.inflight) { 660 smp_mb(); /* RCU core before ->inflight = 1. */ 661 rbi.inflight = 1; 662 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 663 if (jiffies - call_rcu_time > 664 test_boost_duration * HZ - HZ / 2) { 665 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 666 n_rcu_torture_boost_failure++; 667 } 668 call_rcu_time = jiffies; 669 } 670 cond_resched(); 671 stutter_wait("rcu_torture_boost"); 672 if (torture_must_stop()) 673 goto checkwait; 674 } 675 676 /* 677 * Set the start time of the next test interval. 678 * Yes, this is vulnerable to long delays, but such 679 * delays simply cause a false negative for the next 680 * interval. Besides, we are running at RT priority, 681 * so delays should be relatively rare. 682 */ 683 while (oldstarttime == boost_starttime && 684 !kthread_should_stop()) { 685 if (mutex_trylock(&boost_mutex)) { 686 boost_starttime = jiffies + 687 test_boost_interval * HZ; 688 n_rcu_torture_boosts++; 689 mutex_unlock(&boost_mutex); 690 break; 691 } 692 schedule_timeout_uninterruptible(1); 693 } 694 695 /* Go do the stutter. */ 696 checkwait: stutter_wait("rcu_torture_boost"); 697 } while (!torture_must_stop()); 698 699 /* Clean up and exit. */ 700 while (!kthread_should_stop() || rbi.inflight) { 701 torture_shutdown_absorb("rcu_torture_boost"); 702 schedule_timeout_uninterruptible(1); 703 } 704 smp_mb(); /* order accesses to ->inflight before stack-frame death. */ 705 destroy_rcu_head_on_stack(&rbi.rcu); 706 torture_kthread_stopping("rcu_torture_boost"); 707 return 0; 708 } 709 710 /* 711 * RCU torture force-quiescent-state kthread. Repeatedly induces 712 * bursts of calls to force_quiescent_state(), increasing the probability 713 * of occurrence of some important types of race conditions. 714 */ 715 static int 716 rcu_torture_fqs(void *arg) 717 { 718 unsigned long fqs_resume_time; 719 int fqs_burst_remaining; 720 721 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 722 do { 723 fqs_resume_time = jiffies + fqs_stutter * HZ; 724 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 725 !kthread_should_stop()) { 726 schedule_timeout_interruptible(1); 727 } 728 fqs_burst_remaining = fqs_duration; 729 while (fqs_burst_remaining > 0 && 730 !kthread_should_stop()) { 731 cur_ops->fqs(); 732 udelay(fqs_holdoff); 733 fqs_burst_remaining -= fqs_holdoff; 734 } 735 stutter_wait("rcu_torture_fqs"); 736 } while (!torture_must_stop()); 737 torture_kthread_stopping("rcu_torture_fqs"); 738 return 0; 739 } 740 741 /* 742 * RCU torture writer kthread. Repeatedly substitutes a new structure 743 * for that pointed to by rcu_torture_current, freeing the old structure 744 * after a series of grace periods (the "pipeline"). 745 */ 746 static int 747 rcu_torture_writer(void *arg) 748 { 749 unsigned long gp_snap; 750 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 751 bool gp_sync1 = gp_sync; 752 int i; 753 struct rcu_torture *rp; 754 struct rcu_torture *old_rp; 755 static DEFINE_TORTURE_RANDOM(rand); 756 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 757 RTWS_COND_GET, RTWS_SYNC }; 758 int nsynctypes = 0; 759 760 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 761 762 /* Initialize synctype[] array. If none set, take default. */ 763 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync) 764 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 765 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) 766 synctype[nsynctypes++] = RTWS_COND_GET; 767 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) 768 pr_alert("rcu_torture_writer: gp_cond without primitives.\n"); 769 if (gp_exp1 && cur_ops->exp_sync) 770 synctype[nsynctypes++] = RTWS_EXP_SYNC; 771 else if (gp_exp && !cur_ops->exp_sync) 772 pr_alert("rcu_torture_writer: gp_exp without primitives.\n"); 773 if (gp_normal1 && cur_ops->deferred_free) 774 synctype[nsynctypes++] = RTWS_DEF_FREE; 775 else if (gp_normal && !cur_ops->deferred_free) 776 pr_alert("rcu_torture_writer: gp_normal without primitives.\n"); 777 if (gp_sync1 && cur_ops->sync) 778 synctype[nsynctypes++] = RTWS_SYNC; 779 else if (gp_sync && !cur_ops->sync) 780 pr_alert("rcu_torture_writer: gp_sync without primitives.\n"); 781 if (WARN_ONCE(nsynctypes == 0, 782 "rcu_torture_writer: No update-side primitives.\n")) { 783 /* 784 * No updates primitives, so don't try updating. 785 * The resulting test won't be testing much, hence the 786 * above WARN_ONCE(). 787 */ 788 rcu_torture_writer_state = RTWS_STOPPING; 789 torture_kthread_stopping("rcu_torture_writer"); 790 } 791 792 do { 793 rcu_torture_writer_state = RTWS_FIXED_DELAY; 794 schedule_timeout_uninterruptible(1); 795 rp = rcu_torture_alloc(); 796 if (rp == NULL) 797 continue; 798 rp->rtort_pipe_count = 0; 799 rcu_torture_writer_state = RTWS_DELAY; 800 udelay(torture_random(&rand) & 0x3ff); 801 rcu_torture_writer_state = RTWS_REPLACE; 802 old_rp = rcu_dereference_check(rcu_torture_current, 803 current == writer_task); 804 rp->rtort_mbtest = 1; 805 rcu_assign_pointer(rcu_torture_current, rp); 806 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 807 if (old_rp) { 808 i = old_rp->rtort_pipe_count; 809 if (i > RCU_TORTURE_PIPE_LEN) 810 i = RCU_TORTURE_PIPE_LEN; 811 atomic_inc(&rcu_torture_wcount[i]); 812 old_rp->rtort_pipe_count++; 813 switch (synctype[torture_random(&rand) % nsynctypes]) { 814 case RTWS_DEF_FREE: 815 rcu_torture_writer_state = RTWS_DEF_FREE; 816 cur_ops->deferred_free(old_rp); 817 break; 818 case RTWS_EXP_SYNC: 819 rcu_torture_writer_state = RTWS_EXP_SYNC; 820 cur_ops->exp_sync(); 821 rcu_torture_pipe_update(old_rp); 822 break; 823 case RTWS_COND_GET: 824 rcu_torture_writer_state = RTWS_COND_GET; 825 gp_snap = cur_ops->get_state(); 826 i = torture_random(&rand) % 16; 827 if (i != 0) 828 schedule_timeout_interruptible(i); 829 udelay(torture_random(&rand) % 1000); 830 rcu_torture_writer_state = RTWS_COND_SYNC; 831 cur_ops->cond_sync(gp_snap); 832 rcu_torture_pipe_update(old_rp); 833 break; 834 case RTWS_SYNC: 835 rcu_torture_writer_state = RTWS_SYNC; 836 cur_ops->sync(); 837 rcu_torture_pipe_update(old_rp); 838 break; 839 default: 840 WARN_ON_ONCE(1); 841 break; 842 } 843 } 844 rcutorture_record_progress(++rcu_torture_current_version); 845 rcu_torture_writer_state = RTWS_STUTTER; 846 stutter_wait("rcu_torture_writer"); 847 } while (!torture_must_stop()); 848 rcu_torture_writer_state = RTWS_STOPPING; 849 torture_kthread_stopping("rcu_torture_writer"); 850 return 0; 851 } 852 853 /* 854 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 855 * delay between calls. 856 */ 857 static int 858 rcu_torture_fakewriter(void *arg) 859 { 860 DEFINE_TORTURE_RANDOM(rand); 861 862 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 863 set_user_nice(current, MAX_NICE); 864 865 do { 866 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 867 udelay(torture_random(&rand) & 0x3ff); 868 if (cur_ops->cb_barrier != NULL && 869 torture_random(&rand) % (nfakewriters * 8) == 0) { 870 cur_ops->cb_barrier(); 871 } else if (gp_normal == gp_exp) { 872 if (torture_random(&rand) & 0x80) 873 cur_ops->sync(); 874 else 875 cur_ops->exp_sync(); 876 } else if (gp_normal) { 877 cur_ops->sync(); 878 } else { 879 cur_ops->exp_sync(); 880 } 881 stutter_wait("rcu_torture_fakewriter"); 882 } while (!torture_must_stop()); 883 884 torture_kthread_stopping("rcu_torture_fakewriter"); 885 return 0; 886 } 887 888 static void rcutorture_trace_dump(void) 889 { 890 static atomic_t beenhere = ATOMIC_INIT(0); 891 892 if (atomic_read(&beenhere)) 893 return; 894 if (atomic_xchg(&beenhere, 1) != 0) 895 return; 896 ftrace_dump(DUMP_ALL); 897 } 898 899 /* 900 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 901 * incrementing the corresponding element of the pipeline array. The 902 * counter in the element should never be greater than 1, otherwise, the 903 * RCU implementation is broken. 904 */ 905 static void rcu_torture_timer(unsigned long unused) 906 { 907 int idx; 908 int completed; 909 int completed_end; 910 static DEFINE_TORTURE_RANDOM(rand); 911 static DEFINE_SPINLOCK(rand_lock); 912 struct rcu_torture *p; 913 int pipe_count; 914 unsigned long long ts; 915 916 idx = cur_ops->readlock(); 917 completed = cur_ops->completed(); 918 ts = rcu_trace_clock_local(); 919 p = rcu_dereference_check(rcu_torture_current, 920 rcu_read_lock_bh_held() || 921 rcu_read_lock_sched_held() || 922 srcu_read_lock_held(&srcu_ctl)); 923 if (p == NULL) { 924 /* Leave because rcu_torture_writer is not yet underway */ 925 cur_ops->readunlock(idx); 926 return; 927 } 928 if (p->rtort_mbtest == 0) 929 atomic_inc(&n_rcu_torture_mberror); 930 spin_lock(&rand_lock); 931 cur_ops->read_delay(&rand); 932 n_rcu_torture_timers++; 933 spin_unlock(&rand_lock); 934 preempt_disable(); 935 pipe_count = p->rtort_pipe_count; 936 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 937 /* Should not happen, but... */ 938 pipe_count = RCU_TORTURE_PIPE_LEN; 939 } 940 completed_end = cur_ops->completed(); 941 if (pipe_count > 1) { 942 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts, 943 completed, completed_end); 944 rcutorture_trace_dump(); 945 } 946 __this_cpu_inc(rcu_torture_count[pipe_count]); 947 completed = completed_end - completed; 948 if (completed > RCU_TORTURE_PIPE_LEN) { 949 /* Should not happen, but... */ 950 completed = RCU_TORTURE_PIPE_LEN; 951 } 952 __this_cpu_inc(rcu_torture_batch[completed]); 953 preempt_enable(); 954 cur_ops->readunlock(idx); 955 } 956 957 /* 958 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 959 * incrementing the corresponding element of the pipeline array. The 960 * counter in the element should never be greater than 1, otherwise, the 961 * RCU implementation is broken. 962 */ 963 static int 964 rcu_torture_reader(void *arg) 965 { 966 int completed; 967 int completed_end; 968 int idx; 969 DEFINE_TORTURE_RANDOM(rand); 970 struct rcu_torture *p; 971 int pipe_count; 972 struct timer_list t; 973 unsigned long long ts; 974 975 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 976 set_user_nice(current, MAX_NICE); 977 if (irqreader && cur_ops->irq_capable) 978 setup_timer_on_stack(&t, rcu_torture_timer, 0); 979 980 do { 981 if (irqreader && cur_ops->irq_capable) { 982 if (!timer_pending(&t)) 983 mod_timer(&t, jiffies + 1); 984 } 985 idx = cur_ops->readlock(); 986 completed = cur_ops->completed(); 987 ts = rcu_trace_clock_local(); 988 p = rcu_dereference_check(rcu_torture_current, 989 rcu_read_lock_bh_held() || 990 rcu_read_lock_sched_held() || 991 srcu_read_lock_held(&srcu_ctl)); 992 if (p == NULL) { 993 /* Wait for rcu_torture_writer to get underway */ 994 cur_ops->readunlock(idx); 995 schedule_timeout_interruptible(HZ); 996 continue; 997 } 998 if (p->rtort_mbtest == 0) 999 atomic_inc(&n_rcu_torture_mberror); 1000 cur_ops->read_delay(&rand); 1001 preempt_disable(); 1002 pipe_count = p->rtort_pipe_count; 1003 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1004 /* Should not happen, but... */ 1005 pipe_count = RCU_TORTURE_PIPE_LEN; 1006 } 1007 completed_end = cur_ops->completed(); 1008 if (pipe_count > 1) { 1009 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1010 ts, completed, completed_end); 1011 rcutorture_trace_dump(); 1012 } 1013 __this_cpu_inc(rcu_torture_count[pipe_count]); 1014 completed = completed_end - completed; 1015 if (completed > RCU_TORTURE_PIPE_LEN) { 1016 /* Should not happen, but... */ 1017 completed = RCU_TORTURE_PIPE_LEN; 1018 } 1019 __this_cpu_inc(rcu_torture_batch[completed]); 1020 preempt_enable(); 1021 cur_ops->readunlock(idx); 1022 cond_resched(); 1023 stutter_wait("rcu_torture_reader"); 1024 } while (!torture_must_stop()); 1025 if (irqreader && cur_ops->irq_capable) { 1026 del_timer_sync(&t); 1027 destroy_timer_on_stack(&t); 1028 } 1029 torture_kthread_stopping("rcu_torture_reader"); 1030 return 0; 1031 } 1032 1033 /* 1034 * Create an RCU-torture statistics message in the specified buffer. 1035 */ 1036 static void 1037 rcu_torture_printk(char *page) 1038 { 1039 int cpu; 1040 int i; 1041 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1042 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1043 static unsigned long rtcv_snap = ULONG_MAX; 1044 1045 for_each_possible_cpu(cpu) { 1046 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1047 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1048 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1049 } 1050 } 1051 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1052 if (pipesummary[i] != 0) 1053 break; 1054 } 1055 page += sprintf(page, "%s%s ", torture_type, TORTURE_FLAG); 1056 page += sprintf(page, 1057 "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1058 rcu_torture_current, 1059 rcu_torture_current_version, 1060 list_empty(&rcu_torture_freelist), 1061 atomic_read(&n_rcu_torture_alloc), 1062 atomic_read(&n_rcu_torture_alloc_fail), 1063 atomic_read(&n_rcu_torture_free)); 1064 page += sprintf(page, "rtmbe: %d rtbke: %ld rtbre: %ld ", 1065 atomic_read(&n_rcu_torture_mberror), 1066 n_rcu_torture_boost_ktrerror, 1067 n_rcu_torture_boost_rterror); 1068 page += sprintf(page, "rtbf: %ld rtb: %ld nt: %ld ", 1069 n_rcu_torture_boost_failure, 1070 n_rcu_torture_boosts, 1071 n_rcu_torture_timers); 1072 page = torture_onoff_stats(page); 1073 page += sprintf(page, "barrier: %ld/%ld:%ld", 1074 n_barrier_successes, 1075 n_barrier_attempts, 1076 n_rcu_torture_barrier_error); 1077 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 1078 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1079 n_rcu_torture_barrier_error != 0 || 1080 n_rcu_torture_boost_ktrerror != 0 || 1081 n_rcu_torture_boost_rterror != 0 || 1082 n_rcu_torture_boost_failure != 0 || 1083 i > 1) { 1084 page += sprintf(page, "!!! "); 1085 atomic_inc(&n_rcu_torture_error); 1086 WARN_ON_ONCE(1); 1087 } 1088 page += sprintf(page, "Reader Pipe: "); 1089 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1090 page += sprintf(page, " %ld", pipesummary[i]); 1091 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 1092 page += sprintf(page, "Reader Batch: "); 1093 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1094 page += sprintf(page, " %ld", batchsummary[i]); 1095 page += sprintf(page, "\n%s%s ", torture_type, TORTURE_FLAG); 1096 page += sprintf(page, "Free-Block Circulation: "); 1097 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1098 page += sprintf(page, " %d", 1099 atomic_read(&rcu_torture_wcount[i])); 1100 } 1101 page += sprintf(page, "\n"); 1102 if (cur_ops->stats) 1103 cur_ops->stats(page); 1104 if (rtcv_snap == rcu_torture_current_version && 1105 rcu_torture_current != NULL) { 1106 int __maybe_unused flags; 1107 unsigned long __maybe_unused gpnum; 1108 unsigned long __maybe_unused completed; 1109 1110 rcutorture_get_gp_data(cur_ops->ttype, 1111 &flags, &gpnum, &completed); 1112 page += sprintf(page, 1113 "??? Writer stall state %d g%lu c%lu f%#x\n", 1114 rcu_torture_writer_state, 1115 gpnum, completed, flags); 1116 show_rcu_gp_kthreads(); 1117 rcutorture_trace_dump(); 1118 } 1119 rtcv_snap = rcu_torture_current_version; 1120 } 1121 1122 /* 1123 * Print torture statistics. Caller must ensure that there is only 1124 * one call to this function at a given time!!! This is normally 1125 * accomplished by relying on the module system to only have one copy 1126 * of the module loaded, and then by giving the rcu_torture_stats 1127 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1128 * thread is not running). 1129 */ 1130 static void 1131 rcu_torture_stats_print(void) 1132 { 1133 int size = nr_cpu_ids * 200 + 8192; 1134 char *buf; 1135 1136 buf = kmalloc(size, GFP_KERNEL); 1137 if (!buf) { 1138 pr_err("rcu-torture: Out of memory, need: %d", size); 1139 return; 1140 } 1141 rcu_torture_printk(buf); 1142 pr_alert("%s", buf); 1143 kfree(buf); 1144 } 1145 1146 /* 1147 * Periodically prints torture statistics, if periodic statistics printing 1148 * was specified via the stat_interval module parameter. 1149 */ 1150 static int 1151 rcu_torture_stats(void *arg) 1152 { 1153 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1154 do { 1155 schedule_timeout_interruptible(stat_interval * HZ); 1156 rcu_torture_stats_print(); 1157 torture_shutdown_absorb("rcu_torture_stats"); 1158 } while (!torture_must_stop()); 1159 torture_kthread_stopping("rcu_torture_stats"); 1160 return 0; 1161 } 1162 1163 static inline void 1164 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1165 { 1166 pr_alert("%s" TORTURE_FLAG 1167 "--- %s: nreaders=%d nfakewriters=%d " 1168 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1169 "shuffle_interval=%d stutter=%d irqreader=%d " 1170 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1171 "test_boost=%d/%d test_boost_interval=%d " 1172 "test_boost_duration=%d shutdown_secs=%d " 1173 "stall_cpu=%d stall_cpu_holdoff=%d " 1174 "n_barrier_cbs=%d " 1175 "onoff_interval=%d onoff_holdoff=%d\n", 1176 torture_type, tag, nrealreaders, nfakewriters, 1177 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1178 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1179 test_boost, cur_ops->can_boost, 1180 test_boost_interval, test_boost_duration, shutdown_secs, 1181 stall_cpu, stall_cpu_holdoff, 1182 n_barrier_cbs, 1183 onoff_interval, onoff_holdoff); 1184 } 1185 1186 static void rcutorture_booster_cleanup(int cpu) 1187 { 1188 struct task_struct *t; 1189 1190 if (boost_tasks[cpu] == NULL) 1191 return; 1192 mutex_lock(&boost_mutex); 1193 t = boost_tasks[cpu]; 1194 boost_tasks[cpu] = NULL; 1195 mutex_unlock(&boost_mutex); 1196 1197 /* This must be outside of the mutex, otherwise deadlock! */ 1198 torture_stop_kthread(rcu_torture_boost, t); 1199 } 1200 1201 static int rcutorture_booster_init(int cpu) 1202 { 1203 int retval; 1204 1205 if (boost_tasks[cpu] != NULL) 1206 return 0; /* Already created, nothing more to do. */ 1207 1208 /* Don't allow time recalculation while creating a new task. */ 1209 mutex_lock(&boost_mutex); 1210 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1211 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1212 cpu_to_node(cpu), 1213 "rcu_torture_boost"); 1214 if (IS_ERR(boost_tasks[cpu])) { 1215 retval = PTR_ERR(boost_tasks[cpu]); 1216 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1217 n_rcu_torture_boost_ktrerror++; 1218 boost_tasks[cpu] = NULL; 1219 mutex_unlock(&boost_mutex); 1220 return retval; 1221 } 1222 kthread_bind(boost_tasks[cpu], cpu); 1223 wake_up_process(boost_tasks[cpu]); 1224 mutex_unlock(&boost_mutex); 1225 return 0; 1226 } 1227 1228 /* 1229 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1230 * induces a CPU stall for the time specified by stall_cpu. 1231 */ 1232 static int rcu_torture_stall(void *args) 1233 { 1234 unsigned long stop_at; 1235 1236 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1237 if (stall_cpu_holdoff > 0) { 1238 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1239 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1240 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1241 } 1242 if (!kthread_should_stop()) { 1243 stop_at = get_seconds() + stall_cpu; 1244 /* RCU CPU stall is expected behavior in following code. */ 1245 pr_alert("rcu_torture_stall start.\n"); 1246 rcu_read_lock(); 1247 preempt_disable(); 1248 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1249 continue; /* Induce RCU CPU stall warning. */ 1250 preempt_enable(); 1251 rcu_read_unlock(); 1252 pr_alert("rcu_torture_stall end.\n"); 1253 } 1254 torture_shutdown_absorb("rcu_torture_stall"); 1255 while (!kthread_should_stop()) 1256 schedule_timeout_interruptible(10 * HZ); 1257 return 0; 1258 } 1259 1260 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1261 static int __init rcu_torture_stall_init(void) 1262 { 1263 if (stall_cpu <= 0) 1264 return 0; 1265 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1266 } 1267 1268 /* Callback function for RCU barrier testing. */ 1269 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1270 { 1271 atomic_inc(&barrier_cbs_invoked); 1272 } 1273 1274 /* kthread function to register callbacks used to test RCU barriers. */ 1275 static int rcu_torture_barrier_cbs(void *arg) 1276 { 1277 long myid = (long)arg; 1278 bool lastphase = 0; 1279 bool newphase; 1280 struct rcu_head rcu; 1281 1282 init_rcu_head_on_stack(&rcu); 1283 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 1284 set_user_nice(current, MAX_NICE); 1285 do { 1286 wait_event(barrier_cbs_wq[myid], 1287 (newphase = 1288 ACCESS_ONCE(barrier_phase)) != lastphase || 1289 torture_must_stop()); 1290 lastphase = newphase; 1291 smp_mb(); /* ensure barrier_phase load before ->call(). */ 1292 if (torture_must_stop()) 1293 break; 1294 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1295 if (atomic_dec_and_test(&barrier_cbs_count)) 1296 wake_up(&barrier_wq); 1297 } while (!torture_must_stop()); 1298 cur_ops->cb_barrier(); 1299 destroy_rcu_head_on_stack(&rcu); 1300 torture_kthread_stopping("rcu_torture_barrier_cbs"); 1301 return 0; 1302 } 1303 1304 /* kthread function to drive and coordinate RCU barrier testing. */ 1305 static int rcu_torture_barrier(void *arg) 1306 { 1307 int i; 1308 1309 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 1310 do { 1311 atomic_set(&barrier_cbs_invoked, 0); 1312 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1313 smp_mb(); /* Ensure barrier_phase after prior assignments. */ 1314 barrier_phase = !barrier_phase; 1315 for (i = 0; i < n_barrier_cbs; i++) 1316 wake_up(&barrier_cbs_wq[i]); 1317 wait_event(barrier_wq, 1318 atomic_read(&barrier_cbs_count) == 0 || 1319 torture_must_stop()); 1320 if (torture_must_stop()) 1321 break; 1322 n_barrier_attempts++; 1323 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 1324 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1325 n_rcu_torture_barrier_error++; 1326 WARN_ON_ONCE(1); 1327 } 1328 n_barrier_successes++; 1329 schedule_timeout_interruptible(HZ / 10); 1330 } while (!torture_must_stop()); 1331 torture_kthread_stopping("rcu_torture_barrier"); 1332 return 0; 1333 } 1334 1335 /* Initialize RCU barrier testing. */ 1336 static int rcu_torture_barrier_init(void) 1337 { 1338 int i; 1339 int ret; 1340 1341 if (n_barrier_cbs == 0) 1342 return 0; 1343 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1344 pr_alert("%s" TORTURE_FLAG 1345 " Call or barrier ops missing for %s,\n", 1346 torture_type, cur_ops->name); 1347 pr_alert("%s" TORTURE_FLAG 1348 " RCU barrier testing omitted from run.\n", 1349 torture_type); 1350 return 0; 1351 } 1352 atomic_set(&barrier_cbs_count, 0); 1353 atomic_set(&barrier_cbs_invoked, 0); 1354 barrier_cbs_tasks = 1355 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1356 GFP_KERNEL); 1357 barrier_cbs_wq = 1358 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1359 GFP_KERNEL); 1360 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 1361 return -ENOMEM; 1362 for (i = 0; i < n_barrier_cbs; i++) { 1363 init_waitqueue_head(&barrier_cbs_wq[i]); 1364 ret = torture_create_kthread(rcu_torture_barrier_cbs, 1365 (void *)(long)i, 1366 barrier_cbs_tasks[i]); 1367 if (ret) 1368 return ret; 1369 } 1370 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 1371 } 1372 1373 /* Clean up after RCU barrier testing. */ 1374 static void rcu_torture_barrier_cleanup(void) 1375 { 1376 int i; 1377 1378 torture_stop_kthread(rcu_torture_barrier, barrier_task); 1379 if (barrier_cbs_tasks != NULL) { 1380 for (i = 0; i < n_barrier_cbs; i++) 1381 torture_stop_kthread(rcu_torture_barrier_cbs, 1382 barrier_cbs_tasks[i]); 1383 kfree(barrier_cbs_tasks); 1384 barrier_cbs_tasks = NULL; 1385 } 1386 if (barrier_cbs_wq != NULL) { 1387 kfree(barrier_cbs_wq); 1388 barrier_cbs_wq = NULL; 1389 } 1390 } 1391 1392 static int rcutorture_cpu_notify(struct notifier_block *self, 1393 unsigned long action, void *hcpu) 1394 { 1395 long cpu = (long)hcpu; 1396 1397 switch (action) { 1398 case CPU_ONLINE: 1399 case CPU_DOWN_FAILED: 1400 (void)rcutorture_booster_init(cpu); 1401 break; 1402 case CPU_DOWN_PREPARE: 1403 rcutorture_booster_cleanup(cpu); 1404 break; 1405 default: 1406 break; 1407 } 1408 return NOTIFY_OK; 1409 } 1410 1411 static struct notifier_block rcutorture_cpu_nb = { 1412 .notifier_call = rcutorture_cpu_notify, 1413 }; 1414 1415 static void 1416 rcu_torture_cleanup(void) 1417 { 1418 int i; 1419 1420 rcutorture_record_test_transition(); 1421 if (torture_cleanup()) { 1422 if (cur_ops->cb_barrier != NULL) 1423 cur_ops->cb_barrier(); 1424 return; 1425 } 1426 1427 rcu_torture_barrier_cleanup(); 1428 torture_stop_kthread(rcu_torture_stall, stall_task); 1429 torture_stop_kthread(rcu_torture_writer, writer_task); 1430 1431 if (reader_tasks) { 1432 for (i = 0; i < nrealreaders; i++) 1433 torture_stop_kthread(rcu_torture_reader, 1434 reader_tasks[i]); 1435 kfree(reader_tasks); 1436 } 1437 rcu_torture_current = NULL; 1438 1439 if (fakewriter_tasks) { 1440 for (i = 0; i < nfakewriters; i++) { 1441 torture_stop_kthread(rcu_torture_fakewriter, 1442 fakewriter_tasks[i]); 1443 } 1444 kfree(fakewriter_tasks); 1445 fakewriter_tasks = NULL; 1446 } 1447 1448 torture_stop_kthread(rcu_torture_stats, stats_task); 1449 torture_stop_kthread(rcu_torture_fqs, fqs_task); 1450 if ((test_boost == 1 && cur_ops->can_boost) || 1451 test_boost == 2) { 1452 unregister_cpu_notifier(&rcutorture_cpu_nb); 1453 for_each_possible_cpu(i) 1454 rcutorture_booster_cleanup(i); 1455 } 1456 1457 /* Wait for all RCU callbacks to fire. */ 1458 1459 if (cur_ops->cb_barrier != NULL) 1460 cur_ops->cb_barrier(); 1461 1462 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1463 1464 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1465 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1466 else if (torture_onoff_failures()) 1467 rcu_torture_print_module_parms(cur_ops, 1468 "End of test: RCU_HOTPLUG"); 1469 else 1470 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1471 } 1472 1473 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1474 static void rcu_torture_leak_cb(struct rcu_head *rhp) 1475 { 1476 } 1477 1478 static void rcu_torture_err_cb(struct rcu_head *rhp) 1479 { 1480 /* 1481 * This -might- happen due to race conditions, but is unlikely. 1482 * The scenario that leads to this happening is that the 1483 * first of the pair of duplicate callbacks is queued, 1484 * someone else starts a grace period that includes that 1485 * callback, then the second of the pair must wait for the 1486 * next grace period. Unlikely, but can happen. If it 1487 * does happen, the debug-objects subsystem won't have splatted. 1488 */ 1489 pr_alert("rcutorture: duplicated callback was invoked.\n"); 1490 } 1491 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1492 1493 /* 1494 * Verify that double-free causes debug-objects to complain, but only 1495 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 1496 * cannot be carried out. 1497 */ 1498 static void rcu_test_debug_objects(void) 1499 { 1500 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 1501 struct rcu_head rh1; 1502 struct rcu_head rh2; 1503 1504 init_rcu_head_on_stack(&rh1); 1505 init_rcu_head_on_stack(&rh2); 1506 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n"); 1507 1508 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 1509 preempt_disable(); /* Prevent preemption from interrupting test. */ 1510 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 1511 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 1512 local_irq_disable(); /* Make it harder to start a new grace period. */ 1513 call_rcu(&rh2, rcu_torture_leak_cb); 1514 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 1515 local_irq_enable(); 1516 rcu_read_unlock(); 1517 preempt_enable(); 1518 1519 /* Wait for them all to get done so we can safely return. */ 1520 rcu_barrier(); 1521 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n"); 1522 destroy_rcu_head_on_stack(&rh1); 1523 destroy_rcu_head_on_stack(&rh2); 1524 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1525 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n"); 1526 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 1527 } 1528 1529 static int __init 1530 rcu_torture_init(void) 1531 { 1532 int i; 1533 int cpu; 1534 int firsterr = 0; 1535 static struct rcu_torture_ops *torture_ops[] = { 1536 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &sched_ops, 1537 }; 1538 1539 if (!torture_init_begin(torture_type, verbose, &rcutorture_runnable)) 1540 return -EBUSY; 1541 1542 /* Process args and tell the world that the torturer is on the job. */ 1543 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1544 cur_ops = torture_ops[i]; 1545 if (strcmp(torture_type, cur_ops->name) == 0) 1546 break; 1547 } 1548 if (i == ARRAY_SIZE(torture_ops)) { 1549 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1550 torture_type); 1551 pr_alert("rcu-torture types:"); 1552 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1553 pr_alert(" %s", torture_ops[i]->name); 1554 pr_alert("\n"); 1555 torture_init_end(); 1556 return -EINVAL; 1557 } 1558 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1559 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1560 fqs_duration = 0; 1561 } 1562 if (cur_ops->init) 1563 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1564 1565 if (nreaders >= 0) { 1566 nrealreaders = nreaders; 1567 } else { 1568 nrealreaders = num_online_cpus() - 1; 1569 if (nrealreaders <= 0) 1570 nrealreaders = 1; 1571 } 1572 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1573 1574 /* Set up the freelist. */ 1575 1576 INIT_LIST_HEAD(&rcu_torture_freelist); 1577 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1578 rcu_tortures[i].rtort_mbtest = 0; 1579 list_add_tail(&rcu_tortures[i].rtort_free, 1580 &rcu_torture_freelist); 1581 } 1582 1583 /* Initialize the statistics so that each run gets its own numbers. */ 1584 1585 rcu_torture_current = NULL; 1586 rcu_torture_current_version = 0; 1587 atomic_set(&n_rcu_torture_alloc, 0); 1588 atomic_set(&n_rcu_torture_alloc_fail, 0); 1589 atomic_set(&n_rcu_torture_free, 0); 1590 atomic_set(&n_rcu_torture_mberror, 0); 1591 atomic_set(&n_rcu_torture_error, 0); 1592 n_rcu_torture_barrier_error = 0; 1593 n_rcu_torture_boost_ktrerror = 0; 1594 n_rcu_torture_boost_rterror = 0; 1595 n_rcu_torture_boost_failure = 0; 1596 n_rcu_torture_boosts = 0; 1597 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1598 atomic_set(&rcu_torture_wcount[i], 0); 1599 for_each_possible_cpu(cpu) { 1600 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1601 per_cpu(rcu_torture_count, cpu)[i] = 0; 1602 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1603 } 1604 } 1605 1606 /* Start up the kthreads. */ 1607 1608 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 1609 writer_task); 1610 if (firsterr) 1611 goto unwind; 1612 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 1613 GFP_KERNEL); 1614 if (fakewriter_tasks == NULL) { 1615 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1616 firsterr = -ENOMEM; 1617 goto unwind; 1618 } 1619 for (i = 0; i < nfakewriters; i++) { 1620 firsterr = torture_create_kthread(rcu_torture_fakewriter, 1621 NULL, fakewriter_tasks[i]); 1622 if (firsterr) 1623 goto unwind; 1624 } 1625 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1626 GFP_KERNEL); 1627 if (reader_tasks == NULL) { 1628 VERBOSE_TOROUT_ERRSTRING("out of memory"); 1629 firsterr = -ENOMEM; 1630 goto unwind; 1631 } 1632 for (i = 0; i < nrealreaders; i++) { 1633 firsterr = torture_create_kthread(rcu_torture_reader, NULL, 1634 reader_tasks[i]); 1635 if (firsterr) 1636 goto unwind; 1637 } 1638 if (stat_interval > 0) { 1639 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 1640 stats_task); 1641 if (firsterr) 1642 goto unwind; 1643 } 1644 if (test_no_idle_hz) { 1645 firsterr = torture_shuffle_init(shuffle_interval * HZ); 1646 if (firsterr) 1647 goto unwind; 1648 } 1649 if (stutter < 0) 1650 stutter = 0; 1651 if (stutter) { 1652 firsterr = torture_stutter_init(stutter * HZ); 1653 if (firsterr) 1654 goto unwind; 1655 } 1656 if (fqs_duration < 0) 1657 fqs_duration = 0; 1658 if (fqs_duration) { 1659 /* Create the fqs thread */ 1660 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 1661 fqs_task); 1662 if (firsterr) 1663 goto unwind; 1664 } 1665 if (test_boost_interval < 1) 1666 test_boost_interval = 1; 1667 if (test_boost_duration < 2) 1668 test_boost_duration = 2; 1669 if ((test_boost == 1 && cur_ops->can_boost) || 1670 test_boost == 2) { 1671 1672 boost_starttime = jiffies + test_boost_interval * HZ; 1673 register_cpu_notifier(&rcutorture_cpu_nb); 1674 for_each_possible_cpu(i) { 1675 if (cpu_is_offline(i)) 1676 continue; /* Heuristic: CPU can go offline. */ 1677 firsterr = rcutorture_booster_init(i); 1678 if (firsterr) 1679 goto unwind; 1680 } 1681 } 1682 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 1683 if (firsterr) 1684 goto unwind; 1685 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ); 1686 if (firsterr) 1687 goto unwind; 1688 firsterr = rcu_torture_stall_init(); 1689 if (firsterr) 1690 goto unwind; 1691 firsterr = rcu_torture_barrier_init(); 1692 if (firsterr) 1693 goto unwind; 1694 if (object_debug) 1695 rcu_test_debug_objects(); 1696 rcutorture_record_test_transition(); 1697 torture_init_end(); 1698 return 0; 1699 1700 unwind: 1701 torture_init_end(); 1702 rcu_torture_cleanup(); 1703 return firsterr; 1704 } 1705 1706 module_init(rcu_torture_init); 1707 module_exit(rcu_torture_cleanup); 1708