1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.txt 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate.h> 24 #include <linux/interrupt.h> 25 #include <linux/sched/signal.h> 26 #include <uapi/linux/sched/types.h> 27 #include <linux/atomic.h> 28 #include <linux/bitops.h> 29 #include <linux/completion.h> 30 #include <linux/moduleparam.h> 31 #include <linux/percpu.h> 32 #include <linux/notifier.h> 33 #include <linux/reboot.h> 34 #include <linux/freezer.h> 35 #include <linux/cpu.h> 36 #include <linux/delay.h> 37 #include <linux/stat.h> 38 #include <linux/srcu.h> 39 #include <linux/slab.h> 40 #include <linux/trace_clock.h> 41 #include <asm/byteorder.h> 42 #include <linux/torture.h> 43 #include <linux/vmalloc.h> 44 #include <linux/sched/debug.h> 45 #include <linux/sched/sysctl.h> 46 #include <linux/oom.h> 47 #include <linux/tick.h> 48 49 #include "rcu.h" 50 51 MODULE_LICENSE("GPL"); 52 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 53 54 55 /* Bits for ->extendables field, extendables param, and related definitions. */ 56 #define RCUTORTURE_RDR_SHIFT 8 /* Put SRCU index in upper bits. */ 57 #define RCUTORTURE_RDR_MASK ((1 << RCUTORTURE_RDR_SHIFT) - 1) 58 #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ 59 #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ 60 #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ 61 #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ 62 #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ 63 #define RCUTORTURE_RDR_RCU 0x20 /* ... entering another RCU reader. */ 64 #define RCUTORTURE_RDR_NBITS 6 /* Number of bits defined above. */ 65 #define RCUTORTURE_MAX_EXTEND \ 66 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 67 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) 68 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 69 /* Must be power of two minus one. */ 70 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 71 72 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 73 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 74 torture_param(int, fqs_duration, 0, 75 "Duration of fqs bursts (us), 0 to disable"); 76 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 77 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 78 torture_param(bool, fwd_progress, 1, "Test grace-period forward progress"); 79 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 80 torture_param(int, fwd_progress_holdoff, 60, 81 "Time between forward-progress tests (s)"); 82 torture_param(bool, fwd_progress_need_resched, 1, 83 "Hide cond_resched() behind need_resched()"); 84 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 85 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 86 torture_param(bool, gp_normal, false, 87 "Use normal (non-expedited) GP wait primitives"); 88 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 89 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 90 torture_param(int, n_barrier_cbs, 0, 91 "# of callbacks/kthreads for barrier testing"); 92 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 93 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 94 torture_param(int, object_debug, 0, 95 "Enable debug-object double call_rcu() testing"); 96 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 97 torture_param(int, onoff_interval, 0, 98 "Time between CPU hotplugs (jiffies), 0=disable"); 99 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 100 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 101 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 102 torture_param(int, stall_cpu_holdoff, 10, 103 "Time to wait before starting stall (s)."); 104 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 105 torture_param(int, stat_interval, 60, 106 "Number of seconds between stats printk()s"); 107 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 108 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 109 torture_param(int, test_boost_duration, 4, 110 "Duration of each boost test, seconds."); 111 torture_param(int, test_boost_interval, 7, 112 "Interval between boost tests, seconds."); 113 torture_param(bool, test_no_idle_hz, true, 114 "Test support for tickless idle CPUs"); 115 torture_param(int, verbose, 1, 116 "Enable verbose debugging printk()s"); 117 118 static char *torture_type = "rcu"; 119 module_param(torture_type, charp, 0444); 120 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 121 122 static int nrealreaders; 123 static struct task_struct *writer_task; 124 static struct task_struct **fakewriter_tasks; 125 static struct task_struct **reader_tasks; 126 static struct task_struct *stats_task; 127 static struct task_struct *fqs_task; 128 static struct task_struct *boost_tasks[NR_CPUS]; 129 static struct task_struct *stall_task; 130 static struct task_struct *fwd_prog_task; 131 static struct task_struct **barrier_cbs_tasks; 132 static struct task_struct *barrier_task; 133 134 #define RCU_TORTURE_PIPE_LEN 10 135 136 struct rcu_torture { 137 struct rcu_head rtort_rcu; 138 int rtort_pipe_count; 139 struct list_head rtort_free; 140 int rtort_mbtest; 141 }; 142 143 static LIST_HEAD(rcu_torture_freelist); 144 static struct rcu_torture __rcu *rcu_torture_current; 145 static unsigned long rcu_torture_current_version; 146 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 147 static DEFINE_SPINLOCK(rcu_torture_lock); 148 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 149 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 150 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 151 static atomic_t n_rcu_torture_alloc; 152 static atomic_t n_rcu_torture_alloc_fail; 153 static atomic_t n_rcu_torture_free; 154 static atomic_t n_rcu_torture_mberror; 155 static atomic_t n_rcu_torture_error; 156 static long n_rcu_torture_barrier_error; 157 static long n_rcu_torture_boost_ktrerror; 158 static long n_rcu_torture_boost_rterror; 159 static long n_rcu_torture_boost_failure; 160 static long n_rcu_torture_boosts; 161 static atomic_long_t n_rcu_torture_timers; 162 static long n_barrier_attempts; 163 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 164 static struct list_head rcu_torture_removed; 165 static unsigned long shutdown_jiffies; 166 167 static int rcu_torture_writer_state; 168 #define RTWS_FIXED_DELAY 0 169 #define RTWS_DELAY 1 170 #define RTWS_REPLACE 2 171 #define RTWS_DEF_FREE 3 172 #define RTWS_EXP_SYNC 4 173 #define RTWS_COND_GET 5 174 #define RTWS_COND_SYNC 6 175 #define RTWS_SYNC 7 176 #define RTWS_STUTTER 8 177 #define RTWS_STOPPING 9 178 static const char * const rcu_torture_writer_state_names[] = { 179 "RTWS_FIXED_DELAY", 180 "RTWS_DELAY", 181 "RTWS_REPLACE", 182 "RTWS_DEF_FREE", 183 "RTWS_EXP_SYNC", 184 "RTWS_COND_GET", 185 "RTWS_COND_SYNC", 186 "RTWS_SYNC", 187 "RTWS_STUTTER", 188 "RTWS_STOPPING", 189 }; 190 191 /* Record reader segment types and duration for first failing read. */ 192 struct rt_read_seg { 193 int rt_readstate; 194 unsigned long rt_delay_jiffies; 195 unsigned long rt_delay_ms; 196 unsigned long rt_delay_us; 197 bool rt_preempted; 198 }; 199 static int err_segs_recorded; 200 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 201 static int rt_read_nsegs; 202 203 static const char *rcu_torture_writer_state_getname(void) 204 { 205 unsigned int i = READ_ONCE(rcu_torture_writer_state); 206 207 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 208 return "???"; 209 return rcu_torture_writer_state_names[i]; 210 } 211 212 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 213 #define rcu_can_boost() 1 214 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 215 #define rcu_can_boost() 0 216 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 217 218 #ifdef CONFIG_RCU_TRACE 219 static u64 notrace rcu_trace_clock_local(void) 220 { 221 u64 ts = trace_clock_local(); 222 223 (void)do_div(ts, NSEC_PER_USEC); 224 return ts; 225 } 226 #else /* #ifdef CONFIG_RCU_TRACE */ 227 static u64 notrace rcu_trace_clock_local(void) 228 { 229 return 0ULL; 230 } 231 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 232 233 /* 234 * Stop aggressive CPU-hog tests a bit before the end of the test in order 235 * to avoid interfering with test shutdown. 236 */ 237 static bool shutdown_time_arrived(void) 238 { 239 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 240 } 241 242 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 243 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 244 /* and boost task create/destroy. */ 245 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 246 static bool barrier_phase; /* Test phase. */ 247 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 248 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 249 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 250 251 static bool rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 252 253 /* 254 * Allocate an element from the rcu_tortures pool. 255 */ 256 static struct rcu_torture * 257 rcu_torture_alloc(void) 258 { 259 struct list_head *p; 260 261 spin_lock_bh(&rcu_torture_lock); 262 if (list_empty(&rcu_torture_freelist)) { 263 atomic_inc(&n_rcu_torture_alloc_fail); 264 spin_unlock_bh(&rcu_torture_lock); 265 return NULL; 266 } 267 atomic_inc(&n_rcu_torture_alloc); 268 p = rcu_torture_freelist.next; 269 list_del_init(p); 270 spin_unlock_bh(&rcu_torture_lock); 271 return container_of(p, struct rcu_torture, rtort_free); 272 } 273 274 /* 275 * Free an element to the rcu_tortures pool. 276 */ 277 static void 278 rcu_torture_free(struct rcu_torture *p) 279 { 280 atomic_inc(&n_rcu_torture_free); 281 spin_lock_bh(&rcu_torture_lock); 282 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 283 spin_unlock_bh(&rcu_torture_lock); 284 } 285 286 /* 287 * Operations vector for selecting different types of tests. 288 */ 289 290 struct rcu_torture_ops { 291 int ttype; 292 void (*init)(void); 293 void (*cleanup)(void); 294 int (*readlock)(void); 295 void (*read_delay)(struct torture_random_state *rrsp, 296 struct rt_read_seg *rtrsp); 297 void (*readunlock)(int idx); 298 unsigned long (*get_gp_seq)(void); 299 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 300 void (*deferred_free)(struct rcu_torture *p); 301 void (*sync)(void); 302 void (*exp_sync)(void); 303 unsigned long (*get_state)(void); 304 void (*cond_sync)(unsigned long oldstate); 305 call_rcu_func_t call; 306 void (*cb_barrier)(void); 307 void (*fqs)(void); 308 void (*stats)(void); 309 int (*stall_dur)(void); 310 int irq_capable; 311 int can_boost; 312 int extendables; 313 int slow_gps; 314 const char *name; 315 }; 316 317 static struct rcu_torture_ops *cur_ops; 318 319 /* 320 * Definitions for rcu torture testing. 321 */ 322 323 static int rcu_torture_read_lock(void) __acquires(RCU) 324 { 325 rcu_read_lock(); 326 return 0; 327 } 328 329 static void 330 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 331 { 332 unsigned long started; 333 unsigned long completed; 334 const unsigned long shortdelay_us = 200; 335 unsigned long longdelay_ms = 300; 336 unsigned long long ts; 337 338 /* We want a short delay sometimes to make a reader delay the grace 339 * period, and we want a long delay occasionally to trigger 340 * force_quiescent_state. */ 341 342 if (!READ_ONCE(rcu_fwd_cb_nodelay) && 343 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 344 started = cur_ops->get_gp_seq(); 345 ts = rcu_trace_clock_local(); 346 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) 347 longdelay_ms = 5; /* Avoid triggering BH limits. */ 348 mdelay(longdelay_ms); 349 rtrsp->rt_delay_ms = longdelay_ms; 350 completed = cur_ops->get_gp_seq(); 351 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 352 started, completed); 353 } 354 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 355 udelay(shortdelay_us); 356 rtrsp->rt_delay_us = shortdelay_us; 357 } 358 if (!preempt_count() && 359 !(torture_random(rrsp) % (nrealreaders * 500))) { 360 torture_preempt_schedule(); /* QS only if preemptible. */ 361 rtrsp->rt_preempted = true; 362 } 363 } 364 365 static void rcu_torture_read_unlock(int idx) __releases(RCU) 366 { 367 rcu_read_unlock(); 368 } 369 370 /* 371 * Update callback in the pipe. This should be invoked after a grace period. 372 */ 373 static bool 374 rcu_torture_pipe_update_one(struct rcu_torture *rp) 375 { 376 int i; 377 378 i = READ_ONCE(rp->rtort_pipe_count); 379 if (i > RCU_TORTURE_PIPE_LEN) 380 i = RCU_TORTURE_PIPE_LEN; 381 atomic_inc(&rcu_torture_wcount[i]); 382 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 383 if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 384 rp->rtort_mbtest = 0; 385 return true; 386 } 387 return false; 388 } 389 390 /* 391 * Update all callbacks in the pipe. Suitable for synchronous grace-period 392 * primitives. 393 */ 394 static void 395 rcu_torture_pipe_update(struct rcu_torture *old_rp) 396 { 397 struct rcu_torture *rp; 398 struct rcu_torture *rp1; 399 400 if (old_rp) 401 list_add(&old_rp->rtort_free, &rcu_torture_removed); 402 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 403 if (rcu_torture_pipe_update_one(rp)) { 404 list_del(&rp->rtort_free); 405 rcu_torture_free(rp); 406 } 407 } 408 } 409 410 static void 411 rcu_torture_cb(struct rcu_head *p) 412 { 413 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 414 415 if (torture_must_stop_irq()) { 416 /* Test is ending, just drop callbacks on the floor. */ 417 /* The next initialization will pick up the pieces. */ 418 return; 419 } 420 if (rcu_torture_pipe_update_one(rp)) 421 rcu_torture_free(rp); 422 else 423 cur_ops->deferred_free(rp); 424 } 425 426 static unsigned long rcu_no_completed(void) 427 { 428 return 0; 429 } 430 431 static void rcu_torture_deferred_free(struct rcu_torture *p) 432 { 433 call_rcu(&p->rtort_rcu, rcu_torture_cb); 434 } 435 436 static void rcu_sync_torture_init(void) 437 { 438 INIT_LIST_HEAD(&rcu_torture_removed); 439 } 440 441 static struct rcu_torture_ops rcu_ops = { 442 .ttype = RCU_FLAVOR, 443 .init = rcu_sync_torture_init, 444 .readlock = rcu_torture_read_lock, 445 .read_delay = rcu_read_delay, 446 .readunlock = rcu_torture_read_unlock, 447 .get_gp_seq = rcu_get_gp_seq, 448 .gp_diff = rcu_seq_diff, 449 .deferred_free = rcu_torture_deferred_free, 450 .sync = synchronize_rcu, 451 .exp_sync = synchronize_rcu_expedited, 452 .get_state = get_state_synchronize_rcu, 453 .cond_sync = cond_synchronize_rcu, 454 .call = call_rcu, 455 .cb_barrier = rcu_barrier, 456 .fqs = rcu_force_quiescent_state, 457 .stats = NULL, 458 .stall_dur = rcu_jiffies_till_stall_check, 459 .irq_capable = 1, 460 .can_boost = rcu_can_boost(), 461 .extendables = RCUTORTURE_MAX_EXTEND, 462 .name = "rcu" 463 }; 464 465 /* 466 * Don't even think about trying any of these in real life!!! 467 * The names includes "busted", and they really means it! 468 * The only purpose of these functions is to provide a buggy RCU 469 * implementation to make sure that rcutorture correctly emits 470 * buggy-RCU error messages. 471 */ 472 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 473 { 474 /* This is a deliberate bug for testing purposes only! */ 475 rcu_torture_cb(&p->rtort_rcu); 476 } 477 478 static void synchronize_rcu_busted(void) 479 { 480 /* This is a deliberate bug for testing purposes only! */ 481 } 482 483 static void 484 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 485 { 486 /* This is a deliberate bug for testing purposes only! */ 487 func(head); 488 } 489 490 static struct rcu_torture_ops rcu_busted_ops = { 491 .ttype = INVALID_RCU_FLAVOR, 492 .init = rcu_sync_torture_init, 493 .readlock = rcu_torture_read_lock, 494 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 495 .readunlock = rcu_torture_read_unlock, 496 .get_gp_seq = rcu_no_completed, 497 .deferred_free = rcu_busted_torture_deferred_free, 498 .sync = synchronize_rcu_busted, 499 .exp_sync = synchronize_rcu_busted, 500 .call = call_rcu_busted, 501 .cb_barrier = NULL, 502 .fqs = NULL, 503 .stats = NULL, 504 .irq_capable = 1, 505 .name = "busted" 506 }; 507 508 /* 509 * Definitions for srcu torture testing. 510 */ 511 512 DEFINE_STATIC_SRCU(srcu_ctl); 513 static struct srcu_struct srcu_ctld; 514 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 515 516 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) 517 { 518 return srcu_read_lock(srcu_ctlp); 519 } 520 521 static void 522 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 523 { 524 long delay; 525 const long uspertick = 1000000 / HZ; 526 const long longdelay = 10; 527 528 /* We want there to be long-running readers, but not all the time. */ 529 530 delay = torture_random(rrsp) % 531 (nrealreaders * 2 * longdelay * uspertick); 532 if (!delay && in_task()) { 533 schedule_timeout_interruptible(longdelay); 534 rtrsp->rt_delay_jiffies = longdelay; 535 } else { 536 rcu_read_delay(rrsp, rtrsp); 537 } 538 } 539 540 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) 541 { 542 srcu_read_unlock(srcu_ctlp, idx); 543 } 544 545 static unsigned long srcu_torture_completed(void) 546 { 547 return srcu_batches_completed(srcu_ctlp); 548 } 549 550 static void srcu_torture_deferred_free(struct rcu_torture *rp) 551 { 552 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 553 } 554 555 static void srcu_torture_synchronize(void) 556 { 557 synchronize_srcu(srcu_ctlp); 558 } 559 560 static void srcu_torture_call(struct rcu_head *head, 561 rcu_callback_t func) 562 { 563 call_srcu(srcu_ctlp, head, func); 564 } 565 566 static void srcu_torture_barrier(void) 567 { 568 srcu_barrier(srcu_ctlp); 569 } 570 571 static void srcu_torture_stats(void) 572 { 573 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 574 } 575 576 static void srcu_torture_synchronize_expedited(void) 577 { 578 synchronize_srcu_expedited(srcu_ctlp); 579 } 580 581 static struct rcu_torture_ops srcu_ops = { 582 .ttype = SRCU_FLAVOR, 583 .init = rcu_sync_torture_init, 584 .readlock = srcu_torture_read_lock, 585 .read_delay = srcu_read_delay, 586 .readunlock = srcu_torture_read_unlock, 587 .get_gp_seq = srcu_torture_completed, 588 .deferred_free = srcu_torture_deferred_free, 589 .sync = srcu_torture_synchronize, 590 .exp_sync = srcu_torture_synchronize_expedited, 591 .call = srcu_torture_call, 592 .cb_barrier = srcu_torture_barrier, 593 .stats = srcu_torture_stats, 594 .irq_capable = 1, 595 .name = "srcu" 596 }; 597 598 static void srcu_torture_init(void) 599 { 600 rcu_sync_torture_init(); 601 WARN_ON(init_srcu_struct(&srcu_ctld)); 602 srcu_ctlp = &srcu_ctld; 603 } 604 605 static void srcu_torture_cleanup(void) 606 { 607 cleanup_srcu_struct(&srcu_ctld); 608 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 609 } 610 611 /* As above, but dynamically allocated. */ 612 static struct rcu_torture_ops srcud_ops = { 613 .ttype = SRCU_FLAVOR, 614 .init = srcu_torture_init, 615 .cleanup = srcu_torture_cleanup, 616 .readlock = srcu_torture_read_lock, 617 .read_delay = srcu_read_delay, 618 .readunlock = srcu_torture_read_unlock, 619 .get_gp_seq = srcu_torture_completed, 620 .deferred_free = srcu_torture_deferred_free, 621 .sync = srcu_torture_synchronize, 622 .exp_sync = srcu_torture_synchronize_expedited, 623 .call = srcu_torture_call, 624 .cb_barrier = srcu_torture_barrier, 625 .stats = srcu_torture_stats, 626 .irq_capable = 1, 627 .name = "srcud" 628 }; 629 630 /* As above, but broken due to inappropriate reader extension. */ 631 static struct rcu_torture_ops busted_srcud_ops = { 632 .ttype = SRCU_FLAVOR, 633 .init = srcu_torture_init, 634 .cleanup = srcu_torture_cleanup, 635 .readlock = srcu_torture_read_lock, 636 .read_delay = rcu_read_delay, 637 .readunlock = srcu_torture_read_unlock, 638 .get_gp_seq = srcu_torture_completed, 639 .deferred_free = srcu_torture_deferred_free, 640 .sync = srcu_torture_synchronize, 641 .exp_sync = srcu_torture_synchronize_expedited, 642 .call = srcu_torture_call, 643 .cb_barrier = srcu_torture_barrier, 644 .stats = srcu_torture_stats, 645 .irq_capable = 1, 646 .extendables = RCUTORTURE_MAX_EXTEND, 647 .name = "busted_srcud" 648 }; 649 650 /* 651 * Definitions for RCU-tasks torture testing. 652 */ 653 654 static int tasks_torture_read_lock(void) 655 { 656 return 0; 657 } 658 659 static void tasks_torture_read_unlock(int idx) 660 { 661 } 662 663 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 664 { 665 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 666 } 667 668 static struct rcu_torture_ops tasks_ops = { 669 .ttype = RCU_TASKS_FLAVOR, 670 .init = rcu_sync_torture_init, 671 .readlock = tasks_torture_read_lock, 672 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 673 .readunlock = tasks_torture_read_unlock, 674 .get_gp_seq = rcu_no_completed, 675 .deferred_free = rcu_tasks_torture_deferred_free, 676 .sync = synchronize_rcu_tasks, 677 .exp_sync = synchronize_rcu_tasks, 678 .call = call_rcu_tasks, 679 .cb_barrier = rcu_barrier_tasks, 680 .fqs = NULL, 681 .stats = NULL, 682 .irq_capable = 1, 683 .slow_gps = 1, 684 .name = "tasks" 685 }; 686 687 /* 688 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 689 * This implementation does not necessarily work well with CPU hotplug. 690 */ 691 692 static void synchronize_rcu_trivial(void) 693 { 694 int cpu; 695 696 for_each_online_cpu(cpu) { 697 rcutorture_sched_setaffinity(current->pid, cpumask_of(cpu)); 698 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 699 } 700 } 701 702 static int rcu_torture_read_lock_trivial(void) __acquires(RCU) 703 { 704 preempt_disable(); 705 return 0; 706 } 707 708 static void rcu_torture_read_unlock_trivial(int idx) __releases(RCU) 709 { 710 preempt_enable(); 711 } 712 713 static struct rcu_torture_ops trivial_ops = { 714 .ttype = RCU_TRIVIAL_FLAVOR, 715 .init = rcu_sync_torture_init, 716 .readlock = rcu_torture_read_lock_trivial, 717 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 718 .readunlock = rcu_torture_read_unlock_trivial, 719 .get_gp_seq = rcu_no_completed, 720 .sync = synchronize_rcu_trivial, 721 .exp_sync = synchronize_rcu_trivial, 722 .fqs = NULL, 723 .stats = NULL, 724 .irq_capable = 1, 725 .name = "trivial" 726 }; 727 728 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 729 { 730 if (!cur_ops->gp_diff) 731 return new - old; 732 return cur_ops->gp_diff(new, old); 733 } 734 735 static bool __maybe_unused torturing_tasks(void) 736 { 737 return cur_ops == &tasks_ops; 738 } 739 740 /* 741 * RCU torture priority-boost testing. Runs one real-time thread per 742 * CPU for moderate bursts, repeatedly registering RCU callbacks and 743 * spinning waiting for them to be invoked. If a given callback takes 744 * too long to be invoked, we assume that priority inversion has occurred. 745 */ 746 747 struct rcu_boost_inflight { 748 struct rcu_head rcu; 749 int inflight; 750 }; 751 752 static void rcu_torture_boost_cb(struct rcu_head *head) 753 { 754 struct rcu_boost_inflight *rbip = 755 container_of(head, struct rcu_boost_inflight, rcu); 756 757 /* Ensure RCU-core accesses precede clearing ->inflight */ 758 smp_store_release(&rbip->inflight, 0); 759 } 760 761 static int old_rt_runtime = -1; 762 763 static void rcu_torture_disable_rt_throttle(void) 764 { 765 /* 766 * Disable RT throttling so that rcutorture's boost threads don't get 767 * throttled. Only possible if rcutorture is built-in otherwise the 768 * user should manually do this by setting the sched_rt_period_us and 769 * sched_rt_runtime sysctls. 770 */ 771 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 772 return; 773 774 old_rt_runtime = sysctl_sched_rt_runtime; 775 sysctl_sched_rt_runtime = -1; 776 } 777 778 static void rcu_torture_enable_rt_throttle(void) 779 { 780 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 781 return; 782 783 sysctl_sched_rt_runtime = old_rt_runtime; 784 old_rt_runtime = -1; 785 } 786 787 static bool rcu_torture_boost_failed(unsigned long start, unsigned long end) 788 { 789 if (end - start > test_boost_duration * HZ - HZ / 2) { 790 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 791 n_rcu_torture_boost_failure++; 792 793 return true; /* failed */ 794 } 795 796 return false; /* passed */ 797 } 798 799 static int rcu_torture_boost(void *arg) 800 { 801 unsigned long call_rcu_time; 802 unsigned long endtime; 803 unsigned long oldstarttime; 804 struct rcu_boost_inflight rbi = { .inflight = 0 }; 805 struct sched_param sp; 806 807 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 808 809 /* Set real-time priority. */ 810 sp.sched_priority = 1; 811 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 812 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!"); 813 n_rcu_torture_boost_rterror++; 814 } 815 816 init_rcu_head_on_stack(&rbi.rcu); 817 /* Each pass through the following loop does one boost-test cycle. */ 818 do { 819 /* Track if the test failed already in this test interval? */ 820 bool failed = false; 821 822 /* Increment n_rcu_torture_boosts once per boost-test */ 823 while (!kthread_should_stop()) { 824 if (mutex_trylock(&boost_mutex)) { 825 n_rcu_torture_boosts++; 826 mutex_unlock(&boost_mutex); 827 break; 828 } 829 schedule_timeout_uninterruptible(1); 830 } 831 if (kthread_should_stop()) 832 goto checkwait; 833 834 /* Wait for the next test interval. */ 835 oldstarttime = boost_starttime; 836 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 837 schedule_timeout_interruptible(oldstarttime - jiffies); 838 stutter_wait("rcu_torture_boost"); 839 if (torture_must_stop()) 840 goto checkwait; 841 } 842 843 /* Do one boost-test interval. */ 844 endtime = oldstarttime + test_boost_duration * HZ; 845 call_rcu_time = jiffies; 846 while (ULONG_CMP_LT(jiffies, endtime)) { 847 /* If we don't have a callback in flight, post one. */ 848 if (!smp_load_acquire(&rbi.inflight)) { 849 /* RCU core before ->inflight = 1. */ 850 smp_store_release(&rbi.inflight, 1); 851 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 852 /* Check if the boost test failed */ 853 failed = failed || 854 rcu_torture_boost_failed(call_rcu_time, 855 jiffies); 856 call_rcu_time = jiffies; 857 } 858 stutter_wait("rcu_torture_boost"); 859 if (torture_must_stop()) 860 goto checkwait; 861 } 862 863 /* 864 * If boost never happened, then inflight will always be 1, in 865 * this case the boost check would never happen in the above 866 * loop so do another one here. 867 */ 868 if (!failed && smp_load_acquire(&rbi.inflight)) 869 rcu_torture_boost_failed(call_rcu_time, jiffies); 870 871 /* 872 * Set the start time of the next test interval. 873 * Yes, this is vulnerable to long delays, but such 874 * delays simply cause a false negative for the next 875 * interval. Besides, we are running at RT priority, 876 * so delays should be relatively rare. 877 */ 878 while (oldstarttime == boost_starttime && 879 !kthread_should_stop()) { 880 if (mutex_trylock(&boost_mutex)) { 881 boost_starttime = jiffies + 882 test_boost_interval * HZ; 883 mutex_unlock(&boost_mutex); 884 break; 885 } 886 schedule_timeout_uninterruptible(1); 887 } 888 889 /* Go do the stutter. */ 890 checkwait: stutter_wait("rcu_torture_boost"); 891 } while (!torture_must_stop()); 892 893 /* Clean up and exit. */ 894 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) { 895 torture_shutdown_absorb("rcu_torture_boost"); 896 schedule_timeout_uninterruptible(1); 897 } 898 destroy_rcu_head_on_stack(&rbi.rcu); 899 torture_kthread_stopping("rcu_torture_boost"); 900 return 0; 901 } 902 903 /* 904 * RCU torture force-quiescent-state kthread. Repeatedly induces 905 * bursts of calls to force_quiescent_state(), increasing the probability 906 * of occurrence of some important types of race conditions. 907 */ 908 static int 909 rcu_torture_fqs(void *arg) 910 { 911 unsigned long fqs_resume_time; 912 int fqs_burst_remaining; 913 914 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 915 do { 916 fqs_resume_time = jiffies + fqs_stutter * HZ; 917 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 918 !kthread_should_stop()) { 919 schedule_timeout_interruptible(1); 920 } 921 fqs_burst_remaining = fqs_duration; 922 while (fqs_burst_remaining > 0 && 923 !kthread_should_stop()) { 924 cur_ops->fqs(); 925 udelay(fqs_holdoff); 926 fqs_burst_remaining -= fqs_holdoff; 927 } 928 stutter_wait("rcu_torture_fqs"); 929 } while (!torture_must_stop()); 930 torture_kthread_stopping("rcu_torture_fqs"); 931 return 0; 932 } 933 934 /* 935 * RCU torture writer kthread. Repeatedly substitutes a new structure 936 * for that pointed to by rcu_torture_current, freeing the old structure 937 * after a series of grace periods (the "pipeline"). 938 */ 939 static int 940 rcu_torture_writer(void *arg) 941 { 942 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 943 int expediting = 0; 944 unsigned long gp_snap; 945 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal; 946 bool gp_sync1 = gp_sync; 947 int i; 948 struct rcu_torture *rp; 949 struct rcu_torture *old_rp; 950 static DEFINE_TORTURE_RANDOM(rand); 951 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC, 952 RTWS_COND_GET, RTWS_SYNC }; 953 int nsynctypes = 0; 954 955 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 956 if (!can_expedite) 957 pr_alert("%s" TORTURE_FLAG 958 " GP expediting controlled from boot/sysfs for %s.\n", 959 torture_type, cur_ops->name); 960 961 /* Initialize synctype[] array. If none set, take default. */ 962 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) 963 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true; 964 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) { 965 synctype[nsynctypes++] = RTWS_COND_GET; 966 pr_info("%s: Testing conditional GPs.\n", __func__); 967 } else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync)) { 968 pr_alert("%s: gp_cond without primitives.\n", __func__); 969 } 970 if (gp_exp1 && cur_ops->exp_sync) { 971 synctype[nsynctypes++] = RTWS_EXP_SYNC; 972 pr_info("%s: Testing expedited GPs.\n", __func__); 973 } else if (gp_exp && !cur_ops->exp_sync) { 974 pr_alert("%s: gp_exp without primitives.\n", __func__); 975 } 976 if (gp_normal1 && cur_ops->deferred_free) { 977 synctype[nsynctypes++] = RTWS_DEF_FREE; 978 pr_info("%s: Testing asynchronous GPs.\n", __func__); 979 } else if (gp_normal && !cur_ops->deferred_free) { 980 pr_alert("%s: gp_normal without primitives.\n", __func__); 981 } 982 if (gp_sync1 && cur_ops->sync) { 983 synctype[nsynctypes++] = RTWS_SYNC; 984 pr_info("%s: Testing normal GPs.\n", __func__); 985 } else if (gp_sync && !cur_ops->sync) { 986 pr_alert("%s: gp_sync without primitives.\n", __func__); 987 } 988 if (WARN_ONCE(nsynctypes == 0, 989 "rcu_torture_writer: No update-side primitives.\n")) { 990 /* 991 * No updates primitives, so don't try updating. 992 * The resulting test won't be testing much, hence the 993 * above WARN_ONCE(). 994 */ 995 rcu_torture_writer_state = RTWS_STOPPING; 996 torture_kthread_stopping("rcu_torture_writer"); 997 } 998 999 do { 1000 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1001 schedule_timeout_uninterruptible(1); 1002 rp = rcu_torture_alloc(); 1003 if (rp == NULL) 1004 continue; 1005 rp->rtort_pipe_count = 0; 1006 rcu_torture_writer_state = RTWS_DELAY; 1007 udelay(torture_random(&rand) & 0x3ff); 1008 rcu_torture_writer_state = RTWS_REPLACE; 1009 old_rp = rcu_dereference_check(rcu_torture_current, 1010 current == writer_task); 1011 rp->rtort_mbtest = 1; 1012 rcu_assign_pointer(rcu_torture_current, rp); 1013 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1014 if (old_rp) { 1015 i = old_rp->rtort_pipe_count; 1016 if (i > RCU_TORTURE_PIPE_LEN) 1017 i = RCU_TORTURE_PIPE_LEN; 1018 atomic_inc(&rcu_torture_wcount[i]); 1019 WRITE_ONCE(old_rp->rtort_pipe_count, 1020 old_rp->rtort_pipe_count + 1); 1021 switch (synctype[torture_random(&rand) % nsynctypes]) { 1022 case RTWS_DEF_FREE: 1023 rcu_torture_writer_state = RTWS_DEF_FREE; 1024 cur_ops->deferred_free(old_rp); 1025 break; 1026 case RTWS_EXP_SYNC: 1027 rcu_torture_writer_state = RTWS_EXP_SYNC; 1028 cur_ops->exp_sync(); 1029 rcu_torture_pipe_update(old_rp); 1030 break; 1031 case RTWS_COND_GET: 1032 rcu_torture_writer_state = RTWS_COND_GET; 1033 gp_snap = cur_ops->get_state(); 1034 i = torture_random(&rand) % 16; 1035 if (i != 0) 1036 schedule_timeout_interruptible(i); 1037 udelay(torture_random(&rand) % 1000); 1038 rcu_torture_writer_state = RTWS_COND_SYNC; 1039 cur_ops->cond_sync(gp_snap); 1040 rcu_torture_pipe_update(old_rp); 1041 break; 1042 case RTWS_SYNC: 1043 rcu_torture_writer_state = RTWS_SYNC; 1044 cur_ops->sync(); 1045 rcu_torture_pipe_update(old_rp); 1046 break; 1047 default: 1048 WARN_ON_ONCE(1); 1049 break; 1050 } 1051 } 1052 WRITE_ONCE(rcu_torture_current_version, 1053 rcu_torture_current_version + 1); 1054 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1055 if (can_expedite && 1056 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1057 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1058 if (expediting >= 0) 1059 rcu_expedite_gp(); 1060 else 1061 rcu_unexpedite_gp(); 1062 if (++expediting > 3) 1063 expediting = -expediting; 1064 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1065 can_expedite = !rcu_gp_is_expedited() && 1066 !rcu_gp_is_normal(); 1067 } 1068 rcu_torture_writer_state = RTWS_STUTTER; 1069 if (stutter_wait("rcu_torture_writer") && 1070 !READ_ONCE(rcu_fwd_cb_nodelay) && 1071 !cur_ops->slow_gps && 1072 !torture_must_stop() && 1073 rcu_inkernel_boot_has_ended()) 1074 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1075 if (list_empty(&rcu_tortures[i].rtort_free) && 1076 rcu_access_pointer(rcu_torture_current) != 1077 &rcu_tortures[i]) { 1078 rcu_ftrace_dump(DUMP_ALL); 1079 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1080 } 1081 } while (!torture_must_stop()); 1082 /* Reset expediting back to unexpedited. */ 1083 if (expediting > 0) 1084 expediting = -expediting; 1085 while (can_expedite && expediting++ < 0) 1086 rcu_unexpedite_gp(); 1087 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1088 if (!can_expedite) 1089 pr_alert("%s" TORTURE_FLAG 1090 " Dynamic grace-period expediting was disabled.\n", 1091 torture_type); 1092 rcu_torture_writer_state = RTWS_STOPPING; 1093 torture_kthread_stopping("rcu_torture_writer"); 1094 return 0; 1095 } 1096 1097 /* 1098 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1099 * delay between calls. 1100 */ 1101 static int 1102 rcu_torture_fakewriter(void *arg) 1103 { 1104 DEFINE_TORTURE_RANDOM(rand); 1105 1106 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1107 set_user_nice(current, MAX_NICE); 1108 1109 do { 1110 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10); 1111 udelay(torture_random(&rand) & 0x3ff); 1112 if (cur_ops->cb_barrier != NULL && 1113 torture_random(&rand) % (nfakewriters * 8) == 0) { 1114 cur_ops->cb_barrier(); 1115 } else if (gp_normal == gp_exp) { 1116 if (cur_ops->sync && torture_random(&rand) & 0x80) 1117 cur_ops->sync(); 1118 else if (cur_ops->exp_sync) 1119 cur_ops->exp_sync(); 1120 } else if (gp_normal && cur_ops->sync) { 1121 cur_ops->sync(); 1122 } else if (cur_ops->exp_sync) { 1123 cur_ops->exp_sync(); 1124 } 1125 stutter_wait("rcu_torture_fakewriter"); 1126 } while (!torture_must_stop()); 1127 1128 torture_kthread_stopping("rcu_torture_fakewriter"); 1129 return 0; 1130 } 1131 1132 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1133 { 1134 kfree(rhp); 1135 } 1136 1137 /* 1138 * Do one extension of an RCU read-side critical section using the 1139 * current reader state in readstate (set to zero for initial entry 1140 * to extended critical section), set the new state as specified by 1141 * newstate (set to zero for final exit from extended critical section), 1142 * and random-number-generator state in trsp. If this is neither the 1143 * beginning or end of the critical section and if there was actually a 1144 * change, do a ->read_delay(). 1145 */ 1146 static void rcutorture_one_extend(int *readstate, int newstate, 1147 struct torture_random_state *trsp, 1148 struct rt_read_seg *rtrsp) 1149 { 1150 int idxnew = -1; 1151 int idxold = *readstate; 1152 int statesnew = ~*readstate & newstate; 1153 int statesold = *readstate & ~newstate; 1154 1155 WARN_ON_ONCE(idxold < 0); 1156 WARN_ON_ONCE((idxold >> RCUTORTURE_RDR_SHIFT) > 1); 1157 rtrsp->rt_readstate = newstate; 1158 1159 /* First, put new protection in place to avoid critical-section gap. */ 1160 if (statesnew & RCUTORTURE_RDR_BH) 1161 local_bh_disable(); 1162 if (statesnew & RCUTORTURE_RDR_IRQ) 1163 local_irq_disable(); 1164 if (statesnew & RCUTORTURE_RDR_PREEMPT) 1165 preempt_disable(); 1166 if (statesnew & RCUTORTURE_RDR_RBH) 1167 rcu_read_lock_bh(); 1168 if (statesnew & RCUTORTURE_RDR_SCHED) 1169 rcu_read_lock_sched(); 1170 if (statesnew & RCUTORTURE_RDR_RCU) 1171 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT; 1172 1173 /* Next, remove old protection, irq first due to bh conflict. */ 1174 if (statesold & RCUTORTURE_RDR_IRQ) 1175 local_irq_enable(); 1176 if (statesold & RCUTORTURE_RDR_BH) 1177 local_bh_enable(); 1178 if (statesold & RCUTORTURE_RDR_PREEMPT) 1179 preempt_enable(); 1180 if (statesold & RCUTORTURE_RDR_RBH) 1181 rcu_read_unlock_bh(); 1182 if (statesold & RCUTORTURE_RDR_SCHED) 1183 rcu_read_unlock_sched(); 1184 if (statesold & RCUTORTURE_RDR_RCU) 1185 cur_ops->readunlock(idxold >> RCUTORTURE_RDR_SHIFT); 1186 1187 /* Delay if neither beginning nor end and there was a change. */ 1188 if ((statesnew || statesold) && *readstate && newstate) 1189 cur_ops->read_delay(trsp, rtrsp); 1190 1191 /* Update the reader state. */ 1192 if (idxnew == -1) 1193 idxnew = idxold & ~RCUTORTURE_RDR_MASK; 1194 WARN_ON_ONCE(idxnew < 0); 1195 WARN_ON_ONCE((idxnew >> RCUTORTURE_RDR_SHIFT) > 1); 1196 *readstate = idxnew | newstate; 1197 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) < 0); 1198 WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT) > 1); 1199 } 1200 1201 /* Return the biggest extendables mask given current RCU and boot parameters. */ 1202 static int rcutorture_extend_mask_max(void) 1203 { 1204 int mask; 1205 1206 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 1207 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 1208 mask = mask | RCUTORTURE_RDR_RCU; 1209 return mask; 1210 } 1211 1212 /* Return a random protection state mask, but with at least one bit set. */ 1213 static int 1214 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 1215 { 1216 int mask = rcutorture_extend_mask_max(); 1217 unsigned long randmask1 = torture_random(trsp) >> 8; 1218 unsigned long randmask2 = randmask1 >> 3; 1219 1220 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT); 1221 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 1222 if (!(randmask1 & 0x7)) 1223 mask = mask & randmask2; 1224 else 1225 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 1226 /* Can't enable bh w/irq disabled. */ 1227 if ((mask & RCUTORTURE_RDR_IRQ) && 1228 ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) || 1229 (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH)))) 1230 mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1231 return mask ?: RCUTORTURE_RDR_RCU; 1232 } 1233 1234 /* 1235 * Do a randomly selected number of extensions of an existing RCU read-side 1236 * critical section. 1237 */ 1238 static struct rt_read_seg * 1239 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, 1240 struct rt_read_seg *rtrsp) 1241 { 1242 int i; 1243 int j; 1244 int mask = rcutorture_extend_mask_max(); 1245 1246 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 1247 if (!((mask - 1) & mask)) 1248 return rtrsp; /* Current RCU reader not extendable. */ 1249 /* Bias towards larger numbers of loops. */ 1250 i = (torture_random(trsp) >> 3); 1251 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 1252 for (j = 0; j < i; j++) { 1253 mask = rcutorture_extend_mask(*readstate, trsp); 1254 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 1255 } 1256 return &rtrsp[j]; 1257 } 1258 1259 /* 1260 * Do one read-side critical section, returning false if there was 1261 * no data to read. Can be invoked both from process context and 1262 * from a timer handler. 1263 */ 1264 static bool rcu_torture_one_read(struct torture_random_state *trsp) 1265 { 1266 int i; 1267 unsigned long started; 1268 unsigned long completed; 1269 int newstate; 1270 struct rcu_torture *p; 1271 int pipe_count; 1272 int readstate = 0; 1273 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } }; 1274 struct rt_read_seg *rtrsp = &rtseg[0]; 1275 struct rt_read_seg *rtrsp1; 1276 unsigned long long ts; 1277 1278 newstate = rcutorture_extend_mask(readstate, trsp); 1279 rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); 1280 started = cur_ops->get_gp_seq(); 1281 ts = rcu_trace_clock_local(); 1282 p = rcu_dereference_check(rcu_torture_current, 1283 rcu_read_lock_bh_held() || 1284 rcu_read_lock_sched_held() || 1285 srcu_read_lock_held(srcu_ctlp) || 1286 torturing_tasks()); 1287 if (p == NULL) { 1288 /* Wait for rcu_torture_writer to get underway */ 1289 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 1290 return false; 1291 } 1292 if (p->rtort_mbtest == 0) 1293 atomic_inc(&n_rcu_torture_mberror); 1294 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp); 1295 preempt_disable(); 1296 pipe_count = READ_ONCE(p->rtort_pipe_count); 1297 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1298 /* Should not happen, but... */ 1299 pipe_count = RCU_TORTURE_PIPE_LEN; 1300 } 1301 completed = cur_ops->get_gp_seq(); 1302 if (pipe_count > 1) { 1303 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 1304 ts, started, completed); 1305 rcu_ftrace_dump(DUMP_ALL); 1306 } 1307 __this_cpu_inc(rcu_torture_count[pipe_count]); 1308 completed = rcutorture_seq_diff(completed, started); 1309 if (completed > RCU_TORTURE_PIPE_LEN) { 1310 /* Should not happen, but... */ 1311 completed = RCU_TORTURE_PIPE_LEN; 1312 } 1313 __this_cpu_inc(rcu_torture_batch[completed]); 1314 preempt_enable(); 1315 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 1316 WARN_ON_ONCE(readstate & RCUTORTURE_RDR_MASK); 1317 1318 /* If error or close call, record the sequence of reader protections. */ 1319 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 1320 i = 0; 1321 for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++) 1322 err_segs[i++] = *rtrsp1; 1323 rt_read_nsegs = i; 1324 } 1325 1326 return true; 1327 } 1328 1329 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 1330 1331 /* 1332 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1333 * incrementing the corresponding element of the pipeline array. The 1334 * counter in the element should never be greater than 1, otherwise, the 1335 * RCU implementation is broken. 1336 */ 1337 static void rcu_torture_timer(struct timer_list *unused) 1338 { 1339 atomic_long_inc(&n_rcu_torture_timers); 1340 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand)); 1341 1342 /* Test call_rcu() invocation from interrupt handler. */ 1343 if (cur_ops->call) { 1344 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 1345 1346 if (rhp) 1347 cur_ops->call(rhp, rcu_torture_timer_cb); 1348 } 1349 } 1350 1351 /* 1352 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1353 * incrementing the corresponding element of the pipeline array. The 1354 * counter in the element should never be greater than 1, otherwise, the 1355 * RCU implementation is broken. 1356 */ 1357 static int 1358 rcu_torture_reader(void *arg) 1359 { 1360 unsigned long lastsleep = jiffies; 1361 long myid = (long)arg; 1362 int mynumonline = myid; 1363 DEFINE_TORTURE_RANDOM(rand); 1364 struct timer_list t; 1365 1366 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 1367 set_user_nice(current, MAX_NICE); 1368 if (irqreader && cur_ops->irq_capable) 1369 timer_setup_on_stack(&t, rcu_torture_timer, 0); 1370 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 1371 do { 1372 if (irqreader && cur_ops->irq_capable) { 1373 if (!timer_pending(&t)) 1374 mod_timer(&t, jiffies + 1); 1375 } 1376 if (!rcu_torture_one_read(&rand) && !torture_must_stop()) 1377 schedule_timeout_interruptible(HZ); 1378 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 1379 schedule_timeout_interruptible(1); 1380 lastsleep = jiffies + 10; 1381 } 1382 while (num_online_cpus() < mynumonline && !torture_must_stop()) 1383 schedule_timeout_interruptible(HZ / 5); 1384 stutter_wait("rcu_torture_reader"); 1385 } while (!torture_must_stop()); 1386 if (irqreader && cur_ops->irq_capable) { 1387 del_timer_sync(&t); 1388 destroy_timer_on_stack(&t); 1389 } 1390 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 1391 torture_kthread_stopping("rcu_torture_reader"); 1392 return 0; 1393 } 1394 1395 /* 1396 * Print torture statistics. Caller must ensure that there is only 1397 * one call to this function at a given time!!! This is normally 1398 * accomplished by relying on the module system to only have one copy 1399 * of the module loaded, and then by giving the rcu_torture_stats 1400 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1401 * thread is not running). 1402 */ 1403 static void 1404 rcu_torture_stats_print(void) 1405 { 1406 int cpu; 1407 int i; 1408 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1409 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1410 struct rcu_torture *rtcp; 1411 static unsigned long rtcv_snap = ULONG_MAX; 1412 static bool splatted; 1413 struct task_struct *wtp; 1414 1415 for_each_possible_cpu(cpu) { 1416 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1417 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 1418 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 1419 } 1420 } 1421 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1422 if (pipesummary[i] != 0) 1423 break; 1424 } 1425 1426 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1427 rtcp = rcu_access_pointer(rcu_torture_current); 1428 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1429 rtcp, 1430 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 1431 rcu_torture_current_version, 1432 list_empty(&rcu_torture_freelist), 1433 atomic_read(&n_rcu_torture_alloc), 1434 atomic_read(&n_rcu_torture_alloc_fail), 1435 atomic_read(&n_rcu_torture_free)); 1436 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ", 1437 atomic_read(&n_rcu_torture_mberror), 1438 n_rcu_torture_barrier_error, 1439 n_rcu_torture_boost_ktrerror, 1440 n_rcu_torture_boost_rterror); 1441 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 1442 n_rcu_torture_boost_failure, 1443 n_rcu_torture_boosts, 1444 atomic_long_read(&n_rcu_torture_timers)); 1445 torture_onoff_stats(); 1446 pr_cont("barrier: %ld/%ld:%ld\n", 1447 n_barrier_successes, 1448 n_barrier_attempts, 1449 n_rcu_torture_barrier_error); 1450 1451 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1452 if (atomic_read(&n_rcu_torture_mberror) || 1453 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 1454 n_rcu_torture_boost_rterror || n_rcu_torture_boost_failure || 1455 i > 1) { 1456 pr_cont("%s", "!!! "); 1457 atomic_inc(&n_rcu_torture_error); 1458 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 1459 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 1460 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 1461 WARN_ON_ONCE(n_rcu_torture_boost_rterror); // can't set RT prio 1462 WARN_ON_ONCE(n_rcu_torture_boost_failure); // RCU boost failed 1463 WARN_ON_ONCE(i > 1); // Too-short grace period 1464 } 1465 pr_cont("Reader Pipe: "); 1466 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1467 pr_cont(" %ld", pipesummary[i]); 1468 pr_cont("\n"); 1469 1470 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1471 pr_cont("Reader Batch: "); 1472 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1473 pr_cont(" %ld", batchsummary[i]); 1474 pr_cont("\n"); 1475 1476 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 1477 pr_cont("Free-Block Circulation: "); 1478 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1479 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 1480 } 1481 pr_cont("\n"); 1482 1483 if (cur_ops->stats) 1484 cur_ops->stats(); 1485 if (rtcv_snap == rcu_torture_current_version && 1486 rcu_access_pointer(rcu_torture_current) && 1487 !rcu_stall_is_suppressed()) { 1488 int __maybe_unused flags = 0; 1489 unsigned long __maybe_unused gp_seq = 0; 1490 1491 rcutorture_get_gp_data(cur_ops->ttype, 1492 &flags, &gp_seq); 1493 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, 1494 &flags, &gp_seq); 1495 wtp = READ_ONCE(writer_task); 1496 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#lx cpu %d\n", 1497 rcu_torture_writer_state_getname(), 1498 rcu_torture_writer_state, gp_seq, flags, 1499 wtp == NULL ? ~0UL : wtp->state, 1500 wtp == NULL ? -1 : (int)task_cpu(wtp)); 1501 if (!splatted && wtp) { 1502 sched_show_task(wtp); 1503 splatted = true; 1504 } 1505 show_rcu_gp_kthreads(); 1506 rcu_ftrace_dump(DUMP_ALL); 1507 } 1508 rtcv_snap = rcu_torture_current_version; 1509 } 1510 1511 /* 1512 * Periodically prints torture statistics, if periodic statistics printing 1513 * was specified via the stat_interval module parameter. 1514 */ 1515 static int 1516 rcu_torture_stats(void *arg) 1517 { 1518 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 1519 do { 1520 schedule_timeout_interruptible(stat_interval * HZ); 1521 rcu_torture_stats_print(); 1522 torture_shutdown_absorb("rcu_torture_stats"); 1523 } while (!torture_must_stop()); 1524 torture_kthread_stopping("rcu_torture_stats"); 1525 return 0; 1526 } 1527 1528 static void 1529 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 1530 { 1531 pr_alert("%s" TORTURE_FLAG 1532 "--- %s: nreaders=%d nfakewriters=%d " 1533 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1534 "shuffle_interval=%d stutter=%d irqreader=%d " 1535 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1536 "test_boost=%d/%d test_boost_interval=%d " 1537 "test_boost_duration=%d shutdown_secs=%d " 1538 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 1539 "n_barrier_cbs=%d " 1540 "onoff_interval=%d onoff_holdoff=%d\n", 1541 torture_type, tag, nrealreaders, nfakewriters, 1542 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1543 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1544 test_boost, cur_ops->can_boost, 1545 test_boost_interval, test_boost_duration, shutdown_secs, 1546 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 1547 n_barrier_cbs, 1548 onoff_interval, onoff_holdoff); 1549 } 1550 1551 static int rcutorture_booster_cleanup(unsigned int cpu) 1552 { 1553 struct task_struct *t; 1554 1555 if (boost_tasks[cpu] == NULL) 1556 return 0; 1557 mutex_lock(&boost_mutex); 1558 t = boost_tasks[cpu]; 1559 boost_tasks[cpu] = NULL; 1560 rcu_torture_enable_rt_throttle(); 1561 mutex_unlock(&boost_mutex); 1562 1563 /* This must be outside of the mutex, otherwise deadlock! */ 1564 torture_stop_kthread(rcu_torture_boost, t); 1565 return 0; 1566 } 1567 1568 static int rcutorture_booster_init(unsigned int cpu) 1569 { 1570 int retval; 1571 1572 if (boost_tasks[cpu] != NULL) 1573 return 0; /* Already created, nothing more to do. */ 1574 1575 /* Don't allow time recalculation while creating a new task. */ 1576 mutex_lock(&boost_mutex); 1577 rcu_torture_disable_rt_throttle(); 1578 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 1579 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1580 cpu_to_node(cpu), 1581 "rcu_torture_boost"); 1582 if (IS_ERR(boost_tasks[cpu])) { 1583 retval = PTR_ERR(boost_tasks[cpu]); 1584 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 1585 n_rcu_torture_boost_ktrerror++; 1586 boost_tasks[cpu] = NULL; 1587 mutex_unlock(&boost_mutex); 1588 return retval; 1589 } 1590 kthread_bind(boost_tasks[cpu], cpu); 1591 wake_up_process(boost_tasks[cpu]); 1592 mutex_unlock(&boost_mutex); 1593 return 0; 1594 } 1595 1596 /* 1597 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1598 * induces a CPU stall for the time specified by stall_cpu. 1599 */ 1600 static int rcu_torture_stall(void *args) 1601 { 1602 unsigned long stop_at; 1603 1604 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 1605 if (stall_cpu_holdoff > 0) { 1606 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 1607 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1608 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 1609 } 1610 if (!kthread_should_stop()) { 1611 stop_at = ktime_get_seconds() + stall_cpu; 1612 /* RCU CPU stall is expected behavior in following code. */ 1613 rcu_read_lock(); 1614 if (stall_cpu_irqsoff) 1615 local_irq_disable(); 1616 else 1617 preempt_disable(); 1618 pr_alert("rcu_torture_stall start on CPU %d.\n", 1619 smp_processor_id()); 1620 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), 1621 stop_at)) 1622 continue; /* Induce RCU CPU stall warning. */ 1623 if (stall_cpu_irqsoff) 1624 local_irq_enable(); 1625 else 1626 preempt_enable(); 1627 rcu_read_unlock(); 1628 pr_alert("rcu_torture_stall end.\n"); 1629 } 1630 torture_shutdown_absorb("rcu_torture_stall"); 1631 while (!kthread_should_stop()) 1632 schedule_timeout_interruptible(10 * HZ); 1633 return 0; 1634 } 1635 1636 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1637 static int __init rcu_torture_stall_init(void) 1638 { 1639 if (stall_cpu <= 0) 1640 return 0; 1641 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 1642 } 1643 1644 /* State structure for forward-progress self-propagating RCU callback. */ 1645 struct fwd_cb_state { 1646 struct rcu_head rh; 1647 int stop; 1648 }; 1649 1650 /* 1651 * Forward-progress self-propagating RCU callback function. Because 1652 * callbacks run from softirq, this function is an implicit RCU read-side 1653 * critical section. 1654 */ 1655 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 1656 { 1657 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 1658 1659 if (READ_ONCE(fcsp->stop)) { 1660 WRITE_ONCE(fcsp->stop, 2); 1661 return; 1662 } 1663 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 1664 } 1665 1666 /* State for continuous-flood RCU callbacks. */ 1667 struct rcu_fwd_cb { 1668 struct rcu_head rh; 1669 struct rcu_fwd_cb *rfc_next; 1670 struct rcu_fwd *rfc_rfp; 1671 int rfc_gps; 1672 }; 1673 1674 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 1675 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 1676 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 1677 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 1678 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 1679 1680 struct rcu_launder_hist { 1681 long n_launders; 1682 unsigned long launder_gp_seq; 1683 }; 1684 1685 struct rcu_fwd { 1686 spinlock_t rcu_fwd_lock; 1687 struct rcu_fwd_cb *rcu_fwd_cb_head; 1688 struct rcu_fwd_cb **rcu_fwd_cb_tail; 1689 long n_launders_cb; 1690 unsigned long rcu_fwd_startat; 1691 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 1692 unsigned long rcu_launder_gp_seq_start; 1693 }; 1694 1695 struct rcu_fwd *rcu_fwds; 1696 bool rcu_fwd_emergency_stop; 1697 1698 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 1699 { 1700 unsigned long gps; 1701 unsigned long gps_old; 1702 int i; 1703 int j; 1704 1705 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 1706 if (rfp->n_launders_hist[i].n_launders > 0) 1707 break; 1708 pr_alert("%s: Callback-invocation histogram (duration %lu jiffies):", 1709 __func__, jiffies - rfp->rcu_fwd_startat); 1710 gps_old = rfp->rcu_launder_gp_seq_start; 1711 for (j = 0; j <= i; j++) { 1712 gps = rfp->n_launders_hist[j].launder_gp_seq; 1713 pr_cont(" %ds/%d: %ld:%ld", 1714 j + 1, FWD_CBS_HIST_DIV, 1715 rfp->n_launders_hist[j].n_launders, 1716 rcutorture_seq_diff(gps, gps_old)); 1717 gps_old = gps; 1718 } 1719 pr_cont("\n"); 1720 } 1721 1722 /* Callback function for continuous-flood RCU callbacks. */ 1723 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 1724 { 1725 unsigned long flags; 1726 int i; 1727 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 1728 struct rcu_fwd_cb **rfcpp; 1729 struct rcu_fwd *rfp = rfcp->rfc_rfp; 1730 1731 rfcp->rfc_next = NULL; 1732 rfcp->rfc_gps++; 1733 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 1734 rfcpp = rfp->rcu_fwd_cb_tail; 1735 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 1736 WRITE_ONCE(*rfcpp, rfcp); 1737 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 1738 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 1739 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 1740 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 1741 rfp->n_launders_hist[i].n_launders++; 1742 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 1743 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1744 } 1745 1746 // Give the scheduler a chance, even on nohz_full CPUs. 1747 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 1748 { 1749 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 1750 // Real call_rcu() floods hit userspace, so emulate that. 1751 if (need_resched() || (iter & 0xfff)) 1752 schedule(); 1753 return; 1754 } 1755 // No userspace emulation: CB invocation throttles call_rcu() 1756 cond_resched(); 1757 } 1758 1759 /* 1760 * Free all callbacks on the rcu_fwd_cb_head list, either because the 1761 * test is over or because we hit an OOM event. 1762 */ 1763 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 1764 { 1765 unsigned long flags; 1766 unsigned long freed = 0; 1767 struct rcu_fwd_cb *rfcp; 1768 1769 for (;;) { 1770 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 1771 rfcp = rfp->rcu_fwd_cb_head; 1772 if (!rfcp) { 1773 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1774 break; 1775 } 1776 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 1777 if (!rfp->rcu_fwd_cb_head) 1778 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 1779 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 1780 kfree(rfcp); 1781 freed++; 1782 rcu_torture_fwd_prog_cond_resched(freed); 1783 if (tick_nohz_full_enabled()) { 1784 local_irq_save(flags); 1785 rcu_momentary_dyntick_idle(); 1786 local_irq_restore(flags); 1787 } 1788 } 1789 return freed; 1790 } 1791 1792 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 1793 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 1794 int *tested, int *tested_tries) 1795 { 1796 unsigned long cver; 1797 unsigned long dur; 1798 struct fwd_cb_state fcs; 1799 unsigned long gps; 1800 int idx; 1801 int sd; 1802 int sd4; 1803 bool selfpropcb = false; 1804 unsigned long stopat; 1805 static DEFINE_TORTURE_RANDOM(trs); 1806 1807 if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) { 1808 init_rcu_head_on_stack(&fcs.rh); 1809 selfpropcb = true; 1810 } 1811 1812 /* Tight loop containing cond_resched(). */ 1813 WRITE_ONCE(rcu_fwd_cb_nodelay, true); 1814 cur_ops->sync(); /* Later readers see above write. */ 1815 if (selfpropcb) { 1816 WRITE_ONCE(fcs.stop, 0); 1817 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 1818 } 1819 cver = READ_ONCE(rcu_torture_current_version); 1820 gps = cur_ops->get_gp_seq(); 1821 sd = cur_ops->stall_dur() + 1; 1822 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 1823 dur = sd4 + torture_random(&trs) % (sd - sd4); 1824 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 1825 stopat = rfp->rcu_fwd_startat + dur; 1826 while (time_before(jiffies, stopat) && 1827 !shutdown_time_arrived() && 1828 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1829 idx = cur_ops->readlock(); 1830 udelay(10); 1831 cur_ops->readunlock(idx); 1832 if (!fwd_progress_need_resched || need_resched()) 1833 cond_resched(); 1834 } 1835 (*tested_tries)++; 1836 if (!time_before(jiffies, stopat) && 1837 !shutdown_time_arrived() && 1838 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1839 (*tested)++; 1840 cver = READ_ONCE(rcu_torture_current_version) - cver; 1841 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 1842 WARN_ON(!cver && gps < 2); 1843 pr_alert("%s: Duration %ld cver %ld gps %ld\n", __func__, dur, cver, gps); 1844 } 1845 if (selfpropcb) { 1846 WRITE_ONCE(fcs.stop, 1); 1847 cur_ops->sync(); /* Wait for running CB to complete. */ 1848 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 1849 } 1850 1851 if (selfpropcb) { 1852 WARN_ON(READ_ONCE(fcs.stop) != 2); 1853 destroy_rcu_head_on_stack(&fcs.rh); 1854 } 1855 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 1856 WRITE_ONCE(rcu_fwd_cb_nodelay, false); 1857 } 1858 1859 /* Carry out call_rcu() forward-progress testing. */ 1860 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 1861 { 1862 unsigned long cver; 1863 unsigned long flags; 1864 unsigned long gps; 1865 int i; 1866 long n_launders; 1867 long n_launders_cb_snap; 1868 long n_launders_sa; 1869 long n_max_cbs; 1870 long n_max_gps; 1871 struct rcu_fwd_cb *rfcp; 1872 struct rcu_fwd_cb *rfcpn; 1873 unsigned long stopat; 1874 unsigned long stoppedat; 1875 1876 if (READ_ONCE(rcu_fwd_emergency_stop)) 1877 return; /* Get out of the way quickly, no GP wait! */ 1878 if (!cur_ops->call) 1879 return; /* Can't do call_rcu() fwd prog without ->call. */ 1880 1881 /* Loop continuously posting RCU callbacks. */ 1882 WRITE_ONCE(rcu_fwd_cb_nodelay, true); 1883 cur_ops->sync(); /* Later readers see above write. */ 1884 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 1885 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 1886 n_launders = 0; 1887 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 1888 n_launders_sa = 0; 1889 n_max_cbs = 0; 1890 n_max_gps = 0; 1891 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 1892 rfp->n_launders_hist[i].n_launders = 0; 1893 cver = READ_ONCE(rcu_torture_current_version); 1894 gps = cur_ops->get_gp_seq(); 1895 rfp->rcu_launder_gp_seq_start = gps; 1896 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 1897 while (time_before(jiffies, stopat) && 1898 !shutdown_time_arrived() && 1899 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 1900 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 1901 rfcpn = NULL; 1902 if (rfcp) 1903 rfcpn = READ_ONCE(rfcp->rfc_next); 1904 if (rfcpn) { 1905 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 1906 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 1907 break; 1908 rfp->rcu_fwd_cb_head = rfcpn; 1909 n_launders++; 1910 n_launders_sa++; 1911 } else { 1912 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 1913 if (WARN_ON_ONCE(!rfcp)) { 1914 schedule_timeout_interruptible(1); 1915 continue; 1916 } 1917 n_max_cbs++; 1918 n_launders_sa = 0; 1919 rfcp->rfc_gps = 0; 1920 rfcp->rfc_rfp = rfp; 1921 } 1922 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 1923 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 1924 if (tick_nohz_full_enabled()) { 1925 local_irq_save(flags); 1926 rcu_momentary_dyntick_idle(); 1927 local_irq_restore(flags); 1928 } 1929 } 1930 stoppedat = jiffies; 1931 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 1932 cver = READ_ONCE(rcu_torture_current_version) - cver; 1933 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 1934 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 1935 (void)rcu_torture_fwd_prog_cbfree(rfp); 1936 1937 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 1938 !shutdown_time_arrived()) { 1939 WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED); 1940 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n", 1941 __func__, 1942 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 1943 n_launders + n_max_cbs - n_launders_cb_snap, 1944 n_launders, n_launders_sa, 1945 n_max_gps, n_max_cbs, cver, gps); 1946 rcu_torture_fwd_cb_hist(rfp); 1947 } 1948 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 1949 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 1950 WRITE_ONCE(rcu_fwd_cb_nodelay, false); 1951 } 1952 1953 1954 /* 1955 * OOM notifier, but this only prints diagnostic information for the 1956 * current forward-progress test. 1957 */ 1958 static int rcutorture_oom_notify(struct notifier_block *self, 1959 unsigned long notused, void *nfreed) 1960 { 1961 struct rcu_fwd *rfp = rcu_fwds; 1962 1963 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 1964 __func__); 1965 rcu_torture_fwd_cb_hist(rfp); 1966 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2); 1967 WRITE_ONCE(rcu_fwd_emergency_stop, true); 1968 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 1969 pr_info("%s: Freed %lu RCU callbacks.\n", 1970 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1971 rcu_barrier(); 1972 pr_info("%s: Freed %lu RCU callbacks.\n", 1973 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1974 rcu_barrier(); 1975 pr_info("%s: Freed %lu RCU callbacks.\n", 1976 __func__, rcu_torture_fwd_prog_cbfree(rfp)); 1977 smp_mb(); /* Frees before return to avoid redoing OOM. */ 1978 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 1979 pr_info("%s returning after OOM processing.\n", __func__); 1980 return NOTIFY_OK; 1981 } 1982 1983 static struct notifier_block rcutorture_oom_nb = { 1984 .notifier_call = rcutorture_oom_notify 1985 }; 1986 1987 /* Carry out grace-period forward-progress testing. */ 1988 static int rcu_torture_fwd_prog(void *args) 1989 { 1990 struct rcu_fwd *rfp = args; 1991 int tested = 0; 1992 int tested_tries = 0; 1993 1994 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 1995 rcu_bind_current_to_nocb(); 1996 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 1997 set_user_nice(current, MAX_NICE); 1998 do { 1999 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 2000 WRITE_ONCE(rcu_fwd_emergency_stop, false); 2001 register_oom_notifier(&rcutorture_oom_nb); 2002 if (!IS_ENABLED(CONFIG_TINY_RCU) || 2003 rcu_inkernel_boot_has_ended()) 2004 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 2005 if (rcu_inkernel_boot_has_ended()) 2006 rcu_torture_fwd_prog_cr(rfp); 2007 unregister_oom_notifier(&rcutorture_oom_nb); 2008 2009 /* Avoid slow periods, better to test when busy. */ 2010 stutter_wait("rcu_torture_fwd_prog"); 2011 } while (!torture_must_stop()); 2012 /* Short runs might not contain a valid forward-progress attempt. */ 2013 WARN_ON(!tested && tested_tries >= 5); 2014 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 2015 torture_kthread_stopping("rcu_torture_fwd_prog"); 2016 return 0; 2017 } 2018 2019 /* If forward-progress checking is requested and feasible, spawn the thread. */ 2020 static int __init rcu_torture_fwd_prog_init(void) 2021 { 2022 struct rcu_fwd *rfp; 2023 2024 if (!fwd_progress) 2025 return 0; /* Not requested, so don't do it. */ 2026 if (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || 2027 cur_ops == &rcu_busted_ops) { 2028 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 2029 return 0; 2030 } 2031 if (stall_cpu > 0) { 2032 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); 2033 if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS)) 2034 return -EINVAL; /* In module, can fail back to user. */ 2035 WARN_ON(1); /* Make sure rcutorture notices conflict. */ 2036 return 0; 2037 } 2038 if (fwd_progress_holdoff <= 0) 2039 fwd_progress_holdoff = 1; 2040 if (fwd_progress_div <= 0) 2041 fwd_progress_div = 4; 2042 rfp = kzalloc(sizeof(*rfp), GFP_KERNEL); 2043 if (!rfp) 2044 return -ENOMEM; 2045 spin_lock_init(&rfp->rcu_fwd_lock); 2046 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 2047 return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task); 2048 } 2049 2050 /* Callback function for RCU barrier testing. */ 2051 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 2052 { 2053 atomic_inc(&barrier_cbs_invoked); 2054 } 2055 2056 /* IPI handler to get callback posted on desired CPU, if online. */ 2057 static void rcu_torture_barrier1cb(void *rcu_void) 2058 { 2059 struct rcu_head *rhp = rcu_void; 2060 2061 cur_ops->call(rhp, rcu_torture_barrier_cbf); 2062 } 2063 2064 /* kthread function to register callbacks used to test RCU barriers. */ 2065 static int rcu_torture_barrier_cbs(void *arg) 2066 { 2067 long myid = (long)arg; 2068 bool lastphase = 0; 2069 bool newphase; 2070 struct rcu_head rcu; 2071 2072 init_rcu_head_on_stack(&rcu); 2073 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 2074 set_user_nice(current, MAX_NICE); 2075 do { 2076 wait_event(barrier_cbs_wq[myid], 2077 (newphase = 2078 smp_load_acquire(&barrier_phase)) != lastphase || 2079 torture_must_stop()); 2080 lastphase = newphase; 2081 if (torture_must_stop()) 2082 break; 2083 /* 2084 * The above smp_load_acquire() ensures barrier_phase load 2085 * is ordered before the following ->call(). 2086 */ 2087 if (smp_call_function_single(myid, rcu_torture_barrier1cb, 2088 &rcu, 1)) { 2089 // IPI failed, so use direct call from current CPU. 2090 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 2091 } 2092 if (atomic_dec_and_test(&barrier_cbs_count)) 2093 wake_up(&barrier_wq); 2094 } while (!torture_must_stop()); 2095 if (cur_ops->cb_barrier != NULL) 2096 cur_ops->cb_barrier(); 2097 destroy_rcu_head_on_stack(&rcu); 2098 torture_kthread_stopping("rcu_torture_barrier_cbs"); 2099 return 0; 2100 } 2101 2102 /* kthread function to drive and coordinate RCU barrier testing. */ 2103 static int rcu_torture_barrier(void *arg) 2104 { 2105 int i; 2106 2107 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 2108 do { 2109 atomic_set(&barrier_cbs_invoked, 0); 2110 atomic_set(&barrier_cbs_count, n_barrier_cbs); 2111 /* Ensure barrier_phase ordered after prior assignments. */ 2112 smp_store_release(&barrier_phase, !barrier_phase); 2113 for (i = 0; i < n_barrier_cbs; i++) 2114 wake_up(&barrier_cbs_wq[i]); 2115 wait_event(barrier_wq, 2116 atomic_read(&barrier_cbs_count) == 0 || 2117 torture_must_stop()); 2118 if (torture_must_stop()) 2119 break; 2120 n_barrier_attempts++; 2121 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 2122 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 2123 n_rcu_torture_barrier_error++; 2124 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 2125 atomic_read(&barrier_cbs_invoked), 2126 n_barrier_cbs); 2127 WARN_ON(1); 2128 // Wait manually for the remaining callbacks 2129 i = 0; 2130 do { 2131 if (WARN_ON(i++ > HZ)) 2132 i = INT_MIN; 2133 schedule_timeout_interruptible(1); 2134 cur_ops->cb_barrier(); 2135 } while (atomic_read(&barrier_cbs_invoked) != 2136 n_barrier_cbs && 2137 !torture_must_stop()); 2138 smp_mb(); // Can't trust ordering if broken. 2139 if (!torture_must_stop()) 2140 pr_err("Recovered: barrier_cbs_invoked = %d\n", 2141 atomic_read(&barrier_cbs_invoked)); 2142 } else { 2143 n_barrier_successes++; 2144 } 2145 schedule_timeout_interruptible(HZ / 10); 2146 } while (!torture_must_stop()); 2147 torture_kthread_stopping("rcu_torture_barrier"); 2148 return 0; 2149 } 2150 2151 /* Initialize RCU barrier testing. */ 2152 static int rcu_torture_barrier_init(void) 2153 { 2154 int i; 2155 int ret; 2156 2157 if (n_barrier_cbs <= 0) 2158 return 0; 2159 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 2160 pr_alert("%s" TORTURE_FLAG 2161 " Call or barrier ops missing for %s,\n", 2162 torture_type, cur_ops->name); 2163 pr_alert("%s" TORTURE_FLAG 2164 " RCU barrier testing omitted from run.\n", 2165 torture_type); 2166 return 0; 2167 } 2168 atomic_set(&barrier_cbs_count, 0); 2169 atomic_set(&barrier_cbs_invoked, 0); 2170 barrier_cbs_tasks = 2171 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 2172 GFP_KERNEL); 2173 barrier_cbs_wq = 2174 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 2175 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 2176 return -ENOMEM; 2177 for (i = 0; i < n_barrier_cbs; i++) { 2178 init_waitqueue_head(&barrier_cbs_wq[i]); 2179 ret = torture_create_kthread(rcu_torture_barrier_cbs, 2180 (void *)(long)i, 2181 barrier_cbs_tasks[i]); 2182 if (ret) 2183 return ret; 2184 } 2185 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 2186 } 2187 2188 /* Clean up after RCU barrier testing. */ 2189 static void rcu_torture_barrier_cleanup(void) 2190 { 2191 int i; 2192 2193 torture_stop_kthread(rcu_torture_barrier, barrier_task); 2194 if (barrier_cbs_tasks != NULL) { 2195 for (i = 0; i < n_barrier_cbs; i++) 2196 torture_stop_kthread(rcu_torture_barrier_cbs, 2197 barrier_cbs_tasks[i]); 2198 kfree(barrier_cbs_tasks); 2199 barrier_cbs_tasks = NULL; 2200 } 2201 if (barrier_cbs_wq != NULL) { 2202 kfree(barrier_cbs_wq); 2203 barrier_cbs_wq = NULL; 2204 } 2205 } 2206 2207 static bool rcu_torture_can_boost(void) 2208 { 2209 static int boost_warn_once; 2210 int prio; 2211 2212 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 2213 return false; 2214 2215 prio = rcu_get_gp_kthreads_prio(); 2216 if (!prio) 2217 return false; 2218 2219 if (prio < 2) { 2220 if (boost_warn_once == 1) 2221 return false; 2222 2223 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 2224 boost_warn_once = 1; 2225 return false; 2226 } 2227 2228 return true; 2229 } 2230 2231 static enum cpuhp_state rcutor_hp; 2232 2233 static void 2234 rcu_torture_cleanup(void) 2235 { 2236 int firsttime; 2237 int flags = 0; 2238 unsigned long gp_seq = 0; 2239 int i; 2240 2241 if (torture_cleanup_begin()) { 2242 if (cur_ops->cb_barrier != NULL) 2243 cur_ops->cb_barrier(); 2244 return; 2245 } 2246 if (!cur_ops) { 2247 torture_cleanup_end(); 2248 return; 2249 } 2250 2251 show_rcu_gp_kthreads(); 2252 rcu_torture_barrier_cleanup(); 2253 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_task); 2254 torture_stop_kthread(rcu_torture_stall, stall_task); 2255 torture_stop_kthread(rcu_torture_writer, writer_task); 2256 2257 if (reader_tasks) { 2258 for (i = 0; i < nrealreaders; i++) 2259 torture_stop_kthread(rcu_torture_reader, 2260 reader_tasks[i]); 2261 kfree(reader_tasks); 2262 } 2263 rcu_torture_current = NULL; 2264 2265 if (fakewriter_tasks) { 2266 for (i = 0; i < nfakewriters; i++) { 2267 torture_stop_kthread(rcu_torture_fakewriter, 2268 fakewriter_tasks[i]); 2269 } 2270 kfree(fakewriter_tasks); 2271 fakewriter_tasks = NULL; 2272 } 2273 2274 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); 2275 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); 2276 pr_alert("%s: End-test grace-period state: g%lu f%#x\n", 2277 cur_ops->name, gp_seq, flags); 2278 torture_stop_kthread(rcu_torture_stats, stats_task); 2279 torture_stop_kthread(rcu_torture_fqs, fqs_task); 2280 if (rcu_torture_can_boost()) 2281 cpuhp_remove_state(rcutor_hp); 2282 2283 /* 2284 * Wait for all RCU callbacks to fire, then do torture-type-specific 2285 * cleanup operations. 2286 */ 2287 if (cur_ops->cb_barrier != NULL) 2288 cur_ops->cb_barrier(); 2289 if (cur_ops->cleanup != NULL) 2290 cur_ops->cleanup(); 2291 2292 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 2293 2294 if (err_segs_recorded) { 2295 pr_alert("Failure/close-call rcutorture reader segments:\n"); 2296 if (rt_read_nsegs == 0) 2297 pr_alert("\t: No segments recorded!!!\n"); 2298 firsttime = 1; 2299 for (i = 0; i < rt_read_nsegs; i++) { 2300 pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate); 2301 if (err_segs[i].rt_delay_jiffies != 0) { 2302 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 2303 err_segs[i].rt_delay_jiffies); 2304 firsttime = 0; 2305 } 2306 if (err_segs[i].rt_delay_ms != 0) { 2307 pr_cont("%s%ldms", firsttime ? "" : "+", 2308 err_segs[i].rt_delay_ms); 2309 firsttime = 0; 2310 } 2311 if (err_segs[i].rt_delay_us != 0) { 2312 pr_cont("%s%ldus", firsttime ? "" : "+", 2313 err_segs[i].rt_delay_us); 2314 firsttime = 0; 2315 } 2316 pr_cont("%s\n", 2317 err_segs[i].rt_preempted ? "preempted" : ""); 2318 2319 } 2320 } 2321 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 2322 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 2323 else if (torture_onoff_failures()) 2324 rcu_torture_print_module_parms(cur_ops, 2325 "End of test: RCU_HOTPLUG"); 2326 else 2327 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 2328 torture_cleanup_end(); 2329 } 2330 2331 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 2332 static void rcu_torture_leak_cb(struct rcu_head *rhp) 2333 { 2334 } 2335 2336 static void rcu_torture_err_cb(struct rcu_head *rhp) 2337 { 2338 /* 2339 * This -might- happen due to race conditions, but is unlikely. 2340 * The scenario that leads to this happening is that the 2341 * first of the pair of duplicate callbacks is queued, 2342 * someone else starts a grace period that includes that 2343 * callback, then the second of the pair must wait for the 2344 * next grace period. Unlikely, but can happen. If it 2345 * does happen, the debug-objects subsystem won't have splatted. 2346 */ 2347 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 2348 } 2349 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2350 2351 /* 2352 * Verify that double-free causes debug-objects to complain, but only 2353 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 2354 * cannot be carried out. 2355 */ 2356 static void rcu_test_debug_objects(void) 2357 { 2358 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD 2359 struct rcu_head rh1; 2360 struct rcu_head rh2; 2361 2362 init_rcu_head_on_stack(&rh1); 2363 init_rcu_head_on_stack(&rh2); 2364 pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME); 2365 2366 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 2367 preempt_disable(); /* Prevent preemption from interrupting test. */ 2368 rcu_read_lock(); /* Make it impossible to finish a grace period. */ 2369 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 2370 local_irq_disable(); /* Make it harder to start a new grace period. */ 2371 call_rcu(&rh2, rcu_torture_leak_cb); 2372 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 2373 local_irq_enable(); 2374 rcu_read_unlock(); 2375 preempt_enable(); 2376 2377 /* Wait for them all to get done so we can safely return. */ 2378 rcu_barrier(); 2379 pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); 2380 destroy_rcu_head_on_stack(&rh1); 2381 destroy_rcu_head_on_stack(&rh2); 2382 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2383 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); 2384 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 2385 } 2386 2387 static void rcutorture_sync(void) 2388 { 2389 static unsigned long n; 2390 2391 if (cur_ops->sync && !(++n & 0xfff)) 2392 cur_ops->sync(); 2393 } 2394 2395 static int __init 2396 rcu_torture_init(void) 2397 { 2398 long i; 2399 int cpu; 2400 int firsterr = 0; 2401 static struct rcu_torture_ops *torture_ops[] = { 2402 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, 2403 &busted_srcud_ops, &tasks_ops, &trivial_ops, 2404 }; 2405 2406 if (!torture_init_begin(torture_type, verbose)) 2407 return -EBUSY; 2408 2409 /* Process args and tell the world that the torturer is on the job. */ 2410 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 2411 cur_ops = torture_ops[i]; 2412 if (strcmp(torture_type, cur_ops->name) == 0) 2413 break; 2414 } 2415 if (i == ARRAY_SIZE(torture_ops)) { 2416 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 2417 torture_type); 2418 pr_alert("rcu-torture types:"); 2419 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 2420 pr_cont(" %s", torture_ops[i]->name); 2421 pr_cont("\n"); 2422 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 2423 firsterr = -EINVAL; 2424 cur_ops = NULL; 2425 goto unwind; 2426 } 2427 if (cur_ops->fqs == NULL && fqs_duration != 0) { 2428 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 2429 fqs_duration = 0; 2430 } 2431 if (cur_ops->init) 2432 cur_ops->init(); 2433 2434 if (nreaders >= 0) { 2435 nrealreaders = nreaders; 2436 } else { 2437 nrealreaders = num_online_cpus() - 2 - nreaders; 2438 if (nrealreaders <= 0) 2439 nrealreaders = 1; 2440 } 2441 rcu_torture_print_module_parms(cur_ops, "Start of test"); 2442 2443 /* Set up the freelist. */ 2444 2445 INIT_LIST_HEAD(&rcu_torture_freelist); 2446 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 2447 rcu_tortures[i].rtort_mbtest = 0; 2448 list_add_tail(&rcu_tortures[i].rtort_free, 2449 &rcu_torture_freelist); 2450 } 2451 2452 /* Initialize the statistics so that each run gets its own numbers. */ 2453 2454 rcu_torture_current = NULL; 2455 rcu_torture_current_version = 0; 2456 atomic_set(&n_rcu_torture_alloc, 0); 2457 atomic_set(&n_rcu_torture_alloc_fail, 0); 2458 atomic_set(&n_rcu_torture_free, 0); 2459 atomic_set(&n_rcu_torture_mberror, 0); 2460 atomic_set(&n_rcu_torture_error, 0); 2461 n_rcu_torture_barrier_error = 0; 2462 n_rcu_torture_boost_ktrerror = 0; 2463 n_rcu_torture_boost_rterror = 0; 2464 n_rcu_torture_boost_failure = 0; 2465 n_rcu_torture_boosts = 0; 2466 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2467 atomic_set(&rcu_torture_wcount[i], 0); 2468 for_each_possible_cpu(cpu) { 2469 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2470 per_cpu(rcu_torture_count, cpu)[i] = 0; 2471 per_cpu(rcu_torture_batch, cpu)[i] = 0; 2472 } 2473 } 2474 err_segs_recorded = 0; 2475 rt_read_nsegs = 0; 2476 2477 /* Start up the kthreads. */ 2478 2479 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 2480 writer_task); 2481 if (firsterr) 2482 goto unwind; 2483 if (nfakewriters > 0) { 2484 fakewriter_tasks = kcalloc(nfakewriters, 2485 sizeof(fakewriter_tasks[0]), 2486 GFP_KERNEL); 2487 if (fakewriter_tasks == NULL) { 2488 VERBOSE_TOROUT_ERRSTRING("out of memory"); 2489 firsterr = -ENOMEM; 2490 goto unwind; 2491 } 2492 } 2493 for (i = 0; i < nfakewriters; i++) { 2494 firsterr = torture_create_kthread(rcu_torture_fakewriter, 2495 NULL, fakewriter_tasks[i]); 2496 if (firsterr) 2497 goto unwind; 2498 } 2499 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 2500 GFP_KERNEL); 2501 if (reader_tasks == NULL) { 2502 VERBOSE_TOROUT_ERRSTRING("out of memory"); 2503 firsterr = -ENOMEM; 2504 goto unwind; 2505 } 2506 for (i = 0; i < nrealreaders; i++) { 2507 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 2508 reader_tasks[i]); 2509 if (firsterr) 2510 goto unwind; 2511 } 2512 if (stat_interval > 0) { 2513 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 2514 stats_task); 2515 if (firsterr) 2516 goto unwind; 2517 } 2518 if (test_no_idle_hz && shuffle_interval > 0) { 2519 firsterr = torture_shuffle_init(shuffle_interval * HZ); 2520 if (firsterr) 2521 goto unwind; 2522 } 2523 if (stutter < 0) 2524 stutter = 0; 2525 if (stutter) { 2526 int t; 2527 2528 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 2529 firsterr = torture_stutter_init(stutter * HZ, t); 2530 if (firsterr) 2531 goto unwind; 2532 } 2533 if (fqs_duration < 0) 2534 fqs_duration = 0; 2535 if (fqs_duration) { 2536 /* Create the fqs thread */ 2537 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 2538 fqs_task); 2539 if (firsterr) 2540 goto unwind; 2541 } 2542 if (test_boost_interval < 1) 2543 test_boost_interval = 1; 2544 if (test_boost_duration < 2) 2545 test_boost_duration = 2; 2546 if (rcu_torture_can_boost()) { 2547 2548 boost_starttime = jiffies + test_boost_interval * HZ; 2549 2550 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 2551 rcutorture_booster_init, 2552 rcutorture_booster_cleanup); 2553 if (firsterr < 0) 2554 goto unwind; 2555 rcutor_hp = firsterr; 2556 } 2557 shutdown_jiffies = jiffies + shutdown_secs * HZ; 2558 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 2559 if (firsterr) 2560 goto unwind; 2561 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 2562 rcutorture_sync); 2563 if (firsterr) 2564 goto unwind; 2565 firsterr = rcu_torture_stall_init(); 2566 if (firsterr) 2567 goto unwind; 2568 firsterr = rcu_torture_fwd_prog_init(); 2569 if (firsterr) 2570 goto unwind; 2571 firsterr = rcu_torture_barrier_init(); 2572 if (firsterr) 2573 goto unwind; 2574 if (object_debug) 2575 rcu_test_debug_objects(); 2576 torture_init_end(); 2577 return 0; 2578 2579 unwind: 2580 torture_init_end(); 2581 rcu_torture_cleanup(); 2582 return firsterr; 2583 } 2584 2585 module_init(rcu_torture_init); 2586 module_exit(rcu_torture_cleanup); 2587