1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.rst 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/rcu_notifier.h> 25 #include <linux/interrupt.h> 26 #include <linux/sched/signal.h> 27 #include <uapi/linux/sched/types.h> 28 #include <linux/atomic.h> 29 #include <linux/bitops.h> 30 #include <linux/completion.h> 31 #include <linux/moduleparam.h> 32 #include <linux/percpu.h> 33 #include <linux/notifier.h> 34 #include <linux/reboot.h> 35 #include <linux/freezer.h> 36 #include <linux/cpu.h> 37 #include <linux/delay.h> 38 #include <linux/stat.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/trace_clock.h> 42 #include <asm/byteorder.h> 43 #include <linux/torture.h> 44 #include <linux/vmalloc.h> 45 #include <linux/sched/debug.h> 46 #include <linux/sched/sysctl.h> 47 #include <linux/oom.h> 48 #include <linux/tick.h> 49 #include <linux/rcupdate_trace.h> 50 #include <linux/nmi.h> 51 52 #include "rcu.h" 53 54 MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility"); 55 MODULE_LICENSE("GPL"); 56 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 57 58 /* Bits for ->extendables field, extendables param, and related definitions. */ 59 #define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */ 60 #define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1) 61 #define RCUTORTURE_RDR_SHIFT_2 16 /* Put SRCU index in upper bits. */ 62 #define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2) 63 #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ 64 #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ 65 #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ 66 #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ 67 #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ 68 #define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */ 69 #define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */ 70 #define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */ 71 #define RCUTORTURE_MAX_EXTEND \ 72 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 73 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) 74 #define RCUTORTURE_RDR_ALLBITS \ 75 (RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \ 76 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2) 77 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 78 /* Must be power of two minus one. */ 79 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 80 81 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 82 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 83 torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); 84 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 85 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 86 torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); 87 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 88 torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); 89 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); 90 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 91 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); 92 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); 93 torture_param(bool, gp_cond_exp_full, false, 94 "Use conditional/async full-stateexpedited GP wait primitives"); 95 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 96 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); 97 torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); 98 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); 99 torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); 100 torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); 101 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 102 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 103 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); 104 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); 105 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 106 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 107 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); 108 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 109 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); 110 torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); 111 torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); 112 torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); 113 torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); 114 torture_param(int, reader_flavor, 0x1, "Reader flavors to use, one per bit."); 115 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 116 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 117 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 118 torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); 119 torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); 120 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 121 torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); 122 torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); 123 torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); 124 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 125 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 126 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 127 torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); 128 torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); 129 torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); 130 torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); 131 torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); 132 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 133 134 static char *torture_type = "rcu"; 135 module_param(torture_type, charp, 0444); 136 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 137 138 static int nrealnocbers; 139 static int nrealreaders; 140 static struct task_struct *writer_task; 141 static struct task_struct **fakewriter_tasks; 142 static struct task_struct **reader_tasks; 143 static struct task_struct **nocb_tasks; 144 static struct task_struct *stats_task; 145 static struct task_struct *fqs_task; 146 static struct task_struct *boost_tasks[NR_CPUS]; 147 static struct task_struct *stall_task; 148 static struct task_struct **fwd_prog_tasks; 149 static struct task_struct **barrier_cbs_tasks; 150 static struct task_struct *barrier_task; 151 static struct task_struct *read_exit_task; 152 153 #define RCU_TORTURE_PIPE_LEN 10 154 155 // Mailbox-like structure to check RCU global memory ordering. 156 struct rcu_torture_reader_check { 157 unsigned long rtc_myloops; 158 int rtc_chkrdr; 159 unsigned long rtc_chkloops; 160 int rtc_ready; 161 struct rcu_torture_reader_check *rtc_assigner; 162 } ____cacheline_internodealigned_in_smp; 163 164 // Update-side data structure used to check RCU readers. 165 struct rcu_torture { 166 struct rcu_head rtort_rcu; 167 int rtort_pipe_count; 168 struct list_head rtort_free; 169 int rtort_mbtest; 170 struct rcu_torture_reader_check *rtort_chkp; 171 }; 172 173 static LIST_HEAD(rcu_torture_freelist); 174 static struct rcu_torture __rcu *rcu_torture_current; 175 static unsigned long rcu_torture_current_version; 176 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 177 static DEFINE_SPINLOCK(rcu_torture_lock); 178 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 179 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 180 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 181 static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; 182 static atomic_t n_rcu_torture_alloc; 183 static atomic_t n_rcu_torture_alloc_fail; 184 static atomic_t n_rcu_torture_free; 185 static atomic_t n_rcu_torture_mberror; 186 static atomic_t n_rcu_torture_mbchk_fail; 187 static atomic_t n_rcu_torture_mbchk_tries; 188 static atomic_t n_rcu_torture_error; 189 static long n_rcu_torture_barrier_error; 190 static long n_rcu_torture_boost_ktrerror; 191 static long n_rcu_torture_boost_failure; 192 static long n_rcu_torture_boosts; 193 static atomic_long_t n_rcu_torture_timers; 194 static long n_barrier_attempts; 195 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 196 static unsigned long n_read_exits; 197 static struct list_head rcu_torture_removed; 198 static unsigned long shutdown_jiffies; 199 static unsigned long start_gp_seq; 200 static atomic_long_t n_nocb_offload; 201 static atomic_long_t n_nocb_deoffload; 202 203 static int rcu_torture_writer_state; 204 #define RTWS_FIXED_DELAY 0 205 #define RTWS_DELAY 1 206 #define RTWS_REPLACE 2 207 #define RTWS_DEF_FREE 3 208 #define RTWS_EXP_SYNC 4 209 #define RTWS_COND_GET 5 210 #define RTWS_COND_GET_FULL 6 211 #define RTWS_COND_GET_EXP 7 212 #define RTWS_COND_GET_EXP_FULL 8 213 #define RTWS_COND_SYNC 9 214 #define RTWS_COND_SYNC_FULL 10 215 #define RTWS_COND_SYNC_EXP 11 216 #define RTWS_COND_SYNC_EXP_FULL 12 217 #define RTWS_POLL_GET 13 218 #define RTWS_POLL_GET_FULL 14 219 #define RTWS_POLL_GET_EXP 15 220 #define RTWS_POLL_GET_EXP_FULL 16 221 #define RTWS_POLL_WAIT 17 222 #define RTWS_POLL_WAIT_FULL 18 223 #define RTWS_POLL_WAIT_EXP 19 224 #define RTWS_POLL_WAIT_EXP_FULL 20 225 #define RTWS_SYNC 21 226 #define RTWS_STUTTER 22 227 #define RTWS_STOPPING 23 228 static const char * const rcu_torture_writer_state_names[] = { 229 "RTWS_FIXED_DELAY", 230 "RTWS_DELAY", 231 "RTWS_REPLACE", 232 "RTWS_DEF_FREE", 233 "RTWS_EXP_SYNC", 234 "RTWS_COND_GET", 235 "RTWS_COND_GET_FULL", 236 "RTWS_COND_GET_EXP", 237 "RTWS_COND_GET_EXP_FULL", 238 "RTWS_COND_SYNC", 239 "RTWS_COND_SYNC_FULL", 240 "RTWS_COND_SYNC_EXP", 241 "RTWS_COND_SYNC_EXP_FULL", 242 "RTWS_POLL_GET", 243 "RTWS_POLL_GET_FULL", 244 "RTWS_POLL_GET_EXP", 245 "RTWS_POLL_GET_EXP_FULL", 246 "RTWS_POLL_WAIT", 247 "RTWS_POLL_WAIT_FULL", 248 "RTWS_POLL_WAIT_EXP", 249 "RTWS_POLL_WAIT_EXP_FULL", 250 "RTWS_SYNC", 251 "RTWS_STUTTER", 252 "RTWS_STOPPING", 253 }; 254 255 /* Record reader segment types and duration for first failing read. */ 256 struct rt_read_seg { 257 int rt_readstate; 258 unsigned long rt_delay_jiffies; 259 unsigned long rt_delay_ms; 260 unsigned long rt_delay_us; 261 bool rt_preempted; 262 }; 263 static int err_segs_recorded; 264 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 265 static int rt_read_nsegs; 266 267 static const char *rcu_torture_writer_state_getname(void) 268 { 269 unsigned int i = READ_ONCE(rcu_torture_writer_state); 270 271 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 272 return "???"; 273 return rcu_torture_writer_state_names[i]; 274 } 275 276 #ifdef CONFIG_RCU_TRACE 277 static u64 notrace rcu_trace_clock_local(void) 278 { 279 u64 ts = trace_clock_local(); 280 281 (void)do_div(ts, NSEC_PER_USEC); 282 return ts; 283 } 284 #else /* #ifdef CONFIG_RCU_TRACE */ 285 static u64 notrace rcu_trace_clock_local(void) 286 { 287 return 0ULL; 288 } 289 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 290 291 /* 292 * Stop aggressive CPU-hog tests a bit before the end of the test in order 293 * to avoid interfering with test shutdown. 294 */ 295 static bool shutdown_time_arrived(void) 296 { 297 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 298 } 299 300 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 301 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 302 /* and boost task create/destroy. */ 303 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 304 static bool barrier_phase; /* Test phase. */ 305 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 306 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 307 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 308 309 static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 310 311 /* 312 * Allocate an element from the rcu_tortures pool. 313 */ 314 static struct rcu_torture * 315 rcu_torture_alloc(void) 316 { 317 struct list_head *p; 318 319 spin_lock_bh(&rcu_torture_lock); 320 if (list_empty(&rcu_torture_freelist)) { 321 atomic_inc(&n_rcu_torture_alloc_fail); 322 spin_unlock_bh(&rcu_torture_lock); 323 return NULL; 324 } 325 atomic_inc(&n_rcu_torture_alloc); 326 p = rcu_torture_freelist.next; 327 list_del_init(p); 328 spin_unlock_bh(&rcu_torture_lock); 329 return container_of(p, struct rcu_torture, rtort_free); 330 } 331 332 /* 333 * Free an element to the rcu_tortures pool. 334 */ 335 static void 336 rcu_torture_free(struct rcu_torture *p) 337 { 338 atomic_inc(&n_rcu_torture_free); 339 spin_lock_bh(&rcu_torture_lock); 340 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 341 spin_unlock_bh(&rcu_torture_lock); 342 } 343 344 /* 345 * Operations vector for selecting different types of tests. 346 */ 347 348 struct rcu_torture_ops { 349 int ttype; 350 void (*init)(void); 351 void (*cleanup)(void); 352 int (*readlock)(void); 353 void (*read_delay)(struct torture_random_state *rrsp, 354 struct rt_read_seg *rtrsp); 355 void (*readunlock)(int idx); 356 int (*readlock_held)(void); 357 unsigned long (*get_gp_seq)(void); 358 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 359 void (*deferred_free)(struct rcu_torture *p); 360 void (*sync)(void); 361 void (*exp_sync)(void); 362 unsigned long (*get_gp_state_exp)(void); 363 unsigned long (*start_gp_poll_exp)(void); 364 void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); 365 bool (*poll_gp_state_exp)(unsigned long oldstate); 366 void (*cond_sync_exp)(unsigned long oldstate); 367 void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); 368 unsigned long (*get_comp_state)(void); 369 void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); 370 bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); 371 bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); 372 unsigned long (*get_gp_state)(void); 373 void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); 374 unsigned long (*start_gp_poll)(void); 375 void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); 376 bool (*poll_gp_state)(unsigned long oldstate); 377 bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); 378 bool (*poll_need_2gp)(bool poll, bool poll_full); 379 void (*cond_sync)(unsigned long oldstate); 380 void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); 381 int poll_active; 382 int poll_active_full; 383 call_rcu_func_t call; 384 void (*cb_barrier)(void); 385 void (*fqs)(void); 386 void (*stats)(void); 387 void (*gp_kthread_dbg)(void); 388 bool (*check_boost_failed)(unsigned long gp_state, int *cpup); 389 int (*stall_dur)(void); 390 void (*get_gp_data)(int *flags, unsigned long *gp_seq); 391 void (*gp_slow_register)(atomic_t *rgssp); 392 void (*gp_slow_unregister)(atomic_t *rgssp); 393 long cbflood_max; 394 int irq_capable; 395 int can_boost; 396 int extendables; 397 int slow_gps; 398 int no_pi_lock; 399 int debug_objects; 400 int start_poll_irqsoff; 401 const char *name; 402 }; 403 404 static struct rcu_torture_ops *cur_ops; 405 406 /* 407 * Definitions for rcu torture testing. 408 */ 409 410 static int torture_readlock_not_held(void) 411 { 412 return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); 413 } 414 415 static int rcu_torture_read_lock(void) 416 { 417 rcu_read_lock(); 418 return 0; 419 } 420 421 static void 422 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 423 { 424 unsigned long started; 425 unsigned long completed; 426 const unsigned long shortdelay_us = 200; 427 unsigned long longdelay_ms = 300; 428 unsigned long long ts; 429 430 /* We want a short delay sometimes to make a reader delay the grace 431 * period, and we want a long delay occasionally to trigger 432 * force_quiescent_state. */ 433 434 if (!atomic_read(&rcu_fwd_cb_nodelay) && 435 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 436 started = cur_ops->get_gp_seq(); 437 ts = rcu_trace_clock_local(); 438 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) 439 longdelay_ms = 5; /* Avoid triggering BH limits. */ 440 mdelay(longdelay_ms); 441 rtrsp->rt_delay_ms = longdelay_ms; 442 completed = cur_ops->get_gp_seq(); 443 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 444 started, completed); 445 } 446 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 447 udelay(shortdelay_us); 448 rtrsp->rt_delay_us = shortdelay_us; 449 } 450 if (!preempt_count() && 451 !(torture_random(rrsp) % (nrealreaders * 500))) { 452 torture_preempt_schedule(); /* QS only if preemptible. */ 453 rtrsp->rt_preempted = true; 454 } 455 } 456 457 static void rcu_torture_read_unlock(int idx) 458 { 459 rcu_read_unlock(); 460 } 461 462 /* 463 * Update callback in the pipe. This should be invoked after a grace period. 464 */ 465 static bool 466 rcu_torture_pipe_update_one(struct rcu_torture *rp) 467 { 468 int i; 469 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); 470 471 if (rtrcp) { 472 WRITE_ONCE(rp->rtort_chkp, NULL); 473 smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). 474 } 475 i = rp->rtort_pipe_count; 476 if (i > RCU_TORTURE_PIPE_LEN) 477 i = RCU_TORTURE_PIPE_LEN; 478 atomic_inc(&rcu_torture_wcount[i]); 479 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 480 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 481 if (i + 1 >= RCU_TORTURE_PIPE_LEN) { 482 rp->rtort_mbtest = 0; 483 return true; 484 } 485 return false; 486 } 487 488 /* 489 * Update all callbacks in the pipe. Suitable for synchronous grace-period 490 * primitives. 491 */ 492 static void 493 rcu_torture_pipe_update(struct rcu_torture *old_rp) 494 { 495 struct rcu_torture *rp; 496 struct rcu_torture *rp1; 497 498 if (old_rp) 499 list_add(&old_rp->rtort_free, &rcu_torture_removed); 500 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 501 if (rcu_torture_pipe_update_one(rp)) { 502 list_del(&rp->rtort_free); 503 rcu_torture_free(rp); 504 } 505 } 506 } 507 508 static void 509 rcu_torture_cb(struct rcu_head *p) 510 { 511 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 512 513 if (torture_must_stop_irq()) { 514 /* Test is ending, just drop callbacks on the floor. */ 515 /* The next initialization will pick up the pieces. */ 516 return; 517 } 518 if (rcu_torture_pipe_update_one(rp)) 519 rcu_torture_free(rp); 520 else 521 cur_ops->deferred_free(rp); 522 } 523 524 static unsigned long rcu_no_completed(void) 525 { 526 return 0; 527 } 528 529 static void rcu_torture_deferred_free(struct rcu_torture *p) 530 { 531 call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); 532 } 533 534 static void rcu_sync_torture_init(void) 535 { 536 INIT_LIST_HEAD(&rcu_torture_removed); 537 } 538 539 static bool rcu_poll_need_2gp(bool poll, bool poll_full) 540 { 541 return poll; 542 } 543 544 static struct rcu_torture_ops rcu_ops = { 545 .ttype = RCU_FLAVOR, 546 .init = rcu_sync_torture_init, 547 .readlock = rcu_torture_read_lock, 548 .read_delay = rcu_read_delay, 549 .readunlock = rcu_torture_read_unlock, 550 .readlock_held = torture_readlock_not_held, 551 .get_gp_seq = rcu_get_gp_seq, 552 .gp_diff = rcu_seq_diff, 553 .deferred_free = rcu_torture_deferred_free, 554 .sync = synchronize_rcu, 555 .exp_sync = synchronize_rcu_expedited, 556 .same_gp_state = same_state_synchronize_rcu, 557 .same_gp_state_full = same_state_synchronize_rcu_full, 558 .get_comp_state = get_completed_synchronize_rcu, 559 .get_comp_state_full = get_completed_synchronize_rcu_full, 560 .get_gp_state = get_state_synchronize_rcu, 561 .get_gp_state_full = get_state_synchronize_rcu_full, 562 .start_gp_poll = start_poll_synchronize_rcu, 563 .start_gp_poll_full = start_poll_synchronize_rcu_full, 564 .poll_gp_state = poll_state_synchronize_rcu, 565 .poll_gp_state_full = poll_state_synchronize_rcu_full, 566 .poll_need_2gp = rcu_poll_need_2gp, 567 .cond_sync = cond_synchronize_rcu, 568 .cond_sync_full = cond_synchronize_rcu_full, 569 .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE, 570 .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, 571 .get_gp_state_exp = get_state_synchronize_rcu, 572 .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, 573 .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, 574 .poll_gp_state_exp = poll_state_synchronize_rcu, 575 .cond_sync_exp = cond_synchronize_rcu_expedited, 576 .call = call_rcu_hurry, 577 .cb_barrier = rcu_barrier, 578 .fqs = rcu_force_quiescent_state, 579 .gp_kthread_dbg = show_rcu_gp_kthreads, 580 .check_boost_failed = rcu_check_boost_fail, 581 .stall_dur = rcu_jiffies_till_stall_check, 582 .get_gp_data = rcutorture_get_gp_data, 583 .gp_slow_register = rcu_gp_slow_register, 584 .gp_slow_unregister = rcu_gp_slow_unregister, 585 .irq_capable = 1, 586 .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), 587 .extendables = RCUTORTURE_MAX_EXTEND, 588 .debug_objects = 1, 589 .start_poll_irqsoff = 1, 590 .name = "rcu" 591 }; 592 593 /* 594 * Don't even think about trying any of these in real life!!! 595 * The names includes "busted", and they really means it! 596 * The only purpose of these functions is to provide a buggy RCU 597 * implementation to make sure that rcutorture correctly emits 598 * buggy-RCU error messages. 599 */ 600 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 601 { 602 /* This is a deliberate bug for testing purposes only! */ 603 rcu_torture_cb(&p->rtort_rcu); 604 } 605 606 static void synchronize_rcu_busted(void) 607 { 608 /* This is a deliberate bug for testing purposes only! */ 609 } 610 611 static void 612 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 613 { 614 /* This is a deliberate bug for testing purposes only! */ 615 func(head); 616 } 617 618 static struct rcu_torture_ops rcu_busted_ops = { 619 .ttype = INVALID_RCU_FLAVOR, 620 .init = rcu_sync_torture_init, 621 .readlock = rcu_torture_read_lock, 622 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 623 .readunlock = rcu_torture_read_unlock, 624 .readlock_held = torture_readlock_not_held, 625 .get_gp_seq = rcu_no_completed, 626 .deferred_free = rcu_busted_torture_deferred_free, 627 .sync = synchronize_rcu_busted, 628 .exp_sync = synchronize_rcu_busted, 629 .call = call_rcu_busted, 630 .irq_capable = 1, 631 .name = "busted" 632 }; 633 634 /* 635 * Definitions for srcu torture testing. 636 */ 637 638 DEFINE_STATIC_SRCU(srcu_ctl); 639 static struct srcu_struct srcu_ctld; 640 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 641 static struct rcu_torture_ops srcud_ops; 642 643 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) 644 { 645 srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq); 646 } 647 648 static int srcu_torture_read_lock(void) 649 { 650 int idx; 651 int ret = 0; 652 653 if ((reader_flavor & 0x1) || !(reader_flavor & 0x7)) { 654 idx = srcu_read_lock(srcu_ctlp); 655 WARN_ON_ONCE(idx & ~0x1); 656 ret += idx; 657 } 658 if (reader_flavor & 0x2) { 659 idx = srcu_read_lock_nmisafe(srcu_ctlp); 660 WARN_ON_ONCE(idx & ~0x1); 661 ret += idx << 1; 662 } 663 if (reader_flavor & 0x4) { 664 idx = srcu_read_lock_lite(srcu_ctlp); 665 WARN_ON_ONCE(idx & ~0x1); 666 ret += idx << 2; 667 } 668 return ret; 669 } 670 671 static void 672 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 673 { 674 long delay; 675 const long uspertick = 1000000 / HZ; 676 const long longdelay = 10; 677 678 /* We want there to be long-running readers, but not all the time. */ 679 680 delay = torture_random(rrsp) % 681 (nrealreaders * 2 * longdelay * uspertick); 682 if (!delay && in_task()) { 683 schedule_timeout_interruptible(longdelay); 684 rtrsp->rt_delay_jiffies = longdelay; 685 } else { 686 rcu_read_delay(rrsp, rtrsp); 687 } 688 } 689 690 static void srcu_torture_read_unlock(int idx) 691 { 692 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 693 if (reader_flavor & 0x4) 694 srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2); 695 if (reader_flavor & 0x2) 696 srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1); 697 if ((reader_flavor & 0x1) || !(reader_flavor & 0x7)) 698 srcu_read_unlock(srcu_ctlp, idx & 0x1); 699 } 700 701 static int torture_srcu_read_lock_held(void) 702 { 703 return srcu_read_lock_held(srcu_ctlp); 704 } 705 706 static unsigned long srcu_torture_completed(void) 707 { 708 return srcu_batches_completed(srcu_ctlp); 709 } 710 711 static void srcu_torture_deferred_free(struct rcu_torture *rp) 712 { 713 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 714 } 715 716 static void srcu_torture_synchronize(void) 717 { 718 synchronize_srcu(srcu_ctlp); 719 } 720 721 static unsigned long srcu_torture_get_gp_state(void) 722 { 723 return get_state_synchronize_srcu(srcu_ctlp); 724 } 725 726 static unsigned long srcu_torture_start_gp_poll(void) 727 { 728 return start_poll_synchronize_srcu(srcu_ctlp); 729 } 730 731 static bool srcu_torture_poll_gp_state(unsigned long oldstate) 732 { 733 return poll_state_synchronize_srcu(srcu_ctlp, oldstate); 734 } 735 736 static void srcu_torture_call(struct rcu_head *head, 737 rcu_callback_t func) 738 { 739 call_srcu(srcu_ctlp, head, func); 740 } 741 742 static void srcu_torture_barrier(void) 743 { 744 srcu_barrier(srcu_ctlp); 745 } 746 747 static void srcu_torture_stats(void) 748 { 749 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 750 } 751 752 static void srcu_torture_synchronize_expedited(void) 753 { 754 synchronize_srcu_expedited(srcu_ctlp); 755 } 756 757 static struct rcu_torture_ops srcu_ops = { 758 .ttype = SRCU_FLAVOR, 759 .init = rcu_sync_torture_init, 760 .readlock = srcu_torture_read_lock, 761 .read_delay = srcu_read_delay, 762 .readunlock = srcu_torture_read_unlock, 763 .readlock_held = torture_srcu_read_lock_held, 764 .get_gp_seq = srcu_torture_completed, 765 .deferred_free = srcu_torture_deferred_free, 766 .sync = srcu_torture_synchronize, 767 .exp_sync = srcu_torture_synchronize_expedited, 768 .same_gp_state = same_state_synchronize_srcu, 769 .get_comp_state = get_completed_synchronize_srcu, 770 .get_gp_state = srcu_torture_get_gp_state, 771 .start_gp_poll = srcu_torture_start_gp_poll, 772 .poll_gp_state = srcu_torture_poll_gp_state, 773 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 774 .call = srcu_torture_call, 775 .cb_barrier = srcu_torture_barrier, 776 .stats = srcu_torture_stats, 777 .get_gp_data = srcu_get_gp_data, 778 .cbflood_max = 50000, 779 .irq_capable = 1, 780 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 781 .debug_objects = 1, 782 .name = "srcu" 783 }; 784 785 static void srcu_torture_init(void) 786 { 787 rcu_sync_torture_init(); 788 WARN_ON(init_srcu_struct(&srcu_ctld)); 789 srcu_ctlp = &srcu_ctld; 790 } 791 792 static void srcu_torture_cleanup(void) 793 { 794 cleanup_srcu_struct(&srcu_ctld); 795 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 796 } 797 798 /* As above, but dynamically allocated. */ 799 static struct rcu_torture_ops srcud_ops = { 800 .ttype = SRCU_FLAVOR, 801 .init = srcu_torture_init, 802 .cleanup = srcu_torture_cleanup, 803 .readlock = srcu_torture_read_lock, 804 .read_delay = srcu_read_delay, 805 .readunlock = srcu_torture_read_unlock, 806 .readlock_held = torture_srcu_read_lock_held, 807 .get_gp_seq = srcu_torture_completed, 808 .deferred_free = srcu_torture_deferred_free, 809 .sync = srcu_torture_synchronize, 810 .exp_sync = srcu_torture_synchronize_expedited, 811 .same_gp_state = same_state_synchronize_srcu, 812 .get_comp_state = get_completed_synchronize_srcu, 813 .get_gp_state = srcu_torture_get_gp_state, 814 .start_gp_poll = srcu_torture_start_gp_poll, 815 .poll_gp_state = srcu_torture_poll_gp_state, 816 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 817 .call = srcu_torture_call, 818 .cb_barrier = srcu_torture_barrier, 819 .stats = srcu_torture_stats, 820 .get_gp_data = srcu_get_gp_data, 821 .cbflood_max = 50000, 822 .irq_capable = 1, 823 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 824 .debug_objects = 1, 825 .name = "srcud" 826 }; 827 828 /* As above, but broken due to inappropriate reader extension. */ 829 static struct rcu_torture_ops busted_srcud_ops = { 830 .ttype = SRCU_FLAVOR, 831 .init = srcu_torture_init, 832 .cleanup = srcu_torture_cleanup, 833 .readlock = srcu_torture_read_lock, 834 .read_delay = rcu_read_delay, 835 .readunlock = srcu_torture_read_unlock, 836 .readlock_held = torture_srcu_read_lock_held, 837 .get_gp_seq = srcu_torture_completed, 838 .deferred_free = srcu_torture_deferred_free, 839 .sync = srcu_torture_synchronize, 840 .exp_sync = srcu_torture_synchronize_expedited, 841 .call = srcu_torture_call, 842 .cb_barrier = srcu_torture_barrier, 843 .stats = srcu_torture_stats, 844 .irq_capable = 1, 845 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 846 .extendables = RCUTORTURE_MAX_EXTEND, 847 .name = "busted_srcud" 848 }; 849 850 /* 851 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 852 * This implementation does not necessarily work well with CPU hotplug. 853 */ 854 855 static void synchronize_rcu_trivial(void) 856 { 857 int cpu; 858 859 for_each_online_cpu(cpu) { 860 torture_sched_setaffinity(current->pid, cpumask_of(cpu)); 861 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 862 } 863 } 864 865 static int rcu_torture_read_lock_trivial(void) 866 { 867 preempt_disable(); 868 return 0; 869 } 870 871 static void rcu_torture_read_unlock_trivial(int idx) 872 { 873 preempt_enable(); 874 } 875 876 static struct rcu_torture_ops trivial_ops = { 877 .ttype = RCU_TRIVIAL_FLAVOR, 878 .init = rcu_sync_torture_init, 879 .readlock = rcu_torture_read_lock_trivial, 880 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 881 .readunlock = rcu_torture_read_unlock_trivial, 882 .readlock_held = torture_readlock_not_held, 883 .get_gp_seq = rcu_no_completed, 884 .sync = synchronize_rcu_trivial, 885 .exp_sync = synchronize_rcu_trivial, 886 .irq_capable = 1, 887 .name = "trivial" 888 }; 889 890 #ifdef CONFIG_TASKS_RCU 891 892 /* 893 * Definitions for RCU-tasks torture testing. 894 */ 895 896 static int tasks_torture_read_lock(void) 897 { 898 return 0; 899 } 900 901 static void tasks_torture_read_unlock(int idx) 902 { 903 } 904 905 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 906 { 907 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 908 } 909 910 static void synchronize_rcu_mult_test(void) 911 { 912 synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); 913 } 914 915 static struct rcu_torture_ops tasks_ops = { 916 .ttype = RCU_TASKS_FLAVOR, 917 .init = rcu_sync_torture_init, 918 .readlock = tasks_torture_read_lock, 919 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 920 .readunlock = tasks_torture_read_unlock, 921 .get_gp_seq = rcu_no_completed, 922 .deferred_free = rcu_tasks_torture_deferred_free, 923 .sync = synchronize_rcu_tasks, 924 .exp_sync = synchronize_rcu_mult_test, 925 .call = call_rcu_tasks, 926 .cb_barrier = rcu_barrier_tasks, 927 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, 928 .get_gp_data = rcu_tasks_get_gp_data, 929 .irq_capable = 1, 930 .slow_gps = 1, 931 .name = "tasks" 932 }; 933 934 #define TASKS_OPS &tasks_ops, 935 936 #else // #ifdef CONFIG_TASKS_RCU 937 938 #define TASKS_OPS 939 940 #endif // #else #ifdef CONFIG_TASKS_RCU 941 942 943 #ifdef CONFIG_TASKS_RUDE_RCU 944 945 /* 946 * Definitions for rude RCU-tasks torture testing. 947 */ 948 949 static struct rcu_torture_ops tasks_rude_ops = { 950 .ttype = RCU_TASKS_RUDE_FLAVOR, 951 .init = rcu_sync_torture_init, 952 .readlock = rcu_torture_read_lock_trivial, 953 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 954 .readunlock = rcu_torture_read_unlock_trivial, 955 .get_gp_seq = rcu_no_completed, 956 .sync = synchronize_rcu_tasks_rude, 957 .exp_sync = synchronize_rcu_tasks_rude, 958 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, 959 .get_gp_data = rcu_tasks_rude_get_gp_data, 960 .cbflood_max = 50000, 961 .irq_capable = 1, 962 .name = "tasks-rude" 963 }; 964 965 #define TASKS_RUDE_OPS &tasks_rude_ops, 966 967 #else // #ifdef CONFIG_TASKS_RUDE_RCU 968 969 #define TASKS_RUDE_OPS 970 971 #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU 972 973 974 #ifdef CONFIG_TASKS_TRACE_RCU 975 976 /* 977 * Definitions for tracing RCU-tasks torture testing. 978 */ 979 980 static int tasks_tracing_torture_read_lock(void) 981 { 982 rcu_read_lock_trace(); 983 return 0; 984 } 985 986 static void tasks_tracing_torture_read_unlock(int idx) 987 { 988 rcu_read_unlock_trace(); 989 } 990 991 static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) 992 { 993 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); 994 } 995 996 static struct rcu_torture_ops tasks_tracing_ops = { 997 .ttype = RCU_TASKS_TRACING_FLAVOR, 998 .init = rcu_sync_torture_init, 999 .readlock = tasks_tracing_torture_read_lock, 1000 .read_delay = srcu_read_delay, /* just reuse srcu's version. */ 1001 .readunlock = tasks_tracing_torture_read_unlock, 1002 .readlock_held = rcu_read_lock_trace_held, 1003 .get_gp_seq = rcu_no_completed, 1004 .deferred_free = rcu_tasks_tracing_torture_deferred_free, 1005 .sync = synchronize_rcu_tasks_trace, 1006 .exp_sync = synchronize_rcu_tasks_trace, 1007 .call = call_rcu_tasks_trace, 1008 .cb_barrier = rcu_barrier_tasks_trace, 1009 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, 1010 .get_gp_data = rcu_tasks_trace_get_gp_data, 1011 .cbflood_max = 50000, 1012 .irq_capable = 1, 1013 .slow_gps = 1, 1014 .name = "tasks-tracing" 1015 }; 1016 1017 #define TASKS_TRACING_OPS &tasks_tracing_ops, 1018 1019 #else // #ifdef CONFIG_TASKS_TRACE_RCU 1020 1021 #define TASKS_TRACING_OPS 1022 1023 #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU 1024 1025 1026 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 1027 { 1028 if (!cur_ops->gp_diff) 1029 return new - old; 1030 return cur_ops->gp_diff(new, old); 1031 } 1032 1033 /* 1034 * RCU torture priority-boost testing. Runs one real-time thread per 1035 * CPU for moderate bursts, repeatedly starting grace periods and waiting 1036 * for them to complete. If a given grace period takes too long, we assume 1037 * that priority inversion has occurred. 1038 */ 1039 1040 static int old_rt_runtime = -1; 1041 1042 static void rcu_torture_disable_rt_throttle(void) 1043 { 1044 /* 1045 * Disable RT throttling so that rcutorture's boost threads don't get 1046 * throttled. Only possible if rcutorture is built-in otherwise the 1047 * user should manually do this by setting the sched_rt_period_us and 1048 * sched_rt_runtime sysctls. 1049 */ 1050 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 1051 return; 1052 1053 old_rt_runtime = sysctl_sched_rt_runtime; 1054 sysctl_sched_rt_runtime = -1; 1055 } 1056 1057 static void rcu_torture_enable_rt_throttle(void) 1058 { 1059 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 1060 return; 1061 1062 sysctl_sched_rt_runtime = old_rt_runtime; 1063 old_rt_runtime = -1; 1064 } 1065 1066 static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) 1067 { 1068 int cpu; 1069 static int dbg_done; 1070 unsigned long end = jiffies; 1071 bool gp_done; 1072 unsigned long j; 1073 static unsigned long last_persist; 1074 unsigned long lp; 1075 unsigned long mininterval = test_boost_duration * HZ - HZ / 2; 1076 1077 if (end - *start > mininterval) { 1078 // Recheck after checking time to avoid false positives. 1079 smp_mb(); // Time check before grace-period check. 1080 if (cur_ops->poll_gp_state(gp_state)) 1081 return false; // passed, though perhaps just barely 1082 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { 1083 // At most one persisted message per boost test. 1084 j = jiffies; 1085 lp = READ_ONCE(last_persist); 1086 if (time_after(j, lp + mininterval) && 1087 cmpxchg(&last_persist, lp, j) == lp) { 1088 if (cpu < 0) 1089 pr_info("Boost inversion persisted: QS from all CPUs\n"); 1090 else 1091 pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); 1092 } 1093 return false; // passed on a technicality 1094 } 1095 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 1096 n_rcu_torture_boost_failure++; 1097 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { 1098 pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", 1099 current->rt_priority, gp_state, end - *start); 1100 cur_ops->gp_kthread_dbg(); 1101 // Recheck after print to flag grace period ending during splat. 1102 gp_done = cur_ops->poll_gp_state(gp_state); 1103 pr_info("Boost inversion: GP %lu %s.\n", gp_state, 1104 gp_done ? "ended already" : "still pending"); 1105 1106 } 1107 1108 return true; // failed 1109 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { 1110 *start = jiffies; 1111 } 1112 1113 return false; // passed 1114 } 1115 1116 static int rcu_torture_boost(void *arg) 1117 { 1118 unsigned long endtime; 1119 unsigned long gp_state; 1120 unsigned long gp_state_time; 1121 unsigned long oldstarttime; 1122 1123 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 1124 1125 /* Set real-time priority. */ 1126 sched_set_fifo_low(current); 1127 1128 /* Each pass through the following loop does one boost-test cycle. */ 1129 do { 1130 bool failed = false; // Test failed already in this test interval 1131 bool gp_initiated = false; 1132 1133 if (kthread_should_stop()) 1134 goto checkwait; 1135 1136 /* Wait for the next test interval. */ 1137 oldstarttime = READ_ONCE(boost_starttime); 1138 while (time_before(jiffies, oldstarttime)) { 1139 schedule_timeout_interruptible(oldstarttime - jiffies); 1140 if (stutter_wait("rcu_torture_boost")) 1141 sched_set_fifo_low(current); 1142 if (torture_must_stop()) 1143 goto checkwait; 1144 } 1145 1146 // Do one boost-test interval. 1147 endtime = oldstarttime + test_boost_duration * HZ; 1148 while (time_before(jiffies, endtime)) { 1149 // Has current GP gone too long? 1150 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1151 failed = rcu_torture_boost_failed(gp_state, &gp_state_time); 1152 // If we don't have a grace period in flight, start one. 1153 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { 1154 gp_state = cur_ops->start_gp_poll(); 1155 gp_initiated = true; 1156 gp_state_time = jiffies; 1157 } 1158 if (stutter_wait("rcu_torture_boost")) { 1159 sched_set_fifo_low(current); 1160 // If the grace period already ended, 1161 // we don't know when that happened, so 1162 // start over. 1163 if (cur_ops->poll_gp_state(gp_state)) 1164 gp_initiated = false; 1165 } 1166 if (torture_must_stop()) 1167 goto checkwait; 1168 } 1169 1170 // In case the grace period extended beyond the end of the loop. 1171 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1172 rcu_torture_boost_failed(gp_state, &gp_state_time); 1173 1174 /* 1175 * Set the start time of the next test interval. 1176 * Yes, this is vulnerable to long delays, but such 1177 * delays simply cause a false negative for the next 1178 * interval. Besides, we are running at RT priority, 1179 * so delays should be relatively rare. 1180 */ 1181 while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { 1182 if (mutex_trylock(&boost_mutex)) { 1183 if (oldstarttime == boost_starttime) { 1184 WRITE_ONCE(boost_starttime, 1185 jiffies + test_boost_interval * HZ); 1186 n_rcu_torture_boosts++; 1187 } 1188 mutex_unlock(&boost_mutex); 1189 break; 1190 } 1191 schedule_timeout_uninterruptible(HZ / 20); 1192 } 1193 1194 /* Go do the stutter. */ 1195 checkwait: if (stutter_wait("rcu_torture_boost")) 1196 sched_set_fifo_low(current); 1197 } while (!torture_must_stop()); 1198 1199 /* Clean up and exit. */ 1200 while (!kthread_should_stop()) { 1201 torture_shutdown_absorb("rcu_torture_boost"); 1202 schedule_timeout_uninterruptible(HZ / 20); 1203 } 1204 torture_kthread_stopping("rcu_torture_boost"); 1205 return 0; 1206 } 1207 1208 /* 1209 * RCU torture force-quiescent-state kthread. Repeatedly induces 1210 * bursts of calls to force_quiescent_state(), increasing the probability 1211 * of occurrence of some important types of race conditions. 1212 */ 1213 static int 1214 rcu_torture_fqs(void *arg) 1215 { 1216 unsigned long fqs_resume_time; 1217 int fqs_burst_remaining; 1218 int oldnice = task_nice(current); 1219 1220 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 1221 do { 1222 fqs_resume_time = jiffies + fqs_stutter * HZ; 1223 while (time_before(jiffies, fqs_resume_time) && 1224 !kthread_should_stop()) { 1225 schedule_timeout_interruptible(HZ / 20); 1226 } 1227 fqs_burst_remaining = fqs_duration; 1228 while (fqs_burst_remaining > 0 && 1229 !kthread_should_stop()) { 1230 cur_ops->fqs(); 1231 udelay(fqs_holdoff); 1232 fqs_burst_remaining -= fqs_holdoff; 1233 } 1234 if (stutter_wait("rcu_torture_fqs")) 1235 sched_set_normal(current, oldnice); 1236 } while (!torture_must_stop()); 1237 torture_kthread_stopping("rcu_torture_fqs"); 1238 return 0; 1239 } 1240 1241 // Used by writers to randomly choose from the available grace-period primitives. 1242 static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; 1243 static int nsynctypes; 1244 1245 /* 1246 * Determine which grace-period primitives are available. 1247 */ 1248 static void rcu_torture_write_types(void) 1249 { 1250 bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; 1251 bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; 1252 bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; 1253 bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; 1254 1255 /* Initialize synctype[] array. If none set, take default. */ 1256 if (!gp_cond1 && 1257 !gp_cond_exp1 && 1258 !gp_cond_full1 && 1259 !gp_cond_exp_full1 && 1260 !gp_exp1 && 1261 !gp_poll_exp1 && 1262 !gp_poll_exp_full1 && 1263 !gp_normal1 && 1264 !gp_poll1 && 1265 !gp_poll_full1 && 1266 !gp_sync1) { 1267 gp_cond1 = true; 1268 gp_cond_exp1 = true; 1269 gp_cond_full1 = true; 1270 gp_cond_exp_full1 = true; 1271 gp_exp1 = true; 1272 gp_poll_exp1 = true; 1273 gp_poll_exp_full1 = true; 1274 gp_normal1 = true; 1275 gp_poll1 = true; 1276 gp_poll_full1 = true; 1277 gp_sync1 = true; 1278 } 1279 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { 1280 synctype[nsynctypes++] = RTWS_COND_GET; 1281 pr_info("%s: Testing conditional GPs.\n", __func__); 1282 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { 1283 pr_alert("%s: gp_cond without primitives.\n", __func__); 1284 } 1285 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { 1286 synctype[nsynctypes++] = RTWS_COND_GET_EXP; 1287 pr_info("%s: Testing conditional expedited GPs.\n", __func__); 1288 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { 1289 pr_alert("%s: gp_cond_exp without primitives.\n", __func__); 1290 } 1291 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { 1292 synctype[nsynctypes++] = RTWS_COND_GET_FULL; 1293 pr_info("%s: Testing conditional full-state GPs.\n", __func__); 1294 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { 1295 pr_alert("%s: gp_cond_full without primitives.\n", __func__); 1296 } 1297 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { 1298 synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; 1299 pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); 1300 } else if (gp_cond_exp_full && 1301 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { 1302 pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); 1303 } 1304 if (gp_exp1 && cur_ops->exp_sync) { 1305 synctype[nsynctypes++] = RTWS_EXP_SYNC; 1306 pr_info("%s: Testing expedited GPs.\n", __func__); 1307 } else if (gp_exp && !cur_ops->exp_sync) { 1308 pr_alert("%s: gp_exp without primitives.\n", __func__); 1309 } 1310 if (gp_normal1 && cur_ops->deferred_free) { 1311 synctype[nsynctypes++] = RTWS_DEF_FREE; 1312 pr_info("%s: Testing asynchronous GPs.\n", __func__); 1313 } else if (gp_normal && !cur_ops->deferred_free) { 1314 pr_alert("%s: gp_normal without primitives.\n", __func__); 1315 } 1316 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state && 1317 cur_ops->start_gp_poll && cur_ops->poll_gp_state) { 1318 synctype[nsynctypes++] = RTWS_POLL_GET; 1319 pr_info("%s: Testing polling GPs.\n", __func__); 1320 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { 1321 pr_alert("%s: gp_poll without primitives.\n", __func__); 1322 } 1323 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full 1324 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { 1325 synctype[nsynctypes++] = RTWS_POLL_GET_FULL; 1326 pr_info("%s: Testing polling full-state GPs.\n", __func__); 1327 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { 1328 pr_alert("%s: gp_poll_full without primitives.\n", __func__); 1329 } 1330 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { 1331 synctype[nsynctypes++] = RTWS_POLL_GET_EXP; 1332 pr_info("%s: Testing polling expedited GPs.\n", __func__); 1333 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { 1334 pr_alert("%s: gp_poll_exp without primitives.\n", __func__); 1335 } 1336 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { 1337 synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; 1338 pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); 1339 } else if (gp_poll_exp_full && 1340 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { 1341 pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); 1342 } 1343 if (gp_sync1 && cur_ops->sync) { 1344 synctype[nsynctypes++] = RTWS_SYNC; 1345 pr_info("%s: Testing normal GPs.\n", __func__); 1346 } else if (gp_sync && !cur_ops->sync) { 1347 pr_alert("%s: gp_sync without primitives.\n", __func__); 1348 } 1349 pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes); 1350 } 1351 1352 /* 1353 * Do the specified rcu_torture_writer() synchronous grace period, 1354 * while also testing out the polled APIs. Note well that the single-CPU 1355 * grace-period optimizations must be accounted for. 1356 */ 1357 static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) 1358 { 1359 unsigned long cookie; 1360 struct rcu_gp_oldstate cookie_full; 1361 bool dopoll; 1362 bool dopoll_full; 1363 unsigned long r = torture_random(trsp); 1364 1365 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); 1366 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); 1367 if (dopoll || dopoll_full) 1368 cpus_read_lock(); 1369 if (dopoll) 1370 cookie = cur_ops->get_gp_state(); 1371 if (dopoll_full) 1372 cur_ops->get_gp_state_full(&cookie_full); 1373 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) 1374 sync(); 1375 sync(); 1376 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), 1377 "%s: Cookie check 3 failed %pS() online %*pbl.", 1378 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1379 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), 1380 "%s: Cookie check 4 failed %pS() online %*pbl", 1381 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1382 if (dopoll || dopoll_full) 1383 cpus_read_unlock(); 1384 } 1385 1386 /* 1387 * RCU torture writer kthread. Repeatedly substitutes a new structure 1388 * for that pointed to by rcu_torture_current, freeing the old structure 1389 * after a series of grace periods (the "pipeline"). 1390 */ 1391 static int 1392 rcu_torture_writer(void *arg) 1393 { 1394 bool boot_ended; 1395 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 1396 unsigned long cookie; 1397 struct rcu_gp_oldstate cookie_full; 1398 int expediting = 0; 1399 unsigned long gp_snap; 1400 unsigned long gp_snap1; 1401 struct rcu_gp_oldstate gp_snap_full; 1402 struct rcu_gp_oldstate gp_snap1_full; 1403 int i; 1404 int idx; 1405 int oldnice = task_nice(current); 1406 struct rcu_gp_oldstate *rgo = NULL; 1407 int rgo_size = 0; 1408 struct rcu_torture *rp; 1409 struct rcu_torture *old_rp; 1410 static DEFINE_TORTURE_RANDOM(rand); 1411 unsigned long stallsdone = jiffies; 1412 bool stutter_waited; 1413 unsigned long *ulo = NULL; 1414 int ulo_size = 0; 1415 1416 // If a new stall test is added, this must be adjusted. 1417 if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu) 1418 stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * 1419 HZ * (stall_cpu_repeat + 1); 1420 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 1421 if (!can_expedite) 1422 pr_alert("%s" TORTURE_FLAG 1423 " GP expediting controlled from boot/sysfs for %s.\n", 1424 torture_type, cur_ops->name); 1425 if (WARN_ONCE(nsynctypes == 0, 1426 "%s: No update-side primitives.\n", __func__)) { 1427 /* 1428 * No updates primitives, so don't try updating. 1429 * The resulting test won't be testing much, hence the 1430 * above WARN_ONCE(). 1431 */ 1432 rcu_torture_writer_state = RTWS_STOPPING; 1433 torture_kthread_stopping("rcu_torture_writer"); 1434 return 0; 1435 } 1436 if (cur_ops->poll_active > 0) { 1437 ulo = kzalloc(cur_ops->poll_active * sizeof(ulo[0]), GFP_KERNEL); 1438 if (!WARN_ON(!ulo)) 1439 ulo_size = cur_ops->poll_active; 1440 } 1441 if (cur_ops->poll_active_full > 0) { 1442 rgo = kzalloc(cur_ops->poll_active_full * sizeof(rgo[0]), GFP_KERNEL); 1443 if (!WARN_ON(!rgo)) 1444 rgo_size = cur_ops->poll_active_full; 1445 } 1446 1447 do { 1448 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1449 torture_hrtimeout_us(500, 1000, &rand); 1450 rp = rcu_torture_alloc(); 1451 if (rp == NULL) 1452 continue; 1453 rp->rtort_pipe_count = 0; 1454 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 1455 rcu_torture_writer_state = RTWS_DELAY; 1456 udelay(torture_random(&rand) & 0x3ff); 1457 rcu_torture_writer_state = RTWS_REPLACE; 1458 old_rp = rcu_dereference_check(rcu_torture_current, 1459 current == writer_task); 1460 rp->rtort_mbtest = 1; 1461 rcu_assign_pointer(rcu_torture_current, rp); 1462 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1463 if (old_rp) { 1464 i = old_rp->rtort_pipe_count; 1465 if (i > RCU_TORTURE_PIPE_LEN) 1466 i = RCU_TORTURE_PIPE_LEN; 1467 atomic_inc(&rcu_torture_wcount[i]); 1468 WRITE_ONCE(old_rp->rtort_pipe_count, 1469 old_rp->rtort_pipe_count + 1); 1470 ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count); 1471 1472 // Make sure readers block polled grace periods. 1473 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { 1474 idx = cur_ops->readlock(); 1475 cookie = cur_ops->get_gp_state(); 1476 WARN_ONCE(cur_ops->poll_gp_state(cookie), 1477 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", 1478 __func__, 1479 rcu_torture_writer_state_getname(), 1480 rcu_torture_writer_state, 1481 cookie, cur_ops->get_gp_state()); 1482 if (cur_ops->get_comp_state) { 1483 cookie = cur_ops->get_comp_state(); 1484 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); 1485 } 1486 cur_ops->readunlock(idx); 1487 } 1488 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { 1489 idx = cur_ops->readlock(); 1490 cur_ops->get_gp_state_full(&cookie_full); 1491 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 1492 "%s: Cookie check 5 failed %s(%d) online %*pbl\n", 1493 __func__, 1494 rcu_torture_writer_state_getname(), 1495 rcu_torture_writer_state, 1496 cpumask_pr_args(cpu_online_mask)); 1497 if (cur_ops->get_comp_state_full) { 1498 cur_ops->get_comp_state_full(&cookie_full); 1499 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); 1500 } 1501 cur_ops->readunlock(idx); 1502 } 1503 switch (synctype[torture_random(&rand) % nsynctypes]) { 1504 case RTWS_DEF_FREE: 1505 rcu_torture_writer_state = RTWS_DEF_FREE; 1506 cur_ops->deferred_free(old_rp); 1507 break; 1508 case RTWS_EXP_SYNC: 1509 rcu_torture_writer_state = RTWS_EXP_SYNC; 1510 do_rtws_sync(&rand, cur_ops->exp_sync); 1511 rcu_torture_pipe_update(old_rp); 1512 break; 1513 case RTWS_COND_GET: 1514 rcu_torture_writer_state = RTWS_COND_GET; 1515 gp_snap = cur_ops->get_gp_state(); 1516 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1517 rcu_torture_writer_state = RTWS_COND_SYNC; 1518 cur_ops->cond_sync(gp_snap); 1519 rcu_torture_pipe_update(old_rp); 1520 break; 1521 case RTWS_COND_GET_EXP: 1522 rcu_torture_writer_state = RTWS_COND_GET_EXP; 1523 gp_snap = cur_ops->get_gp_state_exp(); 1524 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1525 rcu_torture_writer_state = RTWS_COND_SYNC_EXP; 1526 cur_ops->cond_sync_exp(gp_snap); 1527 rcu_torture_pipe_update(old_rp); 1528 break; 1529 case RTWS_COND_GET_FULL: 1530 rcu_torture_writer_state = RTWS_COND_GET_FULL; 1531 cur_ops->get_gp_state_full(&gp_snap_full); 1532 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1533 rcu_torture_writer_state = RTWS_COND_SYNC_FULL; 1534 cur_ops->cond_sync_full(&gp_snap_full); 1535 rcu_torture_pipe_update(old_rp); 1536 break; 1537 case RTWS_COND_GET_EXP_FULL: 1538 rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; 1539 cur_ops->get_gp_state_full(&gp_snap_full); 1540 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1541 rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; 1542 cur_ops->cond_sync_exp_full(&gp_snap_full); 1543 rcu_torture_pipe_update(old_rp); 1544 break; 1545 case RTWS_POLL_GET: 1546 rcu_torture_writer_state = RTWS_POLL_GET; 1547 for (i = 0; i < ulo_size; i++) 1548 ulo[i] = cur_ops->get_comp_state(); 1549 gp_snap = cur_ops->start_gp_poll(); 1550 rcu_torture_writer_state = RTWS_POLL_WAIT; 1551 while (!cur_ops->poll_gp_state(gp_snap)) { 1552 gp_snap1 = cur_ops->get_gp_state(); 1553 for (i = 0; i < ulo_size; i++) 1554 if (cur_ops->poll_gp_state(ulo[i]) || 1555 cur_ops->same_gp_state(ulo[i], gp_snap1)) { 1556 ulo[i] = gp_snap1; 1557 break; 1558 } 1559 WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size); 1560 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1561 &rand); 1562 } 1563 rcu_torture_pipe_update(old_rp); 1564 break; 1565 case RTWS_POLL_GET_FULL: 1566 rcu_torture_writer_state = RTWS_POLL_GET_FULL; 1567 for (i = 0; i < rgo_size; i++) 1568 cur_ops->get_comp_state_full(&rgo[i]); 1569 cur_ops->start_gp_poll_full(&gp_snap_full); 1570 rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; 1571 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1572 cur_ops->get_gp_state_full(&gp_snap1_full); 1573 for (i = 0; i < rgo_size; i++) 1574 if (cur_ops->poll_gp_state_full(&rgo[i]) || 1575 cur_ops->same_gp_state_full(&rgo[i], 1576 &gp_snap1_full)) { 1577 rgo[i] = gp_snap1_full; 1578 break; 1579 } 1580 WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size); 1581 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1582 &rand); 1583 } 1584 rcu_torture_pipe_update(old_rp); 1585 break; 1586 case RTWS_POLL_GET_EXP: 1587 rcu_torture_writer_state = RTWS_POLL_GET_EXP; 1588 gp_snap = cur_ops->start_gp_poll_exp(); 1589 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; 1590 while (!cur_ops->poll_gp_state_exp(gp_snap)) 1591 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1592 &rand); 1593 rcu_torture_pipe_update(old_rp); 1594 break; 1595 case RTWS_POLL_GET_EXP_FULL: 1596 rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; 1597 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1598 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; 1599 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) 1600 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1601 &rand); 1602 rcu_torture_pipe_update(old_rp); 1603 break; 1604 case RTWS_SYNC: 1605 rcu_torture_writer_state = RTWS_SYNC; 1606 do_rtws_sync(&rand, cur_ops->sync); 1607 rcu_torture_pipe_update(old_rp); 1608 break; 1609 default: 1610 WARN_ON_ONCE(1); 1611 break; 1612 } 1613 } 1614 WRITE_ONCE(rcu_torture_current_version, 1615 rcu_torture_current_version + 1); 1616 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1617 if (can_expedite && 1618 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1619 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1620 if (expediting >= 0) 1621 rcu_expedite_gp(); 1622 else 1623 rcu_unexpedite_gp(); 1624 if (++expediting > 3) 1625 expediting = -expediting; 1626 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1627 can_expedite = !rcu_gp_is_expedited() && 1628 !rcu_gp_is_normal(); 1629 } 1630 rcu_torture_writer_state = RTWS_STUTTER; 1631 boot_ended = rcu_inkernel_boot_has_ended(); 1632 stutter_waited = stutter_wait("rcu_torture_writer"); 1633 if (stutter_waited && 1634 !atomic_read(&rcu_fwd_cb_nodelay) && 1635 !cur_ops->slow_gps && 1636 !torture_must_stop() && 1637 boot_ended && 1638 time_after(jiffies, stallsdone)) 1639 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1640 if (list_empty(&rcu_tortures[i].rtort_free) && 1641 rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { 1642 tracing_off(); 1643 if (cur_ops->gp_kthread_dbg) 1644 cur_ops->gp_kthread_dbg(); 1645 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1646 rcu_ftrace_dump(DUMP_ALL); 1647 } 1648 if (stutter_waited) 1649 sched_set_normal(current, oldnice); 1650 } while (!torture_must_stop()); 1651 rcu_torture_current = NULL; // Let stats task know that we are done. 1652 /* Reset expediting back to unexpedited. */ 1653 if (expediting > 0) 1654 expediting = -expediting; 1655 while (can_expedite && expediting++ < 0) 1656 rcu_unexpedite_gp(); 1657 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1658 if (!can_expedite) 1659 pr_alert("%s" TORTURE_FLAG 1660 " Dynamic grace-period expediting was disabled.\n", 1661 torture_type); 1662 kfree(ulo); 1663 kfree(rgo); 1664 rcu_torture_writer_state = RTWS_STOPPING; 1665 torture_kthread_stopping("rcu_torture_writer"); 1666 return 0; 1667 } 1668 1669 /* 1670 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1671 * delay between calls. 1672 */ 1673 static int 1674 rcu_torture_fakewriter(void *arg) 1675 { 1676 unsigned long gp_snap; 1677 struct rcu_gp_oldstate gp_snap_full; 1678 DEFINE_TORTURE_RANDOM(rand); 1679 1680 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1681 set_user_nice(current, MAX_NICE); 1682 1683 if (WARN_ONCE(nsynctypes == 0, 1684 "%s: No update-side primitives.\n", __func__)) { 1685 /* 1686 * No updates primitives, so don't try updating. 1687 * The resulting test won't be testing much, hence the 1688 * above WARN_ONCE(). 1689 */ 1690 torture_kthread_stopping("rcu_torture_fakewriter"); 1691 return 0; 1692 } 1693 1694 do { 1695 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); 1696 if (cur_ops->cb_barrier != NULL && 1697 torture_random(&rand) % (nfakewriters * 8) == 0) { 1698 cur_ops->cb_barrier(); 1699 } else { 1700 switch (synctype[torture_random(&rand) % nsynctypes]) { 1701 case RTWS_DEF_FREE: 1702 break; 1703 case RTWS_EXP_SYNC: 1704 cur_ops->exp_sync(); 1705 break; 1706 case RTWS_COND_GET: 1707 gp_snap = cur_ops->get_gp_state(); 1708 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1709 cur_ops->cond_sync(gp_snap); 1710 break; 1711 case RTWS_COND_GET_EXP: 1712 gp_snap = cur_ops->get_gp_state_exp(); 1713 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1714 cur_ops->cond_sync_exp(gp_snap); 1715 break; 1716 case RTWS_COND_GET_FULL: 1717 cur_ops->get_gp_state_full(&gp_snap_full); 1718 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1719 cur_ops->cond_sync_full(&gp_snap_full); 1720 break; 1721 case RTWS_COND_GET_EXP_FULL: 1722 cur_ops->get_gp_state_full(&gp_snap_full); 1723 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1724 cur_ops->cond_sync_exp_full(&gp_snap_full); 1725 break; 1726 case RTWS_POLL_GET: 1727 if (cur_ops->start_poll_irqsoff) 1728 local_irq_disable(); 1729 gp_snap = cur_ops->start_gp_poll(); 1730 if (cur_ops->start_poll_irqsoff) 1731 local_irq_enable(); 1732 while (!cur_ops->poll_gp_state(gp_snap)) { 1733 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1734 &rand); 1735 } 1736 break; 1737 case RTWS_POLL_GET_FULL: 1738 if (cur_ops->start_poll_irqsoff) 1739 local_irq_disable(); 1740 cur_ops->start_gp_poll_full(&gp_snap_full); 1741 if (cur_ops->start_poll_irqsoff) 1742 local_irq_enable(); 1743 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1744 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1745 &rand); 1746 } 1747 break; 1748 case RTWS_POLL_GET_EXP: 1749 gp_snap = cur_ops->start_gp_poll_exp(); 1750 while (!cur_ops->poll_gp_state_exp(gp_snap)) { 1751 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1752 &rand); 1753 } 1754 break; 1755 case RTWS_POLL_GET_EXP_FULL: 1756 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1757 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1758 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1759 &rand); 1760 } 1761 break; 1762 case RTWS_SYNC: 1763 cur_ops->sync(); 1764 break; 1765 default: 1766 WARN_ON_ONCE(1); 1767 break; 1768 } 1769 } 1770 stutter_wait("rcu_torture_fakewriter"); 1771 } while (!torture_must_stop()); 1772 1773 torture_kthread_stopping("rcu_torture_fakewriter"); 1774 return 0; 1775 } 1776 1777 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1778 { 1779 kfree(rhp); 1780 } 1781 1782 // Set up and carry out testing of RCU's global memory ordering 1783 static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, 1784 struct torture_random_state *trsp) 1785 { 1786 unsigned long loops; 1787 int noc = torture_num_online_cpus(); 1788 int rdrchked; 1789 int rdrchker; 1790 struct rcu_torture_reader_check *rtrcp; // Me. 1791 struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. 1792 struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. 1793 struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. 1794 1795 if (myid < 0) 1796 return; // Don't try this from timer handlers. 1797 1798 // Increment my counter. 1799 rtrcp = &rcu_torture_reader_mbchk[myid]; 1800 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); 1801 1802 // Attempt to assign someone else some checking work. 1803 rdrchked = torture_random(trsp) % nrealreaders; 1804 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1805 rdrchker = torture_random(trsp) % nrealreaders; 1806 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; 1807 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && 1808 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. 1809 !READ_ONCE(rtp->rtort_chkp) && 1810 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. 1811 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); 1812 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); 1813 rtrcp->rtc_chkrdr = rdrchked; 1814 WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. 1815 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || 1816 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) 1817 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. 1818 } 1819 1820 // If assigned some completed work, do it! 1821 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); 1822 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) 1823 return; // No work or work not yet ready. 1824 rdrchked = rtrcp_assigner->rtc_chkrdr; 1825 if (WARN_ON_ONCE(rdrchked < 0)) 1826 return; 1827 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1828 loops = READ_ONCE(rtrcp_chked->rtc_myloops); 1829 atomic_inc(&n_rcu_torture_mbchk_tries); 1830 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) 1831 atomic_inc(&n_rcu_torture_mbchk_fail); 1832 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; 1833 rtrcp_assigner->rtc_ready = 0; 1834 smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. 1835 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. 1836 } 1837 1838 /* 1839 * Do one extension of an RCU read-side critical section using the 1840 * current reader state in readstate (set to zero for initial entry 1841 * to extended critical section), set the new state as specified by 1842 * newstate (set to zero for final exit from extended critical section), 1843 * and random-number-generator state in trsp. If this is neither the 1844 * beginning or end of the critical section and if there was actually a 1845 * change, do a ->read_delay(). 1846 */ 1847 static void rcutorture_one_extend(int *readstate, int newstate, 1848 struct torture_random_state *trsp, 1849 struct rt_read_seg *rtrsp) 1850 { 1851 unsigned long flags; 1852 int idxnew1 = -1; 1853 int idxnew2 = -1; 1854 int idxold1 = *readstate; 1855 int idxold2 = idxold1; 1856 int statesnew = ~*readstate & newstate; 1857 int statesold = *readstate & ~newstate; 1858 1859 WARN_ON_ONCE(idxold2 < 0); 1860 WARN_ON_ONCE(idxold2 & ~RCUTORTURE_RDR_ALLBITS); 1861 rtrsp->rt_readstate = newstate; 1862 1863 /* First, put new protection in place to avoid critical-section gap. */ 1864 if (statesnew & RCUTORTURE_RDR_BH) 1865 local_bh_disable(); 1866 if (statesnew & RCUTORTURE_RDR_RBH) 1867 rcu_read_lock_bh(); 1868 if (statesnew & RCUTORTURE_RDR_IRQ) 1869 local_irq_disable(); 1870 if (statesnew & RCUTORTURE_RDR_PREEMPT) 1871 preempt_disable(); 1872 if (statesnew & RCUTORTURE_RDR_SCHED) 1873 rcu_read_lock_sched(); 1874 if (statesnew & RCUTORTURE_RDR_RCU_1) 1875 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 1876 if (statesnew & RCUTORTURE_RDR_RCU_2) 1877 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2; 1878 1879 /* 1880 * Next, remove old protection, in decreasing order of strength 1881 * to avoid unlock paths that aren't safe in the stronger 1882 * context. Namely: BH can not be enabled with disabled interrupts. 1883 * Additionally PREEMPT_RT requires that BH is enabled in preemptible 1884 * context. 1885 */ 1886 if (statesold & RCUTORTURE_RDR_IRQ) 1887 local_irq_enable(); 1888 if (statesold & RCUTORTURE_RDR_PREEMPT) 1889 preempt_enable(); 1890 if (statesold & RCUTORTURE_RDR_SCHED) 1891 rcu_read_unlock_sched(); 1892 if (statesold & RCUTORTURE_RDR_BH) 1893 local_bh_enable(); 1894 if (statesold & RCUTORTURE_RDR_RBH) 1895 rcu_read_unlock_bh(); 1896 if (statesold & RCUTORTURE_RDR_RCU_2) { 1897 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2); 1898 WARN_ON_ONCE(idxnew2 != -1); 1899 idxold2 = 0; 1900 } 1901 if (statesold & RCUTORTURE_RDR_RCU_1) { 1902 bool lockit; 1903 1904 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); 1905 if (lockit) 1906 raw_spin_lock_irqsave(¤t->pi_lock, flags); 1907 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 1908 WARN_ON_ONCE(idxnew1 != -1); 1909 idxold1 = 0; 1910 if (lockit) 1911 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 1912 } 1913 1914 /* Delay if neither beginning nor end and there was a change. */ 1915 if ((statesnew || statesold) && *readstate && newstate) 1916 cur_ops->read_delay(trsp, rtrsp); 1917 1918 /* Update the reader state. */ 1919 if (idxnew1 == -1) 1920 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; 1921 WARN_ON_ONCE(idxnew1 < 0); 1922 if (idxnew2 == -1) 1923 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; 1924 WARN_ON_ONCE(idxnew2 < 0); 1925 *readstate = idxnew1 | idxnew2 | newstate; 1926 WARN_ON_ONCE(*readstate < 0); 1927 if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS)) 1928 pr_info("Unexpected readstate value of %#x\n", *readstate); 1929 } 1930 1931 /* Return the biggest extendables mask given current RCU and boot parameters. */ 1932 static int rcutorture_extend_mask_max(void) 1933 { 1934 int mask; 1935 1936 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 1937 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 1938 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 1939 return mask; 1940 } 1941 1942 /* Return a random protection state mask, but with at least one bit set. */ 1943 static int 1944 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 1945 { 1946 int mask = rcutorture_extend_mask_max(); 1947 unsigned long randmask1 = torture_random(trsp); 1948 unsigned long randmask2 = randmask1 >> 3; 1949 unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 1950 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 1951 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1952 1953 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits. 1954 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 1955 if (!(randmask1 & 0x7)) 1956 mask = mask & randmask2; 1957 else 1958 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 1959 1960 // Can't have nested RCU reader without outer RCU reader. 1961 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { 1962 if (oldmask & RCUTORTURE_RDR_RCU_1) 1963 mask &= ~RCUTORTURE_RDR_RCU_2; 1964 else 1965 mask |= RCUTORTURE_RDR_RCU_1; 1966 } 1967 1968 /* 1969 * Can't enable bh w/irq disabled. 1970 */ 1971 if (mask & RCUTORTURE_RDR_IRQ) 1972 mask |= oldmask & bhs; 1973 1974 /* 1975 * Ideally these sequences would be detected in debug builds 1976 * (regardless of RT), but until then don't stop testing 1977 * them on non-RT. 1978 */ 1979 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1980 /* Can't modify BH in atomic context */ 1981 if (oldmask & preempts_irq) 1982 mask &= ~bhs; 1983 if ((oldmask | mask) & preempts_irq) 1984 mask |= oldmask & bhs; 1985 } 1986 1987 return mask ?: RCUTORTURE_RDR_RCU_1; 1988 } 1989 1990 /* 1991 * Do a randomly selected number of extensions of an existing RCU read-side 1992 * critical section. 1993 */ 1994 static struct rt_read_seg * 1995 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, 1996 struct rt_read_seg *rtrsp) 1997 { 1998 int i; 1999 int j; 2000 int mask = rcutorture_extend_mask_max(); 2001 2002 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 2003 if (!((mask - 1) & mask)) 2004 return rtrsp; /* Current RCU reader not extendable. */ 2005 /* Bias towards larger numbers of loops. */ 2006 i = torture_random(trsp); 2007 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 2008 for (j = 0; j < i; j++) { 2009 mask = rcutorture_extend_mask(*readstate, trsp); 2010 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 2011 } 2012 return &rtrsp[j]; 2013 } 2014 2015 /* 2016 * Do one read-side critical section, returning false if there was 2017 * no data to read. Can be invoked both from process context and 2018 * from a timer handler. 2019 */ 2020 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) 2021 { 2022 bool checkpolling = !(torture_random(trsp) & 0xfff); 2023 unsigned long cookie; 2024 struct rcu_gp_oldstate cookie_full; 2025 int i; 2026 unsigned long started; 2027 unsigned long completed; 2028 int newstate; 2029 struct rcu_torture *p; 2030 int pipe_count; 2031 int readstate = 0; 2032 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } }; 2033 struct rt_read_seg *rtrsp = &rtseg[0]; 2034 struct rt_read_seg *rtrsp1; 2035 unsigned long long ts; 2036 2037 WARN_ON_ONCE(!rcu_is_watching()); 2038 newstate = rcutorture_extend_mask(readstate, trsp); 2039 rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); 2040 if (checkpolling) { 2041 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2042 cookie = cur_ops->get_gp_state(); 2043 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2044 cur_ops->get_gp_state_full(&cookie_full); 2045 } 2046 started = cur_ops->get_gp_seq(); 2047 ts = rcu_trace_clock_local(); 2048 p = rcu_dereference_check(rcu_torture_current, 2049 !cur_ops->readlock_held || cur_ops->readlock_held()); 2050 if (p == NULL) { 2051 /* Wait for rcu_torture_writer to get underway */ 2052 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 2053 return false; 2054 } 2055 if (p->rtort_mbtest == 0) 2056 atomic_inc(&n_rcu_torture_mberror); 2057 rcu_torture_reader_do_mbchk(myid, p, trsp); 2058 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp); 2059 preempt_disable(); 2060 pipe_count = READ_ONCE(p->rtort_pipe_count); 2061 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 2062 // Should not happen in a correct RCU implementation, 2063 // happens quite often for torture_type=busted. 2064 pipe_count = RCU_TORTURE_PIPE_LEN; 2065 } 2066 completed = cur_ops->get_gp_seq(); 2067 if (pipe_count > 1) { 2068 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 2069 ts, started, completed); 2070 rcu_ftrace_dump(DUMP_ALL); 2071 } 2072 __this_cpu_inc(rcu_torture_count[pipe_count]); 2073 completed = rcutorture_seq_diff(completed, started); 2074 if (completed > RCU_TORTURE_PIPE_LEN) { 2075 /* Should not happen, but... */ 2076 completed = RCU_TORTURE_PIPE_LEN; 2077 } 2078 __this_cpu_inc(rcu_torture_batch[completed]); 2079 preempt_enable(); 2080 if (checkpolling) { 2081 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2082 WARN_ONCE(cur_ops->poll_gp_state(cookie), 2083 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", 2084 __func__, 2085 rcu_torture_writer_state_getname(), 2086 rcu_torture_writer_state, 2087 cookie, cur_ops->get_gp_state()); 2088 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2089 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 2090 "%s: Cookie check 6 failed %s(%d) online %*pbl\n", 2091 __func__, 2092 rcu_torture_writer_state_getname(), 2093 rcu_torture_writer_state, 2094 cpumask_pr_args(cpu_online_mask)); 2095 } 2096 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 2097 WARN_ON_ONCE(readstate); 2098 // This next splat is expected behavior if leakpointer, especially 2099 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. 2100 WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1); 2101 2102 /* If error or close call, record the sequence of reader protections. */ 2103 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 2104 i = 0; 2105 for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++) 2106 err_segs[i++] = *rtrsp1; 2107 rt_read_nsegs = i; 2108 } 2109 2110 return true; 2111 } 2112 2113 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 2114 2115 /* 2116 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2117 * incrementing the corresponding element of the pipeline array. The 2118 * counter in the element should never be greater than 1, otherwise, the 2119 * RCU implementation is broken. 2120 */ 2121 static void rcu_torture_timer(struct timer_list *unused) 2122 { 2123 atomic_long_inc(&n_rcu_torture_timers); 2124 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); 2125 2126 /* Test call_rcu() invocation from interrupt handler. */ 2127 if (cur_ops->call) { 2128 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 2129 2130 if (rhp) 2131 cur_ops->call(rhp, rcu_torture_timer_cb); 2132 } 2133 } 2134 2135 /* 2136 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 2137 * incrementing the corresponding element of the pipeline array. The 2138 * counter in the element should never be greater than 1, otherwise, the 2139 * RCU implementation is broken. 2140 */ 2141 static int 2142 rcu_torture_reader(void *arg) 2143 { 2144 unsigned long lastsleep = jiffies; 2145 long myid = (long)arg; 2146 int mynumonline = myid; 2147 DEFINE_TORTURE_RANDOM(rand); 2148 struct timer_list t; 2149 2150 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 2151 set_user_nice(current, MAX_NICE); 2152 if (irqreader && cur_ops->irq_capable) 2153 timer_setup_on_stack(&t, rcu_torture_timer, 0); 2154 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 2155 do { 2156 if (irqreader && cur_ops->irq_capable) { 2157 if (!timer_pending(&t)) 2158 mod_timer(&t, jiffies + 1); 2159 } 2160 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) 2161 schedule_timeout_interruptible(HZ); 2162 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 2163 torture_hrtimeout_us(500, 1000, &rand); 2164 lastsleep = jiffies + 10; 2165 } 2166 while (torture_num_online_cpus() < mynumonline && !torture_must_stop()) 2167 schedule_timeout_interruptible(HZ / 5); 2168 stutter_wait("rcu_torture_reader"); 2169 } while (!torture_must_stop()); 2170 if (irqreader && cur_ops->irq_capable) { 2171 del_timer_sync(&t); 2172 destroy_timer_on_stack(&t); 2173 } 2174 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2175 torture_kthread_stopping("rcu_torture_reader"); 2176 return 0; 2177 } 2178 2179 /* 2180 * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to 2181 * increase race probabilities and fuzzes the interval between toggling. 2182 */ 2183 static int rcu_nocb_toggle(void *arg) 2184 { 2185 int cpu; 2186 int maxcpu = -1; 2187 int oldnice = task_nice(current); 2188 long r; 2189 DEFINE_TORTURE_RANDOM(rand); 2190 ktime_t toggle_delay; 2191 unsigned long toggle_fuzz; 2192 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); 2193 2194 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); 2195 while (!rcu_inkernel_boot_has_ended()) 2196 schedule_timeout_interruptible(HZ / 10); 2197 for_each_possible_cpu(cpu) 2198 maxcpu = cpu; 2199 WARN_ON(maxcpu < 0); 2200 if (toggle_interval > ULONG_MAX) 2201 toggle_fuzz = ULONG_MAX >> 3; 2202 else 2203 toggle_fuzz = toggle_interval >> 3; 2204 if (toggle_fuzz <= 0) 2205 toggle_fuzz = NSEC_PER_USEC; 2206 do { 2207 r = torture_random(&rand); 2208 cpu = (r >> 1) % (maxcpu + 1); 2209 if (r & 0x1) { 2210 rcu_nocb_cpu_offload(cpu); 2211 atomic_long_inc(&n_nocb_offload); 2212 } else { 2213 rcu_nocb_cpu_deoffload(cpu); 2214 atomic_long_inc(&n_nocb_deoffload); 2215 } 2216 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; 2217 set_current_state(TASK_INTERRUPTIBLE); 2218 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); 2219 if (stutter_wait("rcu_nocb_toggle")) 2220 sched_set_normal(current, oldnice); 2221 } while (!torture_must_stop()); 2222 torture_kthread_stopping("rcu_nocb_toggle"); 2223 return 0; 2224 } 2225 2226 /* 2227 * Print torture statistics. Caller must ensure that there is only 2228 * one call to this function at a given time!!! This is normally 2229 * accomplished by relying on the module system to only have one copy 2230 * of the module loaded, and then by giving the rcu_torture_stats 2231 * kthread full control (or the init/cleanup functions when rcu_torture_stats 2232 * thread is not running). 2233 */ 2234 static void 2235 rcu_torture_stats_print(void) 2236 { 2237 int cpu; 2238 int i; 2239 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2240 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2241 struct rcu_torture *rtcp; 2242 static unsigned long rtcv_snap = ULONG_MAX; 2243 static bool splatted; 2244 struct task_struct *wtp; 2245 2246 for_each_possible_cpu(cpu) { 2247 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2248 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 2249 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 2250 } 2251 } 2252 for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { 2253 if (pipesummary[i] != 0) 2254 break; 2255 } 2256 2257 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2258 rtcp = rcu_access_pointer(rcu_torture_current); 2259 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 2260 rtcp, 2261 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 2262 rcu_torture_current_version, 2263 list_empty(&rcu_torture_freelist), 2264 atomic_read(&n_rcu_torture_alloc), 2265 atomic_read(&n_rcu_torture_alloc_fail), 2266 atomic_read(&n_rcu_torture_free)); 2267 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ", 2268 atomic_read(&n_rcu_torture_mberror), 2269 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), 2270 n_rcu_torture_barrier_error, 2271 n_rcu_torture_boost_ktrerror); 2272 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 2273 n_rcu_torture_boost_failure, 2274 n_rcu_torture_boosts, 2275 atomic_long_read(&n_rcu_torture_timers)); 2276 torture_onoff_stats(); 2277 pr_cont("barrier: %ld/%ld:%ld ", 2278 data_race(n_barrier_successes), 2279 data_race(n_barrier_attempts), 2280 data_race(n_rcu_torture_barrier_error)); 2281 pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. 2282 pr_cont("nocb-toggles: %ld:%ld\n", 2283 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); 2284 2285 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2286 if (atomic_read(&n_rcu_torture_mberror) || 2287 atomic_read(&n_rcu_torture_mbchk_fail) || 2288 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 2289 n_rcu_torture_boost_failure || i > 1) { 2290 pr_cont("%s", "!!! "); 2291 atomic_inc(&n_rcu_torture_error); 2292 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 2293 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); 2294 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 2295 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 2296 WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) 2297 WARN_ON_ONCE(i > 1); // Too-short grace period 2298 } 2299 pr_cont("Reader Pipe: "); 2300 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2301 pr_cont(" %ld", pipesummary[i]); 2302 pr_cont("\n"); 2303 2304 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2305 pr_cont("Reader Batch: "); 2306 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2307 pr_cont(" %ld", batchsummary[i]); 2308 pr_cont("\n"); 2309 2310 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2311 pr_cont("Free-Block Circulation: "); 2312 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2313 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 2314 } 2315 pr_cont("\n"); 2316 2317 if (cur_ops->stats) 2318 cur_ops->stats(); 2319 if (rtcv_snap == rcu_torture_current_version && 2320 rcu_access_pointer(rcu_torture_current) && 2321 !rcu_stall_is_suppressed()) { 2322 int __maybe_unused flags = 0; 2323 unsigned long __maybe_unused gp_seq = 0; 2324 2325 if (cur_ops->get_gp_data) 2326 cur_ops->get_gp_data(&flags, &gp_seq); 2327 wtp = READ_ONCE(writer_task); 2328 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", 2329 rcu_torture_writer_state_getname(), 2330 rcu_torture_writer_state, gp_seq, flags, 2331 wtp == NULL ? ~0U : wtp->__state, 2332 wtp == NULL ? -1 : (int)task_cpu(wtp)); 2333 if (!splatted && wtp) { 2334 sched_show_task(wtp); 2335 splatted = true; 2336 } 2337 if (cur_ops->gp_kthread_dbg) 2338 cur_ops->gp_kthread_dbg(); 2339 rcu_ftrace_dump(DUMP_ALL); 2340 } 2341 rtcv_snap = rcu_torture_current_version; 2342 } 2343 2344 /* 2345 * Periodically prints torture statistics, if periodic statistics printing 2346 * was specified via the stat_interval module parameter. 2347 */ 2348 static int 2349 rcu_torture_stats(void *arg) 2350 { 2351 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 2352 do { 2353 schedule_timeout_interruptible(stat_interval * HZ); 2354 rcu_torture_stats_print(); 2355 torture_shutdown_absorb("rcu_torture_stats"); 2356 } while (!torture_must_stop()); 2357 torture_kthread_stopping("rcu_torture_stats"); 2358 return 0; 2359 } 2360 2361 /* Test mem_dump_obj() and friends. */ 2362 static void rcu_torture_mem_dump_obj(void) 2363 { 2364 struct rcu_head *rhp; 2365 struct kmem_cache *kcp; 2366 static int z; 2367 2368 kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); 2369 if (WARN_ON_ONCE(!kcp)) 2370 return; 2371 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); 2372 if (WARN_ON_ONCE(!rhp)) { 2373 kmem_cache_destroy(kcp); 2374 return; 2375 } 2376 pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); 2377 pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); 2378 mem_dump_obj(ZERO_SIZE_PTR); 2379 pr_alert("mem_dump_obj(NULL):"); 2380 mem_dump_obj(NULL); 2381 pr_alert("mem_dump_obj(%px):", &rhp); 2382 mem_dump_obj(&rhp); 2383 pr_alert("mem_dump_obj(%px):", rhp); 2384 mem_dump_obj(rhp); 2385 pr_alert("mem_dump_obj(%px):", &rhp->func); 2386 mem_dump_obj(&rhp->func); 2387 pr_alert("mem_dump_obj(%px):", &z); 2388 mem_dump_obj(&z); 2389 kmem_cache_free(kcp, rhp); 2390 kmem_cache_destroy(kcp); 2391 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 2392 if (WARN_ON_ONCE(!rhp)) 2393 return; 2394 pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2395 pr_alert("mem_dump_obj(kmalloc %px):", rhp); 2396 mem_dump_obj(rhp); 2397 pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); 2398 mem_dump_obj(&rhp->func); 2399 kfree(rhp); 2400 rhp = vmalloc(4096); 2401 if (WARN_ON_ONCE(!rhp)) 2402 return; 2403 pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2404 pr_alert("mem_dump_obj(vmalloc %px):", rhp); 2405 mem_dump_obj(rhp); 2406 pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); 2407 mem_dump_obj(&rhp->func); 2408 vfree(rhp); 2409 } 2410 2411 static void 2412 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 2413 { 2414 pr_alert("%s" TORTURE_FLAG 2415 "--- %s: nreaders=%d nfakewriters=%d " 2416 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 2417 "shuffle_interval=%d stutter=%d irqreader=%d " 2418 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 2419 "test_boost=%d/%d test_boost_interval=%d " 2420 "test_boost_duration=%d shutdown_secs=%d " 2421 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 2422 "stall_cpu_block=%d stall_cpu_repeat=%d " 2423 "n_barrier_cbs=%d " 2424 "onoff_interval=%d onoff_holdoff=%d " 2425 "read_exit_delay=%d read_exit_burst=%d " 2426 "reader_flavor=%x " 2427 "nocbs_nthreads=%d nocbs_toggle=%d " 2428 "test_nmis=%d\n", 2429 torture_type, tag, nrealreaders, nfakewriters, 2430 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 2431 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 2432 test_boost, cur_ops->can_boost, 2433 test_boost_interval, test_boost_duration, shutdown_secs, 2434 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 2435 stall_cpu_block, stall_cpu_repeat, 2436 n_barrier_cbs, 2437 onoff_interval, onoff_holdoff, 2438 read_exit_delay, read_exit_burst, 2439 reader_flavor, 2440 nocbs_nthreads, nocbs_toggle, 2441 test_nmis); 2442 } 2443 2444 static int rcutorture_booster_cleanup(unsigned int cpu) 2445 { 2446 struct task_struct *t; 2447 2448 if (boost_tasks[cpu] == NULL) 2449 return 0; 2450 mutex_lock(&boost_mutex); 2451 t = boost_tasks[cpu]; 2452 boost_tasks[cpu] = NULL; 2453 rcu_torture_enable_rt_throttle(); 2454 mutex_unlock(&boost_mutex); 2455 2456 /* This must be outside of the mutex, otherwise deadlock! */ 2457 torture_stop_kthread(rcu_torture_boost, t); 2458 return 0; 2459 } 2460 2461 static int rcutorture_booster_init(unsigned int cpu) 2462 { 2463 int retval; 2464 2465 if (boost_tasks[cpu] != NULL) 2466 return 0; /* Already created, nothing more to do. */ 2467 2468 // Testing RCU priority boosting requires rcutorture do 2469 // some serious abuse. Counter this by running ksoftirqd 2470 // at higher priority. 2471 if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { 2472 struct sched_param sp; 2473 struct task_struct *t; 2474 2475 t = per_cpu(ksoftirqd, cpu); 2476 WARN_ON_ONCE(!t); 2477 sp.sched_priority = 2; 2478 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2479 } 2480 2481 /* Don't allow time recalculation while creating a new task. */ 2482 mutex_lock(&boost_mutex); 2483 rcu_torture_disable_rt_throttle(); 2484 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 2485 boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, 2486 cpu, "rcu_torture_boost_%u"); 2487 if (IS_ERR(boost_tasks[cpu])) { 2488 retval = PTR_ERR(boost_tasks[cpu]); 2489 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 2490 n_rcu_torture_boost_ktrerror++; 2491 boost_tasks[cpu] = NULL; 2492 mutex_unlock(&boost_mutex); 2493 return retval; 2494 } 2495 mutex_unlock(&boost_mutex); 2496 return 0; 2497 } 2498 2499 static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) 2500 { 2501 pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr); 2502 return NOTIFY_OK; 2503 } 2504 2505 static struct notifier_block rcu_torture_stall_block = { 2506 .notifier_call = rcu_torture_stall_nf, 2507 }; 2508 2509 /* 2510 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 2511 * induces a CPU stall for the time specified by stall_cpu. If a new 2512 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. 2513 */ 2514 static void rcu_torture_stall_one(int rep, int irqsoff) 2515 { 2516 int idx; 2517 unsigned long stop_at; 2518 2519 if (stall_cpu_holdoff > 0) { 2520 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 2521 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 2522 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 2523 } 2524 if (!kthread_should_stop() && stall_gp_kthread > 0) { 2525 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); 2526 rcu_gp_set_torture_wait(stall_gp_kthread * HZ); 2527 for (idx = 0; idx < stall_gp_kthread + 2; idx++) { 2528 if (kthread_should_stop()) 2529 break; 2530 schedule_timeout_uninterruptible(HZ); 2531 } 2532 } 2533 if (!kthread_should_stop() && stall_cpu > 0) { 2534 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); 2535 stop_at = ktime_get_seconds() + stall_cpu; 2536 /* RCU CPU stall is expected behavior in following code. */ 2537 idx = cur_ops->readlock(); 2538 if (irqsoff) 2539 local_irq_disable(); 2540 else if (!stall_cpu_block) 2541 preempt_disable(); 2542 pr_alert("%s start stall episode %d on CPU %d.\n", 2543 __func__, rep + 1, raw_smp_processor_id()); 2544 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) && 2545 !kthread_should_stop()) 2546 if (stall_cpu_block) { 2547 #ifdef CONFIG_PREEMPTION 2548 preempt_schedule(); 2549 #else 2550 schedule_timeout_uninterruptible(HZ); 2551 #endif 2552 } else if (stall_no_softlockup) { 2553 touch_softlockup_watchdog(); 2554 } 2555 if (irqsoff) 2556 local_irq_enable(); 2557 else if (!stall_cpu_block) 2558 preempt_enable(); 2559 cur_ops->readunlock(idx); 2560 } 2561 } 2562 2563 /* 2564 * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many 2565 * additional times as specified by the stall_cpu_repeat module parameter. 2566 * Note that stall_cpu_irqsoff is ignored on the second and subsequent 2567 * stall. 2568 */ 2569 static int rcu_torture_stall(void *args) 2570 { 2571 int i; 2572 int repeat = stall_cpu_repeat; 2573 int ret; 2574 2575 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 2576 if (repeat < 0) { 2577 repeat = 0; 2578 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 2579 } 2580 if (rcu_cpu_stall_notifiers) { 2581 ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block); 2582 if (ret) 2583 pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n", 2584 __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : ""); 2585 } 2586 for (i = 0; i <= repeat; i++) { 2587 if (kthread_should_stop()) 2588 break; 2589 rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0); 2590 } 2591 pr_alert("%s end.\n", __func__); 2592 if (rcu_cpu_stall_notifiers && !ret) { 2593 ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block); 2594 if (ret) 2595 pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret); 2596 } 2597 torture_shutdown_absorb("rcu_torture_stall"); 2598 while (!kthread_should_stop()) 2599 schedule_timeout_interruptible(10 * HZ); 2600 return 0; 2601 } 2602 2603 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 2604 static int __init rcu_torture_stall_init(void) 2605 { 2606 if (stall_cpu <= 0 && stall_gp_kthread <= 0) 2607 return 0; 2608 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 2609 } 2610 2611 /* State structure for forward-progress self-propagating RCU callback. */ 2612 struct fwd_cb_state { 2613 struct rcu_head rh; 2614 int stop; 2615 }; 2616 2617 /* 2618 * Forward-progress self-propagating RCU callback function. Because 2619 * callbacks run from softirq, this function is an implicit RCU read-side 2620 * critical section. 2621 */ 2622 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 2623 { 2624 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 2625 2626 if (READ_ONCE(fcsp->stop)) { 2627 WRITE_ONCE(fcsp->stop, 2); 2628 return; 2629 } 2630 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 2631 } 2632 2633 /* State for continuous-flood RCU callbacks. */ 2634 struct rcu_fwd_cb { 2635 struct rcu_head rh; 2636 struct rcu_fwd_cb *rfc_next; 2637 struct rcu_fwd *rfc_rfp; 2638 int rfc_gps; 2639 }; 2640 2641 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 2642 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 2643 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 2644 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 2645 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 2646 2647 struct rcu_launder_hist { 2648 long n_launders; 2649 unsigned long launder_gp_seq; 2650 }; 2651 2652 struct rcu_fwd { 2653 spinlock_t rcu_fwd_lock; 2654 struct rcu_fwd_cb *rcu_fwd_cb_head; 2655 struct rcu_fwd_cb **rcu_fwd_cb_tail; 2656 long n_launders_cb; 2657 unsigned long rcu_fwd_startat; 2658 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 2659 unsigned long rcu_launder_gp_seq_start; 2660 int rcu_fwd_id; 2661 }; 2662 2663 static DEFINE_MUTEX(rcu_fwd_mutex); 2664 static struct rcu_fwd *rcu_fwds; 2665 static unsigned long rcu_fwd_seq; 2666 static atomic_long_t rcu_fwd_max_cbs; 2667 static bool rcu_fwd_emergency_stop; 2668 2669 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 2670 { 2671 unsigned long gps; 2672 unsigned long gps_old; 2673 int i; 2674 int j; 2675 2676 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 2677 if (rfp->n_launders_hist[i].n_launders > 0) 2678 break; 2679 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", 2680 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); 2681 gps_old = rfp->rcu_launder_gp_seq_start; 2682 for (j = 0; j <= i; j++) { 2683 gps = rfp->n_launders_hist[j].launder_gp_seq; 2684 pr_cont(" %ds/%d: %ld:%ld", 2685 j + 1, FWD_CBS_HIST_DIV, 2686 rfp->n_launders_hist[j].n_launders, 2687 rcutorture_seq_diff(gps, gps_old)); 2688 gps_old = gps; 2689 } 2690 pr_cont("\n"); 2691 } 2692 2693 /* Callback function for continuous-flood RCU callbacks. */ 2694 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 2695 { 2696 unsigned long flags; 2697 int i; 2698 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 2699 struct rcu_fwd_cb **rfcpp; 2700 struct rcu_fwd *rfp = rfcp->rfc_rfp; 2701 2702 rfcp->rfc_next = NULL; 2703 rfcp->rfc_gps++; 2704 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2705 rfcpp = rfp->rcu_fwd_cb_tail; 2706 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 2707 smp_store_release(rfcpp, rfcp); 2708 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 2709 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 2710 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 2711 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 2712 rfp->n_launders_hist[i].n_launders++; 2713 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 2714 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2715 } 2716 2717 // Give the scheduler a chance, even on nohz_full CPUs. 2718 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 2719 { 2720 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 2721 // Real call_rcu() floods hit userspace, so emulate that. 2722 if (need_resched() || (iter & 0xfff)) 2723 schedule(); 2724 return; 2725 } 2726 // No userspace emulation: CB invocation throttles call_rcu() 2727 cond_resched(); 2728 } 2729 2730 /* 2731 * Free all callbacks on the rcu_fwd_cb_head list, either because the 2732 * test is over or because we hit an OOM event. 2733 */ 2734 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 2735 { 2736 unsigned long flags; 2737 unsigned long freed = 0; 2738 struct rcu_fwd_cb *rfcp; 2739 2740 for (;;) { 2741 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2742 rfcp = rfp->rcu_fwd_cb_head; 2743 if (!rfcp) { 2744 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2745 break; 2746 } 2747 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 2748 if (!rfp->rcu_fwd_cb_head) 2749 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 2750 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2751 kfree(rfcp); 2752 freed++; 2753 rcu_torture_fwd_prog_cond_resched(freed); 2754 if (tick_nohz_full_enabled()) { 2755 local_irq_save(flags); 2756 rcu_momentary_eqs(); 2757 local_irq_restore(flags); 2758 } 2759 } 2760 return freed; 2761 } 2762 2763 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 2764 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 2765 int *tested, int *tested_tries) 2766 { 2767 unsigned long cver; 2768 unsigned long dur; 2769 struct fwd_cb_state fcs; 2770 unsigned long gps; 2771 int idx; 2772 int sd; 2773 int sd4; 2774 bool selfpropcb = false; 2775 unsigned long stopat; 2776 static DEFINE_TORTURE_RANDOM(trs); 2777 2778 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 2779 if (!cur_ops->sync) 2780 return; // Cannot do need_resched() forward progress testing without ->sync. 2781 if (cur_ops->call && cur_ops->cb_barrier) { 2782 init_rcu_head_on_stack(&fcs.rh); 2783 selfpropcb = true; 2784 } 2785 2786 /* Tight loop containing cond_resched(). */ 2787 atomic_inc(&rcu_fwd_cb_nodelay); 2788 cur_ops->sync(); /* Later readers see above write. */ 2789 if (selfpropcb) { 2790 WRITE_ONCE(fcs.stop, 0); 2791 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 2792 } 2793 cver = READ_ONCE(rcu_torture_current_version); 2794 gps = cur_ops->get_gp_seq(); 2795 sd = cur_ops->stall_dur() + 1; 2796 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 2797 dur = sd4 + torture_random(&trs) % (sd - sd4); 2798 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 2799 stopat = rfp->rcu_fwd_startat + dur; 2800 while (time_before(jiffies, stopat) && 2801 !shutdown_time_arrived() && 2802 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2803 idx = cur_ops->readlock(); 2804 udelay(10); 2805 cur_ops->readunlock(idx); 2806 if (!fwd_progress_need_resched || need_resched()) 2807 cond_resched(); 2808 } 2809 (*tested_tries)++; 2810 if (!time_before(jiffies, stopat) && 2811 !shutdown_time_arrived() && 2812 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2813 (*tested)++; 2814 cver = READ_ONCE(rcu_torture_current_version) - cver; 2815 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 2816 WARN_ON(!cver && gps < 2); 2817 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, 2818 rfp->rcu_fwd_id, dur, cver, gps); 2819 } 2820 if (selfpropcb) { 2821 WRITE_ONCE(fcs.stop, 1); 2822 cur_ops->sync(); /* Wait for running CB to complete. */ 2823 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 2824 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 2825 } 2826 2827 if (selfpropcb) { 2828 WARN_ON(READ_ONCE(fcs.stop) != 2); 2829 destroy_rcu_head_on_stack(&fcs.rh); 2830 } 2831 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 2832 atomic_dec(&rcu_fwd_cb_nodelay); 2833 } 2834 2835 /* Carry out call_rcu() forward-progress testing. */ 2836 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 2837 { 2838 unsigned long cver; 2839 unsigned long flags; 2840 unsigned long gps; 2841 int i; 2842 long n_launders; 2843 long n_launders_cb_snap; 2844 long n_launders_sa; 2845 long n_max_cbs; 2846 long n_max_gps; 2847 struct rcu_fwd_cb *rfcp; 2848 struct rcu_fwd_cb *rfcpn; 2849 unsigned long stopat; 2850 unsigned long stoppedat; 2851 2852 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 2853 if (READ_ONCE(rcu_fwd_emergency_stop)) 2854 return; /* Get out of the way quickly, no GP wait! */ 2855 if (!cur_ops->call) 2856 return; /* Can't do call_rcu() fwd prog without ->call. */ 2857 2858 /* Loop continuously posting RCU callbacks. */ 2859 atomic_inc(&rcu_fwd_cb_nodelay); 2860 cur_ops->sync(); /* Later readers see above write. */ 2861 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 2862 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 2863 n_launders = 0; 2864 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 2865 n_launders_sa = 0; 2866 n_max_cbs = 0; 2867 n_max_gps = 0; 2868 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 2869 rfp->n_launders_hist[i].n_launders = 0; 2870 cver = READ_ONCE(rcu_torture_current_version); 2871 gps = cur_ops->get_gp_seq(); 2872 rfp->rcu_launder_gp_seq_start = gps; 2873 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 2874 while (time_before(jiffies, stopat) && 2875 !shutdown_time_arrived() && 2876 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2877 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 2878 rfcpn = NULL; 2879 if (rfcp) 2880 rfcpn = READ_ONCE(rfcp->rfc_next); 2881 if (rfcpn) { 2882 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 2883 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 2884 break; 2885 rfp->rcu_fwd_cb_head = rfcpn; 2886 n_launders++; 2887 n_launders_sa++; 2888 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { 2889 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 2890 if (WARN_ON_ONCE(!rfcp)) { 2891 schedule_timeout_interruptible(1); 2892 continue; 2893 } 2894 n_max_cbs++; 2895 n_launders_sa = 0; 2896 rfcp->rfc_gps = 0; 2897 rfcp->rfc_rfp = rfp; 2898 } else { 2899 rfcp = NULL; 2900 } 2901 if (rfcp) 2902 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 2903 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 2904 if (tick_nohz_full_enabled()) { 2905 local_irq_save(flags); 2906 rcu_momentary_eqs(); 2907 local_irq_restore(flags); 2908 } 2909 } 2910 stoppedat = jiffies; 2911 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 2912 cver = READ_ONCE(rcu_torture_current_version) - cver; 2913 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 2914 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 2915 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 2916 (void)rcu_torture_fwd_prog_cbfree(rfp); 2917 2918 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 2919 !shutdown_time_arrived()) { 2920 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg) 2921 cur_ops->gp_kthread_dbg(); 2922 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n", 2923 __func__, 2924 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 2925 n_launders + n_max_cbs - n_launders_cb_snap, 2926 n_launders, n_launders_sa, 2927 n_max_gps, n_max_cbs, cver, gps, num_online_cpus()); 2928 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); 2929 mutex_lock(&rcu_fwd_mutex); // Serialize histograms. 2930 rcu_torture_fwd_cb_hist(rfp); 2931 mutex_unlock(&rcu_fwd_mutex); 2932 } 2933 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 2934 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2935 atomic_dec(&rcu_fwd_cb_nodelay); 2936 } 2937 2938 2939 /* 2940 * OOM notifier, but this only prints diagnostic information for the 2941 * current forward-progress test. 2942 */ 2943 static int rcutorture_oom_notify(struct notifier_block *self, 2944 unsigned long notused, void *nfreed) 2945 { 2946 int i; 2947 long ncbs; 2948 struct rcu_fwd *rfp; 2949 2950 mutex_lock(&rcu_fwd_mutex); 2951 rfp = rcu_fwds; 2952 if (!rfp) { 2953 mutex_unlock(&rcu_fwd_mutex); 2954 return NOTIFY_OK; 2955 } 2956 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 2957 __func__); 2958 for (i = 0; i < fwd_progress; i++) { 2959 rcu_torture_fwd_cb_hist(&rfp[i]); 2960 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); 2961 } 2962 WRITE_ONCE(rcu_fwd_emergency_stop, true); 2963 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 2964 ncbs = 0; 2965 for (i = 0; i < fwd_progress; i++) 2966 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2967 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2968 cur_ops->cb_barrier(); 2969 ncbs = 0; 2970 for (i = 0; i < fwd_progress; i++) 2971 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2972 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2973 cur_ops->cb_barrier(); 2974 ncbs = 0; 2975 for (i = 0; i < fwd_progress; i++) 2976 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2977 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2978 smp_mb(); /* Frees before return to avoid redoing OOM. */ 2979 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 2980 pr_info("%s returning after OOM processing.\n", __func__); 2981 mutex_unlock(&rcu_fwd_mutex); 2982 return NOTIFY_OK; 2983 } 2984 2985 static struct notifier_block rcutorture_oom_nb = { 2986 .notifier_call = rcutorture_oom_notify 2987 }; 2988 2989 /* Carry out grace-period forward-progress testing. */ 2990 static int rcu_torture_fwd_prog(void *args) 2991 { 2992 bool firsttime = true; 2993 long max_cbs; 2994 int oldnice = task_nice(current); 2995 unsigned long oldseq = READ_ONCE(rcu_fwd_seq); 2996 struct rcu_fwd *rfp = args; 2997 int tested = 0; 2998 int tested_tries = 0; 2999 3000 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 3001 rcu_bind_current_to_nocb(); 3002 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 3003 set_user_nice(current, MAX_NICE); 3004 do { 3005 if (!rfp->rcu_fwd_id) { 3006 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 3007 WRITE_ONCE(rcu_fwd_emergency_stop, false); 3008 if (!firsttime) { 3009 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); 3010 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); 3011 } 3012 firsttime = false; 3013 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); 3014 } else { 3015 while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) 3016 schedule_timeout_interruptible(HZ / 20); 3017 oldseq = READ_ONCE(rcu_fwd_seq); 3018 } 3019 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3020 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) 3021 rcu_torture_fwd_prog_cr(rfp); 3022 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && 3023 (!IS_ENABLED(CONFIG_TINY_RCU) || 3024 (rcu_inkernel_boot_has_ended() && 3025 torture_num_online_cpus() > rfp->rcu_fwd_id))) 3026 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 3027 3028 /* Avoid slow periods, better to test when busy. */ 3029 if (stutter_wait("rcu_torture_fwd_prog")) 3030 sched_set_normal(current, oldnice); 3031 } while (!torture_must_stop()); 3032 /* Short runs might not contain a valid forward-progress attempt. */ 3033 if (!rfp->rcu_fwd_id) { 3034 WARN_ON(!tested && tested_tries >= 5); 3035 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 3036 } 3037 torture_kthread_stopping("rcu_torture_fwd_prog"); 3038 return 0; 3039 } 3040 3041 /* If forward-progress checking is requested and feasible, spawn the thread. */ 3042 static int __init rcu_torture_fwd_prog_init(void) 3043 { 3044 int i; 3045 int ret = 0; 3046 struct rcu_fwd *rfp; 3047 3048 if (!fwd_progress) 3049 return 0; /* Not requested, so don't do it. */ 3050 if (fwd_progress >= nr_cpu_ids) { 3051 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); 3052 fwd_progress = nr_cpu_ids; 3053 } else if (fwd_progress < 0) { 3054 fwd_progress = nr_cpu_ids; 3055 } 3056 if ((!cur_ops->sync && !cur_ops->call) || 3057 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || 3058 cur_ops == &rcu_busted_ops) { 3059 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 3060 fwd_progress = 0; 3061 return 0; 3062 } 3063 if (stall_cpu > 0) { 3064 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); 3065 fwd_progress = 0; 3066 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 3067 return -EINVAL; /* In module, can fail back to user. */ 3068 WARN_ON(1); /* Make sure rcutorture notices conflict. */ 3069 return 0; 3070 } 3071 if (fwd_progress_holdoff <= 0) 3072 fwd_progress_holdoff = 1; 3073 if (fwd_progress_div <= 0) 3074 fwd_progress_div = 4; 3075 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL); 3076 fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL); 3077 if (!rfp || !fwd_prog_tasks) { 3078 kfree(rfp); 3079 kfree(fwd_prog_tasks); 3080 fwd_prog_tasks = NULL; 3081 fwd_progress = 0; 3082 return -ENOMEM; 3083 } 3084 for (i = 0; i < fwd_progress; i++) { 3085 spin_lock_init(&rfp[i].rcu_fwd_lock); 3086 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; 3087 rfp[i].rcu_fwd_id = i; 3088 } 3089 mutex_lock(&rcu_fwd_mutex); 3090 rcu_fwds = rfp; 3091 mutex_unlock(&rcu_fwd_mutex); 3092 register_oom_notifier(&rcutorture_oom_nb); 3093 for (i = 0; i < fwd_progress; i++) { 3094 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); 3095 if (ret) { 3096 fwd_progress = i; 3097 return ret; 3098 } 3099 } 3100 return 0; 3101 } 3102 3103 static void rcu_torture_fwd_prog_cleanup(void) 3104 { 3105 int i; 3106 struct rcu_fwd *rfp; 3107 3108 if (!rcu_fwds || !fwd_prog_tasks) 3109 return; 3110 for (i = 0; i < fwd_progress; i++) 3111 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); 3112 unregister_oom_notifier(&rcutorture_oom_nb); 3113 mutex_lock(&rcu_fwd_mutex); 3114 rfp = rcu_fwds; 3115 rcu_fwds = NULL; 3116 mutex_unlock(&rcu_fwd_mutex); 3117 kfree(rfp); 3118 kfree(fwd_prog_tasks); 3119 fwd_prog_tasks = NULL; 3120 } 3121 3122 /* Callback function for RCU barrier testing. */ 3123 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 3124 { 3125 atomic_inc(&barrier_cbs_invoked); 3126 } 3127 3128 /* IPI handler to get callback posted on desired CPU, if online. */ 3129 static int rcu_torture_barrier1cb(void *rcu_void) 3130 { 3131 struct rcu_head *rhp = rcu_void; 3132 3133 cur_ops->call(rhp, rcu_torture_barrier_cbf); 3134 return 0; 3135 } 3136 3137 /* kthread function to register callbacks used to test RCU barriers. */ 3138 static int rcu_torture_barrier_cbs(void *arg) 3139 { 3140 long myid = (long)arg; 3141 bool lastphase = false; 3142 bool newphase; 3143 struct rcu_head rcu; 3144 3145 init_rcu_head_on_stack(&rcu); 3146 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 3147 set_user_nice(current, MAX_NICE); 3148 do { 3149 wait_event(barrier_cbs_wq[myid], 3150 (newphase = 3151 smp_load_acquire(&barrier_phase)) != lastphase || 3152 torture_must_stop()); 3153 lastphase = newphase; 3154 if (torture_must_stop()) 3155 break; 3156 /* 3157 * The above smp_load_acquire() ensures barrier_phase load 3158 * is ordered before the following ->call(). 3159 */ 3160 if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1)) 3161 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 3162 3163 if (atomic_dec_and_test(&barrier_cbs_count)) 3164 wake_up(&barrier_wq); 3165 } while (!torture_must_stop()); 3166 if (cur_ops->cb_barrier != NULL) 3167 cur_ops->cb_barrier(); 3168 destroy_rcu_head_on_stack(&rcu); 3169 torture_kthread_stopping("rcu_torture_barrier_cbs"); 3170 return 0; 3171 } 3172 3173 /* kthread function to drive and coordinate RCU barrier testing. */ 3174 static int rcu_torture_barrier(void *arg) 3175 { 3176 int i; 3177 3178 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 3179 do { 3180 atomic_set(&barrier_cbs_invoked, 0); 3181 atomic_set(&barrier_cbs_count, n_barrier_cbs); 3182 /* Ensure barrier_phase ordered after prior assignments. */ 3183 smp_store_release(&barrier_phase, !barrier_phase); 3184 for (i = 0; i < n_barrier_cbs; i++) 3185 wake_up(&barrier_cbs_wq[i]); 3186 wait_event(barrier_wq, 3187 atomic_read(&barrier_cbs_count) == 0 || 3188 torture_must_stop()); 3189 if (torture_must_stop()) 3190 break; 3191 n_barrier_attempts++; 3192 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 3193 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 3194 n_rcu_torture_barrier_error++; 3195 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 3196 atomic_read(&barrier_cbs_invoked), 3197 n_barrier_cbs); 3198 WARN_ON(1); 3199 // Wait manually for the remaining callbacks 3200 i = 0; 3201 do { 3202 if (WARN_ON(i++ > HZ)) 3203 i = INT_MIN; 3204 schedule_timeout_interruptible(1); 3205 cur_ops->cb_barrier(); 3206 } while (atomic_read(&barrier_cbs_invoked) != 3207 n_barrier_cbs && 3208 !torture_must_stop()); 3209 smp_mb(); // Can't trust ordering if broken. 3210 if (!torture_must_stop()) 3211 pr_err("Recovered: barrier_cbs_invoked = %d\n", 3212 atomic_read(&barrier_cbs_invoked)); 3213 } else { 3214 n_barrier_successes++; 3215 } 3216 schedule_timeout_interruptible(HZ / 10); 3217 } while (!torture_must_stop()); 3218 torture_kthread_stopping("rcu_torture_barrier"); 3219 return 0; 3220 } 3221 3222 /* Initialize RCU barrier testing. */ 3223 static int rcu_torture_barrier_init(void) 3224 { 3225 int i; 3226 int ret; 3227 3228 if (n_barrier_cbs <= 0) 3229 return 0; 3230 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 3231 pr_alert("%s" TORTURE_FLAG 3232 " Call or barrier ops missing for %s,\n", 3233 torture_type, cur_ops->name); 3234 pr_alert("%s" TORTURE_FLAG 3235 " RCU barrier testing omitted from run.\n", 3236 torture_type); 3237 return 0; 3238 } 3239 atomic_set(&barrier_cbs_count, 0); 3240 atomic_set(&barrier_cbs_invoked, 0); 3241 barrier_cbs_tasks = 3242 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 3243 GFP_KERNEL); 3244 barrier_cbs_wq = 3245 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 3246 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 3247 return -ENOMEM; 3248 for (i = 0; i < n_barrier_cbs; i++) { 3249 init_waitqueue_head(&barrier_cbs_wq[i]); 3250 ret = torture_create_kthread(rcu_torture_barrier_cbs, 3251 (void *)(long)i, 3252 barrier_cbs_tasks[i]); 3253 if (ret) 3254 return ret; 3255 } 3256 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 3257 } 3258 3259 /* Clean up after RCU barrier testing. */ 3260 static void rcu_torture_barrier_cleanup(void) 3261 { 3262 int i; 3263 3264 torture_stop_kthread(rcu_torture_barrier, barrier_task); 3265 if (barrier_cbs_tasks != NULL) { 3266 for (i = 0; i < n_barrier_cbs; i++) 3267 torture_stop_kthread(rcu_torture_barrier_cbs, 3268 barrier_cbs_tasks[i]); 3269 kfree(barrier_cbs_tasks); 3270 barrier_cbs_tasks = NULL; 3271 } 3272 if (barrier_cbs_wq != NULL) { 3273 kfree(barrier_cbs_wq); 3274 barrier_cbs_wq = NULL; 3275 } 3276 } 3277 3278 static bool rcu_torture_can_boost(void) 3279 { 3280 static int boost_warn_once; 3281 int prio; 3282 3283 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 3284 return false; 3285 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) 3286 return false; 3287 3288 prio = rcu_get_gp_kthreads_prio(); 3289 if (!prio) 3290 return false; 3291 3292 if (prio < 2) { 3293 if (boost_warn_once == 1) 3294 return false; 3295 3296 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 3297 boost_warn_once = 1; 3298 return false; 3299 } 3300 3301 return true; 3302 } 3303 3304 static bool read_exit_child_stop; 3305 static bool read_exit_child_stopped; 3306 static wait_queue_head_t read_exit_wq; 3307 3308 // Child kthread which just does an rcutorture reader and exits. 3309 static int rcu_torture_read_exit_child(void *trsp_in) 3310 { 3311 struct torture_random_state *trsp = trsp_in; 3312 3313 set_user_nice(current, MAX_NICE); 3314 // Minimize time between reading and exiting. 3315 while (!kthread_should_stop()) 3316 schedule_timeout_uninterruptible(HZ / 20); 3317 (void)rcu_torture_one_read(trsp, -1); 3318 return 0; 3319 } 3320 3321 // Parent kthread which creates and destroys read-exit child kthreads. 3322 static int rcu_torture_read_exit(void *unused) 3323 { 3324 bool errexit = false; 3325 int i; 3326 struct task_struct *tsp; 3327 DEFINE_TORTURE_RANDOM(trs); 3328 3329 // Allocate and initialize. 3330 set_user_nice(current, MAX_NICE); 3331 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); 3332 3333 // Each pass through this loop does one read-exit episode. 3334 do { 3335 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); 3336 for (i = 0; i < read_exit_burst; i++) { 3337 if (READ_ONCE(read_exit_child_stop)) 3338 break; 3339 stutter_wait("rcu_torture_read_exit"); 3340 // Spawn child. 3341 tsp = kthread_run(rcu_torture_read_exit_child, 3342 &trs, "%s", "rcu_torture_read_exit_child"); 3343 if (IS_ERR(tsp)) { 3344 TOROUT_ERRSTRING("out of memory"); 3345 errexit = true; 3346 break; 3347 } 3348 cond_resched(); 3349 kthread_stop(tsp); 3350 n_read_exits++; 3351 } 3352 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); 3353 rcu_barrier(); // Wait for task_struct free, avoid OOM. 3354 i = 0; 3355 for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) 3356 schedule_timeout_uninterruptible(HZ); 3357 } while (!errexit && !READ_ONCE(read_exit_child_stop)); 3358 3359 // Clean up and exit. 3360 smp_store_release(&read_exit_child_stopped, true); // After reaping. 3361 smp_mb(); // Store before wakeup. 3362 wake_up(&read_exit_wq); 3363 while (!torture_must_stop()) 3364 schedule_timeout_uninterruptible(HZ / 20); 3365 torture_kthread_stopping("rcu_torture_read_exit"); 3366 return 0; 3367 } 3368 3369 static int rcu_torture_read_exit_init(void) 3370 { 3371 if (read_exit_burst <= 0) 3372 return 0; 3373 init_waitqueue_head(&read_exit_wq); 3374 read_exit_child_stop = false; 3375 read_exit_child_stopped = false; 3376 return torture_create_kthread(rcu_torture_read_exit, NULL, 3377 read_exit_task); 3378 } 3379 3380 static void rcu_torture_read_exit_cleanup(void) 3381 { 3382 if (!read_exit_task) 3383 return; 3384 WRITE_ONCE(read_exit_child_stop, true); 3385 smp_mb(); // Above write before wait. 3386 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); 3387 torture_stop_kthread(rcutorture_read_exit, read_exit_task); 3388 } 3389 3390 static void rcutorture_test_nmis(int n) 3391 { 3392 #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3393 int cpu; 3394 int dumpcpu; 3395 int i; 3396 3397 for (i = 0; i < n; i++) { 3398 preempt_disable(); 3399 cpu = smp_processor_id(); 3400 dumpcpu = cpu + 1; 3401 if (dumpcpu >= nr_cpu_ids) 3402 dumpcpu = 0; 3403 pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu); 3404 dump_cpu_task(dumpcpu); 3405 preempt_enable(); 3406 schedule_timeout_uninterruptible(15 * HZ); 3407 } 3408 #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3409 WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis); 3410 #endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3411 } 3412 3413 static enum cpuhp_state rcutor_hp; 3414 3415 static void 3416 rcu_torture_cleanup(void) 3417 { 3418 int firsttime; 3419 int flags = 0; 3420 unsigned long gp_seq = 0; 3421 int i; 3422 3423 if (torture_cleanup_begin()) { 3424 if (cur_ops->cb_barrier != NULL) { 3425 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3426 cur_ops->cb_barrier(); 3427 } 3428 if (cur_ops->gp_slow_unregister) 3429 cur_ops->gp_slow_unregister(NULL); 3430 return; 3431 } 3432 if (!cur_ops) { 3433 torture_cleanup_end(); 3434 return; 3435 } 3436 3437 rcutorture_test_nmis(test_nmis); 3438 3439 if (cur_ops->gp_kthread_dbg) 3440 cur_ops->gp_kthread_dbg(); 3441 rcu_torture_read_exit_cleanup(); 3442 rcu_torture_barrier_cleanup(); 3443 rcu_torture_fwd_prog_cleanup(); 3444 torture_stop_kthread(rcu_torture_stall, stall_task); 3445 torture_stop_kthread(rcu_torture_writer, writer_task); 3446 3447 if (nocb_tasks) { 3448 for (i = 0; i < nrealnocbers; i++) 3449 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); 3450 kfree(nocb_tasks); 3451 nocb_tasks = NULL; 3452 } 3453 3454 if (reader_tasks) { 3455 for (i = 0; i < nrealreaders; i++) 3456 torture_stop_kthread(rcu_torture_reader, 3457 reader_tasks[i]); 3458 kfree(reader_tasks); 3459 reader_tasks = NULL; 3460 } 3461 kfree(rcu_torture_reader_mbchk); 3462 rcu_torture_reader_mbchk = NULL; 3463 3464 if (fakewriter_tasks) { 3465 for (i = 0; i < nfakewriters; i++) 3466 torture_stop_kthread(rcu_torture_fakewriter, 3467 fakewriter_tasks[i]); 3468 kfree(fakewriter_tasks); 3469 fakewriter_tasks = NULL; 3470 } 3471 3472 if (cur_ops->get_gp_data) 3473 cur_ops->get_gp_data(&flags, &gp_seq); 3474 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", 3475 cur_ops->name, (long)gp_seq, flags, 3476 rcutorture_seq_diff(gp_seq, start_gp_seq)); 3477 torture_stop_kthread(rcu_torture_stats, stats_task); 3478 torture_stop_kthread(rcu_torture_fqs, fqs_task); 3479 if (rcu_torture_can_boost() && rcutor_hp >= 0) 3480 cpuhp_remove_state(rcutor_hp); 3481 3482 /* 3483 * Wait for all RCU callbacks to fire, then do torture-type-specific 3484 * cleanup operations. 3485 */ 3486 if (cur_ops->cb_barrier != NULL) { 3487 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3488 cur_ops->cb_barrier(); 3489 } 3490 if (cur_ops->cleanup != NULL) 3491 cur_ops->cleanup(); 3492 3493 rcu_torture_mem_dump_obj(); 3494 3495 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 3496 3497 if (err_segs_recorded) { 3498 pr_alert("Failure/close-call rcutorture reader segments:\n"); 3499 if (rt_read_nsegs == 0) 3500 pr_alert("\t: No segments recorded!!!\n"); 3501 firsttime = 1; 3502 for (i = 0; i < rt_read_nsegs; i++) { 3503 pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate); 3504 if (err_segs[i].rt_delay_jiffies != 0) { 3505 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 3506 err_segs[i].rt_delay_jiffies); 3507 firsttime = 0; 3508 } 3509 if (err_segs[i].rt_delay_ms != 0) { 3510 pr_cont("%s%ldms", firsttime ? "" : "+", 3511 err_segs[i].rt_delay_ms); 3512 firsttime = 0; 3513 } 3514 if (err_segs[i].rt_delay_us != 0) { 3515 pr_cont("%s%ldus", firsttime ? "" : "+", 3516 err_segs[i].rt_delay_us); 3517 firsttime = 0; 3518 } 3519 pr_cont("%s\n", 3520 err_segs[i].rt_preempted ? "preempted" : ""); 3521 3522 } 3523 } 3524 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 3525 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 3526 else if (torture_onoff_failures()) 3527 rcu_torture_print_module_parms(cur_ops, 3528 "End of test: RCU_HOTPLUG"); 3529 else 3530 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 3531 torture_cleanup_end(); 3532 if (cur_ops->gp_slow_unregister) 3533 cur_ops->gp_slow_unregister(NULL); 3534 } 3535 3536 static void rcu_torture_leak_cb(struct rcu_head *rhp) 3537 { 3538 } 3539 3540 static void rcu_torture_err_cb(struct rcu_head *rhp) 3541 { 3542 /* 3543 * This -might- happen due to race conditions, but is unlikely. 3544 * The scenario that leads to this happening is that the 3545 * first of the pair of duplicate callbacks is queued, 3546 * someone else starts a grace period that includes that 3547 * callback, then the second of the pair must wait for the 3548 * next grace period. Unlikely, but can happen. If it 3549 * does happen, the debug-objects subsystem won't have splatted. 3550 */ 3551 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 3552 } 3553 3554 /* 3555 * Verify that double-free causes debug-objects to complain, but only 3556 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 3557 * cannot be carried out. 3558 */ 3559 static void rcu_test_debug_objects(void) 3560 { 3561 struct rcu_head rh1; 3562 struct rcu_head rh2; 3563 int idx; 3564 3565 if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) { 3566 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n", 3567 KBUILD_MODNAME, cur_ops->name); 3568 return; 3569 } 3570 3571 if (WARN_ON_ONCE(cur_ops->debug_objects && 3572 (!cur_ops->call || !cur_ops->cb_barrier))) 3573 return; 3574 3575 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 3576 3577 init_rcu_head_on_stack(&rh1); 3578 init_rcu_head_on_stack(&rh2); 3579 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name); 3580 3581 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 3582 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */ 3583 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 3584 cur_ops->call(&rh2, rcu_torture_leak_cb); 3585 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 3586 if (rhp) { 3587 cur_ops->call(rhp, rcu_torture_leak_cb); 3588 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ 3589 } 3590 cur_ops->readunlock(idx); 3591 3592 /* Wait for them all to get done so we can safely return. */ 3593 cur_ops->cb_barrier(); 3594 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name); 3595 destroy_rcu_head_on_stack(&rh1); 3596 destroy_rcu_head_on_stack(&rh2); 3597 kfree(rhp); 3598 } 3599 3600 static void rcutorture_sync(void) 3601 { 3602 static unsigned long n; 3603 3604 if (cur_ops->sync && !(++n & 0xfff)) 3605 cur_ops->sync(); 3606 } 3607 3608 static DEFINE_MUTEX(mut0); 3609 static DEFINE_MUTEX(mut1); 3610 static DEFINE_MUTEX(mut2); 3611 static DEFINE_MUTEX(mut3); 3612 static DEFINE_MUTEX(mut4); 3613 static DEFINE_MUTEX(mut5); 3614 static DEFINE_MUTEX(mut6); 3615 static DEFINE_MUTEX(mut7); 3616 static DEFINE_MUTEX(mut8); 3617 static DEFINE_MUTEX(mut9); 3618 3619 static DECLARE_RWSEM(rwsem0); 3620 static DECLARE_RWSEM(rwsem1); 3621 static DECLARE_RWSEM(rwsem2); 3622 static DECLARE_RWSEM(rwsem3); 3623 static DECLARE_RWSEM(rwsem4); 3624 static DECLARE_RWSEM(rwsem5); 3625 static DECLARE_RWSEM(rwsem6); 3626 static DECLARE_RWSEM(rwsem7); 3627 static DECLARE_RWSEM(rwsem8); 3628 static DECLARE_RWSEM(rwsem9); 3629 3630 DEFINE_STATIC_SRCU(srcu0); 3631 DEFINE_STATIC_SRCU(srcu1); 3632 DEFINE_STATIC_SRCU(srcu2); 3633 DEFINE_STATIC_SRCU(srcu3); 3634 DEFINE_STATIC_SRCU(srcu4); 3635 DEFINE_STATIC_SRCU(srcu5); 3636 DEFINE_STATIC_SRCU(srcu6); 3637 DEFINE_STATIC_SRCU(srcu7); 3638 DEFINE_STATIC_SRCU(srcu8); 3639 DEFINE_STATIC_SRCU(srcu9); 3640 3641 static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, 3642 int cyclelen, int deadlock) 3643 { 3644 int j = i + 1; 3645 3646 if (j >= cyclelen) 3647 j = deadlock ? 0 : -1; 3648 if (j >= 0) 3649 pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i); 3650 else 3651 pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i); 3652 return j; 3653 } 3654 3655 // Test lockdep on SRCU-based deadlock scenarios. 3656 static void rcu_torture_init_srcu_lockdep(void) 3657 { 3658 int cyclelen; 3659 int deadlock; 3660 bool err = false; 3661 int i; 3662 int j; 3663 int idx; 3664 struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4, 3665 &mut5, &mut6, &mut7, &mut8, &mut9 }; 3666 struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4, 3667 &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; 3668 struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, 3669 &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; 3670 int testtype; 3671 3672 if (!test_srcu_lockdep) 3673 return; 3674 3675 deadlock = test_srcu_lockdep / 1000; 3676 testtype = (test_srcu_lockdep / 10) % 100; 3677 cyclelen = test_srcu_lockdep % 10; 3678 WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus)); 3679 if (WARN_ONCE(deadlock != !!deadlock, 3680 "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n", 3681 __func__, test_srcu_lockdep, deadlock)) 3682 err = true; 3683 if (WARN_ONCE(cyclelen <= 0, 3684 "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n", 3685 __func__, test_srcu_lockdep, cyclelen)) 3686 err = true; 3687 if (err) 3688 goto err_out; 3689 3690 if (testtype == 0) { 3691 pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n", 3692 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3693 if (deadlock && cyclelen == 1) 3694 pr_info("%s: Expect hang.\n", __func__); 3695 for (i = 0; i < cyclelen; i++) { 3696 j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu", 3697 "srcu_read_unlock", i, cyclelen, deadlock); 3698 idx = srcu_read_lock(srcus[i]); 3699 if (j >= 0) 3700 synchronize_srcu(srcus[j]); 3701 srcu_read_unlock(srcus[i], idx); 3702 } 3703 return; 3704 } 3705 3706 if (testtype == 1) { 3707 pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n", 3708 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3709 for (i = 0; i < cyclelen; i++) { 3710 pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n", 3711 __func__, i, i, i, i); 3712 idx = srcu_read_lock(srcus[i]); 3713 mutex_lock(muts[i]); 3714 mutex_unlock(muts[i]); 3715 srcu_read_unlock(srcus[i], idx); 3716 3717 j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu", 3718 "mutex_unlock", i, cyclelen, deadlock); 3719 mutex_lock(muts[i]); 3720 if (j >= 0) 3721 synchronize_srcu(srcus[j]); 3722 mutex_unlock(muts[i]); 3723 } 3724 return; 3725 } 3726 3727 if (testtype == 2) { 3728 pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n", 3729 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3730 for (i = 0; i < cyclelen; i++) { 3731 pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n", 3732 __func__, i, i, i, i); 3733 idx = srcu_read_lock(srcus[i]); 3734 down_read(rwsems[i]); 3735 up_read(rwsems[i]); 3736 srcu_read_unlock(srcus[i], idx); 3737 3738 j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu", 3739 "up_write", i, cyclelen, deadlock); 3740 down_write(rwsems[i]); 3741 if (j >= 0) 3742 synchronize_srcu(srcus[j]); 3743 up_write(rwsems[i]); 3744 } 3745 return; 3746 } 3747 3748 #ifdef CONFIG_TASKS_TRACE_RCU 3749 if (testtype == 3) { 3750 pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n", 3751 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3752 if (deadlock && cyclelen == 1) 3753 pr_info("%s: Expect hang.\n", __func__); 3754 for (i = 0; i < cyclelen; i++) { 3755 char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock"; 3756 char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace" 3757 : "synchronize_srcu"; 3758 char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock"; 3759 3760 j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock); 3761 if (i == 0) 3762 rcu_read_lock_trace(); 3763 else 3764 idx = srcu_read_lock(srcus[i]); 3765 if (j >= 0) { 3766 if (i == cyclelen - 1) 3767 synchronize_rcu_tasks_trace(); 3768 else 3769 synchronize_srcu(srcus[j]); 3770 } 3771 if (i == 0) 3772 rcu_read_unlock_trace(); 3773 else 3774 srcu_read_unlock(srcus[i], idx); 3775 } 3776 return; 3777 } 3778 #endif // #ifdef CONFIG_TASKS_TRACE_RCU 3779 3780 err_out: 3781 pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); 3782 pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); 3783 pr_info("%s: D: Deadlock if nonzero.\n", __func__); 3784 pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); 3785 pr_info("%s: L: Cycle length.\n", __func__); 3786 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) 3787 pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); 3788 } 3789 3790 static int __init 3791 rcu_torture_init(void) 3792 { 3793 long i; 3794 int cpu; 3795 int firsterr = 0; 3796 int flags = 0; 3797 unsigned long gp_seq = 0; 3798 static struct rcu_torture_ops *torture_ops[] = { 3799 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, 3800 TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 3801 &trivial_ops, 3802 }; 3803 3804 if (!torture_init_begin(torture_type, verbose)) 3805 return -EBUSY; 3806 3807 /* Process args and tell the world that the torturer is on the job. */ 3808 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 3809 cur_ops = torture_ops[i]; 3810 if (strcmp(torture_type, cur_ops->name) == 0) 3811 break; 3812 } 3813 if (i == ARRAY_SIZE(torture_ops)) { 3814 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 3815 torture_type); 3816 pr_alert("rcu-torture types:"); 3817 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 3818 pr_cont(" %s", torture_ops[i]->name); 3819 pr_cont("\n"); 3820 firsterr = -EINVAL; 3821 cur_ops = NULL; 3822 goto unwind; 3823 } 3824 if (cur_ops->fqs == NULL && fqs_duration != 0) { 3825 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 3826 fqs_duration = 0; 3827 } 3828 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops || 3829 !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 3830 pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n", 3831 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU)); 3832 nocbs_nthreads = 0; 3833 } 3834 if (cur_ops->init) 3835 cur_ops->init(); 3836 3837 rcu_torture_init_srcu_lockdep(); 3838 3839 if (nreaders >= 0) { 3840 nrealreaders = nreaders; 3841 } else { 3842 nrealreaders = num_online_cpus() - 2 - nreaders; 3843 if (nrealreaders <= 0) 3844 nrealreaders = 1; 3845 } 3846 rcu_torture_print_module_parms(cur_ops, "Start of test"); 3847 if (cur_ops->get_gp_data) 3848 cur_ops->get_gp_data(&flags, &gp_seq); 3849 start_gp_seq = gp_seq; 3850 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", 3851 cur_ops->name, (long)gp_seq, flags); 3852 3853 /* Set up the freelist. */ 3854 3855 INIT_LIST_HEAD(&rcu_torture_freelist); 3856 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 3857 rcu_tortures[i].rtort_mbtest = 0; 3858 list_add_tail(&rcu_tortures[i].rtort_free, 3859 &rcu_torture_freelist); 3860 } 3861 3862 /* Initialize the statistics so that each run gets its own numbers. */ 3863 3864 rcu_torture_current = NULL; 3865 rcu_torture_current_version = 0; 3866 atomic_set(&n_rcu_torture_alloc, 0); 3867 atomic_set(&n_rcu_torture_alloc_fail, 0); 3868 atomic_set(&n_rcu_torture_free, 0); 3869 atomic_set(&n_rcu_torture_mberror, 0); 3870 atomic_set(&n_rcu_torture_mbchk_fail, 0); 3871 atomic_set(&n_rcu_torture_mbchk_tries, 0); 3872 atomic_set(&n_rcu_torture_error, 0); 3873 n_rcu_torture_barrier_error = 0; 3874 n_rcu_torture_boost_ktrerror = 0; 3875 n_rcu_torture_boost_failure = 0; 3876 n_rcu_torture_boosts = 0; 3877 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 3878 atomic_set(&rcu_torture_wcount[i], 0); 3879 for_each_possible_cpu(cpu) { 3880 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 3881 per_cpu(rcu_torture_count, cpu)[i] = 0; 3882 per_cpu(rcu_torture_batch, cpu)[i] = 0; 3883 } 3884 } 3885 err_segs_recorded = 0; 3886 rt_read_nsegs = 0; 3887 3888 /* Start up the kthreads. */ 3889 3890 rcu_torture_write_types(); 3891 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 3892 writer_task); 3893 if (torture_init_error(firsterr)) 3894 goto unwind; 3895 if (nfakewriters > 0) { 3896 fakewriter_tasks = kcalloc(nfakewriters, 3897 sizeof(fakewriter_tasks[0]), 3898 GFP_KERNEL); 3899 if (fakewriter_tasks == NULL) { 3900 TOROUT_ERRSTRING("out of memory"); 3901 firsterr = -ENOMEM; 3902 goto unwind; 3903 } 3904 } 3905 for (i = 0; i < nfakewriters; i++) { 3906 firsterr = torture_create_kthread(rcu_torture_fakewriter, 3907 NULL, fakewriter_tasks[i]); 3908 if (torture_init_error(firsterr)) 3909 goto unwind; 3910 } 3911 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 3912 GFP_KERNEL); 3913 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), 3914 GFP_KERNEL); 3915 if (!reader_tasks || !rcu_torture_reader_mbchk) { 3916 TOROUT_ERRSTRING("out of memory"); 3917 firsterr = -ENOMEM; 3918 goto unwind; 3919 } 3920 for (i = 0; i < nrealreaders; i++) { 3921 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 3922 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 3923 reader_tasks[i]); 3924 if (torture_init_error(firsterr)) 3925 goto unwind; 3926 } 3927 nrealnocbers = nocbs_nthreads; 3928 if (WARN_ON(nrealnocbers < 0)) 3929 nrealnocbers = 1; 3930 if (WARN_ON(nocbs_toggle < 0)) 3931 nocbs_toggle = HZ; 3932 if (nrealnocbers > 0) { 3933 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); 3934 if (nocb_tasks == NULL) { 3935 TOROUT_ERRSTRING("out of memory"); 3936 firsterr = -ENOMEM; 3937 goto unwind; 3938 } 3939 } else { 3940 nocb_tasks = NULL; 3941 } 3942 for (i = 0; i < nrealnocbers; i++) { 3943 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 3944 if (torture_init_error(firsterr)) 3945 goto unwind; 3946 } 3947 if (stat_interval > 0) { 3948 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 3949 stats_task); 3950 if (torture_init_error(firsterr)) 3951 goto unwind; 3952 } 3953 if (test_no_idle_hz && shuffle_interval > 0) { 3954 firsterr = torture_shuffle_init(shuffle_interval * HZ); 3955 if (torture_init_error(firsterr)) 3956 goto unwind; 3957 } 3958 if (stutter < 0) 3959 stutter = 0; 3960 if (stutter) { 3961 int t; 3962 3963 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 3964 firsterr = torture_stutter_init(stutter * HZ, t); 3965 if (torture_init_error(firsterr)) 3966 goto unwind; 3967 } 3968 if (fqs_duration < 0) 3969 fqs_duration = 0; 3970 if (fqs_holdoff < 0) 3971 fqs_holdoff = 0; 3972 if (fqs_duration && fqs_holdoff) { 3973 /* Create the fqs thread */ 3974 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 3975 fqs_task); 3976 if (torture_init_error(firsterr)) 3977 goto unwind; 3978 } 3979 if (test_boost_interval < 1) 3980 test_boost_interval = 1; 3981 if (test_boost_duration < 2) 3982 test_boost_duration = 2; 3983 if (rcu_torture_can_boost()) { 3984 3985 boost_starttime = jiffies + test_boost_interval * HZ; 3986 3987 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 3988 rcutorture_booster_init, 3989 rcutorture_booster_cleanup); 3990 rcutor_hp = firsterr; 3991 if (torture_init_error(firsterr)) 3992 goto unwind; 3993 } 3994 shutdown_jiffies = jiffies + shutdown_secs * HZ; 3995 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 3996 if (torture_init_error(firsterr)) 3997 goto unwind; 3998 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 3999 rcutorture_sync); 4000 if (torture_init_error(firsterr)) 4001 goto unwind; 4002 firsterr = rcu_torture_stall_init(); 4003 if (torture_init_error(firsterr)) 4004 goto unwind; 4005 firsterr = rcu_torture_fwd_prog_init(); 4006 if (torture_init_error(firsterr)) 4007 goto unwind; 4008 firsterr = rcu_torture_barrier_init(); 4009 if (torture_init_error(firsterr)) 4010 goto unwind; 4011 firsterr = rcu_torture_read_exit_init(); 4012 if (torture_init_error(firsterr)) 4013 goto unwind; 4014 if (object_debug) 4015 rcu_test_debug_objects(); 4016 torture_init_end(); 4017 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister)) 4018 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay); 4019 return 0; 4020 4021 unwind: 4022 torture_init_end(); 4023 rcu_torture_cleanup(); 4024 if (shutdown_secs) { 4025 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 4026 kernel_power_off(); 4027 } 4028 return firsterr; 4029 } 4030 4031 module_init(rcu_torture_init); 4032 module_exit(rcu_torture_cleanup); 4033