1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.rst 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/rcu_notifier.h> 25 #include <linux/interrupt.h> 26 #include <linux/sched/signal.h> 27 #include <uapi/linux/sched/types.h> 28 #include <linux/atomic.h> 29 #include <linux/bitops.h> 30 #include <linux/completion.h> 31 #include <linux/moduleparam.h> 32 #include <linux/percpu.h> 33 #include <linux/notifier.h> 34 #include <linux/reboot.h> 35 #include <linux/freezer.h> 36 #include <linux/cpu.h> 37 #include <linux/delay.h> 38 #include <linux/stat.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/trace_clock.h> 42 #include <asm/byteorder.h> 43 #include <linux/torture.h> 44 #include <linux/vmalloc.h> 45 #include <linux/sched/debug.h> 46 #include <linux/sched/sysctl.h> 47 #include <linux/oom.h> 48 #include <linux/tick.h> 49 #include <linux/rcupdate_trace.h> 50 #include <linux/nmi.h> 51 52 #include "rcu.h" 53 54 MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility"); 55 MODULE_LICENSE("GPL"); 56 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 57 58 /* Bits for ->extendables field, extendables param, and related definitions. */ 59 #define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */ 60 #define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1) 61 #define RCUTORTURE_RDR_SHIFT_2 16 /* Put SRCU index in upper bits. */ 62 #define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2) 63 #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ 64 #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ 65 #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ 66 #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ 67 #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ 68 #define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */ 69 #define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */ 70 #define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */ 71 #define RCUTORTURE_MAX_EXTEND \ 72 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 73 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) 74 #define RCUTORTURE_RDR_ALLBITS \ 75 (RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \ 76 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2) 77 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 78 /* Must be power of two minus one. */ 79 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 80 81 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 82 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 83 torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); 84 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 85 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 86 torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); 87 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 88 torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); 89 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); 90 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 91 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); 92 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); 93 torture_param(bool, gp_cond_exp_full, false, 94 "Use conditional/async full-stateexpedited GP wait primitives"); 95 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 96 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); 97 torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); 98 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); 99 torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); 100 torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); 101 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 102 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 103 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); 104 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); 105 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 106 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 107 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); 108 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 109 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); 110 torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); 111 torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); 112 torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); 113 torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); 114 torture_param(int, reader_flavor, 0x1, "Reader flavors to use, one per bit."); 115 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 116 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 117 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 118 torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); 119 torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); 120 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 121 torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); 122 torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); 123 torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); 124 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 125 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 126 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 127 torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); 128 torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); 129 torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); 130 torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); 131 torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); 132 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 133 134 static char *torture_type = "rcu"; 135 module_param(torture_type, charp, 0444); 136 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 137 138 static int nrealnocbers; 139 static int nrealreaders; 140 static struct task_struct *writer_task; 141 static struct task_struct **fakewriter_tasks; 142 static struct task_struct **reader_tasks; 143 static struct task_struct **nocb_tasks; 144 static struct task_struct *stats_task; 145 static struct task_struct *fqs_task; 146 static struct task_struct *boost_tasks[NR_CPUS]; 147 static struct task_struct *stall_task; 148 static struct task_struct **fwd_prog_tasks; 149 static struct task_struct **barrier_cbs_tasks; 150 static struct task_struct *barrier_task; 151 static struct task_struct *read_exit_task; 152 153 #define RCU_TORTURE_PIPE_LEN 10 154 155 // Mailbox-like structure to check RCU global memory ordering. 156 struct rcu_torture_reader_check { 157 unsigned long rtc_myloops; 158 int rtc_chkrdr; 159 unsigned long rtc_chkloops; 160 int rtc_ready; 161 struct rcu_torture_reader_check *rtc_assigner; 162 } ____cacheline_internodealigned_in_smp; 163 164 // Update-side data structure used to check RCU readers. 165 struct rcu_torture { 166 struct rcu_head rtort_rcu; 167 int rtort_pipe_count; 168 struct list_head rtort_free; 169 int rtort_mbtest; 170 struct rcu_torture_reader_check *rtort_chkp; 171 }; 172 173 static LIST_HEAD(rcu_torture_freelist); 174 static struct rcu_torture __rcu *rcu_torture_current; 175 static unsigned long rcu_torture_current_version; 176 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 177 static DEFINE_SPINLOCK(rcu_torture_lock); 178 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 179 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 180 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 181 static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; 182 static atomic_t n_rcu_torture_alloc; 183 static atomic_t n_rcu_torture_alloc_fail; 184 static atomic_t n_rcu_torture_free; 185 static atomic_t n_rcu_torture_mberror; 186 static atomic_t n_rcu_torture_mbchk_fail; 187 static atomic_t n_rcu_torture_mbchk_tries; 188 static atomic_t n_rcu_torture_error; 189 static long n_rcu_torture_barrier_error; 190 static long n_rcu_torture_boost_ktrerror; 191 static long n_rcu_torture_boost_failure; 192 static long n_rcu_torture_boosts; 193 static atomic_long_t n_rcu_torture_timers; 194 static long n_barrier_attempts; 195 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 196 static unsigned long n_read_exits; 197 static struct list_head rcu_torture_removed; 198 static unsigned long shutdown_jiffies; 199 static unsigned long start_gp_seq; 200 static atomic_long_t n_nocb_offload; 201 static atomic_long_t n_nocb_deoffload; 202 203 static int rcu_torture_writer_state; 204 #define RTWS_FIXED_DELAY 0 205 #define RTWS_DELAY 1 206 #define RTWS_REPLACE 2 207 #define RTWS_DEF_FREE 3 208 #define RTWS_EXP_SYNC 4 209 #define RTWS_COND_GET 5 210 #define RTWS_COND_GET_FULL 6 211 #define RTWS_COND_GET_EXP 7 212 #define RTWS_COND_GET_EXP_FULL 8 213 #define RTWS_COND_SYNC 9 214 #define RTWS_COND_SYNC_FULL 10 215 #define RTWS_COND_SYNC_EXP 11 216 #define RTWS_COND_SYNC_EXP_FULL 12 217 #define RTWS_POLL_GET 13 218 #define RTWS_POLL_GET_FULL 14 219 #define RTWS_POLL_GET_EXP 15 220 #define RTWS_POLL_GET_EXP_FULL 16 221 #define RTWS_POLL_WAIT 17 222 #define RTWS_POLL_WAIT_FULL 18 223 #define RTWS_POLL_WAIT_EXP 19 224 #define RTWS_POLL_WAIT_EXP_FULL 20 225 #define RTWS_SYNC 21 226 #define RTWS_STUTTER 22 227 #define RTWS_STOPPING 23 228 static const char * const rcu_torture_writer_state_names[] = { 229 "RTWS_FIXED_DELAY", 230 "RTWS_DELAY", 231 "RTWS_REPLACE", 232 "RTWS_DEF_FREE", 233 "RTWS_EXP_SYNC", 234 "RTWS_COND_GET", 235 "RTWS_COND_GET_FULL", 236 "RTWS_COND_GET_EXP", 237 "RTWS_COND_GET_EXP_FULL", 238 "RTWS_COND_SYNC", 239 "RTWS_COND_SYNC_FULL", 240 "RTWS_COND_SYNC_EXP", 241 "RTWS_COND_SYNC_EXP_FULL", 242 "RTWS_POLL_GET", 243 "RTWS_POLL_GET_FULL", 244 "RTWS_POLL_GET_EXP", 245 "RTWS_POLL_GET_EXP_FULL", 246 "RTWS_POLL_WAIT", 247 "RTWS_POLL_WAIT_FULL", 248 "RTWS_POLL_WAIT_EXP", 249 "RTWS_POLL_WAIT_EXP_FULL", 250 "RTWS_SYNC", 251 "RTWS_STUTTER", 252 "RTWS_STOPPING", 253 }; 254 255 /* Record reader segment types and duration for first failing read. */ 256 struct rt_read_seg { 257 int rt_readstate; 258 unsigned long rt_delay_jiffies; 259 unsigned long rt_delay_ms; 260 unsigned long rt_delay_us; 261 bool rt_preempted; 262 }; 263 static int err_segs_recorded; 264 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 265 static int rt_read_nsegs; 266 267 static const char *rcu_torture_writer_state_getname(void) 268 { 269 unsigned int i = READ_ONCE(rcu_torture_writer_state); 270 271 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 272 return "???"; 273 return rcu_torture_writer_state_names[i]; 274 } 275 276 #ifdef CONFIG_RCU_TRACE 277 static u64 notrace rcu_trace_clock_local(void) 278 { 279 u64 ts = trace_clock_local(); 280 281 (void)do_div(ts, NSEC_PER_USEC); 282 return ts; 283 } 284 #else /* #ifdef CONFIG_RCU_TRACE */ 285 static u64 notrace rcu_trace_clock_local(void) 286 { 287 return 0ULL; 288 } 289 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 290 291 /* 292 * Stop aggressive CPU-hog tests a bit before the end of the test in order 293 * to avoid interfering with test shutdown. 294 */ 295 static bool shutdown_time_arrived(void) 296 { 297 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 298 } 299 300 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 301 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 302 /* and boost task create/destroy. */ 303 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 304 static bool barrier_phase; /* Test phase. */ 305 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 306 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 307 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 308 309 static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 310 311 /* 312 * Allocate an element from the rcu_tortures pool. 313 */ 314 static struct rcu_torture * 315 rcu_torture_alloc(void) 316 { 317 struct list_head *p; 318 319 spin_lock_bh(&rcu_torture_lock); 320 if (list_empty(&rcu_torture_freelist)) { 321 atomic_inc(&n_rcu_torture_alloc_fail); 322 spin_unlock_bh(&rcu_torture_lock); 323 return NULL; 324 } 325 atomic_inc(&n_rcu_torture_alloc); 326 p = rcu_torture_freelist.next; 327 list_del_init(p); 328 spin_unlock_bh(&rcu_torture_lock); 329 return container_of(p, struct rcu_torture, rtort_free); 330 } 331 332 /* 333 * Free an element to the rcu_tortures pool. 334 */ 335 static void 336 rcu_torture_free(struct rcu_torture *p) 337 { 338 atomic_inc(&n_rcu_torture_free); 339 spin_lock_bh(&rcu_torture_lock); 340 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 341 spin_unlock_bh(&rcu_torture_lock); 342 } 343 344 /* 345 * Operations vector for selecting different types of tests. 346 */ 347 348 struct rcu_torture_ops { 349 int ttype; 350 void (*init)(void); 351 void (*cleanup)(void); 352 int (*readlock)(void); 353 void (*read_delay)(struct torture_random_state *rrsp, 354 struct rt_read_seg *rtrsp); 355 void (*readunlock)(int idx); 356 int (*readlock_held)(void); 357 unsigned long (*get_gp_seq)(void); 358 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 359 void (*deferred_free)(struct rcu_torture *p); 360 void (*sync)(void); 361 void (*exp_sync)(void); 362 unsigned long (*get_gp_state_exp)(void); 363 unsigned long (*start_gp_poll_exp)(void); 364 void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); 365 bool (*poll_gp_state_exp)(unsigned long oldstate); 366 void (*cond_sync_exp)(unsigned long oldstate); 367 void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); 368 unsigned long (*get_comp_state)(void); 369 void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); 370 bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); 371 bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); 372 unsigned long (*get_gp_state)(void); 373 void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); 374 unsigned long (*start_gp_poll)(void); 375 void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); 376 bool (*poll_gp_state)(unsigned long oldstate); 377 bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); 378 bool (*poll_need_2gp)(bool poll, bool poll_full); 379 void (*cond_sync)(unsigned long oldstate); 380 void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); 381 int poll_active; 382 int poll_active_full; 383 call_rcu_func_t call; 384 void (*cb_barrier)(void); 385 void (*fqs)(void); 386 void (*stats)(void); 387 void (*gp_kthread_dbg)(void); 388 bool (*check_boost_failed)(unsigned long gp_state, int *cpup); 389 int (*stall_dur)(void); 390 void (*get_gp_data)(int *flags, unsigned long *gp_seq); 391 void (*gp_slow_register)(atomic_t *rgssp); 392 void (*gp_slow_unregister)(atomic_t *rgssp); 393 long cbflood_max; 394 int irq_capable; 395 int can_boost; 396 int extendables; 397 int slow_gps; 398 int no_pi_lock; 399 int debug_objects; 400 int start_poll_irqsoff; 401 const char *name; 402 }; 403 404 static struct rcu_torture_ops *cur_ops; 405 406 /* 407 * Definitions for rcu torture testing. 408 */ 409 410 static int torture_readlock_not_held(void) 411 { 412 return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); 413 } 414 415 static int rcu_torture_read_lock(void) 416 { 417 rcu_read_lock(); 418 return 0; 419 } 420 421 static void 422 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 423 { 424 unsigned long started; 425 unsigned long completed; 426 const unsigned long shortdelay_us = 200; 427 unsigned long longdelay_ms = 300; 428 unsigned long long ts; 429 430 /* We want a short delay sometimes to make a reader delay the grace 431 * period, and we want a long delay occasionally to trigger 432 * force_quiescent_state. */ 433 434 if (!atomic_read(&rcu_fwd_cb_nodelay) && 435 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 436 started = cur_ops->get_gp_seq(); 437 ts = rcu_trace_clock_local(); 438 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) 439 longdelay_ms = 5; /* Avoid triggering BH limits. */ 440 mdelay(longdelay_ms); 441 rtrsp->rt_delay_ms = longdelay_ms; 442 completed = cur_ops->get_gp_seq(); 443 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 444 started, completed); 445 } 446 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 447 udelay(shortdelay_us); 448 rtrsp->rt_delay_us = shortdelay_us; 449 } 450 if (!preempt_count() && 451 !(torture_random(rrsp) % (nrealreaders * 500))) { 452 torture_preempt_schedule(); /* QS only if preemptible. */ 453 rtrsp->rt_preempted = true; 454 } 455 } 456 457 static void rcu_torture_read_unlock(int idx) 458 { 459 rcu_read_unlock(); 460 } 461 462 /* 463 * Update callback in the pipe. This should be invoked after a grace period. 464 */ 465 static bool 466 rcu_torture_pipe_update_one(struct rcu_torture *rp) 467 { 468 int i; 469 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); 470 471 if (rtrcp) { 472 WRITE_ONCE(rp->rtort_chkp, NULL); 473 smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). 474 } 475 i = rp->rtort_pipe_count; 476 if (i > RCU_TORTURE_PIPE_LEN) 477 i = RCU_TORTURE_PIPE_LEN; 478 atomic_inc(&rcu_torture_wcount[i]); 479 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 480 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 481 if (i + 1 >= RCU_TORTURE_PIPE_LEN) { 482 rp->rtort_mbtest = 0; 483 return true; 484 } 485 return false; 486 } 487 488 /* 489 * Update all callbacks in the pipe. Suitable for synchronous grace-period 490 * primitives. 491 */ 492 static void 493 rcu_torture_pipe_update(struct rcu_torture *old_rp) 494 { 495 struct rcu_torture *rp; 496 struct rcu_torture *rp1; 497 498 if (old_rp) 499 list_add(&old_rp->rtort_free, &rcu_torture_removed); 500 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 501 if (rcu_torture_pipe_update_one(rp)) { 502 list_del(&rp->rtort_free); 503 rcu_torture_free(rp); 504 } 505 } 506 } 507 508 static void 509 rcu_torture_cb(struct rcu_head *p) 510 { 511 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 512 513 if (torture_must_stop_irq()) { 514 /* Test is ending, just drop callbacks on the floor. */ 515 /* The next initialization will pick up the pieces. */ 516 return; 517 } 518 if (rcu_torture_pipe_update_one(rp)) 519 rcu_torture_free(rp); 520 else 521 cur_ops->deferred_free(rp); 522 } 523 524 static unsigned long rcu_no_completed(void) 525 { 526 return 0; 527 } 528 529 static void rcu_torture_deferred_free(struct rcu_torture *p) 530 { 531 call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); 532 } 533 534 static void rcu_sync_torture_init(void) 535 { 536 INIT_LIST_HEAD(&rcu_torture_removed); 537 } 538 539 static bool rcu_poll_need_2gp(bool poll, bool poll_full) 540 { 541 return poll; 542 } 543 544 static struct rcu_torture_ops rcu_ops = { 545 .ttype = RCU_FLAVOR, 546 .init = rcu_sync_torture_init, 547 .readlock = rcu_torture_read_lock, 548 .read_delay = rcu_read_delay, 549 .readunlock = rcu_torture_read_unlock, 550 .readlock_held = torture_readlock_not_held, 551 .get_gp_seq = rcu_get_gp_seq, 552 .gp_diff = rcu_seq_diff, 553 .deferred_free = rcu_torture_deferred_free, 554 .sync = synchronize_rcu, 555 .exp_sync = synchronize_rcu_expedited, 556 .same_gp_state = same_state_synchronize_rcu, 557 .same_gp_state_full = same_state_synchronize_rcu_full, 558 .get_comp_state = get_completed_synchronize_rcu, 559 .get_comp_state_full = get_completed_synchronize_rcu_full, 560 .get_gp_state = get_state_synchronize_rcu, 561 .get_gp_state_full = get_state_synchronize_rcu_full, 562 .start_gp_poll = start_poll_synchronize_rcu, 563 .start_gp_poll_full = start_poll_synchronize_rcu_full, 564 .poll_gp_state = poll_state_synchronize_rcu, 565 .poll_gp_state_full = poll_state_synchronize_rcu_full, 566 .poll_need_2gp = rcu_poll_need_2gp, 567 .cond_sync = cond_synchronize_rcu, 568 .cond_sync_full = cond_synchronize_rcu_full, 569 .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE, 570 .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, 571 .get_gp_state_exp = get_state_synchronize_rcu, 572 .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, 573 .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, 574 .poll_gp_state_exp = poll_state_synchronize_rcu, 575 .cond_sync_exp = cond_synchronize_rcu_expedited, 576 .call = call_rcu_hurry, 577 .cb_barrier = rcu_barrier, 578 .fqs = rcu_force_quiescent_state, 579 .gp_kthread_dbg = show_rcu_gp_kthreads, 580 .check_boost_failed = rcu_check_boost_fail, 581 .stall_dur = rcu_jiffies_till_stall_check, 582 .get_gp_data = rcutorture_get_gp_data, 583 .gp_slow_register = rcu_gp_slow_register, 584 .gp_slow_unregister = rcu_gp_slow_unregister, 585 .irq_capable = 1, 586 .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), 587 .extendables = RCUTORTURE_MAX_EXTEND, 588 .debug_objects = 1, 589 .start_poll_irqsoff = 1, 590 .name = "rcu" 591 }; 592 593 /* 594 * Don't even think about trying any of these in real life!!! 595 * The names includes "busted", and they really means it! 596 * The only purpose of these functions is to provide a buggy RCU 597 * implementation to make sure that rcutorture correctly emits 598 * buggy-RCU error messages. 599 */ 600 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 601 { 602 /* This is a deliberate bug for testing purposes only! */ 603 rcu_torture_cb(&p->rtort_rcu); 604 } 605 606 static void synchronize_rcu_busted(void) 607 { 608 /* This is a deliberate bug for testing purposes only! */ 609 } 610 611 static void 612 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 613 { 614 /* This is a deliberate bug for testing purposes only! */ 615 func(head); 616 } 617 618 static struct rcu_torture_ops rcu_busted_ops = { 619 .ttype = INVALID_RCU_FLAVOR, 620 .init = rcu_sync_torture_init, 621 .readlock = rcu_torture_read_lock, 622 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 623 .readunlock = rcu_torture_read_unlock, 624 .readlock_held = torture_readlock_not_held, 625 .get_gp_seq = rcu_no_completed, 626 .deferred_free = rcu_busted_torture_deferred_free, 627 .sync = synchronize_rcu_busted, 628 .exp_sync = synchronize_rcu_busted, 629 .call = call_rcu_busted, 630 .irq_capable = 1, 631 .name = "busted" 632 }; 633 634 /* 635 * Definitions for srcu torture testing. 636 */ 637 638 DEFINE_STATIC_SRCU(srcu_ctl); 639 static struct srcu_struct srcu_ctld; 640 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 641 static struct rcu_torture_ops srcud_ops; 642 643 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) 644 { 645 srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq); 646 } 647 648 static int srcu_torture_read_lock(void) 649 { 650 int idx; 651 int ret = 0; 652 653 if ((reader_flavor & 0x1) || !(reader_flavor & 0x7)) { 654 idx = srcu_read_lock(srcu_ctlp); 655 WARN_ON_ONCE(idx & ~0x1); 656 ret += idx; 657 } 658 if (reader_flavor & 0x2) { 659 idx = srcu_read_lock_nmisafe(srcu_ctlp); 660 WARN_ON_ONCE(idx & ~0x1); 661 ret += idx << 1; 662 } 663 if (reader_flavor & 0x4) { 664 idx = srcu_read_lock_lite(srcu_ctlp); 665 WARN_ON_ONCE(idx & ~0x1); 666 ret += idx << 2; 667 } 668 return ret; 669 } 670 671 static void 672 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 673 { 674 long delay; 675 const long uspertick = 1000000 / HZ; 676 const long longdelay = 10; 677 678 /* We want there to be long-running readers, but not all the time. */ 679 680 delay = torture_random(rrsp) % 681 (nrealreaders * 2 * longdelay * uspertick); 682 if (!delay && in_task()) { 683 schedule_timeout_interruptible(longdelay); 684 rtrsp->rt_delay_jiffies = longdelay; 685 } else { 686 rcu_read_delay(rrsp, rtrsp); 687 } 688 } 689 690 static void srcu_torture_read_unlock(int idx) 691 { 692 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 693 if (reader_flavor & 0x4) 694 srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2); 695 if (reader_flavor & 0x2) 696 srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1); 697 if ((reader_flavor & 0x1) || !(reader_flavor & 0x7)) 698 srcu_read_unlock(srcu_ctlp, idx & 0x1); 699 } 700 701 static int torture_srcu_read_lock_held(void) 702 { 703 return srcu_read_lock_held(srcu_ctlp); 704 } 705 706 static unsigned long srcu_torture_completed(void) 707 { 708 return srcu_batches_completed(srcu_ctlp); 709 } 710 711 static void srcu_torture_deferred_free(struct rcu_torture *rp) 712 { 713 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 714 } 715 716 static void srcu_torture_synchronize(void) 717 { 718 synchronize_srcu(srcu_ctlp); 719 } 720 721 static unsigned long srcu_torture_get_gp_state(void) 722 { 723 return get_state_synchronize_srcu(srcu_ctlp); 724 } 725 726 static unsigned long srcu_torture_start_gp_poll(void) 727 { 728 return start_poll_synchronize_srcu(srcu_ctlp); 729 } 730 731 static bool srcu_torture_poll_gp_state(unsigned long oldstate) 732 { 733 return poll_state_synchronize_srcu(srcu_ctlp, oldstate); 734 } 735 736 static void srcu_torture_call(struct rcu_head *head, 737 rcu_callback_t func) 738 { 739 call_srcu(srcu_ctlp, head, func); 740 } 741 742 static void srcu_torture_barrier(void) 743 { 744 srcu_barrier(srcu_ctlp); 745 } 746 747 static void srcu_torture_stats(void) 748 { 749 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 750 } 751 752 static void srcu_torture_synchronize_expedited(void) 753 { 754 synchronize_srcu_expedited(srcu_ctlp); 755 } 756 757 static struct rcu_torture_ops srcu_ops = { 758 .ttype = SRCU_FLAVOR, 759 .init = rcu_sync_torture_init, 760 .readlock = srcu_torture_read_lock, 761 .read_delay = srcu_read_delay, 762 .readunlock = srcu_torture_read_unlock, 763 .readlock_held = torture_srcu_read_lock_held, 764 .get_gp_seq = srcu_torture_completed, 765 .deferred_free = srcu_torture_deferred_free, 766 .sync = srcu_torture_synchronize, 767 .exp_sync = srcu_torture_synchronize_expedited, 768 .same_gp_state = same_state_synchronize_srcu, 769 .get_comp_state = get_completed_synchronize_srcu, 770 .get_gp_state = srcu_torture_get_gp_state, 771 .start_gp_poll = srcu_torture_start_gp_poll, 772 .poll_gp_state = srcu_torture_poll_gp_state, 773 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 774 .call = srcu_torture_call, 775 .cb_barrier = srcu_torture_barrier, 776 .stats = srcu_torture_stats, 777 .get_gp_data = srcu_get_gp_data, 778 .cbflood_max = 50000, 779 .irq_capable = 1, 780 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 781 .debug_objects = 1, 782 .name = "srcu" 783 }; 784 785 static void srcu_torture_init(void) 786 { 787 rcu_sync_torture_init(); 788 WARN_ON(init_srcu_struct(&srcu_ctld)); 789 srcu_ctlp = &srcu_ctld; 790 } 791 792 static void srcu_torture_cleanup(void) 793 { 794 cleanup_srcu_struct(&srcu_ctld); 795 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 796 } 797 798 /* As above, but dynamically allocated. */ 799 static struct rcu_torture_ops srcud_ops = { 800 .ttype = SRCU_FLAVOR, 801 .init = srcu_torture_init, 802 .cleanup = srcu_torture_cleanup, 803 .readlock = srcu_torture_read_lock, 804 .read_delay = srcu_read_delay, 805 .readunlock = srcu_torture_read_unlock, 806 .readlock_held = torture_srcu_read_lock_held, 807 .get_gp_seq = srcu_torture_completed, 808 .deferred_free = srcu_torture_deferred_free, 809 .sync = srcu_torture_synchronize, 810 .exp_sync = srcu_torture_synchronize_expedited, 811 .same_gp_state = same_state_synchronize_srcu, 812 .get_comp_state = get_completed_synchronize_srcu, 813 .get_gp_state = srcu_torture_get_gp_state, 814 .start_gp_poll = srcu_torture_start_gp_poll, 815 .poll_gp_state = srcu_torture_poll_gp_state, 816 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 817 .call = srcu_torture_call, 818 .cb_barrier = srcu_torture_barrier, 819 .stats = srcu_torture_stats, 820 .get_gp_data = srcu_get_gp_data, 821 .cbflood_max = 50000, 822 .irq_capable = 1, 823 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 824 .debug_objects = 1, 825 .name = "srcud" 826 }; 827 828 /* As above, but broken due to inappropriate reader extension. */ 829 static struct rcu_torture_ops busted_srcud_ops = { 830 .ttype = SRCU_FLAVOR, 831 .init = srcu_torture_init, 832 .cleanup = srcu_torture_cleanup, 833 .readlock = srcu_torture_read_lock, 834 .read_delay = rcu_read_delay, 835 .readunlock = srcu_torture_read_unlock, 836 .readlock_held = torture_srcu_read_lock_held, 837 .get_gp_seq = srcu_torture_completed, 838 .deferred_free = srcu_torture_deferred_free, 839 .sync = srcu_torture_synchronize, 840 .exp_sync = srcu_torture_synchronize_expedited, 841 .call = srcu_torture_call, 842 .cb_barrier = srcu_torture_barrier, 843 .stats = srcu_torture_stats, 844 .irq_capable = 1, 845 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 846 .extendables = RCUTORTURE_MAX_EXTEND, 847 .name = "busted_srcud" 848 }; 849 850 /* 851 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 852 * This implementation does not necessarily work well with CPU hotplug. 853 */ 854 855 static void synchronize_rcu_trivial(void) 856 { 857 int cpu; 858 859 for_each_online_cpu(cpu) { 860 torture_sched_setaffinity(current->pid, cpumask_of(cpu)); 861 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 862 } 863 } 864 865 static int rcu_torture_read_lock_trivial(void) 866 { 867 preempt_disable(); 868 return 0; 869 } 870 871 static void rcu_torture_read_unlock_trivial(int idx) 872 { 873 preempt_enable(); 874 } 875 876 static struct rcu_torture_ops trivial_ops = { 877 .ttype = RCU_TRIVIAL_FLAVOR, 878 .init = rcu_sync_torture_init, 879 .readlock = rcu_torture_read_lock_trivial, 880 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 881 .readunlock = rcu_torture_read_unlock_trivial, 882 .readlock_held = torture_readlock_not_held, 883 .get_gp_seq = rcu_no_completed, 884 .sync = synchronize_rcu_trivial, 885 .exp_sync = synchronize_rcu_trivial, 886 .irq_capable = 1, 887 .name = "trivial" 888 }; 889 890 #ifdef CONFIG_TASKS_RCU 891 892 /* 893 * Definitions for RCU-tasks torture testing. 894 */ 895 896 static int tasks_torture_read_lock(void) 897 { 898 return 0; 899 } 900 901 static void tasks_torture_read_unlock(int idx) 902 { 903 } 904 905 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 906 { 907 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 908 } 909 910 static void synchronize_rcu_mult_test(void) 911 { 912 synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); 913 } 914 915 static struct rcu_torture_ops tasks_ops = { 916 .ttype = RCU_TASKS_FLAVOR, 917 .init = rcu_sync_torture_init, 918 .readlock = tasks_torture_read_lock, 919 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 920 .readunlock = tasks_torture_read_unlock, 921 .get_gp_seq = rcu_no_completed, 922 .deferred_free = rcu_tasks_torture_deferred_free, 923 .sync = synchronize_rcu_tasks, 924 .exp_sync = synchronize_rcu_mult_test, 925 .call = call_rcu_tasks, 926 .cb_barrier = rcu_barrier_tasks, 927 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, 928 .get_gp_data = rcu_tasks_get_gp_data, 929 .irq_capable = 1, 930 .slow_gps = 1, 931 .name = "tasks" 932 }; 933 934 #define TASKS_OPS &tasks_ops, 935 936 #else // #ifdef CONFIG_TASKS_RCU 937 938 #define TASKS_OPS 939 940 #endif // #else #ifdef CONFIG_TASKS_RCU 941 942 943 #ifdef CONFIG_TASKS_RUDE_RCU 944 945 /* 946 * Definitions for rude RCU-tasks torture testing. 947 */ 948 949 static struct rcu_torture_ops tasks_rude_ops = { 950 .ttype = RCU_TASKS_RUDE_FLAVOR, 951 .init = rcu_sync_torture_init, 952 .readlock = rcu_torture_read_lock_trivial, 953 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 954 .readunlock = rcu_torture_read_unlock_trivial, 955 .get_gp_seq = rcu_no_completed, 956 .sync = synchronize_rcu_tasks_rude, 957 .exp_sync = synchronize_rcu_tasks_rude, 958 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, 959 .get_gp_data = rcu_tasks_rude_get_gp_data, 960 .cbflood_max = 50000, 961 .irq_capable = 1, 962 .name = "tasks-rude" 963 }; 964 965 #define TASKS_RUDE_OPS &tasks_rude_ops, 966 967 #else // #ifdef CONFIG_TASKS_RUDE_RCU 968 969 #define TASKS_RUDE_OPS 970 971 #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU 972 973 974 #ifdef CONFIG_TASKS_TRACE_RCU 975 976 /* 977 * Definitions for tracing RCU-tasks torture testing. 978 */ 979 980 static int tasks_tracing_torture_read_lock(void) 981 { 982 rcu_read_lock_trace(); 983 return 0; 984 } 985 986 static void tasks_tracing_torture_read_unlock(int idx) 987 { 988 rcu_read_unlock_trace(); 989 } 990 991 static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) 992 { 993 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); 994 } 995 996 static struct rcu_torture_ops tasks_tracing_ops = { 997 .ttype = RCU_TASKS_TRACING_FLAVOR, 998 .init = rcu_sync_torture_init, 999 .readlock = tasks_tracing_torture_read_lock, 1000 .read_delay = srcu_read_delay, /* just reuse srcu's version. */ 1001 .readunlock = tasks_tracing_torture_read_unlock, 1002 .readlock_held = rcu_read_lock_trace_held, 1003 .get_gp_seq = rcu_no_completed, 1004 .deferred_free = rcu_tasks_tracing_torture_deferred_free, 1005 .sync = synchronize_rcu_tasks_trace, 1006 .exp_sync = synchronize_rcu_tasks_trace, 1007 .call = call_rcu_tasks_trace, 1008 .cb_barrier = rcu_barrier_tasks_trace, 1009 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, 1010 .get_gp_data = rcu_tasks_trace_get_gp_data, 1011 .cbflood_max = 50000, 1012 .irq_capable = 1, 1013 .slow_gps = 1, 1014 .name = "tasks-tracing" 1015 }; 1016 1017 #define TASKS_TRACING_OPS &tasks_tracing_ops, 1018 1019 #else // #ifdef CONFIG_TASKS_TRACE_RCU 1020 1021 #define TASKS_TRACING_OPS 1022 1023 #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU 1024 1025 1026 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 1027 { 1028 if (!cur_ops->gp_diff) 1029 return new - old; 1030 return cur_ops->gp_diff(new, old); 1031 } 1032 1033 /* 1034 * RCU torture priority-boost testing. Runs one real-time thread per 1035 * CPU for moderate bursts, repeatedly starting grace periods and waiting 1036 * for them to complete. If a given grace period takes too long, we assume 1037 * that priority inversion has occurred. 1038 */ 1039 1040 static int old_rt_runtime = -1; 1041 1042 static void rcu_torture_disable_rt_throttle(void) 1043 { 1044 /* 1045 * Disable RT throttling so that rcutorture's boost threads don't get 1046 * throttled. Only possible if rcutorture is built-in otherwise the 1047 * user should manually do this by setting the sched_rt_period_us and 1048 * sched_rt_runtime sysctls. 1049 */ 1050 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 1051 return; 1052 1053 old_rt_runtime = sysctl_sched_rt_runtime; 1054 sysctl_sched_rt_runtime = -1; 1055 } 1056 1057 static void rcu_torture_enable_rt_throttle(void) 1058 { 1059 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 1060 return; 1061 1062 sysctl_sched_rt_runtime = old_rt_runtime; 1063 old_rt_runtime = -1; 1064 } 1065 1066 static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) 1067 { 1068 int cpu; 1069 static int dbg_done; 1070 unsigned long end = jiffies; 1071 bool gp_done; 1072 unsigned long j; 1073 static unsigned long last_persist; 1074 unsigned long lp; 1075 unsigned long mininterval = test_boost_duration * HZ - HZ / 2; 1076 1077 if (end - *start > mininterval) { 1078 // Recheck after checking time to avoid false positives. 1079 smp_mb(); // Time check before grace-period check. 1080 if (cur_ops->poll_gp_state(gp_state)) 1081 return false; // passed, though perhaps just barely 1082 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { 1083 // At most one persisted message per boost test. 1084 j = jiffies; 1085 lp = READ_ONCE(last_persist); 1086 if (time_after(j, lp + mininterval) && 1087 cmpxchg(&last_persist, lp, j) == lp) { 1088 if (cpu < 0) 1089 pr_info("Boost inversion persisted: QS from all CPUs\n"); 1090 else 1091 pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); 1092 } 1093 return false; // passed on a technicality 1094 } 1095 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 1096 n_rcu_torture_boost_failure++; 1097 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { 1098 pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", 1099 current->rt_priority, gp_state, end - *start); 1100 cur_ops->gp_kthread_dbg(); 1101 // Recheck after print to flag grace period ending during splat. 1102 gp_done = cur_ops->poll_gp_state(gp_state); 1103 pr_info("Boost inversion: GP %lu %s.\n", gp_state, 1104 gp_done ? "ended already" : "still pending"); 1105 1106 } 1107 1108 return true; // failed 1109 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { 1110 *start = jiffies; 1111 } 1112 1113 return false; // passed 1114 } 1115 1116 static int rcu_torture_boost(void *arg) 1117 { 1118 unsigned long endtime; 1119 unsigned long gp_state; 1120 unsigned long gp_state_time; 1121 unsigned long oldstarttime; 1122 1123 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 1124 1125 /* Set real-time priority. */ 1126 sched_set_fifo_low(current); 1127 1128 /* Each pass through the following loop does one boost-test cycle. */ 1129 do { 1130 bool failed = false; // Test failed already in this test interval 1131 bool gp_initiated = false; 1132 1133 if (kthread_should_stop()) 1134 goto checkwait; 1135 1136 /* Wait for the next test interval. */ 1137 oldstarttime = READ_ONCE(boost_starttime); 1138 while (time_before(jiffies, oldstarttime)) { 1139 schedule_timeout_interruptible(oldstarttime - jiffies); 1140 if (stutter_wait("rcu_torture_boost")) 1141 sched_set_fifo_low(current); 1142 if (torture_must_stop()) 1143 goto checkwait; 1144 } 1145 1146 // Do one boost-test interval. 1147 endtime = oldstarttime + test_boost_duration * HZ; 1148 while (time_before(jiffies, endtime)) { 1149 // Has current GP gone too long? 1150 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1151 failed = rcu_torture_boost_failed(gp_state, &gp_state_time); 1152 // If we don't have a grace period in flight, start one. 1153 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { 1154 gp_state = cur_ops->start_gp_poll(); 1155 gp_initiated = true; 1156 gp_state_time = jiffies; 1157 } 1158 if (stutter_wait("rcu_torture_boost")) { 1159 sched_set_fifo_low(current); 1160 // If the grace period already ended, 1161 // we don't know when that happened, so 1162 // start over. 1163 if (cur_ops->poll_gp_state(gp_state)) 1164 gp_initiated = false; 1165 } 1166 if (torture_must_stop()) 1167 goto checkwait; 1168 } 1169 1170 // In case the grace period extended beyond the end of the loop. 1171 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1172 rcu_torture_boost_failed(gp_state, &gp_state_time); 1173 1174 /* 1175 * Set the start time of the next test interval. 1176 * Yes, this is vulnerable to long delays, but such 1177 * delays simply cause a false negative for the next 1178 * interval. Besides, we are running at RT priority, 1179 * so delays should be relatively rare. 1180 */ 1181 while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { 1182 if (mutex_trylock(&boost_mutex)) { 1183 if (oldstarttime == boost_starttime) { 1184 WRITE_ONCE(boost_starttime, 1185 jiffies + test_boost_interval * HZ); 1186 n_rcu_torture_boosts++; 1187 } 1188 mutex_unlock(&boost_mutex); 1189 break; 1190 } 1191 schedule_timeout_uninterruptible(HZ / 20); 1192 } 1193 1194 /* Go do the stutter. */ 1195 checkwait: if (stutter_wait("rcu_torture_boost")) 1196 sched_set_fifo_low(current); 1197 } while (!torture_must_stop()); 1198 1199 /* Clean up and exit. */ 1200 while (!kthread_should_stop()) { 1201 torture_shutdown_absorb("rcu_torture_boost"); 1202 schedule_timeout_uninterruptible(HZ / 20); 1203 } 1204 torture_kthread_stopping("rcu_torture_boost"); 1205 return 0; 1206 } 1207 1208 /* 1209 * RCU torture force-quiescent-state kthread. Repeatedly induces 1210 * bursts of calls to force_quiescent_state(), increasing the probability 1211 * of occurrence of some important types of race conditions. 1212 */ 1213 static int 1214 rcu_torture_fqs(void *arg) 1215 { 1216 unsigned long fqs_resume_time; 1217 int fqs_burst_remaining; 1218 int oldnice = task_nice(current); 1219 1220 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 1221 do { 1222 fqs_resume_time = jiffies + fqs_stutter * HZ; 1223 while (time_before(jiffies, fqs_resume_time) && 1224 !kthread_should_stop()) { 1225 schedule_timeout_interruptible(HZ / 20); 1226 } 1227 fqs_burst_remaining = fqs_duration; 1228 while (fqs_burst_remaining > 0 && 1229 !kthread_should_stop()) { 1230 cur_ops->fqs(); 1231 udelay(fqs_holdoff); 1232 fqs_burst_remaining -= fqs_holdoff; 1233 } 1234 if (stutter_wait("rcu_torture_fqs")) 1235 sched_set_normal(current, oldnice); 1236 } while (!torture_must_stop()); 1237 torture_kthread_stopping("rcu_torture_fqs"); 1238 return 0; 1239 } 1240 1241 // Used by writers to randomly choose from the available grace-period primitives. 1242 static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; 1243 static int nsynctypes; 1244 1245 /* 1246 * Determine which grace-period primitives are available. 1247 */ 1248 static void rcu_torture_write_types(void) 1249 { 1250 bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; 1251 bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; 1252 bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; 1253 bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; 1254 1255 /* Initialize synctype[] array. If none set, take default. */ 1256 if (!gp_cond1 && 1257 !gp_cond_exp1 && 1258 !gp_cond_full1 && 1259 !gp_cond_exp_full1 && 1260 !gp_exp1 && 1261 !gp_poll_exp1 && 1262 !gp_poll_exp_full1 && 1263 !gp_normal1 && 1264 !gp_poll1 && 1265 !gp_poll_full1 && 1266 !gp_sync1) { 1267 gp_cond1 = true; 1268 gp_cond_exp1 = true; 1269 gp_cond_full1 = true; 1270 gp_cond_exp_full1 = true; 1271 gp_exp1 = true; 1272 gp_poll_exp1 = true; 1273 gp_poll_exp_full1 = true; 1274 gp_normal1 = true; 1275 gp_poll1 = true; 1276 gp_poll_full1 = true; 1277 gp_sync1 = true; 1278 } 1279 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { 1280 synctype[nsynctypes++] = RTWS_COND_GET; 1281 pr_info("%s: Testing conditional GPs.\n", __func__); 1282 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { 1283 pr_alert("%s: gp_cond without primitives.\n", __func__); 1284 } 1285 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { 1286 synctype[nsynctypes++] = RTWS_COND_GET_EXP; 1287 pr_info("%s: Testing conditional expedited GPs.\n", __func__); 1288 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { 1289 pr_alert("%s: gp_cond_exp without primitives.\n", __func__); 1290 } 1291 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { 1292 synctype[nsynctypes++] = RTWS_COND_GET_FULL; 1293 pr_info("%s: Testing conditional full-state GPs.\n", __func__); 1294 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { 1295 pr_alert("%s: gp_cond_full without primitives.\n", __func__); 1296 } 1297 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { 1298 synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; 1299 pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); 1300 } else if (gp_cond_exp_full && 1301 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { 1302 pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); 1303 } 1304 if (gp_exp1 && cur_ops->exp_sync) { 1305 synctype[nsynctypes++] = RTWS_EXP_SYNC; 1306 pr_info("%s: Testing expedited GPs.\n", __func__); 1307 } else if (gp_exp && !cur_ops->exp_sync) { 1308 pr_alert("%s: gp_exp without primitives.\n", __func__); 1309 } 1310 if (gp_normal1 && cur_ops->deferred_free) { 1311 synctype[nsynctypes++] = RTWS_DEF_FREE; 1312 pr_info("%s: Testing asynchronous GPs.\n", __func__); 1313 } else if (gp_normal && !cur_ops->deferred_free) { 1314 pr_alert("%s: gp_normal without primitives.\n", __func__); 1315 } 1316 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state && 1317 cur_ops->start_gp_poll && cur_ops->poll_gp_state) { 1318 synctype[nsynctypes++] = RTWS_POLL_GET; 1319 pr_info("%s: Testing polling GPs.\n", __func__); 1320 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { 1321 pr_alert("%s: gp_poll without primitives.\n", __func__); 1322 } 1323 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full 1324 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { 1325 synctype[nsynctypes++] = RTWS_POLL_GET_FULL; 1326 pr_info("%s: Testing polling full-state GPs.\n", __func__); 1327 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { 1328 pr_alert("%s: gp_poll_full without primitives.\n", __func__); 1329 } 1330 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { 1331 synctype[nsynctypes++] = RTWS_POLL_GET_EXP; 1332 pr_info("%s: Testing polling expedited GPs.\n", __func__); 1333 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { 1334 pr_alert("%s: gp_poll_exp without primitives.\n", __func__); 1335 } 1336 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { 1337 synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; 1338 pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); 1339 } else if (gp_poll_exp_full && 1340 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { 1341 pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); 1342 } 1343 if (gp_sync1 && cur_ops->sync) { 1344 synctype[nsynctypes++] = RTWS_SYNC; 1345 pr_info("%s: Testing normal GPs.\n", __func__); 1346 } else if (gp_sync && !cur_ops->sync) { 1347 pr_alert("%s: gp_sync without primitives.\n", __func__); 1348 } 1349 pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes); 1350 } 1351 1352 /* 1353 * Do the specified rcu_torture_writer() synchronous grace period, 1354 * while also testing out the polled APIs. Note well that the single-CPU 1355 * grace-period optimizations must be accounted for. 1356 */ 1357 static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) 1358 { 1359 unsigned long cookie; 1360 struct rcu_gp_oldstate cookie_full; 1361 bool dopoll; 1362 bool dopoll_full; 1363 unsigned long r = torture_random(trsp); 1364 1365 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); 1366 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); 1367 if (dopoll || dopoll_full) 1368 cpus_read_lock(); 1369 if (dopoll) 1370 cookie = cur_ops->get_gp_state(); 1371 if (dopoll_full) 1372 cur_ops->get_gp_state_full(&cookie_full); 1373 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) 1374 sync(); 1375 sync(); 1376 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), 1377 "%s: Cookie check 3 failed %pS() online %*pbl.", 1378 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1379 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), 1380 "%s: Cookie check 4 failed %pS() online %*pbl", 1381 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1382 if (dopoll || dopoll_full) 1383 cpus_read_unlock(); 1384 } 1385 1386 /* 1387 * RCU torture writer kthread. Repeatedly substitutes a new structure 1388 * for that pointed to by rcu_torture_current, freeing the old structure 1389 * after a series of grace periods (the "pipeline"). 1390 */ 1391 static int 1392 rcu_torture_writer(void *arg) 1393 { 1394 bool boot_ended; 1395 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 1396 unsigned long cookie; 1397 struct rcu_gp_oldstate cookie_full; 1398 int expediting = 0; 1399 unsigned long gp_snap; 1400 unsigned long gp_snap1; 1401 struct rcu_gp_oldstate gp_snap_full; 1402 struct rcu_gp_oldstate gp_snap1_full; 1403 int i; 1404 int idx; 1405 int oldnice = task_nice(current); 1406 struct rcu_gp_oldstate *rgo = NULL; 1407 int rgo_size = 0; 1408 struct rcu_torture *rp; 1409 struct rcu_torture *old_rp; 1410 static DEFINE_TORTURE_RANDOM(rand); 1411 unsigned long stallsdone = jiffies; 1412 bool stutter_waited; 1413 unsigned long *ulo = NULL; 1414 int ulo_size = 0; 1415 1416 // If a new stall test is added, this must be adjusted. 1417 if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu) 1418 stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * 1419 HZ * (stall_cpu_repeat + 1); 1420 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 1421 if (!can_expedite) 1422 pr_alert("%s" TORTURE_FLAG 1423 " GP expediting controlled from boot/sysfs for %s.\n", 1424 torture_type, cur_ops->name); 1425 if (WARN_ONCE(nsynctypes == 0, 1426 "%s: No update-side primitives.\n", __func__)) { 1427 /* 1428 * No updates primitives, so don't try updating. 1429 * The resulting test won't be testing much, hence the 1430 * above WARN_ONCE(). 1431 */ 1432 rcu_torture_writer_state = RTWS_STOPPING; 1433 torture_kthread_stopping("rcu_torture_writer"); 1434 return 0; 1435 } 1436 if (cur_ops->poll_active > 0) { 1437 ulo = kzalloc(cur_ops->poll_active * sizeof(ulo[0]), GFP_KERNEL); 1438 if (!WARN_ON(!ulo)) 1439 ulo_size = cur_ops->poll_active; 1440 } 1441 if (cur_ops->poll_active_full > 0) { 1442 rgo = kzalloc(cur_ops->poll_active_full * sizeof(rgo[0]), GFP_KERNEL); 1443 if (!WARN_ON(!rgo)) 1444 rgo_size = cur_ops->poll_active_full; 1445 } 1446 1447 do { 1448 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1449 torture_hrtimeout_us(500, 1000, &rand); 1450 rp = rcu_torture_alloc(); 1451 if (rp == NULL) 1452 continue; 1453 rp->rtort_pipe_count = 0; 1454 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 1455 rcu_torture_writer_state = RTWS_DELAY; 1456 udelay(torture_random(&rand) & 0x3ff); 1457 rcu_torture_writer_state = RTWS_REPLACE; 1458 old_rp = rcu_dereference_check(rcu_torture_current, 1459 current == writer_task); 1460 rp->rtort_mbtest = 1; 1461 rcu_assign_pointer(rcu_torture_current, rp); 1462 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1463 if (old_rp) { 1464 i = old_rp->rtort_pipe_count; 1465 if (i > RCU_TORTURE_PIPE_LEN) 1466 i = RCU_TORTURE_PIPE_LEN; 1467 atomic_inc(&rcu_torture_wcount[i]); 1468 WRITE_ONCE(old_rp->rtort_pipe_count, 1469 old_rp->rtort_pipe_count + 1); 1470 ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count); 1471 1472 // Make sure readers block polled grace periods. 1473 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { 1474 idx = cur_ops->readlock(); 1475 cookie = cur_ops->get_gp_state(); 1476 WARN_ONCE(cur_ops->poll_gp_state(cookie), 1477 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", 1478 __func__, 1479 rcu_torture_writer_state_getname(), 1480 rcu_torture_writer_state, 1481 cookie, cur_ops->get_gp_state()); 1482 if (cur_ops->get_comp_state) { 1483 cookie = cur_ops->get_comp_state(); 1484 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); 1485 } 1486 cur_ops->readunlock(idx); 1487 } 1488 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { 1489 idx = cur_ops->readlock(); 1490 cur_ops->get_gp_state_full(&cookie_full); 1491 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 1492 "%s: Cookie check 5 failed %s(%d) online %*pbl\n", 1493 __func__, 1494 rcu_torture_writer_state_getname(), 1495 rcu_torture_writer_state, 1496 cpumask_pr_args(cpu_online_mask)); 1497 if (cur_ops->get_comp_state_full) { 1498 cur_ops->get_comp_state_full(&cookie_full); 1499 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); 1500 } 1501 cur_ops->readunlock(idx); 1502 } 1503 switch (synctype[torture_random(&rand) % nsynctypes]) { 1504 case RTWS_DEF_FREE: 1505 rcu_torture_writer_state = RTWS_DEF_FREE; 1506 cur_ops->deferred_free(old_rp); 1507 break; 1508 case RTWS_EXP_SYNC: 1509 rcu_torture_writer_state = RTWS_EXP_SYNC; 1510 do_rtws_sync(&rand, cur_ops->exp_sync); 1511 rcu_torture_pipe_update(old_rp); 1512 break; 1513 case RTWS_COND_GET: 1514 rcu_torture_writer_state = RTWS_COND_GET; 1515 gp_snap = cur_ops->get_gp_state(); 1516 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1517 rcu_torture_writer_state = RTWS_COND_SYNC; 1518 cur_ops->cond_sync(gp_snap); 1519 rcu_torture_pipe_update(old_rp); 1520 break; 1521 case RTWS_COND_GET_EXP: 1522 rcu_torture_writer_state = RTWS_COND_GET_EXP; 1523 gp_snap = cur_ops->get_gp_state_exp(); 1524 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1525 rcu_torture_writer_state = RTWS_COND_SYNC_EXP; 1526 cur_ops->cond_sync_exp(gp_snap); 1527 rcu_torture_pipe_update(old_rp); 1528 break; 1529 case RTWS_COND_GET_FULL: 1530 rcu_torture_writer_state = RTWS_COND_GET_FULL; 1531 cur_ops->get_gp_state_full(&gp_snap_full); 1532 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1533 rcu_torture_writer_state = RTWS_COND_SYNC_FULL; 1534 cur_ops->cond_sync_full(&gp_snap_full); 1535 rcu_torture_pipe_update(old_rp); 1536 break; 1537 case RTWS_COND_GET_EXP_FULL: 1538 rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; 1539 cur_ops->get_gp_state_full(&gp_snap_full); 1540 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1541 rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; 1542 cur_ops->cond_sync_exp_full(&gp_snap_full); 1543 rcu_torture_pipe_update(old_rp); 1544 break; 1545 case RTWS_POLL_GET: 1546 rcu_torture_writer_state = RTWS_POLL_GET; 1547 for (i = 0; i < ulo_size; i++) 1548 ulo[i] = cur_ops->get_comp_state(); 1549 gp_snap = cur_ops->start_gp_poll(); 1550 rcu_torture_writer_state = RTWS_POLL_WAIT; 1551 while (!cur_ops->poll_gp_state(gp_snap)) { 1552 gp_snap1 = cur_ops->get_gp_state(); 1553 for (i = 0; i < ulo_size; i++) 1554 if (cur_ops->poll_gp_state(ulo[i]) || 1555 cur_ops->same_gp_state(ulo[i], gp_snap1)) { 1556 ulo[i] = gp_snap1; 1557 break; 1558 } 1559 WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size); 1560 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1561 &rand); 1562 } 1563 rcu_torture_pipe_update(old_rp); 1564 break; 1565 case RTWS_POLL_GET_FULL: 1566 rcu_torture_writer_state = RTWS_POLL_GET_FULL; 1567 for (i = 0; i < rgo_size; i++) 1568 cur_ops->get_comp_state_full(&rgo[i]); 1569 cur_ops->start_gp_poll_full(&gp_snap_full); 1570 rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; 1571 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1572 cur_ops->get_gp_state_full(&gp_snap1_full); 1573 for (i = 0; i < rgo_size; i++) 1574 if (cur_ops->poll_gp_state_full(&rgo[i]) || 1575 cur_ops->same_gp_state_full(&rgo[i], 1576 &gp_snap1_full)) { 1577 rgo[i] = gp_snap1_full; 1578 break; 1579 } 1580 WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size); 1581 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1582 &rand); 1583 } 1584 rcu_torture_pipe_update(old_rp); 1585 break; 1586 case RTWS_POLL_GET_EXP: 1587 rcu_torture_writer_state = RTWS_POLL_GET_EXP; 1588 gp_snap = cur_ops->start_gp_poll_exp(); 1589 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; 1590 while (!cur_ops->poll_gp_state_exp(gp_snap)) 1591 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1592 &rand); 1593 rcu_torture_pipe_update(old_rp); 1594 break; 1595 case RTWS_POLL_GET_EXP_FULL: 1596 rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; 1597 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1598 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; 1599 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) 1600 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1601 &rand); 1602 rcu_torture_pipe_update(old_rp); 1603 break; 1604 case RTWS_SYNC: 1605 rcu_torture_writer_state = RTWS_SYNC; 1606 do_rtws_sync(&rand, cur_ops->sync); 1607 rcu_torture_pipe_update(old_rp); 1608 break; 1609 default: 1610 WARN_ON_ONCE(1); 1611 break; 1612 } 1613 } 1614 WRITE_ONCE(rcu_torture_current_version, 1615 rcu_torture_current_version + 1); 1616 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1617 if (can_expedite && 1618 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1619 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1620 if (expediting >= 0) 1621 rcu_expedite_gp(); 1622 else 1623 rcu_unexpedite_gp(); 1624 if (++expediting > 3) 1625 expediting = -expediting; 1626 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1627 can_expedite = !rcu_gp_is_expedited() && 1628 !rcu_gp_is_normal(); 1629 } 1630 rcu_torture_writer_state = RTWS_STUTTER; 1631 boot_ended = rcu_inkernel_boot_has_ended(); 1632 stutter_waited = stutter_wait("rcu_torture_writer"); 1633 if (stutter_waited && 1634 !atomic_read(&rcu_fwd_cb_nodelay) && 1635 !cur_ops->slow_gps && 1636 !torture_must_stop() && 1637 boot_ended && 1638 time_after(jiffies, stallsdone)) 1639 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1640 if (list_empty(&rcu_tortures[i].rtort_free) && 1641 rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { 1642 tracing_off(); 1643 if (cur_ops->gp_kthread_dbg) 1644 cur_ops->gp_kthread_dbg(); 1645 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1646 rcu_ftrace_dump(DUMP_ALL); 1647 } 1648 if (stutter_waited) 1649 sched_set_normal(current, oldnice); 1650 } while (!torture_must_stop()); 1651 rcu_torture_current = NULL; // Let stats task know that we are done. 1652 /* Reset expediting back to unexpedited. */ 1653 if (expediting > 0) 1654 expediting = -expediting; 1655 while (can_expedite && expediting++ < 0) 1656 rcu_unexpedite_gp(); 1657 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1658 if (!can_expedite) 1659 pr_alert("%s" TORTURE_FLAG 1660 " Dynamic grace-period expediting was disabled.\n", 1661 torture_type); 1662 kfree(ulo); 1663 kfree(rgo); 1664 rcu_torture_writer_state = RTWS_STOPPING; 1665 torture_kthread_stopping("rcu_torture_writer"); 1666 return 0; 1667 } 1668 1669 /* 1670 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1671 * delay between calls. 1672 */ 1673 static int 1674 rcu_torture_fakewriter(void *arg) 1675 { 1676 unsigned long gp_snap; 1677 struct rcu_gp_oldstate gp_snap_full; 1678 DEFINE_TORTURE_RANDOM(rand); 1679 1680 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1681 set_user_nice(current, MAX_NICE); 1682 1683 if (WARN_ONCE(nsynctypes == 0, 1684 "%s: No update-side primitives.\n", __func__)) { 1685 /* 1686 * No updates primitives, so don't try updating. 1687 * The resulting test won't be testing much, hence the 1688 * above WARN_ONCE(). 1689 */ 1690 torture_kthread_stopping("rcu_torture_fakewriter"); 1691 return 0; 1692 } 1693 1694 do { 1695 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); 1696 if (cur_ops->cb_barrier != NULL && 1697 torture_random(&rand) % (nfakewriters * 8) == 0) { 1698 cur_ops->cb_barrier(); 1699 } else { 1700 switch (synctype[torture_random(&rand) % nsynctypes]) { 1701 case RTWS_DEF_FREE: 1702 break; 1703 case RTWS_EXP_SYNC: 1704 cur_ops->exp_sync(); 1705 break; 1706 case RTWS_COND_GET: 1707 gp_snap = cur_ops->get_gp_state(); 1708 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1709 cur_ops->cond_sync(gp_snap); 1710 break; 1711 case RTWS_COND_GET_EXP: 1712 gp_snap = cur_ops->get_gp_state_exp(); 1713 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1714 cur_ops->cond_sync_exp(gp_snap); 1715 break; 1716 case RTWS_COND_GET_FULL: 1717 cur_ops->get_gp_state_full(&gp_snap_full); 1718 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1719 cur_ops->cond_sync_full(&gp_snap_full); 1720 break; 1721 case RTWS_COND_GET_EXP_FULL: 1722 cur_ops->get_gp_state_full(&gp_snap_full); 1723 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1724 cur_ops->cond_sync_exp_full(&gp_snap_full); 1725 break; 1726 case RTWS_POLL_GET: 1727 if (cur_ops->start_poll_irqsoff) 1728 local_irq_disable(); 1729 gp_snap = cur_ops->start_gp_poll(); 1730 if (cur_ops->start_poll_irqsoff) 1731 local_irq_enable(); 1732 while (!cur_ops->poll_gp_state(gp_snap)) { 1733 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1734 &rand); 1735 } 1736 break; 1737 case RTWS_POLL_GET_FULL: 1738 if (cur_ops->start_poll_irqsoff) 1739 local_irq_disable(); 1740 cur_ops->start_gp_poll_full(&gp_snap_full); 1741 if (cur_ops->start_poll_irqsoff) 1742 local_irq_enable(); 1743 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1744 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1745 &rand); 1746 } 1747 break; 1748 case RTWS_POLL_GET_EXP: 1749 gp_snap = cur_ops->start_gp_poll_exp(); 1750 while (!cur_ops->poll_gp_state_exp(gp_snap)) { 1751 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1752 &rand); 1753 } 1754 break; 1755 case RTWS_POLL_GET_EXP_FULL: 1756 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1757 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1758 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1759 &rand); 1760 } 1761 break; 1762 case RTWS_SYNC: 1763 cur_ops->sync(); 1764 break; 1765 default: 1766 WARN_ON_ONCE(1); 1767 break; 1768 } 1769 } 1770 stutter_wait("rcu_torture_fakewriter"); 1771 } while (!torture_must_stop()); 1772 1773 torture_kthread_stopping("rcu_torture_fakewriter"); 1774 return 0; 1775 } 1776 1777 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1778 { 1779 kfree(rhp); 1780 } 1781 1782 // Set up and carry out testing of RCU's global memory ordering 1783 static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, 1784 struct torture_random_state *trsp) 1785 { 1786 unsigned long loops; 1787 int noc = torture_num_online_cpus(); 1788 int rdrchked; 1789 int rdrchker; 1790 struct rcu_torture_reader_check *rtrcp; // Me. 1791 struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. 1792 struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. 1793 struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. 1794 1795 if (myid < 0) 1796 return; // Don't try this from timer handlers. 1797 1798 // Increment my counter. 1799 rtrcp = &rcu_torture_reader_mbchk[myid]; 1800 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); 1801 1802 // Attempt to assign someone else some checking work. 1803 rdrchked = torture_random(trsp) % nrealreaders; 1804 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1805 rdrchker = torture_random(trsp) % nrealreaders; 1806 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; 1807 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && 1808 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. 1809 !READ_ONCE(rtp->rtort_chkp) && 1810 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. 1811 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); 1812 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); 1813 rtrcp->rtc_chkrdr = rdrchked; 1814 WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. 1815 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || 1816 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) 1817 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. 1818 } 1819 1820 // If assigned some completed work, do it! 1821 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); 1822 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) 1823 return; // No work or work not yet ready. 1824 rdrchked = rtrcp_assigner->rtc_chkrdr; 1825 if (WARN_ON_ONCE(rdrchked < 0)) 1826 return; 1827 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1828 loops = READ_ONCE(rtrcp_chked->rtc_myloops); 1829 atomic_inc(&n_rcu_torture_mbchk_tries); 1830 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) 1831 atomic_inc(&n_rcu_torture_mbchk_fail); 1832 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; 1833 rtrcp_assigner->rtc_ready = 0; 1834 smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. 1835 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. 1836 } 1837 1838 /* 1839 * Do one extension of an RCU read-side critical section using the 1840 * current reader state in readstate (set to zero for initial entry 1841 * to extended critical section), set the new state as specified by 1842 * newstate (set to zero for final exit from extended critical section), 1843 * and random-number-generator state in trsp. If this is neither the 1844 * beginning or end of the critical section and if there was actually a 1845 * change, do a ->read_delay(). 1846 */ 1847 static void rcutorture_one_extend(int *readstate, int newstate, 1848 struct torture_random_state *trsp, 1849 struct rt_read_seg *rtrsp) 1850 { 1851 unsigned long flags; 1852 int idxnew1 = -1; 1853 int idxnew2 = -1; 1854 int idxold1 = *readstate; 1855 int idxold2 = idxold1; 1856 int statesnew = ~*readstate & newstate; 1857 int statesold = *readstate & ~newstate; 1858 1859 WARN_ON_ONCE(idxold2 < 0); 1860 WARN_ON_ONCE(idxold2 & ~RCUTORTURE_RDR_ALLBITS); 1861 rtrsp->rt_readstate = newstate; 1862 1863 /* First, put new protection in place to avoid critical-section gap. */ 1864 if (statesnew & RCUTORTURE_RDR_BH) 1865 local_bh_disable(); 1866 if (statesnew & RCUTORTURE_RDR_RBH) 1867 rcu_read_lock_bh(); 1868 if (statesnew & RCUTORTURE_RDR_IRQ) 1869 local_irq_disable(); 1870 if (statesnew & RCUTORTURE_RDR_PREEMPT) 1871 preempt_disable(); 1872 if (statesnew & RCUTORTURE_RDR_SCHED) 1873 rcu_read_lock_sched(); 1874 if (statesnew & RCUTORTURE_RDR_RCU_1) 1875 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 1876 if (statesnew & RCUTORTURE_RDR_RCU_2) 1877 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2; 1878 1879 /* 1880 * Next, remove old protection, in decreasing order of strength 1881 * to avoid unlock paths that aren't safe in the stronger 1882 * context. Namely: BH can not be enabled with disabled interrupts. 1883 * Additionally PREEMPT_RT requires that BH is enabled in preemptible 1884 * context. 1885 */ 1886 if (statesold & RCUTORTURE_RDR_IRQ) 1887 local_irq_enable(); 1888 if (statesold & RCUTORTURE_RDR_PREEMPT) 1889 preempt_enable(); 1890 if (statesold & RCUTORTURE_RDR_SCHED) 1891 rcu_read_unlock_sched(); 1892 if (statesold & RCUTORTURE_RDR_BH) 1893 local_bh_enable(); 1894 if (statesold & RCUTORTURE_RDR_RBH) 1895 rcu_read_unlock_bh(); 1896 if (statesold & RCUTORTURE_RDR_RCU_2) { 1897 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2); 1898 WARN_ON_ONCE(idxnew2 != -1); 1899 idxold2 = 0; 1900 } 1901 if (statesold & RCUTORTURE_RDR_RCU_1) { 1902 bool lockit; 1903 1904 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); 1905 if (lockit) 1906 raw_spin_lock_irqsave(¤t->pi_lock, flags); 1907 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 1908 WARN_ON_ONCE(idxnew1 != -1); 1909 idxold1 = 0; 1910 if (lockit) 1911 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 1912 } 1913 1914 /* Delay if neither beginning nor end and there was a change. */ 1915 if ((statesnew || statesold) && *readstate && newstate) 1916 cur_ops->read_delay(trsp, rtrsp); 1917 1918 /* Update the reader state. */ 1919 if (idxnew1 == -1) 1920 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; 1921 WARN_ON_ONCE(idxnew1 < 0); 1922 if (idxnew2 == -1) 1923 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; 1924 WARN_ON_ONCE(idxnew2 < 0); 1925 *readstate = idxnew1 | idxnew2 | newstate; 1926 WARN_ON_ONCE(*readstate < 0); 1927 if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS)) 1928 pr_info("Unexpected readstate value of %#x\n", *readstate); 1929 } 1930 1931 /* Return the biggest extendables mask given current RCU and boot parameters. */ 1932 static int rcutorture_extend_mask_max(void) 1933 { 1934 int mask; 1935 1936 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 1937 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 1938 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 1939 return mask; 1940 } 1941 1942 /* Return a random protection state mask, but with at least one bit set. */ 1943 static int 1944 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 1945 { 1946 int mask = rcutorture_extend_mask_max(); 1947 unsigned long randmask1 = torture_random(trsp); 1948 unsigned long randmask2 = randmask1 >> 3; 1949 unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 1950 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 1951 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 1952 1953 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits. 1954 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 1955 if (!(randmask1 & 0x7)) 1956 mask = mask & randmask2; 1957 else 1958 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 1959 1960 // Can't have nested RCU reader without outer RCU reader. 1961 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { 1962 if (oldmask & RCUTORTURE_RDR_RCU_1) 1963 mask &= ~RCUTORTURE_RDR_RCU_2; 1964 else 1965 mask |= RCUTORTURE_RDR_RCU_1; 1966 } 1967 1968 /* 1969 * Can't enable bh w/irq disabled. 1970 */ 1971 if (mask & RCUTORTURE_RDR_IRQ) 1972 mask |= oldmask & bhs; 1973 1974 /* 1975 * Ideally these sequences would be detected in debug builds 1976 * (regardless of RT), but until then don't stop testing 1977 * them on non-RT. 1978 */ 1979 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1980 /* Can't modify BH in atomic context */ 1981 if (oldmask & preempts_irq) 1982 mask &= ~bhs; 1983 if ((oldmask | mask) & preempts_irq) 1984 mask |= oldmask & bhs; 1985 } 1986 1987 return mask ?: RCUTORTURE_RDR_RCU_1; 1988 } 1989 1990 /* 1991 * Do a randomly selected number of extensions of an existing RCU read-side 1992 * critical section. 1993 */ 1994 static struct rt_read_seg * 1995 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, 1996 struct rt_read_seg *rtrsp) 1997 { 1998 int i; 1999 int j; 2000 int mask = rcutorture_extend_mask_max(); 2001 2002 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 2003 if (!((mask - 1) & mask)) 2004 return rtrsp; /* Current RCU reader not extendable. */ 2005 /* Bias towards larger numbers of loops. */ 2006 i = torture_random(trsp); 2007 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 2008 for (j = 0; j < i; j++) { 2009 mask = rcutorture_extend_mask(*readstate, trsp); 2010 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 2011 } 2012 return &rtrsp[j]; 2013 } 2014 2015 /* 2016 * Do one read-side critical section, returning false if there was 2017 * no data to read. Can be invoked both from process context and 2018 * from a timer handler. 2019 */ 2020 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) 2021 { 2022 bool checkpolling = !(torture_random(trsp) & 0xfff); 2023 unsigned long cookie; 2024 struct rcu_gp_oldstate cookie_full; 2025 int i; 2026 unsigned long started; 2027 unsigned long completed; 2028 int newstate; 2029 struct rcu_torture *p; 2030 int pipe_count; 2031 int readstate = 0; 2032 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } }; 2033 struct rt_read_seg *rtrsp = &rtseg[0]; 2034 struct rt_read_seg *rtrsp1; 2035 unsigned long long ts; 2036 2037 WARN_ON_ONCE(!rcu_is_watching()); 2038 newstate = rcutorture_extend_mask(readstate, trsp); 2039 rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); 2040 if (checkpolling) { 2041 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2042 cookie = cur_ops->get_gp_state(); 2043 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2044 cur_ops->get_gp_state_full(&cookie_full); 2045 } 2046 started = cur_ops->get_gp_seq(); 2047 ts = rcu_trace_clock_local(); 2048 p = rcu_dereference_check(rcu_torture_current, 2049 !cur_ops->readlock_held || cur_ops->readlock_held()); 2050 if (p == NULL) { 2051 /* Wait for rcu_torture_writer to get underway */ 2052 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 2053 return false; 2054 } 2055 if (p->rtort_mbtest == 0) 2056 atomic_inc(&n_rcu_torture_mberror); 2057 rcu_torture_reader_do_mbchk(myid, p, trsp); 2058 rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp); 2059 preempt_disable(); 2060 pipe_count = READ_ONCE(p->rtort_pipe_count); 2061 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 2062 // Should not happen in a correct RCU implementation, 2063 // happens quite often for torture_type=busted. 2064 pipe_count = RCU_TORTURE_PIPE_LEN; 2065 } 2066 completed = cur_ops->get_gp_seq(); 2067 if (pipe_count > 1) { 2068 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, 2069 ts, started, completed); 2070 rcu_ftrace_dump(DUMP_ALL); 2071 } 2072 __this_cpu_inc(rcu_torture_count[pipe_count]); 2073 completed = rcutorture_seq_diff(completed, started); 2074 if (completed > RCU_TORTURE_PIPE_LEN) { 2075 /* Should not happen, but... */ 2076 completed = RCU_TORTURE_PIPE_LEN; 2077 } 2078 __this_cpu_inc(rcu_torture_batch[completed]); 2079 preempt_enable(); 2080 if (checkpolling) { 2081 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2082 WARN_ONCE(cur_ops->poll_gp_state(cookie), 2083 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", 2084 __func__, 2085 rcu_torture_writer_state_getname(), 2086 rcu_torture_writer_state, 2087 cookie, cur_ops->get_gp_state()); 2088 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2089 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 2090 "%s: Cookie check 6 failed %s(%d) online %*pbl\n", 2091 __func__, 2092 rcu_torture_writer_state_getname(), 2093 rcu_torture_writer_state, 2094 cpumask_pr_args(cpu_online_mask)); 2095 } 2096 rcutorture_one_extend(&readstate, 0, trsp, rtrsp); 2097 WARN_ON_ONCE(readstate); 2098 // This next splat is expected behavior if leakpointer, especially 2099 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. 2100 WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1); 2101 2102 /* If error or close call, record the sequence of reader protections. */ 2103 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 2104 i = 0; 2105 for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++) 2106 err_segs[i++] = *rtrsp1; 2107 rt_read_nsegs = i; 2108 } 2109 2110 return true; 2111 } 2112 2113 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 2114 2115 /* 2116 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2117 * incrementing the corresponding element of the pipeline array. The 2118 * counter in the element should never be greater than 1, otherwise, the 2119 * RCU implementation is broken. 2120 */ 2121 static void rcu_torture_timer(struct timer_list *unused) 2122 { 2123 atomic_long_inc(&n_rcu_torture_timers); 2124 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); 2125 2126 /* Test call_rcu() invocation from interrupt handler. */ 2127 if (cur_ops->call) { 2128 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 2129 2130 if (rhp) 2131 cur_ops->call(rhp, rcu_torture_timer_cb); 2132 } 2133 } 2134 2135 /* 2136 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 2137 * incrementing the corresponding element of the pipeline array. The 2138 * counter in the element should never be greater than 1, otherwise, the 2139 * RCU implementation is broken. 2140 */ 2141 static int 2142 rcu_torture_reader(void *arg) 2143 { 2144 unsigned long lastsleep = jiffies; 2145 long myid = (long)arg; 2146 int mynumonline = myid; 2147 DEFINE_TORTURE_RANDOM(rand); 2148 struct timer_list t; 2149 2150 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 2151 set_user_nice(current, MAX_NICE); 2152 if (irqreader && cur_ops->irq_capable) 2153 timer_setup_on_stack(&t, rcu_torture_timer, 0); 2154 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 2155 do { 2156 if (irqreader && cur_ops->irq_capable) { 2157 if (!timer_pending(&t)) 2158 mod_timer(&t, jiffies + 1); 2159 } 2160 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) 2161 schedule_timeout_interruptible(HZ); 2162 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 2163 torture_hrtimeout_us(500, 1000, &rand); 2164 lastsleep = jiffies + 10; 2165 } 2166 while (torture_num_online_cpus() < mynumonline && !torture_must_stop()) 2167 schedule_timeout_interruptible(HZ / 5); 2168 stutter_wait("rcu_torture_reader"); 2169 } while (!torture_must_stop()); 2170 if (irqreader && cur_ops->irq_capable) { 2171 del_timer_sync(&t); 2172 destroy_timer_on_stack(&t); 2173 } 2174 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2175 torture_kthread_stopping("rcu_torture_reader"); 2176 return 0; 2177 } 2178 2179 /* 2180 * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to 2181 * increase race probabilities and fuzzes the interval between toggling. 2182 */ 2183 static int rcu_nocb_toggle(void *arg) 2184 { 2185 int cpu; 2186 int maxcpu = -1; 2187 int oldnice = task_nice(current); 2188 long r; 2189 DEFINE_TORTURE_RANDOM(rand); 2190 ktime_t toggle_delay; 2191 unsigned long toggle_fuzz; 2192 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); 2193 2194 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); 2195 while (!rcu_inkernel_boot_has_ended()) 2196 schedule_timeout_interruptible(HZ / 10); 2197 for_each_possible_cpu(cpu) 2198 maxcpu = cpu; 2199 WARN_ON(maxcpu < 0); 2200 if (toggle_interval > ULONG_MAX) 2201 toggle_fuzz = ULONG_MAX >> 3; 2202 else 2203 toggle_fuzz = toggle_interval >> 3; 2204 if (toggle_fuzz <= 0) 2205 toggle_fuzz = NSEC_PER_USEC; 2206 do { 2207 r = torture_random(&rand); 2208 cpu = (r >> 1) % (maxcpu + 1); 2209 if (r & 0x1) { 2210 rcu_nocb_cpu_offload(cpu); 2211 atomic_long_inc(&n_nocb_offload); 2212 } else { 2213 rcu_nocb_cpu_deoffload(cpu); 2214 atomic_long_inc(&n_nocb_deoffload); 2215 } 2216 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; 2217 set_current_state(TASK_INTERRUPTIBLE); 2218 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); 2219 if (stutter_wait("rcu_nocb_toggle")) 2220 sched_set_normal(current, oldnice); 2221 } while (!torture_must_stop()); 2222 torture_kthread_stopping("rcu_nocb_toggle"); 2223 return 0; 2224 } 2225 2226 /* 2227 * Print torture statistics. Caller must ensure that there is only 2228 * one call to this function at a given time!!! This is normally 2229 * accomplished by relying on the module system to only have one copy 2230 * of the module loaded, and then by giving the rcu_torture_stats 2231 * kthread full control (or the init/cleanup functions when rcu_torture_stats 2232 * thread is not running). 2233 */ 2234 static void 2235 rcu_torture_stats_print(void) 2236 { 2237 int cpu; 2238 int i; 2239 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2240 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2241 struct rcu_torture *rtcp; 2242 static unsigned long rtcv_snap = ULONG_MAX; 2243 static bool splatted; 2244 struct task_struct *wtp; 2245 2246 for_each_possible_cpu(cpu) { 2247 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2248 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 2249 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 2250 } 2251 } 2252 for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { 2253 if (pipesummary[i] != 0) 2254 break; 2255 } 2256 2257 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2258 rtcp = rcu_access_pointer(rcu_torture_current); 2259 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 2260 rtcp, 2261 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 2262 rcu_torture_current_version, 2263 list_empty(&rcu_torture_freelist), 2264 atomic_read(&n_rcu_torture_alloc), 2265 atomic_read(&n_rcu_torture_alloc_fail), 2266 atomic_read(&n_rcu_torture_free)); 2267 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ", 2268 atomic_read(&n_rcu_torture_mberror), 2269 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), 2270 n_rcu_torture_barrier_error, 2271 n_rcu_torture_boost_ktrerror); 2272 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 2273 n_rcu_torture_boost_failure, 2274 n_rcu_torture_boosts, 2275 atomic_long_read(&n_rcu_torture_timers)); 2276 torture_onoff_stats(); 2277 pr_cont("barrier: %ld/%ld:%ld ", 2278 data_race(n_barrier_successes), 2279 data_race(n_barrier_attempts), 2280 data_race(n_rcu_torture_barrier_error)); 2281 pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. 2282 pr_cont("nocb-toggles: %ld:%ld\n", 2283 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); 2284 2285 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2286 if (atomic_read(&n_rcu_torture_mberror) || 2287 atomic_read(&n_rcu_torture_mbchk_fail) || 2288 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 2289 n_rcu_torture_boost_failure || i > 1) { 2290 pr_cont("%s", "!!! "); 2291 atomic_inc(&n_rcu_torture_error); 2292 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 2293 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); 2294 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 2295 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 2296 WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) 2297 WARN_ON_ONCE(i > 1); // Too-short grace period 2298 } 2299 pr_cont("Reader Pipe: "); 2300 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2301 pr_cont(" %ld", pipesummary[i]); 2302 pr_cont("\n"); 2303 2304 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2305 pr_cont("Reader Batch: "); 2306 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2307 pr_cont(" %ld", batchsummary[i]); 2308 pr_cont("\n"); 2309 2310 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2311 pr_cont("Free-Block Circulation: "); 2312 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2313 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 2314 } 2315 pr_cont("\n"); 2316 2317 if (cur_ops->stats) 2318 cur_ops->stats(); 2319 if (rtcv_snap == rcu_torture_current_version && 2320 rcu_access_pointer(rcu_torture_current) && 2321 !rcu_stall_is_suppressed()) { 2322 int __maybe_unused flags = 0; 2323 unsigned long __maybe_unused gp_seq = 0; 2324 2325 if (cur_ops->get_gp_data) 2326 cur_ops->get_gp_data(&flags, &gp_seq); 2327 wtp = READ_ONCE(writer_task); 2328 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", 2329 rcu_torture_writer_state_getname(), 2330 rcu_torture_writer_state, gp_seq, flags, 2331 wtp == NULL ? ~0U : wtp->__state, 2332 wtp == NULL ? -1 : (int)task_cpu(wtp)); 2333 if (!splatted && wtp) { 2334 sched_show_task(wtp); 2335 splatted = true; 2336 } 2337 if (cur_ops->gp_kthread_dbg) 2338 cur_ops->gp_kthread_dbg(); 2339 rcu_ftrace_dump(DUMP_ALL); 2340 } 2341 rtcv_snap = rcu_torture_current_version; 2342 } 2343 2344 /* 2345 * Periodically prints torture statistics, if periodic statistics printing 2346 * was specified via the stat_interval module parameter. 2347 */ 2348 static int 2349 rcu_torture_stats(void *arg) 2350 { 2351 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 2352 do { 2353 schedule_timeout_interruptible(stat_interval * HZ); 2354 rcu_torture_stats_print(); 2355 torture_shutdown_absorb("rcu_torture_stats"); 2356 } while (!torture_must_stop()); 2357 torture_kthread_stopping("rcu_torture_stats"); 2358 return 0; 2359 } 2360 2361 /* Test mem_dump_obj() and friends. */ 2362 static void rcu_torture_mem_dump_obj(void) 2363 { 2364 struct rcu_head *rhp; 2365 struct kmem_cache *kcp; 2366 static int z; 2367 2368 kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); 2369 if (WARN_ON_ONCE(!kcp)) 2370 return; 2371 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); 2372 if (WARN_ON_ONCE(!rhp)) { 2373 kmem_cache_destroy(kcp); 2374 return; 2375 } 2376 pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); 2377 pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); 2378 mem_dump_obj(ZERO_SIZE_PTR); 2379 pr_alert("mem_dump_obj(NULL):"); 2380 mem_dump_obj(NULL); 2381 pr_alert("mem_dump_obj(%px):", &rhp); 2382 mem_dump_obj(&rhp); 2383 pr_alert("mem_dump_obj(%px):", rhp); 2384 mem_dump_obj(rhp); 2385 pr_alert("mem_dump_obj(%px):", &rhp->func); 2386 mem_dump_obj(&rhp->func); 2387 pr_alert("mem_dump_obj(%px):", &z); 2388 mem_dump_obj(&z); 2389 kmem_cache_free(kcp, rhp); 2390 kmem_cache_destroy(kcp); 2391 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 2392 if (WARN_ON_ONCE(!rhp)) 2393 return; 2394 pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2395 pr_alert("mem_dump_obj(kmalloc %px):", rhp); 2396 mem_dump_obj(rhp); 2397 pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); 2398 mem_dump_obj(&rhp->func); 2399 kfree(rhp); 2400 rhp = vmalloc(4096); 2401 if (WARN_ON_ONCE(!rhp)) 2402 return; 2403 pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2404 pr_alert("mem_dump_obj(vmalloc %px):", rhp); 2405 mem_dump_obj(rhp); 2406 pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); 2407 mem_dump_obj(&rhp->func); 2408 vfree(rhp); 2409 } 2410 2411 static void 2412 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 2413 { 2414 pr_alert("%s" TORTURE_FLAG 2415 "--- %s: nreaders=%d nfakewriters=%d " 2416 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 2417 "shuffle_interval=%d stutter=%d irqreader=%d " 2418 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 2419 "test_boost=%d/%d test_boost_interval=%d " 2420 "test_boost_duration=%d shutdown_secs=%d " 2421 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 2422 "stall_cpu_block=%d stall_cpu_repeat=%d " 2423 "n_barrier_cbs=%d " 2424 "onoff_interval=%d onoff_holdoff=%d " 2425 "read_exit_delay=%d read_exit_burst=%d " 2426 "reader_flavor=%x " 2427 "nocbs_nthreads=%d nocbs_toggle=%d " 2428 "test_nmis=%d\n", 2429 torture_type, tag, nrealreaders, nfakewriters, 2430 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 2431 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 2432 test_boost, cur_ops->can_boost, 2433 test_boost_interval, test_boost_duration, shutdown_secs, 2434 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 2435 stall_cpu_block, stall_cpu_repeat, 2436 n_barrier_cbs, 2437 onoff_interval, onoff_holdoff, 2438 read_exit_delay, read_exit_burst, 2439 reader_flavor, 2440 nocbs_nthreads, nocbs_toggle, 2441 test_nmis); 2442 } 2443 2444 static int rcutorture_booster_cleanup(unsigned int cpu) 2445 { 2446 struct task_struct *t; 2447 2448 if (boost_tasks[cpu] == NULL) 2449 return 0; 2450 mutex_lock(&boost_mutex); 2451 t = boost_tasks[cpu]; 2452 boost_tasks[cpu] = NULL; 2453 rcu_torture_enable_rt_throttle(); 2454 mutex_unlock(&boost_mutex); 2455 2456 /* This must be outside of the mutex, otherwise deadlock! */ 2457 torture_stop_kthread(rcu_torture_boost, t); 2458 return 0; 2459 } 2460 2461 static int rcutorture_booster_init(unsigned int cpu) 2462 { 2463 int retval; 2464 2465 if (boost_tasks[cpu] != NULL) 2466 return 0; /* Already created, nothing more to do. */ 2467 2468 // Testing RCU priority boosting requires rcutorture do 2469 // some serious abuse. Counter this by running ksoftirqd 2470 // at higher priority. 2471 if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { 2472 struct sched_param sp; 2473 struct task_struct *t; 2474 2475 t = per_cpu(ksoftirqd, cpu); 2476 WARN_ON_ONCE(!t); 2477 sp.sched_priority = 2; 2478 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2479 #ifdef CONFIG_IRQ_FORCED_THREADING 2480 if (force_irqthreads()) { 2481 t = per_cpu(ktimerd, cpu); 2482 WARN_ON_ONCE(!t); 2483 sp.sched_priority = 2; 2484 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2485 } 2486 #endif 2487 } 2488 2489 /* Don't allow time recalculation while creating a new task. */ 2490 mutex_lock(&boost_mutex); 2491 rcu_torture_disable_rt_throttle(); 2492 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 2493 boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, 2494 cpu, "rcu_torture_boost_%u"); 2495 if (IS_ERR(boost_tasks[cpu])) { 2496 retval = PTR_ERR(boost_tasks[cpu]); 2497 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 2498 n_rcu_torture_boost_ktrerror++; 2499 boost_tasks[cpu] = NULL; 2500 mutex_unlock(&boost_mutex); 2501 return retval; 2502 } 2503 mutex_unlock(&boost_mutex); 2504 return 0; 2505 } 2506 2507 static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) 2508 { 2509 pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr); 2510 return NOTIFY_OK; 2511 } 2512 2513 static struct notifier_block rcu_torture_stall_block = { 2514 .notifier_call = rcu_torture_stall_nf, 2515 }; 2516 2517 /* 2518 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 2519 * induces a CPU stall for the time specified by stall_cpu. If a new 2520 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. 2521 */ 2522 static void rcu_torture_stall_one(int rep, int irqsoff) 2523 { 2524 int idx; 2525 unsigned long stop_at; 2526 2527 if (stall_cpu_holdoff > 0) { 2528 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 2529 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 2530 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 2531 } 2532 if (!kthread_should_stop() && stall_gp_kthread > 0) { 2533 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); 2534 rcu_gp_set_torture_wait(stall_gp_kthread * HZ); 2535 for (idx = 0; idx < stall_gp_kthread + 2; idx++) { 2536 if (kthread_should_stop()) 2537 break; 2538 schedule_timeout_uninterruptible(HZ); 2539 } 2540 } 2541 if (!kthread_should_stop() && stall_cpu > 0) { 2542 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); 2543 stop_at = ktime_get_seconds() + stall_cpu; 2544 /* RCU CPU stall is expected behavior in following code. */ 2545 idx = cur_ops->readlock(); 2546 if (irqsoff) 2547 local_irq_disable(); 2548 else if (!stall_cpu_block) 2549 preempt_disable(); 2550 pr_alert("%s start stall episode %d on CPU %d.\n", 2551 __func__, rep + 1, raw_smp_processor_id()); 2552 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) && 2553 !kthread_should_stop()) 2554 if (stall_cpu_block) { 2555 #ifdef CONFIG_PREEMPTION 2556 preempt_schedule(); 2557 #else 2558 schedule_timeout_uninterruptible(HZ); 2559 #endif 2560 } else if (stall_no_softlockup) { 2561 touch_softlockup_watchdog(); 2562 } 2563 if (irqsoff) 2564 local_irq_enable(); 2565 else if (!stall_cpu_block) 2566 preempt_enable(); 2567 cur_ops->readunlock(idx); 2568 } 2569 } 2570 2571 /* 2572 * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many 2573 * additional times as specified by the stall_cpu_repeat module parameter. 2574 * Note that stall_cpu_irqsoff is ignored on the second and subsequent 2575 * stall. 2576 */ 2577 static int rcu_torture_stall(void *args) 2578 { 2579 int i; 2580 int repeat = stall_cpu_repeat; 2581 int ret; 2582 2583 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 2584 if (repeat < 0) { 2585 repeat = 0; 2586 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 2587 } 2588 if (rcu_cpu_stall_notifiers) { 2589 ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block); 2590 if (ret) 2591 pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n", 2592 __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : ""); 2593 } 2594 for (i = 0; i <= repeat; i++) { 2595 if (kthread_should_stop()) 2596 break; 2597 rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0); 2598 } 2599 pr_alert("%s end.\n", __func__); 2600 if (rcu_cpu_stall_notifiers && !ret) { 2601 ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block); 2602 if (ret) 2603 pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret); 2604 } 2605 torture_shutdown_absorb("rcu_torture_stall"); 2606 while (!kthread_should_stop()) 2607 schedule_timeout_interruptible(10 * HZ); 2608 return 0; 2609 } 2610 2611 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 2612 static int __init rcu_torture_stall_init(void) 2613 { 2614 if (stall_cpu <= 0 && stall_gp_kthread <= 0) 2615 return 0; 2616 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 2617 } 2618 2619 /* State structure for forward-progress self-propagating RCU callback. */ 2620 struct fwd_cb_state { 2621 struct rcu_head rh; 2622 int stop; 2623 }; 2624 2625 /* 2626 * Forward-progress self-propagating RCU callback function. Because 2627 * callbacks run from softirq, this function is an implicit RCU read-side 2628 * critical section. 2629 */ 2630 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 2631 { 2632 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 2633 2634 if (READ_ONCE(fcsp->stop)) { 2635 WRITE_ONCE(fcsp->stop, 2); 2636 return; 2637 } 2638 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 2639 } 2640 2641 /* State for continuous-flood RCU callbacks. */ 2642 struct rcu_fwd_cb { 2643 struct rcu_head rh; 2644 struct rcu_fwd_cb *rfc_next; 2645 struct rcu_fwd *rfc_rfp; 2646 int rfc_gps; 2647 }; 2648 2649 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 2650 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 2651 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 2652 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 2653 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 2654 2655 struct rcu_launder_hist { 2656 long n_launders; 2657 unsigned long launder_gp_seq; 2658 }; 2659 2660 struct rcu_fwd { 2661 spinlock_t rcu_fwd_lock; 2662 struct rcu_fwd_cb *rcu_fwd_cb_head; 2663 struct rcu_fwd_cb **rcu_fwd_cb_tail; 2664 long n_launders_cb; 2665 unsigned long rcu_fwd_startat; 2666 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 2667 unsigned long rcu_launder_gp_seq_start; 2668 int rcu_fwd_id; 2669 }; 2670 2671 static DEFINE_MUTEX(rcu_fwd_mutex); 2672 static struct rcu_fwd *rcu_fwds; 2673 static unsigned long rcu_fwd_seq; 2674 static atomic_long_t rcu_fwd_max_cbs; 2675 static bool rcu_fwd_emergency_stop; 2676 2677 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 2678 { 2679 unsigned long gps; 2680 unsigned long gps_old; 2681 int i; 2682 int j; 2683 2684 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 2685 if (rfp->n_launders_hist[i].n_launders > 0) 2686 break; 2687 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", 2688 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); 2689 gps_old = rfp->rcu_launder_gp_seq_start; 2690 for (j = 0; j <= i; j++) { 2691 gps = rfp->n_launders_hist[j].launder_gp_seq; 2692 pr_cont(" %ds/%d: %ld:%ld", 2693 j + 1, FWD_CBS_HIST_DIV, 2694 rfp->n_launders_hist[j].n_launders, 2695 rcutorture_seq_diff(gps, gps_old)); 2696 gps_old = gps; 2697 } 2698 pr_cont("\n"); 2699 } 2700 2701 /* Callback function for continuous-flood RCU callbacks. */ 2702 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 2703 { 2704 unsigned long flags; 2705 int i; 2706 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 2707 struct rcu_fwd_cb **rfcpp; 2708 struct rcu_fwd *rfp = rfcp->rfc_rfp; 2709 2710 rfcp->rfc_next = NULL; 2711 rfcp->rfc_gps++; 2712 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2713 rfcpp = rfp->rcu_fwd_cb_tail; 2714 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 2715 smp_store_release(rfcpp, rfcp); 2716 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 2717 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 2718 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 2719 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 2720 rfp->n_launders_hist[i].n_launders++; 2721 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 2722 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2723 } 2724 2725 // Give the scheduler a chance, even on nohz_full CPUs. 2726 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 2727 { 2728 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 2729 // Real call_rcu() floods hit userspace, so emulate that. 2730 if (need_resched() || (iter & 0xfff)) 2731 schedule(); 2732 return; 2733 } 2734 // No userspace emulation: CB invocation throttles call_rcu() 2735 cond_resched(); 2736 } 2737 2738 /* 2739 * Free all callbacks on the rcu_fwd_cb_head list, either because the 2740 * test is over or because we hit an OOM event. 2741 */ 2742 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 2743 { 2744 unsigned long flags; 2745 unsigned long freed = 0; 2746 struct rcu_fwd_cb *rfcp; 2747 2748 for (;;) { 2749 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2750 rfcp = rfp->rcu_fwd_cb_head; 2751 if (!rfcp) { 2752 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2753 break; 2754 } 2755 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 2756 if (!rfp->rcu_fwd_cb_head) 2757 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 2758 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2759 kfree(rfcp); 2760 freed++; 2761 rcu_torture_fwd_prog_cond_resched(freed); 2762 if (tick_nohz_full_enabled()) { 2763 local_irq_save(flags); 2764 rcu_momentary_eqs(); 2765 local_irq_restore(flags); 2766 } 2767 } 2768 return freed; 2769 } 2770 2771 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 2772 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 2773 int *tested, int *tested_tries) 2774 { 2775 unsigned long cver; 2776 unsigned long dur; 2777 struct fwd_cb_state fcs; 2778 unsigned long gps; 2779 int idx; 2780 int sd; 2781 int sd4; 2782 bool selfpropcb = false; 2783 unsigned long stopat; 2784 static DEFINE_TORTURE_RANDOM(trs); 2785 2786 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 2787 if (!cur_ops->sync) 2788 return; // Cannot do need_resched() forward progress testing without ->sync. 2789 if (cur_ops->call && cur_ops->cb_barrier) { 2790 init_rcu_head_on_stack(&fcs.rh); 2791 selfpropcb = true; 2792 } 2793 2794 /* Tight loop containing cond_resched(). */ 2795 atomic_inc(&rcu_fwd_cb_nodelay); 2796 cur_ops->sync(); /* Later readers see above write. */ 2797 if (selfpropcb) { 2798 WRITE_ONCE(fcs.stop, 0); 2799 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 2800 } 2801 cver = READ_ONCE(rcu_torture_current_version); 2802 gps = cur_ops->get_gp_seq(); 2803 sd = cur_ops->stall_dur() + 1; 2804 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 2805 dur = sd4 + torture_random(&trs) % (sd - sd4); 2806 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 2807 stopat = rfp->rcu_fwd_startat + dur; 2808 while (time_before(jiffies, stopat) && 2809 !shutdown_time_arrived() && 2810 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2811 idx = cur_ops->readlock(); 2812 udelay(10); 2813 cur_ops->readunlock(idx); 2814 if (!fwd_progress_need_resched || need_resched()) 2815 cond_resched(); 2816 } 2817 (*tested_tries)++; 2818 if (!time_before(jiffies, stopat) && 2819 !shutdown_time_arrived() && 2820 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2821 (*tested)++; 2822 cver = READ_ONCE(rcu_torture_current_version) - cver; 2823 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 2824 WARN_ON(!cver && gps < 2); 2825 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, 2826 rfp->rcu_fwd_id, dur, cver, gps); 2827 } 2828 if (selfpropcb) { 2829 WRITE_ONCE(fcs.stop, 1); 2830 cur_ops->sync(); /* Wait for running CB to complete. */ 2831 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 2832 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 2833 } 2834 2835 if (selfpropcb) { 2836 WARN_ON(READ_ONCE(fcs.stop) != 2); 2837 destroy_rcu_head_on_stack(&fcs.rh); 2838 } 2839 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 2840 atomic_dec(&rcu_fwd_cb_nodelay); 2841 } 2842 2843 /* Carry out call_rcu() forward-progress testing. */ 2844 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 2845 { 2846 unsigned long cver; 2847 unsigned long flags; 2848 unsigned long gps; 2849 int i; 2850 long n_launders; 2851 long n_launders_cb_snap; 2852 long n_launders_sa; 2853 long n_max_cbs; 2854 long n_max_gps; 2855 struct rcu_fwd_cb *rfcp; 2856 struct rcu_fwd_cb *rfcpn; 2857 unsigned long stopat; 2858 unsigned long stoppedat; 2859 2860 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 2861 if (READ_ONCE(rcu_fwd_emergency_stop)) 2862 return; /* Get out of the way quickly, no GP wait! */ 2863 if (!cur_ops->call) 2864 return; /* Can't do call_rcu() fwd prog without ->call. */ 2865 2866 /* Loop continuously posting RCU callbacks. */ 2867 atomic_inc(&rcu_fwd_cb_nodelay); 2868 cur_ops->sync(); /* Later readers see above write. */ 2869 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 2870 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 2871 n_launders = 0; 2872 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 2873 n_launders_sa = 0; 2874 n_max_cbs = 0; 2875 n_max_gps = 0; 2876 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 2877 rfp->n_launders_hist[i].n_launders = 0; 2878 cver = READ_ONCE(rcu_torture_current_version); 2879 gps = cur_ops->get_gp_seq(); 2880 rfp->rcu_launder_gp_seq_start = gps; 2881 tick_dep_set_task(current, TICK_DEP_BIT_RCU); 2882 while (time_before(jiffies, stopat) && 2883 !shutdown_time_arrived() && 2884 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 2885 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 2886 rfcpn = NULL; 2887 if (rfcp) 2888 rfcpn = READ_ONCE(rfcp->rfc_next); 2889 if (rfcpn) { 2890 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 2891 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 2892 break; 2893 rfp->rcu_fwd_cb_head = rfcpn; 2894 n_launders++; 2895 n_launders_sa++; 2896 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { 2897 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 2898 if (WARN_ON_ONCE(!rfcp)) { 2899 schedule_timeout_interruptible(1); 2900 continue; 2901 } 2902 n_max_cbs++; 2903 n_launders_sa = 0; 2904 rfcp->rfc_gps = 0; 2905 rfcp->rfc_rfp = rfp; 2906 } else { 2907 rfcp = NULL; 2908 } 2909 if (rfcp) 2910 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 2911 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 2912 if (tick_nohz_full_enabled()) { 2913 local_irq_save(flags); 2914 rcu_momentary_eqs(); 2915 local_irq_restore(flags); 2916 } 2917 } 2918 stoppedat = jiffies; 2919 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 2920 cver = READ_ONCE(rcu_torture_current_version) - cver; 2921 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 2922 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 2923 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 2924 (void)rcu_torture_fwd_prog_cbfree(rfp); 2925 2926 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 2927 !shutdown_time_arrived()) { 2928 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg) 2929 cur_ops->gp_kthread_dbg(); 2930 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n", 2931 __func__, 2932 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 2933 n_launders + n_max_cbs - n_launders_cb_snap, 2934 n_launders, n_launders_sa, 2935 n_max_gps, n_max_cbs, cver, gps, num_online_cpus()); 2936 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); 2937 mutex_lock(&rcu_fwd_mutex); // Serialize histograms. 2938 rcu_torture_fwd_cb_hist(rfp); 2939 mutex_unlock(&rcu_fwd_mutex); 2940 } 2941 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 2942 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2943 atomic_dec(&rcu_fwd_cb_nodelay); 2944 } 2945 2946 2947 /* 2948 * OOM notifier, but this only prints diagnostic information for the 2949 * current forward-progress test. 2950 */ 2951 static int rcutorture_oom_notify(struct notifier_block *self, 2952 unsigned long notused, void *nfreed) 2953 { 2954 int i; 2955 long ncbs; 2956 struct rcu_fwd *rfp; 2957 2958 mutex_lock(&rcu_fwd_mutex); 2959 rfp = rcu_fwds; 2960 if (!rfp) { 2961 mutex_unlock(&rcu_fwd_mutex); 2962 return NOTIFY_OK; 2963 } 2964 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 2965 __func__); 2966 for (i = 0; i < fwd_progress; i++) { 2967 rcu_torture_fwd_cb_hist(&rfp[i]); 2968 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); 2969 } 2970 WRITE_ONCE(rcu_fwd_emergency_stop, true); 2971 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 2972 ncbs = 0; 2973 for (i = 0; i < fwd_progress; i++) 2974 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2975 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2976 cur_ops->cb_barrier(); 2977 ncbs = 0; 2978 for (i = 0; i < fwd_progress; i++) 2979 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2980 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2981 cur_ops->cb_barrier(); 2982 ncbs = 0; 2983 for (i = 0; i < fwd_progress; i++) 2984 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 2985 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 2986 smp_mb(); /* Frees before return to avoid redoing OOM. */ 2987 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 2988 pr_info("%s returning after OOM processing.\n", __func__); 2989 mutex_unlock(&rcu_fwd_mutex); 2990 return NOTIFY_OK; 2991 } 2992 2993 static struct notifier_block rcutorture_oom_nb = { 2994 .notifier_call = rcutorture_oom_notify 2995 }; 2996 2997 /* Carry out grace-period forward-progress testing. */ 2998 static int rcu_torture_fwd_prog(void *args) 2999 { 3000 bool firsttime = true; 3001 long max_cbs; 3002 int oldnice = task_nice(current); 3003 unsigned long oldseq = READ_ONCE(rcu_fwd_seq); 3004 struct rcu_fwd *rfp = args; 3005 int tested = 0; 3006 int tested_tries = 0; 3007 3008 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 3009 rcu_bind_current_to_nocb(); 3010 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 3011 set_user_nice(current, MAX_NICE); 3012 do { 3013 if (!rfp->rcu_fwd_id) { 3014 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 3015 WRITE_ONCE(rcu_fwd_emergency_stop, false); 3016 if (!firsttime) { 3017 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); 3018 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); 3019 } 3020 firsttime = false; 3021 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); 3022 } else { 3023 while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) 3024 schedule_timeout_interruptible(HZ / 20); 3025 oldseq = READ_ONCE(rcu_fwd_seq); 3026 } 3027 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3028 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) 3029 rcu_torture_fwd_prog_cr(rfp); 3030 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && 3031 (!IS_ENABLED(CONFIG_TINY_RCU) || 3032 (rcu_inkernel_boot_has_ended() && 3033 torture_num_online_cpus() > rfp->rcu_fwd_id))) 3034 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 3035 3036 /* Avoid slow periods, better to test when busy. */ 3037 if (stutter_wait("rcu_torture_fwd_prog")) 3038 sched_set_normal(current, oldnice); 3039 } while (!torture_must_stop()); 3040 /* Short runs might not contain a valid forward-progress attempt. */ 3041 if (!rfp->rcu_fwd_id) { 3042 WARN_ON(!tested && tested_tries >= 5); 3043 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 3044 } 3045 torture_kthread_stopping("rcu_torture_fwd_prog"); 3046 return 0; 3047 } 3048 3049 /* If forward-progress checking is requested and feasible, spawn the thread. */ 3050 static int __init rcu_torture_fwd_prog_init(void) 3051 { 3052 int i; 3053 int ret = 0; 3054 struct rcu_fwd *rfp; 3055 3056 if (!fwd_progress) 3057 return 0; /* Not requested, so don't do it. */ 3058 if (fwd_progress >= nr_cpu_ids) { 3059 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); 3060 fwd_progress = nr_cpu_ids; 3061 } else if (fwd_progress < 0) { 3062 fwd_progress = nr_cpu_ids; 3063 } 3064 if ((!cur_ops->sync && !cur_ops->call) || 3065 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || 3066 cur_ops == &rcu_busted_ops) { 3067 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 3068 fwd_progress = 0; 3069 return 0; 3070 } 3071 if (stall_cpu > 0) { 3072 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); 3073 fwd_progress = 0; 3074 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 3075 return -EINVAL; /* In module, can fail back to user. */ 3076 WARN_ON(1); /* Make sure rcutorture notices conflict. */ 3077 return 0; 3078 } 3079 if (fwd_progress_holdoff <= 0) 3080 fwd_progress_holdoff = 1; 3081 if (fwd_progress_div <= 0) 3082 fwd_progress_div = 4; 3083 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL); 3084 fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL); 3085 if (!rfp || !fwd_prog_tasks) { 3086 kfree(rfp); 3087 kfree(fwd_prog_tasks); 3088 fwd_prog_tasks = NULL; 3089 fwd_progress = 0; 3090 return -ENOMEM; 3091 } 3092 for (i = 0; i < fwd_progress; i++) { 3093 spin_lock_init(&rfp[i].rcu_fwd_lock); 3094 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; 3095 rfp[i].rcu_fwd_id = i; 3096 } 3097 mutex_lock(&rcu_fwd_mutex); 3098 rcu_fwds = rfp; 3099 mutex_unlock(&rcu_fwd_mutex); 3100 register_oom_notifier(&rcutorture_oom_nb); 3101 for (i = 0; i < fwd_progress; i++) { 3102 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); 3103 if (ret) { 3104 fwd_progress = i; 3105 return ret; 3106 } 3107 } 3108 return 0; 3109 } 3110 3111 static void rcu_torture_fwd_prog_cleanup(void) 3112 { 3113 int i; 3114 struct rcu_fwd *rfp; 3115 3116 if (!rcu_fwds || !fwd_prog_tasks) 3117 return; 3118 for (i = 0; i < fwd_progress; i++) 3119 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); 3120 unregister_oom_notifier(&rcutorture_oom_nb); 3121 mutex_lock(&rcu_fwd_mutex); 3122 rfp = rcu_fwds; 3123 rcu_fwds = NULL; 3124 mutex_unlock(&rcu_fwd_mutex); 3125 kfree(rfp); 3126 kfree(fwd_prog_tasks); 3127 fwd_prog_tasks = NULL; 3128 } 3129 3130 /* Callback function for RCU barrier testing. */ 3131 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 3132 { 3133 atomic_inc(&barrier_cbs_invoked); 3134 } 3135 3136 /* IPI handler to get callback posted on desired CPU, if online. */ 3137 static int rcu_torture_barrier1cb(void *rcu_void) 3138 { 3139 struct rcu_head *rhp = rcu_void; 3140 3141 cur_ops->call(rhp, rcu_torture_barrier_cbf); 3142 return 0; 3143 } 3144 3145 /* kthread function to register callbacks used to test RCU barriers. */ 3146 static int rcu_torture_barrier_cbs(void *arg) 3147 { 3148 long myid = (long)arg; 3149 bool lastphase = false; 3150 bool newphase; 3151 struct rcu_head rcu; 3152 3153 init_rcu_head_on_stack(&rcu); 3154 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 3155 set_user_nice(current, MAX_NICE); 3156 do { 3157 wait_event(barrier_cbs_wq[myid], 3158 (newphase = 3159 smp_load_acquire(&barrier_phase)) != lastphase || 3160 torture_must_stop()); 3161 lastphase = newphase; 3162 if (torture_must_stop()) 3163 break; 3164 /* 3165 * The above smp_load_acquire() ensures barrier_phase load 3166 * is ordered before the following ->call(). 3167 */ 3168 if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1)) 3169 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 3170 3171 if (atomic_dec_and_test(&barrier_cbs_count)) 3172 wake_up(&barrier_wq); 3173 } while (!torture_must_stop()); 3174 if (cur_ops->cb_barrier != NULL) 3175 cur_ops->cb_barrier(); 3176 destroy_rcu_head_on_stack(&rcu); 3177 torture_kthread_stopping("rcu_torture_barrier_cbs"); 3178 return 0; 3179 } 3180 3181 /* kthread function to drive and coordinate RCU barrier testing. */ 3182 static int rcu_torture_barrier(void *arg) 3183 { 3184 int i; 3185 3186 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 3187 do { 3188 atomic_set(&barrier_cbs_invoked, 0); 3189 atomic_set(&barrier_cbs_count, n_barrier_cbs); 3190 /* Ensure barrier_phase ordered after prior assignments. */ 3191 smp_store_release(&barrier_phase, !barrier_phase); 3192 for (i = 0; i < n_barrier_cbs; i++) 3193 wake_up(&barrier_cbs_wq[i]); 3194 wait_event(barrier_wq, 3195 atomic_read(&barrier_cbs_count) == 0 || 3196 torture_must_stop()); 3197 if (torture_must_stop()) 3198 break; 3199 n_barrier_attempts++; 3200 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 3201 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 3202 n_rcu_torture_barrier_error++; 3203 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 3204 atomic_read(&barrier_cbs_invoked), 3205 n_barrier_cbs); 3206 WARN_ON(1); 3207 // Wait manually for the remaining callbacks 3208 i = 0; 3209 do { 3210 if (WARN_ON(i++ > HZ)) 3211 i = INT_MIN; 3212 schedule_timeout_interruptible(1); 3213 cur_ops->cb_barrier(); 3214 } while (atomic_read(&barrier_cbs_invoked) != 3215 n_barrier_cbs && 3216 !torture_must_stop()); 3217 smp_mb(); // Can't trust ordering if broken. 3218 if (!torture_must_stop()) 3219 pr_err("Recovered: barrier_cbs_invoked = %d\n", 3220 atomic_read(&barrier_cbs_invoked)); 3221 } else { 3222 n_barrier_successes++; 3223 } 3224 schedule_timeout_interruptible(HZ / 10); 3225 } while (!torture_must_stop()); 3226 torture_kthread_stopping("rcu_torture_barrier"); 3227 return 0; 3228 } 3229 3230 /* Initialize RCU barrier testing. */ 3231 static int rcu_torture_barrier_init(void) 3232 { 3233 int i; 3234 int ret; 3235 3236 if (n_barrier_cbs <= 0) 3237 return 0; 3238 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 3239 pr_alert("%s" TORTURE_FLAG 3240 " Call or barrier ops missing for %s,\n", 3241 torture_type, cur_ops->name); 3242 pr_alert("%s" TORTURE_FLAG 3243 " RCU barrier testing omitted from run.\n", 3244 torture_type); 3245 return 0; 3246 } 3247 atomic_set(&barrier_cbs_count, 0); 3248 atomic_set(&barrier_cbs_invoked, 0); 3249 barrier_cbs_tasks = 3250 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 3251 GFP_KERNEL); 3252 barrier_cbs_wq = 3253 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 3254 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 3255 return -ENOMEM; 3256 for (i = 0; i < n_barrier_cbs; i++) { 3257 init_waitqueue_head(&barrier_cbs_wq[i]); 3258 ret = torture_create_kthread(rcu_torture_barrier_cbs, 3259 (void *)(long)i, 3260 barrier_cbs_tasks[i]); 3261 if (ret) 3262 return ret; 3263 } 3264 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 3265 } 3266 3267 /* Clean up after RCU barrier testing. */ 3268 static void rcu_torture_barrier_cleanup(void) 3269 { 3270 int i; 3271 3272 torture_stop_kthread(rcu_torture_barrier, barrier_task); 3273 if (barrier_cbs_tasks != NULL) { 3274 for (i = 0; i < n_barrier_cbs; i++) 3275 torture_stop_kthread(rcu_torture_barrier_cbs, 3276 barrier_cbs_tasks[i]); 3277 kfree(barrier_cbs_tasks); 3278 barrier_cbs_tasks = NULL; 3279 } 3280 if (barrier_cbs_wq != NULL) { 3281 kfree(barrier_cbs_wq); 3282 barrier_cbs_wq = NULL; 3283 } 3284 } 3285 3286 static bool rcu_torture_can_boost(void) 3287 { 3288 static int boost_warn_once; 3289 int prio; 3290 3291 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 3292 return false; 3293 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) 3294 return false; 3295 3296 prio = rcu_get_gp_kthreads_prio(); 3297 if (!prio) 3298 return false; 3299 3300 if (prio < 2) { 3301 if (boost_warn_once == 1) 3302 return false; 3303 3304 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 3305 boost_warn_once = 1; 3306 return false; 3307 } 3308 3309 return true; 3310 } 3311 3312 static bool read_exit_child_stop; 3313 static bool read_exit_child_stopped; 3314 static wait_queue_head_t read_exit_wq; 3315 3316 // Child kthread which just does an rcutorture reader and exits. 3317 static int rcu_torture_read_exit_child(void *trsp_in) 3318 { 3319 struct torture_random_state *trsp = trsp_in; 3320 3321 set_user_nice(current, MAX_NICE); 3322 // Minimize time between reading and exiting. 3323 while (!kthread_should_stop()) 3324 schedule_timeout_uninterruptible(HZ / 20); 3325 (void)rcu_torture_one_read(trsp, -1); 3326 return 0; 3327 } 3328 3329 // Parent kthread which creates and destroys read-exit child kthreads. 3330 static int rcu_torture_read_exit(void *unused) 3331 { 3332 bool errexit = false; 3333 int i; 3334 struct task_struct *tsp; 3335 DEFINE_TORTURE_RANDOM(trs); 3336 3337 // Allocate and initialize. 3338 set_user_nice(current, MAX_NICE); 3339 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); 3340 3341 // Each pass through this loop does one read-exit episode. 3342 do { 3343 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); 3344 for (i = 0; i < read_exit_burst; i++) { 3345 if (READ_ONCE(read_exit_child_stop)) 3346 break; 3347 stutter_wait("rcu_torture_read_exit"); 3348 // Spawn child. 3349 tsp = kthread_run(rcu_torture_read_exit_child, 3350 &trs, "%s", "rcu_torture_read_exit_child"); 3351 if (IS_ERR(tsp)) { 3352 TOROUT_ERRSTRING("out of memory"); 3353 errexit = true; 3354 break; 3355 } 3356 cond_resched(); 3357 kthread_stop(tsp); 3358 n_read_exits++; 3359 } 3360 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); 3361 rcu_barrier(); // Wait for task_struct free, avoid OOM. 3362 i = 0; 3363 for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) 3364 schedule_timeout_uninterruptible(HZ); 3365 } while (!errexit && !READ_ONCE(read_exit_child_stop)); 3366 3367 // Clean up and exit. 3368 smp_store_release(&read_exit_child_stopped, true); // After reaping. 3369 smp_mb(); // Store before wakeup. 3370 wake_up(&read_exit_wq); 3371 while (!torture_must_stop()) 3372 schedule_timeout_uninterruptible(HZ / 20); 3373 torture_kthread_stopping("rcu_torture_read_exit"); 3374 return 0; 3375 } 3376 3377 static int rcu_torture_read_exit_init(void) 3378 { 3379 if (read_exit_burst <= 0) 3380 return 0; 3381 init_waitqueue_head(&read_exit_wq); 3382 read_exit_child_stop = false; 3383 read_exit_child_stopped = false; 3384 return torture_create_kthread(rcu_torture_read_exit, NULL, 3385 read_exit_task); 3386 } 3387 3388 static void rcu_torture_read_exit_cleanup(void) 3389 { 3390 if (!read_exit_task) 3391 return; 3392 WRITE_ONCE(read_exit_child_stop, true); 3393 smp_mb(); // Above write before wait. 3394 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); 3395 torture_stop_kthread(rcutorture_read_exit, read_exit_task); 3396 } 3397 3398 static void rcutorture_test_nmis(int n) 3399 { 3400 #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3401 int cpu; 3402 int dumpcpu; 3403 int i; 3404 3405 for (i = 0; i < n; i++) { 3406 preempt_disable(); 3407 cpu = smp_processor_id(); 3408 dumpcpu = cpu + 1; 3409 if (dumpcpu >= nr_cpu_ids) 3410 dumpcpu = 0; 3411 pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu); 3412 dump_cpu_task(dumpcpu); 3413 preempt_enable(); 3414 schedule_timeout_uninterruptible(15 * HZ); 3415 } 3416 #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3417 WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis); 3418 #endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3419 } 3420 3421 static enum cpuhp_state rcutor_hp; 3422 3423 static void 3424 rcu_torture_cleanup(void) 3425 { 3426 int firsttime; 3427 int flags = 0; 3428 unsigned long gp_seq = 0; 3429 int i; 3430 3431 if (torture_cleanup_begin()) { 3432 if (cur_ops->cb_barrier != NULL) { 3433 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3434 cur_ops->cb_barrier(); 3435 } 3436 if (cur_ops->gp_slow_unregister) 3437 cur_ops->gp_slow_unregister(NULL); 3438 return; 3439 } 3440 if (!cur_ops) { 3441 torture_cleanup_end(); 3442 return; 3443 } 3444 3445 rcutorture_test_nmis(test_nmis); 3446 3447 if (cur_ops->gp_kthread_dbg) 3448 cur_ops->gp_kthread_dbg(); 3449 rcu_torture_read_exit_cleanup(); 3450 rcu_torture_barrier_cleanup(); 3451 rcu_torture_fwd_prog_cleanup(); 3452 torture_stop_kthread(rcu_torture_stall, stall_task); 3453 torture_stop_kthread(rcu_torture_writer, writer_task); 3454 3455 if (nocb_tasks) { 3456 for (i = 0; i < nrealnocbers; i++) 3457 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); 3458 kfree(nocb_tasks); 3459 nocb_tasks = NULL; 3460 } 3461 3462 if (reader_tasks) { 3463 for (i = 0; i < nrealreaders; i++) 3464 torture_stop_kthread(rcu_torture_reader, 3465 reader_tasks[i]); 3466 kfree(reader_tasks); 3467 reader_tasks = NULL; 3468 } 3469 kfree(rcu_torture_reader_mbchk); 3470 rcu_torture_reader_mbchk = NULL; 3471 3472 if (fakewriter_tasks) { 3473 for (i = 0; i < nfakewriters; i++) 3474 torture_stop_kthread(rcu_torture_fakewriter, 3475 fakewriter_tasks[i]); 3476 kfree(fakewriter_tasks); 3477 fakewriter_tasks = NULL; 3478 } 3479 3480 if (cur_ops->get_gp_data) 3481 cur_ops->get_gp_data(&flags, &gp_seq); 3482 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", 3483 cur_ops->name, (long)gp_seq, flags, 3484 rcutorture_seq_diff(gp_seq, start_gp_seq)); 3485 torture_stop_kthread(rcu_torture_stats, stats_task); 3486 torture_stop_kthread(rcu_torture_fqs, fqs_task); 3487 if (rcu_torture_can_boost() && rcutor_hp >= 0) 3488 cpuhp_remove_state(rcutor_hp); 3489 3490 /* 3491 * Wait for all RCU callbacks to fire, then do torture-type-specific 3492 * cleanup operations. 3493 */ 3494 if (cur_ops->cb_barrier != NULL) { 3495 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3496 cur_ops->cb_barrier(); 3497 } 3498 if (cur_ops->cleanup != NULL) 3499 cur_ops->cleanup(); 3500 3501 rcu_torture_mem_dump_obj(); 3502 3503 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 3504 3505 if (err_segs_recorded) { 3506 pr_alert("Failure/close-call rcutorture reader segments:\n"); 3507 if (rt_read_nsegs == 0) 3508 pr_alert("\t: No segments recorded!!!\n"); 3509 firsttime = 1; 3510 for (i = 0; i < rt_read_nsegs; i++) { 3511 pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate); 3512 if (err_segs[i].rt_delay_jiffies != 0) { 3513 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 3514 err_segs[i].rt_delay_jiffies); 3515 firsttime = 0; 3516 } 3517 if (err_segs[i].rt_delay_ms != 0) { 3518 pr_cont("%s%ldms", firsttime ? "" : "+", 3519 err_segs[i].rt_delay_ms); 3520 firsttime = 0; 3521 } 3522 if (err_segs[i].rt_delay_us != 0) { 3523 pr_cont("%s%ldus", firsttime ? "" : "+", 3524 err_segs[i].rt_delay_us); 3525 firsttime = 0; 3526 } 3527 pr_cont("%s\n", 3528 err_segs[i].rt_preempted ? "preempted" : ""); 3529 3530 } 3531 } 3532 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 3533 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 3534 else if (torture_onoff_failures()) 3535 rcu_torture_print_module_parms(cur_ops, 3536 "End of test: RCU_HOTPLUG"); 3537 else 3538 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 3539 torture_cleanup_end(); 3540 if (cur_ops->gp_slow_unregister) 3541 cur_ops->gp_slow_unregister(NULL); 3542 } 3543 3544 static void rcu_torture_leak_cb(struct rcu_head *rhp) 3545 { 3546 } 3547 3548 static void rcu_torture_err_cb(struct rcu_head *rhp) 3549 { 3550 /* 3551 * This -might- happen due to race conditions, but is unlikely. 3552 * The scenario that leads to this happening is that the 3553 * first of the pair of duplicate callbacks is queued, 3554 * someone else starts a grace period that includes that 3555 * callback, then the second of the pair must wait for the 3556 * next grace period. Unlikely, but can happen. If it 3557 * does happen, the debug-objects subsystem won't have splatted. 3558 */ 3559 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 3560 } 3561 3562 /* 3563 * Verify that double-free causes debug-objects to complain, but only 3564 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 3565 * cannot be carried out. 3566 */ 3567 static void rcu_test_debug_objects(void) 3568 { 3569 struct rcu_head rh1; 3570 struct rcu_head rh2; 3571 int idx; 3572 3573 if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) { 3574 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n", 3575 KBUILD_MODNAME, cur_ops->name); 3576 return; 3577 } 3578 3579 if (WARN_ON_ONCE(cur_ops->debug_objects && 3580 (!cur_ops->call || !cur_ops->cb_barrier))) 3581 return; 3582 3583 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 3584 3585 init_rcu_head_on_stack(&rh1); 3586 init_rcu_head_on_stack(&rh2); 3587 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name); 3588 3589 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 3590 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */ 3591 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 3592 cur_ops->call(&rh2, rcu_torture_leak_cb); 3593 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 3594 if (rhp) { 3595 cur_ops->call(rhp, rcu_torture_leak_cb); 3596 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ 3597 } 3598 cur_ops->readunlock(idx); 3599 3600 /* Wait for them all to get done so we can safely return. */ 3601 cur_ops->cb_barrier(); 3602 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name); 3603 destroy_rcu_head_on_stack(&rh1); 3604 destroy_rcu_head_on_stack(&rh2); 3605 kfree(rhp); 3606 } 3607 3608 static void rcutorture_sync(void) 3609 { 3610 static unsigned long n; 3611 3612 if (cur_ops->sync && !(++n & 0xfff)) 3613 cur_ops->sync(); 3614 } 3615 3616 static DEFINE_MUTEX(mut0); 3617 static DEFINE_MUTEX(mut1); 3618 static DEFINE_MUTEX(mut2); 3619 static DEFINE_MUTEX(mut3); 3620 static DEFINE_MUTEX(mut4); 3621 static DEFINE_MUTEX(mut5); 3622 static DEFINE_MUTEX(mut6); 3623 static DEFINE_MUTEX(mut7); 3624 static DEFINE_MUTEX(mut8); 3625 static DEFINE_MUTEX(mut9); 3626 3627 static DECLARE_RWSEM(rwsem0); 3628 static DECLARE_RWSEM(rwsem1); 3629 static DECLARE_RWSEM(rwsem2); 3630 static DECLARE_RWSEM(rwsem3); 3631 static DECLARE_RWSEM(rwsem4); 3632 static DECLARE_RWSEM(rwsem5); 3633 static DECLARE_RWSEM(rwsem6); 3634 static DECLARE_RWSEM(rwsem7); 3635 static DECLARE_RWSEM(rwsem8); 3636 static DECLARE_RWSEM(rwsem9); 3637 3638 DEFINE_STATIC_SRCU(srcu0); 3639 DEFINE_STATIC_SRCU(srcu1); 3640 DEFINE_STATIC_SRCU(srcu2); 3641 DEFINE_STATIC_SRCU(srcu3); 3642 DEFINE_STATIC_SRCU(srcu4); 3643 DEFINE_STATIC_SRCU(srcu5); 3644 DEFINE_STATIC_SRCU(srcu6); 3645 DEFINE_STATIC_SRCU(srcu7); 3646 DEFINE_STATIC_SRCU(srcu8); 3647 DEFINE_STATIC_SRCU(srcu9); 3648 3649 static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, 3650 int cyclelen, int deadlock) 3651 { 3652 int j = i + 1; 3653 3654 if (j >= cyclelen) 3655 j = deadlock ? 0 : -1; 3656 if (j >= 0) 3657 pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i); 3658 else 3659 pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i); 3660 return j; 3661 } 3662 3663 // Test lockdep on SRCU-based deadlock scenarios. 3664 static void rcu_torture_init_srcu_lockdep(void) 3665 { 3666 int cyclelen; 3667 int deadlock; 3668 bool err = false; 3669 int i; 3670 int j; 3671 int idx; 3672 struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4, 3673 &mut5, &mut6, &mut7, &mut8, &mut9 }; 3674 struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4, 3675 &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; 3676 struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, 3677 &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; 3678 int testtype; 3679 3680 if (!test_srcu_lockdep) 3681 return; 3682 3683 deadlock = test_srcu_lockdep / 1000; 3684 testtype = (test_srcu_lockdep / 10) % 100; 3685 cyclelen = test_srcu_lockdep % 10; 3686 WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus)); 3687 if (WARN_ONCE(deadlock != !!deadlock, 3688 "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n", 3689 __func__, test_srcu_lockdep, deadlock)) 3690 err = true; 3691 if (WARN_ONCE(cyclelen <= 0, 3692 "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n", 3693 __func__, test_srcu_lockdep, cyclelen)) 3694 err = true; 3695 if (err) 3696 goto err_out; 3697 3698 if (testtype == 0) { 3699 pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n", 3700 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3701 if (deadlock && cyclelen == 1) 3702 pr_info("%s: Expect hang.\n", __func__); 3703 for (i = 0; i < cyclelen; i++) { 3704 j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu", 3705 "srcu_read_unlock", i, cyclelen, deadlock); 3706 idx = srcu_read_lock(srcus[i]); 3707 if (j >= 0) 3708 synchronize_srcu(srcus[j]); 3709 srcu_read_unlock(srcus[i], idx); 3710 } 3711 return; 3712 } 3713 3714 if (testtype == 1) { 3715 pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n", 3716 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3717 for (i = 0; i < cyclelen; i++) { 3718 pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n", 3719 __func__, i, i, i, i); 3720 idx = srcu_read_lock(srcus[i]); 3721 mutex_lock(muts[i]); 3722 mutex_unlock(muts[i]); 3723 srcu_read_unlock(srcus[i], idx); 3724 3725 j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu", 3726 "mutex_unlock", i, cyclelen, deadlock); 3727 mutex_lock(muts[i]); 3728 if (j >= 0) 3729 synchronize_srcu(srcus[j]); 3730 mutex_unlock(muts[i]); 3731 } 3732 return; 3733 } 3734 3735 if (testtype == 2) { 3736 pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n", 3737 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3738 for (i = 0; i < cyclelen; i++) { 3739 pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n", 3740 __func__, i, i, i, i); 3741 idx = srcu_read_lock(srcus[i]); 3742 down_read(rwsems[i]); 3743 up_read(rwsems[i]); 3744 srcu_read_unlock(srcus[i], idx); 3745 3746 j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu", 3747 "up_write", i, cyclelen, deadlock); 3748 down_write(rwsems[i]); 3749 if (j >= 0) 3750 synchronize_srcu(srcus[j]); 3751 up_write(rwsems[i]); 3752 } 3753 return; 3754 } 3755 3756 #ifdef CONFIG_TASKS_TRACE_RCU 3757 if (testtype == 3) { 3758 pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n", 3759 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 3760 if (deadlock && cyclelen == 1) 3761 pr_info("%s: Expect hang.\n", __func__); 3762 for (i = 0; i < cyclelen; i++) { 3763 char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock"; 3764 char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace" 3765 : "synchronize_srcu"; 3766 char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock"; 3767 3768 j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock); 3769 if (i == 0) 3770 rcu_read_lock_trace(); 3771 else 3772 idx = srcu_read_lock(srcus[i]); 3773 if (j >= 0) { 3774 if (i == cyclelen - 1) 3775 synchronize_rcu_tasks_trace(); 3776 else 3777 synchronize_srcu(srcus[j]); 3778 } 3779 if (i == 0) 3780 rcu_read_unlock_trace(); 3781 else 3782 srcu_read_unlock(srcus[i], idx); 3783 } 3784 return; 3785 } 3786 #endif // #ifdef CONFIG_TASKS_TRACE_RCU 3787 3788 err_out: 3789 pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); 3790 pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); 3791 pr_info("%s: D: Deadlock if nonzero.\n", __func__); 3792 pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); 3793 pr_info("%s: L: Cycle length.\n", __func__); 3794 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) 3795 pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); 3796 } 3797 3798 static int __init 3799 rcu_torture_init(void) 3800 { 3801 long i; 3802 int cpu; 3803 int firsterr = 0; 3804 int flags = 0; 3805 unsigned long gp_seq = 0; 3806 static struct rcu_torture_ops *torture_ops[] = { 3807 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, 3808 TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 3809 &trivial_ops, 3810 }; 3811 3812 if (!torture_init_begin(torture_type, verbose)) 3813 return -EBUSY; 3814 3815 /* Process args and tell the world that the torturer is on the job. */ 3816 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 3817 cur_ops = torture_ops[i]; 3818 if (strcmp(torture_type, cur_ops->name) == 0) 3819 break; 3820 } 3821 if (i == ARRAY_SIZE(torture_ops)) { 3822 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 3823 torture_type); 3824 pr_alert("rcu-torture types:"); 3825 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 3826 pr_cont(" %s", torture_ops[i]->name); 3827 pr_cont("\n"); 3828 firsterr = -EINVAL; 3829 cur_ops = NULL; 3830 goto unwind; 3831 } 3832 if (cur_ops->fqs == NULL && fqs_duration != 0) { 3833 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 3834 fqs_duration = 0; 3835 } 3836 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops || 3837 !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 3838 pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n", 3839 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU)); 3840 nocbs_nthreads = 0; 3841 } 3842 if (cur_ops->init) 3843 cur_ops->init(); 3844 3845 rcu_torture_init_srcu_lockdep(); 3846 3847 if (nreaders >= 0) { 3848 nrealreaders = nreaders; 3849 } else { 3850 nrealreaders = num_online_cpus() - 2 - nreaders; 3851 if (nrealreaders <= 0) 3852 nrealreaders = 1; 3853 } 3854 rcu_torture_print_module_parms(cur_ops, "Start of test"); 3855 if (cur_ops->get_gp_data) 3856 cur_ops->get_gp_data(&flags, &gp_seq); 3857 start_gp_seq = gp_seq; 3858 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", 3859 cur_ops->name, (long)gp_seq, flags); 3860 3861 /* Set up the freelist. */ 3862 3863 INIT_LIST_HEAD(&rcu_torture_freelist); 3864 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 3865 rcu_tortures[i].rtort_mbtest = 0; 3866 list_add_tail(&rcu_tortures[i].rtort_free, 3867 &rcu_torture_freelist); 3868 } 3869 3870 /* Initialize the statistics so that each run gets its own numbers. */ 3871 3872 rcu_torture_current = NULL; 3873 rcu_torture_current_version = 0; 3874 atomic_set(&n_rcu_torture_alloc, 0); 3875 atomic_set(&n_rcu_torture_alloc_fail, 0); 3876 atomic_set(&n_rcu_torture_free, 0); 3877 atomic_set(&n_rcu_torture_mberror, 0); 3878 atomic_set(&n_rcu_torture_mbchk_fail, 0); 3879 atomic_set(&n_rcu_torture_mbchk_tries, 0); 3880 atomic_set(&n_rcu_torture_error, 0); 3881 n_rcu_torture_barrier_error = 0; 3882 n_rcu_torture_boost_ktrerror = 0; 3883 n_rcu_torture_boost_failure = 0; 3884 n_rcu_torture_boosts = 0; 3885 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 3886 atomic_set(&rcu_torture_wcount[i], 0); 3887 for_each_possible_cpu(cpu) { 3888 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 3889 per_cpu(rcu_torture_count, cpu)[i] = 0; 3890 per_cpu(rcu_torture_batch, cpu)[i] = 0; 3891 } 3892 } 3893 err_segs_recorded = 0; 3894 rt_read_nsegs = 0; 3895 3896 /* Start up the kthreads. */ 3897 3898 rcu_torture_write_types(); 3899 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 3900 writer_task); 3901 if (torture_init_error(firsterr)) 3902 goto unwind; 3903 if (nfakewriters > 0) { 3904 fakewriter_tasks = kcalloc(nfakewriters, 3905 sizeof(fakewriter_tasks[0]), 3906 GFP_KERNEL); 3907 if (fakewriter_tasks == NULL) { 3908 TOROUT_ERRSTRING("out of memory"); 3909 firsterr = -ENOMEM; 3910 goto unwind; 3911 } 3912 } 3913 for (i = 0; i < nfakewriters; i++) { 3914 firsterr = torture_create_kthread(rcu_torture_fakewriter, 3915 NULL, fakewriter_tasks[i]); 3916 if (torture_init_error(firsterr)) 3917 goto unwind; 3918 } 3919 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 3920 GFP_KERNEL); 3921 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), 3922 GFP_KERNEL); 3923 if (!reader_tasks || !rcu_torture_reader_mbchk) { 3924 TOROUT_ERRSTRING("out of memory"); 3925 firsterr = -ENOMEM; 3926 goto unwind; 3927 } 3928 for (i = 0; i < nrealreaders; i++) { 3929 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 3930 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 3931 reader_tasks[i]); 3932 if (torture_init_error(firsterr)) 3933 goto unwind; 3934 } 3935 nrealnocbers = nocbs_nthreads; 3936 if (WARN_ON(nrealnocbers < 0)) 3937 nrealnocbers = 1; 3938 if (WARN_ON(nocbs_toggle < 0)) 3939 nocbs_toggle = HZ; 3940 if (nrealnocbers > 0) { 3941 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); 3942 if (nocb_tasks == NULL) { 3943 TOROUT_ERRSTRING("out of memory"); 3944 firsterr = -ENOMEM; 3945 goto unwind; 3946 } 3947 } else { 3948 nocb_tasks = NULL; 3949 } 3950 for (i = 0; i < nrealnocbers; i++) { 3951 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 3952 if (torture_init_error(firsterr)) 3953 goto unwind; 3954 } 3955 if (stat_interval > 0) { 3956 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 3957 stats_task); 3958 if (torture_init_error(firsterr)) 3959 goto unwind; 3960 } 3961 if (test_no_idle_hz && shuffle_interval > 0) { 3962 firsterr = torture_shuffle_init(shuffle_interval * HZ); 3963 if (torture_init_error(firsterr)) 3964 goto unwind; 3965 } 3966 if (stutter < 0) 3967 stutter = 0; 3968 if (stutter) { 3969 int t; 3970 3971 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 3972 firsterr = torture_stutter_init(stutter * HZ, t); 3973 if (torture_init_error(firsterr)) 3974 goto unwind; 3975 } 3976 if (fqs_duration < 0) 3977 fqs_duration = 0; 3978 if (fqs_holdoff < 0) 3979 fqs_holdoff = 0; 3980 if (fqs_duration && fqs_holdoff) { 3981 /* Create the fqs thread */ 3982 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 3983 fqs_task); 3984 if (torture_init_error(firsterr)) 3985 goto unwind; 3986 } 3987 if (test_boost_interval < 1) 3988 test_boost_interval = 1; 3989 if (test_boost_duration < 2) 3990 test_boost_duration = 2; 3991 if (rcu_torture_can_boost()) { 3992 3993 boost_starttime = jiffies + test_boost_interval * HZ; 3994 3995 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 3996 rcutorture_booster_init, 3997 rcutorture_booster_cleanup); 3998 rcutor_hp = firsterr; 3999 if (torture_init_error(firsterr)) 4000 goto unwind; 4001 } 4002 shutdown_jiffies = jiffies + shutdown_secs * HZ; 4003 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 4004 if (torture_init_error(firsterr)) 4005 goto unwind; 4006 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 4007 rcutorture_sync); 4008 if (torture_init_error(firsterr)) 4009 goto unwind; 4010 firsterr = rcu_torture_stall_init(); 4011 if (torture_init_error(firsterr)) 4012 goto unwind; 4013 firsterr = rcu_torture_fwd_prog_init(); 4014 if (torture_init_error(firsterr)) 4015 goto unwind; 4016 firsterr = rcu_torture_barrier_init(); 4017 if (torture_init_error(firsterr)) 4018 goto unwind; 4019 firsterr = rcu_torture_read_exit_init(); 4020 if (torture_init_error(firsterr)) 4021 goto unwind; 4022 if (object_debug) 4023 rcu_test_debug_objects(); 4024 torture_init_end(); 4025 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister)) 4026 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay); 4027 return 0; 4028 4029 unwind: 4030 torture_init_end(); 4031 rcu_torture_cleanup(); 4032 if (shutdown_secs) { 4033 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 4034 kernel_power_off(); 4035 } 4036 return firsterr; 4037 } 4038 4039 module_init(rcu_torture_init); 4040 module_exit(rcu_torture_cleanup); 4041