1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.rst 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/rcu_notifier.h> 25 #include <linux/interrupt.h> 26 #include <linux/sched/signal.h> 27 #include <uapi/linux/sched/types.h> 28 #include <linux/atomic.h> 29 #include <linux/bitops.h> 30 #include <linux/completion.h> 31 #include <linux/moduleparam.h> 32 #include <linux/percpu.h> 33 #include <linux/notifier.h> 34 #include <linux/reboot.h> 35 #include <linux/freezer.h> 36 #include <linux/cpu.h> 37 #include <linux/delay.h> 38 #include <linux/stat.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/trace_clock.h> 42 #include <asm/byteorder.h> 43 #include <linux/torture.h> 44 #include <linux/vmalloc.h> 45 #include <linux/sched/debug.h> 46 #include <linux/sched/sysctl.h> 47 #include <linux/oom.h> 48 #include <linux/tick.h> 49 #include <linux/rcupdate_trace.h> 50 #include <linux/nmi.h> 51 52 #include "rcu.h" 53 54 MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility"); 55 MODULE_LICENSE("GPL"); 56 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 57 58 // Bits for ->extendables field, extendables param, and related definitions. 59 #define RCUTORTURE_RDR_SHIFT_1 8 // Put SRCU index in upper bits. 60 #define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1) 61 #define RCUTORTURE_RDR_SHIFT_2 16 // Put SRCU index in upper bits. 62 #define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2) 63 #define RCUTORTURE_RDR_BH 0x01 // Extend readers by disabling bh. 64 #define RCUTORTURE_RDR_IRQ 0x02 // ... disabling interrupts. 65 #define RCUTORTURE_RDR_PREEMPT 0x04 // ... disabling preemption. 66 #define RCUTORTURE_RDR_RBH 0x08 // ... rcu_read_lock_bh(). 67 #define RCUTORTURE_RDR_SCHED 0x10 // ... rcu_read_lock_sched(). 68 #define RCUTORTURE_RDR_RCU_1 0x20 // ... entering another RCU reader. 69 #define RCUTORTURE_RDR_RCU_2 0x40 // ... entering another RCU reader. 70 #define RCUTORTURE_RDR_UPDOWN 0x80 // ... up-read from task, down-read from timer. 71 // Note: Manual start, automatic end. 72 #define RCUTORTURE_RDR_NBITS 8 // Number of bits defined above. 73 #define RCUTORTURE_MAX_EXTEND \ 74 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 75 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) // Intentionally omit RCUTORTURE_RDR_UPDOWN. 76 #define RCUTORTURE_RDR_ALLBITS \ 77 (RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \ 78 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2) 79 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 80 /* Must be power of two minus one. */ 81 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 82 83 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 84 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 85 torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); 86 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 87 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 88 torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); 89 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 90 torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); 91 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); 92 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 93 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); 94 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); 95 torture_param(bool, gp_cond_exp_full, false, 96 "Use conditional/async full-stateexpedited GP wait primitives"); 97 torture_param(int, gp_cond_wi, 16 * USEC_PER_SEC / HZ, 98 "Wait interval for normal conditional grace periods, us (default 16 jiffies)"); 99 torture_param(int, gp_cond_wi_exp, 128, 100 "Wait interval for expedited conditional grace periods, us (default 128 us)"); 101 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 102 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); 103 torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); 104 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); 105 torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); 106 torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); 107 torture_param(int, gp_poll_wi, 16 * USEC_PER_SEC / HZ, 108 "Wait interval for normal polled grace periods, us (default 16 jiffies)"); 109 torture_param(int, gp_poll_wi_exp, 128, 110 "Wait interval for expedited polled grace periods, us (default 128 us)"); 111 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 112 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 113 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); 114 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); 115 torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); 116 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 117 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 118 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); 119 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 120 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); 121 torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing"); 122 torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period."); 123 torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)"); 124 torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)"); 125 torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); 126 torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); 127 torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); 128 torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)"); 129 torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); 130 torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); 131 torture_param(int, reader_flavor, SRCU_READ_FLAVOR_NORMAL, "Reader flavors to use, one per bit."); 132 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 133 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 134 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 135 torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); 136 torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); 137 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 138 torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); 139 torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); 140 torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); 141 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 142 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 143 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 144 torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); 145 torture_param(int, test_boost_holdoff, 0, "Holdoff time from rcutorture start, seconds."); 146 torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); 147 torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); 148 torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); 149 torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); 150 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 151 152 static char *torture_type = "rcu"; 153 module_param(torture_type, charp, 0444); 154 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 155 156 static int nrealnocbers; 157 static int nrealreaders; 158 static int nrealfakewriters; 159 static struct task_struct *writer_task; 160 static struct task_struct **fakewriter_tasks; 161 static struct task_struct **reader_tasks; 162 static struct task_struct *updown_task; 163 static struct task_struct **nocb_tasks; 164 static struct task_struct *stats_task; 165 static struct task_struct *fqs_task; 166 static struct task_struct *boost_tasks[NR_CPUS]; 167 static struct task_struct *stall_task; 168 static struct task_struct **fwd_prog_tasks; 169 static struct task_struct **barrier_cbs_tasks; 170 static struct task_struct *barrier_task; 171 static struct task_struct *read_exit_task; 172 static struct task_struct *preempt_task; 173 174 #define RCU_TORTURE_PIPE_LEN 10 175 176 // Mailbox-like structure to check RCU global memory ordering. 177 struct rcu_torture_reader_check { 178 unsigned long rtc_myloops; 179 int rtc_chkrdr; 180 unsigned long rtc_chkloops; 181 int rtc_ready; 182 struct rcu_torture_reader_check *rtc_assigner; 183 } ____cacheline_internodealigned_in_smp; 184 185 // Update-side data structure used to check RCU readers. 186 struct rcu_torture { 187 struct rcu_head rtort_rcu; 188 int rtort_pipe_count; 189 struct list_head rtort_free; 190 int rtort_mbtest; 191 struct rcu_torture_reader_check *rtort_chkp; 192 }; 193 194 static LIST_HEAD(rcu_torture_freelist); 195 static struct rcu_torture __rcu *rcu_torture_current; 196 static unsigned long rcu_torture_current_version; 197 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 198 static DEFINE_SPINLOCK(rcu_torture_lock); 199 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 200 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 201 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 202 static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; 203 static atomic_t n_rcu_torture_alloc; 204 static atomic_t n_rcu_torture_alloc_fail; 205 static atomic_t n_rcu_torture_free; 206 static atomic_t n_rcu_torture_mberror; 207 static atomic_t n_rcu_torture_mbchk_fail; 208 static atomic_t n_rcu_torture_mbchk_tries; 209 static atomic_t n_rcu_torture_error; 210 static long n_rcu_torture_barrier_error; 211 static long n_rcu_torture_boost_ktrerror; 212 static long n_rcu_torture_boost_failure; 213 static long n_rcu_torture_boosts; 214 static atomic_long_t n_rcu_torture_timers; 215 static long n_barrier_attempts; 216 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 217 static unsigned long n_read_exits; 218 static struct list_head rcu_torture_removed; 219 static unsigned long shutdown_jiffies; 220 static unsigned long start_gp_seq; 221 static atomic_long_t n_nocb_offload; 222 static atomic_long_t n_nocb_deoffload; 223 224 static int rcu_torture_writer_state; 225 #define RTWS_FIXED_DELAY 0 226 #define RTWS_DELAY 1 227 #define RTWS_REPLACE 2 228 #define RTWS_DEF_FREE 3 229 #define RTWS_EXP_SYNC 4 230 #define RTWS_COND_GET 5 231 #define RTWS_COND_GET_FULL 6 232 #define RTWS_COND_GET_EXP 7 233 #define RTWS_COND_GET_EXP_FULL 8 234 #define RTWS_COND_SYNC 9 235 #define RTWS_COND_SYNC_FULL 10 236 #define RTWS_COND_SYNC_EXP 11 237 #define RTWS_COND_SYNC_EXP_FULL 12 238 #define RTWS_POLL_GET 13 239 #define RTWS_POLL_GET_FULL 14 240 #define RTWS_POLL_GET_EXP 15 241 #define RTWS_POLL_GET_EXP_FULL 16 242 #define RTWS_POLL_WAIT 17 243 #define RTWS_POLL_WAIT_FULL 18 244 #define RTWS_POLL_WAIT_EXP 19 245 #define RTWS_POLL_WAIT_EXP_FULL 20 246 #define RTWS_SYNC 21 247 #define RTWS_STUTTER 22 248 #define RTWS_STOPPING 23 249 static const char * const rcu_torture_writer_state_names[] = { 250 "RTWS_FIXED_DELAY", 251 "RTWS_DELAY", 252 "RTWS_REPLACE", 253 "RTWS_DEF_FREE", 254 "RTWS_EXP_SYNC", 255 "RTWS_COND_GET", 256 "RTWS_COND_GET_FULL", 257 "RTWS_COND_GET_EXP", 258 "RTWS_COND_GET_EXP_FULL", 259 "RTWS_COND_SYNC", 260 "RTWS_COND_SYNC_FULL", 261 "RTWS_COND_SYNC_EXP", 262 "RTWS_COND_SYNC_EXP_FULL", 263 "RTWS_POLL_GET", 264 "RTWS_POLL_GET_FULL", 265 "RTWS_POLL_GET_EXP", 266 "RTWS_POLL_GET_EXP_FULL", 267 "RTWS_POLL_WAIT", 268 "RTWS_POLL_WAIT_FULL", 269 "RTWS_POLL_WAIT_EXP", 270 "RTWS_POLL_WAIT_EXP_FULL", 271 "RTWS_SYNC", 272 "RTWS_STUTTER", 273 "RTWS_STOPPING", 274 }; 275 276 /* Record reader segment types and duration for first failing read. */ 277 struct rt_read_seg { 278 int rt_readstate; 279 unsigned long rt_delay_jiffies; 280 unsigned long rt_delay_ms; 281 unsigned long rt_delay_us; 282 bool rt_preempted; 283 int rt_cpu; 284 int rt_end_cpu; 285 unsigned long long rt_gp_seq; 286 unsigned long long rt_gp_seq_end; 287 u64 rt_ts; 288 }; 289 static int err_segs_recorded; 290 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 291 static int rt_read_nsegs; 292 static int rt_read_preempted; 293 294 static const char *rcu_torture_writer_state_getname(void) 295 { 296 unsigned int i = READ_ONCE(rcu_torture_writer_state); 297 298 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 299 return "???"; 300 return rcu_torture_writer_state_names[i]; 301 } 302 303 #ifdef CONFIG_RCU_TRACE 304 static u64 notrace rcu_trace_clock_local(void) 305 { 306 u64 ts = trace_clock_local(); 307 308 (void)do_div(ts, NSEC_PER_USEC); 309 return ts; 310 } 311 #else /* #ifdef CONFIG_RCU_TRACE */ 312 static u64 notrace rcu_trace_clock_local(void) 313 { 314 return 0ULL; 315 } 316 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 317 318 /* 319 * Stop aggressive CPU-hog tests a bit before the end of the test in order 320 * to avoid interfering with test shutdown. 321 */ 322 static bool shutdown_time_arrived(void) 323 { 324 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 325 } 326 327 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 328 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 329 /* and boost task create/destroy. */ 330 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 331 static bool barrier_phase; /* Test phase. */ 332 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 333 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 334 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 335 336 static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 337 338 /* 339 * Allocate an element from the rcu_tortures pool. 340 */ 341 static struct rcu_torture * 342 rcu_torture_alloc(void) 343 { 344 struct list_head *p; 345 346 spin_lock_bh(&rcu_torture_lock); 347 if (list_empty(&rcu_torture_freelist)) { 348 atomic_inc(&n_rcu_torture_alloc_fail); 349 spin_unlock_bh(&rcu_torture_lock); 350 return NULL; 351 } 352 atomic_inc(&n_rcu_torture_alloc); 353 p = rcu_torture_freelist.next; 354 list_del_init(p); 355 spin_unlock_bh(&rcu_torture_lock); 356 return container_of(p, struct rcu_torture, rtort_free); 357 } 358 359 /* 360 * Free an element to the rcu_tortures pool. 361 */ 362 static void 363 rcu_torture_free(struct rcu_torture *p) 364 { 365 atomic_inc(&n_rcu_torture_free); 366 spin_lock_bh(&rcu_torture_lock); 367 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 368 spin_unlock_bh(&rcu_torture_lock); 369 } 370 371 /* 372 * Operations vector for selecting different types of tests. 373 */ 374 375 struct rcu_torture_ops { 376 int ttype; 377 void (*init)(void); 378 void (*cleanup)(void); 379 int (*readlock)(void); 380 void (*read_delay)(struct torture_random_state *rrsp, 381 struct rt_read_seg *rtrsp); 382 void (*readunlock)(int idx); 383 int (*readlock_held)(void); // lockdep. 384 int (*readlock_nesting)(void); // actual nesting, if available, -1 if not. 385 int (*down_read)(void); 386 void (*up_read)(int idx); 387 unsigned long (*get_gp_seq)(void); 388 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 389 void (*deferred_free)(struct rcu_torture *p); 390 void (*sync)(void); 391 void (*exp_sync)(void); 392 unsigned long (*get_gp_state_exp)(void); 393 unsigned long (*start_gp_poll_exp)(void); 394 void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); 395 bool (*poll_gp_state_exp)(unsigned long oldstate); 396 void (*cond_sync_exp)(unsigned long oldstate); 397 void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); 398 unsigned long (*get_comp_state)(void); 399 void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); 400 bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); 401 bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); 402 unsigned long (*get_gp_state)(void); 403 void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); 404 unsigned long (*start_gp_poll)(void); 405 void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); 406 bool (*poll_gp_state)(unsigned long oldstate); 407 bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); 408 bool (*poll_need_2gp)(bool poll, bool poll_full); 409 void (*cond_sync)(unsigned long oldstate); 410 void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); 411 int poll_active; 412 int poll_active_full; 413 call_rcu_func_t call; 414 void (*cb_barrier)(void); 415 void (*fqs)(void); 416 void (*stats)(void); 417 void (*gp_kthread_dbg)(void); 418 bool (*check_boost_failed)(unsigned long gp_state, int *cpup); 419 int (*stall_dur)(void); 420 void (*get_gp_data)(int *flags, unsigned long *gp_seq); 421 void (*gp_slow_register)(atomic_t *rgssp); 422 void (*gp_slow_unregister)(atomic_t *rgssp); 423 bool (*reader_blocked)(void); 424 unsigned long long (*gather_gp_seqs)(void); 425 void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); 426 void (*set_gpwrap_lag)(unsigned long lag); 427 int (*get_gpwrap_count)(int cpu); 428 long cbflood_max; 429 int irq_capable; 430 int can_boost; 431 int extendables; 432 int slow_gps; 433 int no_pi_lock; 434 int debug_objects; 435 int start_poll_irqsoff; 436 int have_up_down; 437 const char *name; 438 }; 439 440 static struct rcu_torture_ops *cur_ops; 441 442 /* 443 * Definitions for rcu torture testing. 444 */ 445 446 static int torture_readlock_not_held(void) 447 { 448 return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); 449 } 450 451 static int rcu_torture_read_lock(void) 452 { 453 rcu_read_lock(); 454 return 0; 455 } 456 457 static void 458 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 459 { 460 unsigned long started; 461 unsigned long completed; 462 const unsigned long shortdelay_us = 200; 463 unsigned long longdelay_ms = 300; 464 unsigned long long ts; 465 466 /* We want a short delay sometimes to make a reader delay the grace 467 * period, and we want a long delay occasionally to trigger 468 * force_quiescent_state. */ 469 470 if (!atomic_read(&rcu_fwd_cb_nodelay) && 471 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 472 started = cur_ops->get_gp_seq(); 473 ts = rcu_trace_clock_local(); 474 if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) 475 longdelay_ms = 5; /* Avoid triggering BH limits. */ 476 mdelay(longdelay_ms); 477 rtrsp->rt_delay_ms = longdelay_ms; 478 completed = cur_ops->get_gp_seq(); 479 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 480 started, completed); 481 } 482 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 483 udelay(shortdelay_us); 484 rtrsp->rt_delay_us = shortdelay_us; 485 } 486 if (!preempt_count() && 487 !(torture_random(rrsp) % (nrealreaders * 500))) 488 torture_preempt_schedule(); /* QS only if preemptible. */ 489 } 490 491 static void rcu_torture_read_unlock(int idx) 492 { 493 rcu_read_unlock(); 494 } 495 496 static int rcu_torture_readlock_nesting(void) 497 { 498 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) 499 return rcu_preempt_depth(); 500 if (IS_ENABLED(CONFIG_PREEMPT_COUNT)) 501 return (preempt_count() & PREEMPT_MASK); 502 return -1; 503 } 504 505 /* 506 * Update callback in the pipe. This should be invoked after a grace period. 507 */ 508 static bool 509 rcu_torture_pipe_update_one(struct rcu_torture *rp) 510 { 511 int i; 512 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); 513 514 if (rtrcp) { 515 WRITE_ONCE(rp->rtort_chkp, NULL); 516 smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). 517 } 518 i = rp->rtort_pipe_count; 519 if (i > RCU_TORTURE_PIPE_LEN) 520 i = RCU_TORTURE_PIPE_LEN; 521 atomic_inc(&rcu_torture_wcount[i]); 522 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 523 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 524 if (i + 1 >= RCU_TORTURE_PIPE_LEN) { 525 rp->rtort_mbtest = 0; 526 return true; 527 } 528 return false; 529 } 530 531 /* 532 * Update all callbacks in the pipe. Suitable for synchronous grace-period 533 * primitives. 534 */ 535 static void 536 rcu_torture_pipe_update(struct rcu_torture *old_rp) 537 { 538 struct rcu_torture *rp; 539 struct rcu_torture *rp1; 540 541 if (old_rp) 542 list_add(&old_rp->rtort_free, &rcu_torture_removed); 543 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 544 if (rcu_torture_pipe_update_one(rp)) { 545 list_del(&rp->rtort_free); 546 rcu_torture_free(rp); 547 } 548 } 549 } 550 551 static void 552 rcu_torture_cb(struct rcu_head *p) 553 { 554 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 555 556 if (torture_must_stop_irq()) { 557 /* Test is ending, just drop callbacks on the floor. */ 558 /* The next initialization will pick up the pieces. */ 559 return; 560 } 561 if (rcu_torture_pipe_update_one(rp)) 562 rcu_torture_free(rp); 563 else 564 cur_ops->deferred_free(rp); 565 } 566 567 static unsigned long rcu_no_completed(void) 568 { 569 return 0; 570 } 571 572 static void rcu_torture_deferred_free(struct rcu_torture *p) 573 { 574 call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); 575 } 576 577 static void rcu_sync_torture_init(void) 578 { 579 INIT_LIST_HEAD(&rcu_torture_removed); 580 } 581 582 static bool rcu_poll_need_2gp(bool poll, bool poll_full) 583 { 584 return poll; 585 } 586 587 static struct rcu_torture_ops rcu_ops = { 588 .ttype = RCU_FLAVOR, 589 .init = rcu_sync_torture_init, 590 .readlock = rcu_torture_read_lock, 591 .read_delay = rcu_read_delay, 592 .readunlock = rcu_torture_read_unlock, 593 .readlock_held = torture_readlock_not_held, 594 .readlock_nesting = rcu_torture_readlock_nesting, 595 .get_gp_seq = rcu_get_gp_seq, 596 .gp_diff = rcu_seq_diff, 597 .deferred_free = rcu_torture_deferred_free, 598 .sync = synchronize_rcu, 599 .exp_sync = synchronize_rcu_expedited, 600 .same_gp_state = same_state_synchronize_rcu, 601 .same_gp_state_full = same_state_synchronize_rcu_full, 602 .get_comp_state = get_completed_synchronize_rcu, 603 .get_comp_state_full = get_completed_synchronize_rcu_full, 604 .get_gp_state = get_state_synchronize_rcu, 605 .get_gp_state_full = get_state_synchronize_rcu_full, 606 .start_gp_poll = start_poll_synchronize_rcu, 607 .start_gp_poll_full = start_poll_synchronize_rcu_full, 608 .poll_gp_state = poll_state_synchronize_rcu, 609 .poll_gp_state_full = poll_state_synchronize_rcu_full, 610 .poll_need_2gp = rcu_poll_need_2gp, 611 .cond_sync = cond_synchronize_rcu, 612 .cond_sync_full = cond_synchronize_rcu_full, 613 .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE, 614 .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, 615 .get_gp_state_exp = get_state_synchronize_rcu, 616 .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, 617 .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, 618 .poll_gp_state_exp = poll_state_synchronize_rcu, 619 .cond_sync_exp = cond_synchronize_rcu_expedited, 620 .cond_sync_exp_full = cond_synchronize_rcu_expedited_full, 621 .call = call_rcu_hurry, 622 .cb_barrier = rcu_barrier, 623 .fqs = rcu_force_quiescent_state, 624 .gp_kthread_dbg = show_rcu_gp_kthreads, 625 .check_boost_failed = rcu_check_boost_fail, 626 .stall_dur = rcu_jiffies_till_stall_check, 627 .get_gp_data = rcutorture_get_gp_data, 628 .gp_slow_register = rcu_gp_slow_register, 629 .gp_slow_unregister = rcu_gp_slow_unregister, 630 .reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU) 631 ? has_rcu_reader_blocked 632 : NULL, 633 .gather_gp_seqs = rcutorture_gather_gp_seqs, 634 .format_gp_seqs = rcutorture_format_gp_seqs, 635 .set_gpwrap_lag = rcu_set_gpwrap_lag, 636 .get_gpwrap_count = rcu_get_gpwrap_count, 637 .irq_capable = 1, 638 .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), 639 .extendables = RCUTORTURE_MAX_EXTEND, 640 .debug_objects = 1, 641 .start_poll_irqsoff = 1, 642 .name = "rcu" 643 }; 644 645 /* 646 * Don't even think about trying any of these in real life!!! 647 * The names includes "busted", and they really means it! 648 * The only purpose of these functions is to provide a buggy RCU 649 * implementation to make sure that rcutorture correctly emits 650 * buggy-RCU error messages. 651 */ 652 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 653 { 654 /* This is a deliberate bug for testing purposes only! */ 655 rcu_torture_cb(&p->rtort_rcu); 656 } 657 658 static void synchronize_rcu_busted(void) 659 { 660 /* This is a deliberate bug for testing purposes only! */ 661 } 662 663 static void 664 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 665 { 666 /* This is a deliberate bug for testing purposes only! */ 667 func(head); 668 } 669 670 static struct rcu_torture_ops rcu_busted_ops = { 671 .ttype = INVALID_RCU_FLAVOR, 672 .init = rcu_sync_torture_init, 673 .readlock = rcu_torture_read_lock, 674 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 675 .readunlock = rcu_torture_read_unlock, 676 .readlock_held = torture_readlock_not_held, 677 .get_gp_seq = rcu_no_completed, 678 .deferred_free = rcu_busted_torture_deferred_free, 679 .sync = synchronize_rcu_busted, 680 .exp_sync = synchronize_rcu_busted, 681 .call = call_rcu_busted, 682 .gather_gp_seqs = rcutorture_gather_gp_seqs, 683 .format_gp_seqs = rcutorture_format_gp_seqs, 684 .irq_capable = 1, 685 .extendables = RCUTORTURE_MAX_EXTEND, 686 .name = "busted" 687 }; 688 689 /* 690 * Definitions for srcu torture testing. 691 */ 692 693 DEFINE_STATIC_SRCU(srcu_ctl); 694 static struct srcu_struct srcu_ctld; 695 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 696 static struct rcu_torture_ops srcud_ops; 697 698 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) 699 { 700 srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq); 701 } 702 703 static int srcu_torture_read_lock(void) 704 { 705 int idx; 706 struct srcu_ctr __percpu *scp; 707 int ret = 0; 708 709 WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL); 710 711 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) { 712 idx = srcu_read_lock(srcu_ctlp); 713 WARN_ON_ONCE(idx & ~0x1); 714 ret += idx; 715 } 716 if (reader_flavor & SRCU_READ_FLAVOR_NMI) { 717 idx = srcu_read_lock_nmisafe(srcu_ctlp); 718 WARN_ON_ONCE(idx & ~0x1); 719 ret += idx << 1; 720 } 721 if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 722 scp = srcu_read_lock_fast(srcu_ctlp); 723 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 724 WARN_ON_ONCE(idx & ~0x1); 725 ret += idx << 3; 726 } 727 return ret; 728 } 729 730 static void 731 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 732 { 733 long delay; 734 const long uspertick = 1000000 / HZ; 735 const long longdelay = 10; 736 737 /* We want there to be long-running readers, but not all the time. */ 738 739 delay = torture_random(rrsp) % 740 (nrealreaders * 2 * longdelay * uspertick); 741 if (!delay && in_task()) { 742 schedule_timeout_interruptible(longdelay); 743 rtrsp->rt_delay_jiffies = longdelay; 744 } else { 745 rcu_read_delay(rrsp, rtrsp); 746 } 747 } 748 749 static void srcu_torture_read_unlock(int idx) 750 { 751 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 752 if (reader_flavor & SRCU_READ_FLAVOR_FAST) 753 srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); 754 if (reader_flavor & SRCU_READ_FLAVOR_NMI) 755 srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1); 756 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) 757 srcu_read_unlock(srcu_ctlp, idx & 0x1); 758 } 759 760 static int torture_srcu_read_lock_held(void) 761 { 762 return srcu_read_lock_held(srcu_ctlp); 763 } 764 765 static bool srcu_torture_have_up_down(void) 766 { 767 int rf = reader_flavor; 768 769 if (!rf) 770 rf = SRCU_READ_FLAVOR_NORMAL; 771 return !!(cur_ops->have_up_down & rf); 772 } 773 774 static int srcu_torture_down_read(void) 775 { 776 int idx; 777 struct srcu_ctr __percpu *scp; 778 779 WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL); 780 WARN_ON_ONCE(reader_flavor & (reader_flavor - 1)); 781 782 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) { 783 idx = srcu_down_read(srcu_ctlp); 784 WARN_ON_ONCE(idx & ~0x1); 785 return idx; 786 } 787 if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 788 scp = srcu_down_read_fast(srcu_ctlp); 789 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 790 WARN_ON_ONCE(idx & ~0x1); 791 return idx << 3; 792 } 793 WARN_ON_ONCE(1); 794 return 0; 795 } 796 797 static void srcu_torture_up_read(int idx) 798 { 799 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 800 if (reader_flavor & SRCU_READ_FLAVOR_FAST) 801 srcu_up_read_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); 802 else if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || 803 !(reader_flavor & SRCU_READ_FLAVOR_ALL)) 804 srcu_up_read(srcu_ctlp, idx & 0x1); 805 else 806 WARN_ON_ONCE(1); 807 } 808 809 static unsigned long srcu_torture_completed(void) 810 { 811 return srcu_batches_completed(srcu_ctlp); 812 } 813 814 static void srcu_torture_deferred_free(struct rcu_torture *rp) 815 { 816 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 817 } 818 819 static void srcu_torture_synchronize(void) 820 { 821 synchronize_srcu(srcu_ctlp); 822 } 823 824 static unsigned long srcu_torture_get_gp_state(void) 825 { 826 return get_state_synchronize_srcu(srcu_ctlp); 827 } 828 829 static unsigned long srcu_torture_start_gp_poll(void) 830 { 831 return start_poll_synchronize_srcu(srcu_ctlp); 832 } 833 834 static bool srcu_torture_poll_gp_state(unsigned long oldstate) 835 { 836 return poll_state_synchronize_srcu(srcu_ctlp, oldstate); 837 } 838 839 static void srcu_torture_call(struct rcu_head *head, 840 rcu_callback_t func) 841 { 842 call_srcu(srcu_ctlp, head, func); 843 } 844 845 static void srcu_torture_barrier(void) 846 { 847 srcu_barrier(srcu_ctlp); 848 } 849 850 static void srcu_torture_stats(void) 851 { 852 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 853 } 854 855 static void srcu_torture_synchronize_expedited(void) 856 { 857 synchronize_srcu_expedited(srcu_ctlp); 858 } 859 860 static struct rcu_torture_ops srcu_ops = { 861 .ttype = SRCU_FLAVOR, 862 .init = rcu_sync_torture_init, 863 .readlock = srcu_torture_read_lock, 864 .read_delay = srcu_read_delay, 865 .readunlock = srcu_torture_read_unlock, 866 .down_read = srcu_torture_down_read, 867 .up_read = srcu_torture_up_read, 868 .readlock_held = torture_srcu_read_lock_held, 869 .get_gp_seq = srcu_torture_completed, 870 .gp_diff = rcu_seq_diff, 871 .deferred_free = srcu_torture_deferred_free, 872 .sync = srcu_torture_synchronize, 873 .exp_sync = srcu_torture_synchronize_expedited, 874 .same_gp_state = same_state_synchronize_srcu, 875 .get_comp_state = get_completed_synchronize_srcu, 876 .get_gp_state = srcu_torture_get_gp_state, 877 .start_gp_poll = srcu_torture_start_gp_poll, 878 .poll_gp_state = srcu_torture_poll_gp_state, 879 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 880 .call = srcu_torture_call, 881 .cb_barrier = srcu_torture_barrier, 882 .stats = srcu_torture_stats, 883 .get_gp_data = srcu_get_gp_data, 884 .cbflood_max = 50000, 885 .irq_capable = 1, 886 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 887 .debug_objects = 1, 888 .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) 889 ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST, 890 .name = "srcu" 891 }; 892 893 static void srcu_torture_init(void) 894 { 895 rcu_sync_torture_init(); 896 WARN_ON(init_srcu_struct(&srcu_ctld)); 897 srcu_ctlp = &srcu_ctld; 898 } 899 900 static void srcu_torture_cleanup(void) 901 { 902 cleanup_srcu_struct(&srcu_ctld); 903 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 904 } 905 906 /* As above, but dynamically allocated. */ 907 static struct rcu_torture_ops srcud_ops = { 908 .ttype = SRCU_FLAVOR, 909 .init = srcu_torture_init, 910 .cleanup = srcu_torture_cleanup, 911 .readlock = srcu_torture_read_lock, 912 .read_delay = srcu_read_delay, 913 .readunlock = srcu_torture_read_unlock, 914 .readlock_held = torture_srcu_read_lock_held, 915 .down_read = srcu_torture_down_read, 916 .up_read = srcu_torture_up_read, 917 .get_gp_seq = srcu_torture_completed, 918 .gp_diff = rcu_seq_diff, 919 .deferred_free = srcu_torture_deferred_free, 920 .sync = srcu_torture_synchronize, 921 .exp_sync = srcu_torture_synchronize_expedited, 922 .same_gp_state = same_state_synchronize_srcu, 923 .get_comp_state = get_completed_synchronize_srcu, 924 .get_gp_state = srcu_torture_get_gp_state, 925 .start_gp_poll = srcu_torture_start_gp_poll, 926 .poll_gp_state = srcu_torture_poll_gp_state, 927 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 928 .call = srcu_torture_call, 929 .cb_barrier = srcu_torture_barrier, 930 .stats = srcu_torture_stats, 931 .get_gp_data = srcu_get_gp_data, 932 .cbflood_max = 50000, 933 .irq_capable = 1, 934 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 935 .debug_objects = 1, 936 .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) 937 ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST, 938 .name = "srcud" 939 }; 940 941 /* As above, but broken due to inappropriate reader extension. */ 942 static struct rcu_torture_ops busted_srcud_ops = { 943 .ttype = SRCU_FLAVOR, 944 .init = srcu_torture_init, 945 .cleanup = srcu_torture_cleanup, 946 .readlock = srcu_torture_read_lock, 947 .read_delay = rcu_read_delay, 948 .readunlock = srcu_torture_read_unlock, 949 .readlock_held = torture_srcu_read_lock_held, 950 .get_gp_seq = srcu_torture_completed, 951 .deferred_free = srcu_torture_deferred_free, 952 .sync = srcu_torture_synchronize, 953 .exp_sync = srcu_torture_synchronize_expedited, 954 .call = srcu_torture_call, 955 .cb_barrier = srcu_torture_barrier, 956 .stats = srcu_torture_stats, 957 .irq_capable = 1, 958 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 959 .extendables = RCUTORTURE_MAX_EXTEND, 960 .name = "busted_srcud" 961 }; 962 963 /* 964 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 965 * This implementation does not work well with CPU hotplug nor 966 * with rcutorture's shuffling. 967 */ 968 969 static void synchronize_rcu_trivial(void) 970 { 971 int cpu; 972 973 for_each_online_cpu(cpu) { 974 torture_sched_setaffinity(current->pid, cpumask_of(cpu), true); 975 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 976 } 977 } 978 979 static void rcu_sync_torture_init_trivial(void) 980 { 981 rcu_sync_torture_init(); 982 // if (onoff_interval || shuffle_interval) { 983 if (WARN_ONCE(onoff_interval || shuffle_interval, "%s: Non-zero onoff_interval (%d) or shuffle_interval (%d) breaks trivial RCU, resetting to zero", __func__, onoff_interval, shuffle_interval)) { 984 onoff_interval = 0; 985 shuffle_interval = 0; 986 } 987 } 988 989 static int rcu_torture_read_lock_trivial(void) 990 { 991 preempt_disable(); 992 return 0; 993 } 994 995 static void rcu_torture_read_unlock_trivial(int idx) 996 { 997 preempt_enable(); 998 } 999 1000 static struct rcu_torture_ops trivial_ops = { 1001 .ttype = RCU_TRIVIAL_FLAVOR, 1002 .init = rcu_sync_torture_init_trivial, 1003 .readlock = rcu_torture_read_lock_trivial, 1004 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1005 .readunlock = rcu_torture_read_unlock_trivial, 1006 .readlock_held = torture_readlock_not_held, 1007 .get_gp_seq = rcu_no_completed, 1008 .sync = synchronize_rcu_trivial, 1009 .exp_sync = synchronize_rcu_trivial, 1010 .irq_capable = 1, 1011 .name = "trivial" 1012 }; 1013 1014 #ifdef CONFIG_TASKS_RCU 1015 1016 /* 1017 * Definitions for RCU-tasks torture testing. 1018 */ 1019 1020 static int tasks_torture_read_lock(void) 1021 { 1022 return 0; 1023 } 1024 1025 static void tasks_torture_read_unlock(int idx) 1026 { 1027 } 1028 1029 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 1030 { 1031 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 1032 } 1033 1034 static void synchronize_rcu_mult_test(void) 1035 { 1036 synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); 1037 } 1038 1039 static struct rcu_torture_ops tasks_ops = { 1040 .ttype = RCU_TASKS_FLAVOR, 1041 .init = rcu_sync_torture_init, 1042 .readlock = tasks_torture_read_lock, 1043 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1044 .readunlock = tasks_torture_read_unlock, 1045 .get_gp_seq = rcu_no_completed, 1046 .deferred_free = rcu_tasks_torture_deferred_free, 1047 .sync = synchronize_rcu_tasks, 1048 .exp_sync = synchronize_rcu_mult_test, 1049 .call = call_rcu_tasks, 1050 .cb_barrier = rcu_barrier_tasks, 1051 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, 1052 .get_gp_data = rcu_tasks_get_gp_data, 1053 .irq_capable = 1, 1054 .slow_gps = 1, 1055 .name = "tasks" 1056 }; 1057 1058 #define TASKS_OPS &tasks_ops, 1059 1060 #else // #ifdef CONFIG_TASKS_RCU 1061 1062 #define TASKS_OPS 1063 1064 #endif // #else #ifdef CONFIG_TASKS_RCU 1065 1066 1067 #ifdef CONFIG_TASKS_RUDE_RCU 1068 1069 /* 1070 * Definitions for rude RCU-tasks torture testing. 1071 */ 1072 1073 static struct rcu_torture_ops tasks_rude_ops = { 1074 .ttype = RCU_TASKS_RUDE_FLAVOR, 1075 .init = rcu_sync_torture_init, 1076 .readlock = rcu_torture_read_lock_trivial, 1077 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1078 .readunlock = rcu_torture_read_unlock_trivial, 1079 .get_gp_seq = rcu_no_completed, 1080 .sync = synchronize_rcu_tasks_rude, 1081 .exp_sync = synchronize_rcu_tasks_rude, 1082 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, 1083 .get_gp_data = rcu_tasks_rude_get_gp_data, 1084 .cbflood_max = 50000, 1085 .irq_capable = 1, 1086 .name = "tasks-rude" 1087 }; 1088 1089 #define TASKS_RUDE_OPS &tasks_rude_ops, 1090 1091 #else // #ifdef CONFIG_TASKS_RUDE_RCU 1092 1093 #define TASKS_RUDE_OPS 1094 1095 #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU 1096 1097 1098 #ifdef CONFIG_TASKS_TRACE_RCU 1099 1100 /* 1101 * Definitions for tracing RCU-tasks torture testing. 1102 */ 1103 1104 static int tasks_tracing_torture_read_lock(void) 1105 { 1106 rcu_read_lock_trace(); 1107 return 0; 1108 } 1109 1110 static void tasks_tracing_torture_read_unlock(int idx) 1111 { 1112 rcu_read_unlock_trace(); 1113 } 1114 1115 static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) 1116 { 1117 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); 1118 } 1119 1120 static struct rcu_torture_ops tasks_tracing_ops = { 1121 .ttype = RCU_TASKS_TRACING_FLAVOR, 1122 .init = rcu_sync_torture_init, 1123 .readlock = tasks_tracing_torture_read_lock, 1124 .read_delay = srcu_read_delay, /* just reuse srcu's version. */ 1125 .readunlock = tasks_tracing_torture_read_unlock, 1126 .readlock_held = rcu_read_lock_trace_held, 1127 .get_gp_seq = rcu_no_completed, 1128 .deferred_free = rcu_tasks_tracing_torture_deferred_free, 1129 .sync = synchronize_rcu_tasks_trace, 1130 .exp_sync = synchronize_rcu_tasks_trace, 1131 .call = call_rcu_tasks_trace, 1132 .cb_barrier = rcu_barrier_tasks_trace, 1133 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, 1134 .get_gp_data = rcu_tasks_trace_get_gp_data, 1135 .cbflood_max = 50000, 1136 .irq_capable = 1, 1137 .slow_gps = 1, 1138 .name = "tasks-tracing" 1139 }; 1140 1141 #define TASKS_TRACING_OPS &tasks_tracing_ops, 1142 1143 #else // #ifdef CONFIG_TASKS_TRACE_RCU 1144 1145 #define TASKS_TRACING_OPS 1146 1147 #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU 1148 1149 1150 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 1151 { 1152 if (!cur_ops->gp_diff) 1153 return new - old; 1154 return cur_ops->gp_diff(new, old); 1155 } 1156 1157 /* 1158 * RCU torture priority-boost testing. Runs one real-time thread per 1159 * CPU for moderate bursts, repeatedly starting grace periods and waiting 1160 * for them to complete. If a given grace period takes too long, we assume 1161 * that priority inversion has occurred. 1162 */ 1163 1164 static int old_rt_runtime = -1; 1165 1166 static void rcu_torture_disable_rt_throttle(void) 1167 { 1168 /* 1169 * Disable RT throttling so that rcutorture's boost threads don't get 1170 * throttled. Only possible if rcutorture is built-in otherwise the 1171 * user should manually do this by setting the sched_rt_period_us and 1172 * sched_rt_runtime sysctls. 1173 */ 1174 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 1175 return; 1176 1177 old_rt_runtime = sysctl_sched_rt_runtime; 1178 sysctl_sched_rt_runtime = -1; 1179 } 1180 1181 static void rcu_torture_enable_rt_throttle(void) 1182 { 1183 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 1184 return; 1185 1186 sysctl_sched_rt_runtime = old_rt_runtime; 1187 old_rt_runtime = -1; 1188 } 1189 1190 static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) 1191 { 1192 int cpu; 1193 static int dbg_done; 1194 unsigned long end = jiffies; 1195 bool gp_done; 1196 unsigned long j; 1197 static unsigned long last_persist; 1198 unsigned long lp; 1199 unsigned long mininterval = test_boost_duration * HZ - HZ / 2; 1200 1201 if (end - *start > mininterval) { 1202 // Recheck after checking time to avoid false positives. 1203 smp_mb(); // Time check before grace-period check. 1204 if (cur_ops->poll_gp_state(gp_state)) 1205 return false; // passed, though perhaps just barely 1206 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { 1207 // At most one persisted message per boost test. 1208 j = jiffies; 1209 lp = READ_ONCE(last_persist); 1210 if (time_after(j, lp + mininterval) && 1211 cmpxchg(&last_persist, lp, j) == lp) { 1212 if (cpu < 0) 1213 pr_info("Boost inversion persisted: QS from all CPUs\n"); 1214 else 1215 pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); 1216 } 1217 return false; // passed on a technicality 1218 } 1219 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 1220 n_rcu_torture_boost_failure++; 1221 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { 1222 pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", 1223 current->rt_priority, gp_state, end - *start); 1224 cur_ops->gp_kthread_dbg(); 1225 // Recheck after print to flag grace period ending during splat. 1226 gp_done = cur_ops->poll_gp_state(gp_state); 1227 pr_info("Boost inversion: GP %lu %s.\n", gp_state, 1228 gp_done ? "ended already" : "still pending"); 1229 1230 } 1231 1232 return true; // failed 1233 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { 1234 *start = jiffies; 1235 } 1236 1237 return false; // passed 1238 } 1239 1240 static int rcu_torture_boost(void *arg) 1241 { 1242 unsigned long endtime; 1243 unsigned long gp_state; 1244 unsigned long gp_state_time; 1245 unsigned long oldstarttime; 1246 unsigned long booststarttime = get_torture_init_jiffies() + test_boost_holdoff * HZ; 1247 1248 if (test_boost_holdoff <= 0 || time_after(jiffies, booststarttime)) { 1249 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 1250 } else { 1251 VERBOSE_TOROUT_STRING("rcu_torture_boost started holdoff period"); 1252 while (time_before(jiffies, booststarttime)) { 1253 schedule_timeout_idle(HZ); 1254 if (kthread_should_stop()) 1255 goto cleanup; 1256 } 1257 VERBOSE_TOROUT_STRING("rcu_torture_boost finished holdoff period"); 1258 } 1259 1260 /* Set real-time priority. */ 1261 sched_set_fifo_low(current); 1262 1263 /* Each pass through the following loop does one boost-test cycle. */ 1264 do { 1265 bool failed = false; // Test failed already in this test interval 1266 bool gp_initiated = false; 1267 1268 if (kthread_should_stop()) 1269 goto checkwait; 1270 1271 /* Wait for the next test interval. */ 1272 oldstarttime = READ_ONCE(boost_starttime); 1273 while (time_before(jiffies, oldstarttime)) { 1274 schedule_timeout_interruptible(oldstarttime - jiffies); 1275 if (stutter_wait("rcu_torture_boost")) 1276 sched_set_fifo_low(current); 1277 if (torture_must_stop()) 1278 goto checkwait; 1279 } 1280 1281 // Do one boost-test interval. 1282 endtime = oldstarttime + test_boost_duration * HZ; 1283 while (time_before(jiffies, endtime)) { 1284 // Has current GP gone too long? 1285 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1286 failed = rcu_torture_boost_failed(gp_state, &gp_state_time); 1287 // If we don't have a grace period in flight, start one. 1288 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { 1289 gp_state = cur_ops->start_gp_poll(); 1290 gp_initiated = true; 1291 gp_state_time = jiffies; 1292 } 1293 if (stutter_wait("rcu_torture_boost")) { 1294 sched_set_fifo_low(current); 1295 // If the grace period already ended, 1296 // we don't know when that happened, so 1297 // start over. 1298 if (cur_ops->poll_gp_state(gp_state)) 1299 gp_initiated = false; 1300 } 1301 if (torture_must_stop()) 1302 goto checkwait; 1303 } 1304 1305 // In case the grace period extended beyond the end of the loop. 1306 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1307 rcu_torture_boost_failed(gp_state, &gp_state_time); 1308 1309 /* 1310 * Set the start time of the next test interval. 1311 * Yes, this is vulnerable to long delays, but such 1312 * delays simply cause a false negative for the next 1313 * interval. Besides, we are running at RT priority, 1314 * so delays should be relatively rare. 1315 */ 1316 while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { 1317 if (mutex_trylock(&boost_mutex)) { 1318 if (oldstarttime == boost_starttime) { 1319 WRITE_ONCE(boost_starttime, 1320 jiffies + test_boost_interval * HZ); 1321 n_rcu_torture_boosts++; 1322 } 1323 mutex_unlock(&boost_mutex); 1324 break; 1325 } 1326 schedule_timeout_uninterruptible(HZ / 20); 1327 } 1328 1329 /* Go do the stutter. */ 1330 checkwait: if (stutter_wait("rcu_torture_boost")) 1331 sched_set_fifo_low(current); 1332 } while (!torture_must_stop()); 1333 1334 cleanup: 1335 /* Clean up and exit. */ 1336 while (!kthread_should_stop()) { 1337 torture_shutdown_absorb("rcu_torture_boost"); 1338 schedule_timeout_uninterruptible(HZ / 20); 1339 } 1340 torture_kthread_stopping("rcu_torture_boost"); 1341 return 0; 1342 } 1343 1344 /* 1345 * RCU torture force-quiescent-state kthread. Repeatedly induces 1346 * bursts of calls to force_quiescent_state(), increasing the probability 1347 * of occurrence of some important types of race conditions. 1348 */ 1349 static int 1350 rcu_torture_fqs(void *arg) 1351 { 1352 unsigned long fqs_resume_time; 1353 int fqs_burst_remaining; 1354 int oldnice = task_nice(current); 1355 1356 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 1357 do { 1358 fqs_resume_time = jiffies + fqs_stutter * HZ; 1359 while (time_before(jiffies, fqs_resume_time) && 1360 !kthread_should_stop()) { 1361 schedule_timeout_interruptible(HZ / 20); 1362 } 1363 fqs_burst_remaining = fqs_duration; 1364 while (fqs_burst_remaining > 0 && 1365 !kthread_should_stop()) { 1366 cur_ops->fqs(); 1367 udelay(fqs_holdoff); 1368 fqs_burst_remaining -= fqs_holdoff; 1369 } 1370 if (stutter_wait("rcu_torture_fqs")) 1371 sched_set_normal(current, oldnice); 1372 } while (!torture_must_stop()); 1373 torture_kthread_stopping("rcu_torture_fqs"); 1374 return 0; 1375 } 1376 1377 // Used by writers to randomly choose from the available grace-period primitives. 1378 static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; 1379 static int nsynctypes; 1380 1381 /* 1382 * Determine which grace-period primitives are available. 1383 */ 1384 static void rcu_torture_write_types(void) 1385 { 1386 bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; 1387 bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; 1388 bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; 1389 bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; 1390 1391 /* Initialize synctype[] array. If none set, take default. */ 1392 if (!gp_cond1 && 1393 !gp_cond_exp1 && 1394 !gp_cond_full1 && 1395 !gp_cond_exp_full1 && 1396 !gp_exp1 && 1397 !gp_poll_exp1 && 1398 !gp_poll_exp_full1 && 1399 !gp_normal1 && 1400 !gp_poll1 && 1401 !gp_poll_full1 && 1402 !gp_sync1) { 1403 gp_cond1 = true; 1404 gp_cond_exp1 = true; 1405 gp_cond_full1 = true; 1406 gp_cond_exp_full1 = true; 1407 gp_exp1 = true; 1408 gp_poll_exp1 = true; 1409 gp_poll_exp_full1 = true; 1410 gp_normal1 = true; 1411 gp_poll1 = true; 1412 gp_poll_full1 = true; 1413 gp_sync1 = true; 1414 } 1415 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { 1416 synctype[nsynctypes++] = RTWS_COND_GET; 1417 pr_info("%s: Testing conditional GPs.\n", __func__); 1418 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { 1419 pr_alert("%s: gp_cond without primitives.\n", __func__); 1420 } 1421 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { 1422 synctype[nsynctypes++] = RTWS_COND_GET_EXP; 1423 pr_info("%s: Testing conditional expedited GPs.\n", __func__); 1424 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { 1425 pr_alert("%s: gp_cond_exp without primitives.\n", __func__); 1426 } 1427 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { 1428 synctype[nsynctypes++] = RTWS_COND_GET_FULL; 1429 pr_info("%s: Testing conditional full-state GPs.\n", __func__); 1430 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { 1431 pr_alert("%s: gp_cond_full without primitives.\n", __func__); 1432 } 1433 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { 1434 synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; 1435 pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); 1436 } else if (gp_cond_exp_full && 1437 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { 1438 pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); 1439 } 1440 if (gp_exp1 && cur_ops->exp_sync) { 1441 synctype[nsynctypes++] = RTWS_EXP_SYNC; 1442 pr_info("%s: Testing expedited GPs.\n", __func__); 1443 } else if (gp_exp && !cur_ops->exp_sync) { 1444 pr_alert("%s: gp_exp without primitives.\n", __func__); 1445 } 1446 if (gp_normal1 && cur_ops->deferred_free) { 1447 synctype[nsynctypes++] = RTWS_DEF_FREE; 1448 pr_info("%s: Testing asynchronous GPs.\n", __func__); 1449 } else if (gp_normal && !cur_ops->deferred_free) { 1450 pr_alert("%s: gp_normal without primitives.\n", __func__); 1451 } 1452 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state && 1453 cur_ops->start_gp_poll && cur_ops->poll_gp_state) { 1454 synctype[nsynctypes++] = RTWS_POLL_GET; 1455 pr_info("%s: Testing polling GPs.\n", __func__); 1456 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { 1457 pr_alert("%s: gp_poll without primitives.\n", __func__); 1458 } 1459 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full 1460 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { 1461 synctype[nsynctypes++] = RTWS_POLL_GET_FULL; 1462 pr_info("%s: Testing polling full-state GPs.\n", __func__); 1463 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { 1464 pr_alert("%s: gp_poll_full without primitives.\n", __func__); 1465 } 1466 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { 1467 synctype[nsynctypes++] = RTWS_POLL_GET_EXP; 1468 pr_info("%s: Testing polling expedited GPs.\n", __func__); 1469 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { 1470 pr_alert("%s: gp_poll_exp without primitives.\n", __func__); 1471 } 1472 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { 1473 synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; 1474 pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); 1475 } else if (gp_poll_exp_full && 1476 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { 1477 pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); 1478 } 1479 if (gp_sync1 && cur_ops->sync) { 1480 synctype[nsynctypes++] = RTWS_SYNC; 1481 pr_info("%s: Testing normal GPs.\n", __func__); 1482 } else if (gp_sync && !cur_ops->sync) { 1483 pr_alert("%s: gp_sync without primitives.\n", __func__); 1484 } 1485 pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes); 1486 pr_info("%s: gp_cond_wi %d gp_cond_wi_exp %d gp_poll_wi %d gp_poll_wi_exp %d\n", __func__, gp_cond_wi, gp_cond_wi_exp, gp_poll_wi, gp_poll_wi_exp); 1487 } 1488 1489 /* 1490 * Do the specified rcu_torture_writer() synchronous grace period, 1491 * while also testing out the polled APIs. Note well that the single-CPU 1492 * grace-period optimizations must be accounted for. 1493 */ 1494 static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) 1495 { 1496 unsigned long cookie; 1497 struct rcu_gp_oldstate cookie_full; 1498 bool dopoll; 1499 bool dopoll_full; 1500 unsigned long r = torture_random(trsp); 1501 1502 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); 1503 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); 1504 if (dopoll || dopoll_full) 1505 cpus_read_lock(); 1506 if (dopoll) 1507 cookie = cur_ops->get_gp_state(); 1508 if (dopoll_full) 1509 cur_ops->get_gp_state_full(&cookie_full); 1510 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) 1511 sync(); 1512 sync(); 1513 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), 1514 "%s: Cookie check 3 failed %pS() online %*pbl.", 1515 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1516 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), 1517 "%s: Cookie check 4 failed %pS() online %*pbl", 1518 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1519 if (dopoll || dopoll_full) 1520 cpus_read_unlock(); 1521 } 1522 1523 /* 1524 * RCU torture writer kthread. Repeatedly substitutes a new structure 1525 * for that pointed to by rcu_torture_current, freeing the old structure 1526 * after a series of grace periods (the "pipeline"). 1527 */ 1528 static int 1529 rcu_torture_writer(void *arg) 1530 { 1531 bool booting_still = false; 1532 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 1533 unsigned long cookie; 1534 struct rcu_gp_oldstate cookie_full; 1535 int expediting = 0; 1536 unsigned long gp_snap; 1537 unsigned long gp_snap1; 1538 struct rcu_gp_oldstate gp_snap_full; 1539 struct rcu_gp_oldstate gp_snap1_full; 1540 int i; 1541 int idx; 1542 unsigned long j; 1543 int oldnice = task_nice(current); 1544 struct rcu_gp_oldstate *rgo = NULL; 1545 int rgo_size = 0; 1546 struct rcu_torture *rp; 1547 struct rcu_torture *old_rp; 1548 static DEFINE_TORTURE_RANDOM(rand); 1549 unsigned long stallsdone = jiffies; 1550 bool stutter_waited; 1551 unsigned long *ulo = NULL; 1552 int ulo_size = 0; 1553 1554 // If a new stall test is added, this must be adjusted. 1555 if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu) 1556 stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * 1557 HZ * (stall_cpu_repeat + 1); 1558 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 1559 if (!can_expedite) 1560 pr_alert("%s" TORTURE_FLAG 1561 " GP expediting controlled from boot/sysfs for %s.\n", 1562 torture_type, cur_ops->name); 1563 if (WARN_ONCE(nsynctypes == 0, 1564 "%s: No update-side primitives.\n", __func__)) { 1565 /* 1566 * No updates primitives, so don't try updating. 1567 * The resulting test won't be testing much, hence the 1568 * above WARN_ONCE(). 1569 */ 1570 rcu_torture_writer_state = RTWS_STOPPING; 1571 torture_kthread_stopping("rcu_torture_writer"); 1572 return 0; 1573 } 1574 if (cur_ops->poll_active > 0) { 1575 ulo = kcalloc(cur_ops->poll_active, sizeof(*ulo), GFP_KERNEL); 1576 if (!WARN_ON(!ulo)) 1577 ulo_size = cur_ops->poll_active; 1578 } 1579 if (cur_ops->poll_active_full > 0) { 1580 rgo = kcalloc(cur_ops->poll_active_full, sizeof(*rgo), GFP_KERNEL); 1581 if (!WARN_ON(!rgo)) 1582 rgo_size = cur_ops->poll_active_full; 1583 } 1584 1585 // If the system is still booting, let it finish. 1586 j = jiffies; 1587 while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) { 1588 booting_still = true; 1589 schedule_timeout_interruptible(HZ); 1590 } 1591 if (booting_still) 1592 pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", 1593 torture_type, jiffies - j); 1594 1595 do { 1596 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1597 torture_hrtimeout_us(500, 1000, &rand); 1598 rp = rcu_torture_alloc(); 1599 if (rp == NULL) 1600 continue; 1601 rp->rtort_pipe_count = 0; 1602 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 1603 rcu_torture_writer_state = RTWS_DELAY; 1604 udelay(torture_random(&rand) & 0x3ff); 1605 rcu_torture_writer_state = RTWS_REPLACE; 1606 old_rp = rcu_dereference_check(rcu_torture_current, 1607 current == writer_task); 1608 rp->rtort_mbtest = 1; 1609 rcu_assign_pointer(rcu_torture_current, rp); 1610 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1611 if (old_rp) { 1612 i = old_rp->rtort_pipe_count; 1613 if (i > RCU_TORTURE_PIPE_LEN) 1614 i = RCU_TORTURE_PIPE_LEN; 1615 atomic_inc(&rcu_torture_wcount[i]); 1616 WRITE_ONCE(old_rp->rtort_pipe_count, 1617 old_rp->rtort_pipe_count + 1); 1618 ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count); 1619 1620 // Make sure readers block polled grace periods. 1621 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { 1622 idx = cur_ops->readlock(); 1623 cookie = cur_ops->get_gp_state(); 1624 WARN_ONCE(cur_ops->poll_gp_state(cookie), 1625 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", 1626 __func__, 1627 rcu_torture_writer_state_getname(), 1628 rcu_torture_writer_state, 1629 cookie, cur_ops->get_gp_state()); 1630 if (cur_ops->get_comp_state) { 1631 cookie = cur_ops->get_comp_state(); 1632 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); 1633 } 1634 cur_ops->readunlock(idx); 1635 } 1636 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { 1637 idx = cur_ops->readlock(); 1638 cur_ops->get_gp_state_full(&cookie_full); 1639 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 1640 "%s: Cookie check 5 failed %s(%d) online %*pbl\n", 1641 __func__, 1642 rcu_torture_writer_state_getname(), 1643 rcu_torture_writer_state, 1644 cpumask_pr_args(cpu_online_mask)); 1645 if (cur_ops->get_comp_state_full) { 1646 cur_ops->get_comp_state_full(&cookie_full); 1647 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); 1648 } 1649 cur_ops->readunlock(idx); 1650 } 1651 switch (synctype[torture_random(&rand) % nsynctypes]) { 1652 case RTWS_DEF_FREE: 1653 rcu_torture_writer_state = RTWS_DEF_FREE; 1654 cur_ops->deferred_free(old_rp); 1655 break; 1656 case RTWS_EXP_SYNC: 1657 rcu_torture_writer_state = RTWS_EXP_SYNC; 1658 do_rtws_sync(&rand, cur_ops->exp_sync); 1659 rcu_torture_pipe_update(old_rp); 1660 break; 1661 case RTWS_COND_GET: 1662 rcu_torture_writer_state = RTWS_COND_GET; 1663 gp_snap = cur_ops->get_gp_state(); 1664 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1665 1000, &rand); 1666 rcu_torture_writer_state = RTWS_COND_SYNC; 1667 cur_ops->cond_sync(gp_snap); 1668 rcu_torture_pipe_update(old_rp); 1669 break; 1670 case RTWS_COND_GET_EXP: 1671 rcu_torture_writer_state = RTWS_COND_GET_EXP; 1672 gp_snap = cur_ops->get_gp_state_exp(); 1673 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1674 1000, &rand); 1675 rcu_torture_writer_state = RTWS_COND_SYNC_EXP; 1676 cur_ops->cond_sync_exp(gp_snap); 1677 rcu_torture_pipe_update(old_rp); 1678 break; 1679 case RTWS_COND_GET_FULL: 1680 rcu_torture_writer_state = RTWS_COND_GET_FULL; 1681 cur_ops->get_gp_state_full(&gp_snap_full); 1682 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1683 1000, &rand); 1684 rcu_torture_writer_state = RTWS_COND_SYNC_FULL; 1685 cur_ops->cond_sync_full(&gp_snap_full); 1686 rcu_torture_pipe_update(old_rp); 1687 break; 1688 case RTWS_COND_GET_EXP_FULL: 1689 rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; 1690 cur_ops->get_gp_state_full(&gp_snap_full); 1691 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1692 1000, &rand); 1693 rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; 1694 cur_ops->cond_sync_exp_full(&gp_snap_full); 1695 rcu_torture_pipe_update(old_rp); 1696 break; 1697 case RTWS_POLL_GET: 1698 rcu_torture_writer_state = RTWS_POLL_GET; 1699 for (i = 0; i < ulo_size; i++) 1700 ulo[i] = cur_ops->get_comp_state(); 1701 gp_snap = cur_ops->start_gp_poll(); 1702 rcu_torture_writer_state = RTWS_POLL_WAIT; 1703 while (!cur_ops->poll_gp_state(gp_snap)) { 1704 gp_snap1 = cur_ops->get_gp_state(); 1705 for (i = 0; i < ulo_size; i++) 1706 if (cur_ops->poll_gp_state(ulo[i]) || 1707 cur_ops->same_gp_state(ulo[i], gp_snap1)) { 1708 ulo[i] = gp_snap1; 1709 break; 1710 } 1711 WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size); 1712 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1713 1000, &rand); 1714 } 1715 rcu_torture_pipe_update(old_rp); 1716 break; 1717 case RTWS_POLL_GET_FULL: 1718 rcu_torture_writer_state = RTWS_POLL_GET_FULL; 1719 for (i = 0; i < rgo_size; i++) 1720 cur_ops->get_comp_state_full(&rgo[i]); 1721 cur_ops->start_gp_poll_full(&gp_snap_full); 1722 rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; 1723 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1724 cur_ops->get_gp_state_full(&gp_snap1_full); 1725 for (i = 0; i < rgo_size; i++) 1726 if (cur_ops->poll_gp_state_full(&rgo[i]) || 1727 cur_ops->same_gp_state_full(&rgo[i], 1728 &gp_snap1_full)) { 1729 rgo[i] = gp_snap1_full; 1730 break; 1731 } 1732 WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size); 1733 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1734 1000, &rand); 1735 } 1736 rcu_torture_pipe_update(old_rp); 1737 break; 1738 case RTWS_POLL_GET_EXP: 1739 rcu_torture_writer_state = RTWS_POLL_GET_EXP; 1740 gp_snap = cur_ops->start_gp_poll_exp(); 1741 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; 1742 while (!cur_ops->poll_gp_state_exp(gp_snap)) 1743 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1744 1000, &rand); 1745 rcu_torture_pipe_update(old_rp); 1746 break; 1747 case RTWS_POLL_GET_EXP_FULL: 1748 rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; 1749 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1750 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; 1751 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) 1752 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1753 1000, &rand); 1754 rcu_torture_pipe_update(old_rp); 1755 break; 1756 case RTWS_SYNC: 1757 rcu_torture_writer_state = RTWS_SYNC; 1758 do_rtws_sync(&rand, cur_ops->sync); 1759 rcu_torture_pipe_update(old_rp); 1760 break; 1761 default: 1762 WARN_ON_ONCE(1); 1763 break; 1764 } 1765 } 1766 WRITE_ONCE(rcu_torture_current_version, 1767 rcu_torture_current_version + 1); 1768 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1769 if (can_expedite && 1770 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1771 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1772 if (expediting >= 0) 1773 rcu_expedite_gp(); 1774 else 1775 rcu_unexpedite_gp(); 1776 if (++expediting > 3) 1777 expediting = -expediting; 1778 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1779 can_expedite = !rcu_gp_is_expedited() && 1780 !rcu_gp_is_normal(); 1781 } 1782 rcu_torture_writer_state = RTWS_STUTTER; 1783 stutter_waited = stutter_wait("rcu_torture_writer"); 1784 if (stutter_waited && 1785 !atomic_read(&rcu_fwd_cb_nodelay) && 1786 !cur_ops->slow_gps && 1787 !torture_must_stop() && 1788 time_after(jiffies, stallsdone)) 1789 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1790 if (list_empty(&rcu_tortures[i].rtort_free) && 1791 rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { 1792 tracing_off(); 1793 if (cur_ops->gp_kthread_dbg) 1794 cur_ops->gp_kthread_dbg(); 1795 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1796 rcu_ftrace_dump(DUMP_ALL); 1797 break; 1798 } 1799 if (stutter_waited) 1800 sched_set_normal(current, oldnice); 1801 } while (!torture_must_stop()); 1802 rcu_torture_current = NULL; // Let stats task know that we are done. 1803 /* Reset expediting back to unexpedited. */ 1804 if (expediting > 0) 1805 expediting = -expediting; 1806 while (can_expedite && expediting++ < 0) 1807 rcu_unexpedite_gp(); 1808 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1809 if (!can_expedite) 1810 pr_alert("%s" TORTURE_FLAG 1811 " Dynamic grace-period expediting was disabled.\n", 1812 torture_type); 1813 kfree(ulo); 1814 kfree(rgo); 1815 rcu_torture_writer_state = RTWS_STOPPING; 1816 torture_kthread_stopping("rcu_torture_writer"); 1817 return 0; 1818 } 1819 1820 /* 1821 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1822 * delay between calls. 1823 */ 1824 static int 1825 rcu_torture_fakewriter(void *arg) 1826 { 1827 unsigned long gp_snap; 1828 struct rcu_gp_oldstate gp_snap_full; 1829 DEFINE_TORTURE_RANDOM(rand); 1830 1831 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1832 set_user_nice(current, MAX_NICE); 1833 1834 if (WARN_ONCE(nsynctypes == 0, 1835 "%s: No update-side primitives.\n", __func__)) { 1836 /* 1837 * No updates primitives, so don't try updating. 1838 * The resulting test won't be testing much, hence the 1839 * above WARN_ONCE(). 1840 */ 1841 torture_kthread_stopping("rcu_torture_fakewriter"); 1842 return 0; 1843 } 1844 1845 do { 1846 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); 1847 if (cur_ops->cb_barrier != NULL && 1848 torture_random(&rand) % (nrealfakewriters * 8) == 0) { 1849 cur_ops->cb_barrier(); 1850 } else { 1851 switch (synctype[torture_random(&rand) % nsynctypes]) { 1852 case RTWS_DEF_FREE: 1853 break; 1854 case RTWS_EXP_SYNC: 1855 cur_ops->exp_sync(); 1856 break; 1857 case RTWS_COND_GET: 1858 gp_snap = cur_ops->get_gp_state(); 1859 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1860 cur_ops->cond_sync(gp_snap); 1861 break; 1862 case RTWS_COND_GET_EXP: 1863 gp_snap = cur_ops->get_gp_state_exp(); 1864 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1865 cur_ops->cond_sync_exp(gp_snap); 1866 break; 1867 case RTWS_COND_GET_FULL: 1868 cur_ops->get_gp_state_full(&gp_snap_full); 1869 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1870 cur_ops->cond_sync_full(&gp_snap_full); 1871 break; 1872 case RTWS_COND_GET_EXP_FULL: 1873 cur_ops->get_gp_state_full(&gp_snap_full); 1874 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1875 cur_ops->cond_sync_exp_full(&gp_snap_full); 1876 break; 1877 case RTWS_POLL_GET: 1878 if (cur_ops->start_poll_irqsoff) 1879 local_irq_disable(); 1880 gp_snap = cur_ops->start_gp_poll(); 1881 if (cur_ops->start_poll_irqsoff) 1882 local_irq_enable(); 1883 while (!cur_ops->poll_gp_state(gp_snap)) { 1884 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1885 &rand); 1886 } 1887 break; 1888 case RTWS_POLL_GET_FULL: 1889 if (cur_ops->start_poll_irqsoff) 1890 local_irq_disable(); 1891 cur_ops->start_gp_poll_full(&gp_snap_full); 1892 if (cur_ops->start_poll_irqsoff) 1893 local_irq_enable(); 1894 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1895 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1896 &rand); 1897 } 1898 break; 1899 case RTWS_POLL_GET_EXP: 1900 gp_snap = cur_ops->start_gp_poll_exp(); 1901 while (!cur_ops->poll_gp_state_exp(gp_snap)) { 1902 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1903 &rand); 1904 } 1905 break; 1906 case RTWS_POLL_GET_EXP_FULL: 1907 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1908 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1909 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1910 &rand); 1911 } 1912 break; 1913 case RTWS_SYNC: 1914 cur_ops->sync(); 1915 break; 1916 default: 1917 WARN_ON_ONCE(1); 1918 break; 1919 } 1920 } 1921 stutter_wait("rcu_torture_fakewriter"); 1922 } while (!torture_must_stop()); 1923 1924 torture_kthread_stopping("rcu_torture_fakewriter"); 1925 return 0; 1926 } 1927 1928 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1929 { 1930 kfree(rhp); 1931 } 1932 1933 // Set up and carry out testing of RCU's global memory ordering 1934 static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, 1935 struct torture_random_state *trsp) 1936 { 1937 unsigned long loops; 1938 int noc = torture_num_online_cpus(); 1939 int rdrchked; 1940 int rdrchker; 1941 struct rcu_torture_reader_check *rtrcp; // Me. 1942 struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. 1943 struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. 1944 struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. 1945 1946 if (myid < 0) 1947 return; // Don't try this from timer handlers. 1948 1949 // Increment my counter. 1950 rtrcp = &rcu_torture_reader_mbchk[myid]; 1951 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); 1952 1953 // Attempt to assign someone else some checking work. 1954 rdrchked = torture_random(trsp) % nrealreaders; 1955 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1956 rdrchker = torture_random(trsp) % nrealreaders; 1957 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; 1958 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && 1959 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. 1960 !READ_ONCE(rtp->rtort_chkp) && 1961 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. 1962 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); 1963 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); 1964 rtrcp->rtc_chkrdr = rdrchked; 1965 WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. 1966 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || 1967 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) 1968 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. 1969 } 1970 1971 // If assigned some completed work, do it! 1972 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); 1973 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) 1974 return; // No work or work not yet ready. 1975 rdrchked = rtrcp_assigner->rtc_chkrdr; 1976 if (WARN_ON_ONCE(rdrchked < 0)) 1977 return; 1978 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1979 loops = READ_ONCE(rtrcp_chked->rtc_myloops); 1980 atomic_inc(&n_rcu_torture_mbchk_tries); 1981 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) 1982 atomic_inc(&n_rcu_torture_mbchk_fail); 1983 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; 1984 rtrcp_assigner->rtc_ready = 0; 1985 smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. 1986 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. 1987 } 1988 1989 // Verify the specified RCUTORTURE_RDR* state. 1990 #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() 1991 static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) 1992 { 1993 int mask; 1994 1995 if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) 1996 return; 1997 1998 WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS); 1999 WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); 2000 2001 // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. 2002 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 2003 return; 2004 2005 WARN_ONCE((curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 2006 !softirq_count(), ROEC_ARGS); 2007 WARN_ONCE((curstate & (RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED)) && 2008 !(preempt_count() & PREEMPT_MASK), ROEC_ARGS); 2009 WARN_ONCE(cur_ops->readlock_nesting && 2010 (curstate & (RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2)) && 2011 cur_ops->readlock_nesting() == 0, ROEC_ARGS); 2012 2013 // Interrupt handlers have all sorts of stuff disabled, so ignore 2014 // unintended disabling. 2015 if (in_serving_softirq() || in_hardirq()) 2016 return; 2017 2018 WARN_ONCE(cur_ops->extendables && 2019 !(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 2020 softirq_count(), ROEC_ARGS); 2021 2022 /* 2023 * non-preemptible RCU in a preemptible kernel uses preempt_disable() 2024 * as rcu_read_lock(). 2025 */ 2026 mask = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2027 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 2028 mask |= RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2029 2030 WARN_ONCE(cur_ops->extendables && !(curstate & mask) && 2031 (preempt_count() & PREEMPT_MASK), ROEC_ARGS); 2032 2033 /* 2034 * non-preemptible RCU in a preemptible kernel uses "preempt_count() & 2035 * PREEMPT_MASK" as ->readlock_nesting(). 2036 */ 2037 mask = RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2038 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 2039 mask |= RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2040 2041 if (IS_ENABLED(CONFIG_PREEMPT_RT) && softirq_count()) 2042 mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 2043 2044 WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) && 2045 cur_ops->readlock_nesting() > 0, ROEC_ARGS); 2046 } 2047 2048 /* 2049 * Do one extension of an RCU read-side critical section using the 2050 * current reader state in readstate (set to zero for initial entry 2051 * to extended critical section), set the new state as specified by 2052 * newstate (set to zero for final exit from extended critical section), 2053 * and random-number-generator state in trsp. If this is neither the 2054 * beginning or end of the critical section and if there was actually a 2055 * change, do a ->read_delay(). 2056 */ 2057 static void rcutorture_one_extend(int *readstate, int newstate, struct torture_random_state *trsp, 2058 struct rt_read_seg *rtrsp) 2059 { 2060 bool first; 2061 unsigned long flags; 2062 int idxnew1 = -1; 2063 int idxnew2 = -1; 2064 int idxold1 = *readstate; 2065 int idxold2 = idxold1; 2066 int statesnew = ~*readstate & newstate; 2067 int statesold = *readstate & ~newstate; 2068 2069 first = idxold1 == 0; 2070 WARN_ON_ONCE(idxold2 < 0); 2071 WARN_ON_ONCE(idxold2 & ~(RCUTORTURE_RDR_ALLBITS | RCUTORTURE_RDR_UPDOWN)); 2072 rcutorture_one_extend_check("before change", idxold1, statesnew, statesold); 2073 rtrsp->rt_readstate = newstate; 2074 2075 /* First, put new protection in place to avoid critical-section gap. */ 2076 if (statesnew & RCUTORTURE_RDR_BH) 2077 local_bh_disable(); 2078 if (statesnew & RCUTORTURE_RDR_RBH) 2079 rcu_read_lock_bh(); 2080 if (statesnew & RCUTORTURE_RDR_IRQ) 2081 local_irq_disable(); 2082 if (statesnew & RCUTORTURE_RDR_PREEMPT) 2083 preempt_disable(); 2084 if (statesnew & RCUTORTURE_RDR_SCHED) 2085 rcu_read_lock_sched(); 2086 if (statesnew & RCUTORTURE_RDR_RCU_1) 2087 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 2088 if (statesnew & RCUTORTURE_RDR_RCU_2) 2089 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2; 2090 2091 // Complain unless both the old and the new protection is in place. 2092 rcutorture_one_extend_check("during change", idxold1 | statesnew, statesnew, statesold); 2093 2094 // Sample CPU under both sets of protections to reduce confusion. 2095 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 2096 int cpu = raw_smp_processor_id(); 2097 rtrsp->rt_cpu = cpu; 2098 if (!first) { 2099 rtrsp[-1].rt_end_cpu = cpu; 2100 if (cur_ops->reader_blocked) 2101 rtrsp[-1].rt_preempted = cur_ops->reader_blocked(); 2102 } 2103 } 2104 // Sample grace-period sequence number, as good a place as any. 2105 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) { 2106 rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs(); 2107 rtrsp->rt_ts = ktime_get_mono_fast_ns(); 2108 if (!first) 2109 rtrsp[-1].rt_gp_seq_end = rtrsp->rt_gp_seq; 2110 } 2111 2112 /* 2113 * Next, remove old protection, in decreasing order of strength 2114 * to avoid unlock paths that aren't safe in the stronger 2115 * context. Namely: BH can not be enabled with disabled interrupts. 2116 * Additionally PREEMPT_RT requires that BH is enabled in preemptible 2117 * context. 2118 */ 2119 if (statesold & RCUTORTURE_RDR_IRQ) 2120 local_irq_enable(); 2121 if (statesold & RCUTORTURE_RDR_PREEMPT) 2122 preempt_enable(); 2123 if (statesold & RCUTORTURE_RDR_SCHED) 2124 rcu_read_unlock_sched(); 2125 if (statesold & RCUTORTURE_RDR_BH) 2126 local_bh_enable(); 2127 if (statesold & RCUTORTURE_RDR_RBH) 2128 rcu_read_unlock_bh(); 2129 if (statesold & RCUTORTURE_RDR_RCU_2) { 2130 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2); 2131 WARN_ON_ONCE(idxnew2 != -1); 2132 idxold2 = 0; 2133 } 2134 if (statesold & RCUTORTURE_RDR_RCU_1) { 2135 bool lockit; 2136 2137 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); 2138 if (lockit) 2139 raw_spin_lock_irqsave(¤t->pi_lock, flags); 2140 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 2141 WARN_ON_ONCE(idxnew1 != -1); 2142 idxold1 = 0; 2143 if (lockit) 2144 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 2145 } 2146 if (statesold & RCUTORTURE_RDR_UPDOWN) { 2147 cur_ops->up_read((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 2148 WARN_ON_ONCE(idxnew1 != -1); 2149 idxold1 = 0; 2150 } 2151 2152 /* Delay if neither beginning nor end and there was a change. */ 2153 if ((statesnew || statesold) && *readstate && newstate) 2154 cur_ops->read_delay(trsp, rtrsp); 2155 2156 /* Update the reader state. */ 2157 if (idxnew1 == -1) 2158 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; 2159 WARN_ON_ONCE(idxnew1 < 0); 2160 if (idxnew2 == -1) 2161 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; 2162 WARN_ON_ONCE(idxnew2 < 0); 2163 *readstate = idxnew1 | idxnew2 | newstate; 2164 WARN_ON_ONCE(*readstate < 0); 2165 if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS)) 2166 pr_info("Unexpected readstate value of %#x\n", *readstate); 2167 rcutorture_one_extend_check("after change", *readstate, statesnew, statesold); 2168 } 2169 2170 /* Return the biggest extendables mask given current RCU and boot parameters. */ 2171 static int rcutorture_extend_mask_max(void) 2172 { 2173 int mask; 2174 2175 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 2176 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 2177 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2178 return mask; 2179 } 2180 2181 /* Return a random protection state mask, but with at least one bit set. */ 2182 static int 2183 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 2184 { 2185 int mask = rcutorture_extend_mask_max(); 2186 unsigned long randmask1 = torture_random(trsp); 2187 unsigned long randmask2 = randmask1 >> 3; 2188 unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2189 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 2190 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 2191 2192 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits. 2193 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 2194 if (!(randmask1 & 0x7)) 2195 mask = mask & randmask2; 2196 else 2197 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 2198 2199 // Can't have nested RCU reader without outer RCU reader. 2200 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { 2201 if (oldmask & RCUTORTURE_RDR_RCU_1) 2202 mask &= ~RCUTORTURE_RDR_RCU_2; 2203 else 2204 mask |= RCUTORTURE_RDR_RCU_1; 2205 } 2206 2207 /* 2208 * Can't enable bh w/irq disabled. 2209 */ 2210 if (mask & RCUTORTURE_RDR_IRQ) 2211 mask |= oldmask & bhs; 2212 2213 /* 2214 * Ideally these sequences would be detected in debug builds 2215 * (regardless of RT), but until then don't stop testing 2216 * them on non-RT. 2217 */ 2218 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 2219 /* Can't modify BH in atomic context */ 2220 if (oldmask & preempts_irq) 2221 mask &= ~bhs; 2222 if ((oldmask | mask) & preempts_irq) 2223 mask |= oldmask & bhs; 2224 } 2225 2226 return mask ?: RCUTORTURE_RDR_RCU_1; 2227 } 2228 2229 /* 2230 * Do a randomly selected number of extensions of an existing RCU read-side 2231 * critical section. 2232 */ 2233 static struct rt_read_seg * 2234 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct rt_read_seg *rtrsp) 2235 { 2236 int i; 2237 int j; 2238 int mask = rcutorture_extend_mask_max(); 2239 2240 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 2241 if (!((mask - 1) & mask)) 2242 return rtrsp; /* Current RCU reader not extendable. */ 2243 /* Bias towards larger numbers of loops. */ 2244 i = torture_random(trsp); 2245 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 2246 for (j = 0; j < i; j++) { 2247 mask = rcutorture_extend_mask(*readstate, trsp); 2248 WARN_ON_ONCE(mask & RCUTORTURE_RDR_UPDOWN); 2249 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 2250 } 2251 return &rtrsp[j]; 2252 } 2253 2254 struct rcu_torture_one_read_state { 2255 bool checkpolling; 2256 unsigned long cookie; 2257 struct rcu_gp_oldstate cookie_full; 2258 unsigned long started; 2259 struct rcu_torture *p; 2260 int readstate; 2261 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS]; 2262 struct rt_read_seg *rtrsp; 2263 unsigned long long ts; 2264 }; 2265 2266 static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, 2267 struct torture_random_state *trsp) 2268 { 2269 memset(rtorsp, 0, sizeof(*rtorsp)); 2270 rtorsp->checkpolling = !(torture_random(trsp) & 0xfff); 2271 rtorsp->rtrsp = &rtorsp->rtseg[0]; 2272 } 2273 2274 /* 2275 * Set up the first segment of a series of overlapping read-side 2276 * critical sections. The caller must have actually initiated the 2277 * outermost read-side critical section. 2278 */ 2279 static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp, 2280 struct torture_random_state *trsp, long myid) 2281 { 2282 if (rtorsp->checkpolling) { 2283 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2284 rtorsp->cookie = cur_ops->get_gp_state(); 2285 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2286 cur_ops->get_gp_state_full(&rtorsp->cookie_full); 2287 } 2288 rtorsp->started = cur_ops->get_gp_seq(); 2289 rtorsp->ts = rcu_trace_clock_local(); 2290 rtorsp->p = rcu_dereference_check(rcu_torture_current, 2291 !cur_ops->readlock_held || cur_ops->readlock_held() || 2292 (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN)); 2293 if (rtorsp->p == NULL) { 2294 /* Wait for rcu_torture_writer to get underway */ 2295 rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); 2296 return false; 2297 } 2298 if (rtorsp->p->rtort_mbtest == 0) 2299 atomic_inc(&n_rcu_torture_mberror); 2300 rcu_torture_reader_do_mbchk(myid, rtorsp->p, trsp); 2301 return true; 2302 } 2303 2304 /* 2305 * Complete the last segment of a series of overlapping read-side 2306 * critical sections and check for errors. 2307 */ 2308 static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, 2309 struct torture_random_state *trsp) 2310 { 2311 int i; 2312 unsigned long completed; 2313 int pipe_count; 2314 bool preempted = false; 2315 struct rt_read_seg *rtrsp1; 2316 2317 preempt_disable(); 2318 pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count); 2319 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 2320 // Should not happen in a correct RCU implementation, 2321 // happens quite often for torture_type=busted. 2322 pipe_count = RCU_TORTURE_PIPE_LEN; 2323 } 2324 completed = cur_ops->get_gp_seq(); 2325 if (pipe_count > 1) { 2326 do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu, 2327 rtorsp->ts, rtorsp->started, completed); 2328 rcu_ftrace_dump(DUMP_ALL); 2329 } 2330 __this_cpu_inc(rcu_torture_count[pipe_count]); 2331 completed = rcutorture_seq_diff(completed, rtorsp->started); 2332 if (completed > RCU_TORTURE_PIPE_LEN) { 2333 /* Should not happen, but... */ 2334 completed = RCU_TORTURE_PIPE_LEN; 2335 } 2336 __this_cpu_inc(rcu_torture_batch[completed]); 2337 preempt_enable(); 2338 if (rtorsp->checkpolling) { 2339 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2340 WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie), 2341 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", 2342 __func__, 2343 rcu_torture_writer_state_getname(), 2344 rcu_torture_writer_state, 2345 rtorsp->cookie, cur_ops->get_gp_state()); 2346 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2347 WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full), 2348 "%s: Cookie check 6 failed %s(%d) online %*pbl\n", 2349 __func__, 2350 rcu_torture_writer_state_getname(), 2351 rcu_torture_writer_state, 2352 cpumask_pr_args(cpu_online_mask)); 2353 } 2354 if (cur_ops->reader_blocked) 2355 preempted = cur_ops->reader_blocked(); 2356 rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); 2357 WARN_ON_ONCE(rtorsp->readstate); 2358 // This next splat is expected behavior if leakpointer, especially 2359 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. 2360 WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1); 2361 2362 /* If error or close call, record the sequence of reader protections. */ 2363 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 2364 i = 0; 2365 for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++) 2366 err_segs[i++] = *rtrsp1; 2367 rt_read_nsegs = i; 2368 rt_read_preempted = preempted; 2369 } 2370 } 2371 2372 /* 2373 * Do one read-side critical section, returning false if there was 2374 * no data to read. Can be invoked both from process context and 2375 * from a timer handler. 2376 */ 2377 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) 2378 { 2379 int newstate; 2380 struct rcu_torture_one_read_state rtors; 2381 2382 WARN_ON_ONCE(!rcu_is_watching()); 2383 init_rcu_torture_one_read_state(&rtors, trsp); 2384 newstate = rcutorture_extend_mask(rtors.readstate, trsp); 2385 WARN_ON_ONCE(newstate & RCUTORTURE_RDR_UPDOWN); 2386 rcutorture_one_extend(&rtors.readstate, newstate, trsp, rtors.rtrsp++); 2387 if (!rcu_torture_one_read_start(&rtors, trsp, myid)) { 2388 rcutorture_one_extend(&rtors.readstate, 0, trsp, rtors.rtrsp); 2389 return false; 2390 } 2391 rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); 2392 rcu_torture_one_read_end(&rtors, trsp); 2393 return true; 2394 } 2395 2396 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 2397 2398 /* 2399 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2400 * incrementing the corresponding element of the pipeline array. The 2401 * counter in the element should never be greater than 1, otherwise, the 2402 * RCU implementation is broken. 2403 */ 2404 static void rcu_torture_timer(struct timer_list *unused) 2405 { 2406 atomic_long_inc(&n_rcu_torture_timers); 2407 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); 2408 2409 /* Test call_rcu() invocation from interrupt handler. */ 2410 if (cur_ops->call) { 2411 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 2412 2413 if (rhp) 2414 cur_ops->call(rhp, rcu_torture_timer_cb); 2415 } 2416 } 2417 2418 /* 2419 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 2420 * incrementing the corresponding element of the pipeline array. The 2421 * counter in the element should never be greater than 1, otherwise, the 2422 * RCU implementation is broken. 2423 */ 2424 static int 2425 rcu_torture_reader(void *arg) 2426 { 2427 unsigned long lastsleep = jiffies; 2428 long myid = (long)arg; 2429 int mynumonline = myid; 2430 DEFINE_TORTURE_RANDOM(rand); 2431 struct timer_list t; 2432 2433 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 2434 set_user_nice(current, MAX_NICE); 2435 if (irqreader && cur_ops->irq_capable) 2436 timer_setup_on_stack(&t, rcu_torture_timer, 0); 2437 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 2438 do { 2439 if (irqreader && cur_ops->irq_capable) { 2440 if (!timer_pending(&t)) 2441 mod_timer(&t, jiffies + 1); 2442 } 2443 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) 2444 schedule_timeout_interruptible(HZ); 2445 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 2446 torture_hrtimeout_us(500, 1000, &rand); 2447 lastsleep = jiffies + 10; 2448 } 2449 while (!torture_must_stop() && 2450 (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) 2451 schedule_timeout_interruptible(HZ / 5); 2452 stutter_wait("rcu_torture_reader"); 2453 } while (!torture_must_stop()); 2454 if (irqreader && cur_ops->irq_capable) { 2455 timer_delete_sync(&t); 2456 timer_destroy_on_stack(&t); 2457 } 2458 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2459 torture_kthread_stopping("rcu_torture_reader"); 2460 return 0; 2461 } 2462 2463 struct rcu_torture_one_read_state_updown { 2464 struct hrtimer rtorsu_hrt; 2465 bool rtorsu_inuse; 2466 ktime_t rtorsu_kt; 2467 int rtorsu_cpu; 2468 unsigned long rtorsu_j; 2469 unsigned long rtorsu_ndowns; 2470 unsigned long rtorsu_nups; 2471 unsigned long rtorsu_nmigrates; 2472 struct torture_random_state rtorsu_trs; 2473 struct rcu_torture_one_read_state rtorsu_rtors; 2474 }; 2475 2476 static struct rcu_torture_one_read_state_updown *updownreaders; 2477 static DEFINE_TORTURE_RANDOM(rcu_torture_updown_rand); 2478 static int rcu_torture_updown(void *arg); 2479 2480 static enum hrtimer_restart rcu_torture_updown_hrt(struct hrtimer *hrtp) 2481 { 2482 int cpu = raw_smp_processor_id(); 2483 struct rcu_torture_one_read_state_updown *rtorsup; 2484 2485 rtorsup = container_of(hrtp, struct rcu_torture_one_read_state_updown, rtorsu_hrt); 2486 rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 2487 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 2488 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 2489 WRITE_ONCE(rtorsup->rtorsu_nmigrates, 2490 rtorsup->rtorsu_nmigrates + (cpu != rtorsup->rtorsu_cpu)); 2491 smp_store_release(&rtorsup->rtorsu_inuse, false); 2492 return HRTIMER_NORESTART; 2493 } 2494 2495 static int rcu_torture_updown_init(void) 2496 { 2497 int i; 2498 struct torture_random_state *rand = &rcu_torture_updown_rand; 2499 int ret; 2500 2501 if (n_up_down < 0) 2502 return 0; 2503 if (!srcu_torture_have_up_down()) { 2504 VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Disabling up/down reader tests due to lack of primitives"); 2505 return 0; 2506 } 2507 updownreaders = kcalloc(n_up_down, sizeof(*updownreaders), GFP_KERNEL); 2508 if (!updownreaders) { 2509 VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Out of memory, disabling up/down reader tests"); 2510 return -ENOMEM; 2511 } 2512 for (i = 0; i < n_up_down; i++) { 2513 init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors, rand); 2514 hrtimer_setup(&updownreaders[i].rtorsu_hrt, rcu_torture_updown_hrt, CLOCK_MONOTONIC, 2515 HRTIMER_MODE_REL | HRTIMER_MODE_HARD); 2516 torture_random_init(&updownreaders[i].rtorsu_trs); 2517 init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors, 2518 &updownreaders[i].rtorsu_trs); 2519 } 2520 ret = torture_create_kthread(rcu_torture_updown, rand, updown_task); 2521 if (ret) { 2522 kfree(updownreaders); 2523 updownreaders = NULL; 2524 } 2525 return ret; 2526 } 2527 2528 static void rcu_torture_updown_cleanup(void) 2529 { 2530 struct rcu_torture_one_read_state_updown *rtorsup; 2531 2532 for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) { 2533 if (!smp_load_acquire(&rtorsup->rtorsu_inuse)) 2534 continue; 2535 if (hrtimer_cancel(&rtorsup->rtorsu_hrt) || WARN_ON_ONCE(rtorsup->rtorsu_inuse)) { 2536 rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 2537 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 2538 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 2539 smp_store_release(&rtorsup->rtorsu_inuse, false); 2540 } 2541 2542 } 2543 kfree(updownreaders); 2544 updownreaders = NULL; 2545 } 2546 2547 // Do one reader for rcu_torture_updown(). 2548 static void rcu_torture_updown_one(struct rcu_torture_one_read_state_updown *rtorsup) 2549 { 2550 int idx; 2551 int rawidx; 2552 ktime_t t; 2553 2554 init_rcu_torture_one_read_state(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 2555 rawidx = cur_ops->down_read(); 2556 WRITE_ONCE(rtorsup->rtorsu_ndowns, rtorsup->rtorsu_ndowns + 1); 2557 idx = (rawidx << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 2558 rtorsup->rtorsu_rtors.readstate = idx | RCUTORTURE_RDR_UPDOWN; 2559 rtorsup->rtorsu_rtors.rtrsp++; 2560 rtorsup->rtorsu_cpu = raw_smp_processor_id(); 2561 if (!rcu_torture_one_read_start(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs, -1)) { 2562 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 2563 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 2564 schedule_timeout_idle(HZ); 2565 return; 2566 } 2567 smp_store_release(&rtorsup->rtorsu_inuse, true); 2568 t = torture_random(&rtorsup->rtorsu_trs) & 0xfffff; // One per million. 2569 if (t < 10 * 1000) 2570 t = 200 * 1000 * 1000; 2571 hrtimer_start(&rtorsup->rtorsu_hrt, t, HRTIMER_MODE_REL | HRTIMER_MODE_HARD); 2572 smp_mb(); // Sample jiffies after posting hrtimer. 2573 rtorsup->rtorsu_j = jiffies; // Not used by hrtimer handler. 2574 rtorsup->rtorsu_kt = t; 2575 } 2576 2577 /* 2578 * RCU torture up/down reader kthread, starting RCU readers in kthread 2579 * context and ending them in hrtimer handlers. Otherwise similar to 2580 * rcu_torture_reader(). 2581 */ 2582 static int 2583 rcu_torture_updown(void *arg) 2584 { 2585 unsigned long j; 2586 struct rcu_torture_one_read_state_updown *rtorsup; 2587 2588 VERBOSE_TOROUT_STRING("rcu_torture_updown task started"); 2589 do { 2590 for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) { 2591 if (torture_must_stop()) 2592 break; 2593 j = smp_load_acquire(&jiffies); // Time before ->rtorsu_inuse. 2594 if (smp_load_acquire(&rtorsup->rtorsu_inuse)) { 2595 WARN_ONCE(time_after(j, rtorsup->rtorsu_j + 1 + HZ * 10), 2596 "hrtimer queued at jiffies %lu for %lld ns took %lu jiffies\n", rtorsup->rtorsu_j, rtorsup->rtorsu_kt, j - rtorsup->rtorsu_j); 2597 continue; 2598 } 2599 rcu_torture_updown_one(rtorsup); 2600 } 2601 torture_hrtimeout_ms(1, 1000, &rcu_torture_updown_rand); 2602 stutter_wait("rcu_torture_updown"); 2603 } while (!torture_must_stop()); 2604 rcu_torture_updown_cleanup(); 2605 torture_kthread_stopping("rcu_torture_updown"); 2606 return 0; 2607 } 2608 2609 /* 2610 * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to 2611 * increase race probabilities and fuzzes the interval between toggling. 2612 */ 2613 static int rcu_nocb_toggle(void *arg) 2614 { 2615 int cpu; 2616 int maxcpu = -1; 2617 int oldnice = task_nice(current); 2618 long r; 2619 DEFINE_TORTURE_RANDOM(rand); 2620 ktime_t toggle_delay; 2621 unsigned long toggle_fuzz; 2622 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); 2623 2624 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); 2625 while (!rcu_inkernel_boot_has_ended()) 2626 schedule_timeout_interruptible(HZ / 10); 2627 for_each_possible_cpu(cpu) 2628 maxcpu = cpu; 2629 WARN_ON(maxcpu < 0); 2630 if (toggle_interval > ULONG_MAX) 2631 toggle_fuzz = ULONG_MAX >> 3; 2632 else 2633 toggle_fuzz = toggle_interval >> 3; 2634 if (toggle_fuzz <= 0) 2635 toggle_fuzz = NSEC_PER_USEC; 2636 do { 2637 r = torture_random(&rand); 2638 cpu = (r >> 1) % (maxcpu + 1); 2639 if (r & 0x1) { 2640 rcu_nocb_cpu_offload(cpu); 2641 atomic_long_inc(&n_nocb_offload); 2642 } else { 2643 rcu_nocb_cpu_deoffload(cpu); 2644 atomic_long_inc(&n_nocb_deoffload); 2645 } 2646 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; 2647 set_current_state(TASK_INTERRUPTIBLE); 2648 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); 2649 if (stutter_wait("rcu_nocb_toggle")) 2650 sched_set_normal(current, oldnice); 2651 } while (!torture_must_stop()); 2652 torture_kthread_stopping("rcu_nocb_toggle"); 2653 return 0; 2654 } 2655 2656 /* 2657 * Print torture statistics. Caller must ensure that there is only 2658 * one call to this function at a given time!!! This is normally 2659 * accomplished by relying on the module system to only have one copy 2660 * of the module loaded, and then by giving the rcu_torture_stats 2661 * kthread full control (or the init/cleanup functions when rcu_torture_stats 2662 * thread is not running). 2663 */ 2664 static void 2665 rcu_torture_stats_print(void) 2666 { 2667 int cpu; 2668 int i; 2669 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2670 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2671 long n_gpwraps = 0; 2672 unsigned long ndowns = 0; 2673 unsigned long nunexpired = 0; 2674 unsigned long nmigrates = 0; 2675 unsigned long nups = 0; 2676 struct rcu_torture *rtcp; 2677 static unsigned long rtcv_snap = ULONG_MAX; 2678 static bool splatted; 2679 struct task_struct *wtp; 2680 2681 for_each_possible_cpu(cpu) { 2682 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2683 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 2684 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 2685 } 2686 if (cur_ops->get_gpwrap_count) 2687 n_gpwraps += cur_ops->get_gpwrap_count(cpu); 2688 } 2689 if (updownreaders) { 2690 for (i = 0; i < n_up_down; i++) { 2691 ndowns += READ_ONCE(updownreaders[i].rtorsu_ndowns); 2692 nups += READ_ONCE(updownreaders[i].rtorsu_nups); 2693 nunexpired += READ_ONCE(updownreaders[i].rtorsu_inuse); 2694 nmigrates += READ_ONCE(updownreaders[i].rtorsu_nmigrates); 2695 } 2696 } 2697 for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { 2698 if (pipesummary[i] != 0) 2699 break; 2700 } // The value of variable "i" is used later, so don't clobber it! 2701 2702 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2703 rtcp = rcu_access_pointer(rcu_torture_current); 2704 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 2705 rtcp, 2706 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 2707 rcu_torture_current_version, 2708 list_empty(&rcu_torture_freelist), 2709 atomic_read(&n_rcu_torture_alloc), 2710 atomic_read(&n_rcu_torture_alloc_fail), 2711 atomic_read(&n_rcu_torture_free)); 2712 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ", 2713 atomic_read(&n_rcu_torture_mberror), 2714 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), 2715 n_rcu_torture_barrier_error, 2716 n_rcu_torture_boost_ktrerror); 2717 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 2718 n_rcu_torture_boost_failure, 2719 n_rcu_torture_boosts, 2720 atomic_long_read(&n_rcu_torture_timers)); 2721 if (updownreaders) 2722 pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); 2723 torture_onoff_stats(); 2724 pr_cont("barrier: %ld/%ld:%ld ", 2725 data_race(n_barrier_successes), 2726 data_race(n_barrier_attempts), 2727 data_race(n_rcu_torture_barrier_error)); 2728 pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. 2729 pr_cont("nocb-toggles: %ld:%ld ", 2730 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); 2731 pr_cont("gpwraps: %ld\n", n_gpwraps); 2732 2733 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2734 if (atomic_read(&n_rcu_torture_mberror) || 2735 atomic_read(&n_rcu_torture_mbchk_fail) || 2736 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 2737 n_rcu_torture_boost_failure || i > 1) { 2738 pr_cont("%s", "!!! "); 2739 atomic_inc(&n_rcu_torture_error); 2740 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 2741 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); 2742 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 2743 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 2744 WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) 2745 WARN_ON_ONCE(i > 1); // Too-short grace period 2746 } 2747 pr_cont("Reader Pipe: "); 2748 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2749 pr_cont(" %ld", pipesummary[i]); 2750 pr_cont("\n"); 2751 2752 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2753 pr_cont("Reader Batch: "); 2754 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2755 pr_cont(" %ld", batchsummary[i]); 2756 pr_cont("\n"); 2757 2758 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2759 pr_cont("Free-Block Circulation: "); 2760 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2761 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 2762 } 2763 pr_cont("\n"); 2764 2765 if (cur_ops->stats) 2766 cur_ops->stats(); 2767 if (rtcv_snap == rcu_torture_current_version && 2768 rcu_access_pointer(rcu_torture_current) && 2769 !rcu_stall_is_suppressed() && 2770 rcu_inkernel_boot_has_ended()) { 2771 int __maybe_unused flags = 0; 2772 unsigned long __maybe_unused gp_seq = 0; 2773 2774 if (cur_ops->get_gp_data) 2775 cur_ops->get_gp_data(&flags, &gp_seq); 2776 wtp = READ_ONCE(writer_task); 2777 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", 2778 rcu_torture_writer_state_getname(), 2779 rcu_torture_writer_state, gp_seq, flags, 2780 wtp == NULL ? ~0U : wtp->__state, 2781 wtp == NULL ? -1 : (int)task_cpu(wtp)); 2782 if (!splatted && wtp) { 2783 sched_show_task(wtp); 2784 splatted = true; 2785 } 2786 if (cur_ops->gp_kthread_dbg) 2787 cur_ops->gp_kthread_dbg(); 2788 rcu_ftrace_dump(DUMP_ALL); 2789 } 2790 rtcv_snap = rcu_torture_current_version; 2791 } 2792 2793 /* 2794 * Periodically prints torture statistics, if periodic statistics printing 2795 * was specified via the stat_interval module parameter. 2796 */ 2797 static int 2798 rcu_torture_stats(void *arg) 2799 { 2800 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 2801 do { 2802 schedule_timeout_interruptible(stat_interval * HZ); 2803 rcu_torture_stats_print(); 2804 torture_shutdown_absorb("rcu_torture_stats"); 2805 } while (!torture_must_stop()); 2806 torture_kthread_stopping("rcu_torture_stats"); 2807 return 0; 2808 } 2809 2810 /* Test mem_dump_obj() and friends. */ 2811 static void rcu_torture_mem_dump_obj(void) 2812 { 2813 struct rcu_head *rhp; 2814 struct kmem_cache *kcp; 2815 static int z; 2816 2817 kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); 2818 if (WARN_ON_ONCE(!kcp)) 2819 return; 2820 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); 2821 if (WARN_ON_ONCE(!rhp)) { 2822 kmem_cache_destroy(kcp); 2823 return; 2824 } 2825 pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); 2826 pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); 2827 mem_dump_obj(ZERO_SIZE_PTR); 2828 pr_alert("mem_dump_obj(NULL):"); 2829 mem_dump_obj(NULL); 2830 pr_alert("mem_dump_obj(%px):", &rhp); 2831 mem_dump_obj(&rhp); 2832 pr_alert("mem_dump_obj(%px):", rhp); 2833 mem_dump_obj(rhp); 2834 pr_alert("mem_dump_obj(%px):", &rhp->func); 2835 mem_dump_obj(&rhp->func); 2836 pr_alert("mem_dump_obj(%px):", &z); 2837 mem_dump_obj(&z); 2838 kmem_cache_free(kcp, rhp); 2839 kmem_cache_destroy(kcp); 2840 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 2841 if (WARN_ON_ONCE(!rhp)) 2842 return; 2843 pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2844 pr_alert("mem_dump_obj(kmalloc %px):", rhp); 2845 mem_dump_obj(rhp); 2846 pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); 2847 mem_dump_obj(&rhp->func); 2848 kfree(rhp); 2849 rhp = vmalloc(4096); 2850 if (WARN_ON_ONCE(!rhp)) 2851 return; 2852 pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2853 pr_alert("mem_dump_obj(vmalloc %px):", rhp); 2854 mem_dump_obj(rhp); 2855 pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); 2856 mem_dump_obj(&rhp->func); 2857 vfree(rhp); 2858 } 2859 2860 static void 2861 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 2862 { 2863 pr_alert("%s" TORTURE_FLAG 2864 "--- %s: nreaders=%d nfakewriters=%d " 2865 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 2866 "shuffle_interval=%d stutter=%d irqreader=%d " 2867 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 2868 "test_boost=%d/%d test_boost_interval=%d " 2869 "test_boost_duration=%d test_boost_holdoff=%d shutdown_secs=%d " 2870 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 2871 "stall_cpu_block=%d stall_cpu_repeat=%d " 2872 "n_barrier_cbs=%d " 2873 "onoff_interval=%d onoff_holdoff=%d " 2874 "read_exit_delay=%d read_exit_burst=%d " 2875 "reader_flavor=%x " 2876 "nocbs_nthreads=%d nocbs_toggle=%d " 2877 "test_nmis=%d " 2878 "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", 2879 torture_type, tag, nrealreaders, nrealfakewriters, 2880 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 2881 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 2882 test_boost, cur_ops->can_boost, 2883 test_boost_interval, test_boost_duration, test_boost_holdoff, shutdown_secs, 2884 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 2885 stall_cpu_block, stall_cpu_repeat, 2886 n_barrier_cbs, 2887 onoff_interval, onoff_holdoff, 2888 read_exit_delay, read_exit_burst, 2889 reader_flavor, 2890 nocbs_nthreads, nocbs_toggle, 2891 test_nmis, 2892 preempt_duration, preempt_interval, n_up_down); 2893 } 2894 2895 static int rcutorture_booster_cleanup(unsigned int cpu) 2896 { 2897 struct task_struct *t; 2898 2899 if (boost_tasks[cpu] == NULL) 2900 return 0; 2901 mutex_lock(&boost_mutex); 2902 t = boost_tasks[cpu]; 2903 boost_tasks[cpu] = NULL; 2904 rcu_torture_enable_rt_throttle(); 2905 mutex_unlock(&boost_mutex); 2906 2907 /* This must be outside of the mutex, otherwise deadlock! */ 2908 torture_stop_kthread(rcu_torture_boost, t); 2909 return 0; 2910 } 2911 2912 static int rcutorture_booster_init(unsigned int cpu) 2913 { 2914 int retval; 2915 2916 if (boost_tasks[cpu] != NULL) 2917 return 0; /* Already created, nothing more to do. */ 2918 2919 // Testing RCU priority boosting requires rcutorture do 2920 // some serious abuse. Counter this by running ksoftirqd 2921 // at higher priority. 2922 if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { 2923 struct sched_param sp; 2924 struct task_struct *t; 2925 2926 t = per_cpu(ksoftirqd, cpu); 2927 WARN_ON_ONCE(!t); 2928 sp.sched_priority = 2; 2929 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2930 #ifdef CONFIG_IRQ_FORCED_THREADING 2931 if (force_irqthreads()) { 2932 t = per_cpu(ktimerd, cpu); 2933 WARN_ON_ONCE(!t); 2934 sp.sched_priority = 2; 2935 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2936 } 2937 #endif 2938 } 2939 2940 /* Don't allow time recalculation while creating a new task. */ 2941 mutex_lock(&boost_mutex); 2942 rcu_torture_disable_rt_throttle(); 2943 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 2944 boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, 2945 cpu, "rcu_torture_boost_%u"); 2946 if (IS_ERR(boost_tasks[cpu])) { 2947 retval = PTR_ERR(boost_tasks[cpu]); 2948 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 2949 n_rcu_torture_boost_ktrerror++; 2950 boost_tasks[cpu] = NULL; 2951 mutex_unlock(&boost_mutex); 2952 return retval; 2953 } 2954 mutex_unlock(&boost_mutex); 2955 return 0; 2956 } 2957 2958 static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) 2959 { 2960 pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr); 2961 return NOTIFY_OK; 2962 } 2963 2964 static struct notifier_block rcu_torture_stall_block = { 2965 .notifier_call = rcu_torture_stall_nf, 2966 }; 2967 2968 /* 2969 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 2970 * induces a CPU stall for the time specified by stall_cpu. If a new 2971 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. 2972 */ 2973 static void rcu_torture_stall_one(int rep, int irqsoff) 2974 { 2975 int idx; 2976 unsigned long stop_at; 2977 2978 if (stall_cpu_holdoff > 0) { 2979 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 2980 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 2981 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 2982 } 2983 if (!kthread_should_stop() && stall_gp_kthread > 0) { 2984 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); 2985 rcu_gp_set_torture_wait(stall_gp_kthread * HZ); 2986 for (idx = 0; idx < stall_gp_kthread + 2; idx++) { 2987 if (kthread_should_stop()) 2988 break; 2989 schedule_timeout_uninterruptible(HZ); 2990 } 2991 } 2992 if (!kthread_should_stop() && stall_cpu > 0) { 2993 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); 2994 stop_at = ktime_get_seconds() + stall_cpu; 2995 /* RCU CPU stall is expected behavior in following code. */ 2996 idx = cur_ops->readlock(); 2997 if (irqsoff) 2998 local_irq_disable(); 2999 else if (!stall_cpu_block) 3000 preempt_disable(); 3001 pr_alert("%s start stall episode %d on CPU %d.\n", 3002 __func__, rep + 1, raw_smp_processor_id()); 3003 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) && 3004 !kthread_should_stop()) 3005 if (stall_cpu_block) { 3006 #ifdef CONFIG_PREEMPTION 3007 preempt_schedule(); 3008 #else 3009 schedule_timeout_uninterruptible(HZ); 3010 #endif 3011 } else if (stall_no_softlockup) { 3012 touch_softlockup_watchdog(); 3013 } 3014 if (irqsoff) 3015 local_irq_enable(); 3016 else if (!stall_cpu_block) 3017 preempt_enable(); 3018 cur_ops->readunlock(idx); 3019 } 3020 } 3021 3022 /* 3023 * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many 3024 * additional times as specified by the stall_cpu_repeat module parameter. 3025 * Note that stall_cpu_irqsoff is ignored on the second and subsequent 3026 * stall. 3027 */ 3028 static int rcu_torture_stall(void *args) 3029 { 3030 int i; 3031 int repeat = stall_cpu_repeat; 3032 int ret; 3033 3034 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 3035 if (repeat < 0) { 3036 repeat = 0; 3037 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 3038 } 3039 if (rcu_cpu_stall_notifiers) { 3040 ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block); 3041 if (ret) 3042 pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n", 3043 __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : ""); 3044 } 3045 for (i = 0; i <= repeat; i++) { 3046 if (kthread_should_stop()) 3047 break; 3048 rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0); 3049 } 3050 pr_alert("%s end.\n", __func__); 3051 if (rcu_cpu_stall_notifiers && !ret) { 3052 ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block); 3053 if (ret) 3054 pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret); 3055 } 3056 torture_shutdown_absorb("rcu_torture_stall"); 3057 while (!kthread_should_stop()) 3058 schedule_timeout_interruptible(10 * HZ); 3059 return 0; 3060 } 3061 3062 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 3063 static int __init rcu_torture_stall_init(void) 3064 { 3065 if (stall_cpu <= 0 && stall_gp_kthread <= 0) 3066 return 0; 3067 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 3068 } 3069 3070 /* State structure for forward-progress self-propagating RCU callback. */ 3071 struct fwd_cb_state { 3072 struct rcu_head rh; 3073 int stop; 3074 }; 3075 3076 /* 3077 * Forward-progress self-propagating RCU callback function. Because 3078 * callbacks run from softirq, this function is an implicit RCU read-side 3079 * critical section. 3080 */ 3081 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 3082 { 3083 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 3084 3085 if (READ_ONCE(fcsp->stop)) { 3086 WRITE_ONCE(fcsp->stop, 2); 3087 return; 3088 } 3089 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 3090 } 3091 3092 /* State for continuous-flood RCU callbacks. */ 3093 struct rcu_fwd_cb { 3094 struct rcu_head rh; 3095 struct rcu_fwd_cb *rfc_next; 3096 struct rcu_fwd *rfc_rfp; 3097 int rfc_gps; 3098 }; 3099 3100 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 3101 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 3102 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 3103 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 3104 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 3105 3106 struct rcu_launder_hist { 3107 long n_launders; 3108 unsigned long launder_gp_seq; 3109 }; 3110 3111 struct rcu_fwd { 3112 spinlock_t rcu_fwd_lock; 3113 struct rcu_fwd_cb *rcu_fwd_cb_head; 3114 struct rcu_fwd_cb **rcu_fwd_cb_tail; 3115 long n_launders_cb; 3116 unsigned long rcu_fwd_startat; 3117 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 3118 unsigned long rcu_launder_gp_seq_start; 3119 int rcu_fwd_id; 3120 }; 3121 3122 static DEFINE_MUTEX(rcu_fwd_mutex); 3123 static struct rcu_fwd *rcu_fwds; 3124 static unsigned long rcu_fwd_seq; 3125 static atomic_long_t rcu_fwd_max_cbs; 3126 static bool rcu_fwd_emergency_stop; 3127 3128 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 3129 { 3130 unsigned long gps; 3131 unsigned long gps_old; 3132 int i; 3133 int j; 3134 3135 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 3136 if (rfp->n_launders_hist[i].n_launders > 0) 3137 break; 3138 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", 3139 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); 3140 gps_old = rfp->rcu_launder_gp_seq_start; 3141 for (j = 0; j <= i; j++) { 3142 gps = rfp->n_launders_hist[j].launder_gp_seq; 3143 pr_cont(" %ds/%d: %ld:%ld", 3144 j + 1, FWD_CBS_HIST_DIV, 3145 rfp->n_launders_hist[j].n_launders, 3146 rcutorture_seq_diff(gps, gps_old)); 3147 gps_old = gps; 3148 } 3149 pr_cont("\n"); 3150 } 3151 3152 /* Callback function for continuous-flood RCU callbacks. */ 3153 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 3154 { 3155 unsigned long flags; 3156 int i; 3157 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 3158 struct rcu_fwd_cb **rfcpp; 3159 struct rcu_fwd *rfp = rfcp->rfc_rfp; 3160 3161 rfcp->rfc_next = NULL; 3162 rfcp->rfc_gps++; 3163 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 3164 rfcpp = rfp->rcu_fwd_cb_tail; 3165 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 3166 smp_store_release(rfcpp, rfcp); 3167 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 3168 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 3169 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 3170 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 3171 rfp->n_launders_hist[i].n_launders++; 3172 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 3173 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3174 } 3175 3176 // Give the scheduler a chance, even on nohz_full CPUs. 3177 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 3178 { 3179 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 3180 // Real call_rcu() floods hit userspace, so emulate that. 3181 if (need_resched() || (iter & 0xfff)) 3182 schedule(); 3183 return; 3184 } 3185 // No userspace emulation: CB invocation throttles call_rcu() 3186 cond_resched(); 3187 } 3188 3189 /* 3190 * Free all callbacks on the rcu_fwd_cb_head list, either because the 3191 * test is over or because we hit an OOM event. 3192 */ 3193 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 3194 { 3195 unsigned long flags; 3196 unsigned long freed = 0; 3197 struct rcu_fwd_cb *rfcp; 3198 3199 for (;;) { 3200 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 3201 rfcp = rfp->rcu_fwd_cb_head; 3202 if (!rfcp) { 3203 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3204 break; 3205 } 3206 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 3207 if (!rfp->rcu_fwd_cb_head) 3208 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 3209 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3210 kfree(rfcp); 3211 freed++; 3212 rcu_torture_fwd_prog_cond_resched(freed); 3213 if (tick_nohz_full_enabled()) { 3214 local_irq_save(flags); 3215 rcu_momentary_eqs(); 3216 local_irq_restore(flags); 3217 } 3218 } 3219 return freed; 3220 } 3221 3222 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 3223 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 3224 int *tested, int *tested_tries) 3225 { 3226 unsigned long cver; 3227 unsigned long dur; 3228 struct fwd_cb_state fcs; 3229 unsigned long gps; 3230 int idx; 3231 int sd; 3232 int sd4; 3233 bool selfpropcb = false; 3234 unsigned long stopat; 3235 static DEFINE_TORTURE_RANDOM(trs); 3236 3237 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3238 if (!cur_ops->sync) 3239 return; // Cannot do need_resched() forward progress testing without ->sync. 3240 if (cur_ops->call && cur_ops->cb_barrier) { 3241 init_rcu_head_on_stack(&fcs.rh); 3242 selfpropcb = true; 3243 } 3244 3245 /* Tight loop containing cond_resched(). */ 3246 atomic_inc(&rcu_fwd_cb_nodelay); 3247 cur_ops->sync(); /* Later readers see above write. */ 3248 if (selfpropcb) { 3249 WRITE_ONCE(fcs.stop, 0); 3250 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 3251 } 3252 cver = READ_ONCE(rcu_torture_current_version); 3253 gps = cur_ops->get_gp_seq(); 3254 sd = cur_ops->stall_dur() + 1; 3255 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 3256 dur = sd4 + torture_random(&trs) % (sd - sd4); 3257 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3258 stopat = rfp->rcu_fwd_startat + dur; 3259 while (time_before(jiffies, stopat) && 3260 !shutdown_time_arrived() && 3261 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3262 idx = cur_ops->readlock(); 3263 udelay(10); 3264 cur_ops->readunlock(idx); 3265 if (!fwd_progress_need_resched || need_resched()) 3266 cond_resched(); 3267 } 3268 (*tested_tries)++; 3269 if (!time_before(jiffies, stopat) && 3270 !shutdown_time_arrived() && 3271 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3272 (*tested)++; 3273 cver = READ_ONCE(rcu_torture_current_version) - cver; 3274 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3275 WARN_ON(!cver && gps < 2); 3276 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, 3277 rfp->rcu_fwd_id, dur, cver, gps); 3278 } 3279 if (selfpropcb) { 3280 WRITE_ONCE(fcs.stop, 1); 3281 cur_ops->sync(); /* Wait for running CB to complete. */ 3282 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3283 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 3284 } 3285 3286 if (selfpropcb) { 3287 WARN_ON(READ_ONCE(fcs.stop) != 2); 3288 destroy_rcu_head_on_stack(&fcs.rh); 3289 } 3290 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 3291 atomic_dec(&rcu_fwd_cb_nodelay); 3292 } 3293 3294 /* Carry out call_rcu() forward-progress testing. */ 3295 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 3296 { 3297 unsigned long cver; 3298 unsigned long flags; 3299 unsigned long gps; 3300 int i; 3301 long n_launders; 3302 long n_launders_cb_snap; 3303 long n_launders_sa; 3304 long n_max_cbs; 3305 long n_max_gps; 3306 struct rcu_fwd_cb *rfcp; 3307 struct rcu_fwd_cb *rfcpn; 3308 unsigned long stopat; 3309 unsigned long stoppedat; 3310 3311 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3312 if (READ_ONCE(rcu_fwd_emergency_stop)) 3313 return; /* Get out of the way quickly, no GP wait! */ 3314 if (!cur_ops->call) 3315 return; /* Can't do call_rcu() fwd prog without ->call. */ 3316 3317 /* Loop continuously posting RCU callbacks. */ 3318 atomic_inc(&rcu_fwd_cb_nodelay); 3319 cur_ops->sync(); /* Later readers see above write. */ 3320 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3321 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 3322 n_launders = 0; 3323 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 3324 n_launders_sa = 0; 3325 n_max_cbs = 0; 3326 n_max_gps = 0; 3327 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 3328 rfp->n_launders_hist[i].n_launders = 0; 3329 cver = READ_ONCE(rcu_torture_current_version); 3330 gps = cur_ops->get_gp_seq(); 3331 rfp->rcu_launder_gp_seq_start = gps; 3332 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 3333 while (time_before(jiffies, stopat) && 3334 !shutdown_time_arrived() && 3335 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3336 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 3337 rfcpn = NULL; 3338 if (rfcp) 3339 rfcpn = READ_ONCE(rfcp->rfc_next); 3340 if (rfcpn) { 3341 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 3342 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 3343 break; 3344 rfp->rcu_fwd_cb_head = rfcpn; 3345 n_launders++; 3346 n_launders_sa++; 3347 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { 3348 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 3349 if (WARN_ON_ONCE(!rfcp)) { 3350 schedule_timeout_interruptible(1); 3351 continue; 3352 } 3353 n_max_cbs++; 3354 n_launders_sa = 0; 3355 rfcp->rfc_gps = 0; 3356 rfcp->rfc_rfp = rfp; 3357 } else { 3358 rfcp = NULL; 3359 } 3360 if (rfcp) 3361 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 3362 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 3363 if (tick_nohz_full_enabled()) { 3364 local_irq_save(flags); 3365 rcu_momentary_eqs(); 3366 local_irq_restore(flags); 3367 } 3368 } 3369 stoppedat = jiffies; 3370 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 3371 cver = READ_ONCE(rcu_torture_current_version) - cver; 3372 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3373 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3374 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 3375 (void)rcu_torture_fwd_prog_cbfree(rfp); 3376 3377 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 3378 !shutdown_time_arrived()) { 3379 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg) 3380 cur_ops->gp_kthread_dbg(); 3381 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n", 3382 __func__, 3383 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 3384 n_launders + n_max_cbs - n_launders_cb_snap, 3385 n_launders, n_launders_sa, 3386 n_max_gps, n_max_cbs, cver, gps, num_online_cpus()); 3387 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); 3388 mutex_lock(&rcu_fwd_mutex); // Serialize histograms. 3389 rcu_torture_fwd_cb_hist(rfp); 3390 mutex_unlock(&rcu_fwd_mutex); 3391 } 3392 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 3393 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 3394 atomic_dec(&rcu_fwd_cb_nodelay); 3395 } 3396 3397 3398 /* 3399 * OOM notifier, but this only prints diagnostic information for the 3400 * current forward-progress test. 3401 */ 3402 static int rcutorture_oom_notify(struct notifier_block *self, 3403 unsigned long notused, void *nfreed) 3404 { 3405 int i; 3406 long ncbs; 3407 struct rcu_fwd *rfp; 3408 3409 mutex_lock(&rcu_fwd_mutex); 3410 rfp = rcu_fwds; 3411 if (!rfp) { 3412 mutex_unlock(&rcu_fwd_mutex); 3413 return NOTIFY_OK; 3414 } 3415 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 3416 __func__); 3417 for (i = 0; i < fwd_progress; i++) { 3418 rcu_torture_fwd_cb_hist(&rfp[i]); 3419 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); 3420 } 3421 WRITE_ONCE(rcu_fwd_emergency_stop, true); 3422 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 3423 ncbs = 0; 3424 for (i = 0; i < fwd_progress; i++) 3425 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3426 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3427 cur_ops->cb_barrier(); 3428 ncbs = 0; 3429 for (i = 0; i < fwd_progress; i++) 3430 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3431 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3432 cur_ops->cb_barrier(); 3433 ncbs = 0; 3434 for (i = 0; i < fwd_progress; i++) 3435 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3436 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3437 smp_mb(); /* Frees before return to avoid redoing OOM. */ 3438 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 3439 pr_info("%s returning after OOM processing.\n", __func__); 3440 mutex_unlock(&rcu_fwd_mutex); 3441 return NOTIFY_OK; 3442 } 3443 3444 static struct notifier_block rcutorture_oom_nb = { 3445 .notifier_call = rcutorture_oom_notify 3446 }; 3447 3448 /* Carry out grace-period forward-progress testing. */ 3449 static int rcu_torture_fwd_prog(void *args) 3450 { 3451 bool firsttime = true; 3452 long max_cbs; 3453 int oldnice = task_nice(current); 3454 unsigned long oldseq = READ_ONCE(rcu_fwd_seq); 3455 struct rcu_fwd *rfp = args; 3456 int tested = 0; 3457 int tested_tries = 0; 3458 3459 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 3460 while (!rcu_inkernel_boot_has_ended()) 3461 schedule_timeout_interruptible(HZ / 10); 3462 rcu_bind_current_to_nocb(); 3463 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 3464 set_user_nice(current, MAX_NICE); 3465 do { 3466 if (!rfp->rcu_fwd_id) { 3467 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 3468 WRITE_ONCE(rcu_fwd_emergency_stop, false); 3469 if (!firsttime) { 3470 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); 3471 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); 3472 } 3473 firsttime = false; 3474 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); 3475 } else { 3476 while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) 3477 schedule_timeout_interruptible(HZ / 20); 3478 oldseq = READ_ONCE(rcu_fwd_seq); 3479 } 3480 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3481 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) 3482 rcu_torture_fwd_prog_cr(rfp); 3483 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && 3484 (!IS_ENABLED(CONFIG_TINY_RCU) || 3485 (rcu_inkernel_boot_has_ended() && 3486 torture_num_online_cpus() > rfp->rcu_fwd_id))) 3487 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 3488 3489 /* Avoid slow periods, better to test when busy. */ 3490 if (stutter_wait("rcu_torture_fwd_prog")) 3491 sched_set_normal(current, oldnice); 3492 } while (!torture_must_stop()); 3493 /* Short runs might not contain a valid forward-progress attempt. */ 3494 if (!rfp->rcu_fwd_id) { 3495 WARN_ON(!tested && tested_tries >= 5); 3496 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 3497 } 3498 torture_kthread_stopping("rcu_torture_fwd_prog"); 3499 return 0; 3500 } 3501 3502 /* If forward-progress checking is requested and feasible, spawn the thread. */ 3503 static int __init rcu_torture_fwd_prog_init(void) 3504 { 3505 int i; 3506 int ret = 0; 3507 struct rcu_fwd *rfp; 3508 3509 if (!fwd_progress) 3510 return 0; /* Not requested, so don't do it. */ 3511 if (fwd_progress >= nr_cpu_ids) { 3512 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); 3513 fwd_progress = nr_cpu_ids; 3514 } else if (fwd_progress < 0) { 3515 fwd_progress = nr_cpu_ids; 3516 } 3517 if ((!cur_ops->sync && !cur_ops->call) || 3518 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || 3519 cur_ops == &rcu_busted_ops) { 3520 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 3521 fwd_progress = 0; 3522 return 0; 3523 } 3524 if (stall_cpu > 0 || (preempt_duration > 0 && IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 3525 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall and/or preemption testing"); 3526 fwd_progress = 0; 3527 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 3528 return -EINVAL; /* In module, can fail back to user. */ 3529 WARN_ON(1); /* Make sure rcutorture scripting notices conflict. */ 3530 return 0; 3531 } 3532 if (fwd_progress_holdoff <= 0) 3533 fwd_progress_holdoff = 1; 3534 if (fwd_progress_div <= 0) 3535 fwd_progress_div = 4; 3536 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL); 3537 fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL); 3538 if (!rfp || !fwd_prog_tasks) { 3539 kfree(rfp); 3540 kfree(fwd_prog_tasks); 3541 fwd_prog_tasks = NULL; 3542 fwd_progress = 0; 3543 return -ENOMEM; 3544 } 3545 for (i = 0; i < fwd_progress; i++) { 3546 spin_lock_init(&rfp[i].rcu_fwd_lock); 3547 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; 3548 rfp[i].rcu_fwd_id = i; 3549 } 3550 mutex_lock(&rcu_fwd_mutex); 3551 rcu_fwds = rfp; 3552 mutex_unlock(&rcu_fwd_mutex); 3553 register_oom_notifier(&rcutorture_oom_nb); 3554 for (i = 0; i < fwd_progress; i++) { 3555 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); 3556 if (ret) { 3557 fwd_progress = i; 3558 return ret; 3559 } 3560 } 3561 return 0; 3562 } 3563 3564 static void rcu_torture_fwd_prog_cleanup(void) 3565 { 3566 int i; 3567 struct rcu_fwd *rfp; 3568 3569 if (!rcu_fwds || !fwd_prog_tasks) 3570 return; 3571 for (i = 0; i < fwd_progress; i++) 3572 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); 3573 unregister_oom_notifier(&rcutorture_oom_nb); 3574 mutex_lock(&rcu_fwd_mutex); 3575 rfp = rcu_fwds; 3576 rcu_fwds = NULL; 3577 mutex_unlock(&rcu_fwd_mutex); 3578 kfree(rfp); 3579 kfree(fwd_prog_tasks); 3580 fwd_prog_tasks = NULL; 3581 } 3582 3583 /* Callback function for RCU barrier testing. */ 3584 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 3585 { 3586 atomic_inc(&barrier_cbs_invoked); 3587 } 3588 3589 /* IPI handler to get callback posted on desired CPU, if online. */ 3590 static int rcu_torture_barrier1cb(void *rcu_void) 3591 { 3592 struct rcu_head *rhp = rcu_void; 3593 3594 cur_ops->call(rhp, rcu_torture_barrier_cbf); 3595 return 0; 3596 } 3597 3598 /* kthread function to register callbacks used to test RCU barriers. */ 3599 static int rcu_torture_barrier_cbs(void *arg) 3600 { 3601 long myid = (long)arg; 3602 bool lastphase = false; 3603 bool newphase; 3604 struct rcu_head rcu; 3605 3606 init_rcu_head_on_stack(&rcu); 3607 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 3608 set_user_nice(current, MAX_NICE); 3609 do { 3610 wait_event(barrier_cbs_wq[myid], 3611 (newphase = 3612 smp_load_acquire(&barrier_phase)) != lastphase || 3613 torture_must_stop()); 3614 lastphase = newphase; 3615 if (torture_must_stop()) 3616 break; 3617 /* 3618 * The above smp_load_acquire() ensures barrier_phase load 3619 * is ordered before the following ->call(). 3620 */ 3621 if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1)) 3622 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 3623 3624 if (atomic_dec_and_test(&barrier_cbs_count)) 3625 wake_up(&barrier_wq); 3626 } while (!torture_must_stop()); 3627 if (cur_ops->cb_barrier != NULL) 3628 cur_ops->cb_barrier(); 3629 destroy_rcu_head_on_stack(&rcu); 3630 torture_kthread_stopping("rcu_torture_barrier_cbs"); 3631 return 0; 3632 } 3633 3634 /* kthread function to drive and coordinate RCU barrier testing. */ 3635 static int rcu_torture_barrier(void *arg) 3636 { 3637 int i; 3638 3639 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 3640 do { 3641 atomic_set(&barrier_cbs_invoked, 0); 3642 atomic_set(&barrier_cbs_count, n_barrier_cbs); 3643 /* Ensure barrier_phase ordered after prior assignments. */ 3644 smp_store_release(&barrier_phase, !barrier_phase); 3645 for (i = 0; i < n_barrier_cbs; i++) 3646 wake_up(&barrier_cbs_wq[i]); 3647 wait_event(barrier_wq, 3648 atomic_read(&barrier_cbs_count) == 0 || 3649 torture_must_stop()); 3650 if (torture_must_stop()) 3651 break; 3652 n_barrier_attempts++; 3653 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 3654 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 3655 n_rcu_torture_barrier_error++; 3656 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 3657 atomic_read(&barrier_cbs_invoked), 3658 n_barrier_cbs); 3659 WARN_ON(1); 3660 // Wait manually for the remaining callbacks 3661 i = 0; 3662 do { 3663 if (WARN_ON(i++ > HZ)) 3664 i = INT_MIN; 3665 schedule_timeout_interruptible(1); 3666 cur_ops->cb_barrier(); 3667 } while (atomic_read(&barrier_cbs_invoked) != 3668 n_barrier_cbs && 3669 !torture_must_stop()); 3670 smp_mb(); // Can't trust ordering if broken. 3671 if (!torture_must_stop()) 3672 pr_err("Recovered: barrier_cbs_invoked = %d\n", 3673 atomic_read(&barrier_cbs_invoked)); 3674 } else { 3675 n_barrier_successes++; 3676 } 3677 schedule_timeout_interruptible(HZ / 10); 3678 } while (!torture_must_stop()); 3679 torture_kthread_stopping("rcu_torture_barrier"); 3680 return 0; 3681 } 3682 3683 /* Initialize RCU barrier testing. */ 3684 static int rcu_torture_barrier_init(void) 3685 { 3686 int i; 3687 int ret; 3688 3689 if (n_barrier_cbs <= 0) 3690 return 0; 3691 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 3692 pr_alert("%s" TORTURE_FLAG 3693 " Call or barrier ops missing for %s,\n", 3694 torture_type, cur_ops->name); 3695 pr_alert("%s" TORTURE_FLAG 3696 " RCU barrier testing omitted from run.\n", 3697 torture_type); 3698 return 0; 3699 } 3700 atomic_set(&barrier_cbs_count, 0); 3701 atomic_set(&barrier_cbs_invoked, 0); 3702 barrier_cbs_tasks = 3703 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 3704 GFP_KERNEL); 3705 barrier_cbs_wq = 3706 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 3707 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 3708 return -ENOMEM; 3709 for (i = 0; i < n_barrier_cbs; i++) { 3710 init_waitqueue_head(&barrier_cbs_wq[i]); 3711 ret = torture_create_kthread(rcu_torture_barrier_cbs, 3712 (void *)(long)i, 3713 barrier_cbs_tasks[i]); 3714 if (ret) 3715 return ret; 3716 } 3717 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 3718 } 3719 3720 /* Clean up after RCU barrier testing. */ 3721 static void rcu_torture_barrier_cleanup(void) 3722 { 3723 int i; 3724 3725 torture_stop_kthread(rcu_torture_barrier, barrier_task); 3726 if (barrier_cbs_tasks != NULL) { 3727 for (i = 0; i < n_barrier_cbs; i++) 3728 torture_stop_kthread(rcu_torture_barrier_cbs, 3729 barrier_cbs_tasks[i]); 3730 kfree(barrier_cbs_tasks); 3731 barrier_cbs_tasks = NULL; 3732 } 3733 if (barrier_cbs_wq != NULL) { 3734 kfree(barrier_cbs_wq); 3735 barrier_cbs_wq = NULL; 3736 } 3737 } 3738 3739 static bool rcu_torture_can_boost(void) 3740 { 3741 static int boost_warn_once; 3742 int prio; 3743 3744 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 3745 return false; 3746 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) 3747 return false; 3748 3749 prio = rcu_get_gp_kthreads_prio(); 3750 if (!prio) 3751 return false; 3752 3753 if (prio < 2) { 3754 if (boost_warn_once == 1) 3755 return false; 3756 3757 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 3758 boost_warn_once = 1; 3759 return false; 3760 } 3761 3762 return true; 3763 } 3764 3765 static bool read_exit_child_stop; 3766 static bool read_exit_child_stopped; 3767 static wait_queue_head_t read_exit_wq; 3768 3769 // Child kthread which just does an rcutorture reader and exits. 3770 static int rcu_torture_read_exit_child(void *trsp_in) 3771 { 3772 struct torture_random_state *trsp = trsp_in; 3773 3774 set_user_nice(current, MAX_NICE); 3775 // Minimize time between reading and exiting. 3776 while (!kthread_should_stop()) 3777 schedule_timeout_uninterruptible(HZ / 20); 3778 (void)rcu_torture_one_read(trsp, -1); 3779 return 0; 3780 } 3781 3782 // Parent kthread which creates and destroys read-exit child kthreads. 3783 static int rcu_torture_read_exit(void *unused) 3784 { 3785 bool errexit = false; 3786 int i; 3787 struct task_struct *tsp; 3788 DEFINE_TORTURE_RANDOM(trs); 3789 3790 // Allocate and initialize. 3791 set_user_nice(current, MAX_NICE); 3792 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); 3793 3794 // Each pass through this loop does one read-exit episode. 3795 do { 3796 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); 3797 for (i = 0; i < read_exit_burst; i++) { 3798 if (READ_ONCE(read_exit_child_stop)) 3799 break; 3800 stutter_wait("rcu_torture_read_exit"); 3801 // Spawn child. 3802 tsp = kthread_run(rcu_torture_read_exit_child, 3803 &trs, "%s", "rcu_torture_read_exit_child"); 3804 if (IS_ERR(tsp)) { 3805 TOROUT_ERRSTRING("out of memory"); 3806 errexit = true; 3807 break; 3808 } 3809 cond_resched(); 3810 kthread_stop(tsp); 3811 n_read_exits++; 3812 } 3813 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); 3814 rcu_barrier(); // Wait for task_struct free, avoid OOM. 3815 i = 0; 3816 for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) 3817 schedule_timeout_uninterruptible(HZ); 3818 } while (!errexit && !READ_ONCE(read_exit_child_stop)); 3819 3820 // Clean up and exit. 3821 smp_store_release(&read_exit_child_stopped, true); // After reaping. 3822 smp_mb(); // Store before wakeup. 3823 wake_up(&read_exit_wq); 3824 while (!torture_must_stop()) 3825 schedule_timeout_uninterruptible(HZ / 20); 3826 torture_kthread_stopping("rcu_torture_read_exit"); 3827 return 0; 3828 } 3829 3830 static int rcu_torture_read_exit_init(void) 3831 { 3832 if (read_exit_burst <= 0) 3833 return 0; 3834 init_waitqueue_head(&read_exit_wq); 3835 read_exit_child_stop = false; 3836 read_exit_child_stopped = false; 3837 return torture_create_kthread(rcu_torture_read_exit, NULL, 3838 read_exit_task); 3839 } 3840 3841 static void rcu_torture_read_exit_cleanup(void) 3842 { 3843 if (!read_exit_task) 3844 return; 3845 WRITE_ONCE(read_exit_child_stop, true); 3846 smp_mb(); // Above write before wait. 3847 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); 3848 torture_stop_kthread(rcutorture_read_exit, read_exit_task); 3849 } 3850 3851 static void rcutorture_test_nmis(int n) 3852 { 3853 #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3854 int cpu; 3855 int dumpcpu; 3856 int i; 3857 3858 for (i = 0; i < n; i++) { 3859 preempt_disable(); 3860 cpu = smp_processor_id(); 3861 dumpcpu = cpu + 1; 3862 if (dumpcpu >= nr_cpu_ids) 3863 dumpcpu = 0; 3864 pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu); 3865 dump_cpu_task(dumpcpu); 3866 preempt_enable(); 3867 schedule_timeout_uninterruptible(15 * HZ); 3868 } 3869 #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3870 WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis); 3871 #endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3872 } 3873 3874 // Randomly preempt online CPUs. 3875 static int rcu_torture_preempt(void *unused) 3876 { 3877 int cpu = -1; 3878 DEFINE_TORTURE_RANDOM(rand); 3879 3880 schedule_timeout_idle(stall_cpu_holdoff); 3881 do { 3882 // Wait for preempt_interval ms with up to 100us fuzz. 3883 torture_hrtimeout_ms(preempt_interval, 100, &rand); 3884 // Select online CPU. 3885 cpu = cpumask_next(cpu, cpu_online_mask); 3886 if (cpu >= nr_cpu_ids) 3887 cpu = cpumask_next(-1, cpu_online_mask); 3888 WARN_ON_ONCE(cpu >= nr_cpu_ids); 3889 // Move to that CPU, if can't do so, retry later. 3890 if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) 3891 continue; 3892 // Preempt at high-ish priority, then reset to normal. 3893 sched_set_fifo(current); 3894 torture_sched_setaffinity(current->pid, cpu_present_mask, true); 3895 mdelay(preempt_duration); 3896 sched_set_normal(current, 0); 3897 stutter_wait("rcu_torture_preempt"); 3898 } while (!torture_must_stop()); 3899 torture_kthread_stopping("rcu_torture_preempt"); 3900 return 0; 3901 } 3902 3903 static enum cpuhp_state rcutor_hp; 3904 3905 static struct hrtimer gpwrap_lag_timer; 3906 static bool gpwrap_lag_active; 3907 3908 /* Timer handler for toggling RCU grace-period sequence overflow test lag value */ 3909 static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer) 3910 { 3911 ktime_t next_delay; 3912 3913 if (gpwrap_lag_active) { 3914 pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n"); 3915 cur_ops->set_gpwrap_lag(0); 3916 gpwrap_lag_active = false; 3917 next_delay = ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0); 3918 } else { 3919 pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps); 3920 cur_ops->set_gpwrap_lag(gpwrap_lag_gps); 3921 gpwrap_lag_active = true; 3922 next_delay = ktime_set(gpwrap_lag_active_mins * 60, 0); 3923 } 3924 3925 if (torture_must_stop_irq()) 3926 return HRTIMER_NORESTART; 3927 3928 hrtimer_forward_now(timer, next_delay); 3929 return HRTIMER_RESTART; 3930 } 3931 3932 static int rcu_gpwrap_lag_init(void) 3933 { 3934 if (!gpwrap_lag) 3935 return 0; 3936 3937 if (gpwrap_lag_cycle_mins <= 0 || gpwrap_lag_active_mins <= 0) { 3938 pr_alert("rcu-torture: lag timing parameters must be positive\n"); 3939 return -EINVAL; 3940 } 3941 3942 hrtimer_setup(&gpwrap_lag_timer, rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3943 gpwrap_lag_active = false; 3944 hrtimer_start(&gpwrap_lag_timer, 3945 ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0), HRTIMER_MODE_REL); 3946 3947 return 0; 3948 } 3949 3950 static void rcu_gpwrap_lag_cleanup(void) 3951 { 3952 hrtimer_cancel(&gpwrap_lag_timer); 3953 cur_ops->set_gpwrap_lag(0); 3954 gpwrap_lag_active = false; 3955 } 3956 static void 3957 rcu_torture_cleanup(void) 3958 { 3959 int firsttime; 3960 int flags = 0; 3961 unsigned long gp_seq = 0; 3962 int i; 3963 int j; 3964 3965 if (torture_cleanup_begin()) { 3966 if (cur_ops->cb_barrier != NULL) { 3967 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3968 cur_ops->cb_barrier(); 3969 } 3970 if (cur_ops->gp_slow_unregister) 3971 cur_ops->gp_slow_unregister(NULL); 3972 return; 3973 } 3974 if (!cur_ops) { 3975 torture_cleanup_end(); 3976 return; 3977 } 3978 3979 rcutorture_test_nmis(test_nmis); 3980 3981 if (cur_ops->gp_kthread_dbg) 3982 cur_ops->gp_kthread_dbg(); 3983 torture_stop_kthread(rcu_torture_preempt, preempt_task); 3984 rcu_torture_read_exit_cleanup(); 3985 rcu_torture_barrier_cleanup(); 3986 rcu_torture_fwd_prog_cleanup(); 3987 torture_stop_kthread(rcu_torture_stall, stall_task); 3988 torture_stop_kthread(rcu_torture_writer, writer_task); 3989 3990 if (nocb_tasks) { 3991 for (i = 0; i < nrealnocbers; i++) 3992 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); 3993 kfree(nocb_tasks); 3994 nocb_tasks = NULL; 3995 } 3996 3997 if (updown_task) { 3998 torture_stop_kthread(rcu_torture_updown, updown_task); 3999 updown_task = NULL; 4000 } 4001 if (reader_tasks) { 4002 for (i = 0; i < nrealreaders; i++) 4003 torture_stop_kthread(rcu_torture_reader, 4004 reader_tasks[i]); 4005 kfree(reader_tasks); 4006 reader_tasks = NULL; 4007 } 4008 kfree(rcu_torture_reader_mbchk); 4009 rcu_torture_reader_mbchk = NULL; 4010 4011 if (fakewriter_tasks) { 4012 for (i = 0; i < nrealfakewriters; i++) 4013 torture_stop_kthread(rcu_torture_fakewriter, 4014 fakewriter_tasks[i]); 4015 kfree(fakewriter_tasks); 4016 fakewriter_tasks = NULL; 4017 } 4018 4019 if (cur_ops->get_gp_data) 4020 cur_ops->get_gp_data(&flags, &gp_seq); 4021 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", 4022 cur_ops->name, (long)gp_seq, flags, 4023 rcutorture_seq_diff(gp_seq, start_gp_seq)); 4024 torture_stop_kthread(rcu_torture_stats, stats_task); 4025 torture_stop_kthread(rcu_torture_fqs, fqs_task); 4026 if (rcu_torture_can_boost() && rcutor_hp >= 0) 4027 cpuhp_remove_state(rcutor_hp); 4028 4029 /* 4030 * Wait for all RCU callbacks to fire, then do torture-type-specific 4031 * cleanup operations. 4032 */ 4033 if (cur_ops->cb_barrier != NULL) { 4034 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 4035 cur_ops->cb_barrier(); 4036 } 4037 if (cur_ops->cleanup != NULL) 4038 cur_ops->cleanup(); 4039 4040 rcu_torture_mem_dump_obj(); 4041 4042 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 4043 4044 if (err_segs_recorded) { 4045 pr_alert("Failure/close-call rcutorture reader segments:\n"); 4046 if (rt_read_nsegs == 0) 4047 pr_alert("\t: No segments recorded!!!\n"); 4048 firsttime = 1; 4049 for (i = 0; i < rt_read_nsegs; i++) { 4050 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) 4051 pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); 4052 else 4053 pr_alert("\t"); 4054 pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); 4055 if (err_segs[i].rt_delay_jiffies != 0) { 4056 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 4057 err_segs[i].rt_delay_jiffies); 4058 firsttime = 0; 4059 } 4060 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 4061 pr_cont(" CPU %2d", err_segs[i].rt_cpu); 4062 if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) 4063 pr_cont("->%-2d", err_segs[i].rt_end_cpu); 4064 else 4065 pr_cont(" ..."); 4066 } 4067 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && 4068 cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { 4069 char buf1[20+1]; 4070 char buf2[20+1]; 4071 char sepchar = '-'; 4072 4073 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, 4074 buf1, ARRAY_SIZE(buf1)); 4075 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, 4076 buf2, ARRAY_SIZE(buf2)); 4077 if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { 4078 if (buf2[0]) { 4079 for (j = 0; buf2[j]; j++) 4080 buf2[j] = '.'; 4081 if (j) 4082 buf2[j - 1] = ' '; 4083 } 4084 sepchar = ' '; 4085 } 4086 pr_cont(" %s%c%s", buf1, sepchar, buf2); 4087 } 4088 if (err_segs[i].rt_delay_ms != 0) { 4089 pr_cont(" %s%ldms", firsttime ? "" : "+", 4090 err_segs[i].rt_delay_ms); 4091 firsttime = 0; 4092 } 4093 if (err_segs[i].rt_delay_us != 0) { 4094 pr_cont(" %s%ldus", firsttime ? "" : "+", 4095 err_segs[i].rt_delay_us); 4096 firsttime = 0; 4097 } 4098 pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); 4099 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) 4100 pr_cont(" BH"); 4101 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) 4102 pr_cont(" IRQ"); 4103 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) 4104 pr_cont(" PREEMPT"); 4105 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) 4106 pr_cont(" RBH"); 4107 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) 4108 pr_cont(" SCHED"); 4109 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) 4110 pr_cont(" RCU_1"); 4111 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) 4112 pr_cont(" RCU_2"); 4113 pr_cont("\n"); 4114 4115 } 4116 if (rt_read_preempted) 4117 pr_alert("\tReader was preempted.\n"); 4118 } 4119 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 4120 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 4121 else if (torture_onoff_failures()) 4122 rcu_torture_print_module_parms(cur_ops, 4123 "End of test: RCU_HOTPLUG"); 4124 else 4125 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 4126 torture_cleanup_end(); 4127 if (cur_ops->gp_slow_unregister) 4128 cur_ops->gp_slow_unregister(NULL); 4129 4130 if (gpwrap_lag && cur_ops->set_gpwrap_lag) 4131 rcu_gpwrap_lag_cleanup(); 4132 } 4133 4134 static void rcu_torture_leak_cb(struct rcu_head *rhp) 4135 { 4136 } 4137 4138 static void rcu_torture_err_cb(struct rcu_head *rhp) 4139 { 4140 /* 4141 * This -might- happen due to race conditions, but is unlikely. 4142 * The scenario that leads to this happening is that the 4143 * first of the pair of duplicate callbacks is queued, 4144 * someone else starts a grace period that includes that 4145 * callback, then the second of the pair must wait for the 4146 * next grace period. Unlikely, but can happen. If it 4147 * does happen, the debug-objects subsystem won't have splatted. 4148 */ 4149 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 4150 } 4151 4152 /* 4153 * Verify that double-free causes debug-objects to complain, but only 4154 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 4155 * cannot be carried out. 4156 */ 4157 static void rcu_test_debug_objects(void) 4158 { 4159 struct rcu_head rh1; 4160 struct rcu_head rh2; 4161 int idx; 4162 4163 if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) { 4164 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n", 4165 KBUILD_MODNAME, cur_ops->name); 4166 return; 4167 } 4168 4169 if (WARN_ON_ONCE(cur_ops->debug_objects && 4170 (!cur_ops->call || !cur_ops->cb_barrier))) 4171 return; 4172 4173 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 4174 4175 init_rcu_head_on_stack(&rh1); 4176 init_rcu_head_on_stack(&rh2); 4177 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name); 4178 4179 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 4180 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */ 4181 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 4182 cur_ops->call(&rh2, rcu_torture_leak_cb); 4183 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 4184 if (rhp) { 4185 cur_ops->call(rhp, rcu_torture_leak_cb); 4186 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ 4187 } 4188 cur_ops->readunlock(idx); 4189 4190 /* Wait for them all to get done so we can safely return. */ 4191 cur_ops->cb_barrier(); 4192 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name); 4193 destroy_rcu_head_on_stack(&rh1); 4194 destroy_rcu_head_on_stack(&rh2); 4195 kfree(rhp); 4196 } 4197 4198 static void rcutorture_sync(void) 4199 { 4200 static unsigned long n; 4201 4202 if (cur_ops->sync && !(++n & 0xfff)) 4203 cur_ops->sync(); 4204 } 4205 4206 static DEFINE_MUTEX(mut0); 4207 static DEFINE_MUTEX(mut1); 4208 static DEFINE_MUTEX(mut2); 4209 static DEFINE_MUTEX(mut3); 4210 static DEFINE_MUTEX(mut4); 4211 static DEFINE_MUTEX(mut5); 4212 static DEFINE_MUTEX(mut6); 4213 static DEFINE_MUTEX(mut7); 4214 static DEFINE_MUTEX(mut8); 4215 static DEFINE_MUTEX(mut9); 4216 4217 static DECLARE_RWSEM(rwsem0); 4218 static DECLARE_RWSEM(rwsem1); 4219 static DECLARE_RWSEM(rwsem2); 4220 static DECLARE_RWSEM(rwsem3); 4221 static DECLARE_RWSEM(rwsem4); 4222 static DECLARE_RWSEM(rwsem5); 4223 static DECLARE_RWSEM(rwsem6); 4224 static DECLARE_RWSEM(rwsem7); 4225 static DECLARE_RWSEM(rwsem8); 4226 static DECLARE_RWSEM(rwsem9); 4227 4228 DEFINE_STATIC_SRCU(srcu0); 4229 DEFINE_STATIC_SRCU(srcu1); 4230 DEFINE_STATIC_SRCU(srcu2); 4231 DEFINE_STATIC_SRCU(srcu3); 4232 DEFINE_STATIC_SRCU(srcu4); 4233 DEFINE_STATIC_SRCU(srcu5); 4234 DEFINE_STATIC_SRCU(srcu6); 4235 DEFINE_STATIC_SRCU(srcu7); 4236 DEFINE_STATIC_SRCU(srcu8); 4237 DEFINE_STATIC_SRCU(srcu9); 4238 4239 static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, 4240 int cyclelen, int deadlock) 4241 { 4242 int j = i + 1; 4243 4244 if (j >= cyclelen) 4245 j = deadlock ? 0 : -1; 4246 if (j >= 0) 4247 pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i); 4248 else 4249 pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i); 4250 return j; 4251 } 4252 4253 // Test lockdep on SRCU-based deadlock scenarios. 4254 static void rcu_torture_init_srcu_lockdep(void) 4255 { 4256 int cyclelen; 4257 int deadlock; 4258 bool err = false; 4259 int i; 4260 int j; 4261 int idx; 4262 struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4, 4263 &mut5, &mut6, &mut7, &mut8, &mut9 }; 4264 struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4, 4265 &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; 4266 struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, 4267 &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; 4268 int testtype; 4269 4270 if (!test_srcu_lockdep) 4271 return; 4272 4273 deadlock = test_srcu_lockdep / 1000; 4274 testtype = (test_srcu_lockdep / 10) % 100; 4275 cyclelen = test_srcu_lockdep % 10; 4276 WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus)); 4277 if (WARN_ONCE(deadlock != !!deadlock, 4278 "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n", 4279 __func__, test_srcu_lockdep, deadlock)) 4280 err = true; 4281 if (WARN_ONCE(cyclelen <= 0, 4282 "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n", 4283 __func__, test_srcu_lockdep, cyclelen)) 4284 err = true; 4285 if (err) 4286 goto err_out; 4287 4288 if (testtype == 0) { 4289 pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n", 4290 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4291 if (deadlock && cyclelen == 1) 4292 pr_info("%s: Expect hang.\n", __func__); 4293 for (i = 0; i < cyclelen; i++) { 4294 j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu", 4295 "srcu_read_unlock", i, cyclelen, deadlock); 4296 idx = srcu_read_lock(srcus[i]); 4297 if (j >= 0) 4298 synchronize_srcu(srcus[j]); 4299 srcu_read_unlock(srcus[i], idx); 4300 } 4301 return; 4302 } 4303 4304 if (testtype == 1) { 4305 pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n", 4306 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4307 for (i = 0; i < cyclelen; i++) { 4308 pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n", 4309 __func__, i, i, i, i); 4310 idx = srcu_read_lock(srcus[i]); 4311 mutex_lock(muts[i]); 4312 mutex_unlock(muts[i]); 4313 srcu_read_unlock(srcus[i], idx); 4314 4315 j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu", 4316 "mutex_unlock", i, cyclelen, deadlock); 4317 mutex_lock(muts[i]); 4318 if (j >= 0) 4319 synchronize_srcu(srcus[j]); 4320 mutex_unlock(muts[i]); 4321 } 4322 return; 4323 } 4324 4325 if (testtype == 2) { 4326 pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n", 4327 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4328 for (i = 0; i < cyclelen; i++) { 4329 pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n", 4330 __func__, i, i, i, i); 4331 idx = srcu_read_lock(srcus[i]); 4332 down_read(rwsems[i]); 4333 up_read(rwsems[i]); 4334 srcu_read_unlock(srcus[i], idx); 4335 4336 j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu", 4337 "up_write", i, cyclelen, deadlock); 4338 down_write(rwsems[i]); 4339 if (j >= 0) 4340 synchronize_srcu(srcus[j]); 4341 up_write(rwsems[i]); 4342 } 4343 return; 4344 } 4345 4346 #ifdef CONFIG_TASKS_TRACE_RCU 4347 if (testtype == 3) { 4348 pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n", 4349 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4350 if (deadlock && cyclelen == 1) 4351 pr_info("%s: Expect hang.\n", __func__); 4352 for (i = 0; i < cyclelen; i++) { 4353 char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock"; 4354 char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace" 4355 : "synchronize_srcu"; 4356 char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock"; 4357 4358 j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock); 4359 if (i == 0) 4360 rcu_read_lock_trace(); 4361 else 4362 idx = srcu_read_lock(srcus[i]); 4363 if (j >= 0) { 4364 if (i == cyclelen - 1) 4365 synchronize_rcu_tasks_trace(); 4366 else 4367 synchronize_srcu(srcus[j]); 4368 } 4369 if (i == 0) 4370 rcu_read_unlock_trace(); 4371 else 4372 srcu_read_unlock(srcus[i], idx); 4373 } 4374 return; 4375 } 4376 #endif // #ifdef CONFIG_TASKS_TRACE_RCU 4377 4378 err_out: 4379 pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); 4380 pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); 4381 pr_info("%s: D: Deadlock if nonzero.\n", __func__); 4382 pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); 4383 pr_info("%s: L: Cycle length.\n", __func__); 4384 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) 4385 pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); 4386 } 4387 4388 static int __init 4389 rcu_torture_init(void) 4390 { 4391 long i; 4392 int cpu; 4393 int firsterr = 0; 4394 int flags = 0; 4395 unsigned long gp_seq = 0; 4396 static struct rcu_torture_ops *torture_ops[] = { 4397 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, 4398 TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 4399 &trivial_ops, 4400 }; 4401 4402 if (!torture_init_begin(torture_type, verbose)) 4403 return -EBUSY; 4404 4405 /* Process args and tell the world that the torturer is on the job. */ 4406 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 4407 cur_ops = torture_ops[i]; 4408 if (strcmp(torture_type, cur_ops->name) == 0) 4409 break; 4410 } 4411 if (i == ARRAY_SIZE(torture_ops)) { 4412 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 4413 torture_type); 4414 pr_alert("rcu-torture types:"); 4415 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 4416 pr_cont(" %s", torture_ops[i]->name); 4417 pr_cont("\n"); 4418 firsterr = -EINVAL; 4419 cur_ops = NULL; 4420 goto unwind; 4421 } 4422 if (cur_ops->fqs == NULL && fqs_duration != 0) { 4423 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 4424 fqs_duration = 0; 4425 } 4426 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops || 4427 !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 4428 pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n", 4429 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU)); 4430 nocbs_nthreads = 0; 4431 } 4432 if (cur_ops->init) 4433 cur_ops->init(); 4434 4435 rcu_torture_init_srcu_lockdep(); 4436 4437 if (nfakewriters >= 0) { 4438 nrealfakewriters = nfakewriters; 4439 } else { 4440 nrealfakewriters = num_online_cpus() - 2 - nfakewriters; 4441 if (nrealfakewriters <= 0) 4442 nrealfakewriters = 1; 4443 } 4444 4445 if (nreaders >= 0) { 4446 nrealreaders = nreaders; 4447 } else { 4448 nrealreaders = num_online_cpus() - 2 - nreaders; 4449 if (nrealreaders <= 0) 4450 nrealreaders = 1; 4451 } 4452 rcu_torture_print_module_parms(cur_ops, "Start of test"); 4453 if (cur_ops->get_gp_data) 4454 cur_ops->get_gp_data(&flags, &gp_seq); 4455 start_gp_seq = gp_seq; 4456 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", 4457 cur_ops->name, (long)gp_seq, flags); 4458 4459 /* Set up the freelist. */ 4460 4461 INIT_LIST_HEAD(&rcu_torture_freelist); 4462 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 4463 rcu_tortures[i].rtort_mbtest = 0; 4464 list_add_tail(&rcu_tortures[i].rtort_free, 4465 &rcu_torture_freelist); 4466 } 4467 4468 /* Initialize the statistics so that each run gets its own numbers. */ 4469 4470 rcu_torture_current = NULL; 4471 rcu_torture_current_version = 0; 4472 atomic_set(&n_rcu_torture_alloc, 0); 4473 atomic_set(&n_rcu_torture_alloc_fail, 0); 4474 atomic_set(&n_rcu_torture_free, 0); 4475 atomic_set(&n_rcu_torture_mberror, 0); 4476 atomic_set(&n_rcu_torture_mbchk_fail, 0); 4477 atomic_set(&n_rcu_torture_mbchk_tries, 0); 4478 atomic_set(&n_rcu_torture_error, 0); 4479 n_rcu_torture_barrier_error = 0; 4480 n_rcu_torture_boost_ktrerror = 0; 4481 n_rcu_torture_boost_failure = 0; 4482 n_rcu_torture_boosts = 0; 4483 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 4484 atomic_set(&rcu_torture_wcount[i], 0); 4485 for_each_possible_cpu(cpu) { 4486 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 4487 per_cpu(rcu_torture_count, cpu)[i] = 0; 4488 per_cpu(rcu_torture_batch, cpu)[i] = 0; 4489 } 4490 } 4491 err_segs_recorded = 0; 4492 rt_read_nsegs = 0; 4493 4494 /* Start up the kthreads. */ 4495 4496 rcu_torture_write_types(); 4497 if (nrealfakewriters > 0) { 4498 fakewriter_tasks = kcalloc(nrealfakewriters, 4499 sizeof(fakewriter_tasks[0]), 4500 GFP_KERNEL); 4501 if (fakewriter_tasks == NULL) { 4502 TOROUT_ERRSTRING("out of memory"); 4503 firsterr = -ENOMEM; 4504 goto unwind; 4505 } 4506 } 4507 for (i = 0; i < nrealfakewriters; i++) { 4508 firsterr = torture_create_kthread(rcu_torture_fakewriter, 4509 NULL, fakewriter_tasks[i]); 4510 if (torture_init_error(firsterr)) 4511 goto unwind; 4512 } 4513 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 4514 GFP_KERNEL); 4515 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), 4516 GFP_KERNEL); 4517 if (!reader_tasks || !rcu_torture_reader_mbchk) { 4518 TOROUT_ERRSTRING("out of memory"); 4519 firsterr = -ENOMEM; 4520 goto unwind; 4521 } 4522 for (i = 0; i < nrealreaders; i++) { 4523 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 4524 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 4525 reader_tasks[i]); 4526 if (torture_init_error(firsterr)) 4527 goto unwind; 4528 } 4529 4530 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 4531 writer_task); 4532 if (torture_init_error(firsterr)) 4533 goto unwind; 4534 4535 firsterr = rcu_torture_updown_init(); 4536 if (torture_init_error(firsterr)) 4537 goto unwind; 4538 nrealnocbers = nocbs_nthreads; 4539 if (WARN_ON(nrealnocbers < 0)) 4540 nrealnocbers = 1; 4541 if (WARN_ON(nocbs_toggle < 0)) 4542 nocbs_toggle = HZ; 4543 if (nrealnocbers > 0) { 4544 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); 4545 if (nocb_tasks == NULL) { 4546 TOROUT_ERRSTRING("out of memory"); 4547 firsterr = -ENOMEM; 4548 goto unwind; 4549 } 4550 } else { 4551 nocb_tasks = NULL; 4552 } 4553 for (i = 0; i < nrealnocbers; i++) { 4554 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 4555 if (torture_init_error(firsterr)) 4556 goto unwind; 4557 } 4558 if (stat_interval > 0) { 4559 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 4560 stats_task); 4561 if (torture_init_error(firsterr)) 4562 goto unwind; 4563 } 4564 if (test_no_idle_hz && shuffle_interval > 0) { 4565 firsterr = torture_shuffle_init(shuffle_interval * HZ); 4566 if (torture_init_error(firsterr)) 4567 goto unwind; 4568 } 4569 if (stutter < 0) 4570 stutter = 0; 4571 if (stutter) { 4572 int t; 4573 4574 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 4575 firsterr = torture_stutter_init(stutter * HZ, t); 4576 if (torture_init_error(firsterr)) 4577 goto unwind; 4578 } 4579 if (fqs_duration < 0) 4580 fqs_duration = 0; 4581 if (fqs_holdoff < 0) 4582 fqs_holdoff = 0; 4583 if (fqs_duration && fqs_holdoff) { 4584 /* Create the fqs thread */ 4585 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 4586 fqs_task); 4587 if (torture_init_error(firsterr)) 4588 goto unwind; 4589 } 4590 if (test_boost_interval < 1) 4591 test_boost_interval = 1; 4592 if (test_boost_duration < 2) 4593 test_boost_duration = 2; 4594 if (rcu_torture_can_boost()) { 4595 4596 boost_starttime = jiffies + test_boost_interval * HZ; 4597 4598 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 4599 rcutorture_booster_init, 4600 rcutorture_booster_cleanup); 4601 rcutor_hp = firsterr; 4602 if (torture_init_error(firsterr)) 4603 goto unwind; 4604 } 4605 shutdown_jiffies = jiffies + shutdown_secs * HZ; 4606 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 4607 if (torture_init_error(firsterr)) 4608 goto unwind; 4609 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 4610 rcutorture_sync); 4611 if (torture_init_error(firsterr)) 4612 goto unwind; 4613 firsterr = rcu_torture_stall_init(); 4614 if (torture_init_error(firsterr)) 4615 goto unwind; 4616 firsterr = rcu_torture_fwd_prog_init(); 4617 if (torture_init_error(firsterr)) 4618 goto unwind; 4619 firsterr = rcu_torture_barrier_init(); 4620 if (torture_init_error(firsterr)) 4621 goto unwind; 4622 firsterr = rcu_torture_read_exit_init(); 4623 if (torture_init_error(firsterr)) 4624 goto unwind; 4625 if (preempt_duration > 0) { 4626 firsterr = torture_create_kthread(rcu_torture_preempt, NULL, preempt_task); 4627 if (torture_init_error(firsterr)) 4628 goto unwind; 4629 } 4630 if (object_debug) 4631 rcu_test_debug_objects(); 4632 4633 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister)) 4634 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay); 4635 4636 if (gpwrap_lag && cur_ops->set_gpwrap_lag) { 4637 firsterr = rcu_gpwrap_lag_init(); 4638 if (torture_init_error(firsterr)) 4639 goto unwind; 4640 } 4641 4642 torture_init_end(); 4643 return 0; 4644 4645 unwind: 4646 torture_init_end(); 4647 rcu_torture_cleanup(); 4648 if (shutdown_secs) { 4649 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 4650 kernel_power_off(); 4651 } 4652 return firsterr; 4653 } 4654 4655 module_init(rcu_torture_init); 4656 module_exit(rcu_torture_cleanup); 4657