1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.rst 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/rcu_notifier.h> 25 #include <linux/interrupt.h> 26 #include <linux/sched/signal.h> 27 #include <uapi/linux/sched/types.h> 28 #include <linux/atomic.h> 29 #include <linux/bitops.h> 30 #include <linux/completion.h> 31 #include <linux/moduleparam.h> 32 #include <linux/percpu.h> 33 #include <linux/notifier.h> 34 #include <linux/reboot.h> 35 #include <linux/freezer.h> 36 #include <linux/cpu.h> 37 #include <linux/delay.h> 38 #include <linux/stat.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/trace_clock.h> 42 #include <asm/byteorder.h> 43 #include <linux/torture.h> 44 #include <linux/vmalloc.h> 45 #include <linux/sched/debug.h> 46 #include <linux/sched/sysctl.h> 47 #include <linux/oom.h> 48 #include <linux/tick.h> 49 #include <linux/rcupdate_trace.h> 50 #include <linux/nmi.h> 51 #include <linux/perf_event.h> 52 53 #include "rcu.h" 54 55 MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility"); 56 MODULE_LICENSE("GPL"); 57 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 58 59 // Bits for ->extendables field, extendables param, and related definitions. 60 #define RCUTORTURE_RDR_SHIFT_1 8 // Put SRCU index in upper bits. 61 #define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1) 62 #define RCUTORTURE_RDR_SHIFT_2 16 // Put SRCU index in upper bits. 63 #define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2) 64 #define RCUTORTURE_RDR_BH 0x01 // Extend readers by disabling bh. 65 #define RCUTORTURE_RDR_IRQ 0x02 // ... disabling interrupts. 66 #define RCUTORTURE_RDR_PREEMPT 0x04 // ... disabling preemption. 67 #define RCUTORTURE_RDR_RBH 0x08 // ... rcu_read_lock_bh(). 68 #define RCUTORTURE_RDR_SCHED 0x10 // ... rcu_read_lock_sched(). 69 #define RCUTORTURE_RDR_RCU_1 0x20 // ... entering another RCU reader. 70 #define RCUTORTURE_RDR_RCU_2 0x40 // ... entering another RCU reader. 71 #define RCUTORTURE_RDR_UPDOWN 0x80 // ... up-read from task, down-read from timer. 72 // Note: Manual start, automatic end. 73 #define RCUTORTURE_RDR_NBITS 8 // Number of bits defined above. 74 #define RCUTORTURE_MAX_EXTEND \ 75 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 76 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) // Intentionally omit RCUTORTURE_RDR_UPDOWN. 77 #define RCUTORTURE_RDR_ALLBITS \ 78 (RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \ 79 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2) 80 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 81 /* Must be power of two minus one. */ 82 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 83 84 torture_param(bool, deboost_timeliness_check, 0, "Enable checks for immediate deboosting"); 85 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 86 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 87 torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); 88 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 89 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 90 torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); 91 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 92 torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); 93 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); 94 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 95 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); 96 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); 97 torture_param(bool, gp_cond_exp_full, false, 98 "Use conditional/async full-stateexpedited GP wait primitives"); 99 torture_param(int, gp_cond_wi, 16 * USEC_PER_SEC / HZ, 100 "Wait interval for normal conditional grace periods, us (default 16 jiffies)"); 101 torture_param(int, gp_cond_wi_exp, 128, 102 "Wait interval for expedited conditional grace periods, us (default 128 us)"); 103 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 104 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); 105 torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); 106 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); 107 torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); 108 torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); 109 torture_param(int, gp_poll_wi, 16 * USEC_PER_SEC / HZ, 110 "Wait interval for normal polled grace periods, us (default 16 jiffies)"); 111 torture_param(int, gp_poll_wi_exp, 128, 112 "Wait interval for expedited polled grace periods, us (default 128 us)"); 113 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 114 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 115 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); 116 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); 117 torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); 118 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 119 torture_param(bool, nmi_calls, true, "Exercise ->call() from NMI on nmi_capable flavors"); 120 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 121 torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)"); 122 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); 123 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 124 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); 125 torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing"); 126 torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period."); 127 torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)"); 128 torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)"); 129 torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); 130 torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); 131 torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); 132 torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)"); 133 torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); 134 torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); 135 torture_param(int, reader_flavor, SRCU_READ_FLAVOR_NORMAL, "Reader flavors to use, one per bit."); 136 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 137 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 138 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 139 torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); 140 torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); 141 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 142 torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); 143 torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); 144 torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); 145 torture_param(bool, stall_only, 0, "Suppress all non-CPU-stall kthreads."); 146 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 147 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 148 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 149 torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); 150 torture_param(int, test_boost_holdoff, 0, "Holdoff time from rcutorture start, seconds."); 151 torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); 152 torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); 153 torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); 154 torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); 155 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 156 157 static char *torture_type = "rcu"; 158 module_param(torture_type, charp, 0444); 159 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 160 161 static int nrealnocbers; 162 static int nrealreaders; 163 static int nrealfakewriters; 164 static struct task_struct *writer_task; 165 static struct task_struct **fakewriter_tasks; 166 static struct task_struct **reader_tasks; 167 static struct task_struct *updown_task; 168 static struct task_struct **nocb_tasks; 169 static struct task_struct *stats_task; 170 static struct task_struct *fqs_task; 171 static struct task_struct *boost_tasks[NR_CPUS]; 172 static struct task_struct *stall_task; 173 static struct task_struct **fwd_prog_tasks; 174 static struct task_struct **barrier_cbs_tasks; 175 static struct task_struct *barrier_task; 176 static struct task_struct *read_exit_task; 177 static struct task_struct *preempt_task; 178 179 #define RCU_TORTURE_PIPE_LEN 10 180 181 // Mailbox-like structure to check RCU global memory ordering. 182 struct rcu_torture_reader_check { 183 unsigned long rtc_myloops; 184 int rtc_chkrdr; 185 unsigned long rtc_chkloops; 186 int rtc_ready; 187 struct rcu_torture_reader_check *rtc_assigner; 188 } ____cacheline_internodealigned_in_smp; 189 190 // Update-side data structure used to check RCU readers. 191 struct rcu_torture { 192 struct rcu_head rtort_rcu; 193 int rtort_pipe_count; 194 struct list_head rtort_free; 195 int rtort_mbtest; 196 struct rcu_torture_reader_check *rtort_chkp; 197 }; 198 199 static LIST_HEAD(rcu_torture_freelist); 200 static struct rcu_torture __rcu *rcu_torture_current; 201 static unsigned long rcu_torture_current_version; 202 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 203 static DEFINE_SPINLOCK(rcu_torture_lock); 204 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 205 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 206 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 207 static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; 208 static atomic_t n_rcu_torture_alloc; 209 static atomic_t n_rcu_torture_alloc_fail; 210 static atomic_t n_rcu_torture_free; 211 static atomic_t n_rcu_torture_mberror; 212 static atomic_t n_rcu_torture_mbchk_fail; 213 static atomic_t n_rcu_torture_mbchk_tries; 214 static atomic_t n_rcu_torture_error; 215 static long n_rcu_torture_barrier_error; 216 static long n_rcu_torture_boost_ktrerror; 217 static long n_rcu_torture_boost_failure; 218 static long n_rcu_torture_boosts; 219 static atomic_long_t n_rcu_torture_timers; 220 static atomic_long_t n_rcu_torture_irqs; 221 static atomic_long_t n_rcu_torture_nmi_call; 222 static atomic_long_t n_rcu_torture_nmi_cb; 223 static long n_barrier_attempts; 224 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 225 static unsigned long n_read_exits; 226 static struct list_head rcu_torture_removed; 227 static unsigned long shutdown_jiffies; 228 static unsigned long start_gp_seq; 229 static atomic_long_t n_nocb_offload; 230 static atomic_long_t n_nocb_deoffload; 231 232 static int rcu_torture_writer_state; 233 #define RTWS_FIXED_DELAY 0 234 #define RTWS_DELAY 1 235 #define RTWS_REPLACE 2 236 #define RTWS_DEF_FREE 3 237 #define RTWS_EXP_SYNC 4 238 #define RTWS_COND_GET 5 239 #define RTWS_COND_GET_FULL 6 240 #define RTWS_COND_GET_EXP 7 241 #define RTWS_COND_GET_EXP_FULL 8 242 #define RTWS_COND_SYNC 9 243 #define RTWS_COND_SYNC_FULL 10 244 #define RTWS_COND_SYNC_EXP 11 245 #define RTWS_COND_SYNC_EXP_FULL 12 246 #define RTWS_POLL_GET 13 247 #define RTWS_POLL_GET_FULL 14 248 #define RTWS_POLL_GET_EXP 15 249 #define RTWS_POLL_GET_EXP_FULL 16 250 #define RTWS_POLL_WAIT 17 251 #define RTWS_POLL_WAIT_FULL 18 252 #define RTWS_POLL_WAIT_EXP 19 253 #define RTWS_POLL_WAIT_EXP_FULL 20 254 #define RTWS_SYNC 21 255 #define RTWS_STUTTER 22 256 #define RTWS_STOPPING 23 257 static const char * const rcu_torture_writer_state_names[] = { 258 "RTWS_FIXED_DELAY", 259 "RTWS_DELAY", 260 "RTWS_REPLACE", 261 "RTWS_DEF_FREE", 262 "RTWS_EXP_SYNC", 263 "RTWS_COND_GET", 264 "RTWS_COND_GET_FULL", 265 "RTWS_COND_GET_EXP", 266 "RTWS_COND_GET_EXP_FULL", 267 "RTWS_COND_SYNC", 268 "RTWS_COND_SYNC_FULL", 269 "RTWS_COND_SYNC_EXP", 270 "RTWS_COND_SYNC_EXP_FULL", 271 "RTWS_POLL_GET", 272 "RTWS_POLL_GET_FULL", 273 "RTWS_POLL_GET_EXP", 274 "RTWS_POLL_GET_EXP_FULL", 275 "RTWS_POLL_WAIT", 276 "RTWS_POLL_WAIT_FULL", 277 "RTWS_POLL_WAIT_EXP", 278 "RTWS_POLL_WAIT_EXP_FULL", 279 "RTWS_SYNC", 280 "RTWS_STUTTER", 281 "RTWS_STOPPING", 282 }; 283 284 /* Record reader segment types and duration for first failing read. */ 285 struct rt_read_seg { 286 int rt_readstate; 287 unsigned long rt_delay_jiffies; 288 unsigned long rt_delay_ms; 289 unsigned long rt_delay_us; 290 bool rt_preempted; 291 int rt_cpu; 292 int rt_end_cpu; 293 unsigned long long rt_gp_seq; 294 unsigned long long rt_gp_seq_end; 295 u64 rt_ts; 296 }; 297 static int err_segs_recorded; 298 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 299 static int rt_read_nsegs; 300 static int rt_read_preempted; 301 302 static const char *rcu_torture_writer_state_getname(void) 303 { 304 unsigned int i = READ_ONCE(rcu_torture_writer_state); 305 306 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 307 return "???"; 308 return rcu_torture_writer_state_names[i]; 309 } 310 311 #ifdef CONFIG_RCU_TRACE 312 static u64 notrace rcu_trace_clock_local(void) 313 { 314 u64 ts = trace_clock_local(); 315 316 (void)do_div(ts, NSEC_PER_USEC); 317 return ts; 318 } 319 #else /* #ifdef CONFIG_RCU_TRACE */ 320 static u64 notrace rcu_trace_clock_local(void) 321 { 322 return 0ULL; 323 } 324 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 325 326 /* 327 * Stop aggressive CPU-hog tests a bit before the end of the test in order 328 * to avoid interfering with test shutdown. 329 */ 330 static bool shutdown_time_arrived(void) 331 { 332 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 333 } 334 335 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 336 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 337 /* and boost task create/destroy. */ 338 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 339 static bool barrier_phase; /* Test phase. */ 340 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 341 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 342 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 343 344 static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 345 346 /* 347 * Allocate an element from the rcu_tortures pool. 348 */ 349 static struct rcu_torture * 350 rcu_torture_alloc(void) 351 { 352 struct list_head *p; 353 354 spin_lock_bh(&rcu_torture_lock); 355 if (list_empty(&rcu_torture_freelist)) { 356 atomic_inc(&n_rcu_torture_alloc_fail); 357 spin_unlock_bh(&rcu_torture_lock); 358 return NULL; 359 } 360 atomic_inc(&n_rcu_torture_alloc); 361 p = rcu_torture_freelist.next; 362 list_del_init(p); 363 spin_unlock_bh(&rcu_torture_lock); 364 return container_of(p, struct rcu_torture, rtort_free); 365 } 366 367 /* 368 * Free an element to the rcu_tortures pool. 369 */ 370 static void 371 rcu_torture_free(struct rcu_torture *p) 372 { 373 atomic_inc(&n_rcu_torture_free); 374 spin_lock_bh(&rcu_torture_lock); 375 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 376 spin_unlock_bh(&rcu_torture_lock); 377 } 378 379 /* 380 * Operations vector for selecting different types of tests. 381 */ 382 383 struct rcu_torture_ops { 384 int ttype; 385 void (*init)(void); 386 void (*cleanup)(void); 387 int (*readlock)(void); 388 void (*read_delay)(struct torture_random_state *rrsp, 389 struct rt_read_seg *rtrsp); 390 void (*readunlock)(int idx); 391 int (*readlock_held)(void); // lockdep. 392 int (*readlock_nesting)(void); // actual nesting, if available, -1 if not. 393 int (*down_read)(void); 394 void (*up_read)(int idx); 395 unsigned long (*get_gp_seq)(void); 396 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 397 void (*deferred_free)(struct rcu_torture *p); 398 void (*sync)(void); 399 void (*exp_sync)(void); 400 void (*exp_current)(void); 401 unsigned long (*get_gp_state_exp)(void); 402 unsigned long (*start_gp_poll_exp)(void); 403 void (*start_gp_poll_exp_full)(struct rcu_gp_seq *gsp); 404 bool (*poll_gp_state_exp)(unsigned long oldstate); 405 void (*cond_sync_exp)(unsigned long oldstate); 406 void (*cond_sync_exp_full)(struct rcu_gp_seq *gsp); 407 unsigned long (*get_comp_state)(void); 408 void (*get_comp_state_full)(struct rcu_gp_seq *gsp); 409 bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); 410 bool (*same_gp_state_full)(struct rcu_gp_seq *rgosp1, struct rcu_gp_seq *rgosp2); 411 unsigned long (*get_gp_state)(void); 412 void (*get_gp_state_full)(struct rcu_gp_seq *gsp); 413 unsigned long (*start_gp_poll)(void); 414 void (*start_gp_poll_full)(struct rcu_gp_seq *gsp); 415 bool (*poll_gp_state)(unsigned long oldstate); 416 bool (*poll_gp_state_full)(struct rcu_gp_seq *gsp); 417 bool (*poll_need_2gp)(bool poll, bool poll_full); 418 void (*cond_sync)(unsigned long oldstate); 419 void (*cond_sync_full)(struct rcu_gp_seq *gsp); 420 int poll_active; 421 int poll_active_full; 422 call_rcu_func_t call; 423 void (*cb_barrier)(void); 424 void (*fqs)(void); 425 void (*stats)(void); 426 void (*gp_kthread_dbg)(void); 427 bool (*check_boost_failed)(unsigned long gp_state, int *cpup); 428 int (*stall_dur)(void); 429 void (*get_gp_data)(int *flags, unsigned long *gp_seq); 430 void (*gp_slow_register)(atomic_t *rgssp); 431 void (*gp_slow_unregister)(atomic_t *rgssp); 432 bool (*reader_blocked)(void); 433 unsigned long long (*gather_gp_seqs)(void); 434 void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); 435 void (*set_gpwrap_lag)(unsigned long lag); 436 int (*get_gpwrap_count)(int cpu); 437 bool (*is_task_rcu_boosted)(void); 438 long cbflood_max; 439 int irq_capable; 440 int nmi_capable; 441 int can_boost; 442 int extendables; 443 int slow_gps; 444 int no_pi_lock; 445 int debug_objects; 446 int start_poll_irqsoff; 447 int have_up_down; 448 int rdrs_handle_load; 449 const char *name; 450 }; 451 452 static struct rcu_torture_ops *cur_ops; 453 454 /* 455 * Definitions for rcu torture testing. 456 */ 457 458 static int torture_readlock_not_held(void) 459 { 460 return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); 461 } 462 463 static int rcu_torture_read_lock(void) 464 { 465 rcu_read_lock(); 466 return 0; 467 } 468 469 static void 470 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 471 { 472 unsigned long started; 473 unsigned long completed; 474 const unsigned long shortdelay_us = 200; 475 unsigned long longdelay_ms = 300; 476 unsigned long long ts; 477 478 // If there is a forward-progress test in flight, don't delay. 479 if (atomic_read(&rcu_fwd_cb_nodelay)) 480 return; 481 482 // We want a short delay sometimes to make a reader delay the grace 483 // period, and we want a long delay occasionally to trigger 484 // force_quiescent_state. 485 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 486 started = cur_ops->get_gp_seq(); 487 ts = rcu_trace_clock_local(); 488 if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) 489 longdelay_ms = 5; /* Avoid triggering BH limits. */ 490 mdelay(longdelay_ms); 491 rtrsp->rt_delay_ms = longdelay_ms; 492 completed = cur_ops->get_gp_seq(); 493 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 494 started, completed); 495 } 496 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 497 udelay(shortdelay_us); 498 rtrsp->rt_delay_us = shortdelay_us; 499 } 500 if (!preempt_count() && 501 !(torture_random(rrsp) % (nrealreaders * 500))) 502 torture_preempt_schedule(); /* QS only if preemptible. */ 503 } 504 505 static void rcu_torture_read_unlock(int idx) 506 { 507 rcu_read_unlock(); 508 } 509 510 static int rcu_torture_readlock_nesting(void) 511 { 512 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) 513 return rcu_preempt_depth(); 514 if (IS_ENABLED(CONFIG_PREEMPT_COUNT)) 515 return (preempt_count() & PREEMPT_MASK); 516 return -1; 517 } 518 519 /* 520 * Update callback in the pipe. This should be invoked after a grace period. 521 */ 522 static bool 523 rcu_torture_pipe_update_one(struct rcu_torture *rp) 524 { 525 int i; 526 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); 527 528 if (rtrcp) { 529 WRITE_ONCE(rp->rtort_chkp, NULL); 530 smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). 531 } 532 i = rp->rtort_pipe_count; 533 if (i > RCU_TORTURE_PIPE_LEN) 534 i = RCU_TORTURE_PIPE_LEN; 535 atomic_inc(&rcu_torture_wcount[i]); 536 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 537 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 538 if (i + 1 >= RCU_TORTURE_PIPE_LEN) { 539 rp->rtort_mbtest = 0; 540 return true; 541 } 542 return false; 543 } 544 545 /* 546 * Update all callbacks in the pipe. Suitable for synchronous grace-period 547 * primitives. 548 */ 549 static void 550 rcu_torture_pipe_update(struct rcu_torture *old_rp) 551 { 552 struct rcu_torture *rp; 553 struct rcu_torture *rp1; 554 555 if (old_rp) 556 list_add(&old_rp->rtort_free, &rcu_torture_removed); 557 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 558 if (rcu_torture_pipe_update_one(rp)) { 559 list_del(&rp->rtort_free); 560 rcu_torture_free(rp); 561 } 562 } 563 } 564 565 static void 566 rcu_torture_cb(struct rcu_head *p) 567 { 568 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 569 570 if (torture_must_stop_irq()) { 571 /* Test is ending, just drop callbacks on the floor. */ 572 /* The next initialization will pick up the pieces. */ 573 return; 574 } 575 if (rcu_torture_pipe_update_one(rp)) 576 rcu_torture_free(rp); 577 else 578 cur_ops->deferred_free(rp); 579 } 580 581 static unsigned long rcu_no_completed(void) 582 { 583 return 0; 584 } 585 586 static void rcu_torture_deferred_free(struct rcu_torture *p) 587 { 588 call_rcu(&p->rtort_rcu, rcu_torture_cb); 589 } 590 591 static void rcu_sync_torture_init(void) 592 { 593 INIT_LIST_HEAD(&rcu_torture_removed); 594 } 595 596 static bool rcu_poll_need_2gp(bool poll, bool poll_full) 597 { 598 return poll; 599 } 600 601 static struct rcu_torture_ops rcu_ops = { 602 .ttype = RCU_FLAVOR, 603 .init = rcu_sync_torture_init, 604 .readlock = rcu_torture_read_lock, 605 .read_delay = rcu_read_delay, 606 .readunlock = rcu_torture_read_unlock, 607 .readlock_held = torture_readlock_not_held, 608 .readlock_nesting = rcu_torture_readlock_nesting, 609 .get_gp_seq = rcu_get_gp_seq, 610 .gp_diff = rcu_seq_diff, 611 .deferred_free = rcu_torture_deferred_free, 612 .sync = synchronize_rcu, 613 .exp_sync = synchronize_rcu_expedited, 614 .same_gp_state = same_state_synchronize_rcu, 615 .same_gp_state_full = same_state_synchronize_rcu_full, 616 .get_comp_state = get_completed_synchronize_rcu, 617 .get_comp_state_full = get_completed_synchronize_rcu_full, 618 .get_gp_state = get_state_synchronize_rcu, 619 .get_gp_state_full = get_state_synchronize_rcu_full, 620 .start_gp_poll = start_poll_synchronize_rcu, 621 .start_gp_poll_full = start_poll_synchronize_rcu_full, 622 .poll_gp_state = poll_state_synchronize_rcu, 623 .poll_gp_state_full = poll_state_synchronize_rcu_full, 624 .poll_need_2gp = rcu_poll_need_2gp, 625 .cond_sync = cond_synchronize_rcu, 626 .cond_sync_full = cond_synchronize_rcu_full, 627 .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE, 628 .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, 629 .get_gp_state_exp = get_state_synchronize_rcu, 630 .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, 631 .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, 632 .poll_gp_state_exp = poll_state_synchronize_rcu, 633 .cond_sync_exp = cond_synchronize_rcu_expedited, 634 .cond_sync_exp_full = cond_synchronize_rcu_expedited_full, 635 .call = call_rcu, 636 .cb_barrier = rcu_barrier, 637 .fqs = rcu_force_quiescent_state, 638 .gp_kthread_dbg = show_rcu_gp_kthreads, 639 .check_boost_failed = rcu_check_boost_fail, 640 .stall_dur = rcu_jiffies_till_stall_check, 641 .get_gp_data = rcutorture_get_gp_data, 642 .gp_slow_register = rcu_gp_slow_register, 643 .gp_slow_unregister = rcu_gp_slow_unregister, 644 .reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU) 645 ? has_rcu_reader_blocked 646 : NULL, 647 .gather_gp_seqs = rcutorture_gather_gp_seqs, 648 .format_gp_seqs = rcutorture_format_gp_seqs, 649 .set_gpwrap_lag = rcu_set_gpwrap_lag, 650 .get_gpwrap_count = rcu_get_gpwrap_count, 651 .is_task_rcu_boosted = rcu_is_task_rcu_boosted, 652 .irq_capable = 1, 653 .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), 654 .extendables = RCUTORTURE_MAX_EXTEND, 655 .debug_objects = 1, 656 .start_poll_irqsoff = 1, 657 .rdrs_handle_load = !IS_ENABLED(CONFIG_PREEMPT_RCU) || IS_ENABLED(CONFIG_RCU_BOOST), 658 .nmi_capable = 1, 659 .name = "rcu" 660 }; 661 662 /* 663 * Don't even think about trying any of these in real life!!! 664 * The names includes "busted", and they really means it! 665 * The only purpose of these functions is to provide a buggy RCU 666 * implementation to make sure that rcutorture correctly emits 667 * buggy-RCU error messages. 668 */ 669 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 670 { 671 /* This is a deliberate bug for testing purposes only! */ 672 rcu_torture_cb(&p->rtort_rcu); 673 } 674 675 static void synchronize_rcu_busted(void) 676 { 677 /* This is a deliberate bug for testing purposes only! */ 678 } 679 680 static void 681 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 682 { 683 /* This is a deliberate bug for testing purposes only! */ 684 func(head); 685 } 686 687 static struct rcu_torture_ops rcu_busted_ops = { 688 .ttype = INVALID_RCU_FLAVOR, 689 .init = rcu_sync_torture_init, 690 .readlock = rcu_torture_read_lock, 691 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 692 .readunlock = rcu_torture_read_unlock, 693 .readlock_held = torture_readlock_not_held, 694 .get_gp_seq = rcu_no_completed, 695 .deferred_free = rcu_busted_torture_deferred_free, 696 .sync = synchronize_rcu_busted, 697 .exp_sync = synchronize_rcu_busted, 698 .call = call_rcu_busted, 699 .gather_gp_seqs = rcutorture_gather_gp_seqs, 700 .format_gp_seqs = rcutorture_format_gp_seqs, 701 .irq_capable = 1, 702 .extendables = RCUTORTURE_MAX_EXTEND, 703 .name = "busted" 704 }; 705 706 /* 707 * Definitions for srcu torture testing. 708 */ 709 710 DEFINE_STATIC_SRCU(srcu_ctl); 711 DEFINE_STATIC_SRCU_FAST(srcu_ctlf); 712 DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_ctlfud); 713 DEFINE_STATIC_SRCU_ATOMIC(srcu_ctla); 714 static struct srcu_struct srcu_ctld; 715 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 716 static struct rcu_torture_ops srcud_ops; 717 718 // Restrict APIs permitted for atomic SRCU. 719 static void srcu_torture_init_forbidden_apis(void) 720 { 721 cur_ops->call = NULL; 722 cur_ops->cb_barrier = NULL; 723 cur_ops->deferred_free = NULL; 724 cur_ops->exp_current = NULL; 725 cur_ops->start_gp_poll = NULL; 726 } 727 728 static void srcu_torture_init(void) 729 { 730 rcu_sync_torture_init(); 731 if (!reader_flavor || (reader_flavor & SRCU_READ_FLAVOR_NORMAL)) 732 VERBOSE_TOROUT_STRING("srcu_torture_init normal SRCU"); 733 if (reader_flavor & SRCU_READ_FLAVOR_NMI) 734 VERBOSE_TOROUT_STRING("srcu_torture_init NMI-safe SRCU"); 735 if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 736 srcu_ctlp = &srcu_ctlf; 737 VERBOSE_TOROUT_STRING("srcu_torture_init fast SRCU"); 738 } 739 if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { 740 srcu_ctlp = &srcu_ctlfud; 741 VERBOSE_TOROUT_STRING("srcu_torture_init fast-up/down SRCU"); 742 } 743 if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { 744 srcu_ctlp = &srcu_ctla; 745 VERBOSE_TOROUT_STRING("srcu_torture_init atomic SRCU"); 746 srcu_torture_init_forbidden_apis(); 747 } 748 } 749 750 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) 751 { 752 srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq); 753 } 754 755 static int srcu_torture_read_lock(void) 756 { 757 int idx; 758 struct srcu_ctr __percpu *scp; 759 int ret = 0; 760 761 WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL); 762 763 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) { 764 idx = srcu_read_lock(srcu_ctlp); 765 WARN_ON_ONCE(idx & ~0x1); 766 ret += idx; 767 } 768 if (reader_flavor & SRCU_READ_FLAVOR_NMI) { 769 idx = srcu_read_lock_nmisafe(srcu_ctlp); 770 WARN_ON_ONCE(idx & ~0x1); 771 ret += idx << 1; 772 } 773 if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 774 scp = srcu_read_lock_fast(srcu_ctlp); 775 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 776 WARN_ON_ONCE(idx & ~0x1); 777 ret += idx << 2; 778 } 779 if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { 780 scp = srcu_read_lock_fast_updown(srcu_ctlp); 781 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 782 WARN_ON_ONCE(idx & ~0x1); 783 ret += idx << 3; 784 } 785 if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { 786 idx = srcu_read_lock_atomic(srcu_ctlp); 787 WARN_ON_ONCE(idx & ~0x1); 788 ret += idx << 4; 789 } 790 return ret; 791 } 792 793 static void 794 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 795 { 796 long delay; 797 const long uspertick = 1000000 / HZ; 798 const long longdelay = 10; 799 800 // If there is a forward-progress test in flight, don't delay. 801 if (atomic_read(&rcu_fwd_cb_nodelay)) 802 return; 803 804 // We want there to be long-running readers, but not all the time. 805 // The !rcu_preempt_depth() is for RCU Tasks Trace. 806 807 delay = torture_random(rrsp) % 808 (nrealreaders * 2 * longdelay * uspertick); 809 if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled() && 810 !(reader_flavor & SRCU_READ_FLAVOR_ATOMIC)) { 811 schedule_timeout_interruptible(longdelay); 812 rtrsp->rt_delay_jiffies = longdelay; 813 } else { 814 rcu_read_delay(rrsp, rtrsp); 815 } 816 } 817 818 static void srcu_torture_read_unlock(int idx) 819 { 820 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 821 if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) 822 srcu_read_unlock_atomic(srcu_ctlp, (idx & 0x10) >> 4); 823 if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) 824 srcu_read_unlock_fast_updown(srcu_ctlp, 825 __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); 826 if (reader_flavor & SRCU_READ_FLAVOR_FAST) 827 srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x4) >> 2)); 828 if (reader_flavor & SRCU_READ_FLAVOR_NMI) 829 srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1); 830 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) 831 srcu_read_unlock(srcu_ctlp, idx & 0x1); 832 } 833 834 static int torture_srcu_read_lock_held(void) 835 { 836 return srcu_read_lock_held(srcu_ctlp); 837 } 838 839 static bool srcu_torture_have_up_down(void) 840 { 841 int rf = reader_flavor; 842 843 if (!rf) 844 rf = SRCU_READ_FLAVOR_NORMAL; 845 return !!(cur_ops->have_up_down & rf); 846 } 847 848 static int srcu_torture_down_read(void) 849 { 850 int idx; 851 struct srcu_ctr __percpu *scp; 852 853 WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL); 854 WARN_ON_ONCE(reader_flavor & (reader_flavor - 1)); 855 856 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) { 857 idx = srcu_down_read(srcu_ctlp); 858 WARN_ON_ONCE(idx & ~0x1); 859 return idx; 860 } 861 if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { 862 scp = srcu_down_read_fast(srcu_ctlp); 863 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 864 WARN_ON_ONCE(idx & ~0x1); 865 return idx << 3; 866 } 867 WARN_ON_ONCE(1); 868 return 0; 869 } 870 871 static void srcu_torture_up_read(int idx) 872 { 873 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 874 if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) 875 srcu_up_read_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); 876 else if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || 877 !(reader_flavor & SRCU_READ_FLAVOR_ALL)) 878 srcu_up_read(srcu_ctlp, idx & 0x1); 879 else 880 WARN_ON_ONCE(1); 881 } 882 883 static unsigned long srcu_torture_completed(void) 884 { 885 return srcu_batches_completed(srcu_ctlp); 886 } 887 888 static void srcu_torture_deferred_free(struct rcu_torture *rp) 889 { 890 unsigned long flags; 891 bool lockit = jiffies & 0x1; 892 893 if (lockit) 894 raw_spin_lock_irqsave(¤t->pi_lock, flags); 895 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 896 if (lockit) 897 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 898 } 899 900 static void srcu_torture_synchronize(void) 901 { 902 if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) 903 synchronize_srcu_atomic(srcu_ctlp); 904 else 905 synchronize_srcu(srcu_ctlp); 906 } 907 908 static unsigned long srcu_torture_get_gp_state(void) 909 { 910 return get_state_synchronize_srcu(srcu_ctlp); 911 } 912 913 static unsigned long srcu_torture_start_gp_poll(void) 914 { 915 return start_poll_synchronize_srcu(srcu_ctlp); 916 } 917 918 static bool srcu_torture_poll_gp_state(unsigned long oldstate) 919 { 920 return poll_state_synchronize_srcu(srcu_ctlp, oldstate); 921 } 922 923 static void srcu_torture_call(struct rcu_head *head, 924 rcu_callback_t func) 925 { 926 call_srcu(srcu_ctlp, head, func); 927 } 928 929 static void srcu_torture_barrier(void) 930 { 931 srcu_barrier(srcu_ctlp); 932 } 933 934 static void srcu_torture_stats(void) 935 { 936 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 937 } 938 939 static void srcu_torture_synchronize_expedited(void) 940 { 941 if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) 942 synchronize_srcu_atomic(srcu_ctlp); 943 else 944 synchronize_srcu_expedited(srcu_ctlp); 945 } 946 947 static void srcu_torture_expedite_current(void) 948 { 949 srcu_expedite_current(srcu_ctlp); 950 } 951 952 static struct rcu_torture_ops srcu_ops = { 953 .ttype = SRCU_FLAVOR, 954 .init = srcu_torture_init, 955 .readlock = srcu_torture_read_lock, 956 .read_delay = srcu_read_delay, 957 .readunlock = srcu_torture_read_unlock, 958 .down_read = srcu_torture_down_read, 959 .up_read = srcu_torture_up_read, 960 .readlock_held = torture_srcu_read_lock_held, 961 .get_gp_seq = srcu_torture_completed, 962 .gp_diff = rcu_seq_diff, 963 .deferred_free = srcu_torture_deferred_free, 964 .sync = srcu_torture_synchronize, 965 .exp_sync = srcu_torture_synchronize_expedited, 966 .exp_current = srcu_torture_expedite_current, 967 .same_gp_state = same_state_synchronize_srcu, 968 .get_comp_state = get_completed_synchronize_srcu, 969 .get_gp_state = srcu_torture_get_gp_state, 970 .start_gp_poll = srcu_torture_start_gp_poll, 971 .poll_gp_state = srcu_torture_poll_gp_state, 972 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 973 .call = srcu_torture_call, 974 .cb_barrier = srcu_torture_barrier, 975 .stats = srcu_torture_stats, 976 .get_gp_data = srcu_get_gp_data, 977 .cbflood_max = 50000, 978 .irq_capable = 1, 979 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 980 .debug_objects = 1, 981 .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) 982 ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, 983 .nmi_capable = 1, 984 .name = "srcu" 985 }; 986 987 static void srcud_torture_init(void) 988 { 989 rcu_sync_torture_init(); 990 if (!reader_flavor || (reader_flavor & SRCU_READ_FLAVOR_NORMAL)) { 991 WARN_ON(init_srcu_struct(&srcu_ctld)); 992 VERBOSE_TOROUT_STRING("srcud_torture_init normal SRCU"); 993 } else if (reader_flavor & SRCU_READ_FLAVOR_NMI) { 994 WARN_ON(init_srcu_struct(&srcu_ctld)); 995 VERBOSE_TOROUT_STRING("srcud_torture_init NMI-safe SRCU"); 996 } else if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 997 WARN_ON(init_srcu_struct_fast(&srcu_ctld)); 998 VERBOSE_TOROUT_STRING("srcud_torture_init fast SRCU"); 999 } else if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { 1000 WARN_ON(init_srcu_struct_fast_updown(&srcu_ctld)); 1001 VERBOSE_TOROUT_STRING("srcud_torture_init fast-up/down SRCU"); 1002 } else if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { 1003 WARN_ON(init_srcu_struct_atomic(&srcu_ctld)); 1004 VERBOSE_TOROUT_STRING("srcud_torture_init atomic SRCU"); 1005 srcu_torture_init_forbidden_apis(); 1006 } else { 1007 WARN_ON(init_srcu_struct(&srcu_ctld)); 1008 } 1009 srcu_ctlp = &srcu_ctld; 1010 } 1011 1012 static void srcu_torture_cleanup(void) 1013 { 1014 cleanup_srcu_struct(&srcu_ctld); 1015 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 1016 } 1017 1018 /* As above, but dynamically allocated. */ 1019 static struct rcu_torture_ops srcud_ops = { 1020 .ttype = SRCU_FLAVOR, 1021 .init = srcud_torture_init, 1022 .cleanup = srcu_torture_cleanup, 1023 .readlock = srcu_torture_read_lock, 1024 .read_delay = srcu_read_delay, 1025 .readunlock = srcu_torture_read_unlock, 1026 .readlock_held = torture_srcu_read_lock_held, 1027 .down_read = srcu_torture_down_read, 1028 .up_read = srcu_torture_up_read, 1029 .get_gp_seq = srcu_torture_completed, 1030 .gp_diff = rcu_seq_diff, 1031 .deferred_free = srcu_torture_deferred_free, 1032 .sync = srcu_torture_synchronize, 1033 .exp_sync = srcu_torture_synchronize_expedited, 1034 .exp_current = srcu_torture_expedite_current, 1035 .same_gp_state = same_state_synchronize_srcu, 1036 .get_comp_state = get_completed_synchronize_srcu, 1037 .get_gp_state = srcu_torture_get_gp_state, 1038 .start_gp_poll = srcu_torture_start_gp_poll, 1039 .poll_gp_state = srcu_torture_poll_gp_state, 1040 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 1041 .call = srcu_torture_call, 1042 .cb_barrier = srcu_torture_barrier, 1043 .stats = srcu_torture_stats, 1044 .get_gp_data = srcu_get_gp_data, 1045 .cbflood_max = 50000, 1046 .irq_capable = 1, 1047 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 1048 .debug_objects = 1, 1049 .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) 1050 ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, 1051 .nmi_capable = 1, 1052 .name = "srcud" 1053 }; 1054 1055 /* As above, but broken due to inappropriate reader extension. */ 1056 static struct rcu_torture_ops busted_srcud_ops = { 1057 .ttype = SRCU_FLAVOR, 1058 .init = srcu_torture_init, 1059 .cleanup = srcu_torture_cleanup, 1060 .readlock = srcu_torture_read_lock, 1061 .read_delay = rcu_read_delay, 1062 .readunlock = srcu_torture_read_unlock, 1063 .readlock_held = torture_srcu_read_lock_held, 1064 .get_gp_seq = srcu_torture_completed, 1065 .deferred_free = srcu_torture_deferred_free, 1066 .sync = srcu_torture_synchronize, 1067 .exp_sync = srcu_torture_synchronize_expedited, 1068 .call = srcu_torture_call, 1069 .cb_barrier = srcu_torture_barrier, 1070 .stats = srcu_torture_stats, 1071 .irq_capable = 1, 1072 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 1073 .extendables = RCUTORTURE_MAX_EXTEND, 1074 .name = "busted_srcud" 1075 }; 1076 1077 /* 1078 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 1079 * This implementation does not work well with CPU hotplug nor 1080 * with rcutorture's shuffling. 1081 */ 1082 1083 static void synchronize_rcu_trivial(void) 1084 { 1085 int cpu; 1086 1087 for_each_online_cpu(cpu) { 1088 torture_sched_setaffinity(current->pid, cpumask_of(cpu), true); 1089 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 1090 } 1091 } 1092 1093 static void rcu_sync_torture_init_trivial(void) 1094 { 1095 rcu_sync_torture_init(); 1096 // if (onoff_interval || shuffle_interval) { 1097 if (WARN_ONCE(onoff_interval || shuffle_interval, "%s: Non-zero onoff_interval (%d) or shuffle_interval (%d) breaks trivial RCU, resetting to zero", __func__, onoff_interval, shuffle_interval)) { 1098 onoff_interval = 0; 1099 shuffle_interval = 0; 1100 } 1101 } 1102 1103 static int rcu_torture_read_lock_trivial(void) 1104 { 1105 preempt_disable(); 1106 return 0; 1107 } 1108 1109 static void rcu_torture_read_unlock_trivial(int idx) 1110 { 1111 preempt_enable(); 1112 } 1113 1114 static struct rcu_torture_ops trivial_ops = { 1115 .ttype = RCU_TRIVIAL_FLAVOR, 1116 .init = rcu_sync_torture_init_trivial, 1117 .readlock = rcu_torture_read_lock_trivial, 1118 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1119 .readunlock = rcu_torture_read_unlock_trivial, 1120 .readlock_held = torture_readlock_not_held, 1121 .get_gp_seq = rcu_no_completed, 1122 .sync = synchronize_rcu_trivial, 1123 .exp_sync = synchronize_rcu_trivial, 1124 .irq_capable = 1, 1125 .name = "trivial" 1126 }; 1127 1128 #ifdef CONFIG_TRIVIAL_PREEMPT_RCU 1129 1130 /* 1131 * Definitions for trivial CONFIG_PREEMPT=y torture testing. This 1132 * implementation does not work well with large numbers of tasks or with 1133 * long-term preemption. Either or both get you RCU CPU stall warnings. 1134 */ 1135 1136 static void rcu_sync_torture_init_trivial_preempt(void) 1137 { 1138 rcu_sync_torture_init(); 1139 if (WARN_ONCE(onoff_interval || shuffle_interval, "%s: Non-zero onoff_interval (%d) or shuffle_interval (%d) breaks trivial RCU, resetting to zero", __func__, onoff_interval, shuffle_interval)) { 1140 onoff_interval = 0; 1141 shuffle_interval = 0; 1142 } 1143 } 1144 1145 static int rcu_torture_read_lock_trivial_preempt(void) 1146 { 1147 struct task_struct *t = current; 1148 1149 WRITE_ONCE(t->rcu_trivial_preempt_nesting, t->rcu_trivial_preempt_nesting + 1); 1150 smp_mb(); 1151 return 0; 1152 } 1153 1154 static void rcu_torture_read_unlock_trivial_preempt(int idx) 1155 { 1156 struct task_struct *t = current; 1157 1158 smp_store_release(&t->rcu_trivial_preempt_nesting, t->rcu_trivial_preempt_nesting - 1); 1159 } 1160 1161 static struct rcu_torture_ops trivial_preempt_ops = { 1162 .ttype = RCU_TRIVIAL_FLAVOR, 1163 .init = rcu_sync_torture_init_trivial_preempt, 1164 .readlock = rcu_torture_read_lock_trivial_preempt, 1165 .read_delay = rcu_read_delay, // just reuse rcu's version. 1166 .readunlock = rcu_torture_read_unlock_trivial_preempt, 1167 .readlock_held = torture_readlock_not_held, 1168 .get_gp_seq = rcu_no_completed, 1169 .sync = synchronize_rcu_trivial_preempt, 1170 .exp_sync = synchronize_rcu_trivial_preempt, 1171 .irq_capable = 0, // In theory it should be, but let's keep it trivial. 1172 .name = "trivial-preempt" 1173 }; 1174 1175 #define TRIVIAL_PREEMPT_OPS &trivial_preempt_ops, 1176 1177 #else // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU 1178 1179 #define TRIVIAL_PREEMPT_OPS 1180 1181 #endif // #else // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU 1182 1183 #ifdef CONFIG_TASKS_RCU 1184 1185 /* 1186 * Definitions for RCU-tasks torture testing. 1187 */ 1188 1189 static int tasks_torture_read_lock(void) 1190 { 1191 return 0; 1192 } 1193 1194 static void tasks_torture_read_unlock(int idx) 1195 { 1196 } 1197 1198 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 1199 { 1200 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 1201 } 1202 1203 static void synchronize_rcu_mult_test(void) 1204 { 1205 synchronize_rcu_mult(call_rcu_tasks, call_rcu); 1206 } 1207 1208 static struct rcu_torture_ops tasks_ops = { 1209 .ttype = RCU_TASKS_FLAVOR, 1210 .init = rcu_sync_torture_init, 1211 .readlock = tasks_torture_read_lock, 1212 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1213 .readunlock = tasks_torture_read_unlock, 1214 .get_gp_seq = rcu_no_completed, 1215 .deferred_free = rcu_tasks_torture_deferred_free, 1216 .sync = synchronize_rcu_tasks, 1217 .exp_sync = synchronize_rcu_mult_test, 1218 .call = call_rcu_tasks, 1219 .cb_barrier = rcu_barrier_tasks, 1220 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, 1221 .get_gp_data = rcu_tasks_get_gp_data, 1222 .irq_capable = 1, 1223 .slow_gps = 1, 1224 .name = "tasks" 1225 }; 1226 1227 #define TASKS_OPS &tasks_ops, 1228 1229 #else // #ifdef CONFIG_TASKS_RCU 1230 1231 #define TASKS_OPS 1232 1233 #endif // #else #ifdef CONFIG_TASKS_RCU 1234 1235 1236 #ifdef CONFIG_TASKS_RUDE_RCU 1237 1238 /* 1239 * Definitions for rude RCU-tasks torture testing. 1240 */ 1241 1242 static struct rcu_torture_ops tasks_rude_ops = { 1243 .ttype = RCU_TASKS_RUDE_FLAVOR, 1244 .init = rcu_sync_torture_init, 1245 .readlock = rcu_torture_read_lock_trivial, 1246 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1247 .readunlock = rcu_torture_read_unlock_trivial, 1248 .get_gp_seq = rcu_no_completed, 1249 .sync = synchronize_rcu_tasks_rude, 1250 .exp_sync = synchronize_rcu_tasks_rude, 1251 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, 1252 .get_gp_data = rcu_tasks_rude_get_gp_data, 1253 .cbflood_max = 50000, 1254 .irq_capable = 1, 1255 .name = "tasks-rude" 1256 }; 1257 1258 #define TASKS_RUDE_OPS &tasks_rude_ops, 1259 1260 #else // #ifdef CONFIG_TASKS_RUDE_RCU 1261 1262 #define TASKS_RUDE_OPS 1263 1264 #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU 1265 1266 1267 #ifdef CONFIG_TASKS_TRACE_RCU 1268 1269 /* 1270 * Definitions for tracing RCU-tasks torture testing. 1271 */ 1272 1273 // Note that an RCU Tasks Trace GP must imply an RCU GP. 1274 static int tasks_tracing_torture_read_lock(void) 1275 { 1276 int use_rcu = !(jiffies & 0xff); 1277 1278 if (use_rcu) 1279 rcu_read_lock(); 1280 else 1281 rcu_read_lock_trace(); 1282 return use_rcu; 1283 } 1284 1285 static void tasks_tracing_torture_read_unlock(int use_rcu) 1286 { 1287 if (use_rcu) 1288 rcu_read_unlock(); 1289 else 1290 rcu_read_unlock_trace(); 1291 } 1292 1293 static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) 1294 { 1295 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); 1296 } 1297 1298 static struct rcu_torture_ops tasks_tracing_ops = { 1299 .ttype = RCU_TASKS_TRACING_FLAVOR, 1300 .init = rcu_sync_torture_init, 1301 .readlock = tasks_tracing_torture_read_lock, 1302 .read_delay = srcu_read_delay, /* just reuse srcu's version. */ 1303 .readunlock = tasks_tracing_torture_read_unlock, 1304 .readlock_held = rcu_read_lock_trace_held, 1305 .get_gp_seq = rcu_tasks_trace_batches_completed, 1306 .gp_diff = rcu_seq_diff, 1307 .deferred_free = rcu_tasks_tracing_torture_deferred_free, 1308 .sync = synchronize_rcu_tasks_trace, 1309 .exp_sync = synchronize_rcu_tasks_trace, 1310 .exp_current = rcu_tasks_trace_expedite_current, 1311 .call = call_rcu_tasks_trace, 1312 .cb_barrier = rcu_barrier_tasks_trace, 1313 .cbflood_max = 50000, 1314 .irq_capable = 1, 1315 .slow_gps = 1, 1316 .nmi_capable = 1, 1317 .name = "tasks-tracing" 1318 }; 1319 1320 #define TASKS_TRACING_OPS &tasks_tracing_ops, 1321 1322 #else // #ifdef CONFIG_TASKS_TRACE_RCU 1323 1324 #define TASKS_TRACING_OPS 1325 1326 #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU 1327 1328 1329 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 1330 { 1331 if (!cur_ops->gp_diff) 1332 return new - old; 1333 return cur_ops->gp_diff(new, old); 1334 } 1335 1336 /* 1337 * RCU torture priority-boost testing. Runs one real-time thread per 1338 * CPU for moderate bursts, repeatedly starting grace periods and waiting 1339 * for them to complete. If a given grace period takes too long, we assume 1340 * that priority inversion has occurred. 1341 */ 1342 1343 static int old_rt_runtime = -1; 1344 1345 static void rcu_torture_disable_rt_throttle(void) 1346 { 1347 /* 1348 * Disable RT throttling so that rcutorture's boost threads don't get 1349 * throttled. Only possible if rcutorture is built-in otherwise the 1350 * user should manually do this by setting the sched_rt_period_us and 1351 * sched_rt_runtime sysctls. 1352 */ 1353 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 1354 return; 1355 1356 old_rt_runtime = sysctl_sched_rt_runtime; 1357 sysctl_sched_rt_runtime = -1; 1358 } 1359 1360 static void rcu_torture_enable_rt_throttle(void) 1361 { 1362 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 1363 return; 1364 1365 sysctl_sched_rt_runtime = old_rt_runtime; 1366 old_rt_runtime = -1; 1367 } 1368 1369 static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) 1370 { 1371 int cpu; 1372 static int dbg_done; 1373 unsigned long end = jiffies; 1374 bool gp_done; 1375 unsigned long j; 1376 static unsigned long last_persist; 1377 unsigned long lp; 1378 unsigned long mininterval = test_boost_duration * HZ - HZ / 2; 1379 1380 if (end - *start > mininterval) { 1381 // Recheck after checking time to avoid false positives. 1382 smp_mb(); // Time check before grace-period check. 1383 if (cur_ops->poll_gp_state(gp_state)) 1384 return false; // passed, though perhaps just barely 1385 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { 1386 // At most one persisted message per boost test. 1387 j = jiffies; 1388 lp = READ_ONCE(last_persist); 1389 if (time_after(j, lp + mininterval) && 1390 cmpxchg(&last_persist, lp, j) == lp) { 1391 if (cpu < 0) 1392 pr_info("Boost inversion persisted: QS from all CPUs\n"); 1393 else 1394 pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); 1395 } 1396 return false; // passed on a technicality 1397 } 1398 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 1399 n_rcu_torture_boost_failure++; 1400 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { 1401 pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", 1402 current->rt_priority, gp_state, end - *start); 1403 cur_ops->gp_kthread_dbg(); 1404 // Recheck after print to flag grace period ending during splat. 1405 gp_done = cur_ops->poll_gp_state(gp_state); 1406 pr_info("Boost inversion: GP %lu %s.\n", gp_state, 1407 gp_done ? "ended already" : "still pending"); 1408 1409 } 1410 1411 return true; // failed 1412 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { 1413 *start = jiffies; 1414 } 1415 1416 return false; // passed 1417 } 1418 1419 static int rcu_torture_boost(void *arg) 1420 { 1421 unsigned long endtime; 1422 unsigned long gp_state; 1423 unsigned long gp_state_time; 1424 unsigned long oldstarttime; 1425 unsigned long booststarttime = get_torture_init_jiffies() + test_boost_holdoff * HZ; 1426 1427 if (test_boost_holdoff <= 0 || time_after(jiffies, booststarttime)) { 1428 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 1429 } else { 1430 VERBOSE_TOROUT_STRING("rcu_torture_boost started holdoff period"); 1431 while (time_before(jiffies, booststarttime)) { 1432 schedule_timeout_idle(HZ); 1433 if (kthread_should_stop()) 1434 goto cleanup; 1435 } 1436 VERBOSE_TOROUT_STRING("rcu_torture_boost finished holdoff period"); 1437 } 1438 1439 /* Set real-time priority. */ 1440 sched_set_fifo_low(current); 1441 1442 /* Each pass through the following loop does one boost-test cycle. */ 1443 do { 1444 bool failed = false; // Test failed already in this test interval 1445 bool gp_initiated = false; 1446 1447 if (kthread_should_stop()) 1448 goto checkwait; 1449 1450 /* Wait for the next test interval. */ 1451 oldstarttime = READ_ONCE(boost_starttime); 1452 while (time_before(jiffies, oldstarttime)) { 1453 schedule_timeout_interruptible(oldstarttime - jiffies); 1454 if (stutter_wait("rcu_torture_boost")) 1455 sched_set_fifo_low(current); 1456 if (torture_must_stop()) 1457 goto checkwait; 1458 } 1459 1460 // Do one boost-test interval. 1461 endtime = oldstarttime + test_boost_duration * HZ; 1462 while (time_before(jiffies, endtime)) { 1463 // Has current GP gone too long? 1464 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1465 failed = rcu_torture_boost_failed(gp_state, &gp_state_time); 1466 // If we don't have a grace period in flight, start one. 1467 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { 1468 gp_state = cur_ops->start_gp_poll(); 1469 gp_initiated = true; 1470 gp_state_time = jiffies; 1471 } 1472 if (stutter_wait("rcu_torture_boost")) { 1473 sched_set_fifo_low(current); 1474 // If the grace period already ended, 1475 // we don't know when that happened, so 1476 // start over. 1477 if (cur_ops->poll_gp_state(gp_state)) 1478 gp_initiated = false; 1479 } 1480 if (torture_must_stop()) 1481 goto checkwait; 1482 } 1483 1484 // In case the grace period extended beyond the end of the loop. 1485 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1486 rcu_torture_boost_failed(gp_state, &gp_state_time); 1487 1488 /* 1489 * Set the start time of the next test interval. 1490 * Yes, this is vulnerable to long delays, but such 1491 * delays simply cause a false negative for the next 1492 * interval. Besides, we are running at RT priority, 1493 * so delays should be relatively rare. 1494 */ 1495 while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { 1496 if (mutex_trylock(&boost_mutex)) { 1497 if (oldstarttime == boost_starttime) { 1498 WRITE_ONCE(boost_starttime, 1499 jiffies + test_boost_interval * HZ); 1500 n_rcu_torture_boosts++; 1501 } 1502 mutex_unlock(&boost_mutex); 1503 break; 1504 } 1505 schedule_timeout_uninterruptible(HZ / 20); 1506 } 1507 1508 /* Go do the stutter. */ 1509 checkwait: if (stutter_wait("rcu_torture_boost")) 1510 sched_set_fifo_low(current); 1511 } while (!torture_must_stop()); 1512 1513 cleanup: 1514 /* Clean up and exit. */ 1515 while (!kthread_should_stop()) { 1516 torture_shutdown_absorb("rcu_torture_boost"); 1517 schedule_timeout_uninterruptible(HZ / 20); 1518 } 1519 torture_kthread_stopping("rcu_torture_boost"); 1520 return 0; 1521 } 1522 1523 /* 1524 * RCU torture force-quiescent-state kthread. Repeatedly induces 1525 * bursts of calls to force_quiescent_state(), increasing the probability 1526 * of occurrence of some important types of race conditions. 1527 */ 1528 static int 1529 rcu_torture_fqs(void *arg) 1530 { 1531 unsigned long fqs_resume_time; 1532 int fqs_burst_remaining; 1533 int oldnice = task_nice(current); 1534 1535 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 1536 do { 1537 fqs_resume_time = jiffies + fqs_stutter * HZ; 1538 while (time_before(jiffies, fqs_resume_time) && 1539 !kthread_should_stop()) { 1540 schedule_timeout_interruptible(HZ / 20); 1541 } 1542 fqs_burst_remaining = fqs_duration; 1543 while (fqs_burst_remaining > 0 && 1544 !kthread_should_stop()) { 1545 cur_ops->fqs(); 1546 udelay(fqs_holdoff); 1547 fqs_burst_remaining -= fqs_holdoff; 1548 } 1549 if (stutter_wait("rcu_torture_fqs")) 1550 sched_set_normal(current, oldnice); 1551 } while (!torture_must_stop()); 1552 torture_kthread_stopping("rcu_torture_fqs"); 1553 return 0; 1554 } 1555 1556 // Used by writers to randomly choose from the available grace-period primitives. 1557 static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; 1558 static int nsynctypes; 1559 1560 /* 1561 * Determine which grace-period primitives are available. 1562 */ 1563 static void rcu_torture_write_types(void) 1564 { 1565 bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; 1566 bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; 1567 bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; 1568 bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; 1569 1570 /* Initialize synctype[] array. If none set, take default. */ 1571 if (!gp_cond1 && 1572 !gp_cond_exp1 && 1573 !gp_cond_full1 && 1574 !gp_cond_exp_full1 && 1575 !gp_exp1 && 1576 !gp_poll_exp1 && 1577 !gp_poll_exp_full1 && 1578 !gp_normal1 && 1579 !gp_poll1 && 1580 !gp_poll_full1 && 1581 !gp_sync1) { 1582 gp_cond1 = true; 1583 gp_cond_exp1 = true; 1584 gp_cond_full1 = true; 1585 gp_cond_exp_full1 = true; 1586 gp_exp1 = true; 1587 gp_poll_exp1 = true; 1588 gp_poll_exp_full1 = true; 1589 gp_normal1 = true; 1590 gp_poll1 = true; 1591 gp_poll_full1 = true; 1592 gp_sync1 = true; 1593 } 1594 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { 1595 synctype[nsynctypes++] = RTWS_COND_GET; 1596 pr_info("%s: Testing conditional GPs.\n", __func__); 1597 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { 1598 pr_alert("%s: gp_cond without primitives.\n", __func__); 1599 } 1600 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { 1601 synctype[nsynctypes++] = RTWS_COND_GET_EXP; 1602 pr_info("%s: Testing conditional expedited GPs.\n", __func__); 1603 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { 1604 pr_alert("%s: gp_cond_exp without primitives.\n", __func__); 1605 } 1606 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { 1607 synctype[nsynctypes++] = RTWS_COND_GET_FULL; 1608 pr_info("%s: Testing conditional full-state GPs.\n", __func__); 1609 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { 1610 pr_alert("%s: gp_cond_full without primitives.\n", __func__); 1611 } 1612 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { 1613 synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; 1614 pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); 1615 } else if (gp_cond_exp_full && 1616 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { 1617 pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); 1618 } 1619 if (gp_exp1 && cur_ops->exp_sync) { 1620 synctype[nsynctypes++] = RTWS_EXP_SYNC; 1621 pr_info("%s: Testing expedited GPs.\n", __func__); 1622 } else if (gp_exp && !cur_ops->exp_sync) { 1623 pr_alert("%s: gp_exp without primitives.\n", __func__); 1624 } 1625 if (gp_normal1 && cur_ops->deferred_free) { 1626 synctype[nsynctypes++] = RTWS_DEF_FREE; 1627 pr_info("%s: Testing asynchronous GPs.\n", __func__); 1628 } else if (gp_normal && !cur_ops->deferred_free) { 1629 pr_alert("%s: gp_normal without primitives.\n", __func__); 1630 } 1631 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state && 1632 cur_ops->start_gp_poll && cur_ops->poll_gp_state) { 1633 synctype[nsynctypes++] = RTWS_POLL_GET; 1634 pr_info("%s: Testing polling GPs.\n", __func__); 1635 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { 1636 pr_alert("%s: gp_poll without primitives.\n", __func__); 1637 } 1638 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full 1639 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { 1640 synctype[nsynctypes++] = RTWS_POLL_GET_FULL; 1641 pr_info("%s: Testing polling full-state GPs.\n", __func__); 1642 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { 1643 pr_alert("%s: gp_poll_full without primitives.\n", __func__); 1644 } 1645 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { 1646 synctype[nsynctypes++] = RTWS_POLL_GET_EXP; 1647 pr_info("%s: Testing polling expedited GPs.\n", __func__); 1648 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { 1649 pr_alert("%s: gp_poll_exp without primitives.\n", __func__); 1650 } 1651 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { 1652 synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; 1653 pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); 1654 } else if (gp_poll_exp_full && 1655 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { 1656 pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); 1657 } 1658 if (gp_sync1 && cur_ops->sync) { 1659 synctype[nsynctypes++] = RTWS_SYNC; 1660 pr_info("%s: Testing normal GPs.\n", __func__); 1661 } else if (gp_sync && !cur_ops->sync) { 1662 pr_alert("%s: gp_sync without primitives.\n", __func__); 1663 } 1664 pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes); 1665 pr_info("%s: gp_cond_wi %d gp_cond_wi_exp %d gp_poll_wi %d gp_poll_wi_exp %d\n", __func__, gp_cond_wi, gp_cond_wi_exp, gp_poll_wi, gp_poll_wi_exp); 1666 } 1667 1668 /* 1669 * Do the specified rcu_torture_writer() synchronous grace period, 1670 * while also testing out the polled APIs. Note well that the single-CPU 1671 * grace-period optimizations must be accounted for. 1672 */ 1673 static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) 1674 { 1675 unsigned long cookie; 1676 struct rcu_gp_seq cookie_full; 1677 bool dopoll; 1678 bool dopoll_full; 1679 unsigned long r = torture_random(trsp); 1680 1681 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); 1682 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); 1683 if (dopoll || dopoll_full) 1684 cpus_read_lock(); 1685 if (dopoll) 1686 cookie = cur_ops->get_gp_state(); 1687 if (dopoll_full) 1688 cur_ops->get_gp_state_full(&cookie_full); 1689 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) 1690 sync(); 1691 sync(); 1692 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), 1693 "%s: Cookie check 3 failed %pS() online %*pbl.", 1694 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1695 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), 1696 "%s: Cookie check 4 failed %pS() online %*pbl", 1697 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1698 if (dopoll || dopoll_full) 1699 cpus_read_unlock(); 1700 } 1701 1702 /* 1703 * Do an rcu_barrier() to motivate lazy callbacks during a stutter 1704 * pause. Without this, we can get false-positives rtort_pipe_count 1705 * splats. 1706 */ 1707 static void rcu_torture_writer_work(struct work_struct *work) 1708 { 1709 if (cur_ops->cb_barrier) 1710 cur_ops->cb_barrier(); 1711 } 1712 1713 /* 1714 * RCU torture writer kthread. Repeatedly substitutes a new structure 1715 * for that pointed to by rcu_torture_current, freeing the old structure 1716 * after a series of grace periods (the "pipeline"). 1717 */ 1718 static int 1719 rcu_torture_writer(void *arg) 1720 { 1721 bool booting_still = false; 1722 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 1723 unsigned long cookie; 1724 struct rcu_gp_seq cookie_full; 1725 int expediting = 0; 1726 unsigned long gp_snap; 1727 unsigned long gp_snap1; 1728 struct rcu_gp_seq gp_snap_full; 1729 struct rcu_gp_seq gp_snap1_full; 1730 int i; 1731 int idx; 1732 unsigned long j; 1733 struct work_struct lazy_work; 1734 int oldnice = task_nice(current); 1735 struct rcu_gp_seq *rgo = NULL; 1736 int rgo_size = 0; 1737 struct rcu_torture *rp; 1738 struct rcu_torture *old_rp; 1739 static DEFINE_TORTURE_RANDOM(rand); 1740 unsigned long stallsdone = jiffies; 1741 bool stutter_waited; 1742 unsigned long *ulo = NULL; 1743 int ulo_size = 0; 1744 1745 // If a new stall test is added, this must be adjusted. 1746 if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu) 1747 stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * 1748 HZ * (stall_cpu_repeat + 1); 1749 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 1750 if (!can_expedite) 1751 pr_alert("%s" TORTURE_FLAG 1752 " GP expediting controlled from boot/sysfs for %s.\n", 1753 torture_type, cur_ops->name); 1754 if (WARN_ONCE(nsynctypes == 0, 1755 "%s: No update-side primitives.\n", __func__)) { 1756 /* 1757 * No updates primitives, so don't try updating. 1758 * The resulting test won't be testing much, hence the 1759 * above WARN_ONCE(). 1760 */ 1761 rcu_torture_writer_state = RTWS_STOPPING; 1762 torture_kthread_stopping("rcu_torture_writer"); 1763 return 0; 1764 } 1765 if (cur_ops->poll_active > 0) { 1766 ulo = kcalloc(cur_ops->poll_active, sizeof(*ulo), GFP_KERNEL); 1767 if (!WARN_ON(!ulo)) 1768 ulo_size = cur_ops->poll_active; 1769 } 1770 if (cur_ops->poll_active_full > 0) { 1771 rgo = kzalloc_objs(*rgo, cur_ops->poll_active_full); 1772 if (!WARN_ON(!rgo)) 1773 rgo_size = cur_ops->poll_active_full; 1774 } 1775 1776 // If the system is still booting, let it finish. 1777 j = jiffies; 1778 while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) { 1779 booting_still = true; 1780 schedule_timeout_interruptible(HZ); 1781 } 1782 if (booting_still) 1783 pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", 1784 torture_type, jiffies - j); 1785 1786 if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) 1787 INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work); 1788 1789 do { 1790 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1791 torture_hrtimeout_us(500, 1000, &rand); 1792 rp = rcu_torture_alloc(); 1793 if (rp == NULL) 1794 continue; 1795 rp->rtort_pipe_count = 0; 1796 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 1797 rcu_torture_writer_state = RTWS_DELAY; 1798 udelay(torture_random(&rand) & 0x3ff); 1799 rcu_torture_writer_state = RTWS_REPLACE; 1800 old_rp = rcu_dereference_check(rcu_torture_current, 1801 current == writer_task); 1802 rp->rtort_mbtest = 1; 1803 rcu_assign_pointer(rcu_torture_current, rp); 1804 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1805 if (old_rp) { 1806 i = old_rp->rtort_pipe_count; 1807 if (i > RCU_TORTURE_PIPE_LEN) 1808 i = RCU_TORTURE_PIPE_LEN; 1809 atomic_inc(&rcu_torture_wcount[i]); 1810 WRITE_ONCE(old_rp->rtort_pipe_count, 1811 old_rp->rtort_pipe_count + 1); 1812 ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count); 1813 1814 // Make sure readers block polled grace periods. 1815 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { 1816 idx = cur_ops->readlock(); 1817 cookie = cur_ops->get_gp_state(); 1818 WARN_ONCE(cur_ops->poll_gp_state(cookie), 1819 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", 1820 __func__, 1821 rcu_torture_writer_state_getname(), 1822 rcu_torture_writer_state, 1823 cookie, cur_ops->get_gp_state()); 1824 if (cur_ops->get_comp_state) { 1825 cookie = cur_ops->get_comp_state(); 1826 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); 1827 } 1828 cur_ops->readunlock(idx); 1829 } 1830 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { 1831 idx = cur_ops->readlock(); 1832 cur_ops->get_gp_state_full(&cookie_full); 1833 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 1834 "%s: Cookie check 5 failed %s(%d) online %*pbl\n", 1835 __func__, 1836 rcu_torture_writer_state_getname(), 1837 rcu_torture_writer_state, 1838 cpumask_pr_args(cpu_online_mask)); 1839 if (cur_ops->get_comp_state_full) { 1840 cur_ops->get_comp_state_full(&cookie_full); 1841 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); 1842 } 1843 cur_ops->readunlock(idx); 1844 } 1845 switch (synctype[torture_random(&rand) % nsynctypes]) { 1846 case RTWS_DEF_FREE: 1847 rcu_torture_writer_state = RTWS_DEF_FREE; 1848 cur_ops->deferred_free(old_rp); 1849 break; 1850 case RTWS_EXP_SYNC: 1851 rcu_torture_writer_state = RTWS_EXP_SYNC; 1852 do_rtws_sync(&rand, cur_ops->exp_sync); 1853 rcu_torture_pipe_update(old_rp); 1854 break; 1855 case RTWS_COND_GET: 1856 rcu_torture_writer_state = RTWS_COND_GET; 1857 gp_snap = cur_ops->get_gp_state(); 1858 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1859 1000, &rand); 1860 rcu_torture_writer_state = RTWS_COND_SYNC; 1861 cur_ops->cond_sync(gp_snap); 1862 rcu_torture_pipe_update(old_rp); 1863 break; 1864 case RTWS_COND_GET_EXP: 1865 rcu_torture_writer_state = RTWS_COND_GET_EXP; 1866 gp_snap = cur_ops->get_gp_state_exp(); 1867 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1868 1000, &rand); 1869 rcu_torture_writer_state = RTWS_COND_SYNC_EXP; 1870 cur_ops->cond_sync_exp(gp_snap); 1871 rcu_torture_pipe_update(old_rp); 1872 break; 1873 case RTWS_COND_GET_FULL: 1874 rcu_torture_writer_state = RTWS_COND_GET_FULL; 1875 cur_ops->get_gp_state_full(&gp_snap_full); 1876 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1877 1000, &rand); 1878 rcu_torture_writer_state = RTWS_COND_SYNC_FULL; 1879 cur_ops->cond_sync_full(&gp_snap_full); 1880 rcu_torture_pipe_update(old_rp); 1881 break; 1882 case RTWS_COND_GET_EXP_FULL: 1883 rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; 1884 cur_ops->get_gp_state_full(&gp_snap_full); 1885 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1886 1000, &rand); 1887 rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; 1888 cur_ops->cond_sync_exp_full(&gp_snap_full); 1889 rcu_torture_pipe_update(old_rp); 1890 break; 1891 case RTWS_POLL_GET: 1892 rcu_torture_writer_state = RTWS_POLL_GET; 1893 for (i = 0; i < ulo_size; i++) 1894 ulo[i] = cur_ops->get_comp_state(); 1895 gp_snap = cur_ops->start_gp_poll(); 1896 rcu_torture_writer_state = RTWS_POLL_WAIT; 1897 if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) 1898 cur_ops->exp_current(); 1899 while (!cur_ops->poll_gp_state(gp_snap)) { 1900 gp_snap1 = cur_ops->get_gp_state(); 1901 for (i = 0; i < ulo_size; i++) 1902 if (cur_ops->poll_gp_state(ulo[i]) || 1903 cur_ops->same_gp_state(ulo[i], gp_snap1)) { 1904 ulo[i] = gp_snap1; 1905 break; 1906 } 1907 WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size); 1908 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1909 1000, &rand); 1910 } 1911 rcu_torture_pipe_update(old_rp); 1912 break; 1913 case RTWS_POLL_GET_FULL: 1914 rcu_torture_writer_state = RTWS_POLL_GET_FULL; 1915 for (i = 0; i < rgo_size; i++) 1916 cur_ops->get_comp_state_full(&rgo[i]); 1917 cur_ops->start_gp_poll_full(&gp_snap_full); 1918 rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; 1919 if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) 1920 cur_ops->exp_current(); 1921 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1922 cur_ops->get_gp_state_full(&gp_snap1_full); 1923 for (i = 0; i < rgo_size; i++) 1924 if (cur_ops->poll_gp_state_full(&rgo[i]) || 1925 cur_ops->same_gp_state_full(&rgo[i], 1926 &gp_snap1_full)) { 1927 rgo[i] = gp_snap1_full; 1928 break; 1929 } 1930 WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size); 1931 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1932 1000, &rand); 1933 } 1934 rcu_torture_pipe_update(old_rp); 1935 break; 1936 case RTWS_POLL_GET_EXP: 1937 rcu_torture_writer_state = RTWS_POLL_GET_EXP; 1938 gp_snap = cur_ops->start_gp_poll_exp(); 1939 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; 1940 while (!cur_ops->poll_gp_state_exp(gp_snap)) 1941 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1942 1000, &rand); 1943 rcu_torture_pipe_update(old_rp); 1944 break; 1945 case RTWS_POLL_GET_EXP_FULL: 1946 rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; 1947 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1948 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; 1949 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) 1950 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1951 1000, &rand); 1952 rcu_torture_pipe_update(old_rp); 1953 break; 1954 case RTWS_SYNC: 1955 rcu_torture_writer_state = RTWS_SYNC; 1956 do_rtws_sync(&rand, cur_ops->sync); 1957 rcu_torture_pipe_update(old_rp); 1958 break; 1959 default: 1960 WARN_ON_ONCE(1); 1961 break; 1962 } 1963 } 1964 WRITE_ONCE(rcu_torture_current_version, 1965 rcu_torture_current_version + 1); 1966 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1967 if (can_expedite && 1968 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1969 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1970 if (expediting >= 0) 1971 rcu_expedite_gp(); 1972 else 1973 rcu_unexpedite_gp(); 1974 if (++expediting > 3) 1975 expediting = -expediting; 1976 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1977 can_expedite = !rcu_gp_is_expedited() && 1978 !rcu_gp_is_normal(); 1979 } 1980 rcu_torture_writer_state = RTWS_STUTTER; 1981 if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) 1982 queue_work(system_percpu_wq, &lazy_work); 1983 stutter_waited = stutter_wait("rcu_torture_writer"); 1984 if (stutter_waited && 1985 !atomic_read(&rcu_fwd_cb_nodelay) && 1986 !cur_ops->slow_gps && 1987 !torture_must_stop() && 1988 time_after(jiffies, stallsdone)) 1989 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1990 if (list_empty(&rcu_tortures[i].rtort_free) && 1991 rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { 1992 tracing_off(); 1993 if (cur_ops->gp_kthread_dbg) 1994 cur_ops->gp_kthread_dbg(); 1995 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1996 rcu_ftrace_dump(DUMP_ALL); 1997 break; 1998 } 1999 if (stutter_waited) 2000 sched_set_normal(current, oldnice); 2001 } while (!torture_must_stop()); 2002 rcu_torture_current = NULL; // Let stats task know that we are done. 2003 /* Reset expediting back to unexpedited. */ 2004 if (expediting > 0) 2005 expediting = -expediting; 2006 while (can_expedite && expediting++ < 0) 2007 rcu_unexpedite_gp(); 2008 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 2009 if (!can_expedite) 2010 pr_alert("%s" TORTURE_FLAG 2011 " Dynamic grace-period expediting was disabled.\n", 2012 torture_type); 2013 2014 if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) { 2015 cancel_work_sync(&lazy_work); 2016 destroy_work_on_stack(&lazy_work); 2017 } 2018 2019 kfree(ulo); 2020 kfree(rgo); 2021 rcu_torture_writer_state = RTWS_STOPPING; 2022 torture_kthread_stopping("rcu_torture_writer"); 2023 return 0; 2024 } 2025 2026 /* 2027 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 2028 * delay between calls. 2029 */ 2030 static int 2031 rcu_torture_fakewriter(void *arg) 2032 { 2033 unsigned long gp_snap; 2034 struct rcu_gp_seq gp_snap_full; 2035 DEFINE_TORTURE_RANDOM(rand); 2036 2037 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 2038 set_user_nice(current, MAX_NICE); 2039 2040 if (WARN_ONCE(nsynctypes == 0, 2041 "%s: No update-side primitives.\n", __func__)) { 2042 /* 2043 * No updates primitives, so don't try updating. 2044 * The resulting test won't be testing much, hence the 2045 * above WARN_ONCE(). 2046 */ 2047 torture_kthread_stopping("rcu_torture_fakewriter"); 2048 return 0; 2049 } 2050 2051 do { 2052 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); 2053 if (cur_ops->cb_barrier != NULL && 2054 torture_random(&rand) % (nrealfakewriters * 8) == 0) { 2055 cur_ops->cb_barrier(); 2056 } else { 2057 switch (synctype[torture_random(&rand) % nsynctypes]) { 2058 case RTWS_DEF_FREE: 2059 break; 2060 case RTWS_EXP_SYNC: 2061 cur_ops->exp_sync(); 2062 break; 2063 case RTWS_COND_GET: 2064 gp_snap = cur_ops->get_gp_state(); 2065 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 2066 cur_ops->cond_sync(gp_snap); 2067 break; 2068 case RTWS_COND_GET_EXP: 2069 gp_snap = cur_ops->get_gp_state_exp(); 2070 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 2071 cur_ops->cond_sync_exp(gp_snap); 2072 break; 2073 case RTWS_COND_GET_FULL: 2074 cur_ops->get_gp_state_full(&gp_snap_full); 2075 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 2076 cur_ops->cond_sync_full(&gp_snap_full); 2077 break; 2078 case RTWS_COND_GET_EXP_FULL: 2079 cur_ops->get_gp_state_full(&gp_snap_full); 2080 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 2081 cur_ops->cond_sync_exp_full(&gp_snap_full); 2082 break; 2083 case RTWS_POLL_GET: 2084 if (cur_ops->start_poll_irqsoff) 2085 local_irq_disable(); 2086 gp_snap = cur_ops->start_gp_poll(); 2087 if (cur_ops->start_poll_irqsoff) 2088 local_irq_enable(); 2089 while (!cur_ops->poll_gp_state(gp_snap)) { 2090 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 2091 &rand); 2092 } 2093 break; 2094 case RTWS_POLL_GET_FULL: 2095 if (cur_ops->start_poll_irqsoff) 2096 local_irq_disable(); 2097 cur_ops->start_gp_poll_full(&gp_snap_full); 2098 if (cur_ops->start_poll_irqsoff) 2099 local_irq_enable(); 2100 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 2101 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 2102 &rand); 2103 } 2104 break; 2105 case RTWS_POLL_GET_EXP: 2106 gp_snap = cur_ops->start_gp_poll_exp(); 2107 while (!cur_ops->poll_gp_state_exp(gp_snap)) { 2108 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 2109 &rand); 2110 } 2111 break; 2112 case RTWS_POLL_GET_EXP_FULL: 2113 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 2114 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 2115 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 2116 &rand); 2117 } 2118 break; 2119 case RTWS_SYNC: 2120 cur_ops->sync(); 2121 break; 2122 default: 2123 WARN_ON_ONCE(1); 2124 break; 2125 } 2126 } 2127 stutter_wait("rcu_torture_fakewriter"); 2128 } while (!torture_must_stop()); 2129 2130 torture_kthread_stopping("rcu_torture_fakewriter"); 2131 return 0; 2132 } 2133 2134 static void rcu_torture_timer_cb(struct rcu_head *rhp) 2135 { 2136 kfree(rhp); 2137 } 2138 2139 // Set up and carry out testing of RCU's global memory ordering 2140 static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, 2141 struct torture_random_state *trsp) 2142 { 2143 unsigned long loops; 2144 int noc = torture_num_online_cpus(); 2145 int rdrchked; 2146 int rdrchker; 2147 struct rcu_torture_reader_check *rtrcp; // Me. 2148 struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. 2149 struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. 2150 struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. 2151 2152 if (myid < 0) 2153 return; // Don't try this from timer handlers. 2154 2155 // Increment my counter. 2156 rtrcp = &rcu_torture_reader_mbchk[myid]; 2157 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); 2158 2159 // Attempt to assign someone else some checking work. 2160 rdrchked = torture_random(trsp) % nrealreaders; 2161 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 2162 rdrchker = torture_random(trsp) % nrealreaders; 2163 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; 2164 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && 2165 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. 2166 !READ_ONCE(rtp->rtort_chkp) && 2167 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. 2168 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); 2169 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); 2170 rtrcp->rtc_chkrdr = rdrchked; 2171 WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. 2172 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || 2173 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) 2174 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. 2175 } 2176 2177 // If assigned some completed work, do it! 2178 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); 2179 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) 2180 return; // No work or work not yet ready. 2181 rdrchked = rtrcp_assigner->rtc_chkrdr; 2182 if (WARN_ON_ONCE(rdrchked < 0)) 2183 return; 2184 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 2185 loops = READ_ONCE(rtrcp_chked->rtc_myloops); 2186 atomic_inc(&n_rcu_torture_mbchk_tries); 2187 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) 2188 atomic_inc(&n_rcu_torture_mbchk_fail); 2189 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; 2190 rtrcp_assigner->rtc_ready = 0; 2191 smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. 2192 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. 2193 } 2194 2195 static DEFINE_PER_CPU(bool, torture_in_scf_handler); 2196 2197 // Verify the specified RCUTORTURE_RDR* state. 2198 #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() 2199 static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) 2200 { 2201 int mask; 2202 2203 if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) 2204 return; 2205 2206 WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq() && !this_cpu_read(torture_in_scf_handler), ROEC_ARGS); 2207 WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); 2208 2209 // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. 2210 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 2211 return; 2212 2213 WARN_ONCE((curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 2214 !softirq_count(), ROEC_ARGS); 2215 WARN_ONCE((curstate & (RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED)) && 2216 !(preempt_count() & PREEMPT_MASK), ROEC_ARGS); 2217 WARN_ONCE(cur_ops->readlock_nesting && 2218 (curstate & (RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2)) && 2219 cur_ops->readlock_nesting() == 0, ROEC_ARGS); 2220 2221 // Interrupt handlers have all sorts of stuff disabled, so ignore 2222 // unintended disabling. 2223 if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler)) 2224 return; 2225 2226 WARN_ONCE(cur_ops->extendables && 2227 !(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 2228 softirq_count(), ROEC_ARGS); 2229 2230 /* 2231 * non-preemptible RCU in a preemptible kernel uses preempt_disable() 2232 * as rcu_read_lock(). 2233 */ 2234 mask = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2235 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 2236 mask |= RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2237 2238 WARN_ONCE(cur_ops->extendables && !(curstate & mask) && 2239 (preempt_count() & PREEMPT_MASK), ROEC_ARGS); 2240 2241 /* 2242 * non-preemptible RCU in a preemptible kernel uses "preempt_count() & 2243 * PREEMPT_MASK" as ->readlock_nesting(). 2244 */ 2245 mask = RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2246 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 2247 mask |= RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2248 2249 if (IS_ENABLED(CONFIG_PREEMPT_RT) && softirq_count()) 2250 mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 2251 2252 WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) && 2253 cur_ops->readlock_nesting() > 0, ROEC_ARGS); 2254 } 2255 2256 /* 2257 * Do one extension of an RCU read-side critical section using the 2258 * current reader state in readstate (set to zero for initial entry 2259 * to extended critical section), set the new state as specified by 2260 * newstate (set to zero for final exit from extended critical section), 2261 * and random-number-generator state in trsp. If this is neither the 2262 * beginning or end of the critical section and if there was actually a 2263 * change, do a ->read_delay(). 2264 */ 2265 static void rcutorture_one_extend(int *readstate, int newstate, struct torture_random_state *trsp, 2266 struct rt_read_seg *rtrsp) 2267 { 2268 bool first; 2269 unsigned long flags; 2270 int idxnew1 = -1; 2271 int idxnew2 = -1; 2272 int idxold1 = *readstate; 2273 int idxold2 = idxold1; 2274 int statesnew = ~*readstate & newstate; 2275 int statesold = *readstate & ~newstate; 2276 2277 first = idxold1 == 0; 2278 WARN_ON_ONCE(idxold2 < 0); 2279 WARN_ON_ONCE(idxold2 & ~(RCUTORTURE_RDR_ALLBITS | RCUTORTURE_RDR_UPDOWN)); 2280 rcutorture_one_extend_check("before change", idxold1, statesnew, statesold); 2281 rtrsp->rt_readstate = newstate; 2282 2283 /* First, put new protection in place to avoid critical-section gap. */ 2284 if (statesnew & RCUTORTURE_RDR_BH) 2285 local_bh_disable(); 2286 if (statesnew & RCUTORTURE_RDR_RBH) 2287 rcu_read_lock_bh(); 2288 if (statesnew & RCUTORTURE_RDR_IRQ) 2289 local_irq_disable(); 2290 if (statesnew & RCUTORTURE_RDR_PREEMPT) 2291 preempt_disable(); 2292 if (statesnew & RCUTORTURE_RDR_SCHED) 2293 rcu_read_lock_sched(); 2294 if (statesnew & RCUTORTURE_RDR_RCU_1) 2295 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 2296 if (statesnew & RCUTORTURE_RDR_RCU_2) 2297 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2; 2298 2299 // Complain unless both the old and the new protection is in place. 2300 rcutorture_one_extend_check("during change", idxold1 | statesnew, statesnew, statesold); 2301 2302 // Sample CPU under both sets of protections to reduce confusion. 2303 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 2304 int cpu = raw_smp_processor_id(); 2305 rtrsp->rt_cpu = cpu; 2306 if (!first) { 2307 rtrsp[-1].rt_end_cpu = cpu; 2308 if (cur_ops->reader_blocked) 2309 rtrsp[-1].rt_preempted = cur_ops->reader_blocked(); 2310 } 2311 } 2312 // Sample grace-period sequence number, as good a place as any. 2313 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) { 2314 rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs(); 2315 rtrsp->rt_ts = ktime_get_mono_fast_ns(); 2316 if (!first) 2317 rtrsp[-1].rt_gp_seq_end = rtrsp->rt_gp_seq; 2318 } 2319 2320 /* 2321 * Next, remove old protection, in decreasing order of strength 2322 * to avoid unlock paths that aren't safe in the stronger 2323 * context. Namely: BH can not be enabled with disabled interrupts. 2324 * Additionally PREEMPT_RT requires that BH is enabled in preemptible 2325 * context. 2326 */ 2327 if (statesold & RCUTORTURE_RDR_IRQ) 2328 local_irq_enable(); 2329 if (statesold & RCUTORTURE_RDR_PREEMPT) 2330 preempt_enable(); 2331 if (statesold & RCUTORTURE_RDR_SCHED) 2332 rcu_read_unlock_sched(); 2333 if (statesold & RCUTORTURE_RDR_BH) 2334 local_bh_enable(); 2335 if (statesold & RCUTORTURE_RDR_RBH) 2336 rcu_read_unlock_bh(); 2337 if (statesold & RCUTORTURE_RDR_RCU_2) { 2338 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2); 2339 WARN_ON_ONCE(idxnew2 != -1); 2340 idxold2 = 0; 2341 } 2342 if (statesold & RCUTORTURE_RDR_RCU_1) { 2343 bool lockit; 2344 2345 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); 2346 if (lockit) 2347 raw_spin_lock_irqsave(¤t->pi_lock, flags); 2348 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 2349 WARN_ON_ONCE(idxnew1 != -1); 2350 idxold1 = 0; 2351 if (lockit) 2352 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 2353 } 2354 if (statesold & RCUTORTURE_RDR_UPDOWN) { 2355 cur_ops->up_read((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 2356 WARN_ON_ONCE(idxnew1 != -1); 2357 idxold1 = 0; 2358 } 2359 2360 /* Delay if neither beginning nor end and there was a change. */ 2361 if ((statesnew || statesold) && *readstate && newstate) 2362 cur_ops->read_delay(trsp, rtrsp); 2363 2364 /* Update the reader state. */ 2365 if (idxnew1 == -1) 2366 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; 2367 WARN_ON_ONCE(idxnew1 < 0); 2368 if (idxnew2 == -1) 2369 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; 2370 WARN_ON_ONCE(idxnew2 < 0); 2371 *readstate = idxnew1 | idxnew2 | newstate; 2372 WARN_ON_ONCE(*readstate < 0); 2373 if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS)) 2374 pr_info("Unexpected readstate value of %#x\n", *readstate); 2375 rcutorture_one_extend_check("after change", *readstate, statesnew, statesold); 2376 } 2377 2378 /* Return the biggest extendables mask given current RCU and boot parameters. */ 2379 static int rcutorture_extend_mask_max(void) 2380 { 2381 int mask; 2382 2383 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 2384 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 2385 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2386 return mask; 2387 } 2388 2389 /* Return a random protection state mask, but with at least one bit set. */ 2390 static int 2391 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 2392 { 2393 int mask = rcutorture_extend_mask_max(); 2394 unsigned long randmask1 = torture_random(trsp); 2395 unsigned long randmask2 = randmask1 >> 3; 2396 unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2397 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 2398 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 2399 2400 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits. 2401 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 2402 if (!(randmask1 & 0x7)) 2403 mask = mask & randmask2; 2404 else 2405 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 2406 2407 // Can't have nested RCU reader without outer RCU reader. 2408 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { 2409 if (oldmask & RCUTORTURE_RDR_RCU_1) 2410 mask &= ~RCUTORTURE_RDR_RCU_2; 2411 else 2412 mask |= RCUTORTURE_RDR_RCU_1; 2413 } 2414 2415 /* 2416 * Don't mess with interrupt masking in interrupt handlers. 2417 */ 2418 if (in_hardirq() || this_cpu_read(torture_in_scf_handler)) 2419 mask &= ~(preempts_irq | bhs); 2420 2421 /* 2422 * Can't enable bh w/irq disabled. 2423 */ 2424 if (mask & RCUTORTURE_RDR_IRQ) 2425 mask |= oldmask & bhs; 2426 2427 2428 /* 2429 * Ideally these sequences would be detected in debug builds 2430 * (regardless of RT), but until then don't stop testing 2431 * them on non-RT. 2432 */ 2433 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 2434 /* Can't modify BH in atomic context */ 2435 if (oldmask & preempts_irq) 2436 mask &= ~bhs; 2437 if ((oldmask | mask) & preempts_irq) 2438 mask |= oldmask & bhs; 2439 } 2440 2441 return mask ?: RCUTORTURE_RDR_RCU_1; 2442 } 2443 2444 /* 2445 * Do a randomly selected number of extensions of an existing RCU read-side 2446 * critical section. 2447 */ 2448 static struct rt_read_seg * 2449 rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct rt_read_seg *rtrsp) 2450 { 2451 int i; 2452 int j; 2453 int mask = rcutorture_extend_mask_max(); 2454 2455 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 2456 if (!((mask - 1) & mask)) 2457 return rtrsp; /* Current RCU reader not extendable. */ 2458 /* Bias towards larger numbers of loops. */ 2459 i = torture_random(trsp); 2460 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 2461 for (j = 0; j < i; j++) { 2462 mask = rcutorture_extend_mask(*readstate, trsp); 2463 WARN_ON_ONCE(mask & RCUTORTURE_RDR_UPDOWN); 2464 rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); 2465 } 2466 return &rtrsp[j]; 2467 } 2468 2469 struct rcu_torture_one_read_state { 2470 bool checkpolling; 2471 unsigned long cookie; 2472 struct rcu_gp_seq cookie_full; 2473 unsigned long started; 2474 struct rcu_torture *p; 2475 int readstate; 2476 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS]; 2477 struct rt_read_seg *rtrsp; 2478 unsigned long long ts; 2479 }; 2480 2481 static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs) 2482 { 2483 bool firsttime; 2484 int i; 2485 int j; 2486 2487 firsttime = 1; 2488 for (i = 0; i < nsegs; i++) { 2489 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) 2490 pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL)); 2491 else 2492 pr_alert("\t"); 2493 pr_cont("%d: %#4x", i, rrsp[i].rt_readstate); 2494 if (rrsp[i].rt_delay_jiffies != 0) { 2495 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 2496 rrsp[i].rt_delay_jiffies); 2497 firsttime = 0; 2498 } 2499 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 2500 pr_cont(" CPU %2d", rrsp[i].rt_cpu); 2501 if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu) 2502 pr_cont("->%-2d", rrsp[i].rt_end_cpu); 2503 else 2504 pr_cont(" ..."); 2505 } 2506 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && 2507 cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { 2508 char buf1[20+1]; 2509 char buf2[20+1]; 2510 char sepchar = '-'; 2511 2512 cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1)); 2513 cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2)); 2514 if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) { 2515 if (buf2[0]) { 2516 for (j = 0; buf2[j]; j++) 2517 buf2[j] = '.'; 2518 if (j) 2519 buf2[j - 1] = ' '; 2520 } 2521 sepchar = ' '; 2522 } 2523 pr_cont(" %s%c%s", buf1, sepchar, buf2); 2524 } 2525 if (rrsp[i].rt_delay_ms != 0) { 2526 pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms); 2527 firsttime = 0; 2528 } 2529 if (rrsp[i].rt_delay_us != 0) { 2530 pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us); 2531 firsttime = 0; 2532 } 2533 pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : ""); 2534 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH) 2535 pr_cont(" BH"); 2536 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ) 2537 pr_cont(" IRQ"); 2538 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) 2539 pr_cont(" PREEMPT"); 2540 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH) 2541 pr_cont(" RBH"); 2542 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED) 2543 pr_cont(" SCHED"); 2544 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1) 2545 pr_cont(" RCU_1"); 2546 if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2) 2547 pr_cont(" RCU_2"); 2548 pr_cont("\n"); 2549 2550 } 2551 if (rt_read_preempted) 2552 pr_alert("\tReader was preempted.\n"); 2553 } 2554 2555 static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, 2556 struct torture_random_state *trsp) 2557 { 2558 memset(rtorsp, 0, sizeof(*rtorsp)); 2559 rtorsp->checkpolling = !(torture_random(trsp) & 0xfff); 2560 rtorsp->rtrsp = &rtorsp->rtseg[0]; 2561 } 2562 2563 /* 2564 * Set up the first segment of a series of overlapping read-side 2565 * critical sections. The caller must have actually initiated the 2566 * outermost read-side critical section. 2567 */ 2568 static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp, 2569 struct torture_random_state *trsp, long myid) 2570 { 2571 if (rtorsp->checkpolling) { 2572 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2573 rtorsp->cookie = cur_ops->get_gp_state(); 2574 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2575 cur_ops->get_gp_state_full(&rtorsp->cookie_full); 2576 } 2577 rtorsp->started = cur_ops->get_gp_seq(); 2578 rtorsp->ts = rcu_trace_clock_local(); 2579 rtorsp->p = rcu_dereference_check(rcu_torture_current, 2580 !cur_ops->readlock_held || cur_ops->readlock_held() || 2581 (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN)); 2582 if ((!cur_ops->rdrs_handle_load && atomic_read(&rcu_fwd_cb_nodelay)) || rtorsp->p == NULL) { 2583 // Wait for rcu_torture_writer to get underway and 2584 // (if readers cannot handle heavy loads) for any 2585 // forward-progress testing to complete. 2586 rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); 2587 return false; 2588 } 2589 if (rtorsp->p->rtort_mbtest == 0) 2590 atomic_inc(&n_rcu_torture_mberror); 2591 rcu_torture_reader_do_mbchk(myid, rtorsp->p, trsp); 2592 return true; 2593 } 2594 2595 /* 2596 * Complete the last segment of a series of overlapping read-side 2597 * critical sections and check for errors. 2598 */ 2599 static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, 2600 struct torture_random_state *trsp) 2601 { 2602 int i; 2603 unsigned long completed; 2604 int pipe_count; 2605 bool preempted = false; 2606 struct rt_read_seg *rtrsp1; 2607 2608 preempt_disable(); 2609 pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count); 2610 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 2611 // Should not happen in a correct RCU implementation, 2612 // happens quite often for torture_type=busted. 2613 pipe_count = RCU_TORTURE_PIPE_LEN; 2614 } 2615 completed = cur_ops->get_gp_seq(); 2616 if (pipe_count > 1) { 2617 do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu, 2618 rtorsp->ts, rtorsp->started, completed); 2619 rcu_ftrace_dump(DUMP_ALL); 2620 } 2621 this_cpu_inc(rcu_torture_count[pipe_count]); 2622 completed = rcutorture_seq_diff(completed, rtorsp->started); 2623 if (completed > RCU_TORTURE_PIPE_LEN) { 2624 /* Should not happen, but... */ 2625 completed = RCU_TORTURE_PIPE_LEN; 2626 } 2627 this_cpu_inc(rcu_torture_batch[completed]); 2628 preempt_enable(); 2629 if (rtorsp->checkpolling) { 2630 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2631 WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie), 2632 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", 2633 __func__, 2634 rcu_torture_writer_state_getname(), 2635 rcu_torture_writer_state, 2636 rtorsp->cookie, cur_ops->get_gp_state()); 2637 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2638 WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full), 2639 "%s: Cookie check 6 failed %s(%d) online %*pbl\n", 2640 __func__, 2641 rcu_torture_writer_state_getname(), 2642 rcu_torture_writer_state, 2643 cpumask_pr_args(cpu_online_mask)); 2644 } 2645 if (cur_ops->reader_blocked) 2646 preempted = cur_ops->reader_blocked(); 2647 rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); 2648 WARN_ON_ONCE(rtorsp->readstate); 2649 // This next splat is expected behavior if leakpointer, especially 2650 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. 2651 WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1); 2652 2653 /* If error or close call, record the sequence of reader protections. */ 2654 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 2655 i = 0; 2656 for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++) 2657 err_segs[i++] = *rtrsp1; 2658 rt_read_nsegs = i; 2659 rt_read_preempted = preempted; 2660 } 2661 } 2662 2663 /* 2664 * Do one read-side critical section, returning false if there was 2665 * no data to read. Can be invoked both from process context and 2666 * from a timer handler. 2667 */ 2668 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) 2669 { 2670 static int firsttime = 1; 2671 int newstate; 2672 unsigned int nsegs; 2673 struct rcu_torture_one_read_state rtors; 2674 2675 WARN_ON_ONCE(!rcu_is_watching()); 2676 init_rcu_torture_one_read_state(&rtors, trsp); 2677 newstate = rcutorture_extend_mask(rtors.readstate, trsp); 2678 WARN_ON_ONCE(newstate & RCUTORTURE_RDR_UPDOWN); 2679 rcutorture_one_extend(&rtors.readstate, newstate, trsp, rtors.rtrsp++); 2680 if (!rcu_torture_one_read_start(&rtors, trsp, myid)) 2681 return false; 2682 rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); 2683 rcu_torture_one_read_end(&rtors, trsp); 2684 2685 // This splat will happen on systems built with CONFIG_IRQ_WORK=n 2686 // and on systems where arch_irq_work_has_interrupt() returns false. 2687 // It might also happen on systems using a short-duration clock 2688 // interrupt instead of a self-IPI (powerpc, s390) or that use 2689 // neither a self-IPI nor a short-duration clock interrupts 2690 // (all architectures using the generic implementation 2691 // of arch_irq_work_raise()). On such systems, RCU cannot 2692 // guarantee to immediately deboost RCU readers when the outermost 2693 // rcu_read_unlock() does not end the full segmented RCU read-side 2694 // critical section. 2695 if (cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && 2696 !in_serving_softirq() && !in_hardirq() && !in_nmi() && 2697 READ_ONCE(firsttime) && xchg(&firsttime, 0)) { 2698 WARN_ON_ONCE(deboost_timeliness_check); 2699 nsegs = rtors.rtrsp - rtors.rtseg; 2700 nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); 2701 pr_alert("Slow-deboost rcutorture reader segments:\n"); 2702 rcu_torture_dump_read_segs(rtors.rtseg, nsegs); 2703 } 2704 return true; 2705 } 2706 2707 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 2708 2709 /* 2710 * Exercise ->call() from NMI context for flavors that set ->nmi_capable. A 2711 * per-CPU hardware perf counter overflows into an NMI, and its handler submits 2712 * a preallocated callback via ->call(). One callback per CPU is in flight at a 2713 * time (guarded by an atomic) to avoid allocating in NMI. 2714 */ 2715 #ifdef CONFIG_PERF_EVENTS 2716 static struct perf_event_attr rcu_torture_nmi_attr = { 2717 .type = PERF_TYPE_HARDWARE, 2718 .config = PERF_COUNT_HW_CPU_CYCLES, 2719 .size = sizeof(struct perf_event_attr), 2720 .pinned = 1, 2721 .disabled = 1, 2722 /* 2723 * A fixed period rather than .freq: a frequency-based event bumps 2724 * nr_freq_events, which sets TICK_DEP_BIT_PERF_EVENTS and would pin the 2725 * tick for the whole run on NO_HZ_FULL kernels. 2726 */ 2727 .sample_period = 20 * 1000 * 1000, 2728 }; 2729 2730 /* One in-flight callback per CPU; ->inuse is released by the callback. */ 2731 struct rcu_torture_nmi_cb { 2732 struct rcu_head rh; 2733 atomic_t inuse; 2734 }; 2735 2736 static struct perf_event **rcu_torture_nmi_events; 2737 static int rcu_torture_nmi_hp_state; 2738 static DEFINE_PER_CPU(struct rcu_torture_nmi_cb, rcu_torture_nmi_cb); 2739 2740 static void rcu_torture_nmi_invoked(struct rcu_head *rhp) 2741 { 2742 struct rcu_torture_nmi_cb *rtncp = container_of(rhp, struct rcu_torture_nmi_cb, rh); 2743 2744 atomic_long_inc(&n_rcu_torture_nmi_cb); 2745 atomic_set(&rtncp->inuse, 0); 2746 } 2747 2748 static void rcu_torture_nmi_overflow(struct perf_event *event, 2749 struct perf_sample_data *data, 2750 struct pt_regs *regs) 2751 { 2752 struct rcu_torture_nmi_cb *rtncp = this_cpu_ptr(&rcu_torture_nmi_cb); 2753 2754 if (!in_nmi()) 2755 return; 2756 if (cur_ops->call && !atomic_xchg(&rtncp->inuse, 1)) { 2757 atomic_long_inc(&n_rcu_torture_nmi_call); 2758 cur_ops->call(&rtncp->rh, rcu_torture_nmi_invoked); 2759 } 2760 } 2761 2762 static int rcu_torture_nmi_online(unsigned int cpu) 2763 { 2764 struct perf_event *event; 2765 2766 event = perf_event_create_kernel_counter(&rcu_torture_nmi_attr, cpu, NULL, 2767 rcu_torture_nmi_overflow, NULL); 2768 if (IS_ERR(event)) 2769 return 0; 2770 rcu_torture_nmi_events[cpu] = event; 2771 perf_event_enable(event); 2772 return 0; 2773 } 2774 2775 static int rcu_torture_nmi_offline(unsigned int cpu) 2776 { 2777 struct perf_event *event = rcu_torture_nmi_events[cpu]; 2778 2779 if (event) { 2780 rcu_torture_nmi_events[cpu] = NULL; 2781 perf_event_disable(event); 2782 perf_event_release_kernel(event); 2783 } 2784 return 0; 2785 } 2786 2787 /* Drive the counters from hotplug callbacks so coverage survives onoff. */ 2788 static void rcu_torture_nmi_init(void) 2789 { 2790 int ret; 2791 2792 if (!nmi_calls || !cur_ops->nmi_capable || !cur_ops->call) 2793 return; 2794 rcu_torture_nmi_events = kcalloc(nr_cpu_ids, sizeof(*rcu_torture_nmi_events), 2795 GFP_KERNEL); 2796 if (!rcu_torture_nmi_events) 2797 return; 2798 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "rcutorture/nmi:online", 2799 rcu_torture_nmi_online, rcu_torture_nmi_offline); 2800 if (ret < 0) { 2801 kfree(rcu_torture_nmi_events); 2802 rcu_torture_nmi_events = NULL; 2803 return; 2804 } 2805 rcu_torture_nmi_hp_state = ret; 2806 } 2807 2808 static void rcu_torture_nmi_cleanup(void) 2809 { 2810 if (!rcu_torture_nmi_events) 2811 return; 2812 if (rcu_torture_nmi_hp_state > 0) { 2813 cpuhp_remove_state(rcu_torture_nmi_hp_state); 2814 rcu_torture_nmi_hp_state = 0; 2815 } 2816 kfree(rcu_torture_nmi_events); 2817 rcu_torture_nmi_events = NULL; 2818 if (!atomic_long_read(&n_rcu_torture_nmi_call)) 2819 pr_alert("%s: nmi_calls set but no ->call() ever issued from NMI, so NMI ->call() went untested (no PMU, or NMIs unavailable here).\n", 2820 __func__); 2821 } 2822 #else /* #ifdef CONFIG_PERF_EVENTS */ 2823 static void rcu_torture_nmi_init(void) { } 2824 static void rcu_torture_nmi_cleanup(void) { } 2825 #endif /* #else #ifdef CONFIG_PERF_EVENTS */ 2826 2827 /* 2828 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2829 * incrementing the corresponding element of the pipeline array. The 2830 * counter in the element should never be greater than 1, otherwise, the 2831 * RCU implementation is broken. 2832 */ 2833 static void rcu_torture_timer(struct timer_list *unused) 2834 { 2835 WARN_ON_ONCE(!in_serving_softirq()); 2836 WARN_ON_ONCE(in_hardirq()); 2837 WARN_ON_ONCE(in_nmi()); 2838 atomic_long_inc(&n_rcu_torture_timers); 2839 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); 2840 2841 /* Test call_rcu() invocation from softirq handler. */ 2842 if (cur_ops->call) { 2843 struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); 2844 2845 if (rhp) 2846 cur_ops->call(rhp, rcu_torture_timer_cb); 2847 } 2848 } 2849 2850 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand); 2851 2852 /* 2853 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2854 * incrementing the corresponding element of the pipeline array. The 2855 * counter in the element should never be greater than 1, otherwise, the 2856 * RCU implementation is broken. 2857 * 2858 * Note that on some systems, "interrupts" from idle are direct calls 2859 * rather than interrupts. The torture_in_scf_handler per-CPU variable 2860 * accounts for this case. 2861 */ 2862 static void rcu_torture_irq(void *unused) 2863 { 2864 WARN_ON_ONCE(in_nmi()); 2865 lockdep_assert_irqs_disabled(); 2866 atomic_long_inc(&n_rcu_torture_irqs); 2867 this_cpu_write(torture_in_scf_handler, true); 2868 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1); 2869 this_cpu_write(torture_in_scf_handler, false); 2870 2871 // Test call_rcu() invocation from interrupt handler. Interrupts 2872 // will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels. 2873 // The "right" thing to do would be to create a special-purpose 2874 // lockless or raw-spinlock-protected allocator, but in the meantime, 2875 // skip testing call_rcu() from interrupt handlers in kernels built 2876 // with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y. 2877 if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { 2878 struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); 2879 2880 if (rhp) 2881 cur_ops->call(rhp, rcu_torture_timer_cb); 2882 } 2883 } 2884 2885 /* 2886 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 2887 * incrementing the corresponding element of the pipeline array. The 2888 * counter in the element should never be greater than 1, otherwise, the 2889 * RCU implementation is broken. 2890 */ 2891 static int 2892 rcu_torture_reader(void *arg) 2893 { 2894 unsigned long lastscf = jiffies; 2895 unsigned long lastsleep = jiffies; 2896 long myid = (long)arg; 2897 int mynumonline = myid; 2898 DEFINE_TORTURE_RANDOM(rand); 2899 struct timer_list t; 2900 2901 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 2902 set_user_nice(current, MAX_NICE); 2903 if (irqreader && cur_ops->irq_capable) 2904 timer_setup_on_stack(&t, rcu_torture_timer, 0); 2905 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 2906 do { 2907 if (irqreader && cur_ops->irq_capable) { 2908 if (!timer_pending(&t)) { 2909 int cpu; 2910 2911 mod_timer(&t, jiffies + 1); 2912 preempt_disable(); 2913 cpu = torture_random(&rand) % nr_cpu_ids; 2914 if (!cpu_online(cpu)) { 2915 cpu = cpumask_next(cpu, cpu_online_mask); 2916 if (cpu >= nr_cpu_ids) 2917 cpu = cpumask_next(-1, cpu_online_mask); 2918 } 2919 // An smp_call_function_single() to self is not an interrupt! 2920 if (cpu != smp_processor_id() && 2921 time_after(jiffies, lastscf + HZ * nrealreaders / 50)) { 2922 smp_call_function_single(cpu, rcu_torture_irq, NULL, 0); 2923 lastscf = jiffies; 2924 } 2925 preempt_enable(); 2926 } 2927 } 2928 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) 2929 schedule_timeout_interruptible(HZ); 2930 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 2931 torture_hrtimeout_us(500, 1000, &rand); 2932 lastsleep = jiffies + 10; 2933 } 2934 while (!torture_must_stop() && 2935 (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) 2936 schedule_timeout_interruptible(HZ / 5); 2937 stutter_wait("rcu_torture_reader"); 2938 } while (!torture_must_stop()); 2939 if (irqreader && cur_ops->irq_capable) { 2940 timer_delete_sync(&t); 2941 timer_destroy_on_stack(&t); 2942 } 2943 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2944 torture_kthread_stopping("rcu_torture_reader"); 2945 return 0; 2946 } 2947 2948 struct rcu_torture_one_read_state_updown { 2949 struct hrtimer rtorsu_hrt; 2950 bool rtorsu_inuse; 2951 ktime_t rtorsu_kt; 2952 int rtorsu_cpu; 2953 unsigned long rtorsu_j; 2954 unsigned long rtorsu_ndowns; 2955 unsigned long rtorsu_nups; 2956 unsigned long rtorsu_nmigrates; 2957 struct torture_random_state rtorsu_trs; 2958 struct rcu_torture_one_read_state rtorsu_rtors; 2959 }; 2960 2961 static struct rcu_torture_one_read_state_updown *updownreaders; 2962 static DEFINE_TORTURE_RANDOM(rcu_torture_updown_rand); 2963 static int rcu_torture_updown(void *arg); 2964 2965 static enum hrtimer_restart rcu_torture_updown_hrt(struct hrtimer *hrtp) 2966 { 2967 int cpu = raw_smp_processor_id(); 2968 struct rcu_torture_one_read_state_updown *rtorsup; 2969 2970 rtorsup = container_of(hrtp, struct rcu_torture_one_read_state_updown, rtorsu_hrt); 2971 rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 2972 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 2973 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 2974 WRITE_ONCE(rtorsup->rtorsu_nmigrates, 2975 rtorsup->rtorsu_nmigrates + (cpu != rtorsup->rtorsu_cpu)); 2976 smp_store_release(&rtorsup->rtorsu_inuse, false); 2977 return HRTIMER_NORESTART; 2978 } 2979 2980 static int rcu_torture_updown_init(void) 2981 { 2982 int i; 2983 struct torture_random_state *rand = &rcu_torture_updown_rand; 2984 int ret; 2985 2986 if (n_up_down < 0) 2987 return 0; 2988 if (!srcu_torture_have_up_down()) { 2989 VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Disabling up/down reader tests due to lack of primitives"); 2990 return 0; 2991 } 2992 updownreaders = kzalloc_objs(*updownreaders, n_up_down); 2993 if (!updownreaders) { 2994 VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Out of memory, disabling up/down reader tests"); 2995 return -ENOMEM; 2996 } 2997 for (i = 0; i < n_up_down; i++) { 2998 init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors, rand); 2999 hrtimer_setup(&updownreaders[i].rtorsu_hrt, rcu_torture_updown_hrt, CLOCK_MONOTONIC, 3000 HRTIMER_MODE_REL | HRTIMER_MODE_HARD); 3001 torture_random_init(&updownreaders[i].rtorsu_trs); 3002 init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors, 3003 &updownreaders[i].rtorsu_trs); 3004 } 3005 ret = torture_create_kthread(rcu_torture_updown, rand, updown_task); 3006 if (ret) { 3007 kfree(updownreaders); 3008 updownreaders = NULL; 3009 } 3010 return ret; 3011 } 3012 3013 static void rcu_torture_updown_cleanup(void) 3014 { 3015 struct rcu_torture_one_read_state_updown *rtorsup; 3016 3017 for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) { 3018 if (!smp_load_acquire(&rtorsup->rtorsu_inuse)) 3019 continue; 3020 if (hrtimer_cancel(&rtorsup->rtorsu_hrt) || WARN_ON_ONCE(rtorsup->rtorsu_inuse)) { 3021 rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 3022 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 3023 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 3024 smp_store_release(&rtorsup->rtorsu_inuse, false); 3025 } 3026 3027 } 3028 kfree(updownreaders); 3029 updownreaders = NULL; 3030 } 3031 3032 // Do one reader for rcu_torture_updown(). 3033 static void rcu_torture_updown_one(struct rcu_torture_one_read_state_updown *rtorsup) 3034 { 3035 int idx; 3036 int rawidx; 3037 ktime_t t; 3038 3039 init_rcu_torture_one_read_state(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs); 3040 rawidx = cur_ops->down_read(); 3041 WRITE_ONCE(rtorsup->rtorsu_ndowns, rtorsup->rtorsu_ndowns + 1); 3042 idx = (rawidx << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 3043 rtorsup->rtorsu_rtors.readstate = idx | RCUTORTURE_RDR_UPDOWN; 3044 rtorsup->rtorsu_rtors.rtrsp++; 3045 rtorsup->rtorsu_cpu = raw_smp_processor_id(); 3046 if (!rcu_torture_one_read_start(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs, -1)) { 3047 WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders); 3048 WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1); 3049 schedule_timeout_idle(HZ); 3050 return; 3051 } 3052 smp_store_release(&rtorsup->rtorsu_inuse, true); 3053 t = torture_random(&rtorsup->rtorsu_trs) & 0xfffff; // One per million. 3054 if (t < 10 * 1000) 3055 t = 200 * 1000 * 1000; 3056 hrtimer_start(&rtorsup->rtorsu_hrt, t, HRTIMER_MODE_REL | HRTIMER_MODE_HARD); 3057 smp_mb(); // Sample jiffies after posting hrtimer. 3058 rtorsup->rtorsu_j = jiffies; // Not used by hrtimer handler. 3059 rtorsup->rtorsu_kt = t; 3060 } 3061 3062 /* 3063 * RCU torture up/down reader kthread, starting RCU readers in kthread 3064 * context and ending them in hrtimer handlers. Otherwise similar to 3065 * rcu_torture_reader(). 3066 */ 3067 static int 3068 rcu_torture_updown(void *arg) 3069 { 3070 unsigned long j; 3071 struct rcu_torture_one_read_state_updown *rtorsup; 3072 3073 VERBOSE_TOROUT_STRING("rcu_torture_updown task started"); 3074 do { 3075 for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) { 3076 if (torture_must_stop()) 3077 break; 3078 j = smp_load_acquire(&jiffies); // Time before ->rtorsu_inuse. 3079 if (smp_load_acquire(&rtorsup->rtorsu_inuse)) { 3080 WARN_ONCE(time_after(j, rtorsup->rtorsu_j + 1 + HZ * 10), 3081 "hrtimer queued at jiffies %lu for %lld ns took %lu jiffies\n", rtorsup->rtorsu_j, rtorsup->rtorsu_kt, j - rtorsup->rtorsu_j); 3082 continue; 3083 } 3084 rcu_torture_updown_one(rtorsup); 3085 } 3086 torture_hrtimeout_ms(1, 1000, &rcu_torture_updown_rand); 3087 stutter_wait("rcu_torture_updown"); 3088 } while (!torture_must_stop()); 3089 rcu_torture_updown_cleanup(); 3090 torture_kthread_stopping("rcu_torture_updown"); 3091 return 0; 3092 } 3093 3094 /* 3095 * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to 3096 * increase race probabilities and fuzzes the interval between toggling. 3097 */ 3098 static int rcu_nocb_toggle(void *arg) 3099 { 3100 int cpu; 3101 int maxcpu = -1; 3102 int oldnice = task_nice(current); 3103 long r; 3104 DEFINE_TORTURE_RANDOM(rand); 3105 ktime_t toggle_delay; 3106 unsigned long toggle_fuzz; 3107 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); 3108 3109 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); 3110 while (!rcu_inkernel_boot_has_ended()) 3111 schedule_timeout_interruptible(HZ / 10); 3112 for_each_possible_cpu(cpu) 3113 maxcpu = cpu; 3114 WARN_ON(maxcpu < 0); 3115 if (toggle_interval > ULONG_MAX) 3116 toggle_fuzz = ULONG_MAX >> 3; 3117 else 3118 toggle_fuzz = toggle_interval >> 3; 3119 if (toggle_fuzz <= 0) 3120 toggle_fuzz = NSEC_PER_USEC; 3121 do { 3122 r = torture_random(&rand); 3123 cpu = (r >> 1) % (maxcpu + 1); 3124 if (r & 0x1) { 3125 rcu_nocb_cpu_offload(cpu); 3126 atomic_long_inc(&n_nocb_offload); 3127 } else { 3128 rcu_nocb_cpu_deoffload(cpu); 3129 atomic_long_inc(&n_nocb_deoffload); 3130 } 3131 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; 3132 set_current_state(TASK_INTERRUPTIBLE); 3133 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); 3134 if (stutter_wait("rcu_nocb_toggle")) 3135 sched_set_normal(current, oldnice); 3136 } while (!torture_must_stop()); 3137 torture_kthread_stopping("rcu_nocb_toggle"); 3138 return 0; 3139 } 3140 3141 /* 3142 * Print torture statistics. Caller must ensure that there is only 3143 * one call to this function at a given time!!! This is normally 3144 * accomplished by relying on the module system to only have one copy 3145 * of the module loaded, and then by giving the rcu_torture_stats 3146 * kthread full control (or the init/cleanup functions when rcu_torture_stats 3147 * thread is not running). 3148 */ 3149 static void 3150 rcu_torture_stats_print(void) 3151 { 3152 int cpu; 3153 int i; 3154 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 3155 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 3156 long n_gpwraps = 0; 3157 unsigned long ndowns = 0; 3158 unsigned long nunexpired = 0; 3159 unsigned long nmigrates = 0; 3160 unsigned long nups = 0; 3161 struct rcu_torture *rtcp; 3162 static unsigned long rtcv_snap = ULONG_MAX; 3163 static bool splatted; 3164 struct task_struct *wtp; 3165 3166 for_each_possible_cpu(cpu) { 3167 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 3168 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 3169 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 3170 } 3171 if (cur_ops->get_gpwrap_count) 3172 n_gpwraps += cur_ops->get_gpwrap_count(cpu); 3173 } 3174 if (updownreaders) { 3175 for (i = 0; i < n_up_down; i++) { 3176 ndowns += READ_ONCE(updownreaders[i].rtorsu_ndowns); 3177 nups += READ_ONCE(updownreaders[i].rtorsu_nups); 3178 nunexpired += READ_ONCE(updownreaders[i].rtorsu_inuse); 3179 nmigrates += READ_ONCE(updownreaders[i].rtorsu_nmigrates); 3180 } 3181 } 3182 for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { 3183 if (pipesummary[i] != 0) 3184 break; 3185 } // The value of variable "i" is used later, so don't clobber it! 3186 3187 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 3188 rtcp = rcu_access_pointer(rcu_torture_current); 3189 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 3190 rtcp, 3191 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 3192 rcu_torture_current_version, 3193 list_empty(&rcu_torture_freelist), 3194 atomic_read(&n_rcu_torture_alloc), 3195 atomic_read(&n_rcu_torture_alloc_fail), 3196 atomic_read(&n_rcu_torture_free)); 3197 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ", 3198 atomic_read(&n_rcu_torture_mberror), 3199 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), 3200 n_rcu_torture_barrier_error, 3201 n_rcu_torture_boost_ktrerror); 3202 pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ", 3203 n_rcu_torture_boost_failure, 3204 n_rcu_torture_boosts, 3205 atomic_long_read(&n_rcu_torture_timers), 3206 atomic_long_read(&n_rcu_torture_irqs)); 3207 if (updownreaders) 3208 pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); 3209 torture_onoff_stats(); 3210 pr_cont("barrier: %ld/%ld:%ld ", 3211 data_race(n_barrier_successes), 3212 data_race(n_barrier_attempts), 3213 data_race(n_rcu_torture_barrier_error)); 3214 pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. 3215 pr_cont("nmi-calls: %ld nmi-cbs: %ld ", 3216 atomic_long_read(&n_rcu_torture_nmi_call), 3217 atomic_long_read(&n_rcu_torture_nmi_cb)); 3218 pr_cont("nocb-toggles: %ld:%ld ", 3219 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); 3220 pr_cont("gpwraps: %ld\n", n_gpwraps); 3221 3222 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 3223 if (atomic_read(&n_rcu_torture_mberror) || 3224 atomic_read(&n_rcu_torture_mbchk_fail) || 3225 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 3226 n_rcu_torture_boost_failure || i > 1) { 3227 pr_cont("%s", "!!! "); 3228 atomic_inc(&n_rcu_torture_error); 3229 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 3230 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); 3231 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 3232 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 3233 WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) 3234 WARN_ON_ONCE(i > 1); // Too-short grace period 3235 } 3236 pr_cont("Reader Pipe: "); 3237 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 3238 pr_cont(" %ld", pipesummary[i]); 3239 pr_cont("\n"); 3240 3241 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 3242 pr_cont("Reader Batch: "); 3243 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 3244 pr_cont(" %ld", batchsummary[i]); 3245 pr_cont("\n"); 3246 3247 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 3248 pr_cont("Free-Block Circulation: "); 3249 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 3250 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 3251 } 3252 pr_cont("\n"); 3253 3254 if (cur_ops->stats) 3255 cur_ops->stats(); 3256 if (rtcv_snap == rcu_torture_current_version && 3257 rcu_access_pointer(rcu_torture_current) && 3258 !rcu_stall_is_suppressed() && 3259 rcu_inkernel_boot_has_ended()) { 3260 int __maybe_unused flags = 0; 3261 unsigned long __maybe_unused gp_seq = 0; 3262 3263 if (cur_ops->get_gp_data) 3264 cur_ops->get_gp_data(&flags, &gp_seq); 3265 wtp = READ_ONCE(writer_task); 3266 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", 3267 rcu_torture_writer_state_getname(), 3268 rcu_torture_writer_state, gp_seq, flags, 3269 wtp == NULL ? '?' : task_state_to_char(wtp), 3270 wtp == NULL ? -1 : (int)task_cpu(wtp)); 3271 if (!splatted && wtp) { 3272 sched_show_task(wtp); 3273 splatted = true; 3274 } 3275 if (cur_ops->gp_kthread_dbg) 3276 cur_ops->gp_kthread_dbg(); 3277 rcu_ftrace_dump(DUMP_ALL); 3278 } 3279 rtcv_snap = rcu_torture_current_version; 3280 } 3281 3282 /* 3283 * Periodically prints torture statistics, if periodic statistics printing 3284 * was specified via the stat_interval module parameter. 3285 */ 3286 static int 3287 rcu_torture_stats(void *arg) 3288 { 3289 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 3290 do { 3291 schedule_timeout_interruptible(stat_interval * HZ); 3292 rcu_torture_stats_print(); 3293 torture_shutdown_absorb("rcu_torture_stats"); 3294 } while (!torture_must_stop()); 3295 torture_kthread_stopping("rcu_torture_stats"); 3296 return 0; 3297 } 3298 3299 /* Test mem_dump_obj() and friends. */ 3300 static void rcu_torture_mem_dump_obj(void) 3301 { 3302 struct rcu_head *rhp; 3303 struct kmem_cache *kcp; 3304 static int z; 3305 3306 kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); 3307 if (WARN_ON_ONCE(!kcp)) 3308 return; 3309 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); 3310 if (WARN_ON_ONCE(!rhp)) { 3311 kmem_cache_destroy(kcp); 3312 return; 3313 } 3314 pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); 3315 pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); 3316 mem_dump_obj(ZERO_SIZE_PTR); 3317 pr_alert("mem_dump_obj(NULL):"); 3318 mem_dump_obj(NULL); 3319 pr_alert("mem_dump_obj(%px):", &rhp); 3320 mem_dump_obj(&rhp); 3321 pr_alert("mem_dump_obj(%px):", rhp); 3322 mem_dump_obj(rhp); 3323 pr_alert("mem_dump_obj(%px):", &rhp->func); 3324 mem_dump_obj(&rhp->func); 3325 pr_alert("mem_dump_obj(%px):", &z); 3326 mem_dump_obj(&z); 3327 kmem_cache_free(kcp, rhp); 3328 kmem_cache_destroy(kcp); 3329 rhp = kmalloc_obj(*rhp); 3330 if (WARN_ON_ONCE(!rhp)) 3331 return; 3332 pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 3333 pr_alert("mem_dump_obj(kmalloc %px):", rhp); 3334 mem_dump_obj(rhp); 3335 pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); 3336 mem_dump_obj(&rhp->func); 3337 kfree(rhp); 3338 rhp = vmalloc(4096); 3339 if (WARN_ON_ONCE(!rhp)) 3340 return; 3341 pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 3342 pr_alert("mem_dump_obj(vmalloc %px):", rhp); 3343 mem_dump_obj(rhp); 3344 pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); 3345 mem_dump_obj(&rhp->func); 3346 vfree(rhp); 3347 } 3348 3349 static void 3350 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 3351 { 3352 pr_alert("%s" TORTURE_FLAG 3353 "--- %s: nreaders=%d nwriters=%d nfakewriters=%d " 3354 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 3355 "shuffle_interval=%d stutter=%d irqreader=%d " 3356 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 3357 "test_boost=%d/%d test_boost_interval=%d " 3358 "test_boost_duration=%d test_boost_holdoff=%d shutdown_secs=%d " 3359 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 3360 "stall_cpu_block=%d stall_cpu_repeat=%d " 3361 "n_barrier_cbs=%d " 3362 "onoff_interval=%d onoff_holdoff=%d " 3363 "read_exit_delay=%d read_exit_burst=%d " 3364 "reader_flavor=%x " 3365 "nocbs_nthreads=%d nocbs_toggle=%d " 3366 "test_nmis=%d nmi_calls=%d " 3367 "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", 3368 torture_type, tag, nrealreaders, nwriters, nrealfakewriters, 3369 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 3370 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 3371 test_boost, cur_ops->can_boost, 3372 test_boost_interval, test_boost_duration, test_boost_holdoff, shutdown_secs, 3373 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 3374 stall_cpu_block, stall_cpu_repeat, 3375 n_barrier_cbs, 3376 onoff_interval, onoff_holdoff, 3377 read_exit_delay, read_exit_burst, 3378 reader_flavor, 3379 nocbs_nthreads, nocbs_toggle, 3380 test_nmis, nmi_calls, 3381 preempt_duration, preempt_interval, n_up_down); 3382 } 3383 3384 static int rcutorture_booster_cleanup(unsigned int cpu) 3385 { 3386 struct task_struct *t; 3387 3388 if (boost_tasks[cpu] == NULL) 3389 return 0; 3390 mutex_lock(&boost_mutex); 3391 t = boost_tasks[cpu]; 3392 boost_tasks[cpu] = NULL; 3393 rcu_torture_enable_rt_throttle(); 3394 mutex_unlock(&boost_mutex); 3395 3396 /* This must be outside of the mutex, otherwise deadlock! */ 3397 torture_stop_kthread(rcu_torture_boost, t); 3398 return 0; 3399 } 3400 3401 static int rcutorture_booster_init(unsigned int cpu) 3402 { 3403 int retval; 3404 3405 if (boost_tasks[cpu] != NULL) 3406 return 0; /* Already created, nothing more to do. */ 3407 3408 // Testing RCU priority boosting requires rcutorture do 3409 // some serious abuse. Counter this by running ksoftirqd 3410 // at higher priority. 3411 if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { 3412 struct sched_param sp; 3413 struct task_struct *t; 3414 3415 t = per_cpu(ksoftirqd, cpu); 3416 WARN_ON_ONCE(!t); 3417 sp.sched_priority = 2; 3418 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 3419 #ifdef CONFIG_IRQ_FORCED_THREADING 3420 if (force_irqthreads()) { 3421 t = per_cpu(ktimerd, cpu); 3422 WARN_ON_ONCE(!t); 3423 sp.sched_priority = 2; 3424 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 3425 } 3426 #endif 3427 } 3428 3429 /* Don't allow time recalculation while creating a new task. */ 3430 mutex_lock(&boost_mutex); 3431 rcu_torture_disable_rt_throttle(); 3432 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 3433 boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, 3434 cpu, "rcu_torture_boost_%u"); 3435 if (IS_ERR(boost_tasks[cpu])) { 3436 retval = PTR_ERR(boost_tasks[cpu]); 3437 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 3438 n_rcu_torture_boost_ktrerror++; 3439 boost_tasks[cpu] = NULL; 3440 mutex_unlock(&boost_mutex); 3441 return retval; 3442 } 3443 mutex_unlock(&boost_mutex); 3444 return 0; 3445 } 3446 3447 static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) 3448 { 3449 pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr); 3450 return NOTIFY_OK; 3451 } 3452 3453 static struct notifier_block rcu_torture_stall_block = { 3454 .notifier_call = rcu_torture_stall_nf, 3455 }; 3456 3457 /* 3458 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 3459 * induces a CPU stall for the time specified by stall_cpu. If a new 3460 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. 3461 */ 3462 static void rcu_torture_stall_one(int rep, int irqsoff) 3463 { 3464 int idx; 3465 unsigned long stop_at; 3466 3467 if (stall_cpu_holdoff > 0) { 3468 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 3469 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 3470 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 3471 } 3472 if (!kthread_should_stop() && stall_gp_kthread > 0) { 3473 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); 3474 rcu_gp_set_torture_wait(stall_gp_kthread * HZ); 3475 for (idx = 0; idx < stall_gp_kthread + 2; idx++) { 3476 if (kthread_should_stop()) 3477 break; 3478 schedule_timeout_uninterruptible(HZ); 3479 } 3480 } 3481 if (!kthread_should_stop() && stall_cpu > 0) { 3482 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); 3483 stop_at = ktime_get_seconds() + stall_cpu; 3484 /* RCU CPU stall is expected behavior in following code. */ 3485 idx = cur_ops->readlock(); 3486 if (irqsoff) 3487 local_irq_disable(); 3488 else if (!stall_cpu_block) 3489 preempt_disable(); 3490 pr_alert("%s start stall episode %d on CPU %d.\n", 3491 __func__, rep + 1, raw_smp_processor_id()); 3492 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) && 3493 !kthread_should_stop()) 3494 if (stall_cpu_block) { 3495 #ifdef CONFIG_PREEMPTION 3496 preempt_schedule(); 3497 #else 3498 schedule_timeout_uninterruptible(HZ); 3499 #endif 3500 } else if (stall_no_softlockup) { 3501 touch_softlockup_watchdog(); 3502 } 3503 if (irqsoff) 3504 local_irq_enable(); 3505 else if (!stall_cpu_block) 3506 preempt_enable(); 3507 cur_ops->readunlock(idx); 3508 } 3509 } 3510 3511 /* 3512 * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many 3513 * additional times as specified by the stall_cpu_repeat module parameter. 3514 * Note that stall_cpu_irqsoff is ignored on the second and subsequent 3515 * stall. 3516 */ 3517 static int rcu_torture_stall(void *args) 3518 { 3519 int i; 3520 int repeat = stall_cpu_repeat; 3521 int ret; 3522 3523 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 3524 if (repeat < 0) { 3525 repeat = 0; 3526 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 3527 } 3528 if (rcu_cpu_stall_notifiers) { 3529 ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block); 3530 if (ret) 3531 pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n", 3532 __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : ""); 3533 } 3534 for (i = 0; i <= repeat; i++) { 3535 if (kthread_should_stop()) 3536 break; 3537 rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0); 3538 } 3539 pr_alert("%s end.\n", __func__); 3540 if (rcu_cpu_stall_notifiers && !ret) { 3541 ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block); 3542 if (ret) 3543 pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret); 3544 } 3545 torture_shutdown_absorb("rcu_torture_stall"); 3546 while (!kthread_should_stop()) 3547 schedule_timeout_interruptible(10 * HZ); 3548 return 0; 3549 } 3550 3551 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 3552 static int __init rcu_torture_stall_init(void) 3553 { 3554 if (stall_cpu <= 0 && stall_gp_kthread <= 0) 3555 return 0; 3556 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 3557 } 3558 3559 /* State structure for forward-progress self-propagating RCU callback. */ 3560 struct fwd_cb_state { 3561 struct rcu_head rh; 3562 int stop; 3563 }; 3564 3565 /* 3566 * Forward-progress self-propagating RCU callback function. Because 3567 * callbacks run from softirq, this function is an implicit RCU read-side 3568 * critical section. 3569 */ 3570 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 3571 { 3572 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 3573 3574 if (READ_ONCE(fcsp->stop)) { 3575 WRITE_ONCE(fcsp->stop, 2); 3576 return; 3577 } 3578 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 3579 } 3580 3581 /* State for continuous-flood RCU callbacks. */ 3582 struct rcu_fwd_cb { 3583 struct rcu_head rh; 3584 struct rcu_fwd_cb *rfc_next; 3585 struct rcu_fwd *rfc_rfp; 3586 int rfc_gps; 3587 }; 3588 3589 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 3590 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 3591 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 3592 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 3593 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 3594 3595 struct rcu_launder_hist { 3596 long n_launders; 3597 unsigned long launder_gp_seq; 3598 }; 3599 3600 struct rcu_fwd { 3601 spinlock_t rcu_fwd_lock; 3602 struct rcu_fwd_cb *rcu_fwd_cb_head; 3603 struct rcu_fwd_cb **rcu_fwd_cb_tail; 3604 long n_launders_cb; 3605 unsigned long rcu_fwd_startat; 3606 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 3607 unsigned long rcu_launder_gp_seq_start; 3608 int rcu_fwd_id; 3609 }; 3610 3611 static DEFINE_MUTEX(rcu_fwd_mutex); 3612 static struct rcu_fwd *rcu_fwds; 3613 static unsigned long rcu_fwd_seq; 3614 static atomic_long_t rcu_fwd_max_cbs; 3615 static bool rcu_fwd_emergency_stop; 3616 3617 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 3618 { 3619 unsigned long gps; 3620 unsigned long gps_old; 3621 int i; 3622 int j; 3623 3624 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 3625 if (rfp->n_launders_hist[i].n_launders > 0) 3626 break; 3627 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", 3628 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); 3629 gps_old = rfp->rcu_launder_gp_seq_start; 3630 for (j = 0; j <= i; j++) { 3631 gps = rfp->n_launders_hist[j].launder_gp_seq; 3632 pr_cont(" %ds/%d: %ld:%ld", 3633 j + 1, FWD_CBS_HIST_DIV, 3634 rfp->n_launders_hist[j].n_launders, 3635 rcutorture_seq_diff(gps, gps_old)); 3636 gps_old = gps; 3637 } 3638 pr_cont("\n"); 3639 } 3640 3641 /* Callback function for continuous-flood RCU callbacks. */ 3642 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 3643 { 3644 unsigned long flags; 3645 int i; 3646 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 3647 struct rcu_fwd_cb **rfcpp; 3648 struct rcu_fwd *rfp = rfcp->rfc_rfp; 3649 3650 rfcp->rfc_next = NULL; 3651 rfcp->rfc_gps++; 3652 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 3653 rfcpp = rfp->rcu_fwd_cb_tail; 3654 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 3655 smp_store_release(rfcpp, rfcp); 3656 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 3657 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 3658 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 3659 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 3660 rfp->n_launders_hist[i].n_launders++; 3661 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 3662 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3663 } 3664 3665 // Give the scheduler a chance, even on nohz_full CPUs. 3666 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 3667 { 3668 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 3669 // Real call_rcu() floods hit userspace, so emulate that. 3670 if (need_resched() || (iter & 0xfff)) 3671 schedule(); 3672 return; 3673 } 3674 // No userspace emulation: CB invocation throttles call_rcu() 3675 cond_resched(); 3676 } 3677 3678 /* 3679 * Free all callbacks on the rcu_fwd_cb_head list, either because the 3680 * test is over or because we hit an OOM event. 3681 */ 3682 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 3683 { 3684 unsigned long flags; 3685 unsigned long freed = 0; 3686 struct rcu_fwd_cb *rfcp; 3687 3688 for (;;) { 3689 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 3690 rfcp = rfp->rcu_fwd_cb_head; 3691 if (!rfcp) { 3692 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3693 break; 3694 } 3695 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 3696 if (!rfp->rcu_fwd_cb_head) 3697 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 3698 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 3699 kfree(rfcp); 3700 freed++; 3701 rcu_torture_fwd_prog_cond_resched(freed); 3702 if (tick_nohz_full_enabled()) { 3703 local_irq_save(flags); 3704 rcu_momentary_eqs(); 3705 local_irq_restore(flags); 3706 } 3707 } 3708 return freed; 3709 } 3710 3711 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 3712 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 3713 int *tested, int *tested_tries) 3714 { 3715 unsigned long cver; 3716 unsigned long dur; 3717 struct fwd_cb_state fcs; 3718 unsigned long gps; 3719 int idx; 3720 int sd; 3721 int sd4; 3722 bool selfpropcb = false; 3723 unsigned long stopat; 3724 static DEFINE_TORTURE_RANDOM(trs); 3725 3726 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3727 if (!cur_ops->sync) 3728 return; // Cannot do need_resched() forward progress testing without ->sync. 3729 if (cur_ops->call && cur_ops->cb_barrier) { 3730 init_rcu_head_on_stack(&fcs.rh); 3731 selfpropcb = true; 3732 } 3733 3734 /* Tight loop containing cond_resched(). */ 3735 atomic_inc(&rcu_fwd_cb_nodelay); 3736 cur_ops->sync(); /* Later readers see above write. */ 3737 if (selfpropcb) { 3738 WRITE_ONCE(fcs.stop, 0); 3739 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 3740 } 3741 cver = READ_ONCE(rcu_torture_current_version); 3742 gps = cur_ops->get_gp_seq(); 3743 sd = cur_ops->stall_dur() + 1; 3744 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 3745 dur = sd4 + torture_random(&trs) % (sd - sd4); 3746 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3747 stopat = rfp->rcu_fwd_startat + dur; 3748 while (time_before(jiffies, stopat) && 3749 !shutdown_time_arrived() && 3750 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3751 idx = cur_ops->readlock(); 3752 udelay(10); 3753 cur_ops->readunlock(idx); 3754 if (!fwd_progress_need_resched || need_resched()) 3755 cond_resched(); 3756 } 3757 (*tested_tries)++; 3758 if (!time_before(jiffies, stopat) && 3759 !shutdown_time_arrived() && 3760 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3761 (*tested)++; 3762 cver = READ_ONCE(rcu_torture_current_version) - cver; 3763 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3764 WARN_ON(!cver && gps < 2); 3765 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, 3766 rfp->rcu_fwd_id, dur, cver, gps); 3767 } 3768 if (selfpropcb) { 3769 WRITE_ONCE(fcs.stop, 1); 3770 cur_ops->sync(); /* Wait for running CB to complete. */ 3771 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3772 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 3773 } 3774 3775 if (selfpropcb) { 3776 WARN_ON(READ_ONCE(fcs.stop) != 2); 3777 destroy_rcu_head_on_stack(&fcs.rh); 3778 } 3779 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 3780 atomic_dec(&rcu_fwd_cb_nodelay); 3781 } 3782 3783 /* Carry out call_rcu() forward-progress testing. */ 3784 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 3785 { 3786 unsigned long cver; 3787 unsigned long flags; 3788 unsigned long gps; 3789 int i; 3790 long n_launders; 3791 long n_launders_cb_snap; 3792 long n_launders_sa; 3793 long n_max_cbs; 3794 long n_max_gps; 3795 struct rcu_fwd_cb *rfcp; 3796 struct rcu_fwd_cb *rfcpn; 3797 unsigned long stopat; 3798 unsigned long stoppedat; 3799 3800 if (READ_ONCE(rcu_fwd_emergency_stop)) { 3801 pr_alert("%s: Emergency stop, so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3802 return; /* Get out of the way quickly, no GP wait! */ 3803 } 3804 if (!cur_ops->call) { 3805 pr_alert("%s: No ->call(), so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3806 return; /* Can't do call_rcu() fwd prog without ->call. */ 3807 } 3808 3809 /* Loop continuously posting RCU callbacks. */ 3810 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3811 atomic_inc(&rcu_fwd_cb_nodelay); 3812 cur_ops->sync(); /* Later readers see above write. */ 3813 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3814 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 3815 n_launders = 0; 3816 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 3817 n_launders_sa = 0; 3818 n_max_cbs = 0; 3819 n_max_gps = 0; 3820 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 3821 rfp->n_launders_hist[i].n_launders = 0; 3822 cver = READ_ONCE(rcu_torture_current_version); 3823 gps = cur_ops->get_gp_seq(); 3824 rfp->rcu_launder_gp_seq_start = gps; 3825 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 3826 while (time_before(jiffies, stopat) && 3827 !shutdown_time_arrived() && 3828 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3829 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 3830 rfcpn = NULL; 3831 if (rfcp) 3832 rfcpn = READ_ONCE(rfcp->rfc_next); 3833 if (rfcpn) { 3834 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 3835 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 3836 break; 3837 rfp->rcu_fwd_cb_head = rfcpn; 3838 n_launders++; 3839 n_launders_sa++; 3840 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { 3841 rfcp = kmalloc_obj(*rfcp); 3842 if (WARN_ON_ONCE(!rfcp)) { 3843 schedule_timeout_interruptible(1); 3844 continue; 3845 } 3846 n_max_cbs++; 3847 n_launders_sa = 0; 3848 rfcp->rfc_gps = 0; 3849 rfcp->rfc_rfp = rfp; 3850 } else { 3851 rfcp = NULL; 3852 } 3853 if (rfcp) 3854 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 3855 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 3856 if (tick_nohz_full_enabled()) { 3857 local_irq_save(flags); 3858 rcu_momentary_eqs(); 3859 local_irq_restore(flags); 3860 } 3861 } 3862 stoppedat = jiffies; 3863 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 3864 cver = READ_ONCE(rcu_torture_current_version) - cver; 3865 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3866 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3867 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 3868 (void)rcu_torture_fwd_prog_cbfree(rfp); 3869 3870 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 3871 !shutdown_time_arrived()) { 3872 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg) 3873 cur_ops->gp_kthread_dbg(); 3874 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n", 3875 __func__, 3876 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 3877 n_launders + n_max_cbs - n_launders_cb_snap, 3878 n_launders, n_launders_sa, 3879 n_max_gps, n_max_cbs, cver, gps, num_online_cpus()); 3880 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); 3881 mutex_lock(&rcu_fwd_mutex); // Serialize histograms. 3882 rcu_torture_fwd_cb_hist(rfp); 3883 mutex_unlock(&rcu_fwd_mutex); 3884 } 3885 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 3886 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 3887 atomic_dec(&rcu_fwd_cb_nodelay); 3888 } 3889 3890 3891 /* 3892 * OOM notifier, but this only prints diagnostic information for the 3893 * current forward-progress test. 3894 */ 3895 static int rcutorture_oom_notify(struct notifier_block *self, 3896 unsigned long notused, void *nfreed) 3897 { 3898 int i; 3899 long ncbs; 3900 struct rcu_fwd *rfp; 3901 3902 mutex_lock(&rcu_fwd_mutex); 3903 rfp = rcu_fwds; 3904 if (!rfp) { 3905 mutex_unlock(&rcu_fwd_mutex); 3906 return NOTIFY_OK; 3907 } 3908 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 3909 __func__); 3910 for (i = 0; i < fwd_progress; i++) { 3911 rcu_torture_fwd_cb_hist(&rfp[i]); 3912 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); 3913 } 3914 WRITE_ONCE(rcu_fwd_emergency_stop, true); 3915 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 3916 ncbs = 0; 3917 for (i = 0; i < fwd_progress; i++) 3918 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3919 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3920 cur_ops->cb_barrier(); 3921 ncbs = 0; 3922 for (i = 0; i < fwd_progress; i++) 3923 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3924 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3925 cur_ops->cb_barrier(); 3926 ncbs = 0; 3927 for (i = 0; i < fwd_progress; i++) 3928 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3929 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3930 smp_mb(); /* Frees before return to avoid redoing OOM. */ 3931 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 3932 pr_info("%s returning after OOM processing.\n", __func__); 3933 mutex_unlock(&rcu_fwd_mutex); 3934 return NOTIFY_OK; 3935 } 3936 3937 static struct notifier_block rcutorture_oom_nb = { 3938 .notifier_call = rcutorture_oom_notify 3939 }; 3940 3941 /* Carry out grace-period forward-progress testing. */ 3942 static int rcu_torture_fwd_prog(void *args) 3943 { 3944 bool firsttime = true; 3945 long max_cbs; 3946 int oldnice = task_nice(current); 3947 unsigned long oldseq = READ_ONCE(rcu_fwd_seq); 3948 struct rcu_fwd *rfp = args; 3949 int tested = 0; 3950 int tested_tries = 0; 3951 3952 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 3953 while (!rcu_inkernel_boot_has_ended()) 3954 schedule_timeout_interruptible(HZ / 10); 3955 rcu_bind_current_to_nocb(); 3956 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 3957 set_user_nice(current, MAX_NICE); 3958 do { 3959 if (!rfp->rcu_fwd_id) { 3960 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 3961 WRITE_ONCE(rcu_fwd_emergency_stop, false); 3962 if (!firsttime) { 3963 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); 3964 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); 3965 } 3966 firsttime = false; 3967 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); 3968 } else { 3969 while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) 3970 schedule_timeout_interruptible(HZ / 20); 3971 oldseq = READ_ONCE(rcu_fwd_seq); 3972 } 3973 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3974 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) 3975 rcu_torture_fwd_prog_cr(rfp); 3976 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && 3977 (!IS_ENABLED(CONFIG_TINY_RCU) || 3978 (rcu_inkernel_boot_has_ended() && 3979 torture_num_online_cpus() > rfp->rcu_fwd_id))) 3980 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 3981 3982 /* Avoid slow periods, better to test when busy. */ 3983 if (stutter_wait("rcu_torture_fwd_prog")) 3984 sched_set_normal(current, oldnice); 3985 } while (!torture_must_stop()); 3986 /* Short runs might not contain a valid forward-progress attempt. */ 3987 if (!rfp->rcu_fwd_id) { 3988 WARN_ON(!tested && tested_tries >= 5); 3989 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 3990 } 3991 torture_kthread_stopping("rcu_torture_fwd_prog"); 3992 return 0; 3993 } 3994 3995 /* If forward-progress checking is requested and feasible, spawn the thread. */ 3996 static int __init rcu_torture_fwd_prog_init(void) 3997 { 3998 int i; 3999 int ret = 0; 4000 struct rcu_fwd *rfp; 4001 4002 if (!fwd_progress) 4003 return 0; /* Not requested, so don't do it. */ 4004 if (fwd_progress >= nr_cpu_ids) { 4005 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); 4006 fwd_progress = nr_cpu_ids; 4007 } else if (fwd_progress < 0) { 4008 fwd_progress = nr_cpu_ids; 4009 } 4010 if ((!cur_ops->sync && !cur_ops->call) || 4011 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || 4012 cur_ops == &rcu_busted_ops) { 4013 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 4014 fwd_progress = 0; 4015 return 0; 4016 } 4017 if (stall_cpu > 0 || (preempt_duration > 0 && IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 4018 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall and/or preemption testing"); 4019 fwd_progress = 0; 4020 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 4021 return -EINVAL; /* In module, can fail back to user. */ 4022 WARN_ON(1); /* Make sure rcutorture scripting notices conflict. */ 4023 return 0; 4024 } 4025 if (fwd_progress_holdoff <= 0) 4026 fwd_progress_holdoff = 1; 4027 if (fwd_progress_div < 2) 4028 fwd_progress_div = 4; 4029 rfp = kzalloc_objs(*rfp, fwd_progress); 4030 fwd_prog_tasks = kzalloc_objs(*fwd_prog_tasks, fwd_progress); 4031 if (!rfp || !fwd_prog_tasks) { 4032 kfree(rfp); 4033 kfree(fwd_prog_tasks); 4034 fwd_prog_tasks = NULL; 4035 fwd_progress = 0; 4036 return -ENOMEM; 4037 } 4038 for (i = 0; i < fwd_progress; i++) { 4039 spin_lock_init(&rfp[i].rcu_fwd_lock); 4040 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; 4041 rfp[i].rcu_fwd_id = i; 4042 } 4043 mutex_lock(&rcu_fwd_mutex); 4044 rcu_fwds = rfp; 4045 mutex_unlock(&rcu_fwd_mutex); 4046 register_oom_notifier(&rcutorture_oom_nb); 4047 for (i = 0; i < fwd_progress; i++) { 4048 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); 4049 if (ret) { 4050 fwd_progress = i; 4051 return ret; 4052 } 4053 } 4054 return 0; 4055 } 4056 4057 static void rcu_torture_fwd_prog_cleanup(void) 4058 { 4059 int i; 4060 struct rcu_fwd *rfp; 4061 4062 if (!rcu_fwds || !fwd_prog_tasks) 4063 return; 4064 for (i = 0; i < fwd_progress; i++) 4065 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); 4066 unregister_oom_notifier(&rcutorture_oom_nb); 4067 mutex_lock(&rcu_fwd_mutex); 4068 rfp = rcu_fwds; 4069 rcu_fwds = NULL; 4070 mutex_unlock(&rcu_fwd_mutex); 4071 kfree(rfp); 4072 kfree(fwd_prog_tasks); 4073 fwd_prog_tasks = NULL; 4074 } 4075 4076 /* Callback function for RCU barrier testing. */ 4077 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 4078 { 4079 atomic_inc(&barrier_cbs_invoked); 4080 } 4081 4082 /* IPI handler to get callback posted on desired CPU, if online. */ 4083 static int rcu_torture_barrier1cb(void *rcu_void) 4084 { 4085 struct rcu_head *rhp = rcu_void; 4086 4087 cur_ops->call(rhp, rcu_torture_barrier_cbf); 4088 return 0; 4089 } 4090 4091 /* kthread function to register callbacks used to test RCU barriers. */ 4092 static int rcu_torture_barrier_cbs(void *arg) 4093 { 4094 long myid = (long)arg; 4095 bool lastphase = false; 4096 bool newphase; 4097 struct rcu_head rcu; 4098 4099 init_rcu_head_on_stack(&rcu); 4100 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 4101 set_user_nice(current, MAX_NICE); 4102 do { 4103 wait_event(barrier_cbs_wq[myid], 4104 (newphase = 4105 smp_load_acquire(&barrier_phase)) != lastphase || 4106 torture_must_stop()); 4107 lastphase = newphase; 4108 if (torture_must_stop()) 4109 break; 4110 /* 4111 * The above smp_load_acquire() ensures barrier_phase load 4112 * is ordered before the following ->call(). 4113 */ 4114 if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1)) 4115 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 4116 4117 if (atomic_dec_and_test(&barrier_cbs_count)) 4118 wake_up(&barrier_wq); 4119 } while (!torture_must_stop()); 4120 if (cur_ops->cb_barrier != NULL) 4121 cur_ops->cb_barrier(); 4122 destroy_rcu_head_on_stack(&rcu); 4123 torture_kthread_stopping("rcu_torture_barrier_cbs"); 4124 return 0; 4125 } 4126 4127 /* kthread function to drive and coordinate RCU barrier testing. */ 4128 static int rcu_torture_barrier(void *arg) 4129 { 4130 int i; 4131 4132 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 4133 do { 4134 atomic_set(&barrier_cbs_invoked, 0); 4135 atomic_set(&barrier_cbs_count, n_barrier_cbs); 4136 /* Ensure barrier_phase ordered after prior assignments. */ 4137 smp_store_release(&barrier_phase, !barrier_phase); 4138 for (i = 0; i < n_barrier_cbs; i++) 4139 wake_up(&barrier_cbs_wq[i]); 4140 wait_event(barrier_wq, 4141 atomic_read(&barrier_cbs_count) == 0 || 4142 torture_must_stop()); 4143 if (torture_must_stop()) 4144 break; 4145 n_barrier_attempts++; 4146 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 4147 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 4148 n_rcu_torture_barrier_error++; 4149 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 4150 atomic_read(&barrier_cbs_invoked), 4151 n_barrier_cbs); 4152 WARN_ON(1); 4153 // Wait manually for the remaining callbacks 4154 i = 0; 4155 do { 4156 if (WARN_ON(i++ > HZ)) 4157 i = INT_MIN; 4158 schedule_timeout_interruptible(1); 4159 cur_ops->cb_barrier(); 4160 } while (atomic_read(&barrier_cbs_invoked) != 4161 n_barrier_cbs && 4162 !torture_must_stop()); 4163 smp_mb(); // Can't trust ordering if broken. 4164 if (!torture_must_stop()) 4165 pr_err("Recovered: barrier_cbs_invoked = %d\n", 4166 atomic_read(&barrier_cbs_invoked)); 4167 } else { 4168 n_barrier_successes++; 4169 } 4170 schedule_timeout_interruptible(HZ / 10); 4171 } while (!torture_must_stop()); 4172 torture_kthread_stopping("rcu_torture_barrier"); 4173 return 0; 4174 } 4175 4176 /* Initialize RCU barrier testing. */ 4177 static int rcu_torture_barrier_init(void) 4178 { 4179 int i; 4180 int ret; 4181 4182 if (n_barrier_cbs <= 0) 4183 return 0; 4184 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 4185 pr_alert("%s" TORTURE_FLAG 4186 " Call or barrier ops missing for %s,\n", 4187 torture_type, cur_ops->name); 4188 pr_alert("%s" TORTURE_FLAG 4189 " RCU barrier testing omitted from run.\n", 4190 torture_type); 4191 return 0; 4192 } 4193 atomic_set(&barrier_cbs_count, 0); 4194 atomic_set(&barrier_cbs_invoked, 0); 4195 barrier_cbs_tasks = 4196 kzalloc_objs(barrier_cbs_tasks[0], n_barrier_cbs); 4197 barrier_cbs_wq = 4198 kzalloc_objs(barrier_cbs_wq[0], n_barrier_cbs); 4199 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 4200 return -ENOMEM; 4201 for (i = 0; i < n_barrier_cbs; i++) { 4202 init_waitqueue_head(&barrier_cbs_wq[i]); 4203 ret = torture_create_kthread(rcu_torture_barrier_cbs, 4204 (void *)(long)i, 4205 barrier_cbs_tasks[i]); 4206 if (ret) 4207 return ret; 4208 } 4209 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 4210 } 4211 4212 /* Clean up after RCU barrier testing. */ 4213 static void rcu_torture_barrier_cleanup(void) 4214 { 4215 int i; 4216 4217 torture_stop_kthread(rcu_torture_barrier, barrier_task); 4218 if (barrier_cbs_tasks != NULL) { 4219 for (i = 0; i < n_barrier_cbs; i++) 4220 torture_stop_kthread(rcu_torture_barrier_cbs, 4221 barrier_cbs_tasks[i]); 4222 kfree(barrier_cbs_tasks); 4223 barrier_cbs_tasks = NULL; 4224 } 4225 if (barrier_cbs_wq != NULL) { 4226 kfree(barrier_cbs_wq); 4227 barrier_cbs_wq = NULL; 4228 } 4229 } 4230 4231 static bool rcu_torture_can_boost(void) 4232 { 4233 static int boost_warn_once; 4234 int prio; 4235 4236 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 4237 return false; 4238 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) 4239 return false; 4240 4241 prio = rcu_get_gp_kthreads_prio(); 4242 if (!prio) 4243 return false; 4244 4245 if (prio < 2) { 4246 if (boost_warn_once == 1) 4247 return false; 4248 4249 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 4250 boost_warn_once = 1; 4251 return false; 4252 } 4253 4254 return true; 4255 } 4256 4257 static bool read_exit_child_stop; 4258 static bool read_exit_child_stopped; 4259 static wait_queue_head_t read_exit_wq; 4260 4261 // Child kthread which just does an rcutorture reader and exits. 4262 static int rcu_torture_read_exit_child(void *trsp_in) 4263 { 4264 struct torture_random_state *trsp = trsp_in; 4265 4266 set_user_nice(current, MAX_NICE); 4267 // Minimize time between reading and exiting. 4268 while (!kthread_should_stop()) 4269 schedule_timeout_uninterruptible(HZ / 20); 4270 (void)rcu_torture_one_read(trsp, -1); 4271 return 0; 4272 } 4273 4274 // Parent kthread which creates and destroys read-exit child kthreads. 4275 static int rcu_torture_read_exit(void *unused) 4276 { 4277 bool errexit = false; 4278 int i; 4279 struct task_struct *tsp; 4280 DEFINE_TORTURE_RANDOM(trs); 4281 4282 // Allocate and initialize. 4283 set_user_nice(current, MAX_NICE); 4284 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); 4285 4286 // Each pass through this loop does one read-exit episode. 4287 do { 4288 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); 4289 for (i = 0; i < read_exit_burst; i++) { 4290 if (READ_ONCE(read_exit_child_stop)) 4291 break; 4292 stutter_wait("rcu_torture_read_exit"); 4293 // Spawn child. 4294 tsp = kthread_run(rcu_torture_read_exit_child, 4295 &trs, "%s", "rcu_torture_read_exit_child"); 4296 if (IS_ERR(tsp)) { 4297 TOROUT_ERRSTRING("out of memory"); 4298 errexit = true; 4299 break; 4300 } 4301 cond_resched(); 4302 kthread_stop(tsp); 4303 n_read_exits++; 4304 } 4305 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); 4306 rcu_barrier(); // Wait for task_struct free, avoid OOM. 4307 i = 0; 4308 for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) 4309 schedule_timeout_uninterruptible(HZ); 4310 } while (!errexit && !READ_ONCE(read_exit_child_stop)); 4311 4312 // Clean up and exit. 4313 smp_store_release(&read_exit_child_stopped, true); // After reaping. 4314 smp_mb(); // Store before wakeup. 4315 wake_up(&read_exit_wq); 4316 while (!torture_must_stop()) 4317 schedule_timeout_uninterruptible(HZ / 20); 4318 torture_kthread_stopping("rcu_torture_read_exit"); 4319 return 0; 4320 } 4321 4322 static int rcu_torture_read_exit_init(void) 4323 { 4324 if (read_exit_burst <= 0) 4325 return 0; 4326 init_waitqueue_head(&read_exit_wq); 4327 read_exit_child_stop = false; 4328 read_exit_child_stopped = false; 4329 return torture_create_kthread(rcu_torture_read_exit, NULL, 4330 read_exit_task); 4331 } 4332 4333 static void rcu_torture_read_exit_cleanup(void) 4334 { 4335 if (!read_exit_task) 4336 return; 4337 WRITE_ONCE(read_exit_child_stop, true); 4338 smp_mb(); // Above write before wait. 4339 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); 4340 torture_stop_kthread(rcutorture_read_exit, read_exit_task); 4341 } 4342 4343 static void rcutorture_test_nmis(int n) 4344 { 4345 #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 4346 int cpu; 4347 int dumpcpu; 4348 int i; 4349 4350 for (i = 0; i < n; i++) { 4351 preempt_disable(); 4352 cpu = smp_processor_id(); 4353 dumpcpu = cpu + 1; 4354 if (dumpcpu >= nr_cpu_ids) 4355 dumpcpu = 0; 4356 pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu); 4357 dump_cpu_task(dumpcpu); 4358 preempt_enable(); 4359 schedule_timeout_uninterruptible(15 * HZ); 4360 } 4361 #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 4362 WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis); 4363 #endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 4364 } 4365 4366 // Randomly preempt online CPUs. 4367 static int rcu_torture_preempt(void *unused) 4368 { 4369 int cpu = -1; 4370 DEFINE_TORTURE_RANDOM(rand); 4371 4372 schedule_timeout_idle(stall_cpu_holdoff); 4373 do { 4374 // Wait for preempt_interval ms with up to 100us fuzz. 4375 torture_hrtimeout_ms(preempt_interval, 100, &rand); 4376 // Select online CPU. 4377 cpu = cpumask_next_wrap(cpu, cpu_online_mask); 4378 WARN_ON_ONCE(cpu >= nr_cpu_ids); 4379 // Move to that CPU, if can't do so, retry later. 4380 if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) 4381 continue; 4382 // Preempt at high-ish priority, then reset to normal. 4383 sched_set_fifo(current); 4384 torture_sched_setaffinity(current->pid, cpu_present_mask, true); 4385 mdelay(preempt_duration); 4386 sched_set_normal(current, 0); 4387 stutter_wait("rcu_torture_preempt"); 4388 } while (!torture_must_stop()); 4389 torture_kthread_stopping("rcu_torture_preempt"); 4390 return 0; 4391 } 4392 4393 static enum cpuhp_state rcutor_hp; 4394 4395 static struct hrtimer gpwrap_lag_timer; 4396 static bool gpwrap_lag_active; 4397 4398 /* Timer handler for toggling RCU grace-period sequence overflow test lag value */ 4399 static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer) 4400 { 4401 ktime_t next_delay; 4402 4403 if (gpwrap_lag_active) { 4404 pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n"); 4405 cur_ops->set_gpwrap_lag(0); 4406 gpwrap_lag_active = false; 4407 next_delay = ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0); 4408 } else { 4409 pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps); 4410 cur_ops->set_gpwrap_lag(gpwrap_lag_gps); 4411 gpwrap_lag_active = true; 4412 next_delay = ktime_set(gpwrap_lag_active_mins * 60, 0); 4413 } 4414 4415 if (torture_must_stop_irq()) 4416 return HRTIMER_NORESTART; 4417 4418 hrtimer_forward_now(timer, next_delay); 4419 return HRTIMER_RESTART; 4420 } 4421 4422 static int rcu_gpwrap_lag_init(void) 4423 { 4424 if (!gpwrap_lag) 4425 return 0; 4426 4427 if (gpwrap_lag_cycle_mins <= 0 || gpwrap_lag_active_mins <= 0) { 4428 pr_alert("rcu-torture: lag timing parameters must be positive\n"); 4429 return -EINVAL; 4430 } 4431 4432 hrtimer_setup(&gpwrap_lag_timer, rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 4433 gpwrap_lag_active = false; 4434 hrtimer_start(&gpwrap_lag_timer, 4435 ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0), HRTIMER_MODE_REL); 4436 4437 return 0; 4438 } 4439 4440 static void rcu_gpwrap_lag_cleanup(void) 4441 { 4442 hrtimer_cancel(&gpwrap_lag_timer); 4443 cur_ops->set_gpwrap_lag(0); 4444 gpwrap_lag_active = false; 4445 } 4446 static void 4447 rcu_torture_cleanup(void) 4448 { 4449 int flags = 0; 4450 unsigned long gp_seq = 0; 4451 int i; 4452 4453 if (torture_cleanup_begin()) { 4454 rcu_torture_nmi_cleanup(); 4455 if (cur_ops->cb_barrier != NULL) { 4456 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 4457 cur_ops->cb_barrier(); 4458 } 4459 if (cur_ops->gp_slow_unregister) 4460 cur_ops->gp_slow_unregister(NULL); 4461 return; 4462 } 4463 if (!cur_ops) { 4464 torture_cleanup_end(); 4465 return; 4466 } 4467 4468 rcutorture_test_nmis(test_nmis); 4469 4470 if (cur_ops->gp_kthread_dbg) 4471 cur_ops->gp_kthread_dbg(); 4472 torture_stop_kthread(rcu_torture_preempt, preempt_task); 4473 rcu_torture_read_exit_cleanup(); 4474 rcu_torture_barrier_cleanup(); 4475 rcu_torture_fwd_prog_cleanup(); 4476 torture_stop_kthread(rcu_torture_stall, stall_task); 4477 torture_stop_kthread(rcu_torture_writer, writer_task); 4478 4479 if (nocb_tasks) { 4480 for (i = 0; i < nrealnocbers; i++) 4481 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); 4482 kfree(nocb_tasks); 4483 nocb_tasks = NULL; 4484 } 4485 4486 if (updown_task) { 4487 torture_stop_kthread(rcu_torture_updown, updown_task); 4488 updown_task = NULL; 4489 } 4490 if (reader_tasks) { 4491 for (i = 0; i < nrealreaders; i++) 4492 torture_stop_kthread(rcu_torture_reader, 4493 reader_tasks[i]); 4494 if (irqreader && cur_ops->irq_capable) 4495 kick_all_cpus_sync(); 4496 kfree(reader_tasks); 4497 reader_tasks = NULL; 4498 } 4499 rcu_torture_nmi_cleanup(); 4500 kfree(rcu_torture_reader_mbchk); 4501 rcu_torture_reader_mbchk = NULL; 4502 4503 if (fakewriter_tasks) { 4504 for (i = 0; i < nrealfakewriters; i++) 4505 torture_stop_kthread(rcu_torture_fakewriter, 4506 fakewriter_tasks[i]); 4507 kfree(fakewriter_tasks); 4508 fakewriter_tasks = NULL; 4509 } 4510 4511 if (cur_ops->get_gp_data) 4512 cur_ops->get_gp_data(&flags, &gp_seq); 4513 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", 4514 cur_ops->name, (long)gp_seq, flags, 4515 rcutorture_seq_diff(gp_seq, start_gp_seq)); 4516 torture_stop_kthread(rcu_torture_stats, stats_task); 4517 torture_stop_kthread(rcu_torture_fqs, fqs_task); 4518 if (rcu_torture_can_boost() && rcutor_hp >= 0) 4519 cpuhp_remove_state(rcutor_hp); 4520 4521 /* 4522 * Wait for all RCU callbacks to fire, then do torture-type-specific 4523 * cleanup operations. 4524 */ 4525 if (cur_ops->cb_barrier != NULL) { 4526 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 4527 cur_ops->cb_barrier(); 4528 } 4529 4530 /* 4531 * cb_barrier() above drained every deferred callback, so the count 4532 * issued from NMI must equal the count invoked. 4533 */ 4534 if (atomic_long_read(&n_rcu_torture_nmi_call) != 4535 atomic_long_read(&n_rcu_torture_nmi_cb)) { 4536 pr_alert("%s: NMI ->call() lost a callback: issued %ld invoked %ld\n", 4537 __func__, atomic_long_read(&n_rcu_torture_nmi_call), 4538 atomic_long_read(&n_rcu_torture_nmi_cb)); 4539 atomic_inc(&n_rcu_torture_error); 4540 } 4541 4542 if (cur_ops->cleanup != NULL) 4543 cur_ops->cleanup(); 4544 4545 rcu_torture_mem_dump_obj(); 4546 4547 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 4548 4549 if (err_segs_recorded) { 4550 pr_alert("Failure/close-call rcutorture reader segments:\n"); 4551 if (rt_read_nsegs == 0) 4552 pr_alert("\t: No segments recorded!!!\n"); 4553 else 4554 rcu_torture_dump_read_segs(err_segs, rt_read_nsegs); 4555 } 4556 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 4557 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 4558 else if (torture_onoff_failures()) 4559 rcu_torture_print_module_parms(cur_ops, 4560 "End of test: RCU_HOTPLUG"); 4561 else 4562 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 4563 torture_cleanup_end(); 4564 if (cur_ops->gp_slow_unregister) 4565 cur_ops->gp_slow_unregister(NULL); 4566 4567 if (gpwrap_lag && cur_ops->set_gpwrap_lag) 4568 rcu_gpwrap_lag_cleanup(); 4569 } 4570 4571 static void rcu_torture_leak_cb(struct rcu_head *rhp) 4572 { 4573 } 4574 4575 static void rcu_torture_err_cb(struct rcu_head *rhp) 4576 { 4577 /* 4578 * This -might- happen due to race conditions, but is unlikely. 4579 * The scenario that leads to this happening is that the 4580 * first of the pair of duplicate callbacks is queued, 4581 * someone else starts a grace period that includes that 4582 * callback, then the second of the pair must wait for the 4583 * next grace period. Unlikely, but can happen. If it 4584 * does happen, the debug-objects subsystem won't have splatted. 4585 */ 4586 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 4587 } 4588 4589 /* 4590 * Verify that double-free causes debug-objects to complain, but only 4591 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 4592 * cannot be carried out. 4593 */ 4594 static void rcu_test_debug_objects(void) 4595 { 4596 struct rcu_head rh1; 4597 struct rcu_head rh2; 4598 int idx; 4599 4600 if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) { 4601 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n", 4602 KBUILD_MODNAME, cur_ops->name); 4603 return; 4604 } 4605 4606 if (WARN_ON_ONCE(cur_ops->debug_objects && 4607 (!cur_ops->call || !cur_ops->cb_barrier))) 4608 return; 4609 4610 struct rcu_head *rhp = kmalloc_obj(*rhp); 4611 4612 init_rcu_head_on_stack(&rh1); 4613 init_rcu_head_on_stack(&rh2); 4614 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name); 4615 4616 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 4617 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */ 4618 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 4619 cur_ops->call(&rh2, rcu_torture_leak_cb); 4620 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 4621 if (rhp) { 4622 cur_ops->call(rhp, rcu_torture_leak_cb); 4623 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ 4624 } 4625 cur_ops->readunlock(idx); 4626 4627 /* Wait for them all to get done so we can safely return. */ 4628 cur_ops->cb_barrier(); 4629 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name); 4630 destroy_rcu_head_on_stack(&rh1); 4631 destroy_rcu_head_on_stack(&rh2); 4632 kfree(rhp); 4633 } 4634 4635 static void rcutorture_sync(void) 4636 { 4637 static unsigned long n; 4638 4639 if (cur_ops->sync && !(++n & 0xfff)) 4640 cur_ops->sync(); 4641 } 4642 4643 static DEFINE_MUTEX(mut0); 4644 static DEFINE_MUTEX(mut1); 4645 static DEFINE_MUTEX(mut2); 4646 static DEFINE_MUTEX(mut3); 4647 static DEFINE_MUTEX(mut4); 4648 static DEFINE_MUTEX(mut5); 4649 static DEFINE_MUTEX(mut6); 4650 static DEFINE_MUTEX(mut7); 4651 static DEFINE_MUTEX(mut8); 4652 static DEFINE_MUTEX(mut9); 4653 4654 static DECLARE_RWSEM(rwsem0); 4655 static DECLARE_RWSEM(rwsem1); 4656 static DECLARE_RWSEM(rwsem2); 4657 static DECLARE_RWSEM(rwsem3); 4658 static DECLARE_RWSEM(rwsem4); 4659 static DECLARE_RWSEM(rwsem5); 4660 static DECLARE_RWSEM(rwsem6); 4661 static DECLARE_RWSEM(rwsem7); 4662 static DECLARE_RWSEM(rwsem8); 4663 static DECLARE_RWSEM(rwsem9); 4664 4665 static DEFINE_RAW_SPINLOCK(spin0); 4666 static DEFINE_RAW_SPINLOCK(spin1); 4667 static DEFINE_RAW_SPINLOCK(spin2); 4668 static DEFINE_RAW_SPINLOCK(spin3); 4669 static DEFINE_RAW_SPINLOCK(spin4); 4670 static DEFINE_RAW_SPINLOCK(spin5); 4671 static DEFINE_RAW_SPINLOCK(spin6); 4672 static DEFINE_RAW_SPINLOCK(spin7); 4673 static DEFINE_RAW_SPINLOCK(spin8); 4674 static DEFINE_RAW_SPINLOCK(spin9); 4675 4676 DEFINE_STATIC_SRCU(srcu0); 4677 DEFINE_STATIC_SRCU(srcu1); 4678 DEFINE_STATIC_SRCU(srcu2); 4679 DEFINE_STATIC_SRCU(srcu3); 4680 DEFINE_STATIC_SRCU(srcu4); 4681 DEFINE_STATIC_SRCU(srcu5); 4682 DEFINE_STATIC_SRCU(srcu6); 4683 DEFINE_STATIC_SRCU(srcu7); 4684 DEFINE_STATIC_SRCU(srcu8); 4685 DEFINE_STATIC_SRCU(srcu9); 4686 4687 DEFINE_STATIC_SRCU_ATOMIC(srcu0_atomic); 4688 DEFINE_STATIC_SRCU_ATOMIC(srcu1_atomic); 4689 DEFINE_STATIC_SRCU_ATOMIC(srcu2_atomic); 4690 DEFINE_STATIC_SRCU_ATOMIC(srcu3_atomic); 4691 DEFINE_STATIC_SRCU_ATOMIC(srcu4_atomic); 4692 DEFINE_STATIC_SRCU_ATOMIC(srcu5_atomic); 4693 DEFINE_STATIC_SRCU_ATOMIC(srcu6_atomic); 4694 DEFINE_STATIC_SRCU_ATOMIC(srcu7_atomic); 4695 DEFINE_STATIC_SRCU_ATOMIC(srcu8_atomic); 4696 DEFINE_STATIC_SRCU_ATOMIC(srcu9_atomic); 4697 4698 DEFINE_STATIC_SRCU_ATOMIC(srcu_irq); 4699 4700 static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, 4701 int cyclelen, int deadlock) 4702 { 4703 int j = i + 1; 4704 4705 if (j >= cyclelen) 4706 j = deadlock ? 0 : -1; 4707 if (j >= 0) 4708 pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i); 4709 else 4710 pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i); 4711 return j; 4712 } 4713 4714 static void srcu_sync_irq(void *unused) 4715 { 4716 synchronize_srcu_atomic(&srcu_irq); 4717 } 4718 4719 // Test lockdep on SRCU-based deadlock scenarios. 4720 static void rcu_torture_init_srcu_lockdep(void) 4721 { 4722 int cyclelen; 4723 int deadlock; 4724 bool err = false; 4725 int i; 4726 int j; 4727 int idx; 4728 struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4, 4729 &mut5, &mut6, &mut7, &mut8, &mut9 }; 4730 struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4, 4731 &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; 4732 struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, 4733 &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; 4734 raw_spinlock_t *spins[] = { &spin0, &spin1, &spin2, &spin3, &spin4, 4735 &spin5, &spin6, &spin7, &spin8, &spin9 }; 4736 struct srcu_struct *srcus_atomic[] = { &srcu0_atomic, &srcu1_atomic, &srcu2_atomic, 4737 &srcu3_atomic, &srcu4_atomic, &srcu5_atomic, 4738 &srcu6_atomic, &srcu7_atomic, &srcu8_atomic, 4739 &srcu9_atomic }; 4740 int testtype; 4741 4742 if (!test_srcu_lockdep) 4743 return; 4744 4745 deadlock = test_srcu_lockdep / 1000; 4746 testtype = (test_srcu_lockdep / 10) % 100; 4747 cyclelen = test_srcu_lockdep % 10; 4748 WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus)); 4749 if (WARN_ONCE(deadlock != !!deadlock, 4750 "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n", 4751 __func__, test_srcu_lockdep, deadlock)) 4752 err = true; 4753 if (WARN_ONCE(cyclelen <= 0, 4754 "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n", 4755 __func__, test_srcu_lockdep, cyclelen)) 4756 err = true; 4757 if (err) 4758 goto err_out; 4759 4760 if (testtype == 0) { 4761 pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n", 4762 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4763 if (deadlock && cyclelen == 1) 4764 pr_info("%s: Expect hang.\n", __func__); 4765 for (i = 0; i < cyclelen; i++) { 4766 j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu", 4767 "srcu_read_unlock", i, cyclelen, deadlock); 4768 idx = srcu_read_lock(srcus[i]); 4769 if (j >= 0) 4770 synchronize_srcu(srcus[j]); 4771 srcu_read_unlock(srcus[i], idx); 4772 } 4773 return; 4774 } 4775 4776 if (testtype == 1) { 4777 pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n", 4778 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4779 for (i = 0; i < cyclelen; i++) { 4780 pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n", 4781 __func__, i, i, i, i); 4782 idx = srcu_read_lock(srcus[i]); 4783 mutex_lock(muts[i]); 4784 mutex_unlock(muts[i]); 4785 srcu_read_unlock(srcus[i], idx); 4786 4787 j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu", 4788 "mutex_unlock", i, cyclelen, deadlock); 4789 mutex_lock(muts[i]); 4790 if (j >= 0) 4791 synchronize_srcu(srcus[j]); 4792 mutex_unlock(muts[i]); 4793 } 4794 return; 4795 } 4796 4797 if (testtype == 2) { 4798 pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n", 4799 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4800 for (i = 0; i < cyclelen; i++) { 4801 pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n", 4802 __func__, i, i, i, i); 4803 idx = srcu_read_lock(srcus[i]); 4804 down_read(rwsems[i]); 4805 up_read(rwsems[i]); 4806 srcu_read_unlock(srcus[i], idx); 4807 4808 j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu", 4809 "up_write", i, cyclelen, deadlock); 4810 down_write(rwsems[i]); 4811 if (j >= 0) 4812 synchronize_srcu(srcus[j]); 4813 up_write(rwsems[i]); 4814 } 4815 return; 4816 } 4817 4818 #ifdef CONFIG_TASKS_TRACE_RCU 4819 if (testtype == 3) { 4820 pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n", 4821 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4822 if (deadlock && cyclelen == 1) 4823 pr_info("%s: Expect hang.\n", __func__); 4824 for (i = 0; i < cyclelen; i++) { 4825 char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock"; 4826 char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace" 4827 : "synchronize_srcu"; 4828 char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock"; 4829 4830 j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock); 4831 if (i == 0) 4832 rcu_read_lock_trace(); 4833 else 4834 idx = srcu_read_lock(srcus[i]); 4835 if (j >= 0) { 4836 if (i == cyclelen - 1) 4837 synchronize_rcu_tasks_trace(); 4838 else 4839 synchronize_srcu(srcus[j]); 4840 } 4841 if (i == 0) 4842 rcu_read_unlock_trace(); 4843 else 4844 srcu_read_unlock(srcus[i], idx); 4845 } 4846 return; 4847 } 4848 #endif // #ifdef CONFIG_TASKS_TRACE_RCU 4849 4850 if (testtype == 4) { 4851 pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC %d-way %sdeadlock.\n", 4852 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4853 if (deadlock && cyclelen == 1) 4854 pr_info("%s: Expect hang.\n", __func__); 4855 for (i = 0; i < cyclelen; i++) { 4856 j = srcu_lockdep_next(__func__, "srcu_read_lock_atomic", 4857 "synchronize_srcu_atomic", 4858 "srcu_read_unlock_atomic", i, 4859 cyclelen, deadlock); 4860 idx = srcu_read_lock_atomic(srcus_atomic[i]); 4861 if (j >= 0) 4862 synchronize_srcu_atomic(srcus_atomic[j]); 4863 srcu_read_unlock_atomic(srcus_atomic[i], idx); 4864 } 4865 return; 4866 } 4867 4868 if (testtype == 5) { 4869 pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC/raw_spinlock %d-way %sdeadlock.\n", 4870 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4871 for (i = 0; i < cyclelen; i++) { 4872 pr_info("%s: srcu_read_lock_atomic(%d), raw_spin_lock(%d), raw_spin_unlock(%d), srcu_read_unlock_atomic(%d)\n", 4873 __func__, i, i, i, i); 4874 idx = srcu_read_lock_atomic(srcus_atomic[i]); 4875 raw_spin_lock(spins[i]); 4876 raw_spin_unlock(spins[i]); 4877 srcu_read_unlock_atomic(srcus_atomic[i], idx); 4878 4879 j = srcu_lockdep_next(__func__, "raw_spin_lock", 4880 "synchronize_srcu_atomic", 4881 "raw_spin_unlock", i, cyclelen, 4882 deadlock); 4883 raw_spin_lock(spins[i]); 4884 if (j >= 0) 4885 synchronize_srcu_atomic(srcus_atomic[j]); 4886 raw_spin_unlock(spins[i]); 4887 } 4888 return; 4889 } 4890 4891 if (testtype == 6) { 4892 pr_info("%s: test_srcu_lockdep = %05d: synchronize_srcu_atomic() inside rcu_read_lock() %d-way.\n", 4893 __func__, test_srcu_lockdep, cyclelen); 4894 for (i = 0; i < cyclelen; i++) { 4895 rcu_read_lock(); 4896 synchronize_srcu_atomic(srcus_atomic[i]); 4897 rcu_read_unlock(); 4898 } 4899 return; 4900 } 4901 4902 if (testtype == 7) { 4903 int cpu; 4904 4905 for (i = 0; i < cyclelen; i++) { 4906 idx = srcu_read_lock_atomic(&srcu_irq); 4907 cpu = cpumask_any_but(cpu_online_mask, 4908 smp_processor_id()); 4909 if (cpu < nr_cpu_ids) { 4910 pr_info("%s: CPU%d sending IPI to CPU%d\n", 4911 __func__, smp_processor_id(), cpu); 4912 smp_call_function_single(cpu, srcu_sync_irq, 4913 NULL, 1); 4914 } 4915 srcu_read_unlock_atomic(&srcu_irq, idx); 4916 } 4917 return; 4918 } 4919 4920 err_out: 4921 pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); 4922 pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); 4923 pr_info("%s: D: Deadlock if nonzero.\n", __func__); 4924 pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU, 4=SRCU_ATOMIC, ", 4925 __func__); 4926 pr_cont("5=SRCU_ATOMIC/raw_spinlock, 6=synchronize_srcu_atomic inside rcu_read_lock, 7=atomic SRCU cross-CPU IRQ context mismatch.\n"); 4927 pr_info("%s: L: Cycle length.\n", __func__); 4928 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) 4929 pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); 4930 } 4931 4932 static int __init 4933 rcu_torture_init(void) 4934 { 4935 long i; 4936 int cpu; 4937 int firsterr = 0; 4938 int flags = 0; 4939 unsigned long gp_seq = 0; 4940 static struct rcu_torture_ops *torture_ops[] = { 4941 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, 4942 TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 4943 &trivial_ops, TRIVIAL_PREEMPT_OPS 4944 }; 4945 4946 if (!torture_init_begin(torture_type, verbose)) 4947 return -EBUSY; 4948 4949 /* Process args and tell the world that the torturer is on the job. */ 4950 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 4951 cur_ops = torture_ops[i]; 4952 if (strcmp(torture_type, cur_ops->name) == 0) 4953 break; 4954 } 4955 if (i == ARRAY_SIZE(torture_ops)) { 4956 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 4957 torture_type); 4958 pr_alert("rcu-torture types:"); 4959 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 4960 pr_cont(" %s", torture_ops[i]->name); 4961 pr_cont("\n"); 4962 firsterr = -EINVAL; 4963 cur_ops = NULL; 4964 goto unwind; 4965 } 4966 if (stall_only) { 4967 pr_alert("rcu-torture: stall_only specified, suppressing all else.\n"); 4968 fqs_stutter = 0; 4969 fwd_progress = 0; 4970 n_barrier_cbs = 0; 4971 nfakewriters = 0; 4972 nocbs_nthreads = 0; 4973 nreaders = 0; 4974 n_up_down = 0; 4975 nwriters = 0; 4976 onoff_interval = 0; 4977 preempt_duration = 0; 4978 read_exit_burst = 0; 4979 shuffle_interval = 0; 4980 stutter = 0; 4981 test_boost = 0; 4982 } 4983 if (cur_ops->fqs == NULL && fqs_duration != 0) { 4984 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 4985 fqs_duration = 0; 4986 } 4987 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops || 4988 !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 4989 pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n", 4990 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU)); 4991 nocbs_nthreads = 0; 4992 } 4993 if (cur_ops->init) 4994 cur_ops->init(); 4995 4996 rcu_torture_init_srcu_lockdep(); 4997 4998 if (nfakewriters >= 0) { 4999 nrealfakewriters = nfakewriters; 5000 } else { 5001 nrealfakewriters = num_online_cpus() - 2 - nfakewriters; 5002 if (nrealfakewriters <= 0) 5003 nrealfakewriters = 1; 5004 } 5005 5006 if (nreaders >= 0) { 5007 nrealreaders = nreaders; 5008 } else { 5009 nrealreaders = num_online_cpus() - 2 - nreaders; 5010 if (nrealreaders <= 0) 5011 nrealreaders = 1; 5012 } 5013 rcu_torture_print_module_parms(cur_ops, "Start of test"); 5014 if (cur_ops->get_gp_data) 5015 cur_ops->get_gp_data(&flags, &gp_seq); 5016 start_gp_seq = gp_seq; 5017 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", 5018 cur_ops->name, (long)gp_seq, flags); 5019 5020 /* Set up the freelist. */ 5021 5022 INIT_LIST_HEAD(&rcu_torture_freelist); 5023 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 5024 rcu_tortures[i].rtort_mbtest = 0; 5025 list_add_tail(&rcu_tortures[i].rtort_free, 5026 &rcu_torture_freelist); 5027 } 5028 5029 /* Initialize the statistics so that each run gets its own numbers. */ 5030 5031 rcu_torture_current = NULL; 5032 rcu_torture_current_version = 0; 5033 atomic_set(&n_rcu_torture_alloc, 0); 5034 atomic_set(&n_rcu_torture_alloc_fail, 0); 5035 atomic_set(&n_rcu_torture_free, 0); 5036 atomic_set(&n_rcu_torture_mberror, 0); 5037 atomic_set(&n_rcu_torture_mbchk_fail, 0); 5038 atomic_set(&n_rcu_torture_mbchk_tries, 0); 5039 atomic_set(&n_rcu_torture_error, 0); 5040 n_rcu_torture_barrier_error = 0; 5041 n_rcu_torture_boost_ktrerror = 0; 5042 n_rcu_torture_boost_failure = 0; 5043 n_rcu_torture_boosts = 0; 5044 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 5045 atomic_set(&rcu_torture_wcount[i], 0); 5046 for_each_possible_cpu(cpu) { 5047 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 5048 per_cpu(rcu_torture_count, cpu)[i] = 0; 5049 per_cpu(rcu_torture_batch, cpu)[i] = 0; 5050 } 5051 } 5052 err_segs_recorded = 0; 5053 rt_read_nsegs = 0; 5054 5055 /* Start up the kthreads. */ 5056 5057 rcu_torture_write_types(); 5058 if (nrealfakewriters > 0) { 5059 fakewriter_tasks = kzalloc_objs(fakewriter_tasks[0], 5060 nrealfakewriters); 5061 if (fakewriter_tasks == NULL) { 5062 TOROUT_ERRSTRING("out of memory"); 5063 firsterr = -ENOMEM; 5064 goto unwind; 5065 } 5066 } 5067 for (i = 0; i < nrealfakewriters; i++) { 5068 firsterr = torture_create_kthread(rcu_torture_fakewriter, 5069 NULL, fakewriter_tasks[i]); 5070 if (torture_init_error(firsterr)) 5071 goto unwind; 5072 } 5073 reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); 5074 rcu_torture_reader_mbchk = kzalloc_objs(*rcu_torture_reader_mbchk, 5075 nrealreaders); 5076 if (!reader_tasks || !rcu_torture_reader_mbchk) { 5077 TOROUT_ERRSTRING("out of memory"); 5078 firsterr = -ENOMEM; 5079 goto unwind; 5080 } 5081 rcu_torture_nmi_init(); 5082 for (i = 0; i < nrealreaders; i++) { 5083 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 5084 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 5085 reader_tasks[i]); 5086 if (torture_init_error(firsterr)) 5087 goto unwind; 5088 } 5089 5090 if (nwriters) { 5091 firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); 5092 if (torture_init_error(firsterr)) 5093 goto unwind; 5094 } 5095 5096 firsterr = rcu_torture_updown_init(); 5097 if (torture_init_error(firsterr)) 5098 goto unwind; 5099 nrealnocbers = nocbs_nthreads; 5100 if (WARN_ON(nrealnocbers < 0)) 5101 nrealnocbers = 1; 5102 if (WARN_ON(nocbs_toggle < 0)) 5103 nocbs_toggle = HZ; 5104 if (nrealnocbers > 0) { 5105 nocb_tasks = kzalloc_objs(nocb_tasks[0], nrealnocbers); 5106 if (nocb_tasks == NULL) { 5107 TOROUT_ERRSTRING("out of memory"); 5108 firsterr = -ENOMEM; 5109 goto unwind; 5110 } 5111 } else { 5112 nocb_tasks = NULL; 5113 } 5114 for (i = 0; i < nrealnocbers; i++) { 5115 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 5116 if (torture_init_error(firsterr)) 5117 goto unwind; 5118 } 5119 if (stat_interval > 0) { 5120 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 5121 stats_task); 5122 if (torture_init_error(firsterr)) 5123 goto unwind; 5124 } 5125 if (test_no_idle_hz && shuffle_interval > 0) { 5126 firsterr = torture_shuffle_init(shuffle_interval * HZ); 5127 if (torture_init_error(firsterr)) 5128 goto unwind; 5129 } 5130 if (stutter < 0) 5131 stutter = 0; 5132 if (stutter) { 5133 int t; 5134 5135 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 5136 firsterr = torture_stutter_init(stutter * HZ, t); 5137 if (torture_init_error(firsterr)) 5138 goto unwind; 5139 } 5140 if (fqs_duration < 0) 5141 fqs_duration = 0; 5142 if (fqs_holdoff < 0) 5143 fqs_holdoff = 0; 5144 if (fqs_duration && fqs_holdoff) { 5145 /* Create the fqs thread */ 5146 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 5147 fqs_task); 5148 if (torture_init_error(firsterr)) 5149 goto unwind; 5150 } 5151 if (test_boost_interval < 1) 5152 test_boost_interval = 1; 5153 if (test_boost_duration < 2) 5154 test_boost_duration = 2; 5155 if (rcu_torture_can_boost()) { 5156 5157 boost_starttime = jiffies + test_boost_interval * HZ; 5158 5159 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 5160 rcutorture_booster_init, 5161 rcutorture_booster_cleanup); 5162 rcutor_hp = firsterr; 5163 if (torture_init_error(firsterr)) 5164 goto unwind; 5165 } 5166 shutdown_jiffies = jiffies + shutdown_secs * HZ; 5167 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 5168 if (torture_init_error(firsterr)) 5169 goto unwind; 5170 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 5171 rcutorture_sync); 5172 if (torture_init_error(firsterr)) 5173 goto unwind; 5174 firsterr = rcu_torture_stall_init(); 5175 if (torture_init_error(firsterr)) 5176 goto unwind; 5177 firsterr = rcu_torture_fwd_prog_init(); 5178 if (torture_init_error(firsterr)) 5179 goto unwind; 5180 firsterr = rcu_torture_barrier_init(); 5181 if (torture_init_error(firsterr)) 5182 goto unwind; 5183 firsterr = rcu_torture_read_exit_init(); 5184 if (torture_init_error(firsterr)) 5185 goto unwind; 5186 if (preempt_duration > 0) { 5187 firsterr = torture_create_kthread(rcu_torture_preempt, NULL, preempt_task); 5188 if (torture_init_error(firsterr)) 5189 goto unwind; 5190 } 5191 if (object_debug) 5192 rcu_test_debug_objects(); 5193 5194 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister)) 5195 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay); 5196 5197 if (gpwrap_lag && cur_ops->set_gpwrap_lag) { 5198 firsterr = rcu_gpwrap_lag_init(); 5199 if (torture_init_error(firsterr)) 5200 goto unwind; 5201 } 5202 5203 torture_init_end(); 5204 return 0; 5205 5206 unwind: 5207 torture_init_end(); 5208 rcu_torture_cleanup(); 5209 if (shutdown_secs) { 5210 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 5211 kernel_power_off(); 5212 } 5213 return firsterr; 5214 } 5215 5216 module_init(rcu_torture_init); 5217 module_exit(rcu_torture_cleanup); 5218