1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Module-based torture test facility for locking 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Davidlohr Bueso <dave@stgolabs.net> 9 * Based on kernel/rcu/torture.c. 10 */ 11 12 #define pr_fmt(fmt) fmt 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/kthread.h> 17 #include <linux/sched/rt.h> 18 #include <linux/spinlock.h> 19 #include <linux/mutex.h> 20 #include <linux/rwsem.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/rtmutex.h> 26 #include <linux/atomic.h> 27 #include <linux/moduleparam.h> 28 #include <linux/delay.h> 29 #include <linux/slab.h> 30 #include <linux/torture.h> 31 #include <linux/reboot.h> 32 33 MODULE_LICENSE("GPL"); 34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 35 36 torture_param(int, nwriters_stress, -1, 37 "Number of write-locking stress-test threads"); 38 torture_param(int, nreaders_stress, -1, 39 "Number of read-locking stress-test threads"); 40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 41 torture_param(int, onoff_interval, 0, 42 "Time between CPU hotplugs (s), 0=disable"); 43 torture_param(int, shuffle_interval, 3, 44 "Number of jiffies between shuffles, 0=disable"); 45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); 46 torture_param(int, stat_interval, 60, 47 "Number of seconds between stats printk()s"); 48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); 49 torture_param(int, rt_boost, 2, 50 "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); 51 torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); 52 torture_param(int, verbose, 1, 53 "Enable verbose debugging printk()s"); 54 torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); 55 /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ 56 #define MAX_NESTED_LOCKS 8 57 58 static char *torture_type = "spin_lock"; 59 module_param(torture_type, charp, 0444); 60 MODULE_PARM_DESC(torture_type, 61 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); 62 63 static struct task_struct *stats_task; 64 static struct task_struct **writer_tasks; 65 static struct task_struct **reader_tasks; 66 67 static bool lock_is_write_held; 68 static atomic_t lock_is_read_held; 69 static unsigned long last_lock_release; 70 71 struct lock_stress_stats { 72 long n_lock_fail; 73 long n_lock_acquired; 74 }; 75 76 /* Forward reference. */ 77 static void lock_torture_cleanup(void); 78 79 /* 80 * Operations vector for selecting different types of tests. 81 */ 82 struct lock_torture_ops { 83 void (*init)(void); 84 void (*exit)(void); 85 int (*nested_lock)(int tid, u32 lockset); 86 int (*writelock)(int tid); 87 void (*write_delay)(struct torture_random_state *trsp); 88 void (*task_boost)(struct torture_random_state *trsp); 89 void (*writeunlock)(int tid); 90 void (*nested_unlock)(int tid, u32 lockset); 91 int (*readlock)(int tid); 92 void (*read_delay)(struct torture_random_state *trsp); 93 void (*readunlock)(int tid); 94 95 unsigned long flags; /* for irq spinlocks */ 96 const char *name; 97 }; 98 99 struct lock_torture_cxt { 100 int nrealwriters_stress; 101 int nrealreaders_stress; 102 bool debug_lock; 103 bool init_called; 104 atomic_t n_lock_torture_errors; 105 struct lock_torture_ops *cur_ops; 106 struct lock_stress_stats *lwsa; /* writer statistics */ 107 struct lock_stress_stats *lrsa; /* reader statistics */ 108 }; 109 static struct lock_torture_cxt cxt = { 0, 0, false, false, 110 ATOMIC_INIT(0), 111 NULL, NULL}; 112 /* 113 * Definitions for lock torture testing. 114 */ 115 116 static int torture_lock_busted_write_lock(int tid __maybe_unused) 117 { 118 return 0; /* BUGGY, do not use in real life!!! */ 119 } 120 121 static void torture_lock_busted_write_delay(struct torture_random_state *trsp) 122 { 123 const unsigned long longdelay_ms = 100; 124 125 /* We want a long delay occasionally to force massive contention. */ 126 if (!(torture_random(trsp) % 127 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 128 mdelay(longdelay_ms); 129 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 130 torture_preempt_schedule(); /* Allow test to be preempted. */ 131 } 132 133 static void torture_lock_busted_write_unlock(int tid __maybe_unused) 134 { 135 /* BUGGY, do not use in real life!!! */ 136 } 137 138 static void __torture_rt_boost(struct torture_random_state *trsp) 139 { 140 const unsigned int factor = rt_boost_factor; 141 142 if (!rt_task(current)) { 143 /* 144 * Boost priority once every rt_boost_factor operations. When 145 * the task tries to take the lock, the rtmutex it will account 146 * for the new priority, and do any corresponding pi-dance. 147 */ 148 if (trsp && !(torture_random(trsp) % 149 (cxt.nrealwriters_stress * factor))) { 150 sched_set_fifo(current); 151 } else /* common case, do nothing */ 152 return; 153 } else { 154 /* 155 * The task will remain boosted for another 10 * rt_boost_factor 156 * operations, then restored back to its original prio, and so 157 * forth. 158 * 159 * When @trsp is nil, we want to force-reset the task for 160 * stopping the kthread. 161 */ 162 if (!trsp || !(torture_random(trsp) % 163 (cxt.nrealwriters_stress * factor * 2))) { 164 sched_set_normal(current, 0); 165 } else /* common case, do nothing */ 166 return; 167 } 168 } 169 170 static void torture_rt_boost(struct torture_random_state *trsp) 171 { 172 if (rt_boost != 2) 173 return; 174 175 __torture_rt_boost(trsp); 176 } 177 178 static struct lock_torture_ops lock_busted_ops = { 179 .writelock = torture_lock_busted_write_lock, 180 .write_delay = torture_lock_busted_write_delay, 181 .task_boost = torture_rt_boost, 182 .writeunlock = torture_lock_busted_write_unlock, 183 .readlock = NULL, 184 .read_delay = NULL, 185 .readunlock = NULL, 186 .name = "lock_busted" 187 }; 188 189 static DEFINE_SPINLOCK(torture_spinlock); 190 191 static int torture_spin_lock_write_lock(int tid __maybe_unused) 192 __acquires(torture_spinlock) 193 { 194 spin_lock(&torture_spinlock); 195 return 0; 196 } 197 198 static void torture_spin_lock_write_delay(struct torture_random_state *trsp) 199 { 200 const unsigned long shortdelay_us = 2; 201 const unsigned long longdelay_ms = 100; 202 203 /* We want a short delay mostly to emulate likely code, and 204 * we want a long delay occasionally to force massive contention. 205 */ 206 if (!(torture_random(trsp) % 207 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 208 mdelay(longdelay_ms); 209 if (!(torture_random(trsp) % 210 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 211 udelay(shortdelay_us); 212 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 213 torture_preempt_schedule(); /* Allow test to be preempted. */ 214 } 215 216 static void torture_spin_lock_write_unlock(int tid __maybe_unused) 217 __releases(torture_spinlock) 218 { 219 spin_unlock(&torture_spinlock); 220 } 221 222 static struct lock_torture_ops spin_lock_ops = { 223 .writelock = torture_spin_lock_write_lock, 224 .write_delay = torture_spin_lock_write_delay, 225 .task_boost = torture_rt_boost, 226 .writeunlock = torture_spin_lock_write_unlock, 227 .readlock = NULL, 228 .read_delay = NULL, 229 .readunlock = NULL, 230 .name = "spin_lock" 231 }; 232 233 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused) 234 __acquires(torture_spinlock) 235 { 236 unsigned long flags; 237 238 spin_lock_irqsave(&torture_spinlock, flags); 239 cxt.cur_ops->flags = flags; 240 return 0; 241 } 242 243 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused) 244 __releases(torture_spinlock) 245 { 246 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); 247 } 248 249 static struct lock_torture_ops spin_lock_irq_ops = { 250 .writelock = torture_spin_lock_write_lock_irq, 251 .write_delay = torture_spin_lock_write_delay, 252 .task_boost = torture_rt_boost, 253 .writeunlock = torture_lock_spin_write_unlock_irq, 254 .readlock = NULL, 255 .read_delay = NULL, 256 .readunlock = NULL, 257 .name = "spin_lock_irq" 258 }; 259 260 static DEFINE_RWLOCK(torture_rwlock); 261 262 static int torture_rwlock_write_lock(int tid __maybe_unused) 263 __acquires(torture_rwlock) 264 { 265 write_lock(&torture_rwlock); 266 return 0; 267 } 268 269 static void torture_rwlock_write_delay(struct torture_random_state *trsp) 270 { 271 const unsigned long shortdelay_us = 2; 272 const unsigned long longdelay_ms = 100; 273 274 /* We want a short delay mostly to emulate likely code, and 275 * we want a long delay occasionally to force massive contention. 276 */ 277 if (!(torture_random(trsp) % 278 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 279 mdelay(longdelay_ms); 280 else 281 udelay(shortdelay_us); 282 } 283 284 static void torture_rwlock_write_unlock(int tid __maybe_unused) 285 __releases(torture_rwlock) 286 { 287 write_unlock(&torture_rwlock); 288 } 289 290 static int torture_rwlock_read_lock(int tid __maybe_unused) 291 __acquires(torture_rwlock) 292 { 293 read_lock(&torture_rwlock); 294 return 0; 295 } 296 297 static void torture_rwlock_read_delay(struct torture_random_state *trsp) 298 { 299 const unsigned long shortdelay_us = 10; 300 const unsigned long longdelay_ms = 100; 301 302 /* We want a short delay mostly to emulate likely code, and 303 * we want a long delay occasionally to force massive contention. 304 */ 305 if (!(torture_random(trsp) % 306 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 307 mdelay(longdelay_ms); 308 else 309 udelay(shortdelay_us); 310 } 311 312 static void torture_rwlock_read_unlock(int tid __maybe_unused) 313 __releases(torture_rwlock) 314 { 315 read_unlock(&torture_rwlock); 316 } 317 318 static struct lock_torture_ops rw_lock_ops = { 319 .writelock = torture_rwlock_write_lock, 320 .write_delay = torture_rwlock_write_delay, 321 .task_boost = torture_rt_boost, 322 .writeunlock = torture_rwlock_write_unlock, 323 .readlock = torture_rwlock_read_lock, 324 .read_delay = torture_rwlock_read_delay, 325 .readunlock = torture_rwlock_read_unlock, 326 .name = "rw_lock" 327 }; 328 329 static int torture_rwlock_write_lock_irq(int tid __maybe_unused) 330 __acquires(torture_rwlock) 331 { 332 unsigned long flags; 333 334 write_lock_irqsave(&torture_rwlock, flags); 335 cxt.cur_ops->flags = flags; 336 return 0; 337 } 338 339 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused) 340 __releases(torture_rwlock) 341 { 342 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 343 } 344 345 static int torture_rwlock_read_lock_irq(int tid __maybe_unused) 346 __acquires(torture_rwlock) 347 { 348 unsigned long flags; 349 350 read_lock_irqsave(&torture_rwlock, flags); 351 cxt.cur_ops->flags = flags; 352 return 0; 353 } 354 355 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused) 356 __releases(torture_rwlock) 357 { 358 read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 359 } 360 361 static struct lock_torture_ops rw_lock_irq_ops = { 362 .writelock = torture_rwlock_write_lock_irq, 363 .write_delay = torture_rwlock_write_delay, 364 .task_boost = torture_rt_boost, 365 .writeunlock = torture_rwlock_write_unlock_irq, 366 .readlock = torture_rwlock_read_lock_irq, 367 .read_delay = torture_rwlock_read_delay, 368 .readunlock = torture_rwlock_read_unlock_irq, 369 .name = "rw_lock_irq" 370 }; 371 372 static DEFINE_MUTEX(torture_mutex); 373 static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS]; 374 static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS]; 375 376 static void torture_mutex_init(void) 377 { 378 int i; 379 380 for (i = 0; i < MAX_NESTED_LOCKS; i++) 381 __mutex_init(&torture_nested_mutexes[i], __func__, 382 &nested_mutex_keys[i]); 383 } 384 385 static int torture_mutex_nested_lock(int tid __maybe_unused, 386 u32 lockset) 387 { 388 int i; 389 390 for (i = 0; i < nested_locks; i++) 391 if (lockset & (1 << i)) 392 mutex_lock(&torture_nested_mutexes[i]); 393 return 0; 394 } 395 396 static int torture_mutex_lock(int tid __maybe_unused) 397 __acquires(torture_mutex) 398 { 399 mutex_lock(&torture_mutex); 400 return 0; 401 } 402 403 static void torture_mutex_delay(struct torture_random_state *trsp) 404 { 405 const unsigned long longdelay_ms = 100; 406 407 /* We want a long delay occasionally to force massive contention. */ 408 if (!(torture_random(trsp) % 409 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 410 mdelay(longdelay_ms * 5); 411 else 412 mdelay(longdelay_ms / 5); 413 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 414 torture_preempt_schedule(); /* Allow test to be preempted. */ 415 } 416 417 static void torture_mutex_unlock(int tid __maybe_unused) 418 __releases(torture_mutex) 419 { 420 mutex_unlock(&torture_mutex); 421 } 422 423 static void torture_mutex_nested_unlock(int tid __maybe_unused, 424 u32 lockset) 425 { 426 int i; 427 428 for (i = nested_locks - 1; i >= 0; i--) 429 if (lockset & (1 << i)) 430 mutex_unlock(&torture_nested_mutexes[i]); 431 } 432 433 static struct lock_torture_ops mutex_lock_ops = { 434 .init = torture_mutex_init, 435 .nested_lock = torture_mutex_nested_lock, 436 .writelock = torture_mutex_lock, 437 .write_delay = torture_mutex_delay, 438 .task_boost = torture_rt_boost, 439 .writeunlock = torture_mutex_unlock, 440 .nested_unlock = torture_mutex_nested_unlock, 441 .readlock = NULL, 442 .read_delay = NULL, 443 .readunlock = NULL, 444 .name = "mutex_lock" 445 }; 446 447 #include <linux/ww_mutex.h> 448 /* 449 * The torture ww_mutexes should belong to the same lock class as 450 * torture_ww_class to avoid lockdep problem. The ww_mutex_init() 451 * function is called for initialization to ensure that. 452 */ 453 static DEFINE_WD_CLASS(torture_ww_class); 454 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2; 455 static struct ww_acquire_ctx *ww_acquire_ctxs; 456 457 static void torture_ww_mutex_init(void) 458 { 459 ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class); 460 ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class); 461 ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class); 462 463 ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress, 464 sizeof(*ww_acquire_ctxs), 465 GFP_KERNEL); 466 if (!ww_acquire_ctxs) 467 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory"); 468 } 469 470 static void torture_ww_mutex_exit(void) 471 { 472 kfree(ww_acquire_ctxs); 473 } 474 475 static int torture_ww_mutex_lock(int tid) 476 __acquires(torture_ww_mutex_0) 477 __acquires(torture_ww_mutex_1) 478 __acquires(torture_ww_mutex_2) 479 { 480 LIST_HEAD(list); 481 struct reorder_lock { 482 struct list_head link; 483 struct ww_mutex *lock; 484 } locks[3], *ll, *ln; 485 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 486 487 locks[0].lock = &torture_ww_mutex_0; 488 list_add(&locks[0].link, &list); 489 490 locks[1].lock = &torture_ww_mutex_1; 491 list_add(&locks[1].link, &list); 492 493 locks[2].lock = &torture_ww_mutex_2; 494 list_add(&locks[2].link, &list); 495 496 ww_acquire_init(ctx, &torture_ww_class); 497 498 list_for_each_entry(ll, &list, link) { 499 int err; 500 501 err = ww_mutex_lock(ll->lock, ctx); 502 if (!err) 503 continue; 504 505 ln = ll; 506 list_for_each_entry_continue_reverse(ln, &list, link) 507 ww_mutex_unlock(ln->lock); 508 509 if (err != -EDEADLK) 510 return err; 511 512 ww_mutex_lock_slow(ll->lock, ctx); 513 list_move(&ll->link, &list); 514 } 515 516 return 0; 517 } 518 519 static void torture_ww_mutex_unlock(int tid) 520 __releases(torture_ww_mutex_0) 521 __releases(torture_ww_mutex_1) 522 __releases(torture_ww_mutex_2) 523 { 524 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 525 526 ww_mutex_unlock(&torture_ww_mutex_0); 527 ww_mutex_unlock(&torture_ww_mutex_1); 528 ww_mutex_unlock(&torture_ww_mutex_2); 529 ww_acquire_fini(ctx); 530 } 531 532 static struct lock_torture_ops ww_mutex_lock_ops = { 533 .init = torture_ww_mutex_init, 534 .exit = torture_ww_mutex_exit, 535 .writelock = torture_ww_mutex_lock, 536 .write_delay = torture_mutex_delay, 537 .task_boost = torture_rt_boost, 538 .writeunlock = torture_ww_mutex_unlock, 539 .readlock = NULL, 540 .read_delay = NULL, 541 .readunlock = NULL, 542 .name = "ww_mutex_lock" 543 }; 544 545 #ifdef CONFIG_RT_MUTEXES 546 static DEFINE_RT_MUTEX(torture_rtmutex); 547 static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS]; 548 static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS]; 549 550 static void torture_rtmutex_init(void) 551 { 552 int i; 553 554 for (i = 0; i < MAX_NESTED_LOCKS; i++) 555 __rt_mutex_init(&torture_nested_rtmutexes[i], __func__, 556 &nested_rtmutex_keys[i]); 557 } 558 559 static int torture_rtmutex_nested_lock(int tid __maybe_unused, 560 u32 lockset) 561 { 562 int i; 563 564 for (i = 0; i < nested_locks; i++) 565 if (lockset & (1 << i)) 566 rt_mutex_lock(&torture_nested_rtmutexes[i]); 567 return 0; 568 } 569 570 static int torture_rtmutex_lock(int tid __maybe_unused) 571 __acquires(torture_rtmutex) 572 { 573 rt_mutex_lock(&torture_rtmutex); 574 return 0; 575 } 576 577 static void torture_rtmutex_delay(struct torture_random_state *trsp) 578 { 579 const unsigned long shortdelay_us = 2; 580 const unsigned long longdelay_ms = 100; 581 582 /* 583 * We want a short delay mostly to emulate likely code, and 584 * we want a long delay occasionally to force massive contention. 585 */ 586 if (!(torture_random(trsp) % 587 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 588 mdelay(longdelay_ms); 589 if (!(torture_random(trsp) % 590 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 591 udelay(shortdelay_us); 592 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 593 torture_preempt_schedule(); /* Allow test to be preempted. */ 594 } 595 596 static void torture_rtmutex_unlock(int tid __maybe_unused) 597 __releases(torture_rtmutex) 598 { 599 rt_mutex_unlock(&torture_rtmutex); 600 } 601 602 static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) 603 { 604 if (!rt_boost) 605 return; 606 607 __torture_rt_boost(trsp); 608 } 609 610 static void torture_rtmutex_nested_unlock(int tid __maybe_unused, 611 u32 lockset) 612 { 613 int i; 614 615 for (i = nested_locks - 1; i >= 0; i--) 616 if (lockset & (1 << i)) 617 rt_mutex_unlock(&torture_nested_rtmutexes[i]); 618 } 619 620 static struct lock_torture_ops rtmutex_lock_ops = { 621 .init = torture_rtmutex_init, 622 .nested_lock = torture_rtmutex_nested_lock, 623 .writelock = torture_rtmutex_lock, 624 .write_delay = torture_rtmutex_delay, 625 .task_boost = torture_rt_boost_rtmutex, 626 .writeunlock = torture_rtmutex_unlock, 627 .nested_unlock = torture_rtmutex_nested_unlock, 628 .readlock = NULL, 629 .read_delay = NULL, 630 .readunlock = NULL, 631 .name = "rtmutex_lock" 632 }; 633 #endif 634 635 static DECLARE_RWSEM(torture_rwsem); 636 static int torture_rwsem_down_write(int tid __maybe_unused) 637 __acquires(torture_rwsem) 638 { 639 down_write(&torture_rwsem); 640 return 0; 641 } 642 643 static void torture_rwsem_write_delay(struct torture_random_state *trsp) 644 { 645 const unsigned long longdelay_ms = 100; 646 647 /* We want a long delay occasionally to force massive contention. */ 648 if (!(torture_random(trsp) % 649 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 650 mdelay(longdelay_ms * 10); 651 else 652 mdelay(longdelay_ms / 10); 653 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 654 torture_preempt_schedule(); /* Allow test to be preempted. */ 655 } 656 657 static void torture_rwsem_up_write(int tid __maybe_unused) 658 __releases(torture_rwsem) 659 { 660 up_write(&torture_rwsem); 661 } 662 663 static int torture_rwsem_down_read(int tid __maybe_unused) 664 __acquires(torture_rwsem) 665 { 666 down_read(&torture_rwsem); 667 return 0; 668 } 669 670 static void torture_rwsem_read_delay(struct torture_random_state *trsp) 671 { 672 const unsigned long longdelay_ms = 100; 673 674 /* We want a long delay occasionally to force massive contention. */ 675 if (!(torture_random(trsp) % 676 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 677 mdelay(longdelay_ms * 2); 678 else 679 mdelay(longdelay_ms / 2); 680 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) 681 torture_preempt_schedule(); /* Allow test to be preempted. */ 682 } 683 684 static void torture_rwsem_up_read(int tid __maybe_unused) 685 __releases(torture_rwsem) 686 { 687 up_read(&torture_rwsem); 688 } 689 690 static struct lock_torture_ops rwsem_lock_ops = { 691 .writelock = torture_rwsem_down_write, 692 .write_delay = torture_rwsem_write_delay, 693 .task_boost = torture_rt_boost, 694 .writeunlock = torture_rwsem_up_write, 695 .readlock = torture_rwsem_down_read, 696 .read_delay = torture_rwsem_read_delay, 697 .readunlock = torture_rwsem_up_read, 698 .name = "rwsem_lock" 699 }; 700 701 #include <linux/percpu-rwsem.h> 702 static struct percpu_rw_semaphore pcpu_rwsem; 703 704 static void torture_percpu_rwsem_init(void) 705 { 706 BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); 707 } 708 709 static void torture_percpu_rwsem_exit(void) 710 { 711 percpu_free_rwsem(&pcpu_rwsem); 712 } 713 714 static int torture_percpu_rwsem_down_write(int tid __maybe_unused) 715 __acquires(pcpu_rwsem) 716 { 717 percpu_down_write(&pcpu_rwsem); 718 return 0; 719 } 720 721 static void torture_percpu_rwsem_up_write(int tid __maybe_unused) 722 __releases(pcpu_rwsem) 723 { 724 percpu_up_write(&pcpu_rwsem); 725 } 726 727 static int torture_percpu_rwsem_down_read(int tid __maybe_unused) 728 __acquires(pcpu_rwsem) 729 { 730 percpu_down_read(&pcpu_rwsem); 731 return 0; 732 } 733 734 static void torture_percpu_rwsem_up_read(int tid __maybe_unused) 735 __releases(pcpu_rwsem) 736 { 737 percpu_up_read(&pcpu_rwsem); 738 } 739 740 static struct lock_torture_ops percpu_rwsem_lock_ops = { 741 .init = torture_percpu_rwsem_init, 742 .exit = torture_percpu_rwsem_exit, 743 .writelock = torture_percpu_rwsem_down_write, 744 .write_delay = torture_rwsem_write_delay, 745 .task_boost = torture_rt_boost, 746 .writeunlock = torture_percpu_rwsem_up_write, 747 .readlock = torture_percpu_rwsem_down_read, 748 .read_delay = torture_rwsem_read_delay, 749 .readunlock = torture_percpu_rwsem_up_read, 750 .name = "percpu_rwsem_lock" 751 }; 752 753 /* 754 * Lock torture writer kthread. Repeatedly acquires and releases 755 * the lock, checking for duplicate acquisitions. 756 */ 757 static int lock_torture_writer(void *arg) 758 { 759 struct lock_stress_stats *lwsp = arg; 760 int tid = lwsp - cxt.lwsa; 761 DEFINE_TORTURE_RANDOM(rand); 762 u32 lockset_mask; 763 764 VERBOSE_TOROUT_STRING("lock_torture_writer task started"); 765 set_user_nice(current, MAX_NICE); 766 767 do { 768 if ((torture_random(&rand) & 0xfffff) == 0) 769 schedule_timeout_uninterruptible(1); 770 771 lockset_mask = torture_random(&rand); 772 cxt.cur_ops->task_boost(&rand); 773 if (cxt.cur_ops->nested_lock) 774 cxt.cur_ops->nested_lock(tid, lockset_mask); 775 cxt.cur_ops->writelock(tid); 776 if (WARN_ON_ONCE(lock_is_write_held)) 777 lwsp->n_lock_fail++; 778 lock_is_write_held = true; 779 if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) 780 lwsp->n_lock_fail++; /* rare, but... */ 781 782 lwsp->n_lock_acquired++; 783 cxt.cur_ops->write_delay(&rand); 784 lock_is_write_held = false; 785 WRITE_ONCE(last_lock_release, jiffies); 786 cxt.cur_ops->writeunlock(tid); 787 if (cxt.cur_ops->nested_unlock) 788 cxt.cur_ops->nested_unlock(tid, lockset_mask); 789 790 stutter_wait("lock_torture_writer"); 791 } while (!torture_must_stop()); 792 793 cxt.cur_ops->task_boost(NULL); /* reset prio */ 794 torture_kthread_stopping("lock_torture_writer"); 795 return 0; 796 } 797 798 /* 799 * Lock torture reader kthread. Repeatedly acquires and releases 800 * the reader lock. 801 */ 802 static int lock_torture_reader(void *arg) 803 { 804 struct lock_stress_stats *lrsp = arg; 805 int tid = lrsp - cxt.lrsa; 806 DEFINE_TORTURE_RANDOM(rand); 807 808 VERBOSE_TOROUT_STRING("lock_torture_reader task started"); 809 set_user_nice(current, MAX_NICE); 810 811 do { 812 if ((torture_random(&rand) & 0xfffff) == 0) 813 schedule_timeout_uninterruptible(1); 814 815 cxt.cur_ops->readlock(tid); 816 atomic_inc(&lock_is_read_held); 817 if (WARN_ON_ONCE(lock_is_write_held)) 818 lrsp->n_lock_fail++; /* rare, but... */ 819 820 lrsp->n_lock_acquired++; 821 cxt.cur_ops->read_delay(&rand); 822 atomic_dec(&lock_is_read_held); 823 cxt.cur_ops->readunlock(tid); 824 825 stutter_wait("lock_torture_reader"); 826 } while (!torture_must_stop()); 827 torture_kthread_stopping("lock_torture_reader"); 828 return 0; 829 } 830 831 /* 832 * Create an lock-torture-statistics message in the specified buffer. 833 */ 834 static void __torture_print_stats(char *page, 835 struct lock_stress_stats *statp, bool write) 836 { 837 long cur; 838 bool fail = false; 839 int i, n_stress; 840 long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0; 841 long long sum = 0; 842 843 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; 844 for (i = 0; i < n_stress; i++) { 845 if (data_race(statp[i].n_lock_fail)) 846 fail = true; 847 cur = data_race(statp[i].n_lock_acquired); 848 sum += cur; 849 if (max < cur) 850 max = cur; 851 if (min > cur) 852 min = cur; 853 } 854 page += sprintf(page, 855 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", 856 write ? "Writes" : "Reads ", 857 sum, max, min, 858 !onoff_interval && max / 2 > min ? "???" : "", 859 fail, fail ? "!!!" : ""); 860 if (fail) 861 atomic_inc(&cxt.n_lock_torture_errors); 862 } 863 864 /* 865 * Print torture statistics. Caller must ensure that there is only one 866 * call to this function at a given time!!! This is normally accomplished 867 * by relying on the module system to only have one copy of the module 868 * loaded, and then by giving the lock_torture_stats kthread full control 869 * (or the init/cleanup functions when lock_torture_stats thread is not 870 * running). 871 */ 872 static void lock_torture_stats_print(void) 873 { 874 int size = cxt.nrealwriters_stress * 200 + 8192; 875 char *buf; 876 877 if (cxt.cur_ops->readlock) 878 size += cxt.nrealreaders_stress * 200 + 8192; 879 880 buf = kmalloc(size, GFP_KERNEL); 881 if (!buf) { 882 pr_err("lock_torture_stats_print: Out of memory, need: %d", 883 size); 884 return; 885 } 886 887 __torture_print_stats(buf, cxt.lwsa, true); 888 pr_alert("%s", buf); 889 kfree(buf); 890 891 if (cxt.cur_ops->readlock) { 892 buf = kmalloc(size, GFP_KERNEL); 893 if (!buf) { 894 pr_err("lock_torture_stats_print: Out of memory, need: %d", 895 size); 896 return; 897 } 898 899 __torture_print_stats(buf, cxt.lrsa, false); 900 pr_alert("%s", buf); 901 kfree(buf); 902 } 903 } 904 905 /* 906 * Periodically prints torture statistics, if periodic statistics printing 907 * was specified via the stat_interval module parameter. 908 * 909 * No need to worry about fullstop here, since this one doesn't reference 910 * volatile state or register callbacks. 911 */ 912 static int lock_torture_stats(void *arg) 913 { 914 VERBOSE_TOROUT_STRING("lock_torture_stats task started"); 915 do { 916 schedule_timeout_interruptible(stat_interval * HZ); 917 lock_torture_stats_print(); 918 torture_shutdown_absorb("lock_torture_stats"); 919 } while (!torture_must_stop()); 920 torture_kthread_stopping("lock_torture_stats"); 921 return 0; 922 } 923 924 static inline void 925 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, 926 const char *tag) 927 { 928 pr_alert("%s" TORTURE_FLAG 929 "--- %s%s: nwriters_stress=%d nreaders_stress=%d nested_locks=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", 930 torture_type, tag, cxt.debug_lock ? " [debug]": "", 931 cxt.nrealwriters_stress, cxt.nrealreaders_stress, 932 nested_locks, stat_interval, verbose, shuffle_interval, 933 stutter, shutdown_secs, onoff_interval, onoff_holdoff); 934 } 935 936 static void lock_torture_cleanup(void) 937 { 938 int i; 939 940 if (torture_cleanup_begin()) 941 return; 942 943 /* 944 * Indicates early cleanup, meaning that the test has not run, 945 * such as when passing bogus args when loading the module. 946 * However cxt->cur_ops.init() may have been invoked, so beside 947 * perform the underlying torture-specific cleanups, cur_ops.exit() 948 * will be invoked if needed. 949 */ 950 if (!cxt.lwsa && !cxt.lrsa) 951 goto end; 952 953 if (writer_tasks) { 954 for (i = 0; i < cxt.nrealwriters_stress; i++) 955 torture_stop_kthread(lock_torture_writer, 956 writer_tasks[i]); 957 kfree(writer_tasks); 958 writer_tasks = NULL; 959 } 960 961 if (reader_tasks) { 962 for (i = 0; i < cxt.nrealreaders_stress; i++) 963 torture_stop_kthread(lock_torture_reader, 964 reader_tasks[i]); 965 kfree(reader_tasks); 966 reader_tasks = NULL; 967 } 968 969 torture_stop_kthread(lock_torture_stats, stats_task); 970 lock_torture_stats_print(); /* -After- the stats thread is stopped! */ 971 972 if (atomic_read(&cxt.n_lock_torture_errors)) 973 lock_torture_print_module_parms(cxt.cur_ops, 974 "End of test: FAILURE"); 975 else if (torture_onoff_failures()) 976 lock_torture_print_module_parms(cxt.cur_ops, 977 "End of test: LOCK_HOTPLUG"); 978 else 979 lock_torture_print_module_parms(cxt.cur_ops, 980 "End of test: SUCCESS"); 981 982 kfree(cxt.lwsa); 983 cxt.lwsa = NULL; 984 kfree(cxt.lrsa); 985 cxt.lrsa = NULL; 986 987 end: 988 if (cxt.init_called) { 989 if (cxt.cur_ops->exit) 990 cxt.cur_ops->exit(); 991 cxt.init_called = false; 992 } 993 torture_cleanup_end(); 994 } 995 996 static int __init lock_torture_init(void) 997 { 998 int i, j; 999 int firsterr = 0; 1000 static struct lock_torture_ops *torture_ops[] = { 1001 &lock_busted_ops, 1002 &spin_lock_ops, &spin_lock_irq_ops, 1003 &rw_lock_ops, &rw_lock_irq_ops, 1004 &mutex_lock_ops, 1005 &ww_mutex_lock_ops, 1006 #ifdef CONFIG_RT_MUTEXES 1007 &rtmutex_lock_ops, 1008 #endif 1009 &rwsem_lock_ops, 1010 &percpu_rwsem_lock_ops, 1011 }; 1012 1013 if (!torture_init_begin(torture_type, verbose)) 1014 return -EBUSY; 1015 1016 /* Process args and tell the world that the torturer is on the job. */ 1017 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1018 cxt.cur_ops = torture_ops[i]; 1019 if (strcmp(torture_type, cxt.cur_ops->name) == 0) 1020 break; 1021 } 1022 if (i == ARRAY_SIZE(torture_ops)) { 1023 pr_alert("lock-torture: invalid torture type: \"%s\"\n", 1024 torture_type); 1025 pr_alert("lock-torture types:"); 1026 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1027 pr_alert(" %s", torture_ops[i]->name); 1028 pr_alert("\n"); 1029 firsterr = -EINVAL; 1030 goto unwind; 1031 } 1032 1033 if (nwriters_stress == 0 && 1034 (!cxt.cur_ops->readlock || nreaders_stress == 0)) { 1035 pr_alert("lock-torture: must run at least one locking thread\n"); 1036 firsterr = -EINVAL; 1037 goto unwind; 1038 } 1039 1040 if (nwriters_stress >= 0) 1041 cxt.nrealwriters_stress = nwriters_stress; 1042 else 1043 cxt.nrealwriters_stress = 2 * num_online_cpus(); 1044 1045 if (cxt.cur_ops->init) { 1046 cxt.cur_ops->init(); 1047 cxt.init_called = true; 1048 } 1049 1050 #ifdef CONFIG_DEBUG_MUTEXES 1051 if (str_has_prefix(torture_type, "mutex")) 1052 cxt.debug_lock = true; 1053 #endif 1054 #ifdef CONFIG_DEBUG_RT_MUTEXES 1055 if (str_has_prefix(torture_type, "rtmutex")) 1056 cxt.debug_lock = true; 1057 #endif 1058 #ifdef CONFIG_DEBUG_SPINLOCK 1059 if ((str_has_prefix(torture_type, "spin")) || 1060 (str_has_prefix(torture_type, "rw_lock"))) 1061 cxt.debug_lock = true; 1062 #endif 1063 1064 /* Initialize the statistics so that each run gets its own numbers. */ 1065 if (nwriters_stress) { 1066 lock_is_write_held = false; 1067 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress, 1068 sizeof(*cxt.lwsa), 1069 GFP_KERNEL); 1070 if (cxt.lwsa == NULL) { 1071 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); 1072 firsterr = -ENOMEM; 1073 goto unwind; 1074 } 1075 1076 for (i = 0; i < cxt.nrealwriters_stress; i++) { 1077 cxt.lwsa[i].n_lock_fail = 0; 1078 cxt.lwsa[i].n_lock_acquired = 0; 1079 } 1080 } 1081 1082 if (cxt.cur_ops->readlock) { 1083 if (nreaders_stress >= 0) 1084 cxt.nrealreaders_stress = nreaders_stress; 1085 else { 1086 /* 1087 * By default distribute evenly the number of 1088 * readers and writers. We still run the same number 1089 * of threads as the writer-only locks default. 1090 */ 1091 if (nwriters_stress < 0) /* user doesn't care */ 1092 cxt.nrealwriters_stress = num_online_cpus(); 1093 cxt.nrealreaders_stress = cxt.nrealwriters_stress; 1094 } 1095 1096 if (nreaders_stress) { 1097 cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress, 1098 sizeof(*cxt.lrsa), 1099 GFP_KERNEL); 1100 if (cxt.lrsa == NULL) { 1101 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); 1102 firsterr = -ENOMEM; 1103 kfree(cxt.lwsa); 1104 cxt.lwsa = NULL; 1105 goto unwind; 1106 } 1107 1108 for (i = 0; i < cxt.nrealreaders_stress; i++) { 1109 cxt.lrsa[i].n_lock_fail = 0; 1110 cxt.lrsa[i].n_lock_acquired = 0; 1111 } 1112 } 1113 } 1114 1115 lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); 1116 1117 /* Prepare torture context. */ 1118 if (onoff_interval > 0) { 1119 firsterr = torture_onoff_init(onoff_holdoff * HZ, 1120 onoff_interval * HZ, NULL); 1121 if (torture_init_error(firsterr)) 1122 goto unwind; 1123 } 1124 if (shuffle_interval > 0) { 1125 firsterr = torture_shuffle_init(shuffle_interval); 1126 if (torture_init_error(firsterr)) 1127 goto unwind; 1128 } 1129 if (shutdown_secs > 0) { 1130 firsterr = torture_shutdown_init(shutdown_secs, 1131 lock_torture_cleanup); 1132 if (torture_init_error(firsterr)) 1133 goto unwind; 1134 } 1135 if (stutter > 0) { 1136 firsterr = torture_stutter_init(stutter, stutter); 1137 if (torture_init_error(firsterr)) 1138 goto unwind; 1139 } 1140 1141 if (nwriters_stress) { 1142 writer_tasks = kcalloc(cxt.nrealwriters_stress, 1143 sizeof(writer_tasks[0]), 1144 GFP_KERNEL); 1145 if (writer_tasks == NULL) { 1146 TOROUT_ERRSTRING("writer_tasks: Out of memory"); 1147 firsterr = -ENOMEM; 1148 goto unwind; 1149 } 1150 } 1151 1152 /* cap nested_locks to MAX_NESTED_LOCKS */ 1153 if (nested_locks > MAX_NESTED_LOCKS) 1154 nested_locks = MAX_NESTED_LOCKS; 1155 1156 if (cxt.cur_ops->readlock) { 1157 reader_tasks = kcalloc(cxt.nrealreaders_stress, 1158 sizeof(reader_tasks[0]), 1159 GFP_KERNEL); 1160 if (reader_tasks == NULL) { 1161 TOROUT_ERRSTRING("reader_tasks: Out of memory"); 1162 kfree(writer_tasks); 1163 writer_tasks = NULL; 1164 firsterr = -ENOMEM; 1165 goto unwind; 1166 } 1167 } 1168 1169 /* 1170 * Create the kthreads and start torturing (oh, those poor little locks). 1171 * 1172 * TODO: Note that we interleave writers with readers, giving writers a 1173 * slight advantage, by creating its kthread first. This can be modified 1174 * for very specific needs, or even let the user choose the policy, if 1175 * ever wanted. 1176 */ 1177 for (i = 0, j = 0; i < cxt.nrealwriters_stress || 1178 j < cxt.nrealreaders_stress; i++, j++) { 1179 if (i >= cxt.nrealwriters_stress) 1180 goto create_reader; 1181 1182 /* Create writer. */ 1183 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], 1184 writer_tasks[i]); 1185 if (torture_init_error(firsterr)) 1186 goto unwind; 1187 1188 create_reader: 1189 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) 1190 continue; 1191 /* Create reader. */ 1192 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], 1193 reader_tasks[j]); 1194 if (torture_init_error(firsterr)) 1195 goto unwind; 1196 } 1197 if (stat_interval > 0) { 1198 firsterr = torture_create_kthread(lock_torture_stats, NULL, 1199 stats_task); 1200 if (torture_init_error(firsterr)) 1201 goto unwind; 1202 } 1203 torture_init_end(); 1204 return 0; 1205 1206 unwind: 1207 torture_init_end(); 1208 lock_torture_cleanup(); 1209 if (shutdown_secs) { 1210 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST)); 1211 kernel_power_off(); 1212 } 1213 return firsterr; 1214 } 1215 1216 module_init(lock_torture_init); 1217 module_exit(lock_torture_cleanup); 1218