1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Module-based torture test facility for locking 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Davidlohr Bueso <dave@stgolabs.net> 9 * Based on kernel/rcu/torture.c. 10 */ 11 12 #define pr_fmt(fmt) fmt 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/kthread.h> 17 #include <linux/sched/rt.h> 18 #include <linux/spinlock.h> 19 #include <linux/mutex.h> 20 #include <linux/rwsem.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/rtmutex.h> 26 #include <linux/atomic.h> 27 #include <linux/moduleparam.h> 28 #include <linux/delay.h> 29 #include <linux/slab.h> 30 #include <linux/torture.h> 31 #include <linux/reboot.h> 32 33 MODULE_LICENSE("GPL"); 34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 35 36 torture_param(int, nwriters_stress, -1, 37 "Number of write-locking stress-test threads"); 38 torture_param(int, nreaders_stress, -1, 39 "Number of read-locking stress-test threads"); 40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 41 torture_param(int, onoff_interval, 0, 42 "Time between CPU hotplugs (s), 0=disable"); 43 torture_param(int, shuffle_interval, 3, 44 "Number of jiffies between shuffles, 0=disable"); 45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); 46 torture_param(int, stat_interval, 60, 47 "Number of seconds between stats printk()s"); 48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); 49 torture_param(int, verbose, 1, 50 "Enable verbose debugging printk()s"); 51 52 static char *torture_type = "spin_lock"; 53 module_param(torture_type, charp, 0444); 54 MODULE_PARM_DESC(torture_type, 55 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); 56 57 static struct task_struct *stats_task; 58 static struct task_struct **writer_tasks; 59 static struct task_struct **reader_tasks; 60 61 static bool lock_is_write_held; 62 static bool lock_is_read_held; 63 static unsigned long last_lock_release; 64 65 struct lock_stress_stats { 66 long n_lock_fail; 67 long n_lock_acquired; 68 }; 69 70 /* Forward reference. */ 71 static void lock_torture_cleanup(void); 72 73 /* 74 * Operations vector for selecting different types of tests. 75 */ 76 struct lock_torture_ops { 77 void (*init)(void); 78 void (*exit)(void); 79 int (*writelock)(int tid); 80 void (*write_delay)(struct torture_random_state *trsp); 81 void (*task_boost)(struct torture_random_state *trsp); 82 void (*writeunlock)(int tid); 83 int (*readlock)(int tid); 84 void (*read_delay)(struct torture_random_state *trsp); 85 void (*readunlock)(int tid); 86 87 unsigned long flags; /* for irq spinlocks */ 88 const char *name; 89 }; 90 91 struct lock_torture_cxt { 92 int nrealwriters_stress; 93 int nrealreaders_stress; 94 bool debug_lock; 95 bool init_called; 96 atomic_t n_lock_torture_errors; 97 struct lock_torture_ops *cur_ops; 98 struct lock_stress_stats *lwsa; /* writer statistics */ 99 struct lock_stress_stats *lrsa; /* reader statistics */ 100 }; 101 static struct lock_torture_cxt cxt = { 0, 0, false, false, 102 ATOMIC_INIT(0), 103 NULL, NULL}; 104 /* 105 * Definitions for lock torture testing. 106 */ 107 108 static int torture_lock_busted_write_lock(int tid __maybe_unused) 109 { 110 return 0; /* BUGGY, do not use in real life!!! */ 111 } 112 113 static void torture_lock_busted_write_delay(struct torture_random_state *trsp) 114 { 115 const unsigned long longdelay_ms = 100; 116 117 /* We want a long delay occasionally to force massive contention. */ 118 if (!(torture_random(trsp) % 119 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 120 mdelay(longdelay_ms); 121 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 122 torture_preempt_schedule(); /* Allow test to be preempted. */ 123 } 124 125 static void torture_lock_busted_write_unlock(int tid __maybe_unused) 126 { 127 /* BUGGY, do not use in real life!!! */ 128 } 129 130 static void torture_boost_dummy(struct torture_random_state *trsp) 131 { 132 /* Only rtmutexes care about priority */ 133 } 134 135 static struct lock_torture_ops lock_busted_ops = { 136 .writelock = torture_lock_busted_write_lock, 137 .write_delay = torture_lock_busted_write_delay, 138 .task_boost = torture_boost_dummy, 139 .writeunlock = torture_lock_busted_write_unlock, 140 .readlock = NULL, 141 .read_delay = NULL, 142 .readunlock = NULL, 143 .name = "lock_busted" 144 }; 145 146 static DEFINE_SPINLOCK(torture_spinlock); 147 148 static int torture_spin_lock_write_lock(int tid __maybe_unused) 149 __acquires(torture_spinlock) 150 { 151 spin_lock(&torture_spinlock); 152 return 0; 153 } 154 155 static void torture_spin_lock_write_delay(struct torture_random_state *trsp) 156 { 157 const unsigned long shortdelay_us = 2; 158 const unsigned long longdelay_ms = 100; 159 160 /* We want a short delay mostly to emulate likely code, and 161 * we want a long delay occasionally to force massive contention. 162 */ 163 if (!(torture_random(trsp) % 164 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 165 mdelay(longdelay_ms); 166 if (!(torture_random(trsp) % 167 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 168 udelay(shortdelay_us); 169 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 170 torture_preempt_schedule(); /* Allow test to be preempted. */ 171 } 172 173 static void torture_spin_lock_write_unlock(int tid __maybe_unused) 174 __releases(torture_spinlock) 175 { 176 spin_unlock(&torture_spinlock); 177 } 178 179 static struct lock_torture_ops spin_lock_ops = { 180 .writelock = torture_spin_lock_write_lock, 181 .write_delay = torture_spin_lock_write_delay, 182 .task_boost = torture_boost_dummy, 183 .writeunlock = torture_spin_lock_write_unlock, 184 .readlock = NULL, 185 .read_delay = NULL, 186 .readunlock = NULL, 187 .name = "spin_lock" 188 }; 189 190 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused) 191 __acquires(torture_spinlock) 192 { 193 unsigned long flags; 194 195 spin_lock_irqsave(&torture_spinlock, flags); 196 cxt.cur_ops->flags = flags; 197 return 0; 198 } 199 200 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused) 201 __releases(torture_spinlock) 202 { 203 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); 204 } 205 206 static struct lock_torture_ops spin_lock_irq_ops = { 207 .writelock = torture_spin_lock_write_lock_irq, 208 .write_delay = torture_spin_lock_write_delay, 209 .task_boost = torture_boost_dummy, 210 .writeunlock = torture_lock_spin_write_unlock_irq, 211 .readlock = NULL, 212 .read_delay = NULL, 213 .readunlock = NULL, 214 .name = "spin_lock_irq" 215 }; 216 217 static DEFINE_RWLOCK(torture_rwlock); 218 219 static int torture_rwlock_write_lock(int tid __maybe_unused) 220 __acquires(torture_rwlock) 221 { 222 write_lock(&torture_rwlock); 223 return 0; 224 } 225 226 static void torture_rwlock_write_delay(struct torture_random_state *trsp) 227 { 228 const unsigned long shortdelay_us = 2; 229 const unsigned long longdelay_ms = 100; 230 231 /* We want a short delay mostly to emulate likely code, and 232 * we want a long delay occasionally to force massive contention. 233 */ 234 if (!(torture_random(trsp) % 235 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 236 mdelay(longdelay_ms); 237 else 238 udelay(shortdelay_us); 239 } 240 241 static void torture_rwlock_write_unlock(int tid __maybe_unused) 242 __releases(torture_rwlock) 243 { 244 write_unlock(&torture_rwlock); 245 } 246 247 static int torture_rwlock_read_lock(int tid __maybe_unused) 248 __acquires(torture_rwlock) 249 { 250 read_lock(&torture_rwlock); 251 return 0; 252 } 253 254 static void torture_rwlock_read_delay(struct torture_random_state *trsp) 255 { 256 const unsigned long shortdelay_us = 10; 257 const unsigned long longdelay_ms = 100; 258 259 /* We want a short delay mostly to emulate likely code, and 260 * we want a long delay occasionally to force massive contention. 261 */ 262 if (!(torture_random(trsp) % 263 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 264 mdelay(longdelay_ms); 265 else 266 udelay(shortdelay_us); 267 } 268 269 static void torture_rwlock_read_unlock(int tid __maybe_unused) 270 __releases(torture_rwlock) 271 { 272 read_unlock(&torture_rwlock); 273 } 274 275 static struct lock_torture_ops rw_lock_ops = { 276 .writelock = torture_rwlock_write_lock, 277 .write_delay = torture_rwlock_write_delay, 278 .task_boost = torture_boost_dummy, 279 .writeunlock = torture_rwlock_write_unlock, 280 .readlock = torture_rwlock_read_lock, 281 .read_delay = torture_rwlock_read_delay, 282 .readunlock = torture_rwlock_read_unlock, 283 .name = "rw_lock" 284 }; 285 286 static int torture_rwlock_write_lock_irq(int tid __maybe_unused) 287 __acquires(torture_rwlock) 288 { 289 unsigned long flags; 290 291 write_lock_irqsave(&torture_rwlock, flags); 292 cxt.cur_ops->flags = flags; 293 return 0; 294 } 295 296 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused) 297 __releases(torture_rwlock) 298 { 299 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 300 } 301 302 static int torture_rwlock_read_lock_irq(int tid __maybe_unused) 303 __acquires(torture_rwlock) 304 { 305 unsigned long flags; 306 307 read_lock_irqsave(&torture_rwlock, flags); 308 cxt.cur_ops->flags = flags; 309 return 0; 310 } 311 312 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused) 313 __releases(torture_rwlock) 314 { 315 read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 316 } 317 318 static struct lock_torture_ops rw_lock_irq_ops = { 319 .writelock = torture_rwlock_write_lock_irq, 320 .write_delay = torture_rwlock_write_delay, 321 .task_boost = torture_boost_dummy, 322 .writeunlock = torture_rwlock_write_unlock_irq, 323 .readlock = torture_rwlock_read_lock_irq, 324 .read_delay = torture_rwlock_read_delay, 325 .readunlock = torture_rwlock_read_unlock_irq, 326 .name = "rw_lock_irq" 327 }; 328 329 static DEFINE_MUTEX(torture_mutex); 330 331 static int torture_mutex_lock(int tid __maybe_unused) 332 __acquires(torture_mutex) 333 { 334 mutex_lock(&torture_mutex); 335 return 0; 336 } 337 338 static void torture_mutex_delay(struct torture_random_state *trsp) 339 { 340 const unsigned long longdelay_ms = 100; 341 342 /* We want a long delay occasionally to force massive contention. */ 343 if (!(torture_random(trsp) % 344 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 345 mdelay(longdelay_ms * 5); 346 else 347 mdelay(longdelay_ms / 5); 348 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 349 torture_preempt_schedule(); /* Allow test to be preempted. */ 350 } 351 352 static void torture_mutex_unlock(int tid __maybe_unused) 353 __releases(torture_mutex) 354 { 355 mutex_unlock(&torture_mutex); 356 } 357 358 static struct lock_torture_ops mutex_lock_ops = { 359 .writelock = torture_mutex_lock, 360 .write_delay = torture_mutex_delay, 361 .task_boost = torture_boost_dummy, 362 .writeunlock = torture_mutex_unlock, 363 .readlock = NULL, 364 .read_delay = NULL, 365 .readunlock = NULL, 366 .name = "mutex_lock" 367 }; 368 369 #include <linux/ww_mutex.h> 370 /* 371 * The torture ww_mutexes should belong to the same lock class as 372 * torture_ww_class to avoid lockdep problem. The ww_mutex_init() 373 * function is called for initialization to ensure that. 374 */ 375 static DEFINE_WD_CLASS(torture_ww_class); 376 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2; 377 static struct ww_acquire_ctx *ww_acquire_ctxs; 378 379 static void torture_ww_mutex_init(void) 380 { 381 ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class); 382 ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class); 383 ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class); 384 385 ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress, 386 sizeof(*ww_acquire_ctxs), 387 GFP_KERNEL); 388 if (!ww_acquire_ctxs) 389 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory"); 390 } 391 392 static void torture_ww_mutex_exit(void) 393 { 394 kfree(ww_acquire_ctxs); 395 } 396 397 static int torture_ww_mutex_lock(int tid) 398 __acquires(torture_ww_mutex_0) 399 __acquires(torture_ww_mutex_1) 400 __acquires(torture_ww_mutex_2) 401 { 402 LIST_HEAD(list); 403 struct reorder_lock { 404 struct list_head link; 405 struct ww_mutex *lock; 406 } locks[3], *ll, *ln; 407 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 408 409 locks[0].lock = &torture_ww_mutex_0; 410 list_add(&locks[0].link, &list); 411 412 locks[1].lock = &torture_ww_mutex_1; 413 list_add(&locks[1].link, &list); 414 415 locks[2].lock = &torture_ww_mutex_2; 416 list_add(&locks[2].link, &list); 417 418 ww_acquire_init(ctx, &torture_ww_class); 419 420 list_for_each_entry(ll, &list, link) { 421 int err; 422 423 err = ww_mutex_lock(ll->lock, ctx); 424 if (!err) 425 continue; 426 427 ln = ll; 428 list_for_each_entry_continue_reverse(ln, &list, link) 429 ww_mutex_unlock(ln->lock); 430 431 if (err != -EDEADLK) 432 return err; 433 434 ww_mutex_lock_slow(ll->lock, ctx); 435 list_move(&ll->link, &list); 436 } 437 438 return 0; 439 } 440 441 static void torture_ww_mutex_unlock(int tid) 442 __releases(torture_ww_mutex_0) 443 __releases(torture_ww_mutex_1) 444 __releases(torture_ww_mutex_2) 445 { 446 struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; 447 448 ww_mutex_unlock(&torture_ww_mutex_0); 449 ww_mutex_unlock(&torture_ww_mutex_1); 450 ww_mutex_unlock(&torture_ww_mutex_2); 451 ww_acquire_fini(ctx); 452 } 453 454 static struct lock_torture_ops ww_mutex_lock_ops = { 455 .init = torture_ww_mutex_init, 456 .exit = torture_ww_mutex_exit, 457 .writelock = torture_ww_mutex_lock, 458 .write_delay = torture_mutex_delay, 459 .task_boost = torture_boost_dummy, 460 .writeunlock = torture_ww_mutex_unlock, 461 .readlock = NULL, 462 .read_delay = NULL, 463 .readunlock = NULL, 464 .name = "ww_mutex_lock" 465 }; 466 467 #ifdef CONFIG_RT_MUTEXES 468 static DEFINE_RT_MUTEX(torture_rtmutex); 469 470 static int torture_rtmutex_lock(int tid __maybe_unused) 471 __acquires(torture_rtmutex) 472 { 473 rt_mutex_lock(&torture_rtmutex); 474 return 0; 475 } 476 477 static void torture_rtmutex_boost(struct torture_random_state *trsp) 478 { 479 const unsigned int factor = 50000; /* yes, quite arbitrary */ 480 481 if (!rt_task(current)) { 482 /* 483 * Boost priority once every ~50k operations. When the 484 * task tries to take the lock, the rtmutex it will account 485 * for the new priority, and do any corresponding pi-dance. 486 */ 487 if (trsp && !(torture_random(trsp) % 488 (cxt.nrealwriters_stress * factor))) { 489 sched_set_fifo(current); 490 } else /* common case, do nothing */ 491 return; 492 } else { 493 /* 494 * The task will remain boosted for another ~500k operations, 495 * then restored back to its original prio, and so forth. 496 * 497 * When @trsp is nil, we want to force-reset the task for 498 * stopping the kthread. 499 */ 500 if (!trsp || !(torture_random(trsp) % 501 (cxt.nrealwriters_stress * factor * 2))) { 502 sched_set_normal(current, 0); 503 } else /* common case, do nothing */ 504 return; 505 } 506 } 507 508 static void torture_rtmutex_delay(struct torture_random_state *trsp) 509 { 510 const unsigned long shortdelay_us = 2; 511 const unsigned long longdelay_ms = 100; 512 513 /* 514 * We want a short delay mostly to emulate likely code, and 515 * we want a long delay occasionally to force massive contention. 516 */ 517 if (!(torture_random(trsp) % 518 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 519 mdelay(longdelay_ms); 520 if (!(torture_random(trsp) % 521 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 522 udelay(shortdelay_us); 523 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 524 torture_preempt_schedule(); /* Allow test to be preempted. */ 525 } 526 527 static void torture_rtmutex_unlock(int tid __maybe_unused) 528 __releases(torture_rtmutex) 529 { 530 rt_mutex_unlock(&torture_rtmutex); 531 } 532 533 static struct lock_torture_ops rtmutex_lock_ops = { 534 .writelock = torture_rtmutex_lock, 535 .write_delay = torture_rtmutex_delay, 536 .task_boost = torture_rtmutex_boost, 537 .writeunlock = torture_rtmutex_unlock, 538 .readlock = NULL, 539 .read_delay = NULL, 540 .readunlock = NULL, 541 .name = "rtmutex_lock" 542 }; 543 #endif 544 545 static DECLARE_RWSEM(torture_rwsem); 546 static int torture_rwsem_down_write(int tid __maybe_unused) 547 __acquires(torture_rwsem) 548 { 549 down_write(&torture_rwsem); 550 return 0; 551 } 552 553 static void torture_rwsem_write_delay(struct torture_random_state *trsp) 554 { 555 const unsigned long longdelay_ms = 100; 556 557 /* We want a long delay occasionally to force massive contention. */ 558 if (!(torture_random(trsp) % 559 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 560 mdelay(longdelay_ms * 10); 561 else 562 mdelay(longdelay_ms / 10); 563 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 564 torture_preempt_schedule(); /* Allow test to be preempted. */ 565 } 566 567 static void torture_rwsem_up_write(int tid __maybe_unused) 568 __releases(torture_rwsem) 569 { 570 up_write(&torture_rwsem); 571 } 572 573 static int torture_rwsem_down_read(int tid __maybe_unused) 574 __acquires(torture_rwsem) 575 { 576 down_read(&torture_rwsem); 577 return 0; 578 } 579 580 static void torture_rwsem_read_delay(struct torture_random_state *trsp) 581 { 582 const unsigned long longdelay_ms = 100; 583 584 /* We want a long delay occasionally to force massive contention. */ 585 if (!(torture_random(trsp) % 586 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 587 mdelay(longdelay_ms * 2); 588 else 589 mdelay(longdelay_ms / 2); 590 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) 591 torture_preempt_schedule(); /* Allow test to be preempted. */ 592 } 593 594 static void torture_rwsem_up_read(int tid __maybe_unused) 595 __releases(torture_rwsem) 596 { 597 up_read(&torture_rwsem); 598 } 599 600 static struct lock_torture_ops rwsem_lock_ops = { 601 .writelock = torture_rwsem_down_write, 602 .write_delay = torture_rwsem_write_delay, 603 .task_boost = torture_boost_dummy, 604 .writeunlock = torture_rwsem_up_write, 605 .readlock = torture_rwsem_down_read, 606 .read_delay = torture_rwsem_read_delay, 607 .readunlock = torture_rwsem_up_read, 608 .name = "rwsem_lock" 609 }; 610 611 #include <linux/percpu-rwsem.h> 612 static struct percpu_rw_semaphore pcpu_rwsem; 613 614 static void torture_percpu_rwsem_init(void) 615 { 616 BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); 617 } 618 619 static void torture_percpu_rwsem_exit(void) 620 { 621 percpu_free_rwsem(&pcpu_rwsem); 622 } 623 624 static int torture_percpu_rwsem_down_write(int tid __maybe_unused) 625 __acquires(pcpu_rwsem) 626 { 627 percpu_down_write(&pcpu_rwsem); 628 return 0; 629 } 630 631 static void torture_percpu_rwsem_up_write(int tid __maybe_unused) 632 __releases(pcpu_rwsem) 633 { 634 percpu_up_write(&pcpu_rwsem); 635 } 636 637 static int torture_percpu_rwsem_down_read(int tid __maybe_unused) 638 __acquires(pcpu_rwsem) 639 { 640 percpu_down_read(&pcpu_rwsem); 641 return 0; 642 } 643 644 static void torture_percpu_rwsem_up_read(int tid __maybe_unused) 645 __releases(pcpu_rwsem) 646 { 647 percpu_up_read(&pcpu_rwsem); 648 } 649 650 static struct lock_torture_ops percpu_rwsem_lock_ops = { 651 .init = torture_percpu_rwsem_init, 652 .exit = torture_percpu_rwsem_exit, 653 .writelock = torture_percpu_rwsem_down_write, 654 .write_delay = torture_rwsem_write_delay, 655 .task_boost = torture_boost_dummy, 656 .writeunlock = torture_percpu_rwsem_up_write, 657 .readlock = torture_percpu_rwsem_down_read, 658 .read_delay = torture_rwsem_read_delay, 659 .readunlock = torture_percpu_rwsem_up_read, 660 .name = "percpu_rwsem_lock" 661 }; 662 663 /* 664 * Lock torture writer kthread. Repeatedly acquires and releases 665 * the lock, checking for duplicate acquisitions. 666 */ 667 static int lock_torture_writer(void *arg) 668 { 669 struct lock_stress_stats *lwsp = arg; 670 int tid = lwsp - cxt.lwsa; 671 DEFINE_TORTURE_RANDOM(rand); 672 673 VERBOSE_TOROUT_STRING("lock_torture_writer task started"); 674 set_user_nice(current, MAX_NICE); 675 676 do { 677 if ((torture_random(&rand) & 0xfffff) == 0) 678 schedule_timeout_uninterruptible(1); 679 680 cxt.cur_ops->task_boost(&rand); 681 cxt.cur_ops->writelock(tid); 682 if (WARN_ON_ONCE(lock_is_write_held)) 683 lwsp->n_lock_fail++; 684 lock_is_write_held = true; 685 if (WARN_ON_ONCE(lock_is_read_held)) 686 lwsp->n_lock_fail++; /* rare, but... */ 687 688 lwsp->n_lock_acquired++; 689 cxt.cur_ops->write_delay(&rand); 690 lock_is_write_held = false; 691 WRITE_ONCE(last_lock_release, jiffies); 692 cxt.cur_ops->writeunlock(tid); 693 694 stutter_wait("lock_torture_writer"); 695 } while (!torture_must_stop()); 696 697 cxt.cur_ops->task_boost(NULL); /* reset prio */ 698 torture_kthread_stopping("lock_torture_writer"); 699 return 0; 700 } 701 702 /* 703 * Lock torture reader kthread. Repeatedly acquires and releases 704 * the reader lock. 705 */ 706 static int lock_torture_reader(void *arg) 707 { 708 struct lock_stress_stats *lrsp = arg; 709 int tid = lrsp - cxt.lrsa; 710 DEFINE_TORTURE_RANDOM(rand); 711 712 VERBOSE_TOROUT_STRING("lock_torture_reader task started"); 713 set_user_nice(current, MAX_NICE); 714 715 do { 716 if ((torture_random(&rand) & 0xfffff) == 0) 717 schedule_timeout_uninterruptible(1); 718 719 cxt.cur_ops->readlock(tid); 720 lock_is_read_held = true; 721 if (WARN_ON_ONCE(lock_is_write_held)) 722 lrsp->n_lock_fail++; /* rare, but... */ 723 724 lrsp->n_lock_acquired++; 725 cxt.cur_ops->read_delay(&rand); 726 lock_is_read_held = false; 727 cxt.cur_ops->readunlock(tid); 728 729 stutter_wait("lock_torture_reader"); 730 } while (!torture_must_stop()); 731 torture_kthread_stopping("lock_torture_reader"); 732 return 0; 733 } 734 735 /* 736 * Create an lock-torture-statistics message in the specified buffer. 737 */ 738 static void __torture_print_stats(char *page, 739 struct lock_stress_stats *statp, bool write) 740 { 741 bool fail = false; 742 int i, n_stress; 743 long max = 0, min = statp ? statp[0].n_lock_acquired : 0; 744 long long sum = 0; 745 746 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; 747 for (i = 0; i < n_stress; i++) { 748 if (statp[i].n_lock_fail) 749 fail = true; 750 sum += statp[i].n_lock_acquired; 751 if (max < statp[i].n_lock_acquired) 752 max = statp[i].n_lock_acquired; 753 if (min > statp[i].n_lock_acquired) 754 min = statp[i].n_lock_acquired; 755 } 756 page += sprintf(page, 757 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", 758 write ? "Writes" : "Reads ", 759 sum, max, min, 760 !onoff_interval && max / 2 > min ? "???" : "", 761 fail, fail ? "!!!" : ""); 762 if (fail) 763 atomic_inc(&cxt.n_lock_torture_errors); 764 } 765 766 /* 767 * Print torture statistics. Caller must ensure that there is only one 768 * call to this function at a given time!!! This is normally accomplished 769 * by relying on the module system to only have one copy of the module 770 * loaded, and then by giving the lock_torture_stats kthread full control 771 * (or the init/cleanup functions when lock_torture_stats thread is not 772 * running). 773 */ 774 static void lock_torture_stats_print(void) 775 { 776 int size = cxt.nrealwriters_stress * 200 + 8192; 777 char *buf; 778 779 if (cxt.cur_ops->readlock) 780 size += cxt.nrealreaders_stress * 200 + 8192; 781 782 buf = kmalloc(size, GFP_KERNEL); 783 if (!buf) { 784 pr_err("lock_torture_stats_print: Out of memory, need: %d", 785 size); 786 return; 787 } 788 789 __torture_print_stats(buf, cxt.lwsa, true); 790 pr_alert("%s", buf); 791 kfree(buf); 792 793 if (cxt.cur_ops->readlock) { 794 buf = kmalloc(size, GFP_KERNEL); 795 if (!buf) { 796 pr_err("lock_torture_stats_print: Out of memory, need: %d", 797 size); 798 return; 799 } 800 801 __torture_print_stats(buf, cxt.lrsa, false); 802 pr_alert("%s", buf); 803 kfree(buf); 804 } 805 } 806 807 /* 808 * Periodically prints torture statistics, if periodic statistics printing 809 * was specified via the stat_interval module parameter. 810 * 811 * No need to worry about fullstop here, since this one doesn't reference 812 * volatile state or register callbacks. 813 */ 814 static int lock_torture_stats(void *arg) 815 { 816 VERBOSE_TOROUT_STRING("lock_torture_stats task started"); 817 do { 818 schedule_timeout_interruptible(stat_interval * HZ); 819 lock_torture_stats_print(); 820 torture_shutdown_absorb("lock_torture_stats"); 821 } while (!torture_must_stop()); 822 torture_kthread_stopping("lock_torture_stats"); 823 return 0; 824 } 825 826 static inline void 827 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, 828 const char *tag) 829 { 830 pr_alert("%s" TORTURE_FLAG 831 "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", 832 torture_type, tag, cxt.debug_lock ? " [debug]": "", 833 cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval, 834 verbose, shuffle_interval, stutter, shutdown_secs, 835 onoff_interval, onoff_holdoff); 836 } 837 838 static void lock_torture_cleanup(void) 839 { 840 int i; 841 842 if (torture_cleanup_begin()) 843 return; 844 845 /* 846 * Indicates early cleanup, meaning that the test has not run, 847 * such as when passing bogus args when loading the module. 848 * However cxt->cur_ops.init() may have been invoked, so beside 849 * perform the underlying torture-specific cleanups, cur_ops.exit() 850 * will be invoked if needed. 851 */ 852 if (!cxt.lwsa && !cxt.lrsa) 853 goto end; 854 855 if (writer_tasks) { 856 for (i = 0; i < cxt.nrealwriters_stress; i++) 857 torture_stop_kthread(lock_torture_writer, 858 writer_tasks[i]); 859 kfree(writer_tasks); 860 writer_tasks = NULL; 861 } 862 863 if (reader_tasks) { 864 for (i = 0; i < cxt.nrealreaders_stress; i++) 865 torture_stop_kthread(lock_torture_reader, 866 reader_tasks[i]); 867 kfree(reader_tasks); 868 reader_tasks = NULL; 869 } 870 871 torture_stop_kthread(lock_torture_stats, stats_task); 872 lock_torture_stats_print(); /* -After- the stats thread is stopped! */ 873 874 if (atomic_read(&cxt.n_lock_torture_errors)) 875 lock_torture_print_module_parms(cxt.cur_ops, 876 "End of test: FAILURE"); 877 else if (torture_onoff_failures()) 878 lock_torture_print_module_parms(cxt.cur_ops, 879 "End of test: LOCK_HOTPLUG"); 880 else 881 lock_torture_print_module_parms(cxt.cur_ops, 882 "End of test: SUCCESS"); 883 884 kfree(cxt.lwsa); 885 cxt.lwsa = NULL; 886 kfree(cxt.lrsa); 887 cxt.lrsa = NULL; 888 889 end: 890 if (cxt.init_called) { 891 if (cxt.cur_ops->exit) 892 cxt.cur_ops->exit(); 893 cxt.init_called = false; 894 } 895 torture_cleanup_end(); 896 } 897 898 static int __init lock_torture_init(void) 899 { 900 int i, j; 901 int firsterr = 0; 902 static struct lock_torture_ops *torture_ops[] = { 903 &lock_busted_ops, 904 &spin_lock_ops, &spin_lock_irq_ops, 905 &rw_lock_ops, &rw_lock_irq_ops, 906 &mutex_lock_ops, 907 &ww_mutex_lock_ops, 908 #ifdef CONFIG_RT_MUTEXES 909 &rtmutex_lock_ops, 910 #endif 911 &rwsem_lock_ops, 912 &percpu_rwsem_lock_ops, 913 }; 914 915 if (!torture_init_begin(torture_type, verbose)) 916 return -EBUSY; 917 918 /* Process args and tell the world that the torturer is on the job. */ 919 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 920 cxt.cur_ops = torture_ops[i]; 921 if (strcmp(torture_type, cxt.cur_ops->name) == 0) 922 break; 923 } 924 if (i == ARRAY_SIZE(torture_ops)) { 925 pr_alert("lock-torture: invalid torture type: \"%s\"\n", 926 torture_type); 927 pr_alert("lock-torture types:"); 928 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 929 pr_alert(" %s", torture_ops[i]->name); 930 pr_alert("\n"); 931 firsterr = -EINVAL; 932 goto unwind; 933 } 934 935 if (nwriters_stress == 0 && 936 (!cxt.cur_ops->readlock || nreaders_stress == 0)) { 937 pr_alert("lock-torture: must run at least one locking thread\n"); 938 firsterr = -EINVAL; 939 goto unwind; 940 } 941 942 if (nwriters_stress >= 0) 943 cxt.nrealwriters_stress = nwriters_stress; 944 else 945 cxt.nrealwriters_stress = 2 * num_online_cpus(); 946 947 if (cxt.cur_ops->init) { 948 cxt.cur_ops->init(); 949 cxt.init_called = true; 950 } 951 952 #ifdef CONFIG_DEBUG_MUTEXES 953 if (str_has_prefix(torture_type, "mutex")) 954 cxt.debug_lock = true; 955 #endif 956 #ifdef CONFIG_DEBUG_RT_MUTEXES 957 if (str_has_prefix(torture_type, "rtmutex")) 958 cxt.debug_lock = true; 959 #endif 960 #ifdef CONFIG_DEBUG_SPINLOCK 961 if ((str_has_prefix(torture_type, "spin")) || 962 (str_has_prefix(torture_type, "rw_lock"))) 963 cxt.debug_lock = true; 964 #endif 965 966 /* Initialize the statistics so that each run gets its own numbers. */ 967 if (nwriters_stress) { 968 lock_is_write_held = false; 969 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress, 970 sizeof(*cxt.lwsa), 971 GFP_KERNEL); 972 if (cxt.lwsa == NULL) { 973 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); 974 firsterr = -ENOMEM; 975 goto unwind; 976 } 977 978 for (i = 0; i < cxt.nrealwriters_stress; i++) { 979 cxt.lwsa[i].n_lock_fail = 0; 980 cxt.lwsa[i].n_lock_acquired = 0; 981 } 982 } 983 984 if (cxt.cur_ops->readlock) { 985 if (nreaders_stress >= 0) 986 cxt.nrealreaders_stress = nreaders_stress; 987 else { 988 /* 989 * By default distribute evenly the number of 990 * readers and writers. We still run the same number 991 * of threads as the writer-only locks default. 992 */ 993 if (nwriters_stress < 0) /* user doesn't care */ 994 cxt.nrealwriters_stress = num_online_cpus(); 995 cxt.nrealreaders_stress = cxt.nrealwriters_stress; 996 } 997 998 if (nreaders_stress) { 999 lock_is_read_held = false; 1000 cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress, 1001 sizeof(*cxt.lrsa), 1002 GFP_KERNEL); 1003 if (cxt.lrsa == NULL) { 1004 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); 1005 firsterr = -ENOMEM; 1006 kfree(cxt.lwsa); 1007 cxt.lwsa = NULL; 1008 goto unwind; 1009 } 1010 1011 for (i = 0; i < cxt.nrealreaders_stress; i++) { 1012 cxt.lrsa[i].n_lock_fail = 0; 1013 cxt.lrsa[i].n_lock_acquired = 0; 1014 } 1015 } 1016 } 1017 1018 lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); 1019 1020 /* Prepare torture context. */ 1021 if (onoff_interval > 0) { 1022 firsterr = torture_onoff_init(onoff_holdoff * HZ, 1023 onoff_interval * HZ, NULL); 1024 if (firsterr) 1025 goto unwind; 1026 } 1027 if (shuffle_interval > 0) { 1028 firsterr = torture_shuffle_init(shuffle_interval); 1029 if (firsterr) 1030 goto unwind; 1031 } 1032 if (shutdown_secs > 0) { 1033 firsterr = torture_shutdown_init(shutdown_secs, 1034 lock_torture_cleanup); 1035 if (firsterr) 1036 goto unwind; 1037 } 1038 if (stutter > 0) { 1039 firsterr = torture_stutter_init(stutter, stutter); 1040 if (firsterr) 1041 goto unwind; 1042 } 1043 1044 if (nwriters_stress) { 1045 writer_tasks = kcalloc(cxt.nrealwriters_stress, 1046 sizeof(writer_tasks[0]), 1047 GFP_KERNEL); 1048 if (writer_tasks == NULL) { 1049 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory"); 1050 firsterr = -ENOMEM; 1051 goto unwind; 1052 } 1053 } 1054 1055 if (cxt.cur_ops->readlock) { 1056 reader_tasks = kcalloc(cxt.nrealreaders_stress, 1057 sizeof(reader_tasks[0]), 1058 GFP_KERNEL); 1059 if (reader_tasks == NULL) { 1060 VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory"); 1061 kfree(writer_tasks); 1062 writer_tasks = NULL; 1063 firsterr = -ENOMEM; 1064 goto unwind; 1065 } 1066 } 1067 1068 /* 1069 * Create the kthreads and start torturing (oh, those poor little locks). 1070 * 1071 * TODO: Note that we interleave writers with readers, giving writers a 1072 * slight advantage, by creating its kthread first. This can be modified 1073 * for very specific needs, or even let the user choose the policy, if 1074 * ever wanted. 1075 */ 1076 for (i = 0, j = 0; i < cxt.nrealwriters_stress || 1077 j < cxt.nrealreaders_stress; i++, j++) { 1078 if (i >= cxt.nrealwriters_stress) 1079 goto create_reader; 1080 1081 /* Create writer. */ 1082 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], 1083 writer_tasks[i]); 1084 if (firsterr) 1085 goto unwind; 1086 1087 create_reader: 1088 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) 1089 continue; 1090 /* Create reader. */ 1091 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], 1092 reader_tasks[j]); 1093 if (firsterr) 1094 goto unwind; 1095 } 1096 if (stat_interval > 0) { 1097 firsterr = torture_create_kthread(lock_torture_stats, NULL, 1098 stats_task); 1099 if (firsterr) 1100 goto unwind; 1101 } 1102 torture_init_end(); 1103 return 0; 1104 1105 unwind: 1106 torture_init_end(); 1107 lock_torture_cleanup(); 1108 if (shutdown_secs) { 1109 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST)); 1110 kernel_power_off(); 1111 } 1112 return firsterr; 1113 } 1114 1115 module_init(lock_torture_init); 1116 module_exit(lock_torture_cleanup); 1117