1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Common functions for in-kernel torture tests. 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Author: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Based on kernel/rcu/torture.c. 9 */ 10 11 #define pr_fmt(fmt) fmt 12 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/interrupt.h> 22 #include <linux/sched.h> 23 #include <linux/sched/clock.h> 24 #include <linux/atomic.h> 25 #include <linux/bitops.h> 26 #include <linux/completion.h> 27 #include <linux/moduleparam.h> 28 #include <linux/percpu.h> 29 #include <linux/notifier.h> 30 #include <linux/reboot.h> 31 #include <linux/freezer.h> 32 #include <linux/cpu.h> 33 #include <linux/delay.h> 34 #include <linux/stat.h> 35 #include <linux/slab.h> 36 #include <linux/trace_clock.h> 37 #include <linux/ktime.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include "rcu/rcu.h" 41 42 MODULE_LICENSE("GPL"); 43 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 44 45 static bool disable_onoff_at_boot; 46 module_param(disable_onoff_at_boot, bool, 0444); 47 48 static bool ftrace_dump_at_shutdown; 49 module_param(ftrace_dump_at_shutdown, bool, 0444); 50 51 static int verbose_sleep_frequency; 52 module_param(verbose_sleep_frequency, int, 0444); 53 54 static int verbose_sleep_duration = 1; 55 module_param(verbose_sleep_duration, int, 0444); 56 57 static char *torture_type; 58 static int verbose; 59 60 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 61 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 62 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */ 63 #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */ 64 static int fullstop = FULLSTOP_RMMOD; 65 static DEFINE_MUTEX(fullstop_mutex); 66 67 static atomic_t verbose_sleep_counter; 68 69 /* 70 * Sleep if needed from VERBOSE_TOROUT*(). 71 */ 72 void verbose_torout_sleep(void) 73 { 74 if (verbose_sleep_frequency > 0 && 75 verbose_sleep_duration > 0 && 76 !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency)) 77 schedule_timeout_uninterruptible(verbose_sleep_duration); 78 } 79 EXPORT_SYMBOL_GPL(verbose_torout_sleep); 80 81 /* 82 * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit 83 * nanosecond random fuzz. This function and its friends desynchronize 84 * testing from the timer wheel. 85 */ 86 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp) 87 { 88 ktime_t hto = baset_ns; 89 90 if (trsp) 91 hto += (torture_random(trsp) >> 3) % fuzzt_ns; 92 set_current_state(TASK_UNINTERRUPTIBLE); 93 return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); 94 } 95 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); 96 97 /* 98 * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit 99 * nanosecond (not microsecond!) random fuzz. 100 */ 101 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp) 102 { 103 ktime_t baset_ns = baset_us * NSEC_PER_USEC; 104 105 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 106 } 107 EXPORT_SYMBOL_GPL(torture_hrtimeout_us); 108 109 /* 110 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 111 * microsecond (not millisecond!) random fuzz. 112 */ 113 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp) 114 { 115 ktime_t baset_ns = baset_ms * NSEC_PER_MSEC; 116 u32 fuzzt_ns; 117 118 if ((u32)~0U / NSEC_PER_USEC < fuzzt_us) 119 fuzzt_ns = (u32)~0U; 120 else 121 fuzzt_ns = fuzzt_us * NSEC_PER_USEC; 122 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 123 } 124 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms); 125 126 /* 127 * Schedule a high-resolution-timer sleep in jiffies, with an 128 * implied one-jiffy random fuzz. This is intended to replace calls to 129 * schedule_timeout_interruptible() and friends. 130 */ 131 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp) 132 { 133 ktime_t baset_ns = jiffies_to_nsecs(baset_j); 134 135 return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp); 136 } 137 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies); 138 139 /* 140 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 141 * millisecond (not second!) random fuzz. 142 */ 143 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp) 144 { 145 ktime_t baset_ns = baset_s * NSEC_PER_SEC; 146 u32 fuzzt_ns; 147 148 if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms) 149 fuzzt_ns = (u32)~0U; 150 else 151 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC; 152 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 153 } 154 EXPORT_SYMBOL_GPL(torture_hrtimeout_s); 155 156 #ifdef CONFIG_HOTPLUG_CPU 157 158 /* 159 * Variables for online-offline handling. Only present if CPU hotplug 160 * is enabled, otherwise does nothing. 161 */ 162 163 static struct task_struct *onoff_task; 164 static long onoff_holdoff; 165 static long onoff_interval; 166 static torture_ofl_func *onoff_f; 167 static long n_offline_attempts; 168 static long n_offline_successes; 169 static unsigned long sum_offline; 170 static int min_offline = -1; 171 static int max_offline; 172 static long n_online_attempts; 173 static long n_online_successes; 174 static unsigned long sum_online; 175 static int min_online = -1; 176 static int max_online; 177 178 static int torture_online_cpus = NR_CPUS; 179 180 /* 181 * Some torture testing leverages confusion as to the number of online 182 * CPUs. This function returns the torture-testing view of this number, 183 * which allows torture tests to load-balance appropriately. 184 */ 185 int torture_num_online_cpus(void) 186 { 187 return READ_ONCE(torture_online_cpus); 188 } 189 EXPORT_SYMBOL_GPL(torture_num_online_cpus); 190 191 /* 192 * Attempt to take a CPU offline. Return false if the CPU is already 193 * offline or if it is not subject to CPU-hotplug operations. The 194 * caller can detect other failures by looking at the statistics. 195 */ 196 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, 197 unsigned long *sum_offl, int *min_offl, int *max_offl) 198 { 199 unsigned long delta; 200 int ret; 201 char *s; 202 unsigned long starttime; 203 204 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 205 return false; 206 if (num_online_cpus() <= 1) 207 return false; /* Can't offline the last CPU. */ 208 209 if (verbose > 1) 210 pr_alert("%s" TORTURE_FLAG 211 "torture_onoff task: offlining %d\n", 212 torture_type, cpu); 213 starttime = jiffies; 214 (*n_offl_attempts)++; 215 ret = remove_cpu(cpu); 216 if (ret) { 217 s = ""; 218 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 219 // PCI probe frequently disables hotplug during boot. 220 (*n_offl_attempts)--; 221 s = " (-EBUSY forgiven during boot)"; 222 } 223 if (verbose) 224 pr_alert("%s" TORTURE_FLAG 225 "torture_onoff task: offline %d failed%s: errno %d\n", 226 torture_type, cpu, s, ret); 227 } else { 228 if (verbose > 1) 229 pr_alert("%s" TORTURE_FLAG 230 "torture_onoff task: offlined %d\n", 231 torture_type, cpu); 232 if (onoff_f) 233 onoff_f(); 234 (*n_offl_successes)++; 235 delta = jiffies - starttime; 236 *sum_offl += delta; 237 if (*min_offl < 0) { 238 *min_offl = delta; 239 *max_offl = delta; 240 } 241 if (*min_offl > delta) 242 *min_offl = delta; 243 if (*max_offl < delta) 244 *max_offl = delta; 245 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1); 246 WARN_ON_ONCE(torture_online_cpus <= 0); 247 } 248 249 return true; 250 } 251 EXPORT_SYMBOL_GPL(torture_offline); 252 253 /* 254 * Attempt to bring a CPU online. Return false if the CPU is already 255 * online or if it is not subject to CPU-hotplug operations. The 256 * caller can detect other failures by looking at the statistics. 257 */ 258 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, 259 unsigned long *sum_onl, int *min_onl, int *max_onl) 260 { 261 unsigned long delta; 262 int ret; 263 char *s; 264 unsigned long starttime; 265 266 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 267 return false; 268 269 if (verbose > 1) 270 pr_alert("%s" TORTURE_FLAG 271 "torture_onoff task: onlining %d\n", 272 torture_type, cpu); 273 starttime = jiffies; 274 (*n_onl_attempts)++; 275 ret = add_cpu(cpu); 276 if (ret) { 277 s = ""; 278 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 279 // PCI probe frequently disables hotplug during boot. 280 (*n_onl_attempts)--; 281 s = " (-EBUSY forgiven during boot)"; 282 } 283 if (verbose) 284 pr_alert("%s" TORTURE_FLAG 285 "torture_onoff task: online %d failed%s: errno %d\n", 286 torture_type, cpu, s, ret); 287 } else { 288 if (verbose > 1) 289 pr_alert("%s" TORTURE_FLAG 290 "torture_onoff task: onlined %d\n", 291 torture_type, cpu); 292 (*n_onl_successes)++; 293 delta = jiffies - starttime; 294 *sum_onl += delta; 295 if (*min_onl < 0) { 296 *min_onl = delta; 297 *max_onl = delta; 298 } 299 if (*min_onl > delta) 300 *min_onl = delta; 301 if (*max_onl < delta) 302 *max_onl = delta; 303 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1); 304 } 305 306 return true; 307 } 308 EXPORT_SYMBOL_GPL(torture_online); 309 310 /* 311 * Get everything online at the beginning and ends of tests. 312 */ 313 static void torture_online_all(char *phase) 314 { 315 int cpu; 316 int ret; 317 318 for_each_possible_cpu(cpu) { 319 if (cpu_online(cpu)) 320 continue; 321 ret = add_cpu(cpu); 322 if (ret && verbose) { 323 pr_alert("%s" TORTURE_FLAG 324 "%s: %s online %d: errno %d\n", 325 __func__, phase, torture_type, cpu, ret); 326 } 327 } 328 } 329 330 /* 331 * Execute random CPU-hotplug operations at the interval specified 332 * by the onoff_interval. 333 */ 334 static int 335 torture_onoff(void *arg) 336 { 337 int cpu; 338 int maxcpu = -1; 339 DEFINE_TORTURE_RANDOM(rand); 340 341 VERBOSE_TOROUT_STRING("torture_onoff task started"); 342 for_each_online_cpu(cpu) 343 maxcpu = cpu; 344 WARN_ON(maxcpu < 0); 345 torture_online_all("Initial"); 346 if (maxcpu == 0) { 347 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled"); 348 goto stop; 349 } 350 351 if (onoff_holdoff > 0) { 352 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); 353 schedule_timeout_interruptible(onoff_holdoff); 354 VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); 355 } 356 while (!torture_must_stop()) { 357 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { 358 schedule_timeout_interruptible(HZ / 10); 359 continue; 360 } 361 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); 362 if (!torture_offline(cpu, 363 &n_offline_attempts, &n_offline_successes, 364 &sum_offline, &min_offline, &max_offline)) 365 torture_online(cpu, 366 &n_online_attempts, &n_online_successes, 367 &sum_online, &min_online, &max_online); 368 schedule_timeout_interruptible(onoff_interval); 369 } 370 371 stop: 372 torture_kthread_stopping("torture_onoff"); 373 torture_online_all("Final"); 374 return 0; 375 } 376 377 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 378 379 /* 380 * Initiate online-offline handling. 381 */ 382 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f) 383 { 384 #ifdef CONFIG_HOTPLUG_CPU 385 onoff_holdoff = ooholdoff; 386 onoff_interval = oointerval; 387 onoff_f = f; 388 if (onoff_interval <= 0) 389 return 0; 390 return torture_create_kthread(torture_onoff, NULL, onoff_task); 391 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 392 return 0; 393 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 394 } 395 EXPORT_SYMBOL_GPL(torture_onoff_init); 396 397 /* 398 * Clean up after online/offline testing. 399 */ 400 static void torture_onoff_cleanup(void) 401 { 402 #ifdef CONFIG_HOTPLUG_CPU 403 if (onoff_task == NULL) 404 return; 405 VERBOSE_TOROUT_STRING("Stopping torture_onoff task"); 406 kthread_stop(onoff_task); 407 onoff_task = NULL; 408 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 409 } 410 411 /* 412 * Print online/offline testing statistics. 413 */ 414 void torture_onoff_stats(void) 415 { 416 #ifdef CONFIG_HOTPLUG_CPU 417 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", 418 n_online_successes, n_online_attempts, 419 n_offline_successes, n_offline_attempts, 420 min_online, max_online, 421 min_offline, max_offline, 422 sum_online, sum_offline, HZ); 423 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 424 } 425 EXPORT_SYMBOL_GPL(torture_onoff_stats); 426 427 /* 428 * Were all the online/offline operations successful? 429 */ 430 bool torture_onoff_failures(void) 431 { 432 #ifdef CONFIG_HOTPLUG_CPU 433 return n_online_successes != n_online_attempts || 434 n_offline_successes != n_offline_attempts; 435 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 436 return false; 437 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 438 } 439 EXPORT_SYMBOL_GPL(torture_onoff_failures); 440 441 #define TORTURE_RANDOM_MULT 39916801 /* prime */ 442 #define TORTURE_RANDOM_ADD 479001701 /* prime */ 443 #define TORTURE_RANDOM_REFRESH 10000 444 445 /* 446 * Crude but fast random-number generator. Uses a linear congruential 447 * generator, with occasional help from cpu_clock(). 448 */ 449 unsigned long 450 torture_random(struct torture_random_state *trsp) 451 { 452 if (--trsp->trs_count < 0) { 453 trsp->trs_state += (unsigned long)local_clock(); 454 trsp->trs_count = TORTURE_RANDOM_REFRESH; 455 } 456 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + 457 TORTURE_RANDOM_ADD; 458 return swahw32(trsp->trs_state); 459 } 460 EXPORT_SYMBOL_GPL(torture_random); 461 462 /* 463 * Variables for shuffling. The idea is to ensure that each CPU stays 464 * idle for an extended period to test interactions with dyntick idle, 465 * as well as interactions with any per-CPU variables. 466 */ 467 struct shuffle_task { 468 struct list_head st_l; 469 struct task_struct *st_t; 470 }; 471 472 static long shuffle_interval; /* In jiffies. */ 473 static struct task_struct *shuffler_task; 474 static cpumask_var_t shuffle_tmp_mask; 475 static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */ 476 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list); 477 static DEFINE_MUTEX(shuffle_task_mutex); 478 479 /* 480 * Register a task to be shuffled. If there is no memory, just splat 481 * and don't bother registering. 482 */ 483 void torture_shuffle_task_register(struct task_struct *tp) 484 { 485 struct shuffle_task *stp; 486 487 if (WARN_ON_ONCE(tp == NULL)) 488 return; 489 stp = kmalloc(sizeof(*stp), GFP_KERNEL); 490 if (WARN_ON_ONCE(stp == NULL)) 491 return; 492 stp->st_t = tp; 493 mutex_lock(&shuffle_task_mutex); 494 list_add(&stp->st_l, &shuffle_task_list); 495 mutex_unlock(&shuffle_task_mutex); 496 } 497 EXPORT_SYMBOL_GPL(torture_shuffle_task_register); 498 499 /* 500 * Unregister all tasks, for example, at the end of the torture run. 501 */ 502 static void torture_shuffle_task_unregister_all(void) 503 { 504 struct shuffle_task *stp; 505 struct shuffle_task *p; 506 507 mutex_lock(&shuffle_task_mutex); 508 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) { 509 list_del(&stp->st_l); 510 kfree(stp); 511 } 512 mutex_unlock(&shuffle_task_mutex); 513 } 514 515 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle. 516 * A special case is when shuffle_idle_cpu = -1, in which case we allow 517 * the tasks to run on all CPUs. 518 */ 519 static void torture_shuffle_tasks(void) 520 { 521 struct shuffle_task *stp; 522 523 cpumask_setall(shuffle_tmp_mask); 524 cpus_read_lock(); 525 526 /* No point in shuffling if there is only one online CPU (ex: UP) */ 527 if (num_online_cpus() == 1) { 528 cpus_read_unlock(); 529 return; 530 } 531 532 /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */ 533 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask); 534 if (shuffle_idle_cpu >= nr_cpu_ids) 535 shuffle_idle_cpu = -1; 536 else 537 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); 538 539 mutex_lock(&shuffle_task_mutex); 540 list_for_each_entry(stp, &shuffle_task_list, st_l) 541 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); 542 mutex_unlock(&shuffle_task_mutex); 543 544 cpus_read_unlock(); 545 } 546 547 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 548 * system to become idle at a time and cut off its timer ticks. This is meant 549 * to test the support for such tickless idle CPU in RCU. 550 */ 551 static int torture_shuffle(void *arg) 552 { 553 VERBOSE_TOROUT_STRING("torture_shuffle task started"); 554 do { 555 schedule_timeout_interruptible(shuffle_interval); 556 torture_shuffle_tasks(); 557 torture_shutdown_absorb("torture_shuffle"); 558 } while (!torture_must_stop()); 559 torture_kthread_stopping("torture_shuffle"); 560 return 0; 561 } 562 563 /* 564 * Start the shuffler, with shuffint in jiffies. 565 */ 566 int torture_shuffle_init(long shuffint) 567 { 568 shuffle_interval = shuffint; 569 570 shuffle_idle_cpu = -1; 571 572 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 573 TOROUT_ERRSTRING("Failed to alloc mask"); 574 return -ENOMEM; 575 } 576 577 /* Create the shuffler thread */ 578 return torture_create_kthread(torture_shuffle, NULL, shuffler_task); 579 } 580 EXPORT_SYMBOL_GPL(torture_shuffle_init); 581 582 /* 583 * Stop the shuffling. 584 */ 585 static void torture_shuffle_cleanup(void) 586 { 587 torture_shuffle_task_unregister_all(); 588 if (shuffler_task) { 589 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task"); 590 kthread_stop(shuffler_task); 591 free_cpumask_var(shuffle_tmp_mask); 592 } 593 shuffler_task = NULL; 594 } 595 596 /* 597 * Variables for auto-shutdown. This allows "lights out" torture runs 598 * to be fully scripted. 599 */ 600 static struct task_struct *shutdown_task; 601 static ktime_t shutdown_time; /* time to system shutdown. */ 602 static void (*torture_shutdown_hook)(void); 603 604 /* 605 * Absorb kthreads into a kernel function that won't return, so that 606 * they won't ever access module text or data again. 607 */ 608 void torture_shutdown_absorb(const char *title) 609 { 610 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 611 pr_notice("torture thread %s parking due to system shutdown\n", 612 title); 613 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 614 } 615 } 616 EXPORT_SYMBOL_GPL(torture_shutdown_absorb); 617 618 /* 619 * Cause the torture test to shutdown the system after the test has 620 * run for the time specified by the shutdown_secs parameter. 621 */ 622 static int torture_shutdown(void *arg) 623 { 624 ktime_t ktime_snap; 625 626 VERBOSE_TOROUT_STRING("torture_shutdown task started"); 627 ktime_snap = ktime_get(); 628 while (ktime_before(ktime_snap, shutdown_time) && 629 !torture_must_stop()) { 630 if (verbose) 631 pr_alert("%s" TORTURE_FLAG 632 "torture_shutdown task: %llu ms remaining\n", 633 torture_type, 634 ktime_ms_delta(shutdown_time, ktime_snap)); 635 set_current_state(TASK_INTERRUPTIBLE); 636 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS); 637 ktime_snap = ktime_get(); 638 } 639 if (torture_must_stop()) { 640 torture_kthread_stopping("torture_shutdown"); 641 return 0; 642 } 643 644 /* OK, shut down the system. */ 645 646 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system"); 647 shutdown_task = NULL; /* Avoid self-kill deadlock. */ 648 if (torture_shutdown_hook) 649 torture_shutdown_hook(); 650 else 651 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping."); 652 if (ftrace_dump_at_shutdown) 653 rcu_ftrace_dump(DUMP_ALL); 654 kernel_power_off(); /* Shut down the system. */ 655 return 0; 656 } 657 658 /* 659 * Start up the shutdown task. 660 */ 661 int torture_shutdown_init(int ssecs, void (*cleanup)(void)) 662 { 663 torture_shutdown_hook = cleanup; 664 if (ssecs > 0) { 665 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0)); 666 return torture_create_kthread(torture_shutdown, NULL, 667 shutdown_task); 668 } 669 return 0; 670 } 671 EXPORT_SYMBOL_GPL(torture_shutdown_init); 672 673 /* 674 * Detect and respond to a system shutdown. 675 */ 676 static int torture_shutdown_notify(struct notifier_block *unused1, 677 unsigned long unused2, void *unused3) 678 { 679 mutex_lock(&fullstop_mutex); 680 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) { 681 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); 682 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN); 683 } else { 684 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 685 } 686 mutex_unlock(&fullstop_mutex); 687 return NOTIFY_DONE; 688 } 689 690 static struct notifier_block torture_shutdown_nb = { 691 .notifier_call = torture_shutdown_notify, 692 }; 693 694 /* 695 * Shut down the shutdown task. Say what??? Heh! This can happen if 696 * the torture module gets an rmmod before the shutdown time arrives. ;-) 697 */ 698 static void torture_shutdown_cleanup(void) 699 { 700 unregister_reboot_notifier(&torture_shutdown_nb); 701 if (shutdown_task != NULL) { 702 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task"); 703 kthread_stop(shutdown_task); 704 } 705 shutdown_task = NULL; 706 } 707 708 /* 709 * Variables for stuttering, which means to periodically pause and 710 * restart testing in order to catch bugs that appear when load is 711 * suddenly applied to or removed from the system. 712 */ 713 static struct task_struct *stutter_task; 714 static int stutter_pause_test; 715 static int stutter; 716 static int stutter_gap; 717 718 /* 719 * Block until the stutter interval ends. This must be called periodically 720 * by all running kthreads that need to be subject to stuttering. 721 */ 722 bool stutter_wait(const char *title) 723 { 724 unsigned int i = 0; 725 bool ret = false; 726 int spt; 727 728 cond_resched_tasks_rcu_qs(); 729 spt = READ_ONCE(stutter_pause_test); 730 for (; spt; spt = READ_ONCE(stutter_pause_test)) { 731 if (!ret) { 732 sched_set_normal(current, MAX_NICE); 733 ret = true; 734 } 735 if (spt == 1) { 736 schedule_timeout_interruptible(1); 737 } else if (spt == 2) { 738 while (READ_ONCE(stutter_pause_test)) { 739 if (!(i++ & 0xffff)) 740 torture_hrtimeout_us(10, 0, NULL); 741 cond_resched(); 742 } 743 } else { 744 schedule_timeout_interruptible(round_jiffies_relative(HZ)); 745 } 746 torture_shutdown_absorb(title); 747 } 748 return ret; 749 } 750 EXPORT_SYMBOL_GPL(stutter_wait); 751 752 /* 753 * Cause the torture test to "stutter", starting and stopping all 754 * threads periodically. 755 */ 756 static int torture_stutter(void *arg) 757 { 758 DEFINE_TORTURE_RANDOM(rand); 759 int wtime; 760 761 VERBOSE_TOROUT_STRING("torture_stutter task started"); 762 do { 763 if (!torture_must_stop() && stutter > 1) { 764 wtime = stutter; 765 if (stutter > 2) { 766 WRITE_ONCE(stutter_pause_test, 1); 767 wtime = stutter - 3; 768 torture_hrtimeout_jiffies(wtime, &rand); 769 wtime = 2; 770 } 771 WRITE_ONCE(stutter_pause_test, 2); 772 torture_hrtimeout_jiffies(wtime, NULL); 773 } 774 WRITE_ONCE(stutter_pause_test, 0); 775 if (!torture_must_stop()) 776 torture_hrtimeout_jiffies(stutter_gap, NULL); 777 torture_shutdown_absorb("torture_stutter"); 778 } while (!torture_must_stop()); 779 torture_kthread_stopping("torture_stutter"); 780 return 0; 781 } 782 783 /* 784 * Initialize and kick off the torture_stutter kthread. 785 */ 786 int torture_stutter_init(const int s, const int sgap) 787 { 788 stutter = s; 789 stutter_gap = sgap; 790 return torture_create_kthread(torture_stutter, NULL, stutter_task); 791 } 792 EXPORT_SYMBOL_GPL(torture_stutter_init); 793 794 /* 795 * Cleanup after the torture_stutter kthread. 796 */ 797 static void torture_stutter_cleanup(void) 798 { 799 if (!stutter_task) 800 return; 801 VERBOSE_TOROUT_STRING("Stopping torture_stutter task"); 802 kthread_stop(stutter_task); 803 stutter_task = NULL; 804 } 805 806 /* 807 * Initialize torture module. Please note that this is -not- invoked via 808 * the usual module_init() mechanism, but rather by an explicit call from 809 * the client torture module. This call must be paired with a later 810 * torture_init_end(). 811 * 812 * The runnable parameter points to a flag that controls whether or not 813 * the test is currently runnable. If there is no such flag, pass in NULL. 814 */ 815 bool torture_init_begin(char *ttype, int v) 816 { 817 mutex_lock(&fullstop_mutex); 818 if (torture_type != NULL) { 819 pr_alert("%s: Refusing %s init: %s running.\n", 820 __func__, ttype, torture_type); 821 pr_alert("%s: One torture test at a time!\n", __func__); 822 mutex_unlock(&fullstop_mutex); 823 return false; 824 } 825 torture_type = ttype; 826 verbose = v; 827 fullstop = FULLSTOP_DONTSTOP; 828 return true; 829 } 830 EXPORT_SYMBOL_GPL(torture_init_begin); 831 832 /* 833 * Tell the torture module that initialization is complete. 834 */ 835 void torture_init_end(void) 836 { 837 mutex_unlock(&fullstop_mutex); 838 register_reboot_notifier(&torture_shutdown_nb); 839 } 840 EXPORT_SYMBOL_GPL(torture_init_end); 841 842 /* 843 * Clean up torture module. Please note that this is -not- invoked via 844 * the usual module_exit() mechanism, but rather by an explicit call from 845 * the client torture module. Returns true if a race with system shutdown 846 * is detected, otherwise, all kthreads started by functions in this file 847 * will be shut down. 848 * 849 * This must be called before the caller starts shutting down its own 850 * kthreads. 851 * 852 * Both torture_cleanup_begin() and torture_cleanup_end() must be paired, 853 * in order to correctly perform the cleanup. They are separated because 854 * threads can still need to reference the torture_type type, thus nullify 855 * only after completing all other relevant calls. 856 */ 857 bool torture_cleanup_begin(void) 858 { 859 mutex_lock(&fullstop_mutex); 860 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 861 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 862 mutex_unlock(&fullstop_mutex); 863 schedule_timeout_uninterruptible(10); 864 return true; 865 } 866 WRITE_ONCE(fullstop, FULLSTOP_RMMOD); 867 mutex_unlock(&fullstop_mutex); 868 torture_shutdown_cleanup(); 869 torture_shuffle_cleanup(); 870 torture_stutter_cleanup(); 871 torture_onoff_cleanup(); 872 return false; 873 } 874 EXPORT_SYMBOL_GPL(torture_cleanup_begin); 875 876 void torture_cleanup_end(void) 877 { 878 mutex_lock(&fullstop_mutex); 879 torture_type = NULL; 880 mutex_unlock(&fullstop_mutex); 881 } 882 EXPORT_SYMBOL_GPL(torture_cleanup_end); 883 884 /* 885 * Is it time for the current torture test to stop? 886 */ 887 bool torture_must_stop(void) 888 { 889 return torture_must_stop_irq() || kthread_should_stop(); 890 } 891 EXPORT_SYMBOL_GPL(torture_must_stop); 892 893 /* 894 * Is it time for the current torture test to stop? This is the irq-safe 895 * version, hence no check for kthread_should_stop(). 896 */ 897 bool torture_must_stop_irq(void) 898 { 899 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP; 900 } 901 EXPORT_SYMBOL_GPL(torture_must_stop_irq); 902 903 /* 904 * Each kthread must wait for kthread_should_stop() before returning from 905 * its top-level function, otherwise segfaults ensue. This function 906 * prints a "stopping" message and waits for kthread_should_stop(), and 907 * should be called from all torture kthreads immediately prior to 908 * returning. 909 */ 910 void torture_kthread_stopping(char *title) 911 { 912 char buf[128]; 913 914 snprintf(buf, sizeof(buf), "%s is stopping", title); 915 VERBOSE_TOROUT_STRING(buf); 916 while (!kthread_should_stop()) { 917 torture_shutdown_absorb(title); 918 schedule_timeout_uninterruptible(1); 919 } 920 } 921 EXPORT_SYMBOL_GPL(torture_kthread_stopping); 922 923 /* 924 * Create a generic torture kthread that is immediately runnable. If you 925 * need the kthread to be stopped so that you can do something to it before 926 * it starts, you will need to open-code your own. 927 */ 928 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, 929 char *f, struct task_struct **tp) 930 { 931 int ret = 0; 932 933 VERBOSE_TOROUT_STRING(m); 934 *tp = kthread_create(fn, arg, "%s", s); 935 if (IS_ERR(*tp)) { 936 ret = PTR_ERR(*tp); 937 TOROUT_ERRSTRING(f); 938 *tp = NULL; 939 return ret; 940 } 941 wake_up_process(*tp); // Process is sleeping, so ordering provided. 942 torture_shuffle_task_register(*tp); 943 return ret; 944 } 945 EXPORT_SYMBOL_GPL(_torture_create_kthread); 946 947 /* 948 * Stop a generic kthread, emitting a message. 949 */ 950 void _torture_stop_kthread(char *m, struct task_struct **tp) 951 { 952 if (*tp == NULL) 953 return; 954 VERBOSE_TOROUT_STRING(m); 955 kthread_stop(*tp); 956 *tp = NULL; 957 } 958 EXPORT_SYMBOL_GPL(_torture_stop_kthread); 959