1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Common functions for in-kernel torture tests. 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Author: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Based on kernel/rcu/torture.c. 9 */ 10 11 #define pr_fmt(fmt) fmt 12 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/interrupt.h> 22 #include <linux/sched.h> 23 #include <linux/sched/clock.h> 24 #include <linux/atomic.h> 25 #include <linux/bitops.h> 26 #include <linux/completion.h> 27 #include <linux/moduleparam.h> 28 #include <linux/percpu.h> 29 #include <linux/notifier.h> 30 #include <linux/reboot.h> 31 #include <linux/freezer.h> 32 #include <linux/cpu.h> 33 #include <linux/delay.h> 34 #include <linux/stat.h> 35 #include <linux/slab.h> 36 #include <linux/trace_clock.h> 37 #include <linux/ktime.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include <linux/sched/rt.h> 41 #include "rcu/rcu.h" 42 43 MODULE_LICENSE("GPL"); 44 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 45 46 static bool disable_onoff_at_boot; 47 module_param(disable_onoff_at_boot, bool, 0444); 48 49 static bool ftrace_dump_at_shutdown; 50 module_param(ftrace_dump_at_shutdown, bool, 0444); 51 52 static int verbose_sleep_frequency; 53 module_param(verbose_sleep_frequency, int, 0444); 54 55 static int verbose_sleep_duration = 1; 56 module_param(verbose_sleep_duration, int, 0444); 57 58 static int random_shuffle; 59 module_param(random_shuffle, int, 0444); 60 61 static char *torture_type; 62 static int verbose; 63 64 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 65 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 66 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */ 67 #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */ 68 static int fullstop = FULLSTOP_RMMOD; 69 static DEFINE_MUTEX(fullstop_mutex); 70 71 static atomic_t verbose_sleep_counter; 72 73 /* 74 * Sleep if needed from VERBOSE_TOROUT*(). 75 */ 76 void verbose_torout_sleep(void) 77 { 78 if (verbose_sleep_frequency > 0 && 79 verbose_sleep_duration > 0 && 80 !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency)) 81 schedule_timeout_uninterruptible(verbose_sleep_duration); 82 } 83 EXPORT_SYMBOL_GPL(verbose_torout_sleep); 84 85 /* 86 * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit 87 * nanosecond random fuzz. This function and its friends desynchronize 88 * testing from the timer wheel. 89 */ 90 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp) 91 { 92 ktime_t hto = baset_ns; 93 94 if (trsp) 95 hto += torture_random(trsp) % fuzzt_ns; 96 set_current_state(TASK_IDLE); 97 return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); 98 } 99 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); 100 101 /* 102 * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit 103 * nanosecond (not microsecond!) random fuzz. 104 */ 105 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp) 106 { 107 ktime_t baset_ns = baset_us * NSEC_PER_USEC; 108 109 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 110 } 111 EXPORT_SYMBOL_GPL(torture_hrtimeout_us); 112 113 /* 114 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 115 * microsecond (not millisecond!) random fuzz. 116 */ 117 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp) 118 { 119 ktime_t baset_ns = baset_ms * NSEC_PER_MSEC; 120 u32 fuzzt_ns; 121 122 if ((u32)~0U / NSEC_PER_USEC < fuzzt_us) 123 fuzzt_ns = (u32)~0U; 124 else 125 fuzzt_ns = fuzzt_us * NSEC_PER_USEC; 126 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 127 } 128 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms); 129 130 /* 131 * Schedule a high-resolution-timer sleep in jiffies, with an 132 * implied one-jiffy random fuzz. This is intended to replace calls to 133 * schedule_timeout_interruptible() and friends. 134 */ 135 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp) 136 { 137 ktime_t baset_ns = jiffies_to_nsecs(baset_j); 138 139 return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp); 140 } 141 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies); 142 143 /* 144 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 145 * millisecond (not second!) random fuzz. 146 */ 147 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp) 148 { 149 ktime_t baset_ns = baset_s * NSEC_PER_SEC; 150 u32 fuzzt_ns; 151 152 if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms) 153 fuzzt_ns = (u32)~0U; 154 else 155 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC; 156 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); 157 } 158 EXPORT_SYMBOL_GPL(torture_hrtimeout_s); 159 160 #ifdef CONFIG_HOTPLUG_CPU 161 162 /* 163 * Variables for online-offline handling. Only present if CPU hotplug 164 * is enabled, otherwise does nothing. 165 */ 166 167 static struct task_struct *onoff_task; 168 static long onoff_holdoff; 169 static long onoff_interval; 170 static torture_ofl_func *onoff_f; 171 static long n_offline_attempts; 172 static long n_offline_successes; 173 static unsigned long sum_offline; 174 static int min_offline = -1; 175 static int max_offline; 176 static long n_online_attempts; 177 static long n_online_successes; 178 static unsigned long sum_online; 179 static int min_online = -1; 180 static int max_online; 181 182 static int torture_online_cpus = NR_CPUS; 183 184 /* 185 * Some torture testing leverages confusion as to the number of online 186 * CPUs. This function returns the torture-testing view of this number, 187 * which allows torture tests to load-balance appropriately. 188 */ 189 int torture_num_online_cpus(void) 190 { 191 return READ_ONCE(torture_online_cpus); 192 } 193 EXPORT_SYMBOL_GPL(torture_num_online_cpus); 194 195 /* 196 * Attempt to take a CPU offline. Return false if the CPU is already 197 * offline or if it is not subject to CPU-hotplug operations. The 198 * caller can detect other failures by looking at the statistics. 199 */ 200 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, 201 unsigned long *sum_offl, int *min_offl, int *max_offl) 202 { 203 unsigned long delta; 204 int ret; 205 char *s; 206 unsigned long starttime; 207 208 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 209 return false; 210 if (num_online_cpus() <= 1) 211 return false; /* Can't offline the last CPU. */ 212 213 if (verbose > 1) 214 pr_alert("%s" TORTURE_FLAG 215 "torture_onoff task: offlining %d\n", 216 torture_type, cpu); 217 starttime = jiffies; 218 (*n_offl_attempts)++; 219 ret = remove_cpu(cpu); 220 if (ret) { 221 s = ""; 222 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 223 // PCI probe frequently disables hotplug during boot. 224 (*n_offl_attempts)--; 225 s = " (-EBUSY forgiven during boot)"; 226 } 227 if (verbose) 228 pr_alert("%s" TORTURE_FLAG 229 "torture_onoff task: offline %d failed%s: errno %d\n", 230 torture_type, cpu, s, ret); 231 } else { 232 if (verbose > 1) 233 pr_alert("%s" TORTURE_FLAG 234 "torture_onoff task: offlined %d\n", 235 torture_type, cpu); 236 if (onoff_f) 237 onoff_f(); 238 (*n_offl_successes)++; 239 delta = jiffies - starttime; 240 *sum_offl += delta; 241 if (*min_offl < 0) { 242 *min_offl = delta; 243 *max_offl = delta; 244 } 245 if (*min_offl > delta) 246 *min_offl = delta; 247 if (*max_offl < delta) 248 *max_offl = delta; 249 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1); 250 WARN_ON_ONCE(torture_online_cpus <= 0); 251 } 252 253 return true; 254 } 255 EXPORT_SYMBOL_GPL(torture_offline); 256 257 /* 258 * Attempt to bring a CPU online. Return false if the CPU is already 259 * online or if it is not subject to CPU-hotplug operations. The 260 * caller can detect other failures by looking at the statistics. 261 */ 262 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, 263 unsigned long *sum_onl, int *min_onl, int *max_onl) 264 { 265 unsigned long delta; 266 int ret; 267 char *s; 268 unsigned long starttime; 269 270 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 271 return false; 272 273 if (verbose > 1) 274 pr_alert("%s" TORTURE_FLAG 275 "torture_onoff task: onlining %d\n", 276 torture_type, cpu); 277 starttime = jiffies; 278 (*n_onl_attempts)++; 279 ret = add_cpu(cpu); 280 if (ret) { 281 s = ""; 282 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 283 // PCI probe frequently disables hotplug during boot. 284 (*n_onl_attempts)--; 285 s = " (-EBUSY forgiven during boot)"; 286 } 287 if (verbose) 288 pr_alert("%s" TORTURE_FLAG 289 "torture_onoff task: online %d failed%s: errno %d\n", 290 torture_type, cpu, s, ret); 291 } else { 292 if (verbose > 1) 293 pr_alert("%s" TORTURE_FLAG 294 "torture_onoff task: onlined %d\n", 295 torture_type, cpu); 296 (*n_onl_successes)++; 297 delta = jiffies - starttime; 298 *sum_onl += delta; 299 if (*min_onl < 0) { 300 *min_onl = delta; 301 *max_onl = delta; 302 } 303 if (*min_onl > delta) 304 *min_onl = delta; 305 if (*max_onl < delta) 306 *max_onl = delta; 307 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1); 308 } 309 310 return true; 311 } 312 EXPORT_SYMBOL_GPL(torture_online); 313 314 /* 315 * Get everything online at the beginning and ends of tests. 316 */ 317 static void torture_online_all(char *phase) 318 { 319 int cpu; 320 int ret; 321 322 for_each_possible_cpu(cpu) { 323 if (cpu_online(cpu)) 324 continue; 325 ret = add_cpu(cpu); 326 if (ret && verbose) { 327 pr_alert("%s" TORTURE_FLAG 328 "%s: %s online %d: errno %d\n", 329 __func__, phase, torture_type, cpu, ret); 330 } 331 } 332 } 333 334 /* 335 * Execute random CPU-hotplug operations at the interval specified 336 * by the onoff_interval. 337 */ 338 static int 339 torture_onoff(void *arg) 340 { 341 int cpu; 342 int maxcpu = -1; 343 DEFINE_TORTURE_RANDOM(rand); 344 345 VERBOSE_TOROUT_STRING("torture_onoff task started"); 346 for_each_online_cpu(cpu) 347 maxcpu = cpu; 348 WARN_ON(maxcpu < 0); 349 torture_online_all("Initial"); 350 if (maxcpu == 0) { 351 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled"); 352 goto stop; 353 } 354 355 if (onoff_holdoff > 0) { 356 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); 357 torture_hrtimeout_jiffies(onoff_holdoff, &rand); 358 VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); 359 } 360 while (!torture_must_stop()) { 361 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { 362 torture_hrtimeout_jiffies(HZ / 10, &rand); 363 continue; 364 } 365 cpu = torture_random(&rand) % (maxcpu + 1); 366 if (!torture_offline(cpu, 367 &n_offline_attempts, &n_offline_successes, 368 &sum_offline, &min_offline, &max_offline)) 369 torture_online(cpu, 370 &n_online_attempts, &n_online_successes, 371 &sum_online, &min_online, &max_online); 372 torture_hrtimeout_jiffies(onoff_interval, &rand); 373 } 374 375 stop: 376 torture_kthread_stopping("torture_onoff"); 377 torture_online_all("Final"); 378 return 0; 379 } 380 381 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 382 383 /* 384 * Initiate online-offline handling. 385 */ 386 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f) 387 { 388 #ifdef CONFIG_HOTPLUG_CPU 389 onoff_holdoff = ooholdoff; 390 onoff_interval = oointerval; 391 onoff_f = f; 392 if (onoff_interval <= 0) 393 return 0; 394 return torture_create_kthread(torture_onoff, NULL, onoff_task); 395 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 396 return 0; 397 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 398 } 399 EXPORT_SYMBOL_GPL(torture_onoff_init); 400 401 /* 402 * Clean up after online/offline testing. 403 */ 404 static void torture_onoff_cleanup(void) 405 { 406 #ifdef CONFIG_HOTPLUG_CPU 407 if (onoff_task == NULL) 408 return; 409 VERBOSE_TOROUT_STRING("Stopping torture_onoff task"); 410 kthread_stop(onoff_task); 411 onoff_task = NULL; 412 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 413 } 414 415 /* 416 * Print online/offline testing statistics. 417 */ 418 void torture_onoff_stats(void) 419 { 420 #ifdef CONFIG_HOTPLUG_CPU 421 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", 422 n_online_successes, n_online_attempts, 423 n_offline_successes, n_offline_attempts, 424 min_online, max_online, 425 min_offline, max_offline, 426 sum_online, sum_offline, HZ); 427 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 428 } 429 EXPORT_SYMBOL_GPL(torture_onoff_stats); 430 431 /* 432 * Were all the online/offline operations successful? 433 */ 434 bool torture_onoff_failures(void) 435 { 436 #ifdef CONFIG_HOTPLUG_CPU 437 return n_online_successes != n_online_attempts || 438 n_offline_successes != n_offline_attempts; 439 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 440 return false; 441 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 442 } 443 EXPORT_SYMBOL_GPL(torture_onoff_failures); 444 445 #define TORTURE_RANDOM_MULT 39916801 /* prime */ 446 #define TORTURE_RANDOM_ADD 479001701 /* prime */ 447 #define TORTURE_RANDOM_REFRESH 10000 448 449 /* 450 * Crude but fast random-number generator. Uses a linear congruential 451 * generator, with occasional help from cpu_clock(). 452 */ 453 unsigned long 454 torture_random(struct torture_random_state *trsp) 455 { 456 if (--trsp->trs_count < 0) { 457 trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id(); 458 trsp->trs_count = TORTURE_RANDOM_REFRESH; 459 } 460 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + 461 TORTURE_RANDOM_ADD; 462 return swahw32(trsp->trs_state); 463 } 464 EXPORT_SYMBOL_GPL(torture_random); 465 466 /* 467 * Variables for shuffling. The idea is to ensure that each CPU stays 468 * idle for an extended period to test interactions with dyntick idle, 469 * as well as interactions with any per-CPU variables. 470 */ 471 struct shuffle_task { 472 struct list_head st_l; 473 struct task_struct *st_t; 474 }; 475 476 static long shuffle_interval; /* In jiffies. */ 477 static struct task_struct *shuffler_task; 478 static cpumask_var_t shuffle_tmp_mask; 479 static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */ 480 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list); 481 static DEFINE_MUTEX(shuffle_task_mutex); 482 483 /* 484 * Register a task to be shuffled. If there is no memory, just splat 485 * and don't bother registering. 486 */ 487 void torture_shuffle_task_register(struct task_struct *tp) 488 { 489 struct shuffle_task *stp; 490 491 if (WARN_ON_ONCE(tp == NULL)) 492 return; 493 stp = kmalloc(sizeof(*stp), GFP_KERNEL); 494 if (WARN_ON_ONCE(stp == NULL)) 495 return; 496 stp->st_t = tp; 497 mutex_lock(&shuffle_task_mutex); 498 list_add(&stp->st_l, &shuffle_task_list); 499 mutex_unlock(&shuffle_task_mutex); 500 } 501 EXPORT_SYMBOL_GPL(torture_shuffle_task_register); 502 503 /* 504 * Unregister all tasks, for example, at the end of the torture run. 505 */ 506 static void torture_shuffle_task_unregister_all(void) 507 { 508 struct shuffle_task *stp; 509 struct shuffle_task *p; 510 511 mutex_lock(&shuffle_task_mutex); 512 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) { 513 list_del(&stp->st_l); 514 kfree(stp); 515 } 516 mutex_unlock(&shuffle_task_mutex); 517 } 518 519 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle. 520 * A special case is when shuffle_idle_cpu = -1, in which case we allow 521 * the tasks to run on all CPUs. 522 */ 523 static void torture_shuffle_tasks(void) 524 { 525 DEFINE_TORTURE_RANDOM(rand); 526 struct shuffle_task *stp; 527 528 cpumask_setall(shuffle_tmp_mask); 529 cpus_read_lock(); 530 531 /* No point in shuffling if there is only one online CPU (ex: UP) */ 532 if (num_online_cpus() == 1) { 533 cpus_read_unlock(); 534 return; 535 } 536 537 /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */ 538 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask); 539 if (shuffle_idle_cpu >= nr_cpu_ids) 540 shuffle_idle_cpu = -1; 541 else 542 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); 543 544 mutex_lock(&shuffle_task_mutex); 545 list_for_each_entry(stp, &shuffle_task_list, st_l) { 546 if (!random_shuffle || torture_random(&rand) & 0x1) 547 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); 548 } 549 mutex_unlock(&shuffle_task_mutex); 550 551 cpus_read_unlock(); 552 } 553 554 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 555 * system to become idle at a time and cut off its timer ticks. This is meant 556 * to test the support for such tickless idle CPU in RCU. 557 */ 558 static int torture_shuffle(void *arg) 559 { 560 DEFINE_TORTURE_RANDOM(rand); 561 562 VERBOSE_TOROUT_STRING("torture_shuffle task started"); 563 do { 564 torture_hrtimeout_jiffies(shuffle_interval, &rand); 565 torture_shuffle_tasks(); 566 torture_shutdown_absorb("torture_shuffle"); 567 } while (!torture_must_stop()); 568 torture_kthread_stopping("torture_shuffle"); 569 return 0; 570 } 571 572 /* 573 * Start the shuffler, with shuffint in jiffies. 574 */ 575 int torture_shuffle_init(long shuffint) 576 { 577 shuffle_interval = shuffint; 578 579 shuffle_idle_cpu = -1; 580 581 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 582 TOROUT_ERRSTRING("Failed to alloc mask"); 583 return -ENOMEM; 584 } 585 586 /* Create the shuffler thread */ 587 return torture_create_kthread(torture_shuffle, NULL, shuffler_task); 588 } 589 EXPORT_SYMBOL_GPL(torture_shuffle_init); 590 591 /* 592 * Stop the shuffling. 593 */ 594 static void torture_shuffle_cleanup(void) 595 { 596 torture_shuffle_task_unregister_all(); 597 if (shuffler_task) { 598 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task"); 599 kthread_stop(shuffler_task); 600 free_cpumask_var(shuffle_tmp_mask); 601 } 602 shuffler_task = NULL; 603 } 604 605 /* 606 * Variables for auto-shutdown. This allows "lights out" torture runs 607 * to be fully scripted. 608 */ 609 static struct task_struct *shutdown_task; 610 static ktime_t shutdown_time; /* time to system shutdown. */ 611 static void (*torture_shutdown_hook)(void); 612 613 /* 614 * Absorb kthreads into a kernel function that won't return, so that 615 * they won't ever access module text or data again. 616 */ 617 void torture_shutdown_absorb(const char *title) 618 { 619 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 620 pr_notice("torture thread %s parking due to system shutdown\n", 621 title); 622 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 623 } 624 } 625 EXPORT_SYMBOL_GPL(torture_shutdown_absorb); 626 627 /* 628 * Cause the torture test to shutdown the system after the test has 629 * run for the time specified by the shutdown_secs parameter. 630 */ 631 static int torture_shutdown(void *arg) 632 { 633 ktime_t ktime_snap; 634 635 VERBOSE_TOROUT_STRING("torture_shutdown task started"); 636 ktime_snap = ktime_get(); 637 while (ktime_before(ktime_snap, shutdown_time) && 638 !torture_must_stop()) { 639 if (verbose) 640 pr_alert("%s" TORTURE_FLAG 641 "torture_shutdown task: %llu ms remaining\n", 642 torture_type, 643 ktime_ms_delta(shutdown_time, ktime_snap)); 644 set_current_state(TASK_INTERRUPTIBLE); 645 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS); 646 ktime_snap = ktime_get(); 647 } 648 if (torture_must_stop()) { 649 torture_kthread_stopping("torture_shutdown"); 650 return 0; 651 } 652 653 /* OK, shut down the system. */ 654 655 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system"); 656 shutdown_task = NULL; /* Avoid self-kill deadlock. */ 657 if (torture_shutdown_hook) 658 torture_shutdown_hook(); 659 else 660 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping."); 661 if (ftrace_dump_at_shutdown) 662 rcu_ftrace_dump(DUMP_ALL); 663 kernel_power_off(); /* Shut down the system. */ 664 return 0; 665 } 666 667 /* 668 * Start up the shutdown task. 669 */ 670 int torture_shutdown_init(int ssecs, void (*cleanup)(void)) 671 { 672 torture_shutdown_hook = cleanup; 673 if (ssecs > 0) { 674 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0)); 675 return torture_create_kthread(torture_shutdown, NULL, 676 shutdown_task); 677 } 678 return 0; 679 } 680 EXPORT_SYMBOL_GPL(torture_shutdown_init); 681 682 /* 683 * Detect and respond to a system shutdown. 684 */ 685 static int torture_shutdown_notify(struct notifier_block *unused1, 686 unsigned long unused2, void *unused3) 687 { 688 mutex_lock(&fullstop_mutex); 689 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) { 690 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); 691 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN); 692 } else { 693 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 694 } 695 mutex_unlock(&fullstop_mutex); 696 return NOTIFY_DONE; 697 } 698 699 static struct notifier_block torture_shutdown_nb = { 700 .notifier_call = torture_shutdown_notify, 701 }; 702 703 /* 704 * Shut down the shutdown task. Say what??? Heh! This can happen if 705 * the torture module gets an rmmod before the shutdown time arrives. ;-) 706 */ 707 static void torture_shutdown_cleanup(void) 708 { 709 unregister_reboot_notifier(&torture_shutdown_nb); 710 if (shutdown_task != NULL) { 711 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task"); 712 kthread_stop(shutdown_task); 713 } 714 shutdown_task = NULL; 715 } 716 717 /* 718 * Variables for stuttering, which means to periodically pause and 719 * restart testing in order to catch bugs that appear when load is 720 * suddenly applied to or removed from the system. 721 */ 722 static struct task_struct *stutter_task; 723 static int stutter_pause_test; 724 static int stutter; 725 static int stutter_gap; 726 727 /* 728 * Block until the stutter interval ends. This must be called periodically 729 * by all running kthreads that need to be subject to stuttering. 730 */ 731 bool stutter_wait(const char *title) 732 { 733 unsigned int i = 0; 734 bool ret = false; 735 int spt; 736 737 cond_resched_tasks_rcu_qs(); 738 spt = READ_ONCE(stutter_pause_test); 739 for (; spt; spt = READ_ONCE(stutter_pause_test)) { 740 if (!ret && !rt_task(current)) { 741 sched_set_normal(current, MAX_NICE); 742 ret = true; 743 } 744 if (spt == 1) { 745 torture_hrtimeout_jiffies(1, NULL); 746 } else if (spt == 2) { 747 while (READ_ONCE(stutter_pause_test)) { 748 if (!(i++ & 0xffff)) 749 torture_hrtimeout_us(10, 0, NULL); 750 cond_resched(); 751 } 752 } else { 753 torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL); 754 } 755 torture_shutdown_absorb(title); 756 } 757 return ret; 758 } 759 EXPORT_SYMBOL_GPL(stutter_wait); 760 761 /* 762 * Cause the torture test to "stutter", starting and stopping all 763 * threads periodically. 764 */ 765 static int torture_stutter(void *arg) 766 { 767 DEFINE_TORTURE_RANDOM(rand); 768 int wtime; 769 770 VERBOSE_TOROUT_STRING("torture_stutter task started"); 771 do { 772 if (!torture_must_stop() && stutter > 1) { 773 wtime = stutter; 774 if (stutter > 2) { 775 WRITE_ONCE(stutter_pause_test, 1); 776 wtime = stutter - 3; 777 torture_hrtimeout_jiffies(wtime, &rand); 778 wtime = 2; 779 } 780 WRITE_ONCE(stutter_pause_test, 2); 781 torture_hrtimeout_jiffies(wtime, NULL); 782 } 783 WRITE_ONCE(stutter_pause_test, 0); 784 if (!torture_must_stop()) 785 torture_hrtimeout_jiffies(stutter_gap, NULL); 786 torture_shutdown_absorb("torture_stutter"); 787 } while (!torture_must_stop()); 788 torture_kthread_stopping("torture_stutter"); 789 return 0; 790 } 791 792 /* 793 * Initialize and kick off the torture_stutter kthread. 794 */ 795 int torture_stutter_init(const int s, const int sgap) 796 { 797 stutter = s; 798 stutter_gap = sgap; 799 return torture_create_kthread(torture_stutter, NULL, stutter_task); 800 } 801 EXPORT_SYMBOL_GPL(torture_stutter_init); 802 803 /* 804 * Cleanup after the torture_stutter kthread. 805 */ 806 static void torture_stutter_cleanup(void) 807 { 808 if (!stutter_task) 809 return; 810 VERBOSE_TOROUT_STRING("Stopping torture_stutter task"); 811 kthread_stop(stutter_task); 812 stutter_task = NULL; 813 } 814 815 /* 816 * Initialize torture module. Please note that this is -not- invoked via 817 * the usual module_init() mechanism, but rather by an explicit call from 818 * the client torture module. This call must be paired with a later 819 * torture_init_end(). 820 * 821 * The runnable parameter points to a flag that controls whether or not 822 * the test is currently runnable. If there is no such flag, pass in NULL. 823 */ 824 bool torture_init_begin(char *ttype, int v) 825 { 826 mutex_lock(&fullstop_mutex); 827 if (torture_type != NULL) { 828 pr_alert("%s: Refusing %s init: %s running.\n", 829 __func__, ttype, torture_type); 830 pr_alert("%s: One torture test at a time!\n", __func__); 831 mutex_unlock(&fullstop_mutex); 832 return false; 833 } 834 torture_type = ttype; 835 verbose = v; 836 fullstop = FULLSTOP_DONTSTOP; 837 return true; 838 } 839 EXPORT_SYMBOL_GPL(torture_init_begin); 840 841 /* 842 * Tell the torture module that initialization is complete. 843 */ 844 void torture_init_end(void) 845 { 846 mutex_unlock(&fullstop_mutex); 847 register_reboot_notifier(&torture_shutdown_nb); 848 } 849 EXPORT_SYMBOL_GPL(torture_init_end); 850 851 /* 852 * Clean up torture module. Please note that this is -not- invoked via 853 * the usual module_exit() mechanism, but rather by an explicit call from 854 * the client torture module. Returns true if a race with system shutdown 855 * is detected, otherwise, all kthreads started by functions in this file 856 * will be shut down. 857 * 858 * This must be called before the caller starts shutting down its own 859 * kthreads. 860 * 861 * Both torture_cleanup_begin() and torture_cleanup_end() must be paired, 862 * in order to correctly perform the cleanup. They are separated because 863 * threads can still need to reference the torture_type type, thus nullify 864 * only after completing all other relevant calls. 865 */ 866 bool torture_cleanup_begin(void) 867 { 868 mutex_lock(&fullstop_mutex); 869 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 870 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 871 mutex_unlock(&fullstop_mutex); 872 schedule_timeout_uninterruptible(10); 873 return true; 874 } 875 WRITE_ONCE(fullstop, FULLSTOP_RMMOD); 876 mutex_unlock(&fullstop_mutex); 877 torture_shutdown_cleanup(); 878 torture_shuffle_cleanup(); 879 torture_stutter_cleanup(); 880 torture_onoff_cleanup(); 881 return false; 882 } 883 EXPORT_SYMBOL_GPL(torture_cleanup_begin); 884 885 void torture_cleanup_end(void) 886 { 887 mutex_lock(&fullstop_mutex); 888 torture_type = NULL; 889 mutex_unlock(&fullstop_mutex); 890 } 891 EXPORT_SYMBOL_GPL(torture_cleanup_end); 892 893 /* 894 * Is it time for the current torture test to stop? 895 */ 896 bool torture_must_stop(void) 897 { 898 return torture_must_stop_irq() || kthread_should_stop(); 899 } 900 EXPORT_SYMBOL_GPL(torture_must_stop); 901 902 /* 903 * Is it time for the current torture test to stop? This is the irq-safe 904 * version, hence no check for kthread_should_stop(). 905 */ 906 bool torture_must_stop_irq(void) 907 { 908 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP; 909 } 910 EXPORT_SYMBOL_GPL(torture_must_stop_irq); 911 912 /* 913 * Each kthread must wait for kthread_should_stop() before returning from 914 * its top-level function, otherwise segfaults ensue. This function 915 * prints a "stopping" message and waits for kthread_should_stop(), and 916 * should be called from all torture kthreads immediately prior to 917 * returning. 918 */ 919 void torture_kthread_stopping(char *title) 920 { 921 char buf[128]; 922 923 snprintf(buf, sizeof(buf), "%s is stopping", title); 924 VERBOSE_TOROUT_STRING(buf); 925 while (!kthread_should_stop()) { 926 torture_shutdown_absorb(title); 927 schedule_timeout_uninterruptible(HZ / 20); 928 } 929 } 930 EXPORT_SYMBOL_GPL(torture_kthread_stopping); 931 932 /* 933 * Create a generic torture kthread that is immediately runnable. If you 934 * need the kthread to be stopped so that you can do something to it before 935 * it starts, you will need to open-code your own. 936 */ 937 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, 938 char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)) 939 { 940 int ret = 0; 941 942 VERBOSE_TOROUT_STRING(m); 943 *tp = kthread_create(fn, arg, "%s", s); 944 if (IS_ERR(*tp)) { 945 ret = PTR_ERR(*tp); 946 TOROUT_ERRSTRING(f); 947 *tp = NULL; 948 return ret; 949 } 950 951 if (cbf) 952 cbf(*tp); 953 954 wake_up_process(*tp); // Process is sleeping, so ordering provided. 955 torture_shuffle_task_register(*tp); 956 return ret; 957 } 958 EXPORT_SYMBOL_GPL(_torture_create_kthread); 959 960 /* 961 * Stop a generic kthread, emitting a message. 962 */ 963 void _torture_stop_kthread(char *m, struct task_struct **tp) 964 { 965 if (*tp == NULL) 966 return; 967 VERBOSE_TOROUT_STRING(m); 968 kthread_stop(*tp); 969 *tp = NULL; 970 } 971 EXPORT_SYMBOL_GPL(_torture_stop_kthread); 972