xref: /linux/kernel/locking/locktorture.c (revision ca55b2fef3a9373fcfc30f82fd26bc7fccbda732)
1 /*
2  * Module-based torture test facility for locking
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, you can access it online at
16  * http://www.gnu.org/licenses/gpl-2.0.html.
17  *
18  * Copyright (C) IBM Corporation, 2014
19  *
20  * Author: Paul E. McKenney <paulmck@us.ibm.com>
21  *	Based on kernel/rcu/torture.c.
22  */
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/kthread.h>
26 #include <linux/spinlock.h>
27 #include <linux/rwlock.h>
28 #include <linux/mutex.h>
29 #include <linux/rwsem.h>
30 #include <linux/smp.h>
31 #include <linux/interrupt.h>
32 #include <linux/sched.h>
33 #include <linux/atomic.h>
34 #include <linux/moduleparam.h>
35 #include <linux/delay.h>
36 #include <linux/slab.h>
37 #include <linux/torture.h>
38 
39 MODULE_LICENSE("GPL");
40 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
41 
42 torture_param(int, nwriters_stress, -1,
43 	     "Number of write-locking stress-test threads");
44 torture_param(int, nreaders_stress, -1,
45 	     "Number of read-locking stress-test threads");
46 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
47 torture_param(int, onoff_interval, 0,
48 	     "Time between CPU hotplugs (s), 0=disable");
49 torture_param(int, shuffle_interval, 3,
50 	     "Number of jiffies between shuffles, 0=disable");
51 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
52 torture_param(int, stat_interval, 60,
53 	     "Number of seconds between stats printk()s");
54 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
55 torture_param(bool, verbose, true,
56 	     "Enable verbose debugging printk()s");
57 
58 static char *torture_type = "spin_lock";
59 module_param(torture_type, charp, 0444);
60 MODULE_PARM_DESC(torture_type,
61 		 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
62 
63 static struct task_struct *stats_task;
64 static struct task_struct **writer_tasks;
65 static struct task_struct **reader_tasks;
66 
67 static bool lock_is_write_held;
68 static bool lock_is_read_held;
69 
70 struct lock_stress_stats {
71 	long n_lock_fail;
72 	long n_lock_acquired;
73 };
74 
75 #if defined(MODULE)
76 #define LOCKTORTURE_RUNNABLE_INIT 1
77 #else
78 #define LOCKTORTURE_RUNNABLE_INIT 0
79 #endif
80 int torture_runnable = LOCKTORTURE_RUNNABLE_INIT;
81 module_param(torture_runnable, int, 0444);
82 MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
83 
84 /* Forward reference. */
85 static void lock_torture_cleanup(void);
86 
87 /*
88  * Operations vector for selecting different types of tests.
89  */
90 struct lock_torture_ops {
91 	void (*init)(void);
92 	int (*writelock)(void);
93 	void (*write_delay)(struct torture_random_state *trsp);
94 	void (*writeunlock)(void);
95 	int (*readlock)(void);
96 	void (*read_delay)(struct torture_random_state *trsp);
97 	void (*readunlock)(void);
98 	unsigned long flags;
99 	const char *name;
100 };
101 
102 struct lock_torture_cxt {
103 	int nrealwriters_stress;
104 	int nrealreaders_stress;
105 	bool debug_lock;
106 	atomic_t n_lock_torture_errors;
107 	struct lock_torture_ops *cur_ops;
108 	struct lock_stress_stats *lwsa; /* writer statistics */
109 	struct lock_stress_stats *lrsa; /* reader statistics */
110 };
111 static struct lock_torture_cxt cxt = { 0, 0, false,
112 				       ATOMIC_INIT(0),
113 				       NULL, NULL};
114 /*
115  * Definitions for lock torture testing.
116  */
117 
118 static int torture_lock_busted_write_lock(void)
119 {
120 	return 0;  /* BUGGY, do not use in real life!!! */
121 }
122 
123 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
124 {
125 	const unsigned long longdelay_ms = 100;
126 
127 	/* We want a long delay occasionally to force massive contention.  */
128 	if (!(torture_random(trsp) %
129 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
130 		mdelay(longdelay_ms);
131 #ifdef CONFIG_PREEMPT
132 	if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
133 		preempt_schedule();  /* Allow test to be preempted. */
134 #endif
135 }
136 
137 static void torture_lock_busted_write_unlock(void)
138 {
139 	  /* BUGGY, do not use in real life!!! */
140 }
141 
142 static struct lock_torture_ops lock_busted_ops = {
143 	.writelock	= torture_lock_busted_write_lock,
144 	.write_delay	= torture_lock_busted_write_delay,
145 	.writeunlock	= torture_lock_busted_write_unlock,
146 	.readlock       = NULL,
147 	.read_delay     = NULL,
148 	.readunlock     = NULL,
149 	.name		= "lock_busted"
150 };
151 
152 static DEFINE_SPINLOCK(torture_spinlock);
153 
154 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
155 {
156 	spin_lock(&torture_spinlock);
157 	return 0;
158 }
159 
160 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
161 {
162 	const unsigned long shortdelay_us = 2;
163 	const unsigned long longdelay_ms = 100;
164 
165 	/* We want a short delay mostly to emulate likely code, and
166 	 * we want a long delay occasionally to force massive contention.
167 	 */
168 	if (!(torture_random(trsp) %
169 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
170 		mdelay(longdelay_ms);
171 	if (!(torture_random(trsp) %
172 	      (cxt.nrealwriters_stress * 2 * shortdelay_us)))
173 		udelay(shortdelay_us);
174 #ifdef CONFIG_PREEMPT
175 	if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
176 		preempt_schedule();  /* Allow test to be preempted. */
177 #endif
178 }
179 
180 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
181 {
182 	spin_unlock(&torture_spinlock);
183 }
184 
185 static struct lock_torture_ops spin_lock_ops = {
186 	.writelock	= torture_spin_lock_write_lock,
187 	.write_delay	= torture_spin_lock_write_delay,
188 	.writeunlock	= torture_spin_lock_write_unlock,
189 	.readlock       = NULL,
190 	.read_delay     = NULL,
191 	.readunlock     = NULL,
192 	.name		= "spin_lock"
193 };
194 
195 static int torture_spin_lock_write_lock_irq(void)
196 __acquires(torture_spinlock)
197 {
198 	unsigned long flags;
199 
200 	spin_lock_irqsave(&torture_spinlock, flags);
201 	cxt.cur_ops->flags = flags;
202 	return 0;
203 }
204 
205 static void torture_lock_spin_write_unlock_irq(void)
206 __releases(torture_spinlock)
207 {
208 	spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
209 }
210 
211 static struct lock_torture_ops spin_lock_irq_ops = {
212 	.writelock	= torture_spin_lock_write_lock_irq,
213 	.write_delay	= torture_spin_lock_write_delay,
214 	.writeunlock	= torture_lock_spin_write_unlock_irq,
215 	.readlock       = NULL,
216 	.read_delay     = NULL,
217 	.readunlock     = NULL,
218 	.name		= "spin_lock_irq"
219 };
220 
221 static DEFINE_RWLOCK(torture_rwlock);
222 
223 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
224 {
225 	write_lock(&torture_rwlock);
226 	return 0;
227 }
228 
229 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
230 {
231 	const unsigned long shortdelay_us = 2;
232 	const unsigned long longdelay_ms = 100;
233 
234 	/* We want a short delay mostly to emulate likely code, and
235 	 * we want a long delay occasionally to force massive contention.
236 	 */
237 	if (!(torture_random(trsp) %
238 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
239 		mdelay(longdelay_ms);
240 	else
241 		udelay(shortdelay_us);
242 }
243 
244 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
245 {
246 	write_unlock(&torture_rwlock);
247 }
248 
249 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
250 {
251 	read_lock(&torture_rwlock);
252 	return 0;
253 }
254 
255 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
256 {
257 	const unsigned long shortdelay_us = 10;
258 	const unsigned long longdelay_ms = 100;
259 
260 	/* We want a short delay mostly to emulate likely code, and
261 	 * we want a long delay occasionally to force massive contention.
262 	 */
263 	if (!(torture_random(trsp) %
264 	      (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
265 		mdelay(longdelay_ms);
266 	else
267 		udelay(shortdelay_us);
268 }
269 
270 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
271 {
272 	read_unlock(&torture_rwlock);
273 }
274 
275 static struct lock_torture_ops rw_lock_ops = {
276 	.writelock	= torture_rwlock_write_lock,
277 	.write_delay	= torture_rwlock_write_delay,
278 	.writeunlock	= torture_rwlock_write_unlock,
279 	.readlock       = torture_rwlock_read_lock,
280 	.read_delay     = torture_rwlock_read_delay,
281 	.readunlock     = torture_rwlock_read_unlock,
282 	.name		= "rw_lock"
283 };
284 
285 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
286 {
287 	unsigned long flags;
288 
289 	write_lock_irqsave(&torture_rwlock, flags);
290 	cxt.cur_ops->flags = flags;
291 	return 0;
292 }
293 
294 static void torture_rwlock_write_unlock_irq(void)
295 __releases(torture_rwlock)
296 {
297 	write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
298 }
299 
300 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
301 {
302 	unsigned long flags;
303 
304 	read_lock_irqsave(&torture_rwlock, flags);
305 	cxt.cur_ops->flags = flags;
306 	return 0;
307 }
308 
309 static void torture_rwlock_read_unlock_irq(void)
310 __releases(torture_rwlock)
311 {
312 	read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
313 }
314 
315 static struct lock_torture_ops rw_lock_irq_ops = {
316 	.writelock	= torture_rwlock_write_lock_irq,
317 	.write_delay	= torture_rwlock_write_delay,
318 	.writeunlock	= torture_rwlock_write_unlock_irq,
319 	.readlock       = torture_rwlock_read_lock_irq,
320 	.read_delay     = torture_rwlock_read_delay,
321 	.readunlock     = torture_rwlock_read_unlock_irq,
322 	.name		= "rw_lock_irq"
323 };
324 
325 static DEFINE_MUTEX(torture_mutex);
326 
327 static int torture_mutex_lock(void) __acquires(torture_mutex)
328 {
329 	mutex_lock(&torture_mutex);
330 	return 0;
331 }
332 
333 static void torture_mutex_delay(struct torture_random_state *trsp)
334 {
335 	const unsigned long longdelay_ms = 100;
336 
337 	/* We want a long delay occasionally to force massive contention.  */
338 	if (!(torture_random(trsp) %
339 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
340 		mdelay(longdelay_ms * 5);
341 	else
342 		mdelay(longdelay_ms / 5);
343 #ifdef CONFIG_PREEMPT
344 	if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
345 		preempt_schedule();  /* Allow test to be preempted. */
346 #endif
347 }
348 
349 static void torture_mutex_unlock(void) __releases(torture_mutex)
350 {
351 	mutex_unlock(&torture_mutex);
352 }
353 
354 static struct lock_torture_ops mutex_lock_ops = {
355 	.writelock	= torture_mutex_lock,
356 	.write_delay	= torture_mutex_delay,
357 	.writeunlock	= torture_mutex_unlock,
358 	.readlock       = NULL,
359 	.read_delay     = NULL,
360 	.readunlock     = NULL,
361 	.name		= "mutex_lock"
362 };
363 
364 static DECLARE_RWSEM(torture_rwsem);
365 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
366 {
367 	down_write(&torture_rwsem);
368 	return 0;
369 }
370 
371 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
372 {
373 	const unsigned long longdelay_ms = 100;
374 
375 	/* We want a long delay occasionally to force massive contention.  */
376 	if (!(torture_random(trsp) %
377 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
378 		mdelay(longdelay_ms * 10);
379 	else
380 		mdelay(longdelay_ms / 10);
381 #ifdef CONFIG_PREEMPT
382 	if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
383 		preempt_schedule();  /* Allow test to be preempted. */
384 #endif
385 }
386 
387 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
388 {
389 	up_write(&torture_rwsem);
390 }
391 
392 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
393 {
394 	down_read(&torture_rwsem);
395 	return 0;
396 }
397 
398 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
399 {
400 	const unsigned long longdelay_ms = 100;
401 
402 	/* We want a long delay occasionally to force massive contention.  */
403 	if (!(torture_random(trsp) %
404 	      (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
405 		mdelay(longdelay_ms * 2);
406 	else
407 		mdelay(longdelay_ms / 2);
408 #ifdef CONFIG_PREEMPT
409 	if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
410 		preempt_schedule();  /* Allow test to be preempted. */
411 #endif
412 }
413 
414 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
415 {
416 	up_read(&torture_rwsem);
417 }
418 
419 static struct lock_torture_ops rwsem_lock_ops = {
420 	.writelock	= torture_rwsem_down_write,
421 	.write_delay	= torture_rwsem_write_delay,
422 	.writeunlock	= torture_rwsem_up_write,
423 	.readlock       = torture_rwsem_down_read,
424 	.read_delay     = torture_rwsem_read_delay,
425 	.readunlock     = torture_rwsem_up_read,
426 	.name		= "rwsem_lock"
427 };
428 
429 /*
430  * Lock torture writer kthread.  Repeatedly acquires and releases
431  * the lock, checking for duplicate acquisitions.
432  */
433 static int lock_torture_writer(void *arg)
434 {
435 	struct lock_stress_stats *lwsp = arg;
436 	static DEFINE_TORTURE_RANDOM(rand);
437 
438 	VERBOSE_TOROUT_STRING("lock_torture_writer task started");
439 	set_user_nice(current, MAX_NICE);
440 
441 	do {
442 		if ((torture_random(&rand) & 0xfffff) == 0)
443 			schedule_timeout_uninterruptible(1);
444 
445 		cxt.cur_ops->writelock();
446 		if (WARN_ON_ONCE(lock_is_write_held))
447 			lwsp->n_lock_fail++;
448 		lock_is_write_held = 1;
449 		if (WARN_ON_ONCE(lock_is_read_held))
450 			lwsp->n_lock_fail++; /* rare, but... */
451 
452 		lwsp->n_lock_acquired++;
453 		cxt.cur_ops->write_delay(&rand);
454 		lock_is_write_held = 0;
455 		cxt.cur_ops->writeunlock();
456 
457 		stutter_wait("lock_torture_writer");
458 	} while (!torture_must_stop());
459 	torture_kthread_stopping("lock_torture_writer");
460 	return 0;
461 }
462 
463 /*
464  * Lock torture reader kthread.  Repeatedly acquires and releases
465  * the reader lock.
466  */
467 static int lock_torture_reader(void *arg)
468 {
469 	struct lock_stress_stats *lrsp = arg;
470 	static DEFINE_TORTURE_RANDOM(rand);
471 
472 	VERBOSE_TOROUT_STRING("lock_torture_reader task started");
473 	set_user_nice(current, MAX_NICE);
474 
475 	do {
476 		if ((torture_random(&rand) & 0xfffff) == 0)
477 			schedule_timeout_uninterruptible(1);
478 
479 		cxt.cur_ops->readlock();
480 		lock_is_read_held = 1;
481 		if (WARN_ON_ONCE(lock_is_write_held))
482 			lrsp->n_lock_fail++; /* rare, but... */
483 
484 		lrsp->n_lock_acquired++;
485 		cxt.cur_ops->read_delay(&rand);
486 		lock_is_read_held = 0;
487 		cxt.cur_ops->readunlock();
488 
489 		stutter_wait("lock_torture_reader");
490 	} while (!torture_must_stop());
491 	torture_kthread_stopping("lock_torture_reader");
492 	return 0;
493 }
494 
495 /*
496  * Create an lock-torture-statistics message in the specified buffer.
497  */
498 static void __torture_print_stats(char *page,
499 				  struct lock_stress_stats *statp, bool write)
500 {
501 	bool fail = 0;
502 	int i, n_stress;
503 	long max = 0;
504 	long min = statp[0].n_lock_acquired;
505 	long long sum = 0;
506 
507 	n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
508 	for (i = 0; i < n_stress; i++) {
509 		if (statp[i].n_lock_fail)
510 			fail = true;
511 		sum += statp[i].n_lock_acquired;
512 		if (max < statp[i].n_lock_fail)
513 			max = statp[i].n_lock_fail;
514 		if (min > statp[i].n_lock_fail)
515 			min = statp[i].n_lock_fail;
516 	}
517 	page += sprintf(page,
518 			"%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
519 			write ? "Writes" : "Reads ",
520 			sum, max, min, max / 2 > min ? "???" : "",
521 			fail, fail ? "!!!" : "");
522 	if (fail)
523 		atomic_inc(&cxt.n_lock_torture_errors);
524 }
525 
526 /*
527  * Print torture statistics.  Caller must ensure that there is only one
528  * call to this function at a given time!!!  This is normally accomplished
529  * by relying on the module system to only have one copy of the module
530  * loaded, and then by giving the lock_torture_stats kthread full control
531  * (or the init/cleanup functions when lock_torture_stats thread is not
532  * running).
533  */
534 static void lock_torture_stats_print(void)
535 {
536 	int size = cxt.nrealwriters_stress * 200 + 8192;
537 	char *buf;
538 
539 	if (cxt.cur_ops->readlock)
540 		size += cxt.nrealreaders_stress * 200 + 8192;
541 
542 	buf = kmalloc(size, GFP_KERNEL);
543 	if (!buf) {
544 		pr_err("lock_torture_stats_print: Out of memory, need: %d",
545 		       size);
546 		return;
547 	}
548 
549 	__torture_print_stats(buf, cxt.lwsa, true);
550 	pr_alert("%s", buf);
551 	kfree(buf);
552 
553 	if (cxt.cur_ops->readlock) {
554 		buf = kmalloc(size, GFP_KERNEL);
555 		if (!buf) {
556 			pr_err("lock_torture_stats_print: Out of memory, need: %d",
557 			       size);
558 			return;
559 		}
560 
561 		__torture_print_stats(buf, cxt.lrsa, false);
562 		pr_alert("%s", buf);
563 		kfree(buf);
564 	}
565 }
566 
567 /*
568  * Periodically prints torture statistics, if periodic statistics printing
569  * was specified via the stat_interval module parameter.
570  *
571  * No need to worry about fullstop here, since this one doesn't reference
572  * volatile state or register callbacks.
573  */
574 static int lock_torture_stats(void *arg)
575 {
576 	VERBOSE_TOROUT_STRING("lock_torture_stats task started");
577 	do {
578 		schedule_timeout_interruptible(stat_interval * HZ);
579 		lock_torture_stats_print();
580 		torture_shutdown_absorb("lock_torture_stats");
581 	} while (!torture_must_stop());
582 	torture_kthread_stopping("lock_torture_stats");
583 	return 0;
584 }
585 
586 static inline void
587 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
588 				const char *tag)
589 {
590 	pr_alert("%s" TORTURE_FLAG
591 		 "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
592 		 torture_type, tag, cxt.debug_lock ? " [debug]": "",
593 		 cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
594 		 verbose, shuffle_interval, stutter, shutdown_secs,
595 		 onoff_interval, onoff_holdoff);
596 }
597 
598 static void lock_torture_cleanup(void)
599 {
600 	int i;
601 
602 	if (torture_cleanup_begin())
603 		return;
604 
605 	if (writer_tasks) {
606 		for (i = 0; i < cxt.nrealwriters_stress; i++)
607 			torture_stop_kthread(lock_torture_writer,
608 					     writer_tasks[i]);
609 		kfree(writer_tasks);
610 		writer_tasks = NULL;
611 	}
612 
613 	if (reader_tasks) {
614 		for (i = 0; i < cxt.nrealreaders_stress; i++)
615 			torture_stop_kthread(lock_torture_reader,
616 					     reader_tasks[i]);
617 		kfree(reader_tasks);
618 		reader_tasks = NULL;
619 	}
620 
621 	torture_stop_kthread(lock_torture_stats, stats_task);
622 	lock_torture_stats_print();  /* -After- the stats thread is stopped! */
623 
624 	if (atomic_read(&cxt.n_lock_torture_errors))
625 		lock_torture_print_module_parms(cxt.cur_ops,
626 						"End of test: FAILURE");
627 	else if (torture_onoff_failures())
628 		lock_torture_print_module_parms(cxt.cur_ops,
629 						"End of test: LOCK_HOTPLUG");
630 	else
631 		lock_torture_print_module_parms(cxt.cur_ops,
632 						"End of test: SUCCESS");
633 	torture_cleanup_end();
634 }
635 
636 static int __init lock_torture_init(void)
637 {
638 	int i, j;
639 	int firsterr = 0;
640 	static struct lock_torture_ops *torture_ops[] = {
641 		&lock_busted_ops,
642 		&spin_lock_ops, &spin_lock_irq_ops,
643 		&rw_lock_ops, &rw_lock_irq_ops,
644 		&mutex_lock_ops,
645 		&rwsem_lock_ops,
646 	};
647 
648 	if (!torture_init_begin(torture_type, verbose, &torture_runnable))
649 		return -EBUSY;
650 
651 	/* Process args and tell the world that the torturer is on the job. */
652 	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
653 		cxt.cur_ops = torture_ops[i];
654 		if (strcmp(torture_type, cxt.cur_ops->name) == 0)
655 			break;
656 	}
657 	if (i == ARRAY_SIZE(torture_ops)) {
658 		pr_alert("lock-torture: invalid torture type: \"%s\"\n",
659 			 torture_type);
660 		pr_alert("lock-torture types:");
661 		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
662 			pr_alert(" %s", torture_ops[i]->name);
663 		pr_alert("\n");
664 		torture_init_end();
665 		return -EINVAL;
666 	}
667 	if (cxt.cur_ops->init)
668 		cxt.cur_ops->init(); /* no "goto unwind" prior to this point!!! */
669 
670 	if (nwriters_stress >= 0)
671 		cxt.nrealwriters_stress = nwriters_stress;
672 	else
673 		cxt.nrealwriters_stress = 2 * num_online_cpus();
674 
675 #ifdef CONFIG_DEBUG_MUTEXES
676 	if (strncmp(torture_type, "mutex", 5) == 0)
677 		cxt.debug_lock = true;
678 #endif
679 #ifdef CONFIG_DEBUG_SPINLOCK
680 	if ((strncmp(torture_type, "spin", 4) == 0) ||
681 	    (strncmp(torture_type, "rw_lock", 7) == 0))
682 		cxt.debug_lock = true;
683 #endif
684 
685 	/* Initialize the statistics so that each run gets its own numbers. */
686 
687 	lock_is_write_held = 0;
688 	cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
689 	if (cxt.lwsa == NULL) {
690 		VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
691 		firsterr = -ENOMEM;
692 		goto unwind;
693 	}
694 	for (i = 0; i < cxt.nrealwriters_stress; i++) {
695 		cxt.lwsa[i].n_lock_fail = 0;
696 		cxt.lwsa[i].n_lock_acquired = 0;
697 	}
698 
699 	if (cxt.cur_ops->readlock) {
700 		if (nreaders_stress >= 0)
701 			cxt.nrealreaders_stress = nreaders_stress;
702 		else {
703 			/*
704 			 * By default distribute evenly the number of
705 			 * readers and writers. We still run the same number
706 			 * of threads as the writer-only locks default.
707 			 */
708 			if (nwriters_stress < 0) /* user doesn't care */
709 				cxt.nrealwriters_stress = num_online_cpus();
710 			cxt.nrealreaders_stress = cxt.nrealwriters_stress;
711 		}
712 
713 		lock_is_read_held = 0;
714 		cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
715 		if (cxt.lrsa == NULL) {
716 			VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
717 			firsterr = -ENOMEM;
718 			kfree(cxt.lwsa);
719 			goto unwind;
720 		}
721 
722 		for (i = 0; i < cxt.nrealreaders_stress; i++) {
723 			cxt.lrsa[i].n_lock_fail = 0;
724 			cxt.lrsa[i].n_lock_acquired = 0;
725 		}
726 	}
727 	lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
728 
729 	/* Prepare torture context. */
730 	if (onoff_interval > 0) {
731 		firsterr = torture_onoff_init(onoff_holdoff * HZ,
732 					      onoff_interval * HZ);
733 		if (firsterr)
734 			goto unwind;
735 	}
736 	if (shuffle_interval > 0) {
737 		firsterr = torture_shuffle_init(shuffle_interval);
738 		if (firsterr)
739 			goto unwind;
740 	}
741 	if (shutdown_secs > 0) {
742 		firsterr = torture_shutdown_init(shutdown_secs,
743 						 lock_torture_cleanup);
744 		if (firsterr)
745 			goto unwind;
746 	}
747 	if (stutter > 0) {
748 		firsterr = torture_stutter_init(stutter);
749 		if (firsterr)
750 			goto unwind;
751 	}
752 
753 	writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
754 			       GFP_KERNEL);
755 	if (writer_tasks == NULL) {
756 		VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
757 		firsterr = -ENOMEM;
758 		goto unwind;
759 	}
760 
761 	if (cxt.cur_ops->readlock) {
762 		reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
763 				       GFP_KERNEL);
764 		if (reader_tasks == NULL) {
765 			VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
766 			firsterr = -ENOMEM;
767 			goto unwind;
768 		}
769 	}
770 
771 	/*
772 	 * Create the kthreads and start torturing (oh, those poor little locks).
773 	 *
774 	 * TODO: Note that we interleave writers with readers, giving writers a
775 	 * slight advantage, by creating its kthread first. This can be modified
776 	 * for very specific needs, or even let the user choose the policy, if
777 	 * ever wanted.
778 	 */
779 	for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
780 		    j < cxt.nrealreaders_stress; i++, j++) {
781 		if (i >= cxt.nrealwriters_stress)
782 			goto create_reader;
783 
784 		/* Create writer. */
785 		firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
786 						  writer_tasks[i]);
787 		if (firsterr)
788 			goto unwind;
789 
790 	create_reader:
791 		if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
792 			continue;
793 		/* Create reader. */
794 		firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
795 						  reader_tasks[j]);
796 		if (firsterr)
797 			goto unwind;
798 	}
799 	if (stat_interval > 0) {
800 		firsterr = torture_create_kthread(lock_torture_stats, NULL,
801 						  stats_task);
802 		if (firsterr)
803 			goto unwind;
804 	}
805 	torture_init_end();
806 	return 0;
807 
808 unwind:
809 	torture_init_end();
810 	lock_torture_cleanup();
811 	return firsterr;
812 }
813 
814 module_init(lock_torture_init);
815 module_exit(lock_torture_cleanup);
816