xref: /linux/drivers/s390/block/dasd.c (revision 2d7f3d1a5866705be2393150e1ffdf67030ab88d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Author(s)......: Holger Smolinski <Holger.Smolinski@de.ibm.com>
4  *		    Horst Hummel <Horst.Hummel@de.ibm.com>
5  *		    Carsten Otte <Cotte@de.ibm.com>
6  *		    Martin Schwidefsky <schwidefsky@de.ibm.com>
7  * Bugreports.to..: <Linux390@de.ibm.com>
8  * Copyright IBM Corp. 1999, 2009
9  */
10 
11 #define KMSG_COMPONENT "dasd"
12 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13 
14 #include <linux/kmod.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
17 #include <linux/ctype.h>
18 #include <linux/major.h>
19 #include <linux/slab.h>
20 #include <linux/hdreg.h>
21 #include <linux/async.h>
22 #include <linux/mutex.h>
23 #include <linux/debugfs.h>
24 #include <linux/seq_file.h>
25 #include <linux/vmalloc.h>
26 
27 #include <asm/ccwdev.h>
28 #include <asm/ebcdic.h>
29 #include <asm/idals.h>
30 #include <asm/itcw.h>
31 #include <asm/diag.h>
32 
33 /* This is ugly... */
34 #define PRINTK_HEADER "dasd:"
35 
36 #include "dasd_int.h"
37 /*
38  * SECTION: Constant definitions to be used within this file
39  */
40 #define DASD_CHANQ_MAX_SIZE 4
41 
42 #define DASD_DIAG_MOD		"dasd_diag_mod"
43 
44 /*
45  * SECTION: exported variables of dasd.c
46  */
47 debug_info_t *dasd_debug_area;
48 EXPORT_SYMBOL(dasd_debug_area);
49 static struct dentry *dasd_debugfs_root_entry;
50 struct dasd_discipline *dasd_diag_discipline_pointer;
51 EXPORT_SYMBOL(dasd_diag_discipline_pointer);
52 void dasd_int_handler(struct ccw_device *, unsigned long, struct irb *);
53 
54 MODULE_AUTHOR("Holger Smolinski <Holger.Smolinski@de.ibm.com>");
55 MODULE_DESCRIPTION("Linux on S/390 DASD device driver,"
56 		   " Copyright IBM Corp. 2000");
57 MODULE_LICENSE("GPL");
58 
59 /*
60  * SECTION: prototypes for static functions of dasd.c
61  */
62 static int dasd_flush_block_queue(struct dasd_block *);
63 static void dasd_device_tasklet(unsigned long);
64 static void dasd_block_tasklet(unsigned long);
65 static void do_kick_device(struct work_struct *);
66 static void do_reload_device(struct work_struct *);
67 static void do_requeue_requests(struct work_struct *);
68 static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *);
69 static void dasd_device_timeout(struct timer_list *);
70 static void dasd_block_timeout(struct timer_list *);
71 static void __dasd_process_erp(struct dasd_device *, struct dasd_ccw_req *);
72 static void dasd_profile_init(struct dasd_profile *, struct dentry *);
73 static void dasd_profile_exit(struct dasd_profile *);
74 static void dasd_hosts_init(struct dentry *, struct dasd_device *);
75 static void dasd_hosts_exit(struct dasd_device *);
76 static int dasd_handle_autoquiesce(struct dasd_device *, struct dasd_ccw_req *,
77 				   unsigned int);
78 /*
79  * SECTION: Operations on the device structure.
80  */
81 static wait_queue_head_t dasd_init_waitq;
82 static wait_queue_head_t dasd_flush_wq;
83 static wait_queue_head_t generic_waitq;
84 static wait_queue_head_t shutdown_waitq;
85 
86 /*
87  * Allocate memory for a new device structure.
88  */
89 struct dasd_device *dasd_alloc_device(void)
90 {
91 	struct dasd_device *device;
92 
93 	device = kzalloc(sizeof(struct dasd_device), GFP_ATOMIC);
94 	if (!device)
95 		return ERR_PTR(-ENOMEM);
96 
97 	/* Get two pages for normal block device operations. */
98 	device->ccw_mem = (void *) __get_free_pages(GFP_ATOMIC | GFP_DMA, 1);
99 	if (!device->ccw_mem) {
100 		kfree(device);
101 		return ERR_PTR(-ENOMEM);
102 	}
103 	/* Get one page for error recovery. */
104 	device->erp_mem = (void *) get_zeroed_page(GFP_ATOMIC | GFP_DMA);
105 	if (!device->erp_mem) {
106 		free_pages((unsigned long) device->ccw_mem, 1);
107 		kfree(device);
108 		return ERR_PTR(-ENOMEM);
109 	}
110 	/* Get two pages for ese format. */
111 	device->ese_mem = (void *)__get_free_pages(GFP_ATOMIC | GFP_DMA, 1);
112 	if (!device->ese_mem) {
113 		free_page((unsigned long) device->erp_mem);
114 		free_pages((unsigned long) device->ccw_mem, 1);
115 		kfree(device);
116 		return ERR_PTR(-ENOMEM);
117 	}
118 
119 	dasd_init_chunklist(&device->ccw_chunks, device->ccw_mem, PAGE_SIZE*2);
120 	dasd_init_chunklist(&device->erp_chunks, device->erp_mem, PAGE_SIZE);
121 	dasd_init_chunklist(&device->ese_chunks, device->ese_mem, PAGE_SIZE * 2);
122 	spin_lock_init(&device->mem_lock);
123 	atomic_set(&device->tasklet_scheduled, 0);
124 	tasklet_init(&device->tasklet, dasd_device_tasklet,
125 		     (unsigned long) device);
126 	INIT_LIST_HEAD(&device->ccw_queue);
127 	timer_setup(&device->timer, dasd_device_timeout, 0);
128 	INIT_WORK(&device->kick_work, do_kick_device);
129 	INIT_WORK(&device->reload_device, do_reload_device);
130 	INIT_WORK(&device->requeue_requests, do_requeue_requests);
131 	device->state = DASD_STATE_NEW;
132 	device->target = DASD_STATE_NEW;
133 	mutex_init(&device->state_mutex);
134 	spin_lock_init(&device->profile.lock);
135 	return device;
136 }
137 
138 /*
139  * Free memory of a device structure.
140  */
141 void dasd_free_device(struct dasd_device *device)
142 {
143 	kfree(device->private);
144 	free_pages((unsigned long) device->ese_mem, 1);
145 	free_page((unsigned long) device->erp_mem);
146 	free_pages((unsigned long) device->ccw_mem, 1);
147 	kfree(device);
148 }
149 
150 /*
151  * Allocate memory for a new device structure.
152  */
153 struct dasd_block *dasd_alloc_block(void)
154 {
155 	struct dasd_block *block;
156 
157 	block = kzalloc(sizeof(*block), GFP_ATOMIC);
158 	if (!block)
159 		return ERR_PTR(-ENOMEM);
160 	/* open_count = 0 means device online but not in use */
161 	atomic_set(&block->open_count, -1);
162 
163 	atomic_set(&block->tasklet_scheduled, 0);
164 	tasklet_init(&block->tasklet, dasd_block_tasklet,
165 		     (unsigned long) block);
166 	INIT_LIST_HEAD(&block->ccw_queue);
167 	spin_lock_init(&block->queue_lock);
168 	INIT_LIST_HEAD(&block->format_list);
169 	spin_lock_init(&block->format_lock);
170 	timer_setup(&block->timer, dasd_block_timeout, 0);
171 	spin_lock_init(&block->profile.lock);
172 
173 	return block;
174 }
175 EXPORT_SYMBOL_GPL(dasd_alloc_block);
176 
177 /*
178  * Free memory of a device structure.
179  */
180 void dasd_free_block(struct dasd_block *block)
181 {
182 	kfree(block);
183 }
184 EXPORT_SYMBOL_GPL(dasd_free_block);
185 
186 /*
187  * Make a new device known to the system.
188  */
189 static int dasd_state_new_to_known(struct dasd_device *device)
190 {
191 	/*
192 	 * As long as the device is not in state DASD_STATE_NEW we want to
193 	 * keep the reference count > 0.
194 	 */
195 	dasd_get_device(device);
196 	device->state = DASD_STATE_KNOWN;
197 	return 0;
198 }
199 
200 /*
201  * Let the system forget about a device.
202  */
203 static int dasd_state_known_to_new(struct dasd_device *device)
204 {
205 	/* Disable extended error reporting for this device. */
206 	dasd_eer_disable(device);
207 	device->state = DASD_STATE_NEW;
208 
209 	/* Give up reference we took in dasd_state_new_to_known. */
210 	dasd_put_device(device);
211 	return 0;
212 }
213 
214 static struct dentry *dasd_debugfs_setup(const char *name,
215 					 struct dentry *base_dentry)
216 {
217 	struct dentry *pde;
218 
219 	if (!base_dentry)
220 		return NULL;
221 	pde = debugfs_create_dir(name, base_dentry);
222 	if (!pde || IS_ERR(pde))
223 		return NULL;
224 	return pde;
225 }
226 
227 /*
228  * Request the irq line for the device.
229  */
230 static int dasd_state_known_to_basic(struct dasd_device *device)
231 {
232 	struct dasd_block *block = device->block;
233 	int rc = 0;
234 
235 	/* Allocate and register gendisk structure. */
236 	if (block) {
237 		rc = dasd_gendisk_alloc(block);
238 		if (rc)
239 			return rc;
240 		block->debugfs_dentry =
241 			dasd_debugfs_setup(block->gdp->disk_name,
242 					   dasd_debugfs_root_entry);
243 		dasd_profile_init(&block->profile, block->debugfs_dentry);
244 		if (dasd_global_profile_level == DASD_PROFILE_ON)
245 			dasd_profile_on(&device->block->profile);
246 	}
247 	device->debugfs_dentry =
248 		dasd_debugfs_setup(dev_name(&device->cdev->dev),
249 				   dasd_debugfs_root_entry);
250 	dasd_profile_init(&device->profile, device->debugfs_dentry);
251 	dasd_hosts_init(device->debugfs_dentry, device);
252 
253 	/* register 'device' debug area, used for all DBF_DEV_XXX calls */
254 	device->debug_area = debug_register(dev_name(&device->cdev->dev), 4, 1,
255 					    8 * sizeof(long));
256 	debug_register_view(device->debug_area, &debug_sprintf_view);
257 	debug_set_level(device->debug_area, DBF_WARNING);
258 	DBF_DEV_EVENT(DBF_EMERG, device, "%s", "debug area created");
259 
260 	device->state = DASD_STATE_BASIC;
261 
262 	return rc;
263 }
264 
265 /*
266  * Release the irq line for the device. Terminate any running i/o.
267  */
268 static int dasd_state_basic_to_known(struct dasd_device *device)
269 {
270 	int rc;
271 
272 	if (device->discipline->basic_to_known) {
273 		rc = device->discipline->basic_to_known(device);
274 		if (rc)
275 			return rc;
276 	}
277 
278 	if (device->block) {
279 		dasd_profile_exit(&device->block->profile);
280 		debugfs_remove(device->block->debugfs_dentry);
281 		dasd_gendisk_free(device->block);
282 		dasd_block_clear_timer(device->block);
283 	}
284 	rc = dasd_flush_device_queue(device);
285 	if (rc)
286 		return rc;
287 	dasd_device_clear_timer(device);
288 	dasd_profile_exit(&device->profile);
289 	dasd_hosts_exit(device);
290 	debugfs_remove(device->debugfs_dentry);
291 	DBF_DEV_EVENT(DBF_EMERG, device, "%p debug area deleted", device);
292 	if (device->debug_area != NULL) {
293 		debug_unregister(device->debug_area);
294 		device->debug_area = NULL;
295 	}
296 	device->state = DASD_STATE_KNOWN;
297 	return 0;
298 }
299 
300 /*
301  * Do the initial analysis. The do_analysis function may return
302  * -EAGAIN in which case the device keeps the state DASD_STATE_BASIC
303  * until the discipline decides to continue the startup sequence
304  * by calling the function dasd_change_state. The eckd disciplines
305  * uses this to start a ccw that detects the format. The completion
306  * interrupt for this detection ccw uses the kernel event daemon to
307  * trigger the call to dasd_change_state. All this is done in the
308  * discipline code, see dasd_eckd.c.
309  * After the analysis ccw is done (do_analysis returned 0) the block
310  * device is setup.
311  * In case the analysis returns an error, the device setup is stopped
312  * (a fake disk was already added to allow formatting).
313  */
314 static int dasd_state_basic_to_ready(struct dasd_device *device)
315 {
316 	int rc;
317 	struct dasd_block *block;
318 	struct gendisk *disk;
319 
320 	rc = 0;
321 	block = device->block;
322 	/* make disk known with correct capacity */
323 	if (block) {
324 		if (block->base->discipline->do_analysis != NULL)
325 			rc = block->base->discipline->do_analysis(block);
326 		if (rc) {
327 			if (rc != -EAGAIN) {
328 				device->state = DASD_STATE_UNFMT;
329 				disk = device->block->gdp;
330 				kobject_uevent(&disk_to_dev(disk)->kobj,
331 					       KOBJ_CHANGE);
332 				goto out;
333 			}
334 			return rc;
335 		}
336 		if (device->discipline->setup_blk_queue)
337 			device->discipline->setup_blk_queue(block);
338 		set_capacity(block->gdp,
339 			     block->blocks << block->s2b_shift);
340 		device->state = DASD_STATE_READY;
341 		rc = dasd_scan_partitions(block);
342 		if (rc) {
343 			device->state = DASD_STATE_BASIC;
344 			return rc;
345 		}
346 	} else {
347 		device->state = DASD_STATE_READY;
348 	}
349 out:
350 	if (device->discipline->basic_to_ready)
351 		rc = device->discipline->basic_to_ready(device);
352 	return rc;
353 }
354 
355 static inline
356 int _wait_for_empty_queues(struct dasd_device *device)
357 {
358 	if (device->block)
359 		return list_empty(&device->ccw_queue) &&
360 			list_empty(&device->block->ccw_queue);
361 	else
362 		return list_empty(&device->ccw_queue);
363 }
364 
365 /*
366  * Remove device from block device layer. Destroy dirty buffers.
367  * Forget format information. Check if the target level is basic
368  * and if it is create fake disk for formatting.
369  */
370 static int dasd_state_ready_to_basic(struct dasd_device *device)
371 {
372 	int rc;
373 
374 	device->state = DASD_STATE_BASIC;
375 	if (device->block) {
376 		struct dasd_block *block = device->block;
377 		rc = dasd_flush_block_queue(block);
378 		if (rc) {
379 			device->state = DASD_STATE_READY;
380 			return rc;
381 		}
382 		dasd_destroy_partitions(block);
383 		block->blocks = 0;
384 		block->bp_block = 0;
385 		block->s2b_shift = 0;
386 	}
387 	return 0;
388 }
389 
390 /*
391  * Back to basic.
392  */
393 static int dasd_state_unfmt_to_basic(struct dasd_device *device)
394 {
395 	device->state = DASD_STATE_BASIC;
396 	return 0;
397 }
398 
399 /*
400  * Make the device online and schedule the bottom half to start
401  * the requeueing of requests from the linux request queue to the
402  * ccw queue.
403  */
404 static int
405 dasd_state_ready_to_online(struct dasd_device * device)
406 {
407 	device->state = DASD_STATE_ONLINE;
408 	if (device->block) {
409 		dasd_schedule_block_bh(device->block);
410 		if ((device->features & DASD_FEATURE_USERAW)) {
411 			kobject_uevent(&disk_to_dev(device->block->gdp)->kobj,
412 					KOBJ_CHANGE);
413 			return 0;
414 		}
415 		disk_uevent(device->block->bdev_handle->bdev->bd_disk,
416 			    KOBJ_CHANGE);
417 	}
418 	return 0;
419 }
420 
421 /*
422  * Stop the requeueing of requests again.
423  */
424 static int dasd_state_online_to_ready(struct dasd_device *device)
425 {
426 	int rc;
427 
428 	if (device->discipline->online_to_ready) {
429 		rc = device->discipline->online_to_ready(device);
430 		if (rc)
431 			return rc;
432 	}
433 
434 	device->state = DASD_STATE_READY;
435 	if (device->block && !(device->features & DASD_FEATURE_USERAW))
436 		disk_uevent(device->block->bdev_handle->bdev->bd_disk,
437 			    KOBJ_CHANGE);
438 	return 0;
439 }
440 
441 /*
442  * Device startup state changes.
443  */
444 static int dasd_increase_state(struct dasd_device *device)
445 {
446 	int rc;
447 
448 	rc = 0;
449 	if (device->state == DASD_STATE_NEW &&
450 	    device->target >= DASD_STATE_KNOWN)
451 		rc = dasd_state_new_to_known(device);
452 
453 	if (!rc &&
454 	    device->state == DASD_STATE_KNOWN &&
455 	    device->target >= DASD_STATE_BASIC)
456 		rc = dasd_state_known_to_basic(device);
457 
458 	if (!rc &&
459 	    device->state == DASD_STATE_BASIC &&
460 	    device->target >= DASD_STATE_READY)
461 		rc = dasd_state_basic_to_ready(device);
462 
463 	if (!rc &&
464 	    device->state == DASD_STATE_UNFMT &&
465 	    device->target > DASD_STATE_UNFMT)
466 		rc = -EPERM;
467 
468 	if (!rc &&
469 	    device->state == DASD_STATE_READY &&
470 	    device->target >= DASD_STATE_ONLINE)
471 		rc = dasd_state_ready_to_online(device);
472 
473 	return rc;
474 }
475 
476 /*
477  * Device shutdown state changes.
478  */
479 static int dasd_decrease_state(struct dasd_device *device)
480 {
481 	int rc;
482 
483 	rc = 0;
484 	if (device->state == DASD_STATE_ONLINE &&
485 	    device->target <= DASD_STATE_READY)
486 		rc = dasd_state_online_to_ready(device);
487 
488 	if (!rc &&
489 	    device->state == DASD_STATE_READY &&
490 	    device->target <= DASD_STATE_BASIC)
491 		rc = dasd_state_ready_to_basic(device);
492 
493 	if (!rc &&
494 	    device->state == DASD_STATE_UNFMT &&
495 	    device->target <= DASD_STATE_BASIC)
496 		rc = dasd_state_unfmt_to_basic(device);
497 
498 	if (!rc &&
499 	    device->state == DASD_STATE_BASIC &&
500 	    device->target <= DASD_STATE_KNOWN)
501 		rc = dasd_state_basic_to_known(device);
502 
503 	if (!rc &&
504 	    device->state == DASD_STATE_KNOWN &&
505 	    device->target <= DASD_STATE_NEW)
506 		rc = dasd_state_known_to_new(device);
507 
508 	return rc;
509 }
510 
511 /*
512  * This is the main startup/shutdown routine.
513  */
514 static void dasd_change_state(struct dasd_device *device)
515 {
516 	int rc;
517 
518 	if (device->state == device->target)
519 		/* Already where we want to go today... */
520 		return;
521 	if (device->state < device->target)
522 		rc = dasd_increase_state(device);
523 	else
524 		rc = dasd_decrease_state(device);
525 	if (rc == -EAGAIN)
526 		return;
527 	if (rc)
528 		device->target = device->state;
529 
530 	/* let user-space know that the device status changed */
531 	kobject_uevent(&device->cdev->dev.kobj, KOBJ_CHANGE);
532 
533 	if (device->state == device->target)
534 		wake_up(&dasd_init_waitq);
535 }
536 
537 /*
538  * Kick starter for devices that did not complete the startup/shutdown
539  * procedure or were sleeping because of a pending state.
540  * dasd_kick_device will schedule a call do do_kick_device to the kernel
541  * event daemon.
542  */
543 static void do_kick_device(struct work_struct *work)
544 {
545 	struct dasd_device *device = container_of(work, struct dasd_device, kick_work);
546 	mutex_lock(&device->state_mutex);
547 	dasd_change_state(device);
548 	mutex_unlock(&device->state_mutex);
549 	dasd_schedule_device_bh(device);
550 	dasd_put_device(device);
551 }
552 
553 void dasd_kick_device(struct dasd_device *device)
554 {
555 	dasd_get_device(device);
556 	/* queue call to dasd_kick_device to the kernel event daemon. */
557 	if (!schedule_work(&device->kick_work))
558 		dasd_put_device(device);
559 }
560 EXPORT_SYMBOL(dasd_kick_device);
561 
562 /*
563  * dasd_reload_device will schedule a call do do_reload_device to the kernel
564  * event daemon.
565  */
566 static void do_reload_device(struct work_struct *work)
567 {
568 	struct dasd_device *device = container_of(work, struct dasd_device,
569 						  reload_device);
570 	device->discipline->reload(device);
571 	dasd_put_device(device);
572 }
573 
574 void dasd_reload_device(struct dasd_device *device)
575 {
576 	dasd_get_device(device);
577 	/* queue call to dasd_reload_device to the kernel event daemon. */
578 	if (!schedule_work(&device->reload_device))
579 		dasd_put_device(device);
580 }
581 EXPORT_SYMBOL(dasd_reload_device);
582 
583 /*
584  * Set the target state for a device and starts the state change.
585  */
586 void dasd_set_target_state(struct dasd_device *device, int target)
587 {
588 	dasd_get_device(device);
589 	mutex_lock(&device->state_mutex);
590 	/* If we are in probeonly mode stop at DASD_STATE_READY. */
591 	if (dasd_probeonly && target > DASD_STATE_READY)
592 		target = DASD_STATE_READY;
593 	if (device->target != target) {
594 		if (device->state == target)
595 			wake_up(&dasd_init_waitq);
596 		device->target = target;
597 	}
598 	if (device->state != device->target)
599 		dasd_change_state(device);
600 	mutex_unlock(&device->state_mutex);
601 	dasd_put_device(device);
602 }
603 
604 /*
605  * Enable devices with device numbers in [from..to].
606  */
607 static inline int _wait_for_device(struct dasd_device *device)
608 {
609 	return (device->state == device->target);
610 }
611 
612 void dasd_enable_device(struct dasd_device *device)
613 {
614 	dasd_set_target_state(device, DASD_STATE_ONLINE);
615 	if (device->state <= DASD_STATE_KNOWN)
616 		/* No discipline for device found. */
617 		dasd_set_target_state(device, DASD_STATE_NEW);
618 	/* Now wait for the devices to come up. */
619 	wait_event(dasd_init_waitq, _wait_for_device(device));
620 
621 	dasd_reload_device(device);
622 	if (device->discipline->kick_validate)
623 		device->discipline->kick_validate(device);
624 }
625 EXPORT_SYMBOL(dasd_enable_device);
626 
627 /*
628  * SECTION: device operation (interrupt handler, start i/o, term i/o ...)
629  */
630 
631 unsigned int dasd_global_profile_level = DASD_PROFILE_OFF;
632 
633 #ifdef CONFIG_DASD_PROFILE
634 struct dasd_profile dasd_global_profile = {
635 	.lock = __SPIN_LOCK_UNLOCKED(dasd_global_profile.lock),
636 };
637 static struct dentry *dasd_debugfs_global_entry;
638 
639 /*
640  * Add profiling information for cqr before execution.
641  */
642 static void dasd_profile_start(struct dasd_block *block,
643 			       struct dasd_ccw_req *cqr,
644 			       struct request *req)
645 {
646 	struct list_head *l;
647 	unsigned int counter;
648 	struct dasd_device *device;
649 
650 	/* count the length of the chanq for statistics */
651 	counter = 0;
652 	if (dasd_global_profile_level || block->profile.data)
653 		list_for_each(l, &block->ccw_queue)
654 			if (++counter >= 31)
655 				break;
656 
657 	spin_lock(&dasd_global_profile.lock);
658 	if (dasd_global_profile.data) {
659 		dasd_global_profile.data->dasd_io_nr_req[counter]++;
660 		if (rq_data_dir(req) == READ)
661 			dasd_global_profile.data->dasd_read_nr_req[counter]++;
662 	}
663 	spin_unlock(&dasd_global_profile.lock);
664 
665 	spin_lock(&block->profile.lock);
666 	if (block->profile.data) {
667 		block->profile.data->dasd_io_nr_req[counter]++;
668 		if (rq_data_dir(req) == READ)
669 			block->profile.data->dasd_read_nr_req[counter]++;
670 	}
671 	spin_unlock(&block->profile.lock);
672 
673 	/*
674 	 * We count the request for the start device, even though it may run on
675 	 * some other device due to error recovery. This way we make sure that
676 	 * we count each request only once.
677 	 */
678 	device = cqr->startdev;
679 	if (!device->profile.data)
680 		return;
681 
682 	spin_lock(get_ccwdev_lock(device->cdev));
683 	counter = 1; /* request is not yet queued on the start device */
684 	list_for_each(l, &device->ccw_queue)
685 		if (++counter >= 31)
686 			break;
687 	spin_unlock(get_ccwdev_lock(device->cdev));
688 
689 	spin_lock(&device->profile.lock);
690 	device->profile.data->dasd_io_nr_req[counter]++;
691 	if (rq_data_dir(req) == READ)
692 		device->profile.data->dasd_read_nr_req[counter]++;
693 	spin_unlock(&device->profile.lock);
694 }
695 
696 /*
697  * Add profiling information for cqr after execution.
698  */
699 
700 #define dasd_profile_counter(value, index)			   \
701 {								   \
702 	for (index = 0; index < 31 && value >> (2+index); index++) \
703 		;						   \
704 }
705 
706 static void dasd_profile_end_add_data(struct dasd_profile_info *data,
707 				      int is_alias,
708 				      int is_tpm,
709 				      int is_read,
710 				      long sectors,
711 				      int sectors_ind,
712 				      int tottime_ind,
713 				      int tottimeps_ind,
714 				      int strtime_ind,
715 				      int irqtime_ind,
716 				      int irqtimeps_ind,
717 				      int endtime_ind)
718 {
719 	/* in case of an overflow, reset the whole profile */
720 	if (data->dasd_io_reqs == UINT_MAX) {
721 			memset(data, 0, sizeof(*data));
722 			ktime_get_real_ts64(&data->starttod);
723 	}
724 	data->dasd_io_reqs++;
725 	data->dasd_io_sects += sectors;
726 	if (is_alias)
727 		data->dasd_io_alias++;
728 	if (is_tpm)
729 		data->dasd_io_tpm++;
730 
731 	data->dasd_io_secs[sectors_ind]++;
732 	data->dasd_io_times[tottime_ind]++;
733 	data->dasd_io_timps[tottimeps_ind]++;
734 	data->dasd_io_time1[strtime_ind]++;
735 	data->dasd_io_time2[irqtime_ind]++;
736 	data->dasd_io_time2ps[irqtimeps_ind]++;
737 	data->dasd_io_time3[endtime_ind]++;
738 
739 	if (is_read) {
740 		data->dasd_read_reqs++;
741 		data->dasd_read_sects += sectors;
742 		if (is_alias)
743 			data->dasd_read_alias++;
744 		if (is_tpm)
745 			data->dasd_read_tpm++;
746 		data->dasd_read_secs[sectors_ind]++;
747 		data->dasd_read_times[tottime_ind]++;
748 		data->dasd_read_time1[strtime_ind]++;
749 		data->dasd_read_time2[irqtime_ind]++;
750 		data->dasd_read_time3[endtime_ind]++;
751 	}
752 }
753 
754 static void dasd_profile_end(struct dasd_block *block,
755 			     struct dasd_ccw_req *cqr,
756 			     struct request *req)
757 {
758 	unsigned long strtime, irqtime, endtime, tottime;
759 	unsigned long tottimeps, sectors;
760 	struct dasd_device *device;
761 	int sectors_ind, tottime_ind, tottimeps_ind, strtime_ind;
762 	int irqtime_ind, irqtimeps_ind, endtime_ind;
763 	struct dasd_profile_info *data;
764 
765 	device = cqr->startdev;
766 	if (!(dasd_global_profile_level ||
767 	      block->profile.data ||
768 	      device->profile.data))
769 		return;
770 
771 	sectors = blk_rq_sectors(req);
772 	if (!cqr->buildclk || !cqr->startclk ||
773 	    !cqr->stopclk || !cqr->endclk ||
774 	    !sectors)
775 		return;
776 
777 	strtime = ((cqr->startclk - cqr->buildclk) >> 12);
778 	irqtime = ((cqr->stopclk - cqr->startclk) >> 12);
779 	endtime = ((cqr->endclk - cqr->stopclk) >> 12);
780 	tottime = ((cqr->endclk - cqr->buildclk) >> 12);
781 	tottimeps = tottime / sectors;
782 
783 	dasd_profile_counter(sectors, sectors_ind);
784 	dasd_profile_counter(tottime, tottime_ind);
785 	dasd_profile_counter(tottimeps, tottimeps_ind);
786 	dasd_profile_counter(strtime, strtime_ind);
787 	dasd_profile_counter(irqtime, irqtime_ind);
788 	dasd_profile_counter(irqtime / sectors, irqtimeps_ind);
789 	dasd_profile_counter(endtime, endtime_ind);
790 
791 	spin_lock(&dasd_global_profile.lock);
792 	if (dasd_global_profile.data) {
793 		data = dasd_global_profile.data;
794 		data->dasd_sum_times += tottime;
795 		data->dasd_sum_time_str += strtime;
796 		data->dasd_sum_time_irq += irqtime;
797 		data->dasd_sum_time_end += endtime;
798 		dasd_profile_end_add_data(dasd_global_profile.data,
799 					  cqr->startdev != block->base,
800 					  cqr->cpmode == 1,
801 					  rq_data_dir(req) == READ,
802 					  sectors, sectors_ind, tottime_ind,
803 					  tottimeps_ind, strtime_ind,
804 					  irqtime_ind, irqtimeps_ind,
805 					  endtime_ind);
806 	}
807 	spin_unlock(&dasd_global_profile.lock);
808 
809 	spin_lock(&block->profile.lock);
810 	if (block->profile.data) {
811 		data = block->profile.data;
812 		data->dasd_sum_times += tottime;
813 		data->dasd_sum_time_str += strtime;
814 		data->dasd_sum_time_irq += irqtime;
815 		data->dasd_sum_time_end += endtime;
816 		dasd_profile_end_add_data(block->profile.data,
817 					  cqr->startdev != block->base,
818 					  cqr->cpmode == 1,
819 					  rq_data_dir(req) == READ,
820 					  sectors, sectors_ind, tottime_ind,
821 					  tottimeps_ind, strtime_ind,
822 					  irqtime_ind, irqtimeps_ind,
823 					  endtime_ind);
824 	}
825 	spin_unlock(&block->profile.lock);
826 
827 	spin_lock(&device->profile.lock);
828 	if (device->profile.data) {
829 		data = device->profile.data;
830 		data->dasd_sum_times += tottime;
831 		data->dasd_sum_time_str += strtime;
832 		data->dasd_sum_time_irq += irqtime;
833 		data->dasd_sum_time_end += endtime;
834 		dasd_profile_end_add_data(device->profile.data,
835 					  cqr->startdev != block->base,
836 					  cqr->cpmode == 1,
837 					  rq_data_dir(req) == READ,
838 					  sectors, sectors_ind, tottime_ind,
839 					  tottimeps_ind, strtime_ind,
840 					  irqtime_ind, irqtimeps_ind,
841 					  endtime_ind);
842 	}
843 	spin_unlock(&device->profile.lock);
844 }
845 
846 void dasd_profile_reset(struct dasd_profile *profile)
847 {
848 	struct dasd_profile_info *data;
849 
850 	spin_lock_bh(&profile->lock);
851 	data = profile->data;
852 	if (!data) {
853 		spin_unlock_bh(&profile->lock);
854 		return;
855 	}
856 	memset(data, 0, sizeof(*data));
857 	ktime_get_real_ts64(&data->starttod);
858 	spin_unlock_bh(&profile->lock);
859 }
860 
861 int dasd_profile_on(struct dasd_profile *profile)
862 {
863 	struct dasd_profile_info *data;
864 
865 	data = kzalloc(sizeof(*data), GFP_KERNEL);
866 	if (!data)
867 		return -ENOMEM;
868 	spin_lock_bh(&profile->lock);
869 	if (profile->data) {
870 		spin_unlock_bh(&profile->lock);
871 		kfree(data);
872 		return 0;
873 	}
874 	ktime_get_real_ts64(&data->starttod);
875 	profile->data = data;
876 	spin_unlock_bh(&profile->lock);
877 	return 0;
878 }
879 
880 void dasd_profile_off(struct dasd_profile *profile)
881 {
882 	spin_lock_bh(&profile->lock);
883 	kfree(profile->data);
884 	profile->data = NULL;
885 	spin_unlock_bh(&profile->lock);
886 }
887 
888 char *dasd_get_user_string(const char __user *user_buf, size_t user_len)
889 {
890 	char *buffer;
891 
892 	buffer = vmalloc(user_len + 1);
893 	if (buffer == NULL)
894 		return ERR_PTR(-ENOMEM);
895 	if (copy_from_user(buffer, user_buf, user_len) != 0) {
896 		vfree(buffer);
897 		return ERR_PTR(-EFAULT);
898 	}
899 	/* got the string, now strip linefeed. */
900 	if (buffer[user_len - 1] == '\n')
901 		buffer[user_len - 1] = 0;
902 	else
903 		buffer[user_len] = 0;
904 	return buffer;
905 }
906 
907 static ssize_t dasd_stats_write(struct file *file,
908 				const char __user *user_buf,
909 				size_t user_len, loff_t *pos)
910 {
911 	char *buffer, *str;
912 	int rc;
913 	struct seq_file *m = (struct seq_file *)file->private_data;
914 	struct dasd_profile *prof = m->private;
915 
916 	if (user_len > 65536)
917 		user_len = 65536;
918 	buffer = dasd_get_user_string(user_buf, user_len);
919 	if (IS_ERR(buffer))
920 		return PTR_ERR(buffer);
921 
922 	str = skip_spaces(buffer);
923 	rc = user_len;
924 	if (strncmp(str, "reset", 5) == 0) {
925 		dasd_profile_reset(prof);
926 	} else if (strncmp(str, "on", 2) == 0) {
927 		rc = dasd_profile_on(prof);
928 		if (rc)
929 			goto out;
930 		rc = user_len;
931 		if (prof == &dasd_global_profile) {
932 			dasd_profile_reset(prof);
933 			dasd_global_profile_level = DASD_PROFILE_GLOBAL_ONLY;
934 		}
935 	} else if (strncmp(str, "off", 3) == 0) {
936 		if (prof == &dasd_global_profile)
937 			dasd_global_profile_level = DASD_PROFILE_OFF;
938 		dasd_profile_off(prof);
939 	} else
940 		rc = -EINVAL;
941 out:
942 	vfree(buffer);
943 	return rc;
944 }
945 
946 static void dasd_stats_array(struct seq_file *m, unsigned int *array)
947 {
948 	int i;
949 
950 	for (i = 0; i < 32; i++)
951 		seq_printf(m, "%u ", array[i]);
952 	seq_putc(m, '\n');
953 }
954 
955 static void dasd_stats_seq_print(struct seq_file *m,
956 				 struct dasd_profile_info *data)
957 {
958 	seq_printf(m, "start_time %lld.%09ld\n",
959 		   (s64)data->starttod.tv_sec, data->starttod.tv_nsec);
960 	seq_printf(m, "total_requests %u\n", data->dasd_io_reqs);
961 	seq_printf(m, "total_sectors %u\n", data->dasd_io_sects);
962 	seq_printf(m, "total_pav %u\n", data->dasd_io_alias);
963 	seq_printf(m, "total_hpf %u\n", data->dasd_io_tpm);
964 	seq_printf(m, "avg_total %lu\n", data->dasd_io_reqs ?
965 		   data->dasd_sum_times / data->dasd_io_reqs : 0UL);
966 	seq_printf(m, "avg_build_to_ssch %lu\n", data->dasd_io_reqs ?
967 		   data->dasd_sum_time_str / data->dasd_io_reqs : 0UL);
968 	seq_printf(m, "avg_ssch_to_irq %lu\n", data->dasd_io_reqs ?
969 		   data->dasd_sum_time_irq / data->dasd_io_reqs : 0UL);
970 	seq_printf(m, "avg_irq_to_end %lu\n", data->dasd_io_reqs ?
971 		   data->dasd_sum_time_end / data->dasd_io_reqs : 0UL);
972 	seq_puts(m, "histogram_sectors ");
973 	dasd_stats_array(m, data->dasd_io_secs);
974 	seq_puts(m, "histogram_io_times ");
975 	dasd_stats_array(m, data->dasd_io_times);
976 	seq_puts(m, "histogram_io_times_weighted ");
977 	dasd_stats_array(m, data->dasd_io_timps);
978 	seq_puts(m, "histogram_time_build_to_ssch ");
979 	dasd_stats_array(m, data->dasd_io_time1);
980 	seq_puts(m, "histogram_time_ssch_to_irq ");
981 	dasd_stats_array(m, data->dasd_io_time2);
982 	seq_puts(m, "histogram_time_ssch_to_irq_weighted ");
983 	dasd_stats_array(m, data->dasd_io_time2ps);
984 	seq_puts(m, "histogram_time_irq_to_end ");
985 	dasd_stats_array(m, data->dasd_io_time3);
986 	seq_puts(m, "histogram_ccw_queue_length ");
987 	dasd_stats_array(m, data->dasd_io_nr_req);
988 	seq_printf(m, "total_read_requests %u\n", data->dasd_read_reqs);
989 	seq_printf(m, "total_read_sectors %u\n", data->dasd_read_sects);
990 	seq_printf(m, "total_read_pav %u\n", data->dasd_read_alias);
991 	seq_printf(m, "total_read_hpf %u\n", data->dasd_read_tpm);
992 	seq_puts(m, "histogram_read_sectors ");
993 	dasd_stats_array(m, data->dasd_read_secs);
994 	seq_puts(m, "histogram_read_times ");
995 	dasd_stats_array(m, data->dasd_read_times);
996 	seq_puts(m, "histogram_read_time_build_to_ssch ");
997 	dasd_stats_array(m, data->dasd_read_time1);
998 	seq_puts(m, "histogram_read_time_ssch_to_irq ");
999 	dasd_stats_array(m, data->dasd_read_time2);
1000 	seq_puts(m, "histogram_read_time_irq_to_end ");
1001 	dasd_stats_array(m, data->dasd_read_time3);
1002 	seq_puts(m, "histogram_read_ccw_queue_length ");
1003 	dasd_stats_array(m, data->dasd_read_nr_req);
1004 }
1005 
1006 static int dasd_stats_show(struct seq_file *m, void *v)
1007 {
1008 	struct dasd_profile *profile;
1009 	struct dasd_profile_info *data;
1010 
1011 	profile = m->private;
1012 	spin_lock_bh(&profile->lock);
1013 	data = profile->data;
1014 	if (!data) {
1015 		spin_unlock_bh(&profile->lock);
1016 		seq_puts(m, "disabled\n");
1017 		return 0;
1018 	}
1019 	dasd_stats_seq_print(m, data);
1020 	spin_unlock_bh(&profile->lock);
1021 	return 0;
1022 }
1023 
1024 static int dasd_stats_open(struct inode *inode, struct file *file)
1025 {
1026 	struct dasd_profile *profile = inode->i_private;
1027 	return single_open(file, dasd_stats_show, profile);
1028 }
1029 
1030 static const struct file_operations dasd_stats_raw_fops = {
1031 	.owner		= THIS_MODULE,
1032 	.open		= dasd_stats_open,
1033 	.read		= seq_read,
1034 	.llseek		= seq_lseek,
1035 	.release	= single_release,
1036 	.write		= dasd_stats_write,
1037 };
1038 
1039 static void dasd_profile_init(struct dasd_profile *profile,
1040 			      struct dentry *base_dentry)
1041 {
1042 	umode_t mode;
1043 	struct dentry *pde;
1044 
1045 	if (!base_dentry)
1046 		return;
1047 	profile->dentry = NULL;
1048 	profile->data = NULL;
1049 	mode = (S_IRUSR | S_IWUSR | S_IFREG);
1050 	pde = debugfs_create_file("statistics", mode, base_dentry,
1051 				  profile, &dasd_stats_raw_fops);
1052 	if (pde && !IS_ERR(pde))
1053 		profile->dentry = pde;
1054 	return;
1055 }
1056 
1057 static void dasd_profile_exit(struct dasd_profile *profile)
1058 {
1059 	dasd_profile_off(profile);
1060 	debugfs_remove(profile->dentry);
1061 	profile->dentry = NULL;
1062 }
1063 
1064 static void dasd_statistics_removeroot(void)
1065 {
1066 	dasd_global_profile_level = DASD_PROFILE_OFF;
1067 	dasd_profile_exit(&dasd_global_profile);
1068 	debugfs_remove(dasd_debugfs_global_entry);
1069 	debugfs_remove(dasd_debugfs_root_entry);
1070 }
1071 
1072 static void dasd_statistics_createroot(void)
1073 {
1074 	struct dentry *pde;
1075 
1076 	dasd_debugfs_root_entry = NULL;
1077 	pde = debugfs_create_dir("dasd", NULL);
1078 	if (!pde || IS_ERR(pde))
1079 		goto error;
1080 	dasd_debugfs_root_entry = pde;
1081 	pde = debugfs_create_dir("global", dasd_debugfs_root_entry);
1082 	if (!pde || IS_ERR(pde))
1083 		goto error;
1084 	dasd_debugfs_global_entry = pde;
1085 	dasd_profile_init(&dasd_global_profile, dasd_debugfs_global_entry);
1086 	return;
1087 
1088 error:
1089 	DBF_EVENT(DBF_ERR, "%s",
1090 		  "Creation of the dasd debugfs interface failed");
1091 	dasd_statistics_removeroot();
1092 	return;
1093 }
1094 
1095 #else
1096 #define dasd_profile_start(block, cqr, req) do {} while (0)
1097 #define dasd_profile_end(block, cqr, req) do {} while (0)
1098 
1099 static void dasd_statistics_createroot(void)
1100 {
1101 	return;
1102 }
1103 
1104 static void dasd_statistics_removeroot(void)
1105 {
1106 	return;
1107 }
1108 
1109 static void dasd_profile_init(struct dasd_profile *profile,
1110 			      struct dentry *base_dentry)
1111 {
1112 	return;
1113 }
1114 
1115 static void dasd_profile_exit(struct dasd_profile *profile)
1116 {
1117 	return;
1118 }
1119 
1120 int dasd_profile_on(struct dasd_profile *profile)
1121 {
1122 	return 0;
1123 }
1124 
1125 #endif				/* CONFIG_DASD_PROFILE */
1126 
1127 static int dasd_hosts_show(struct seq_file *m, void *v)
1128 {
1129 	struct dasd_device *device;
1130 	int rc = -EOPNOTSUPP;
1131 
1132 	device = m->private;
1133 	dasd_get_device(device);
1134 
1135 	if (device->discipline->hosts_print)
1136 		rc = device->discipline->hosts_print(device, m);
1137 
1138 	dasd_put_device(device);
1139 	return rc;
1140 }
1141 
1142 DEFINE_SHOW_ATTRIBUTE(dasd_hosts);
1143 
1144 static void dasd_hosts_exit(struct dasd_device *device)
1145 {
1146 	debugfs_remove(device->hosts_dentry);
1147 	device->hosts_dentry = NULL;
1148 }
1149 
1150 static void dasd_hosts_init(struct dentry *base_dentry,
1151 			    struct dasd_device *device)
1152 {
1153 	struct dentry *pde;
1154 	umode_t mode;
1155 
1156 	if (!base_dentry)
1157 		return;
1158 
1159 	mode = S_IRUSR | S_IFREG;
1160 	pde = debugfs_create_file("host_access_list", mode, base_dentry,
1161 				  device, &dasd_hosts_fops);
1162 	if (pde && !IS_ERR(pde))
1163 		device->hosts_dentry = pde;
1164 }
1165 
1166 struct dasd_ccw_req *dasd_smalloc_request(int magic, int cplength, int datasize,
1167 					  struct dasd_device *device,
1168 					  struct dasd_ccw_req *cqr)
1169 {
1170 	unsigned long flags;
1171 	char *data, *chunk;
1172 	int size = 0;
1173 
1174 	if (cplength > 0)
1175 		size += cplength * sizeof(struct ccw1);
1176 	if (datasize > 0)
1177 		size += datasize;
1178 	if (!cqr)
1179 		size += (sizeof(*cqr) + 7L) & -8L;
1180 
1181 	spin_lock_irqsave(&device->mem_lock, flags);
1182 	data = chunk = dasd_alloc_chunk(&device->ccw_chunks, size);
1183 	spin_unlock_irqrestore(&device->mem_lock, flags);
1184 	if (!chunk)
1185 		return ERR_PTR(-ENOMEM);
1186 	if (!cqr) {
1187 		cqr = (void *) data;
1188 		data += (sizeof(*cqr) + 7L) & -8L;
1189 	}
1190 	memset(cqr, 0, sizeof(*cqr));
1191 	cqr->mem_chunk = chunk;
1192 	if (cplength > 0) {
1193 		cqr->cpaddr = data;
1194 		data += cplength * sizeof(struct ccw1);
1195 		memset(cqr->cpaddr, 0, cplength * sizeof(struct ccw1));
1196 	}
1197 	if (datasize > 0) {
1198 		cqr->data = data;
1199  		memset(cqr->data, 0, datasize);
1200 	}
1201 	cqr->magic = magic;
1202 	set_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags);
1203 	dasd_get_device(device);
1204 	return cqr;
1205 }
1206 EXPORT_SYMBOL(dasd_smalloc_request);
1207 
1208 struct dasd_ccw_req *dasd_fmalloc_request(int magic, int cplength,
1209 					  int datasize,
1210 					  struct dasd_device *device)
1211 {
1212 	struct dasd_ccw_req *cqr;
1213 	unsigned long flags;
1214 	int size, cqr_size;
1215 	char *data;
1216 
1217 	cqr_size = (sizeof(*cqr) + 7L) & -8L;
1218 	size = cqr_size;
1219 	if (cplength > 0)
1220 		size += cplength * sizeof(struct ccw1);
1221 	if (datasize > 0)
1222 		size += datasize;
1223 
1224 	spin_lock_irqsave(&device->mem_lock, flags);
1225 	cqr = dasd_alloc_chunk(&device->ese_chunks, size);
1226 	spin_unlock_irqrestore(&device->mem_lock, flags);
1227 	if (!cqr)
1228 		return ERR_PTR(-ENOMEM);
1229 	memset(cqr, 0, sizeof(*cqr));
1230 	data = (char *)cqr + cqr_size;
1231 	cqr->cpaddr = NULL;
1232 	if (cplength > 0) {
1233 		cqr->cpaddr = data;
1234 		data += cplength * sizeof(struct ccw1);
1235 		memset(cqr->cpaddr, 0, cplength * sizeof(struct ccw1));
1236 	}
1237 	cqr->data = NULL;
1238 	if (datasize > 0) {
1239 		cqr->data = data;
1240 		memset(cqr->data, 0, datasize);
1241 	}
1242 
1243 	cqr->magic = magic;
1244 	set_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags);
1245 	dasd_get_device(device);
1246 
1247 	return cqr;
1248 }
1249 EXPORT_SYMBOL(dasd_fmalloc_request);
1250 
1251 void dasd_sfree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
1252 {
1253 	unsigned long flags;
1254 
1255 	spin_lock_irqsave(&device->mem_lock, flags);
1256 	dasd_free_chunk(&device->ccw_chunks, cqr->mem_chunk);
1257 	spin_unlock_irqrestore(&device->mem_lock, flags);
1258 	dasd_put_device(device);
1259 }
1260 EXPORT_SYMBOL(dasd_sfree_request);
1261 
1262 void dasd_ffree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
1263 {
1264 	unsigned long flags;
1265 
1266 	spin_lock_irqsave(&device->mem_lock, flags);
1267 	dasd_free_chunk(&device->ese_chunks, cqr);
1268 	spin_unlock_irqrestore(&device->mem_lock, flags);
1269 	dasd_put_device(device);
1270 }
1271 EXPORT_SYMBOL(dasd_ffree_request);
1272 
1273 /*
1274  * Check discipline magic in cqr.
1275  */
1276 static inline int dasd_check_cqr(struct dasd_ccw_req *cqr)
1277 {
1278 	struct dasd_device *device;
1279 
1280 	if (cqr == NULL)
1281 		return -EINVAL;
1282 	device = cqr->startdev;
1283 	if (strncmp((char *) &cqr->magic, device->discipline->ebcname, 4)) {
1284 		DBF_DEV_EVENT(DBF_WARNING, device,
1285 			    " dasd_ccw_req 0x%08x magic doesn't match"
1286 			    " discipline 0x%08x",
1287 			    cqr->magic,
1288 			    *(unsigned int *) device->discipline->name);
1289 		return -EINVAL;
1290 	}
1291 	return 0;
1292 }
1293 
1294 /*
1295  * Terminate the current i/o and set the request to clear_pending.
1296  * Timer keeps device runnig.
1297  * ccw_device_clear can fail if the i/o subsystem
1298  * is in a bad mood.
1299  */
1300 int dasd_term_IO(struct dasd_ccw_req *cqr)
1301 {
1302 	struct dasd_device *device;
1303 	int retries, rc;
1304 	char errorstring[ERRORLENGTH];
1305 
1306 	/* Check the cqr */
1307 	rc = dasd_check_cqr(cqr);
1308 	if (rc)
1309 		return rc;
1310 	retries = 0;
1311 	device = (struct dasd_device *) cqr->startdev;
1312 	while ((retries < 5) && (cqr->status == DASD_CQR_IN_IO)) {
1313 		rc = ccw_device_clear(device->cdev, (long) cqr);
1314 		switch (rc) {
1315 		case 0:	/* termination successful */
1316 			cqr->status = DASD_CQR_CLEAR_PENDING;
1317 			cqr->stopclk = get_tod_clock();
1318 			cqr->starttime = 0;
1319 			DBF_DEV_EVENT(DBF_DEBUG, device,
1320 				      "terminate cqr %p successful",
1321 				      cqr);
1322 			break;
1323 		case -ENODEV:
1324 			DBF_DEV_EVENT(DBF_ERR, device, "%s",
1325 				      "device gone, retry");
1326 			break;
1327 		case -EINVAL:
1328 			/*
1329 			 * device not valid so no I/O could be running
1330 			 * handle CQR as termination successful
1331 			 */
1332 			cqr->status = DASD_CQR_CLEARED;
1333 			cqr->stopclk = get_tod_clock();
1334 			cqr->starttime = 0;
1335 			/* no retries for invalid devices */
1336 			cqr->retries = -1;
1337 			DBF_DEV_EVENT(DBF_ERR, device, "%s",
1338 				      "EINVAL, handle as terminated");
1339 			/* fake rc to success */
1340 			rc = 0;
1341 			break;
1342 		default:
1343 			/* internal error 10 - unknown rc*/
1344 			snprintf(errorstring, ERRORLENGTH, "10 %d", rc);
1345 			dev_err(&device->cdev->dev, "An error occurred in the "
1346 				"DASD device driver, reason=%s\n", errorstring);
1347 			BUG();
1348 			break;
1349 		}
1350 		retries++;
1351 	}
1352 	dasd_schedule_device_bh(device);
1353 	return rc;
1354 }
1355 EXPORT_SYMBOL(dasd_term_IO);
1356 
1357 /*
1358  * Start the i/o. This start_IO can fail if the channel is really busy.
1359  * In that case set up a timer to start the request later.
1360  */
1361 int dasd_start_IO(struct dasd_ccw_req *cqr)
1362 {
1363 	struct dasd_device *device;
1364 	int rc;
1365 	char errorstring[ERRORLENGTH];
1366 
1367 	/* Check the cqr */
1368 	rc = dasd_check_cqr(cqr);
1369 	if (rc) {
1370 		cqr->intrc = rc;
1371 		return rc;
1372 	}
1373 	device = (struct dasd_device *) cqr->startdev;
1374 	if (((cqr->block &&
1375 	      test_bit(DASD_FLAG_LOCK_STOLEN, &cqr->block->base->flags)) ||
1376 	     test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags)) &&
1377 	    !test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
1378 		DBF_DEV_EVENT(DBF_DEBUG, device, "start_IO: return request %p "
1379 			      "because of stolen lock", cqr);
1380 		cqr->status = DASD_CQR_ERROR;
1381 		cqr->intrc = -EPERM;
1382 		return -EPERM;
1383 	}
1384 	if (cqr->retries < 0) {
1385 		/* internal error 14 - start_IO run out of retries */
1386 		sprintf(errorstring, "14 %p", cqr);
1387 		dev_err(&device->cdev->dev, "An error occurred in the DASD "
1388 			"device driver, reason=%s\n", errorstring);
1389 		cqr->status = DASD_CQR_ERROR;
1390 		return -EIO;
1391 	}
1392 	cqr->startclk = get_tod_clock();
1393 	cqr->starttime = jiffies;
1394 	cqr->retries--;
1395 	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
1396 		cqr->lpm &= dasd_path_get_opm(device);
1397 		if (!cqr->lpm)
1398 			cqr->lpm = dasd_path_get_opm(device);
1399 	}
1400 	/*
1401 	 * remember the amount of formatted tracks to prevent double format on
1402 	 * ESE devices
1403 	 */
1404 	if (cqr->block)
1405 		cqr->trkcount = atomic_read(&cqr->block->trkcount);
1406 
1407 	if (cqr->cpmode == 1) {
1408 		rc = ccw_device_tm_start(device->cdev, cqr->cpaddr,
1409 					 (long) cqr, cqr->lpm);
1410 	} else {
1411 		rc = ccw_device_start(device->cdev, cqr->cpaddr,
1412 				      (long) cqr, cqr->lpm, 0);
1413 	}
1414 	switch (rc) {
1415 	case 0:
1416 		cqr->status = DASD_CQR_IN_IO;
1417 		break;
1418 	case -EBUSY:
1419 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1420 			      "start_IO: device busy, retry later");
1421 		break;
1422 	case -EACCES:
1423 		/* -EACCES indicates that the request used only a subset of the
1424 		 * available paths and all these paths are gone. If the lpm of
1425 		 * this request was only a subset of the opm (e.g. the ppm) then
1426 		 * we just do a retry with all available paths.
1427 		 * If we already use the full opm, something is amiss, and we
1428 		 * need a full path verification.
1429 		 */
1430 		if (test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
1431 			DBF_DEV_EVENT(DBF_WARNING, device,
1432 				      "start_IO: selected paths gone (%x)",
1433 				      cqr->lpm);
1434 		} else if (cqr->lpm != dasd_path_get_opm(device)) {
1435 			cqr->lpm = dasd_path_get_opm(device);
1436 			DBF_DEV_EVENT(DBF_DEBUG, device, "%s",
1437 				      "start_IO: selected paths gone,"
1438 				      " retry on all paths");
1439 		} else {
1440 			DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1441 				      "start_IO: all paths in opm gone,"
1442 				      " do path verification");
1443 			dasd_generic_last_path_gone(device);
1444 			dasd_path_no_path(device);
1445 			dasd_path_set_tbvpm(device,
1446 					  ccw_device_get_path_mask(
1447 						  device->cdev));
1448 		}
1449 		break;
1450 	case -ENODEV:
1451 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1452 			      "start_IO: -ENODEV device gone, retry");
1453 		/* this is equivalent to CC=3 for SSCH report this to EER */
1454 		dasd_handle_autoquiesce(device, cqr, DASD_EER_STARTIO);
1455 		break;
1456 	case -EIO:
1457 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1458 			      "start_IO: -EIO device gone, retry");
1459 		break;
1460 	case -EINVAL:
1461 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1462 			      "start_IO: -EINVAL device currently "
1463 			      "not accessible");
1464 		break;
1465 	default:
1466 		/* internal error 11 - unknown rc */
1467 		snprintf(errorstring, ERRORLENGTH, "11 %d", rc);
1468 		dev_err(&device->cdev->dev,
1469 			"An error occurred in the DASD device driver, "
1470 			"reason=%s\n", errorstring);
1471 		BUG();
1472 		break;
1473 	}
1474 	cqr->intrc = rc;
1475 	return rc;
1476 }
1477 EXPORT_SYMBOL(dasd_start_IO);
1478 
1479 /*
1480  * Timeout function for dasd devices. This is used for different purposes
1481  *  1) missing interrupt handler for normal operation
1482  *  2) delayed start of request where start_IO failed with -EBUSY
1483  *  3) timeout for missing state change interrupts
1484  * The head of the ccw queue will have status DASD_CQR_IN_IO for 1),
1485  * DASD_CQR_QUEUED for 2) and 3).
1486  */
1487 static void dasd_device_timeout(struct timer_list *t)
1488 {
1489 	unsigned long flags;
1490 	struct dasd_device *device;
1491 
1492 	device = from_timer(device, t, timer);
1493 	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
1494 	/* re-activate request queue */
1495 	dasd_device_remove_stop_bits(device, DASD_STOPPED_PENDING);
1496 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
1497 	dasd_schedule_device_bh(device);
1498 }
1499 
1500 /*
1501  * Setup timeout for a device in jiffies.
1502  */
1503 void dasd_device_set_timer(struct dasd_device *device, int expires)
1504 {
1505 	if (expires == 0)
1506 		del_timer(&device->timer);
1507 	else
1508 		mod_timer(&device->timer, jiffies + expires);
1509 }
1510 EXPORT_SYMBOL(dasd_device_set_timer);
1511 
1512 /*
1513  * Clear timeout for a device.
1514  */
1515 void dasd_device_clear_timer(struct dasd_device *device)
1516 {
1517 	del_timer(&device->timer);
1518 }
1519 EXPORT_SYMBOL(dasd_device_clear_timer);
1520 
1521 static void dasd_handle_killed_request(struct ccw_device *cdev,
1522 				       unsigned long intparm)
1523 {
1524 	struct dasd_ccw_req *cqr;
1525 	struct dasd_device *device;
1526 
1527 	if (!intparm)
1528 		return;
1529 	cqr = (struct dasd_ccw_req *) intparm;
1530 	if (cqr->status != DASD_CQR_IN_IO) {
1531 		DBF_EVENT_DEVID(DBF_DEBUG, cdev,
1532 				"invalid status in handle_killed_request: "
1533 				"%02x", cqr->status);
1534 		return;
1535 	}
1536 
1537 	device = dasd_device_from_cdev_locked(cdev);
1538 	if (IS_ERR(device)) {
1539 		DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1540 				"unable to get device from cdev");
1541 		return;
1542 	}
1543 
1544 	if (!cqr->startdev ||
1545 	    device != cqr->startdev ||
1546 	    strncmp(cqr->startdev->discipline->ebcname,
1547 		    (char *) &cqr->magic, 4)) {
1548 		DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1549 				"invalid device in request");
1550 		dasd_put_device(device);
1551 		return;
1552 	}
1553 
1554 	/* Schedule request to be retried. */
1555 	cqr->status = DASD_CQR_QUEUED;
1556 
1557 	dasd_device_clear_timer(device);
1558 	dasd_schedule_device_bh(device);
1559 	dasd_put_device(device);
1560 }
1561 
1562 void dasd_generic_handle_state_change(struct dasd_device *device)
1563 {
1564 	/* First of all start sense subsystem status request. */
1565 	dasd_eer_snss(device);
1566 
1567 	dasd_device_remove_stop_bits(device, DASD_STOPPED_PENDING);
1568 	dasd_schedule_device_bh(device);
1569 	if (device->block) {
1570 		dasd_schedule_block_bh(device->block);
1571 		if (device->block->gdp)
1572 			blk_mq_run_hw_queues(device->block->gdp->queue, true);
1573 	}
1574 }
1575 EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change);
1576 
1577 static int dasd_check_hpf_error(struct irb *irb)
1578 {
1579 	return (scsw_tm_is_valid_schxs(&irb->scsw) &&
1580 	    (irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX ||
1581 	     irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX));
1582 }
1583 
1584 static int dasd_ese_needs_format(struct dasd_block *block, struct irb *irb)
1585 {
1586 	struct dasd_device *device = NULL;
1587 	u8 *sense = NULL;
1588 
1589 	if (!block)
1590 		return 0;
1591 	device = block->base;
1592 	if (!device || !device->discipline->is_ese)
1593 		return 0;
1594 	if (!device->discipline->is_ese(device))
1595 		return 0;
1596 
1597 	sense = dasd_get_sense(irb);
1598 	if (!sense)
1599 		return 0;
1600 
1601 	return !!(sense[1] & SNS1_NO_REC_FOUND) ||
1602 		!!(sense[1] & SNS1_FILE_PROTECTED) ||
1603 		scsw_cstat(&irb->scsw) == SCHN_STAT_INCORR_LEN;
1604 }
1605 
1606 static int dasd_ese_oos_cond(u8 *sense)
1607 {
1608 	return sense[0] & SNS0_EQUIPMENT_CHECK &&
1609 		sense[1] & SNS1_PERM_ERR &&
1610 		sense[1] & SNS1_WRITE_INHIBITED &&
1611 		sense[25] == 0x01;
1612 }
1613 
1614 /*
1615  * Interrupt handler for "normal" ssch-io based dasd devices.
1616  */
1617 void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
1618 		      struct irb *irb)
1619 {
1620 	struct dasd_ccw_req *cqr, *next, *fcqr;
1621 	struct dasd_device *device;
1622 	unsigned long now;
1623 	int nrf_suppressed = 0;
1624 	int fp_suppressed = 0;
1625 	struct request *req;
1626 	u8 *sense = NULL;
1627 	int expires;
1628 
1629 	cqr = (struct dasd_ccw_req *) intparm;
1630 	if (IS_ERR(irb)) {
1631 		switch (PTR_ERR(irb)) {
1632 		case -EIO:
1633 			if (cqr && cqr->status == DASD_CQR_CLEAR_PENDING) {
1634 				device = cqr->startdev;
1635 				cqr->status = DASD_CQR_CLEARED;
1636 				dasd_device_clear_timer(device);
1637 				wake_up(&dasd_flush_wq);
1638 				dasd_schedule_device_bh(device);
1639 				return;
1640 			}
1641 			break;
1642 		case -ETIMEDOUT:
1643 			DBF_EVENT_DEVID(DBF_WARNING, cdev, "%s: "
1644 					"request timed out\n", __func__);
1645 			break;
1646 		default:
1647 			DBF_EVENT_DEVID(DBF_WARNING, cdev, "%s: "
1648 					"unknown error %ld\n", __func__,
1649 					PTR_ERR(irb));
1650 		}
1651 		dasd_handle_killed_request(cdev, intparm);
1652 		return;
1653 	}
1654 
1655 	now = get_tod_clock();
1656 	/* check for conditions that should be handled immediately */
1657 	if (!cqr ||
1658 	    !(scsw_dstat(&irb->scsw) == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
1659 	      scsw_cstat(&irb->scsw) == 0)) {
1660 		if (cqr)
1661 			memcpy(&cqr->irb, irb, sizeof(*irb));
1662 		device = dasd_device_from_cdev_locked(cdev);
1663 		if (IS_ERR(device))
1664 			return;
1665 		/* ignore unsolicited interrupts for DIAG discipline */
1666 		if (device->discipline == dasd_diag_discipline_pointer) {
1667 			dasd_put_device(device);
1668 			return;
1669 		}
1670 
1671 		/*
1672 		 * In some cases 'File Protected' or 'No Record Found' errors
1673 		 * might be expected and debug log messages for the
1674 		 * corresponding interrupts shouldn't be written then.
1675 		 * Check if either of the according suppress bits is set.
1676 		 */
1677 		sense = dasd_get_sense(irb);
1678 		if (sense) {
1679 			fp_suppressed = (sense[1] & SNS1_FILE_PROTECTED) &&
1680 				test_bit(DASD_CQR_SUPPRESS_FP, &cqr->flags);
1681 			nrf_suppressed = (sense[1] & SNS1_NO_REC_FOUND) &&
1682 				test_bit(DASD_CQR_SUPPRESS_NRF, &cqr->flags);
1683 
1684 			/*
1685 			 * Extent pool probably out-of-space.
1686 			 * Stop device and check exhaust level.
1687 			 */
1688 			if (dasd_ese_oos_cond(sense)) {
1689 				dasd_generic_space_exhaust(device, cqr);
1690 				device->discipline->ext_pool_exhaust(device, cqr);
1691 				dasd_put_device(device);
1692 				return;
1693 			}
1694 		}
1695 		if (!(fp_suppressed || nrf_suppressed))
1696 			device->discipline->dump_sense_dbf(device, irb, "int");
1697 
1698 		if (device->features & DASD_FEATURE_ERPLOG)
1699 			device->discipline->dump_sense(device, cqr, irb);
1700 		device->discipline->check_for_device_change(device, cqr, irb);
1701 		dasd_put_device(device);
1702 	}
1703 
1704 	/* check for attention message */
1705 	if (scsw_dstat(&irb->scsw) & DEV_STAT_ATTENTION) {
1706 		device = dasd_device_from_cdev_locked(cdev);
1707 		if (!IS_ERR(device)) {
1708 			device->discipline->check_attention(device,
1709 							    irb->esw.esw1.lpum);
1710 			dasd_put_device(device);
1711 		}
1712 	}
1713 
1714 	if (!cqr)
1715 		return;
1716 
1717 	device = (struct dasd_device *) cqr->startdev;
1718 	if (!device ||
1719 	    strncmp(device->discipline->ebcname, (char *) &cqr->magic, 4)) {
1720 		DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1721 				"invalid device in request");
1722 		return;
1723 	}
1724 
1725 	if (dasd_ese_needs_format(cqr->block, irb)) {
1726 		req = dasd_get_callback_data(cqr);
1727 		if (!req) {
1728 			cqr->status = DASD_CQR_ERROR;
1729 			return;
1730 		}
1731 		if (rq_data_dir(req) == READ) {
1732 			device->discipline->ese_read(cqr, irb);
1733 			cqr->status = DASD_CQR_SUCCESS;
1734 			cqr->stopclk = now;
1735 			dasd_device_clear_timer(device);
1736 			dasd_schedule_device_bh(device);
1737 			return;
1738 		}
1739 		fcqr = device->discipline->ese_format(device, cqr, irb);
1740 		if (IS_ERR(fcqr)) {
1741 			if (PTR_ERR(fcqr) == -EINVAL) {
1742 				cqr->status = DASD_CQR_ERROR;
1743 				return;
1744 			}
1745 			/*
1746 			 * If we can't format now, let the request go
1747 			 * one extra round. Maybe we can format later.
1748 			 */
1749 			cqr->status = DASD_CQR_QUEUED;
1750 			dasd_schedule_device_bh(device);
1751 			return;
1752 		} else {
1753 			fcqr->status = DASD_CQR_QUEUED;
1754 			cqr->status = DASD_CQR_QUEUED;
1755 			list_add(&fcqr->devlist, &device->ccw_queue);
1756 			dasd_schedule_device_bh(device);
1757 			return;
1758 		}
1759 	}
1760 
1761 	/* Check for clear pending */
1762 	if (cqr->status == DASD_CQR_CLEAR_PENDING &&
1763 	    scsw_fctl(&irb->scsw) & SCSW_FCTL_CLEAR_FUNC) {
1764 		cqr->status = DASD_CQR_CLEARED;
1765 		dasd_device_clear_timer(device);
1766 		wake_up(&dasd_flush_wq);
1767 		dasd_schedule_device_bh(device);
1768 		return;
1769 	}
1770 
1771 	/* check status - the request might have been killed by dyn detach */
1772 	if (cqr->status != DASD_CQR_IN_IO) {
1773 		DBF_DEV_EVENT(DBF_DEBUG, device, "invalid status: bus_id %s, "
1774 			      "status %02x", dev_name(&cdev->dev), cqr->status);
1775 		return;
1776 	}
1777 
1778 	next = NULL;
1779 	expires = 0;
1780 	if (scsw_dstat(&irb->scsw) == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
1781 	    scsw_cstat(&irb->scsw) == 0) {
1782 		/* request was completed successfully */
1783 		cqr->status = DASD_CQR_SUCCESS;
1784 		cqr->stopclk = now;
1785 		/* Start first request on queue if possible -> fast_io. */
1786 		if (cqr->devlist.next != &device->ccw_queue) {
1787 			next = list_entry(cqr->devlist.next,
1788 					  struct dasd_ccw_req, devlist);
1789 		}
1790 	} else {  /* error */
1791 		/* check for HPF error
1792 		 * call discipline function to requeue all requests
1793 		 * and disable HPF accordingly
1794 		 */
1795 		if (cqr->cpmode && dasd_check_hpf_error(irb) &&
1796 		    device->discipline->handle_hpf_error)
1797 			device->discipline->handle_hpf_error(device, irb);
1798 		/*
1799 		 * If we don't want complex ERP for this request, then just
1800 		 * reset this and retry it in the fastpath
1801 		 */
1802 		if (!test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags) &&
1803 		    cqr->retries > 0) {
1804 			if (cqr->lpm == dasd_path_get_opm(device))
1805 				DBF_DEV_EVENT(DBF_DEBUG, device,
1806 					      "default ERP in fastpath "
1807 					      "(%i retries left)",
1808 					      cqr->retries);
1809 			if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags))
1810 				cqr->lpm = dasd_path_get_opm(device);
1811 			cqr->status = DASD_CQR_QUEUED;
1812 			next = cqr;
1813 		} else
1814 			cqr->status = DASD_CQR_ERROR;
1815 	}
1816 	if (next && (next->status == DASD_CQR_QUEUED) &&
1817 	    (!device->stopped)) {
1818 		if (device->discipline->start_IO(next) == 0)
1819 			expires = next->expires;
1820 	}
1821 	if (expires != 0)
1822 		dasd_device_set_timer(device, expires);
1823 	else
1824 		dasd_device_clear_timer(device);
1825 	dasd_schedule_device_bh(device);
1826 }
1827 EXPORT_SYMBOL(dasd_int_handler);
1828 
1829 enum uc_todo dasd_generic_uc_handler(struct ccw_device *cdev, struct irb *irb)
1830 {
1831 	struct dasd_device *device;
1832 
1833 	device = dasd_device_from_cdev_locked(cdev);
1834 
1835 	if (IS_ERR(device))
1836 		goto out;
1837 	if (test_bit(DASD_FLAG_OFFLINE, &device->flags) ||
1838 	   device->state != device->target ||
1839 	   !device->discipline->check_for_device_change){
1840 		dasd_put_device(device);
1841 		goto out;
1842 	}
1843 	if (device->discipline->dump_sense_dbf)
1844 		device->discipline->dump_sense_dbf(device, irb, "uc");
1845 	device->discipline->check_for_device_change(device, NULL, irb);
1846 	dasd_put_device(device);
1847 out:
1848 	return UC_TODO_RETRY;
1849 }
1850 EXPORT_SYMBOL_GPL(dasd_generic_uc_handler);
1851 
1852 /*
1853  * If we have an error on a dasd_block layer request then we cancel
1854  * and return all further requests from the same dasd_block as well.
1855  */
1856 static void __dasd_device_recovery(struct dasd_device *device,
1857 				   struct dasd_ccw_req *ref_cqr)
1858 {
1859 	struct list_head *l, *n;
1860 	struct dasd_ccw_req *cqr;
1861 
1862 	/*
1863 	 * only requeue request that came from the dasd_block layer
1864 	 */
1865 	if (!ref_cqr->block)
1866 		return;
1867 
1868 	list_for_each_safe(l, n, &device->ccw_queue) {
1869 		cqr = list_entry(l, struct dasd_ccw_req, devlist);
1870 		if (cqr->status == DASD_CQR_QUEUED &&
1871 		    ref_cqr->block == cqr->block) {
1872 			cqr->status = DASD_CQR_CLEARED;
1873 		}
1874 	}
1875 };
1876 
1877 /*
1878  * Remove those ccw requests from the queue that need to be returned
1879  * to the upper layer.
1880  */
1881 static void __dasd_device_process_ccw_queue(struct dasd_device *device,
1882 					    struct list_head *final_queue)
1883 {
1884 	struct list_head *l, *n;
1885 	struct dasd_ccw_req *cqr;
1886 
1887 	/* Process request with final status. */
1888 	list_for_each_safe(l, n, &device->ccw_queue) {
1889 		cqr = list_entry(l, struct dasd_ccw_req, devlist);
1890 
1891 		/* Skip any non-final request. */
1892 		if (cqr->status == DASD_CQR_QUEUED ||
1893 		    cqr->status == DASD_CQR_IN_IO ||
1894 		    cqr->status == DASD_CQR_CLEAR_PENDING)
1895 			continue;
1896 		if (cqr->status == DASD_CQR_ERROR) {
1897 			__dasd_device_recovery(device, cqr);
1898 		}
1899 		/* Rechain finished requests to final queue */
1900 		list_move_tail(&cqr->devlist, final_queue);
1901 	}
1902 }
1903 
1904 static void __dasd_process_cqr(struct dasd_device *device,
1905 			       struct dasd_ccw_req *cqr)
1906 {
1907 	char errorstring[ERRORLENGTH];
1908 
1909 	switch (cqr->status) {
1910 	case DASD_CQR_SUCCESS:
1911 		cqr->status = DASD_CQR_DONE;
1912 		break;
1913 	case DASD_CQR_ERROR:
1914 		cqr->status = DASD_CQR_NEED_ERP;
1915 		break;
1916 	case DASD_CQR_CLEARED:
1917 		cqr->status = DASD_CQR_TERMINATED;
1918 		break;
1919 	default:
1920 		/* internal error 12 - wrong cqr status*/
1921 		snprintf(errorstring, ERRORLENGTH, "12 %p %x02", cqr, cqr->status);
1922 		dev_err(&device->cdev->dev,
1923 			"An error occurred in the DASD device driver, "
1924 			"reason=%s\n", errorstring);
1925 		BUG();
1926 	}
1927 	if (cqr->callback)
1928 		cqr->callback(cqr, cqr->callback_data);
1929 }
1930 
1931 /*
1932  * the cqrs from the final queue are returned to the upper layer
1933  * by setting a dasd_block state and calling the callback function
1934  */
1935 static void __dasd_device_process_final_queue(struct dasd_device *device,
1936 					      struct list_head *final_queue)
1937 {
1938 	struct list_head *l, *n;
1939 	struct dasd_ccw_req *cqr;
1940 	struct dasd_block *block;
1941 
1942 	list_for_each_safe(l, n, final_queue) {
1943 		cqr = list_entry(l, struct dasd_ccw_req, devlist);
1944 		list_del_init(&cqr->devlist);
1945 		block = cqr->block;
1946 		if (!block) {
1947 			__dasd_process_cqr(device, cqr);
1948 		} else {
1949 			spin_lock_bh(&block->queue_lock);
1950 			__dasd_process_cqr(device, cqr);
1951 			spin_unlock_bh(&block->queue_lock);
1952 		}
1953 	}
1954 }
1955 
1956 /*
1957  * check if device should be autoquiesced due to too many timeouts
1958  */
1959 static void __dasd_device_check_autoquiesce_timeout(struct dasd_device *device,
1960 						    struct dasd_ccw_req *cqr)
1961 {
1962 	if ((device->default_retries - cqr->retries) >= device->aq_timeouts)
1963 		dasd_handle_autoquiesce(device, cqr, DASD_EER_TIMEOUTS);
1964 }
1965 
1966 /*
1967  * Take a look at the first request on the ccw queue and check
1968  * if it reached its expire time. If so, terminate the IO.
1969  */
1970 static void __dasd_device_check_expire(struct dasd_device *device)
1971 {
1972 	struct dasd_ccw_req *cqr;
1973 
1974 	if (list_empty(&device->ccw_queue))
1975 		return;
1976 	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
1977 	if ((cqr->status == DASD_CQR_IN_IO && cqr->expires != 0) &&
1978 	    (time_after_eq(jiffies, cqr->expires + cqr->starttime))) {
1979 		if (test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
1980 			/*
1981 			 * IO in safe offline processing should not
1982 			 * run out of retries
1983 			 */
1984 			cqr->retries++;
1985 		}
1986 		if (device->discipline->term_IO(cqr) != 0) {
1987 			/* Hmpf, try again in 5 sec */
1988 			dev_err(&device->cdev->dev,
1989 				"cqr %p timed out (%lus) but cannot be "
1990 				"ended, retrying in 5 s\n",
1991 				cqr, (cqr->expires/HZ));
1992 			cqr->expires += 5*HZ;
1993 			dasd_device_set_timer(device, 5*HZ);
1994 		} else {
1995 			dev_err(&device->cdev->dev,
1996 				"cqr %p timed out (%lus), %i retries "
1997 				"remaining\n", cqr, (cqr->expires/HZ),
1998 				cqr->retries);
1999 		}
2000 		__dasd_device_check_autoquiesce_timeout(device, cqr);
2001 	}
2002 }
2003 
2004 /*
2005  * return 1 when device is not eligible for IO
2006  */
2007 static int __dasd_device_is_unusable(struct dasd_device *device,
2008 				     struct dasd_ccw_req *cqr)
2009 {
2010 	int mask = ~(DASD_STOPPED_DC_WAIT | DASD_STOPPED_NOSPC);
2011 
2012 	if (test_bit(DASD_FLAG_OFFLINE, &device->flags) &&
2013 	    !test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
2014 		/*
2015 		 * dasd is being set offline
2016 		 * but it is no safe offline where we have to allow I/O
2017 		 */
2018 		return 1;
2019 	}
2020 	if (device->stopped) {
2021 		if (device->stopped & mask) {
2022 			/* stopped and CQR will not change that. */
2023 			return 1;
2024 		}
2025 		if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
2026 			/* CQR is not able to change device to
2027 			 * operational. */
2028 			return 1;
2029 		}
2030 		/* CQR required to get device operational. */
2031 	}
2032 	return 0;
2033 }
2034 
2035 /*
2036  * Take a look at the first request on the ccw queue and check
2037  * if it needs to be started.
2038  */
2039 static void __dasd_device_start_head(struct dasd_device *device)
2040 {
2041 	struct dasd_ccw_req *cqr;
2042 	int rc;
2043 
2044 	if (list_empty(&device->ccw_queue))
2045 		return;
2046 	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
2047 	if (cqr->status != DASD_CQR_QUEUED)
2048 		return;
2049 	/* if device is not usable return request to upper layer */
2050 	if (__dasd_device_is_unusable(device, cqr)) {
2051 		cqr->intrc = -EAGAIN;
2052 		cqr->status = DASD_CQR_CLEARED;
2053 		dasd_schedule_device_bh(device);
2054 		return;
2055 	}
2056 
2057 	rc = device->discipline->start_IO(cqr);
2058 	if (rc == 0)
2059 		dasd_device_set_timer(device, cqr->expires);
2060 	else if (rc == -EACCES) {
2061 		dasd_schedule_device_bh(device);
2062 	} else
2063 		/* Hmpf, try again in 1/2 sec */
2064 		dasd_device_set_timer(device, 50);
2065 }
2066 
2067 static void __dasd_device_check_path_events(struct dasd_device *device)
2068 {
2069 	__u8 tbvpm, fcsecpm;
2070 	int rc;
2071 
2072 	tbvpm = dasd_path_get_tbvpm(device);
2073 	fcsecpm = dasd_path_get_fcsecpm(device);
2074 
2075 	if (!tbvpm && !fcsecpm)
2076 		return;
2077 
2078 	if (device->stopped & ~(DASD_STOPPED_DC_WAIT))
2079 		return;
2080 
2081 	dasd_path_clear_all_verify(device);
2082 	dasd_path_clear_all_fcsec(device);
2083 
2084 	rc = device->discipline->pe_handler(device, tbvpm, fcsecpm);
2085 	if (rc) {
2086 		dasd_path_add_tbvpm(device, tbvpm);
2087 		dasd_path_add_fcsecpm(device, fcsecpm);
2088 		dasd_device_set_timer(device, 50);
2089 	}
2090 };
2091 
2092 /*
2093  * Go through all request on the dasd_device request queue,
2094  * terminate them on the cdev if necessary, and return them to the
2095  * submitting layer via callback.
2096  * Note:
2097  * Make sure that all 'submitting layers' still exist when
2098  * this function is called!. In other words, when 'device' is a base
2099  * device then all block layer requests must have been removed before
2100  * via dasd_flush_block_queue.
2101  */
2102 int dasd_flush_device_queue(struct dasd_device *device)
2103 {
2104 	struct dasd_ccw_req *cqr, *n;
2105 	int rc;
2106 	struct list_head flush_queue;
2107 
2108 	INIT_LIST_HEAD(&flush_queue);
2109 	spin_lock_irq(get_ccwdev_lock(device->cdev));
2110 	rc = 0;
2111 	list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) {
2112 		/* Check status and move request to flush_queue */
2113 		switch (cqr->status) {
2114 		case DASD_CQR_IN_IO:
2115 			rc = device->discipline->term_IO(cqr);
2116 			if (rc) {
2117 				/* unable to terminate requeust */
2118 				dev_err(&device->cdev->dev,
2119 					"Flushing the DASD request queue "
2120 					"failed for request %p\n", cqr);
2121 				/* stop flush processing */
2122 				goto finished;
2123 			}
2124 			break;
2125 		case DASD_CQR_QUEUED:
2126 			cqr->stopclk = get_tod_clock();
2127 			cqr->status = DASD_CQR_CLEARED;
2128 			break;
2129 		default: /* no need to modify the others */
2130 			break;
2131 		}
2132 		list_move_tail(&cqr->devlist, &flush_queue);
2133 	}
2134 finished:
2135 	spin_unlock_irq(get_ccwdev_lock(device->cdev));
2136 	/*
2137 	 * After this point all requests must be in state CLEAR_PENDING,
2138 	 * CLEARED, SUCCESS or ERROR. Now wait for CLEAR_PENDING to become
2139 	 * one of the others.
2140 	 */
2141 	list_for_each_entry_safe(cqr, n, &flush_queue, devlist)
2142 		wait_event(dasd_flush_wq,
2143 			   (cqr->status != DASD_CQR_CLEAR_PENDING));
2144 	/*
2145 	 * Now set each request back to TERMINATED, DONE or NEED_ERP
2146 	 * and call the callback function of flushed requests
2147 	 */
2148 	__dasd_device_process_final_queue(device, &flush_queue);
2149 	return rc;
2150 }
2151 EXPORT_SYMBOL_GPL(dasd_flush_device_queue);
2152 
2153 /*
2154  * Acquire the device lock and process queues for the device.
2155  */
2156 static void dasd_device_tasklet(unsigned long data)
2157 {
2158 	struct dasd_device *device = (struct dasd_device *) data;
2159 	struct list_head final_queue;
2160 
2161 	atomic_set (&device->tasklet_scheduled, 0);
2162 	INIT_LIST_HEAD(&final_queue);
2163 	spin_lock_irq(get_ccwdev_lock(device->cdev));
2164 	/* Check expire time of first request on the ccw queue. */
2165 	__dasd_device_check_expire(device);
2166 	/* find final requests on ccw queue */
2167 	__dasd_device_process_ccw_queue(device, &final_queue);
2168 	__dasd_device_check_path_events(device);
2169 	spin_unlock_irq(get_ccwdev_lock(device->cdev));
2170 	/* Now call the callback function of requests with final status */
2171 	__dasd_device_process_final_queue(device, &final_queue);
2172 	spin_lock_irq(get_ccwdev_lock(device->cdev));
2173 	/* Now check if the head of the ccw queue needs to be started. */
2174 	__dasd_device_start_head(device);
2175 	spin_unlock_irq(get_ccwdev_lock(device->cdev));
2176 	if (waitqueue_active(&shutdown_waitq))
2177 		wake_up(&shutdown_waitq);
2178 	dasd_put_device(device);
2179 }
2180 
2181 /*
2182  * Schedules a call to dasd_tasklet over the device tasklet.
2183  */
2184 void dasd_schedule_device_bh(struct dasd_device *device)
2185 {
2186 	/* Protect against rescheduling. */
2187 	if (atomic_cmpxchg (&device->tasklet_scheduled, 0, 1) != 0)
2188 		return;
2189 	dasd_get_device(device);
2190 	tasklet_hi_schedule(&device->tasklet);
2191 }
2192 EXPORT_SYMBOL(dasd_schedule_device_bh);
2193 
2194 void dasd_device_set_stop_bits(struct dasd_device *device, int bits)
2195 {
2196 	device->stopped |= bits;
2197 }
2198 EXPORT_SYMBOL_GPL(dasd_device_set_stop_bits);
2199 
2200 void dasd_device_remove_stop_bits(struct dasd_device *device, int bits)
2201 {
2202 	device->stopped &= ~bits;
2203 	if (!device->stopped)
2204 		wake_up(&generic_waitq);
2205 }
2206 EXPORT_SYMBOL_GPL(dasd_device_remove_stop_bits);
2207 
2208 /*
2209  * Queue a request to the head of the device ccw_queue.
2210  * Start the I/O if possible.
2211  */
2212 void dasd_add_request_head(struct dasd_ccw_req *cqr)
2213 {
2214 	struct dasd_device *device;
2215 	unsigned long flags;
2216 
2217 	device = cqr->startdev;
2218 	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2219 	cqr->status = DASD_CQR_QUEUED;
2220 	list_add(&cqr->devlist, &device->ccw_queue);
2221 	/* let the bh start the request to keep them in order */
2222 	dasd_schedule_device_bh(device);
2223 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
2224 }
2225 EXPORT_SYMBOL(dasd_add_request_head);
2226 
2227 /*
2228  * Queue a request to the tail of the device ccw_queue.
2229  * Start the I/O if possible.
2230  */
2231 void dasd_add_request_tail(struct dasd_ccw_req *cqr)
2232 {
2233 	struct dasd_device *device;
2234 	unsigned long flags;
2235 
2236 	device = cqr->startdev;
2237 	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2238 	cqr->status = DASD_CQR_QUEUED;
2239 	list_add_tail(&cqr->devlist, &device->ccw_queue);
2240 	/* let the bh start the request to keep them in order */
2241 	dasd_schedule_device_bh(device);
2242 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
2243 }
2244 EXPORT_SYMBOL(dasd_add_request_tail);
2245 
2246 /*
2247  * Wakeup helper for the 'sleep_on' functions.
2248  */
2249 void dasd_wakeup_cb(struct dasd_ccw_req *cqr, void *data)
2250 {
2251 	spin_lock_irq(get_ccwdev_lock(cqr->startdev->cdev));
2252 	cqr->callback_data = DASD_SLEEPON_END_TAG;
2253 	spin_unlock_irq(get_ccwdev_lock(cqr->startdev->cdev));
2254 	wake_up(&generic_waitq);
2255 }
2256 EXPORT_SYMBOL_GPL(dasd_wakeup_cb);
2257 
2258 static inline int _wait_for_wakeup(struct dasd_ccw_req *cqr)
2259 {
2260 	struct dasd_device *device;
2261 	int rc;
2262 
2263 	device = cqr->startdev;
2264 	spin_lock_irq(get_ccwdev_lock(device->cdev));
2265 	rc = (cqr->callback_data == DASD_SLEEPON_END_TAG);
2266 	spin_unlock_irq(get_ccwdev_lock(device->cdev));
2267 	return rc;
2268 }
2269 
2270 /*
2271  * checks if error recovery is necessary, returns 1 if yes, 0 otherwise.
2272  */
2273 static int __dasd_sleep_on_erp(struct dasd_ccw_req *cqr)
2274 {
2275 	struct dasd_device *device;
2276 	dasd_erp_fn_t erp_fn;
2277 
2278 	if (cqr->status == DASD_CQR_FILLED)
2279 		return 0;
2280 	device = cqr->startdev;
2281 	if (test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags)) {
2282 		if (cqr->status == DASD_CQR_TERMINATED) {
2283 			device->discipline->handle_terminated_request(cqr);
2284 			return 1;
2285 		}
2286 		if (cqr->status == DASD_CQR_NEED_ERP) {
2287 			erp_fn = device->discipline->erp_action(cqr);
2288 			erp_fn(cqr);
2289 			return 1;
2290 		}
2291 		if (cqr->status == DASD_CQR_FAILED)
2292 			dasd_log_sense(cqr, &cqr->irb);
2293 		if (cqr->refers) {
2294 			__dasd_process_erp(device, cqr);
2295 			return 1;
2296 		}
2297 	}
2298 	return 0;
2299 }
2300 
2301 static int __dasd_sleep_on_loop_condition(struct dasd_ccw_req *cqr)
2302 {
2303 	if (test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags)) {
2304 		if (cqr->refers) /* erp is not done yet */
2305 			return 1;
2306 		return ((cqr->status != DASD_CQR_DONE) &&
2307 			(cqr->status != DASD_CQR_FAILED));
2308 	} else
2309 		return (cqr->status == DASD_CQR_FILLED);
2310 }
2311 
2312 static int _dasd_sleep_on(struct dasd_ccw_req *maincqr, int interruptible)
2313 {
2314 	struct dasd_device *device;
2315 	int rc;
2316 	struct list_head ccw_queue;
2317 	struct dasd_ccw_req *cqr;
2318 
2319 	INIT_LIST_HEAD(&ccw_queue);
2320 	maincqr->status = DASD_CQR_FILLED;
2321 	device = maincqr->startdev;
2322 	list_add(&maincqr->blocklist, &ccw_queue);
2323 	for (cqr = maincqr;  __dasd_sleep_on_loop_condition(cqr);
2324 	     cqr = list_first_entry(&ccw_queue,
2325 				    struct dasd_ccw_req, blocklist)) {
2326 
2327 		if (__dasd_sleep_on_erp(cqr))
2328 			continue;
2329 		if (cqr->status != DASD_CQR_FILLED) /* could be failed */
2330 			continue;
2331 		if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2332 		    !test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2333 			cqr->status = DASD_CQR_FAILED;
2334 			cqr->intrc = -EPERM;
2335 			continue;
2336 		}
2337 		/* Non-temporary stop condition will trigger fail fast */
2338 		if (device->stopped & ~DASD_STOPPED_PENDING &&
2339 		    test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2340 		    !dasd_eer_enabled(device) && device->aq_mask == 0) {
2341 			cqr->status = DASD_CQR_FAILED;
2342 			cqr->intrc = -ENOLINK;
2343 			continue;
2344 		}
2345 		/*
2346 		 * Don't try to start requests if device is in
2347 		 * offline processing, it might wait forever
2348 		 */
2349 		if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
2350 			cqr->status = DASD_CQR_FAILED;
2351 			cqr->intrc = -ENODEV;
2352 			continue;
2353 		}
2354 		/*
2355 		 * Don't try to start requests if device is stopped
2356 		 * except path verification requests
2357 		 */
2358 		if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
2359 			if (interruptible) {
2360 				rc = wait_event_interruptible(
2361 					generic_waitq, !(device->stopped));
2362 				if (rc == -ERESTARTSYS) {
2363 					cqr->status = DASD_CQR_FAILED;
2364 					maincqr->intrc = rc;
2365 					continue;
2366 				}
2367 			} else
2368 				wait_event(generic_waitq, !(device->stopped));
2369 		}
2370 		if (!cqr->callback)
2371 			cqr->callback = dasd_wakeup_cb;
2372 
2373 		cqr->callback_data = DASD_SLEEPON_START_TAG;
2374 		dasd_add_request_tail(cqr);
2375 		if (interruptible) {
2376 			rc = wait_event_interruptible(
2377 				generic_waitq, _wait_for_wakeup(cqr));
2378 			if (rc == -ERESTARTSYS) {
2379 				dasd_cancel_req(cqr);
2380 				/* wait (non-interruptible) for final status */
2381 				wait_event(generic_waitq,
2382 					   _wait_for_wakeup(cqr));
2383 				cqr->status = DASD_CQR_FAILED;
2384 				maincqr->intrc = rc;
2385 				continue;
2386 			}
2387 		} else
2388 			wait_event(generic_waitq, _wait_for_wakeup(cqr));
2389 	}
2390 
2391 	maincqr->endclk = get_tod_clock();
2392 	if ((maincqr->status != DASD_CQR_DONE) &&
2393 	    (maincqr->intrc != -ERESTARTSYS))
2394 		dasd_log_sense(maincqr, &maincqr->irb);
2395 	if (maincqr->status == DASD_CQR_DONE)
2396 		rc = 0;
2397 	else if (maincqr->intrc)
2398 		rc = maincqr->intrc;
2399 	else
2400 		rc = -EIO;
2401 	return rc;
2402 }
2403 
2404 static inline int _wait_for_wakeup_queue(struct list_head *ccw_queue)
2405 {
2406 	struct dasd_ccw_req *cqr;
2407 
2408 	list_for_each_entry(cqr, ccw_queue, blocklist) {
2409 		if (cqr->callback_data != DASD_SLEEPON_END_TAG)
2410 			return 0;
2411 	}
2412 
2413 	return 1;
2414 }
2415 
2416 static int _dasd_sleep_on_queue(struct list_head *ccw_queue, int interruptible)
2417 {
2418 	struct dasd_device *device;
2419 	struct dasd_ccw_req *cqr, *n;
2420 	u8 *sense = NULL;
2421 	int rc;
2422 
2423 retry:
2424 	list_for_each_entry_safe(cqr, n, ccw_queue, blocklist) {
2425 		device = cqr->startdev;
2426 		if (cqr->status != DASD_CQR_FILLED) /*could be failed*/
2427 			continue;
2428 
2429 		if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2430 		    !test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2431 			cqr->status = DASD_CQR_FAILED;
2432 			cqr->intrc = -EPERM;
2433 			continue;
2434 		}
2435 		/*Non-temporary stop condition will trigger fail fast*/
2436 		if (device->stopped & ~DASD_STOPPED_PENDING &&
2437 		    test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2438 		    !dasd_eer_enabled(device)) {
2439 			cqr->status = DASD_CQR_FAILED;
2440 			cqr->intrc = -EAGAIN;
2441 			continue;
2442 		}
2443 
2444 		/*Don't try to start requests if device is stopped*/
2445 		if (interruptible) {
2446 			rc = wait_event_interruptible(
2447 				generic_waitq, !device->stopped);
2448 			if (rc == -ERESTARTSYS) {
2449 				cqr->status = DASD_CQR_FAILED;
2450 				cqr->intrc = rc;
2451 				continue;
2452 			}
2453 		} else
2454 			wait_event(generic_waitq, !(device->stopped));
2455 
2456 		if (!cqr->callback)
2457 			cqr->callback = dasd_wakeup_cb;
2458 		cqr->callback_data = DASD_SLEEPON_START_TAG;
2459 		dasd_add_request_tail(cqr);
2460 	}
2461 
2462 	wait_event(generic_waitq, _wait_for_wakeup_queue(ccw_queue));
2463 
2464 	rc = 0;
2465 	list_for_each_entry_safe(cqr, n, ccw_queue, blocklist) {
2466 		/*
2467 		 * In some cases the 'File Protected' or 'Incorrect Length'
2468 		 * error might be expected and error recovery would be
2469 		 * unnecessary in these cases.	Check if the according suppress
2470 		 * bit is set.
2471 		 */
2472 		sense = dasd_get_sense(&cqr->irb);
2473 		if (sense && sense[1] & SNS1_FILE_PROTECTED &&
2474 		    test_bit(DASD_CQR_SUPPRESS_FP, &cqr->flags))
2475 			continue;
2476 		if (scsw_cstat(&cqr->irb.scsw) == 0x40 &&
2477 		    test_bit(DASD_CQR_SUPPRESS_IL, &cqr->flags))
2478 			continue;
2479 
2480 		/*
2481 		 * for alias devices simplify error recovery and
2482 		 * return to upper layer
2483 		 * do not skip ERP requests
2484 		 */
2485 		if (cqr->startdev != cqr->basedev && !cqr->refers &&
2486 		    (cqr->status == DASD_CQR_TERMINATED ||
2487 		     cqr->status == DASD_CQR_NEED_ERP))
2488 			return -EAGAIN;
2489 
2490 		/* normal recovery for basedev IO */
2491 		if (__dasd_sleep_on_erp(cqr))
2492 			/* handle erp first */
2493 			goto retry;
2494 	}
2495 
2496 	return 0;
2497 }
2498 
2499 /*
2500  * Queue a request to the tail of the device ccw_queue and wait for
2501  * it's completion.
2502  */
2503 int dasd_sleep_on(struct dasd_ccw_req *cqr)
2504 {
2505 	return _dasd_sleep_on(cqr, 0);
2506 }
2507 EXPORT_SYMBOL(dasd_sleep_on);
2508 
2509 /*
2510  * Start requests from a ccw_queue and wait for their completion.
2511  */
2512 int dasd_sleep_on_queue(struct list_head *ccw_queue)
2513 {
2514 	return _dasd_sleep_on_queue(ccw_queue, 0);
2515 }
2516 EXPORT_SYMBOL(dasd_sleep_on_queue);
2517 
2518 /*
2519  * Start requests from a ccw_queue and wait interruptible for their completion.
2520  */
2521 int dasd_sleep_on_queue_interruptible(struct list_head *ccw_queue)
2522 {
2523 	return _dasd_sleep_on_queue(ccw_queue, 1);
2524 }
2525 EXPORT_SYMBOL(dasd_sleep_on_queue_interruptible);
2526 
2527 /*
2528  * Queue a request to the tail of the device ccw_queue and wait
2529  * interruptible for it's completion.
2530  */
2531 int dasd_sleep_on_interruptible(struct dasd_ccw_req *cqr)
2532 {
2533 	return _dasd_sleep_on(cqr, 1);
2534 }
2535 EXPORT_SYMBOL(dasd_sleep_on_interruptible);
2536 
2537 /*
2538  * Whoa nelly now it gets really hairy. For some functions (e.g. steal lock
2539  * for eckd devices) the currently running request has to be terminated
2540  * and be put back to status queued, before the special request is added
2541  * to the head of the queue. Then the special request is waited on normally.
2542  */
2543 static inline int _dasd_term_running_cqr(struct dasd_device *device)
2544 {
2545 	struct dasd_ccw_req *cqr;
2546 	int rc;
2547 
2548 	if (list_empty(&device->ccw_queue))
2549 		return 0;
2550 	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
2551 	rc = device->discipline->term_IO(cqr);
2552 	if (!rc)
2553 		/*
2554 		 * CQR terminated because a more important request is pending.
2555 		 * Undo decreasing of retry counter because this is
2556 		 * not an error case.
2557 		 */
2558 		cqr->retries++;
2559 	return rc;
2560 }
2561 
2562 int dasd_sleep_on_immediatly(struct dasd_ccw_req *cqr)
2563 {
2564 	struct dasd_device *device;
2565 	int rc;
2566 
2567 	device = cqr->startdev;
2568 	if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2569 	    !test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2570 		cqr->status = DASD_CQR_FAILED;
2571 		cqr->intrc = -EPERM;
2572 		return -EIO;
2573 	}
2574 	spin_lock_irq(get_ccwdev_lock(device->cdev));
2575 	rc = _dasd_term_running_cqr(device);
2576 	if (rc) {
2577 		spin_unlock_irq(get_ccwdev_lock(device->cdev));
2578 		return rc;
2579 	}
2580 	cqr->callback = dasd_wakeup_cb;
2581 	cqr->callback_data = DASD_SLEEPON_START_TAG;
2582 	cqr->status = DASD_CQR_QUEUED;
2583 	/*
2584 	 * add new request as second
2585 	 * first the terminated cqr needs to be finished
2586 	 */
2587 	list_add(&cqr->devlist, device->ccw_queue.next);
2588 
2589 	/* let the bh start the request to keep them in order */
2590 	dasd_schedule_device_bh(device);
2591 
2592 	spin_unlock_irq(get_ccwdev_lock(device->cdev));
2593 
2594 	wait_event(generic_waitq, _wait_for_wakeup(cqr));
2595 
2596 	if (cqr->status == DASD_CQR_DONE)
2597 		rc = 0;
2598 	else if (cqr->intrc)
2599 		rc = cqr->intrc;
2600 	else
2601 		rc = -EIO;
2602 
2603 	/* kick tasklets */
2604 	dasd_schedule_device_bh(device);
2605 	if (device->block)
2606 		dasd_schedule_block_bh(device->block);
2607 
2608 	return rc;
2609 }
2610 EXPORT_SYMBOL(dasd_sleep_on_immediatly);
2611 
2612 /*
2613  * Cancels a request that was started with dasd_sleep_on_req.
2614  * This is useful to timeout requests. The request will be
2615  * terminated if it is currently in i/o.
2616  * Returns 0 if request termination was successful
2617  *	   negative error code if termination failed
2618  * Cancellation of a request is an asynchronous operation! The calling
2619  * function has to wait until the request is properly returned via callback.
2620  */
2621 static int __dasd_cancel_req(struct dasd_ccw_req *cqr)
2622 {
2623 	struct dasd_device *device = cqr->startdev;
2624 	int rc = 0;
2625 
2626 	switch (cqr->status) {
2627 	case DASD_CQR_QUEUED:
2628 		/* request was not started - just set to cleared */
2629 		cqr->status = DASD_CQR_CLEARED;
2630 		break;
2631 	case DASD_CQR_IN_IO:
2632 		/* request in IO - terminate IO and release again */
2633 		rc = device->discipline->term_IO(cqr);
2634 		if (rc) {
2635 			dev_err(&device->cdev->dev,
2636 				"Cancelling request %p failed with rc=%d\n",
2637 				cqr, rc);
2638 		} else {
2639 			cqr->stopclk = get_tod_clock();
2640 		}
2641 		break;
2642 	default: /* already finished or clear pending - do nothing */
2643 		break;
2644 	}
2645 	dasd_schedule_device_bh(device);
2646 	return rc;
2647 }
2648 
2649 int dasd_cancel_req(struct dasd_ccw_req *cqr)
2650 {
2651 	struct dasd_device *device = cqr->startdev;
2652 	unsigned long flags;
2653 	int rc;
2654 
2655 	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2656 	rc = __dasd_cancel_req(cqr);
2657 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
2658 	return rc;
2659 }
2660 
2661 /*
2662  * SECTION: Operations of the dasd_block layer.
2663  */
2664 
2665 /*
2666  * Timeout function for dasd_block. This is used when the block layer
2667  * is waiting for something that may not come reliably, (e.g. a state
2668  * change interrupt)
2669  */
2670 static void dasd_block_timeout(struct timer_list *t)
2671 {
2672 	unsigned long flags;
2673 	struct dasd_block *block;
2674 
2675 	block = from_timer(block, t, timer);
2676 	spin_lock_irqsave(get_ccwdev_lock(block->base->cdev), flags);
2677 	/* re-activate request queue */
2678 	dasd_device_remove_stop_bits(block->base, DASD_STOPPED_PENDING);
2679 	spin_unlock_irqrestore(get_ccwdev_lock(block->base->cdev), flags);
2680 	dasd_schedule_block_bh(block);
2681 	blk_mq_run_hw_queues(block->gdp->queue, true);
2682 }
2683 
2684 /*
2685  * Setup timeout for a dasd_block in jiffies.
2686  */
2687 void dasd_block_set_timer(struct dasd_block *block, int expires)
2688 {
2689 	if (expires == 0)
2690 		del_timer(&block->timer);
2691 	else
2692 		mod_timer(&block->timer, jiffies + expires);
2693 }
2694 EXPORT_SYMBOL(dasd_block_set_timer);
2695 
2696 /*
2697  * Clear timeout for a dasd_block.
2698  */
2699 void dasd_block_clear_timer(struct dasd_block *block)
2700 {
2701 	del_timer(&block->timer);
2702 }
2703 EXPORT_SYMBOL(dasd_block_clear_timer);
2704 
2705 /*
2706  * Process finished error recovery ccw.
2707  */
2708 static void __dasd_process_erp(struct dasd_device *device,
2709 			       struct dasd_ccw_req *cqr)
2710 {
2711 	dasd_erp_fn_t erp_fn;
2712 
2713 	if (cqr->status == DASD_CQR_DONE)
2714 		DBF_DEV_EVENT(DBF_NOTICE, device, "%s", "ERP successful");
2715 	else
2716 		dev_err(&device->cdev->dev, "ERP failed for the DASD\n");
2717 	erp_fn = device->discipline->erp_postaction(cqr);
2718 	erp_fn(cqr);
2719 }
2720 
2721 static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr)
2722 {
2723 	struct request *req;
2724 	blk_status_t error = BLK_STS_OK;
2725 	unsigned int proc_bytes;
2726 	int status;
2727 
2728 	req = (struct request *) cqr->callback_data;
2729 	dasd_profile_end(cqr->block, cqr, req);
2730 
2731 	proc_bytes = cqr->proc_bytes;
2732 	status = cqr->block->base->discipline->free_cp(cqr, req);
2733 	if (status < 0)
2734 		error = errno_to_blk_status(status);
2735 	else if (status == 0) {
2736 		switch (cqr->intrc) {
2737 		case -EPERM:
2738 			/*
2739 			 * DASD doesn't implement SCSI/NVMe reservations, but it
2740 			 * implements a locking scheme similar to them. We
2741 			 * return this error when we no longer have the lock.
2742 			 */
2743 			error = BLK_STS_RESV_CONFLICT;
2744 			break;
2745 		case -ENOLINK:
2746 			error = BLK_STS_TRANSPORT;
2747 			break;
2748 		case -ETIMEDOUT:
2749 			error = BLK_STS_TIMEOUT;
2750 			break;
2751 		default:
2752 			error = BLK_STS_IOERR;
2753 			break;
2754 		}
2755 	}
2756 
2757 	/*
2758 	 * We need to take care for ETIMEDOUT errors here since the
2759 	 * complete callback does not get called in this case.
2760 	 * Take care of all errors here and avoid additional code to
2761 	 * transfer the error value to the complete callback.
2762 	 */
2763 	if (error) {
2764 		blk_mq_end_request(req, error);
2765 		blk_mq_run_hw_queues(req->q, true);
2766 	} else {
2767 		/*
2768 		 * Partial completed requests can happen with ESE devices.
2769 		 * During read we might have gotten a NRF error and have to
2770 		 * complete a request partially.
2771 		 */
2772 		if (proc_bytes) {
2773 			blk_update_request(req, BLK_STS_OK, proc_bytes);
2774 			blk_mq_requeue_request(req, true);
2775 		} else if (likely(!blk_should_fake_timeout(req->q))) {
2776 			blk_mq_complete_request(req);
2777 		}
2778 	}
2779 }
2780 
2781 /*
2782  * Process ccw request queue.
2783  */
2784 static void __dasd_process_block_ccw_queue(struct dasd_block *block,
2785 					   struct list_head *final_queue)
2786 {
2787 	struct list_head *l, *n;
2788 	struct dasd_ccw_req *cqr;
2789 	dasd_erp_fn_t erp_fn;
2790 	unsigned long flags;
2791 	struct dasd_device *base = block->base;
2792 
2793 restart:
2794 	/* Process request with final status. */
2795 	list_for_each_safe(l, n, &block->ccw_queue) {
2796 		cqr = list_entry(l, struct dasd_ccw_req, blocklist);
2797 		if (cqr->status != DASD_CQR_DONE &&
2798 		    cqr->status != DASD_CQR_FAILED &&
2799 		    cqr->status != DASD_CQR_NEED_ERP &&
2800 		    cqr->status != DASD_CQR_TERMINATED)
2801 			continue;
2802 
2803 		if (cqr->status == DASD_CQR_TERMINATED) {
2804 			base->discipline->handle_terminated_request(cqr);
2805 			goto restart;
2806 		}
2807 
2808 		/*  Process requests that may be recovered */
2809 		if (cqr->status == DASD_CQR_NEED_ERP) {
2810 			erp_fn = base->discipline->erp_action(cqr);
2811 			if (IS_ERR(erp_fn(cqr)))
2812 				continue;
2813 			goto restart;
2814 		}
2815 
2816 		/* log sense for fatal error */
2817 		if (cqr->status == DASD_CQR_FAILED) {
2818 			dasd_log_sense(cqr, &cqr->irb);
2819 		}
2820 
2821 		/*
2822 		 * First call extended error reporting and check for autoquiesce
2823 		 */
2824 		spin_lock_irqsave(get_ccwdev_lock(base->cdev), flags);
2825 		if (cqr->status == DASD_CQR_FAILED &&
2826 		    dasd_handle_autoquiesce(base, cqr, DASD_EER_FATALERROR)) {
2827 			cqr->status = DASD_CQR_FILLED;
2828 			cqr->retries = 255;
2829 			spin_unlock_irqrestore(get_ccwdev_lock(base->cdev), flags);
2830 			goto restart;
2831 		}
2832 		spin_unlock_irqrestore(get_ccwdev_lock(base->cdev), flags);
2833 
2834 		/* Process finished ERP request. */
2835 		if (cqr->refers) {
2836 			__dasd_process_erp(base, cqr);
2837 			goto restart;
2838 		}
2839 
2840 		/* Rechain finished requests to final queue */
2841 		cqr->endclk = get_tod_clock();
2842 		list_move_tail(&cqr->blocklist, final_queue);
2843 	}
2844 }
2845 
2846 static void dasd_return_cqr_cb(struct dasd_ccw_req *cqr, void *data)
2847 {
2848 	dasd_schedule_block_bh(cqr->block);
2849 }
2850 
2851 static void __dasd_block_start_head(struct dasd_block *block)
2852 {
2853 	struct dasd_ccw_req *cqr;
2854 
2855 	if (list_empty(&block->ccw_queue))
2856 		return;
2857 	/* We allways begin with the first requests on the queue, as some
2858 	 * of previously started requests have to be enqueued on a
2859 	 * dasd_device again for error recovery.
2860 	 */
2861 	list_for_each_entry(cqr, &block->ccw_queue, blocklist) {
2862 		if (cqr->status != DASD_CQR_FILLED)
2863 			continue;
2864 		if (test_bit(DASD_FLAG_LOCK_STOLEN, &block->base->flags) &&
2865 		    !test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2866 			cqr->status = DASD_CQR_FAILED;
2867 			cqr->intrc = -EPERM;
2868 			dasd_schedule_block_bh(block);
2869 			continue;
2870 		}
2871 		/* Non-temporary stop condition will trigger fail fast */
2872 		if (block->base->stopped & ~DASD_STOPPED_PENDING &&
2873 		    test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2874 		    !dasd_eer_enabled(block->base) && block->base->aq_mask == 0) {
2875 			cqr->status = DASD_CQR_FAILED;
2876 			cqr->intrc = -ENOLINK;
2877 			dasd_schedule_block_bh(block);
2878 			continue;
2879 		}
2880 		/* Don't try to start requests if device is stopped */
2881 		if (block->base->stopped)
2882 			return;
2883 
2884 		/* just a fail safe check, should not happen */
2885 		if (!cqr->startdev)
2886 			cqr->startdev = block->base;
2887 
2888 		/* make sure that the requests we submit find their way back */
2889 		cqr->callback = dasd_return_cqr_cb;
2890 
2891 		dasd_add_request_tail(cqr);
2892 	}
2893 }
2894 
2895 /*
2896  * Central dasd_block layer routine. Takes requests from the generic
2897  * block layer request queue, creates ccw requests, enqueues them on
2898  * a dasd_device and processes ccw requests that have been returned.
2899  */
2900 static void dasd_block_tasklet(unsigned long data)
2901 {
2902 	struct dasd_block *block = (struct dasd_block *) data;
2903 	struct list_head final_queue;
2904 	struct list_head *l, *n;
2905 	struct dasd_ccw_req *cqr;
2906 	struct dasd_queue *dq;
2907 
2908 	atomic_set(&block->tasklet_scheduled, 0);
2909 	INIT_LIST_HEAD(&final_queue);
2910 	spin_lock_irq(&block->queue_lock);
2911 	/* Finish off requests on ccw queue */
2912 	__dasd_process_block_ccw_queue(block, &final_queue);
2913 	spin_unlock_irq(&block->queue_lock);
2914 
2915 	/* Now call the callback function of requests with final status */
2916 	list_for_each_safe(l, n, &final_queue) {
2917 		cqr = list_entry(l, struct dasd_ccw_req, blocklist);
2918 		dq = cqr->dq;
2919 		spin_lock_irq(&dq->lock);
2920 		list_del_init(&cqr->blocklist);
2921 		__dasd_cleanup_cqr(cqr);
2922 		spin_unlock_irq(&dq->lock);
2923 	}
2924 
2925 	spin_lock_irq(&block->queue_lock);
2926 	/* Now check if the head of the ccw queue needs to be started. */
2927 	__dasd_block_start_head(block);
2928 	spin_unlock_irq(&block->queue_lock);
2929 
2930 	if (waitqueue_active(&shutdown_waitq))
2931 		wake_up(&shutdown_waitq);
2932 	dasd_put_device(block->base);
2933 }
2934 
2935 static void _dasd_wake_block_flush_cb(struct dasd_ccw_req *cqr, void *data)
2936 {
2937 	wake_up(&dasd_flush_wq);
2938 }
2939 
2940 /*
2941  * Requeue a request back to the block request queue
2942  * only works for block requests
2943  */
2944 static void _dasd_requeue_request(struct dasd_ccw_req *cqr)
2945 {
2946 	struct request *req;
2947 
2948 	/*
2949 	 * If the request is an ERP request there is nothing to requeue.
2950 	 * This will be done with the remaining original request.
2951 	 */
2952 	if (cqr->refers)
2953 		return;
2954 	spin_lock_irq(&cqr->dq->lock);
2955 	req = (struct request *) cqr->callback_data;
2956 	blk_mq_requeue_request(req, true);
2957 	spin_unlock_irq(&cqr->dq->lock);
2958 
2959 	return;
2960 }
2961 
2962 static int _dasd_requests_to_flushqueue(struct dasd_block *block,
2963 					struct list_head *flush_queue)
2964 {
2965 	struct dasd_ccw_req *cqr, *n;
2966 	unsigned long flags;
2967 	int rc, i;
2968 
2969 	spin_lock_irqsave(&block->queue_lock, flags);
2970 	rc = 0;
2971 restart:
2972 	list_for_each_entry_safe(cqr, n, &block->ccw_queue, blocklist) {
2973 		/* if this request currently owned by a dasd_device cancel it */
2974 		if (cqr->status >= DASD_CQR_QUEUED)
2975 			rc = dasd_cancel_req(cqr);
2976 		if (rc < 0)
2977 			break;
2978 		/* Rechain request (including erp chain) so it won't be
2979 		 * touched by the dasd_block_tasklet anymore.
2980 		 * Replace the callback so we notice when the request
2981 		 * is returned from the dasd_device layer.
2982 		 */
2983 		cqr->callback = _dasd_wake_block_flush_cb;
2984 		for (i = 0; cqr; cqr = cqr->refers, i++)
2985 			list_move_tail(&cqr->blocklist, flush_queue);
2986 		if (i > 1)
2987 			/* moved more than one request - need to restart */
2988 			goto restart;
2989 	}
2990 	spin_unlock_irqrestore(&block->queue_lock, flags);
2991 
2992 	return rc;
2993 }
2994 
2995 /*
2996  * Go through all request on the dasd_block request queue, cancel them
2997  * on the respective dasd_device, and return them to the generic
2998  * block layer.
2999  */
3000 static int dasd_flush_block_queue(struct dasd_block *block)
3001 {
3002 	struct dasd_ccw_req *cqr, *n;
3003 	struct list_head flush_queue;
3004 	unsigned long flags;
3005 	int rc;
3006 
3007 	INIT_LIST_HEAD(&flush_queue);
3008 	rc = _dasd_requests_to_flushqueue(block, &flush_queue);
3009 
3010 	/* Now call the callback function of flushed requests */
3011 restart_cb:
3012 	list_for_each_entry_safe(cqr, n, &flush_queue, blocklist) {
3013 		wait_event(dasd_flush_wq, (cqr->status < DASD_CQR_QUEUED));
3014 		/* Process finished ERP request. */
3015 		if (cqr->refers) {
3016 			spin_lock_bh(&block->queue_lock);
3017 			__dasd_process_erp(block->base, cqr);
3018 			spin_unlock_bh(&block->queue_lock);
3019 			/* restart list_for_xx loop since dasd_process_erp
3020 			 * might remove multiple elements */
3021 			goto restart_cb;
3022 		}
3023 		/* call the callback function */
3024 		spin_lock_irqsave(&cqr->dq->lock, flags);
3025 		cqr->endclk = get_tod_clock();
3026 		list_del_init(&cqr->blocklist);
3027 		__dasd_cleanup_cqr(cqr);
3028 		spin_unlock_irqrestore(&cqr->dq->lock, flags);
3029 	}
3030 	return rc;
3031 }
3032 
3033 /*
3034  * Schedules a call to dasd_tasklet over the device tasklet.
3035  */
3036 void dasd_schedule_block_bh(struct dasd_block *block)
3037 {
3038 	/* Protect against rescheduling. */
3039 	if (atomic_cmpxchg(&block->tasklet_scheduled, 0, 1) != 0)
3040 		return;
3041 	/* life cycle of block is bound to it's base device */
3042 	dasd_get_device(block->base);
3043 	tasklet_hi_schedule(&block->tasklet);
3044 }
3045 EXPORT_SYMBOL(dasd_schedule_block_bh);
3046 
3047 
3048 /*
3049  * SECTION: external block device operations
3050  * (request queue handling, open, release, etc.)
3051  */
3052 
3053 /*
3054  * Dasd request queue function. Called from ll_rw_blk.c
3055  */
3056 static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx,
3057 				    const struct blk_mq_queue_data *qd)
3058 {
3059 	struct dasd_block *block = hctx->queue->queuedata;
3060 	struct dasd_queue *dq = hctx->driver_data;
3061 	struct request *req = qd->rq;
3062 	struct dasd_device *basedev;
3063 	struct dasd_ccw_req *cqr;
3064 	blk_status_t rc = BLK_STS_OK;
3065 
3066 	basedev = block->base;
3067 	spin_lock_irq(&dq->lock);
3068 	if (basedev->state < DASD_STATE_READY ||
3069 	    test_bit(DASD_FLAG_OFFLINE, &basedev->flags)) {
3070 		DBF_DEV_EVENT(DBF_ERR, basedev,
3071 			      "device not ready for request %p", req);
3072 		rc = BLK_STS_IOERR;
3073 		goto out;
3074 	}
3075 
3076 	/*
3077 	 * if device is stopped do not fetch new requests
3078 	 * except failfast is active which will let requests fail
3079 	 * immediately in __dasd_block_start_head()
3080 	 */
3081 	if (basedev->stopped && !(basedev->features & DASD_FEATURE_FAILFAST)) {
3082 		DBF_DEV_EVENT(DBF_ERR, basedev,
3083 			      "device stopped request %p", req);
3084 		rc = BLK_STS_RESOURCE;
3085 		goto out;
3086 	}
3087 
3088 	if (basedev->features & DASD_FEATURE_READONLY &&
3089 	    rq_data_dir(req) == WRITE) {
3090 		DBF_DEV_EVENT(DBF_ERR, basedev,
3091 			      "Rejecting write request %p", req);
3092 		rc = BLK_STS_IOERR;
3093 		goto out;
3094 	}
3095 
3096 	if (test_bit(DASD_FLAG_ABORTALL, &basedev->flags) &&
3097 	    (basedev->features & DASD_FEATURE_FAILFAST ||
3098 	     blk_noretry_request(req))) {
3099 		DBF_DEV_EVENT(DBF_ERR, basedev,
3100 			      "Rejecting failfast request %p", req);
3101 		rc = BLK_STS_IOERR;
3102 		goto out;
3103 	}
3104 
3105 	cqr = basedev->discipline->build_cp(basedev, block, req);
3106 	if (IS_ERR(cqr)) {
3107 		if (PTR_ERR(cqr) == -EBUSY ||
3108 		    PTR_ERR(cqr) == -ENOMEM ||
3109 		    PTR_ERR(cqr) == -EAGAIN) {
3110 			rc = BLK_STS_RESOURCE;
3111 			goto out;
3112 		}
3113 		DBF_DEV_EVENT(DBF_ERR, basedev,
3114 			      "CCW creation failed (rc=%ld) on request %p",
3115 			      PTR_ERR(cqr), req);
3116 		rc = BLK_STS_IOERR;
3117 		goto out;
3118 	}
3119 	/*
3120 	 *  Note: callback is set to dasd_return_cqr_cb in
3121 	 * __dasd_block_start_head to cover erp requests as well
3122 	 */
3123 	cqr->callback_data = req;
3124 	cqr->status = DASD_CQR_FILLED;
3125 	cqr->dq = dq;
3126 
3127 	blk_mq_start_request(req);
3128 	spin_lock(&block->queue_lock);
3129 	list_add_tail(&cqr->blocklist, &block->ccw_queue);
3130 	INIT_LIST_HEAD(&cqr->devlist);
3131 	dasd_profile_start(block, cqr, req);
3132 	dasd_schedule_block_bh(block);
3133 	spin_unlock(&block->queue_lock);
3134 
3135 out:
3136 	spin_unlock_irq(&dq->lock);
3137 	return rc;
3138 }
3139 
3140 /*
3141  * Block timeout callback, called from the block layer
3142  *
3143  * Return values:
3144  * BLK_EH_RESET_TIMER if the request should be left running
3145  * BLK_EH_DONE if the request is handled or terminated
3146  *		      by the driver.
3147  */
3148 enum blk_eh_timer_return dasd_times_out(struct request *req)
3149 {
3150 	struct dasd_block *block = req->q->queuedata;
3151 	struct dasd_device *device;
3152 	struct dasd_ccw_req *cqr;
3153 	unsigned long flags;
3154 	int rc = 0;
3155 
3156 	cqr = blk_mq_rq_to_pdu(req);
3157 	if (!cqr)
3158 		return BLK_EH_DONE;
3159 
3160 	spin_lock_irqsave(&cqr->dq->lock, flags);
3161 	device = cqr->startdev ? cqr->startdev : block->base;
3162 	if (!device->blk_timeout) {
3163 		spin_unlock_irqrestore(&cqr->dq->lock, flags);
3164 		return BLK_EH_RESET_TIMER;
3165 	}
3166 	DBF_DEV_EVENT(DBF_WARNING, device,
3167 		      " dasd_times_out cqr %p status %x",
3168 		      cqr, cqr->status);
3169 
3170 	spin_lock(&block->queue_lock);
3171 	spin_lock(get_ccwdev_lock(device->cdev));
3172 	cqr->retries = -1;
3173 	cqr->intrc = -ETIMEDOUT;
3174 	if (cqr->status >= DASD_CQR_QUEUED) {
3175 		rc = __dasd_cancel_req(cqr);
3176 	} else if (cqr->status == DASD_CQR_FILLED ||
3177 		   cqr->status == DASD_CQR_NEED_ERP) {
3178 		cqr->status = DASD_CQR_TERMINATED;
3179 	} else if (cqr->status == DASD_CQR_IN_ERP) {
3180 		struct dasd_ccw_req *searchcqr, *nextcqr, *tmpcqr;
3181 
3182 		list_for_each_entry_safe(searchcqr, nextcqr,
3183 					 &block->ccw_queue, blocklist) {
3184 			tmpcqr = searchcqr;
3185 			while (tmpcqr->refers)
3186 				tmpcqr = tmpcqr->refers;
3187 			if (tmpcqr != cqr)
3188 				continue;
3189 			/* searchcqr is an ERP request for cqr */
3190 			searchcqr->retries = -1;
3191 			searchcqr->intrc = -ETIMEDOUT;
3192 			if (searchcqr->status >= DASD_CQR_QUEUED) {
3193 				rc = __dasd_cancel_req(searchcqr);
3194 			} else if ((searchcqr->status == DASD_CQR_FILLED) ||
3195 				   (searchcqr->status == DASD_CQR_NEED_ERP)) {
3196 				searchcqr->status = DASD_CQR_TERMINATED;
3197 				rc = 0;
3198 			} else if (searchcqr->status == DASD_CQR_IN_ERP) {
3199 				/*
3200 				 * Shouldn't happen; most recent ERP
3201 				 * request is at the front of queue
3202 				 */
3203 				continue;
3204 			}
3205 			break;
3206 		}
3207 	}
3208 	spin_unlock(get_ccwdev_lock(device->cdev));
3209 	dasd_schedule_block_bh(block);
3210 	spin_unlock(&block->queue_lock);
3211 	spin_unlock_irqrestore(&cqr->dq->lock, flags);
3212 
3213 	return rc ? BLK_EH_RESET_TIMER : BLK_EH_DONE;
3214 }
3215 
3216 static int dasd_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
3217 			  unsigned int idx)
3218 {
3219 	struct dasd_queue *dq = kzalloc(sizeof(*dq), GFP_KERNEL);
3220 
3221 	if (!dq)
3222 		return -ENOMEM;
3223 
3224 	spin_lock_init(&dq->lock);
3225 	hctx->driver_data = dq;
3226 
3227 	return 0;
3228 }
3229 
3230 static void dasd_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
3231 {
3232 	kfree(hctx->driver_data);
3233 	hctx->driver_data = NULL;
3234 }
3235 
3236 static void dasd_request_done(struct request *req)
3237 {
3238 	blk_mq_end_request(req, 0);
3239 	blk_mq_run_hw_queues(req->q, true);
3240 }
3241 
3242 struct blk_mq_ops dasd_mq_ops = {
3243 	.queue_rq = do_dasd_request,
3244 	.complete = dasd_request_done,
3245 	.timeout = dasd_times_out,
3246 	.init_hctx = dasd_init_hctx,
3247 	.exit_hctx = dasd_exit_hctx,
3248 };
3249 
3250 static int dasd_open(struct gendisk *disk, blk_mode_t mode)
3251 {
3252 	struct dasd_device *base;
3253 	int rc;
3254 
3255 	base = dasd_device_from_gendisk(disk);
3256 	if (!base)
3257 		return -ENODEV;
3258 
3259 	atomic_inc(&base->block->open_count);
3260 	if (test_bit(DASD_FLAG_OFFLINE, &base->flags)) {
3261 		rc = -ENODEV;
3262 		goto unlock;
3263 	}
3264 
3265 	if (!try_module_get(base->discipline->owner)) {
3266 		rc = -EINVAL;
3267 		goto unlock;
3268 	}
3269 
3270 	if (dasd_probeonly) {
3271 		dev_info(&base->cdev->dev,
3272 			 "Accessing the DASD failed because it is in "
3273 			 "probeonly mode\n");
3274 		rc = -EPERM;
3275 		goto out;
3276 	}
3277 
3278 	if (base->state <= DASD_STATE_BASIC) {
3279 		DBF_DEV_EVENT(DBF_ERR, base, " %s",
3280 			      " Cannot open unrecognized device");
3281 		rc = -ENODEV;
3282 		goto out;
3283 	}
3284 	if ((mode & BLK_OPEN_WRITE) &&
3285 	    (test_bit(DASD_FLAG_DEVICE_RO, &base->flags) ||
3286 	     (base->features & DASD_FEATURE_READONLY))) {
3287 		rc = -EROFS;
3288 		goto out;
3289 	}
3290 	dasd_put_device(base);
3291 	return 0;
3292 
3293 out:
3294 	module_put(base->discipline->owner);
3295 unlock:
3296 	atomic_dec(&base->block->open_count);
3297 	dasd_put_device(base);
3298 	return rc;
3299 }
3300 
3301 static void dasd_release(struct gendisk *disk)
3302 {
3303 	struct dasd_device *base = dasd_device_from_gendisk(disk);
3304 	if (base) {
3305 		atomic_dec(&base->block->open_count);
3306 		module_put(base->discipline->owner);
3307 		dasd_put_device(base);
3308 	}
3309 }
3310 
3311 /*
3312  * Return disk geometry.
3313  */
3314 static int dasd_getgeo(struct block_device *bdev, struct hd_geometry *geo)
3315 {
3316 	struct dasd_device *base;
3317 
3318 	base = dasd_device_from_gendisk(bdev->bd_disk);
3319 	if (!base)
3320 		return -ENODEV;
3321 
3322 	if (!base->discipline ||
3323 	    !base->discipline->fill_geometry) {
3324 		dasd_put_device(base);
3325 		return -EINVAL;
3326 	}
3327 	base->discipline->fill_geometry(base->block, geo);
3328 	geo->start = get_start_sect(bdev) >> base->block->s2b_shift;
3329 	dasd_put_device(base);
3330 	return 0;
3331 }
3332 
3333 const struct block_device_operations
3334 dasd_device_operations = {
3335 	.owner		= THIS_MODULE,
3336 	.open		= dasd_open,
3337 	.release	= dasd_release,
3338 	.ioctl		= dasd_ioctl,
3339 	.compat_ioctl	= dasd_ioctl,
3340 	.getgeo		= dasd_getgeo,
3341 	.set_read_only	= dasd_set_read_only,
3342 };
3343 
3344 /*******************************************************************************
3345  * end of block device operations
3346  */
3347 
3348 static void
3349 dasd_exit(void)
3350 {
3351 #ifdef CONFIG_PROC_FS
3352 	dasd_proc_exit();
3353 #endif
3354 	dasd_eer_exit();
3355 	kmem_cache_destroy(dasd_page_cache);
3356 	dasd_page_cache = NULL;
3357 	dasd_gendisk_exit();
3358 	dasd_devmap_exit();
3359 	if (dasd_debug_area != NULL) {
3360 		debug_unregister(dasd_debug_area);
3361 		dasd_debug_area = NULL;
3362 	}
3363 	dasd_statistics_removeroot();
3364 }
3365 
3366 /*
3367  * SECTION: common functions for ccw_driver use
3368  */
3369 
3370 /*
3371  * Is the device read-only?
3372  * Note that this function does not report the setting of the
3373  * readonly device attribute, but how it is configured in z/VM.
3374  */
3375 int dasd_device_is_ro(struct dasd_device *device)
3376 {
3377 	struct ccw_dev_id dev_id;
3378 	struct diag210 diag_data;
3379 	int rc;
3380 
3381 	if (!MACHINE_IS_VM)
3382 		return 0;
3383 	ccw_device_get_id(device->cdev, &dev_id);
3384 	memset(&diag_data, 0, sizeof(diag_data));
3385 	diag_data.vrdcdvno = dev_id.devno;
3386 	diag_data.vrdclen = sizeof(diag_data);
3387 	rc = diag210(&diag_data);
3388 	if (rc == 0 || rc == 2) {
3389 		return diag_data.vrdcvfla & 0x80;
3390 	} else {
3391 		DBF_EVENT(DBF_WARNING, "diag210 failed for dev=%04x with rc=%d",
3392 			  dev_id.devno, rc);
3393 		return 0;
3394 	}
3395 }
3396 EXPORT_SYMBOL_GPL(dasd_device_is_ro);
3397 
3398 static void dasd_generic_auto_online(void *data, async_cookie_t cookie)
3399 {
3400 	struct ccw_device *cdev = data;
3401 	int ret;
3402 
3403 	ret = ccw_device_set_online(cdev);
3404 	if (ret)
3405 		pr_warn("%s: Setting the DASD online failed with rc=%d\n",
3406 			dev_name(&cdev->dev), ret);
3407 }
3408 
3409 /*
3410  * Initial attempt at a probe function. this can be simplified once
3411  * the other detection code is gone.
3412  */
3413 int dasd_generic_probe(struct ccw_device *cdev)
3414 {
3415 	cdev->handler = &dasd_int_handler;
3416 
3417 	/*
3418 	 * Automatically online either all dasd devices (dasd_autodetect)
3419 	 * or all devices specified with dasd= parameters during
3420 	 * initial probe.
3421 	 */
3422 	if ((dasd_get_feature(cdev, DASD_FEATURE_INITIAL_ONLINE) > 0 ) ||
3423 	    (dasd_autodetect && dasd_busid_known(dev_name(&cdev->dev)) != 0))
3424 		async_schedule(dasd_generic_auto_online, cdev);
3425 	return 0;
3426 }
3427 EXPORT_SYMBOL_GPL(dasd_generic_probe);
3428 
3429 void dasd_generic_free_discipline(struct dasd_device *device)
3430 {
3431 	/* Forget the discipline information. */
3432 	if (device->discipline) {
3433 		if (device->discipline->uncheck_device)
3434 			device->discipline->uncheck_device(device);
3435 		module_put(device->discipline->owner);
3436 		device->discipline = NULL;
3437 	}
3438 	if (device->base_discipline) {
3439 		module_put(device->base_discipline->owner);
3440 		device->base_discipline = NULL;
3441 	}
3442 }
3443 EXPORT_SYMBOL_GPL(dasd_generic_free_discipline);
3444 
3445 /*
3446  * This will one day be called from a global not_oper handler.
3447  * It is also used by driver_unregister during module unload.
3448  */
3449 void dasd_generic_remove(struct ccw_device *cdev)
3450 {
3451 	struct dasd_device *device;
3452 	struct dasd_block *block;
3453 
3454 	device = dasd_device_from_cdev(cdev);
3455 	if (IS_ERR(device))
3456 		return;
3457 
3458 	if (test_and_set_bit(DASD_FLAG_OFFLINE, &device->flags) &&
3459 	    !test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3460 		/* Already doing offline processing */
3461 		dasd_put_device(device);
3462 		return;
3463 	}
3464 	/*
3465 	 * This device is removed unconditionally. Set offline
3466 	 * flag to prevent dasd_open from opening it while it is
3467 	 * no quite down yet.
3468 	 */
3469 	dasd_set_target_state(device, DASD_STATE_NEW);
3470 	cdev->handler = NULL;
3471 	/* dasd_delete_device destroys the device reference. */
3472 	block = device->block;
3473 	dasd_delete_device(device);
3474 	/*
3475 	 * life cycle of block is bound to device, so delete it after
3476 	 * device was safely removed
3477 	 */
3478 	if (block)
3479 		dasd_free_block(block);
3480 }
3481 EXPORT_SYMBOL_GPL(dasd_generic_remove);
3482 
3483 /*
3484  * Activate a device. This is called from dasd_{eckd,fba}_probe() when either
3485  * the device is detected for the first time and is supposed to be used
3486  * or the user has started activation through sysfs.
3487  */
3488 int dasd_generic_set_online(struct ccw_device *cdev,
3489 			    struct dasd_discipline *base_discipline)
3490 {
3491 	struct dasd_discipline *discipline;
3492 	struct dasd_device *device;
3493 	int rc;
3494 
3495 	/* first online clears initial online feature flag */
3496 	dasd_set_feature(cdev, DASD_FEATURE_INITIAL_ONLINE, 0);
3497 	device = dasd_create_device(cdev);
3498 	if (IS_ERR(device))
3499 		return PTR_ERR(device);
3500 
3501 	discipline = base_discipline;
3502 	if (device->features & DASD_FEATURE_USEDIAG) {
3503 	  	if (!dasd_diag_discipline_pointer) {
3504 			/* Try to load the required module. */
3505 			rc = request_module(DASD_DIAG_MOD);
3506 			if (rc) {
3507 				pr_warn("%s Setting the DASD online failed "
3508 					"because the required module %s "
3509 					"could not be loaded (rc=%d)\n",
3510 					dev_name(&cdev->dev), DASD_DIAG_MOD,
3511 					rc);
3512 				dasd_delete_device(device);
3513 				return -ENODEV;
3514 			}
3515 		}
3516 		/* Module init could have failed, so check again here after
3517 		 * request_module(). */
3518 		if (!dasd_diag_discipline_pointer) {
3519 			pr_warn("%s Setting the DASD online failed because of missing DIAG discipline\n",
3520 				dev_name(&cdev->dev));
3521 			dasd_delete_device(device);
3522 			return -ENODEV;
3523 		}
3524 		discipline = dasd_diag_discipline_pointer;
3525 	}
3526 	if (!try_module_get(base_discipline->owner)) {
3527 		dasd_delete_device(device);
3528 		return -EINVAL;
3529 	}
3530 	if (!try_module_get(discipline->owner)) {
3531 		module_put(base_discipline->owner);
3532 		dasd_delete_device(device);
3533 		return -EINVAL;
3534 	}
3535 	device->base_discipline = base_discipline;
3536 	device->discipline = discipline;
3537 
3538 	/* check_device will allocate block device if necessary */
3539 	rc = discipline->check_device(device);
3540 	if (rc) {
3541 		pr_warn("%s Setting the DASD online with discipline %s failed with rc=%i\n",
3542 			dev_name(&cdev->dev), discipline->name, rc);
3543 		module_put(discipline->owner);
3544 		module_put(base_discipline->owner);
3545 		dasd_delete_device(device);
3546 		return rc;
3547 	}
3548 
3549 	dasd_set_target_state(device, DASD_STATE_ONLINE);
3550 	if (device->state <= DASD_STATE_KNOWN) {
3551 		pr_warn("%s Setting the DASD online failed because of a missing discipline\n",
3552 			dev_name(&cdev->dev));
3553 		rc = -ENODEV;
3554 		dasd_set_target_state(device, DASD_STATE_NEW);
3555 		if (device->block)
3556 			dasd_free_block(device->block);
3557 		dasd_delete_device(device);
3558 	} else
3559 		pr_debug("dasd_generic device %s found\n",
3560 				dev_name(&cdev->dev));
3561 
3562 	wait_event(dasd_init_waitq, _wait_for_device(device));
3563 
3564 	dasd_put_device(device);
3565 	return rc;
3566 }
3567 EXPORT_SYMBOL_GPL(dasd_generic_set_online);
3568 
3569 int dasd_generic_set_offline(struct ccw_device *cdev)
3570 {
3571 	struct dasd_device *device;
3572 	struct dasd_block *block;
3573 	int max_count, open_count, rc;
3574 	unsigned long flags;
3575 
3576 	rc = 0;
3577 	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3578 	device = dasd_device_from_cdev_locked(cdev);
3579 	if (IS_ERR(device)) {
3580 		spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
3581 		return PTR_ERR(device);
3582 	}
3583 
3584 	/*
3585 	 * We must make sure that this device is currently not in use.
3586 	 * The open_count is increased for every opener, that includes
3587 	 * the blkdev_get in dasd_scan_partitions. We are only interested
3588 	 * in the other openers.
3589 	 */
3590 	if (device->block) {
3591 		max_count = device->block->bdev_handle ? 0 : -1;
3592 		open_count = atomic_read(&device->block->open_count);
3593 		if (open_count > max_count) {
3594 			if (open_count > 0)
3595 				pr_warn("%s: The DASD cannot be set offline with open count %i\n",
3596 					dev_name(&cdev->dev), open_count);
3597 			else
3598 				pr_warn("%s: The DASD cannot be set offline while it is in use\n",
3599 					dev_name(&cdev->dev));
3600 			rc = -EBUSY;
3601 			goto out_err;
3602 		}
3603 	}
3604 
3605 	/*
3606 	 * Test if the offline processing is already running and exit if so.
3607 	 * If a safe offline is being processed this could only be a normal
3608 	 * offline that should be able to overtake the safe offline and
3609 	 * cancel any I/O we do not want to wait for any longer
3610 	 */
3611 	if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
3612 		if (test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3613 			clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING,
3614 				  &device->flags);
3615 		} else {
3616 			rc = -EBUSY;
3617 			goto out_err;
3618 		}
3619 	}
3620 	set_bit(DASD_FLAG_OFFLINE, &device->flags);
3621 
3622 	/*
3623 	 * if safe_offline is called set safe_offline_running flag and
3624 	 * clear safe_offline so that a call to normal offline
3625 	 * can overrun safe_offline processing
3626 	 */
3627 	if (test_and_clear_bit(DASD_FLAG_SAFE_OFFLINE, &device->flags) &&
3628 	    !test_and_set_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3629 		/* need to unlock here to wait for outstanding I/O */
3630 		spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
3631 		/*
3632 		 * If we want to set the device safe offline all IO operations
3633 		 * should be finished before continuing the offline process
3634 		 * so sync bdev first and then wait for our queues to become
3635 		 * empty
3636 		 */
3637 		if (device->block && device->block->bdev_handle)
3638 			bdev_mark_dead(device->block->bdev_handle->bdev, false);
3639 		dasd_schedule_device_bh(device);
3640 		rc = wait_event_interruptible(shutdown_waitq,
3641 					      _wait_for_empty_queues(device));
3642 		if (rc != 0)
3643 			goto interrupted;
3644 
3645 		/*
3646 		 * check if a normal offline process overtook the offline
3647 		 * processing in this case simply do nothing beside returning
3648 		 * that we got interrupted
3649 		 * otherwise mark safe offline as not running any longer and
3650 		 * continue with normal offline
3651 		 */
3652 		spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3653 		if (!test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3654 			rc = -ERESTARTSYS;
3655 			goto out_err;
3656 		}
3657 		clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags);
3658 	}
3659 	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
3660 
3661 	dasd_set_target_state(device, DASD_STATE_NEW);
3662 	/* dasd_delete_device destroys the device reference. */
3663 	block = device->block;
3664 	dasd_delete_device(device);
3665 	/*
3666 	 * life cycle of block is bound to device, so delete it after
3667 	 * device was safely removed
3668 	 */
3669 	if (block)
3670 		dasd_free_block(block);
3671 
3672 	return 0;
3673 
3674 interrupted:
3675 	/* interrupted by signal */
3676 	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3677 	clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags);
3678 	clear_bit(DASD_FLAG_OFFLINE, &device->flags);
3679 out_err:
3680 	dasd_put_device(device);
3681 	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
3682 	return rc;
3683 }
3684 EXPORT_SYMBOL_GPL(dasd_generic_set_offline);
3685 
3686 int dasd_generic_last_path_gone(struct dasd_device *device)
3687 {
3688 	struct dasd_ccw_req *cqr;
3689 
3690 	dev_warn(&device->cdev->dev, "No operational channel path is left "
3691 		 "for the device\n");
3692 	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "last path gone");
3693 	/* First call extended error reporting and check for autoquiesce. */
3694 	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
3695 
3696 	if (device->state < DASD_STATE_BASIC)
3697 		return 0;
3698 	/* Device is active. We want to keep it. */
3699 	list_for_each_entry(cqr, &device->ccw_queue, devlist)
3700 		if ((cqr->status == DASD_CQR_IN_IO) ||
3701 		    (cqr->status == DASD_CQR_CLEAR_PENDING)) {
3702 			cqr->status = DASD_CQR_QUEUED;
3703 			cqr->retries++;
3704 		}
3705 	dasd_device_set_stop_bits(device, DASD_STOPPED_DC_WAIT);
3706 	dasd_device_clear_timer(device);
3707 	dasd_schedule_device_bh(device);
3708 	return 1;
3709 }
3710 EXPORT_SYMBOL_GPL(dasd_generic_last_path_gone);
3711 
3712 int dasd_generic_path_operational(struct dasd_device *device)
3713 {
3714 	dev_info(&device->cdev->dev, "A channel path to the device has become "
3715 		 "operational\n");
3716 	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "path operational");
3717 	dasd_device_remove_stop_bits(device, DASD_STOPPED_DC_WAIT);
3718 	dasd_schedule_device_bh(device);
3719 	if (device->block) {
3720 		dasd_schedule_block_bh(device->block);
3721 		if (device->block->gdp)
3722 			blk_mq_run_hw_queues(device->block->gdp->queue, true);
3723 	}
3724 
3725 	if (!device->stopped)
3726 		wake_up(&generic_waitq);
3727 
3728 	return 1;
3729 }
3730 EXPORT_SYMBOL_GPL(dasd_generic_path_operational);
3731 
3732 int dasd_generic_notify(struct ccw_device *cdev, int event)
3733 {
3734 	struct dasd_device *device;
3735 	int ret;
3736 
3737 	device = dasd_device_from_cdev_locked(cdev);
3738 	if (IS_ERR(device))
3739 		return 0;
3740 	ret = 0;
3741 	switch (event) {
3742 	case CIO_GONE:
3743 	case CIO_BOXED:
3744 	case CIO_NO_PATH:
3745 		dasd_path_no_path(device);
3746 		ret = dasd_generic_last_path_gone(device);
3747 		break;
3748 	case CIO_OPER:
3749 		ret = 1;
3750 		if (dasd_path_get_opm(device))
3751 			ret = dasd_generic_path_operational(device);
3752 		break;
3753 	}
3754 	dasd_put_device(device);
3755 	return ret;
3756 }
3757 EXPORT_SYMBOL_GPL(dasd_generic_notify);
3758 
3759 void dasd_generic_path_event(struct ccw_device *cdev, int *path_event)
3760 {
3761 	struct dasd_device *device;
3762 	int chp, oldopm, hpfpm, ifccpm;
3763 
3764 	device = dasd_device_from_cdev_locked(cdev);
3765 	if (IS_ERR(device))
3766 		return;
3767 
3768 	oldopm = dasd_path_get_opm(device);
3769 	for (chp = 0; chp < 8; chp++) {
3770 		if (path_event[chp] & PE_PATH_GONE) {
3771 			dasd_path_notoper(device, chp);
3772 		}
3773 		if (path_event[chp] & PE_PATH_AVAILABLE) {
3774 			dasd_path_available(device, chp);
3775 			dasd_schedule_device_bh(device);
3776 		}
3777 		if (path_event[chp] & PE_PATHGROUP_ESTABLISHED) {
3778 			if (!dasd_path_is_operational(device, chp) &&
3779 			    !dasd_path_need_verify(device, chp)) {
3780 				/*
3781 				 * we can not establish a pathgroup on an
3782 				 * unavailable path, so trigger a path
3783 				 * verification first
3784 				 */
3785 			dasd_path_available(device, chp);
3786 			dasd_schedule_device_bh(device);
3787 			}
3788 			DBF_DEV_EVENT(DBF_WARNING, device, "%s",
3789 				      "Pathgroup re-established\n");
3790 			if (device->discipline->kick_validate)
3791 				device->discipline->kick_validate(device);
3792 		}
3793 		if (path_event[chp] & PE_PATH_FCES_EVENT) {
3794 			dasd_path_fcsec_update(device, chp);
3795 			dasd_schedule_device_bh(device);
3796 		}
3797 	}
3798 	hpfpm = dasd_path_get_hpfpm(device);
3799 	ifccpm = dasd_path_get_ifccpm(device);
3800 	if (!dasd_path_get_opm(device) && hpfpm) {
3801 		/*
3802 		 * device has no operational paths but at least one path is
3803 		 * disabled due to HPF errors
3804 		 * disable HPF at all and use the path(s) again
3805 		 */
3806 		if (device->discipline->disable_hpf)
3807 			device->discipline->disable_hpf(device);
3808 		dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC);
3809 		dasd_path_set_tbvpm(device, hpfpm);
3810 		dasd_schedule_device_bh(device);
3811 		dasd_schedule_requeue(device);
3812 	} else if (!dasd_path_get_opm(device) && ifccpm) {
3813 		/*
3814 		 * device has no operational paths but at least one path is
3815 		 * disabled due to IFCC errors
3816 		 * trigger path verification on paths with IFCC errors
3817 		 */
3818 		dasd_path_set_tbvpm(device, ifccpm);
3819 		dasd_schedule_device_bh(device);
3820 	}
3821 	if (oldopm && !dasd_path_get_opm(device) && !hpfpm && !ifccpm) {
3822 		dev_warn(&device->cdev->dev,
3823 			 "No verified channel paths remain for the device\n");
3824 		DBF_DEV_EVENT(DBF_WARNING, device,
3825 			      "%s", "last verified path gone");
3826 		/* First call extended error reporting and check for autoquiesce. */
3827 		dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
3828 		dasd_device_set_stop_bits(device,
3829 					  DASD_STOPPED_DC_WAIT);
3830 	}
3831 	dasd_put_device(device);
3832 }
3833 EXPORT_SYMBOL_GPL(dasd_generic_path_event);
3834 
3835 int dasd_generic_verify_path(struct dasd_device *device, __u8 lpm)
3836 {
3837 	if (!dasd_path_get_opm(device) && lpm) {
3838 		dasd_path_set_opm(device, lpm);
3839 		dasd_generic_path_operational(device);
3840 	} else
3841 		dasd_path_add_opm(device, lpm);
3842 	return 0;
3843 }
3844 EXPORT_SYMBOL_GPL(dasd_generic_verify_path);
3845 
3846 void dasd_generic_space_exhaust(struct dasd_device *device,
3847 				struct dasd_ccw_req *cqr)
3848 {
3849 	/* First call extended error reporting and check for autoquiesce. */
3850 	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOSPC);
3851 
3852 	if (device->state < DASD_STATE_BASIC)
3853 		return;
3854 
3855 	if (cqr->status == DASD_CQR_IN_IO ||
3856 	    cqr->status == DASD_CQR_CLEAR_PENDING) {
3857 		cqr->status = DASD_CQR_QUEUED;
3858 		cqr->retries++;
3859 	}
3860 	dasd_device_set_stop_bits(device, DASD_STOPPED_NOSPC);
3861 	dasd_device_clear_timer(device);
3862 	dasd_schedule_device_bh(device);
3863 }
3864 EXPORT_SYMBOL_GPL(dasd_generic_space_exhaust);
3865 
3866 void dasd_generic_space_avail(struct dasd_device *device)
3867 {
3868 	dev_info(&device->cdev->dev, "Extent pool space is available\n");
3869 	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "space available");
3870 
3871 	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOSPC);
3872 	dasd_schedule_device_bh(device);
3873 
3874 	if (device->block) {
3875 		dasd_schedule_block_bh(device->block);
3876 		if (device->block->gdp)
3877 			blk_mq_run_hw_queues(device->block->gdp->queue, true);
3878 	}
3879 	if (!device->stopped)
3880 		wake_up(&generic_waitq);
3881 }
3882 EXPORT_SYMBOL_GPL(dasd_generic_space_avail);
3883 
3884 /*
3885  * clear active requests and requeue them to block layer if possible
3886  */
3887 int dasd_generic_requeue_all_requests(struct dasd_device *device)
3888 {
3889 	struct dasd_block *block = device->block;
3890 	struct list_head requeue_queue;
3891 	struct dasd_ccw_req *cqr, *n;
3892 	int rc;
3893 
3894 	if (!block)
3895 		return 0;
3896 
3897 	INIT_LIST_HEAD(&requeue_queue);
3898 	rc = _dasd_requests_to_flushqueue(block, &requeue_queue);
3899 
3900 	/* Now call the callback function of flushed requests */
3901 restart_cb:
3902 	list_for_each_entry_safe(cqr, n, &requeue_queue, blocklist) {
3903 		wait_event(dasd_flush_wq, (cqr->status < DASD_CQR_QUEUED));
3904 		/* Process finished ERP request. */
3905 		if (cqr->refers) {
3906 			spin_lock_bh(&block->queue_lock);
3907 			__dasd_process_erp(block->base, cqr);
3908 			spin_unlock_bh(&block->queue_lock);
3909 			/* restart list_for_xx loop since dasd_process_erp
3910 			 * might remove multiple elements
3911 			 */
3912 			goto restart_cb;
3913 		}
3914 		_dasd_requeue_request(cqr);
3915 		list_del_init(&cqr->blocklist);
3916 		cqr->block->base->discipline->free_cp(
3917 			cqr, (struct request *) cqr->callback_data);
3918 	}
3919 	dasd_schedule_device_bh(device);
3920 	return rc;
3921 }
3922 EXPORT_SYMBOL_GPL(dasd_generic_requeue_all_requests);
3923 
3924 static void do_requeue_requests(struct work_struct *work)
3925 {
3926 	struct dasd_device *device = container_of(work, struct dasd_device,
3927 						  requeue_requests);
3928 	dasd_generic_requeue_all_requests(device);
3929 	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOT_ACC);
3930 	if (device->block)
3931 		dasd_schedule_block_bh(device->block);
3932 	dasd_put_device(device);
3933 }
3934 
3935 void dasd_schedule_requeue(struct dasd_device *device)
3936 {
3937 	dasd_get_device(device);
3938 	/* queue call to dasd_reload_device to the kernel event daemon. */
3939 	if (!schedule_work(&device->requeue_requests))
3940 		dasd_put_device(device);
3941 }
3942 EXPORT_SYMBOL(dasd_schedule_requeue);
3943 
3944 static int dasd_handle_autoquiesce(struct dasd_device *device,
3945 				   struct dasd_ccw_req *cqr,
3946 				   unsigned int reason)
3947 {
3948 	/* in any case write eer message with reason */
3949 	if (dasd_eer_enabled(device))
3950 		dasd_eer_write(device, cqr, reason);
3951 
3952 	if (!test_bit(reason, &device->aq_mask))
3953 		return 0;
3954 
3955 	/* notify eer about autoquiesce */
3956 	if (dasd_eer_enabled(device))
3957 		dasd_eer_write(device, NULL, DASD_EER_AUTOQUIESCE);
3958 
3959 	pr_info("%s: The DASD has been put in the quiesce state\n",
3960 		dev_name(&device->cdev->dev));
3961 	dasd_device_set_stop_bits(device, DASD_STOPPED_QUIESCE);
3962 
3963 	if (device->features & DASD_FEATURE_REQUEUEQUIESCE)
3964 		dasd_schedule_requeue(device);
3965 
3966 	return 1;
3967 }
3968 
3969 static struct dasd_ccw_req *dasd_generic_build_rdc(struct dasd_device *device,
3970 						   int rdc_buffer_size,
3971 						   int magic)
3972 {
3973 	struct dasd_ccw_req *cqr;
3974 	struct ccw1 *ccw;
3975 
3976 	cqr = dasd_smalloc_request(magic, 1 /* RDC */, rdc_buffer_size, device,
3977 				   NULL);
3978 
3979 	if (IS_ERR(cqr)) {
3980 		/* internal error 13 - Allocating the RDC request failed*/
3981 		dev_err(&device->cdev->dev,
3982 			 "An error occurred in the DASD device driver, "
3983 			 "reason=%s\n", "13");
3984 		return cqr;
3985 	}
3986 
3987 	ccw = cqr->cpaddr;
3988 	ccw->cmd_code = CCW_CMD_RDC;
3989 	ccw->cda = (__u32)virt_to_phys(cqr->data);
3990 	ccw->flags = 0;
3991 	ccw->count = rdc_buffer_size;
3992 	cqr->startdev = device;
3993 	cqr->memdev = device;
3994 	cqr->expires = 10*HZ;
3995 	cqr->retries = 256;
3996 	cqr->buildclk = get_tod_clock();
3997 	cqr->status = DASD_CQR_FILLED;
3998 	return cqr;
3999 }
4000 
4001 
4002 int dasd_generic_read_dev_chars(struct dasd_device *device, int magic,
4003 				void *rdc_buffer, int rdc_buffer_size)
4004 {
4005 	int ret;
4006 	struct dasd_ccw_req *cqr;
4007 
4008 	cqr = dasd_generic_build_rdc(device, rdc_buffer_size, magic);
4009 	if (IS_ERR(cqr))
4010 		return PTR_ERR(cqr);
4011 
4012 	ret = dasd_sleep_on(cqr);
4013 	if (ret == 0)
4014 		memcpy(rdc_buffer, cqr->data, rdc_buffer_size);
4015 	dasd_sfree_request(cqr, cqr->memdev);
4016 	return ret;
4017 }
4018 EXPORT_SYMBOL_GPL(dasd_generic_read_dev_chars);
4019 
4020 /*
4021  *   In command mode and transport mode we need to look for sense
4022  *   data in different places. The sense data itself is allways
4023  *   an array of 32 bytes, so we can unify the sense data access
4024  *   for both modes.
4025  */
4026 char *dasd_get_sense(struct irb *irb)
4027 {
4028 	struct tsb *tsb = NULL;
4029 	char *sense = NULL;
4030 
4031 	if (scsw_is_tm(&irb->scsw) && (irb->scsw.tm.fcxs == 0x01)) {
4032 		if (irb->scsw.tm.tcw)
4033 			tsb = tcw_get_tsb(phys_to_virt(irb->scsw.tm.tcw));
4034 		if (tsb && tsb->length == 64 && tsb->flags)
4035 			switch (tsb->flags & 0x07) {
4036 			case 1:	/* tsa_iostat */
4037 				sense = tsb->tsa.iostat.sense;
4038 				break;
4039 			case 2: /* tsa_ddpc */
4040 				sense = tsb->tsa.ddpc.sense;
4041 				break;
4042 			default:
4043 				/* currently we don't use interrogate data */
4044 				break;
4045 			}
4046 	} else if (irb->esw.esw0.erw.cons) {
4047 		sense = irb->ecw;
4048 	}
4049 	return sense;
4050 }
4051 EXPORT_SYMBOL_GPL(dasd_get_sense);
4052 
4053 void dasd_generic_shutdown(struct ccw_device *cdev)
4054 {
4055 	struct dasd_device *device;
4056 
4057 	device = dasd_device_from_cdev(cdev);
4058 	if (IS_ERR(device))
4059 		return;
4060 
4061 	if (device->block)
4062 		dasd_schedule_block_bh(device->block);
4063 
4064 	dasd_schedule_device_bh(device);
4065 
4066 	wait_event(shutdown_waitq, _wait_for_empty_queues(device));
4067 }
4068 EXPORT_SYMBOL_GPL(dasd_generic_shutdown);
4069 
4070 static int __init dasd_init(void)
4071 {
4072 	int rc;
4073 
4074 	init_waitqueue_head(&dasd_init_waitq);
4075 	init_waitqueue_head(&dasd_flush_wq);
4076 	init_waitqueue_head(&generic_waitq);
4077 	init_waitqueue_head(&shutdown_waitq);
4078 
4079 	/* register 'common' DASD debug area, used for all DBF_XXX calls */
4080 	dasd_debug_area = debug_register("dasd", 1, 1, 8 * sizeof(long));
4081 	if (dasd_debug_area == NULL) {
4082 		rc = -ENOMEM;
4083 		goto failed;
4084 	}
4085 	debug_register_view(dasd_debug_area, &debug_sprintf_view);
4086 	debug_set_level(dasd_debug_area, DBF_WARNING);
4087 
4088 	DBF_EVENT(DBF_EMERG, "%s", "debug area created");
4089 
4090 	dasd_diag_discipline_pointer = NULL;
4091 
4092 	dasd_statistics_createroot();
4093 
4094 	rc = dasd_devmap_init();
4095 	if (rc)
4096 		goto failed;
4097 	rc = dasd_gendisk_init();
4098 	if (rc)
4099 		goto failed;
4100 	rc = dasd_parse();
4101 	if (rc)
4102 		goto failed;
4103 	rc = dasd_eer_init();
4104 	if (rc)
4105 		goto failed;
4106 #ifdef CONFIG_PROC_FS
4107 	rc = dasd_proc_init();
4108 	if (rc)
4109 		goto failed;
4110 #endif
4111 
4112 	return 0;
4113 failed:
4114 	pr_info("The DASD device driver could not be initialized\n");
4115 	dasd_exit();
4116 	return rc;
4117 }
4118 
4119 module_init(dasd_init);
4120 module_exit(dasd_exit);
4121