xref: /linux/block/bsg.c (revision bfd5bb6f90af092aa345b15cd78143956a13c2a8)
1 /*
2  * bsg.c - block layer implementation of the sg v4 interface
3  *
4  * Copyright (C) 2004 Jens Axboe <axboe@suse.de> SUSE Labs
5  * Copyright (C) 2004 Peter M. Jones <pjones@redhat.com>
6  *
7  *  This file is subject to the terms and conditions of the GNU General Public
8  *  License version 2.  See the file "COPYING" in the main directory of this
9  *  archive for more details.
10  *
11  */
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/file.h>
15 #include <linux/blkdev.h>
16 #include <linux/poll.h>
17 #include <linux/cdev.h>
18 #include <linux/jiffies.h>
19 #include <linux/percpu.h>
20 #include <linux/uio.h>
21 #include <linux/idr.h>
22 #include <linux/bsg.h>
23 #include <linux/slab.h>
24 
25 #include <scsi/scsi.h>
26 #include <scsi/scsi_ioctl.h>
27 #include <scsi/scsi_cmnd.h>
28 #include <scsi/scsi_device.h>
29 #include <scsi/scsi_driver.h>
30 #include <scsi/sg.h>
31 
32 #define BSG_DESCRIPTION	"Block layer SCSI generic (bsg) driver"
33 #define BSG_VERSION	"0.4"
34 
35 #define bsg_dbg(bd, fmt, ...) \
36 	pr_debug("%s: " fmt, (bd)->name, ##__VA_ARGS__)
37 
38 struct bsg_device {
39 	struct request_queue *queue;
40 	spinlock_t lock;
41 	struct list_head busy_list;
42 	struct list_head done_list;
43 	struct hlist_node dev_list;
44 	atomic_t ref_count;
45 	int queued_cmds;
46 	int done_cmds;
47 	wait_queue_head_t wq_done;
48 	wait_queue_head_t wq_free;
49 	char name[20];
50 	int max_queue;
51 	unsigned long flags;
52 };
53 
54 enum {
55 	BSG_F_BLOCK		= 1,
56 };
57 
58 #define BSG_DEFAULT_CMDS	64
59 #define BSG_MAX_DEVS		32768
60 
61 static DEFINE_MUTEX(bsg_mutex);
62 static DEFINE_IDR(bsg_minor_idr);
63 
64 #define BSG_LIST_ARRAY_SIZE	8
65 static struct hlist_head bsg_device_list[BSG_LIST_ARRAY_SIZE];
66 
67 static struct class *bsg_class;
68 static int bsg_major;
69 
70 static struct kmem_cache *bsg_cmd_cachep;
71 
72 /*
73  * our internal command type
74  */
75 struct bsg_command {
76 	struct bsg_device *bd;
77 	struct list_head list;
78 	struct request *rq;
79 	struct bio *bio;
80 	struct bio *bidi_bio;
81 	int err;
82 	struct sg_io_v4 hdr;
83 };
84 
85 static void bsg_free_command(struct bsg_command *bc)
86 {
87 	struct bsg_device *bd = bc->bd;
88 	unsigned long flags;
89 
90 	kmem_cache_free(bsg_cmd_cachep, bc);
91 
92 	spin_lock_irqsave(&bd->lock, flags);
93 	bd->queued_cmds--;
94 	spin_unlock_irqrestore(&bd->lock, flags);
95 
96 	wake_up(&bd->wq_free);
97 }
98 
99 static struct bsg_command *bsg_alloc_command(struct bsg_device *bd)
100 {
101 	struct bsg_command *bc = ERR_PTR(-EINVAL);
102 
103 	spin_lock_irq(&bd->lock);
104 
105 	if (bd->queued_cmds >= bd->max_queue)
106 		goto out;
107 
108 	bd->queued_cmds++;
109 	spin_unlock_irq(&bd->lock);
110 
111 	bc = kmem_cache_zalloc(bsg_cmd_cachep, GFP_KERNEL);
112 	if (unlikely(!bc)) {
113 		spin_lock_irq(&bd->lock);
114 		bd->queued_cmds--;
115 		bc = ERR_PTR(-ENOMEM);
116 		goto out;
117 	}
118 
119 	bc->bd = bd;
120 	INIT_LIST_HEAD(&bc->list);
121 	bsg_dbg(bd, "returning free cmd %p\n", bc);
122 	return bc;
123 out:
124 	spin_unlock_irq(&bd->lock);
125 	return bc;
126 }
127 
128 static inline struct hlist_head *bsg_dev_idx_hash(int index)
129 {
130 	return &bsg_device_list[index & (BSG_LIST_ARRAY_SIZE - 1)];
131 }
132 
133 #define uptr64(val) ((void __user *)(uintptr_t)(val))
134 
135 static int bsg_scsi_check_proto(struct sg_io_v4 *hdr)
136 {
137 	if (hdr->protocol != BSG_PROTOCOL_SCSI  ||
138 	    hdr->subprotocol != BSG_SUB_PROTOCOL_SCSI_CMD)
139 		return -EINVAL;
140 	return 0;
141 }
142 
143 static int bsg_scsi_fill_hdr(struct request *rq, struct sg_io_v4 *hdr,
144 		fmode_t mode)
145 {
146 	struct scsi_request *sreq = scsi_req(rq);
147 
148 	sreq->cmd_len = hdr->request_len;
149 	if (sreq->cmd_len > BLK_MAX_CDB) {
150 		sreq->cmd = kzalloc(sreq->cmd_len, GFP_KERNEL);
151 		if (!sreq->cmd)
152 			return -ENOMEM;
153 	}
154 
155 	if (copy_from_user(sreq->cmd, uptr64(hdr->request), sreq->cmd_len))
156 		return -EFAULT;
157 	if (blk_verify_command(sreq->cmd, mode))
158 		return -EPERM;
159 	return 0;
160 }
161 
162 static int bsg_scsi_complete_rq(struct request *rq, struct sg_io_v4 *hdr)
163 {
164 	struct scsi_request *sreq = scsi_req(rq);
165 	int ret = 0;
166 
167 	/*
168 	 * fill in all the output members
169 	 */
170 	hdr->device_status = sreq->result & 0xff;
171 	hdr->transport_status = host_byte(sreq->result);
172 	hdr->driver_status = driver_byte(sreq->result);
173 	hdr->info = 0;
174 	if (hdr->device_status || hdr->transport_status || hdr->driver_status)
175 		hdr->info |= SG_INFO_CHECK;
176 	hdr->response_len = 0;
177 
178 	if (sreq->sense_len && hdr->response) {
179 		int len = min_t(unsigned int, hdr->max_response_len,
180 					sreq->sense_len);
181 
182 		if (copy_to_user(uptr64(hdr->response), sreq->sense, len))
183 			ret = -EFAULT;
184 		else
185 			hdr->response_len = len;
186 	}
187 
188 	if (rq->next_rq) {
189 		hdr->dout_resid = sreq->resid_len;
190 		hdr->din_resid = scsi_req(rq->next_rq)->resid_len;
191 	} else if (rq_data_dir(rq) == READ) {
192 		hdr->din_resid = sreq->resid_len;
193 	} else {
194 		hdr->dout_resid = sreq->resid_len;
195 	}
196 
197 	return ret;
198 }
199 
200 static void bsg_scsi_free_rq(struct request *rq)
201 {
202 	scsi_req_free_cmd(scsi_req(rq));
203 }
204 
205 static const struct bsg_ops bsg_scsi_ops = {
206 	.check_proto		= bsg_scsi_check_proto,
207 	.fill_hdr		= bsg_scsi_fill_hdr,
208 	.complete_rq		= bsg_scsi_complete_rq,
209 	.free_rq		= bsg_scsi_free_rq,
210 };
211 
212 static struct request *
213 bsg_map_hdr(struct request_queue *q, struct sg_io_v4 *hdr, fmode_t mode)
214 {
215 	struct request *rq, *next_rq = NULL;
216 	int ret;
217 
218 	if (!q->bsg_dev.class_dev)
219 		return ERR_PTR(-ENXIO);
220 
221 	if (hdr->guard != 'Q')
222 		return ERR_PTR(-EINVAL);
223 
224 	ret = q->bsg_dev.ops->check_proto(hdr);
225 	if (ret)
226 		return ERR_PTR(ret);
227 
228 	rq = blk_get_request(q, hdr->dout_xfer_len ?
229 			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, 0);
230 	if (IS_ERR(rq))
231 		return rq;
232 
233 	ret = q->bsg_dev.ops->fill_hdr(rq, hdr, mode);
234 	if (ret)
235 		goto out;
236 
237 	rq->timeout = msecs_to_jiffies(hdr->timeout);
238 	if (!rq->timeout)
239 		rq->timeout = q->sg_timeout;
240 	if (!rq->timeout)
241 		rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
242 	if (rq->timeout < BLK_MIN_SG_TIMEOUT)
243 		rq->timeout = BLK_MIN_SG_TIMEOUT;
244 
245 	if (hdr->dout_xfer_len && hdr->din_xfer_len) {
246 		if (!test_bit(QUEUE_FLAG_BIDI, &q->queue_flags)) {
247 			ret = -EOPNOTSUPP;
248 			goto out;
249 		}
250 
251 		next_rq = blk_get_request(q, REQ_OP_SCSI_IN, 0);
252 		if (IS_ERR(next_rq)) {
253 			ret = PTR_ERR(next_rq);
254 			goto out;
255 		}
256 
257 		rq->next_rq = next_rq;
258 		ret = blk_rq_map_user(q, next_rq, NULL, uptr64(hdr->din_xferp),
259 				       hdr->din_xfer_len, GFP_KERNEL);
260 		if (ret)
261 			goto out_free_nextrq;
262 	}
263 
264 	if (hdr->dout_xfer_len) {
265 		ret = blk_rq_map_user(q, rq, NULL, uptr64(hdr->dout_xferp),
266 				hdr->dout_xfer_len, GFP_KERNEL);
267 	} else if (hdr->din_xfer_len) {
268 		ret = blk_rq_map_user(q, rq, NULL, uptr64(hdr->din_xferp),
269 				hdr->din_xfer_len, GFP_KERNEL);
270 	}
271 
272 	if (ret)
273 		goto out_unmap_nextrq;
274 	return rq;
275 
276 out_unmap_nextrq:
277 	if (rq->next_rq)
278 		blk_rq_unmap_user(rq->next_rq->bio);
279 out_free_nextrq:
280 	if (rq->next_rq)
281 		blk_put_request(rq->next_rq);
282 out:
283 	q->bsg_dev.ops->free_rq(rq);
284 	blk_put_request(rq);
285 	return ERR_PTR(ret);
286 }
287 
288 /*
289  * async completion call-back from the block layer, when scsi/ide/whatever
290  * calls end_that_request_last() on a request
291  */
292 static void bsg_rq_end_io(struct request *rq, blk_status_t status)
293 {
294 	struct bsg_command *bc = rq->end_io_data;
295 	struct bsg_device *bd = bc->bd;
296 	unsigned long flags;
297 
298 	bsg_dbg(bd, "finished rq %p bc %p, bio %p\n",
299 		rq, bc, bc->bio);
300 
301 	bc->hdr.duration = jiffies_to_msecs(jiffies - bc->hdr.duration);
302 
303 	spin_lock_irqsave(&bd->lock, flags);
304 	list_move_tail(&bc->list, &bd->done_list);
305 	bd->done_cmds++;
306 	spin_unlock_irqrestore(&bd->lock, flags);
307 
308 	wake_up(&bd->wq_done);
309 }
310 
311 /*
312  * do final setup of a 'bc' and submit the matching 'rq' to the block
313  * layer for io
314  */
315 static void bsg_add_command(struct bsg_device *bd, struct request_queue *q,
316 			    struct bsg_command *bc, struct request *rq)
317 {
318 	int at_head = (0 == (bc->hdr.flags & BSG_FLAG_Q_AT_TAIL));
319 
320 	/*
321 	 * add bc command to busy queue and submit rq for io
322 	 */
323 	bc->rq = rq;
324 	bc->bio = rq->bio;
325 	if (rq->next_rq)
326 		bc->bidi_bio = rq->next_rq->bio;
327 	bc->hdr.duration = jiffies;
328 	spin_lock_irq(&bd->lock);
329 	list_add_tail(&bc->list, &bd->busy_list);
330 	spin_unlock_irq(&bd->lock);
331 
332 	bsg_dbg(bd, "queueing rq %p, bc %p\n", rq, bc);
333 
334 	rq->end_io_data = bc;
335 	blk_execute_rq_nowait(q, NULL, rq, at_head, bsg_rq_end_io);
336 }
337 
338 static struct bsg_command *bsg_next_done_cmd(struct bsg_device *bd)
339 {
340 	struct bsg_command *bc = NULL;
341 
342 	spin_lock_irq(&bd->lock);
343 	if (bd->done_cmds) {
344 		bc = list_first_entry(&bd->done_list, struct bsg_command, list);
345 		list_del(&bc->list);
346 		bd->done_cmds--;
347 	}
348 	spin_unlock_irq(&bd->lock);
349 
350 	return bc;
351 }
352 
353 /*
354  * Get a finished command from the done list
355  */
356 static struct bsg_command *bsg_get_done_cmd(struct bsg_device *bd)
357 {
358 	struct bsg_command *bc;
359 	int ret;
360 
361 	do {
362 		bc = bsg_next_done_cmd(bd);
363 		if (bc)
364 			break;
365 
366 		if (!test_bit(BSG_F_BLOCK, &bd->flags)) {
367 			bc = ERR_PTR(-EAGAIN);
368 			break;
369 		}
370 
371 		ret = wait_event_interruptible(bd->wq_done, bd->done_cmds);
372 		if (ret) {
373 			bc = ERR_PTR(-ERESTARTSYS);
374 			break;
375 		}
376 	} while (1);
377 
378 	bsg_dbg(bd, "returning done %p\n", bc);
379 
380 	return bc;
381 }
382 
383 static int blk_complete_sgv4_hdr_rq(struct request *rq, struct sg_io_v4 *hdr,
384 				    struct bio *bio, struct bio *bidi_bio)
385 {
386 	int ret;
387 
388 	ret = rq->q->bsg_dev.ops->complete_rq(rq, hdr);
389 
390 	if (rq->next_rq) {
391 		blk_rq_unmap_user(bidi_bio);
392 		blk_put_request(rq->next_rq);
393 	}
394 
395 	blk_rq_unmap_user(bio);
396 	rq->q->bsg_dev.ops->free_rq(rq);
397 	blk_put_request(rq);
398 	return ret;
399 }
400 
401 static bool bsg_complete(struct bsg_device *bd)
402 {
403 	bool ret = false;
404 	bool spin;
405 
406 	do {
407 		spin_lock_irq(&bd->lock);
408 
409 		BUG_ON(bd->done_cmds > bd->queued_cmds);
410 
411 		/*
412 		 * All commands consumed.
413 		 */
414 		if (bd->done_cmds == bd->queued_cmds)
415 			ret = true;
416 
417 		spin = !test_bit(BSG_F_BLOCK, &bd->flags);
418 
419 		spin_unlock_irq(&bd->lock);
420 	} while (!ret && spin);
421 
422 	return ret;
423 }
424 
425 static int bsg_complete_all_commands(struct bsg_device *bd)
426 {
427 	struct bsg_command *bc;
428 	int ret, tret;
429 
430 	bsg_dbg(bd, "entered\n");
431 
432 	/*
433 	 * wait for all commands to complete
434 	 */
435 	io_wait_event(bd->wq_done, bsg_complete(bd));
436 
437 	/*
438 	 * discard done commands
439 	 */
440 	ret = 0;
441 	do {
442 		spin_lock_irq(&bd->lock);
443 		if (!bd->queued_cmds) {
444 			spin_unlock_irq(&bd->lock);
445 			break;
446 		}
447 		spin_unlock_irq(&bd->lock);
448 
449 		bc = bsg_get_done_cmd(bd);
450 		if (IS_ERR(bc))
451 			break;
452 
453 		tret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
454 						bc->bidi_bio);
455 		if (!ret)
456 			ret = tret;
457 
458 		bsg_free_command(bc);
459 	} while (1);
460 
461 	return ret;
462 }
463 
464 static int
465 __bsg_read(char __user *buf, size_t count, struct bsg_device *bd,
466 	   const struct iovec *iov, ssize_t *bytes_read)
467 {
468 	struct bsg_command *bc;
469 	int nr_commands, ret;
470 
471 	if (count % sizeof(struct sg_io_v4))
472 		return -EINVAL;
473 
474 	ret = 0;
475 	nr_commands = count / sizeof(struct sg_io_v4);
476 	while (nr_commands) {
477 		bc = bsg_get_done_cmd(bd);
478 		if (IS_ERR(bc)) {
479 			ret = PTR_ERR(bc);
480 			break;
481 		}
482 
483 		/*
484 		 * this is the only case where we need to copy data back
485 		 * after completing the request. so do that here,
486 		 * bsg_complete_work() cannot do that for us
487 		 */
488 		ret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
489 					       bc->bidi_bio);
490 
491 		if (copy_to_user(buf, &bc->hdr, sizeof(bc->hdr)))
492 			ret = -EFAULT;
493 
494 		bsg_free_command(bc);
495 
496 		if (ret)
497 			break;
498 
499 		buf += sizeof(struct sg_io_v4);
500 		*bytes_read += sizeof(struct sg_io_v4);
501 		nr_commands--;
502 	}
503 
504 	return ret;
505 }
506 
507 static inline void bsg_set_block(struct bsg_device *bd, struct file *file)
508 {
509 	if (file->f_flags & O_NONBLOCK)
510 		clear_bit(BSG_F_BLOCK, &bd->flags);
511 	else
512 		set_bit(BSG_F_BLOCK, &bd->flags);
513 }
514 
515 /*
516  * Check if the error is a "real" error that we should return.
517  */
518 static inline int err_block_err(int ret)
519 {
520 	if (ret && ret != -ENOSPC && ret != -ENODATA && ret != -EAGAIN)
521 		return 1;
522 
523 	return 0;
524 }
525 
526 static ssize_t
527 bsg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
528 {
529 	struct bsg_device *bd = file->private_data;
530 	int ret;
531 	ssize_t bytes_read;
532 
533 	bsg_dbg(bd, "read %zd bytes\n", count);
534 
535 	bsg_set_block(bd, file);
536 
537 	bytes_read = 0;
538 	ret = __bsg_read(buf, count, bd, NULL, &bytes_read);
539 	*ppos = bytes_read;
540 
541 	if (!bytes_read || err_block_err(ret))
542 		bytes_read = ret;
543 
544 	return bytes_read;
545 }
546 
547 static int __bsg_write(struct bsg_device *bd, const char __user *buf,
548 		       size_t count, ssize_t *bytes_written, fmode_t mode)
549 {
550 	struct bsg_command *bc;
551 	struct request *rq;
552 	int ret, nr_commands;
553 
554 	if (count % sizeof(struct sg_io_v4))
555 		return -EINVAL;
556 
557 	nr_commands = count / sizeof(struct sg_io_v4);
558 	rq = NULL;
559 	bc = NULL;
560 	ret = 0;
561 	while (nr_commands) {
562 		struct request_queue *q = bd->queue;
563 
564 		bc = bsg_alloc_command(bd);
565 		if (IS_ERR(bc)) {
566 			ret = PTR_ERR(bc);
567 			bc = NULL;
568 			break;
569 		}
570 
571 		if (copy_from_user(&bc->hdr, buf, sizeof(bc->hdr))) {
572 			ret = -EFAULT;
573 			break;
574 		}
575 
576 		/*
577 		 * get a request, fill in the blanks, and add to request queue
578 		 */
579 		rq = bsg_map_hdr(bd->queue, &bc->hdr, mode);
580 		if (IS_ERR(rq)) {
581 			ret = PTR_ERR(rq);
582 			rq = NULL;
583 			break;
584 		}
585 
586 		bsg_add_command(bd, q, bc, rq);
587 		bc = NULL;
588 		rq = NULL;
589 		nr_commands--;
590 		buf += sizeof(struct sg_io_v4);
591 		*bytes_written += sizeof(struct sg_io_v4);
592 	}
593 
594 	if (bc)
595 		bsg_free_command(bc);
596 
597 	return ret;
598 }
599 
600 static ssize_t
601 bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
602 {
603 	struct bsg_device *bd = file->private_data;
604 	ssize_t bytes_written;
605 	int ret;
606 
607 	bsg_dbg(bd, "write %zd bytes\n", count);
608 
609 	if (unlikely(uaccess_kernel()))
610 		return -EINVAL;
611 
612 	bsg_set_block(bd, file);
613 
614 	bytes_written = 0;
615 	ret = __bsg_write(bd, buf, count, &bytes_written, file->f_mode);
616 
617 	*ppos = bytes_written;
618 
619 	/*
620 	 * return bytes written on non-fatal errors
621 	 */
622 	if (!bytes_written || err_block_err(ret))
623 		bytes_written = ret;
624 
625 	bsg_dbg(bd, "returning %zd\n", bytes_written);
626 	return bytes_written;
627 }
628 
629 static struct bsg_device *bsg_alloc_device(void)
630 {
631 	struct bsg_device *bd;
632 
633 	bd = kzalloc(sizeof(struct bsg_device), GFP_KERNEL);
634 	if (unlikely(!bd))
635 		return NULL;
636 
637 	spin_lock_init(&bd->lock);
638 
639 	bd->max_queue = BSG_DEFAULT_CMDS;
640 
641 	INIT_LIST_HEAD(&bd->busy_list);
642 	INIT_LIST_HEAD(&bd->done_list);
643 	INIT_HLIST_NODE(&bd->dev_list);
644 
645 	init_waitqueue_head(&bd->wq_free);
646 	init_waitqueue_head(&bd->wq_done);
647 	return bd;
648 }
649 
650 static int bsg_put_device(struct bsg_device *bd)
651 {
652 	int ret = 0, do_free;
653 	struct request_queue *q = bd->queue;
654 
655 	mutex_lock(&bsg_mutex);
656 
657 	do_free = atomic_dec_and_test(&bd->ref_count);
658 	if (!do_free) {
659 		mutex_unlock(&bsg_mutex);
660 		goto out;
661 	}
662 
663 	hlist_del(&bd->dev_list);
664 	mutex_unlock(&bsg_mutex);
665 
666 	bsg_dbg(bd, "tearing down\n");
667 
668 	/*
669 	 * close can always block
670 	 */
671 	set_bit(BSG_F_BLOCK, &bd->flags);
672 
673 	/*
674 	 * correct error detection baddies here again. it's the responsibility
675 	 * of the app to properly reap commands before close() if it wants
676 	 * fool-proof error detection
677 	 */
678 	ret = bsg_complete_all_commands(bd);
679 
680 	kfree(bd);
681 out:
682 	if (do_free)
683 		blk_put_queue(q);
684 	return ret;
685 }
686 
687 static struct bsg_device *bsg_add_device(struct inode *inode,
688 					 struct request_queue *rq,
689 					 struct file *file)
690 {
691 	struct bsg_device *bd;
692 	unsigned char buf[32];
693 
694 	lockdep_assert_held(&bsg_mutex);
695 
696 	if (!blk_get_queue(rq))
697 		return ERR_PTR(-ENXIO);
698 
699 	bd = bsg_alloc_device();
700 	if (!bd) {
701 		blk_put_queue(rq);
702 		return ERR_PTR(-ENOMEM);
703 	}
704 
705 	bd->queue = rq;
706 
707 	bsg_set_block(bd, file);
708 
709 	atomic_set(&bd->ref_count, 1);
710 	hlist_add_head(&bd->dev_list, bsg_dev_idx_hash(iminor(inode)));
711 
712 	strncpy(bd->name, dev_name(rq->bsg_dev.class_dev), sizeof(bd->name) - 1);
713 	bsg_dbg(bd, "bound to <%s>, max queue %d\n",
714 		format_dev_t(buf, inode->i_rdev), bd->max_queue);
715 
716 	return bd;
717 }
718 
719 static struct bsg_device *__bsg_get_device(int minor, struct request_queue *q)
720 {
721 	struct bsg_device *bd;
722 
723 	lockdep_assert_held(&bsg_mutex);
724 
725 	hlist_for_each_entry(bd, bsg_dev_idx_hash(minor), dev_list) {
726 		if (bd->queue == q) {
727 			atomic_inc(&bd->ref_count);
728 			goto found;
729 		}
730 	}
731 	bd = NULL;
732 found:
733 	return bd;
734 }
735 
736 static struct bsg_device *bsg_get_device(struct inode *inode, struct file *file)
737 {
738 	struct bsg_device *bd;
739 	struct bsg_class_device *bcd;
740 
741 	/*
742 	 * find the class device
743 	 */
744 	mutex_lock(&bsg_mutex);
745 	bcd = idr_find(&bsg_minor_idr, iminor(inode));
746 
747 	if (!bcd) {
748 		bd = ERR_PTR(-ENODEV);
749 		goto out_unlock;
750 	}
751 
752 	bd = __bsg_get_device(iminor(inode), bcd->queue);
753 	if (!bd)
754 		bd = bsg_add_device(inode, bcd->queue, file);
755 
756 out_unlock:
757 	mutex_unlock(&bsg_mutex);
758 	return bd;
759 }
760 
761 static int bsg_open(struct inode *inode, struct file *file)
762 {
763 	struct bsg_device *bd;
764 
765 	bd = bsg_get_device(inode, file);
766 
767 	if (IS_ERR(bd))
768 		return PTR_ERR(bd);
769 
770 	file->private_data = bd;
771 	return 0;
772 }
773 
774 static int bsg_release(struct inode *inode, struct file *file)
775 {
776 	struct bsg_device *bd = file->private_data;
777 
778 	file->private_data = NULL;
779 	return bsg_put_device(bd);
780 }
781 
782 static __poll_t bsg_poll(struct file *file, poll_table *wait)
783 {
784 	struct bsg_device *bd = file->private_data;
785 	__poll_t mask = 0;
786 
787 	poll_wait(file, &bd->wq_done, wait);
788 	poll_wait(file, &bd->wq_free, wait);
789 
790 	spin_lock_irq(&bd->lock);
791 	if (!list_empty(&bd->done_list))
792 		mask |= EPOLLIN | EPOLLRDNORM;
793 	if (bd->queued_cmds < bd->max_queue)
794 		mask |= EPOLLOUT;
795 	spin_unlock_irq(&bd->lock);
796 
797 	return mask;
798 }
799 
800 static long bsg_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
801 {
802 	struct bsg_device *bd = file->private_data;
803 	int __user *uarg = (int __user *) arg;
804 	int ret;
805 
806 	switch (cmd) {
807 		/*
808 		 * our own ioctls
809 		 */
810 	case SG_GET_COMMAND_Q:
811 		return put_user(bd->max_queue, uarg);
812 	case SG_SET_COMMAND_Q: {
813 		int queue;
814 
815 		if (get_user(queue, uarg))
816 			return -EFAULT;
817 		if (queue < 1)
818 			return -EINVAL;
819 
820 		spin_lock_irq(&bd->lock);
821 		bd->max_queue = queue;
822 		spin_unlock_irq(&bd->lock);
823 		return 0;
824 	}
825 
826 	/*
827 	 * SCSI/sg ioctls
828 	 */
829 	case SG_GET_VERSION_NUM:
830 	case SCSI_IOCTL_GET_IDLUN:
831 	case SCSI_IOCTL_GET_BUS_NUMBER:
832 	case SG_SET_TIMEOUT:
833 	case SG_GET_TIMEOUT:
834 	case SG_GET_RESERVED_SIZE:
835 	case SG_SET_RESERVED_SIZE:
836 	case SG_EMULATED_HOST:
837 	case SCSI_IOCTL_SEND_COMMAND: {
838 		void __user *uarg = (void __user *) arg;
839 		return scsi_cmd_ioctl(bd->queue, NULL, file->f_mode, cmd, uarg);
840 	}
841 	case SG_IO: {
842 		struct request *rq;
843 		struct bio *bio, *bidi_bio = NULL;
844 		struct sg_io_v4 hdr;
845 		int at_head;
846 
847 		if (copy_from_user(&hdr, uarg, sizeof(hdr)))
848 			return -EFAULT;
849 
850 		rq = bsg_map_hdr(bd->queue, &hdr, file->f_mode);
851 		if (IS_ERR(rq))
852 			return PTR_ERR(rq);
853 
854 		bio = rq->bio;
855 		if (rq->next_rq)
856 			bidi_bio = rq->next_rq->bio;
857 
858 		at_head = (0 == (hdr.flags & BSG_FLAG_Q_AT_TAIL));
859 		blk_execute_rq(bd->queue, NULL, rq, at_head);
860 		ret = blk_complete_sgv4_hdr_rq(rq, &hdr, bio, bidi_bio);
861 
862 		if (copy_to_user(uarg, &hdr, sizeof(hdr)))
863 			return -EFAULT;
864 
865 		return ret;
866 	}
867 	default:
868 		return -ENOTTY;
869 	}
870 }
871 
872 static const struct file_operations bsg_fops = {
873 	.read		=	bsg_read,
874 	.write		=	bsg_write,
875 	.poll		=	bsg_poll,
876 	.open		=	bsg_open,
877 	.release	=	bsg_release,
878 	.unlocked_ioctl	=	bsg_ioctl,
879 	.owner		=	THIS_MODULE,
880 	.llseek		=	default_llseek,
881 };
882 
883 void bsg_unregister_queue(struct request_queue *q)
884 {
885 	struct bsg_class_device *bcd = &q->bsg_dev;
886 
887 	if (!bcd->class_dev)
888 		return;
889 
890 	mutex_lock(&bsg_mutex);
891 	idr_remove(&bsg_minor_idr, bcd->minor);
892 	if (q->kobj.sd)
893 		sysfs_remove_link(&q->kobj, "bsg");
894 	device_unregister(bcd->class_dev);
895 	bcd->class_dev = NULL;
896 	mutex_unlock(&bsg_mutex);
897 }
898 EXPORT_SYMBOL_GPL(bsg_unregister_queue);
899 
900 int bsg_register_queue(struct request_queue *q, struct device *parent,
901 		const char *name, const struct bsg_ops *ops)
902 {
903 	struct bsg_class_device *bcd;
904 	dev_t dev;
905 	int ret;
906 	struct device *class_dev = NULL;
907 
908 	/*
909 	 * we need a proper transport to send commands, not a stacked device
910 	 */
911 	if (!queue_is_rq_based(q))
912 		return 0;
913 
914 	bcd = &q->bsg_dev;
915 	memset(bcd, 0, sizeof(*bcd));
916 
917 	mutex_lock(&bsg_mutex);
918 
919 	ret = idr_alloc(&bsg_minor_idr, bcd, 0, BSG_MAX_DEVS, GFP_KERNEL);
920 	if (ret < 0) {
921 		if (ret == -ENOSPC) {
922 			printk(KERN_ERR "bsg: too many bsg devices\n");
923 			ret = -EINVAL;
924 		}
925 		goto unlock;
926 	}
927 
928 	bcd->minor = ret;
929 	bcd->queue = q;
930 	bcd->ops = ops;
931 	dev = MKDEV(bsg_major, bcd->minor);
932 	class_dev = device_create(bsg_class, parent, dev, NULL, "%s", name);
933 	if (IS_ERR(class_dev)) {
934 		ret = PTR_ERR(class_dev);
935 		goto idr_remove;
936 	}
937 	bcd->class_dev = class_dev;
938 
939 	if (q->kobj.sd) {
940 		ret = sysfs_create_link(&q->kobj, &bcd->class_dev->kobj, "bsg");
941 		if (ret)
942 			goto unregister_class_dev;
943 	}
944 
945 	mutex_unlock(&bsg_mutex);
946 	return 0;
947 
948 unregister_class_dev:
949 	device_unregister(class_dev);
950 idr_remove:
951 	idr_remove(&bsg_minor_idr, bcd->minor);
952 unlock:
953 	mutex_unlock(&bsg_mutex);
954 	return ret;
955 }
956 
957 int bsg_scsi_register_queue(struct request_queue *q, struct device *parent)
958 {
959 	if (!blk_queue_scsi_passthrough(q)) {
960 		WARN_ONCE(true, "Attempt to register a non-SCSI queue\n");
961 		return -EINVAL;
962 	}
963 
964 	return bsg_register_queue(q, parent, dev_name(parent), &bsg_scsi_ops);
965 }
966 EXPORT_SYMBOL_GPL(bsg_scsi_register_queue);
967 
968 static struct cdev bsg_cdev;
969 
970 static char *bsg_devnode(struct device *dev, umode_t *mode)
971 {
972 	return kasprintf(GFP_KERNEL, "bsg/%s", dev_name(dev));
973 }
974 
975 static int __init bsg_init(void)
976 {
977 	int ret, i;
978 	dev_t devid;
979 
980 	bsg_cmd_cachep = kmem_cache_create("bsg_cmd",
981 				sizeof(struct bsg_command), 0, 0, NULL);
982 	if (!bsg_cmd_cachep) {
983 		printk(KERN_ERR "bsg: failed creating slab cache\n");
984 		return -ENOMEM;
985 	}
986 
987 	for (i = 0; i < BSG_LIST_ARRAY_SIZE; i++)
988 		INIT_HLIST_HEAD(&bsg_device_list[i]);
989 
990 	bsg_class = class_create(THIS_MODULE, "bsg");
991 	if (IS_ERR(bsg_class)) {
992 		ret = PTR_ERR(bsg_class);
993 		goto destroy_kmemcache;
994 	}
995 	bsg_class->devnode = bsg_devnode;
996 
997 	ret = alloc_chrdev_region(&devid, 0, BSG_MAX_DEVS, "bsg");
998 	if (ret)
999 		goto destroy_bsg_class;
1000 
1001 	bsg_major = MAJOR(devid);
1002 
1003 	cdev_init(&bsg_cdev, &bsg_fops);
1004 	ret = cdev_add(&bsg_cdev, MKDEV(bsg_major, 0), BSG_MAX_DEVS);
1005 	if (ret)
1006 		goto unregister_chrdev;
1007 
1008 	printk(KERN_INFO BSG_DESCRIPTION " version " BSG_VERSION
1009 	       " loaded (major %d)\n", bsg_major);
1010 	return 0;
1011 unregister_chrdev:
1012 	unregister_chrdev_region(MKDEV(bsg_major, 0), BSG_MAX_DEVS);
1013 destroy_bsg_class:
1014 	class_destroy(bsg_class);
1015 destroy_kmemcache:
1016 	kmem_cache_destroy(bsg_cmd_cachep);
1017 	return ret;
1018 }
1019 
1020 MODULE_AUTHOR("Jens Axboe");
1021 MODULE_DESCRIPTION(BSG_DESCRIPTION);
1022 MODULE_LICENSE("GPL");
1023 
1024 device_initcall(bsg_init);
1025