1 // SPDX-License-Identifier: GPL-2.0-only 2 //#define DEBUG 3 #include <linux/spinlock.h> 4 #include <linux/slab.h> 5 #include <linux/blkdev.h> 6 #include <linux/hdreg.h> 7 #include <linux/module.h> 8 #include <linux/mutex.h> 9 #include <linux/interrupt.h> 10 #include <linux/virtio.h> 11 #include <linux/virtio_blk.h> 12 #include <linux/scatterlist.h> 13 #include <linux/string_helpers.h> 14 #include <linux/idr.h> 15 #include <linux/blk-mq.h> 16 #include <linux/blk-mq-virtio.h> 17 #include <linux/numa.h> 18 #include <uapi/linux/virtio_ring.h> 19 20 #define PART_BITS 4 21 #define VQ_NAME_LEN 16 22 #define MAX_DISCARD_SEGMENTS 256u 23 24 static int major; 25 static DEFINE_IDA(vd_index_ida); 26 27 static struct workqueue_struct *virtblk_wq; 28 29 struct virtio_blk_vq { 30 struct virtqueue *vq; 31 spinlock_t lock; 32 char name[VQ_NAME_LEN]; 33 } ____cacheline_aligned_in_smp; 34 35 struct virtio_blk { 36 /* 37 * This mutex must be held by anything that may run after 38 * virtblk_remove() sets vblk->vdev to NULL. 39 * 40 * blk-mq, virtqueue processing, and sysfs attribute code paths are 41 * shut down before vblk->vdev is set to NULL and therefore do not need 42 * to hold this mutex. 43 */ 44 struct mutex vdev_mutex; 45 struct virtio_device *vdev; 46 47 /* The disk structure for the kernel. */ 48 struct gendisk *disk; 49 50 /* Block layer tags. */ 51 struct blk_mq_tag_set tag_set; 52 53 /* Process context for config space updates */ 54 struct work_struct config_work; 55 56 /* 57 * Tracks references from block_device_operations open/release and 58 * virtio_driver probe/remove so this object can be freed once no 59 * longer in use. 60 */ 61 refcount_t refs; 62 63 /* What host tells us, plus 2 for header & tailer. */ 64 unsigned int sg_elems; 65 66 /* Ida index - used to track minor number allocations. */ 67 int index; 68 69 /* num of vqs */ 70 int num_vqs; 71 struct virtio_blk_vq *vqs; 72 }; 73 74 struct virtblk_req { 75 struct virtio_blk_outhdr out_hdr; 76 u8 status; 77 struct scatterlist sg[]; 78 }; 79 80 static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 81 { 82 switch (vbr->status) { 83 case VIRTIO_BLK_S_OK: 84 return BLK_STS_OK; 85 case VIRTIO_BLK_S_UNSUPP: 86 return BLK_STS_NOTSUPP; 87 default: 88 return BLK_STS_IOERR; 89 } 90 } 91 92 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, 93 struct scatterlist *data_sg, bool have_data) 94 { 95 struct scatterlist hdr, status, *sgs[3]; 96 unsigned int num_out = 0, num_in = 0; 97 98 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 99 sgs[num_out++] = &hdr; 100 101 if (have_data) { 102 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) 103 sgs[num_out++] = data_sg; 104 else 105 sgs[num_out + num_in++] = data_sg; 106 } 107 108 sg_init_one(&status, &vbr->status, sizeof(vbr->status)); 109 sgs[num_out + num_in++] = &status; 110 111 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); 112 } 113 114 static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap) 115 { 116 unsigned short segments = blk_rq_nr_discard_segments(req); 117 unsigned short n = 0; 118 struct virtio_blk_discard_write_zeroes *range; 119 struct bio *bio; 120 u32 flags = 0; 121 122 if (unmap) 123 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; 124 125 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC); 126 if (!range) 127 return -ENOMEM; 128 129 __rq_for_each_bio(bio, req) { 130 u64 sector = bio->bi_iter.bi_sector; 131 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; 132 133 range[n].flags = cpu_to_le32(flags); 134 range[n].num_sectors = cpu_to_le32(num_sectors); 135 range[n].sector = cpu_to_le64(sector); 136 n++; 137 } 138 139 req->special_vec.bv_page = virt_to_page(range); 140 req->special_vec.bv_offset = offset_in_page(range); 141 req->special_vec.bv_len = sizeof(*range) * segments; 142 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 143 144 return 0; 145 } 146 147 static inline void virtblk_request_done(struct request *req) 148 { 149 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 150 151 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 152 kfree(page_address(req->special_vec.bv_page) + 153 req->special_vec.bv_offset); 154 } 155 156 blk_mq_end_request(req, virtblk_result(vbr)); 157 } 158 159 static void virtblk_done(struct virtqueue *vq) 160 { 161 struct virtio_blk *vblk = vq->vdev->priv; 162 bool req_done = false; 163 int qid = vq->index; 164 struct virtblk_req *vbr; 165 unsigned long flags; 166 unsigned int len; 167 168 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 169 do { 170 virtqueue_disable_cb(vq); 171 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { 172 struct request *req = blk_mq_rq_from_pdu(vbr); 173 174 if (likely(!blk_should_fake_timeout(req->q))) 175 blk_mq_complete_request(req); 176 req_done = true; 177 } 178 if (unlikely(virtqueue_is_broken(vq))) 179 break; 180 } while (!virtqueue_enable_cb(vq)); 181 182 /* In case queue is stopped waiting for more buffers. */ 183 if (req_done) 184 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); 185 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 186 } 187 188 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) 189 { 190 struct virtio_blk *vblk = hctx->queue->queuedata; 191 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; 192 bool kick; 193 194 spin_lock_irq(&vq->lock); 195 kick = virtqueue_kick_prepare(vq->vq); 196 spin_unlock_irq(&vq->lock); 197 198 if (kick) 199 virtqueue_notify(vq->vq); 200 } 201 202 static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 203 const struct blk_mq_queue_data *bd) 204 { 205 struct virtio_blk *vblk = hctx->queue->queuedata; 206 struct request *req = bd->rq; 207 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 208 unsigned long flags; 209 unsigned int num; 210 int qid = hctx->queue_num; 211 int err; 212 bool notify = false; 213 bool unmap = false; 214 u32 type; 215 216 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); 217 218 switch (req_op(req)) { 219 case REQ_OP_READ: 220 case REQ_OP_WRITE: 221 type = 0; 222 break; 223 case REQ_OP_FLUSH: 224 type = VIRTIO_BLK_T_FLUSH; 225 break; 226 case REQ_OP_DISCARD: 227 type = VIRTIO_BLK_T_DISCARD; 228 break; 229 case REQ_OP_WRITE_ZEROES: 230 type = VIRTIO_BLK_T_WRITE_ZEROES; 231 unmap = !(req->cmd_flags & REQ_NOUNMAP); 232 break; 233 case REQ_OP_DRV_IN: 234 type = VIRTIO_BLK_T_GET_ID; 235 break; 236 default: 237 WARN_ON_ONCE(1); 238 return BLK_STS_IOERR; 239 } 240 241 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type); 242 vbr->out_hdr.sector = type ? 243 0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req)); 244 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req)); 245 246 blk_mq_start_request(req); 247 248 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) { 249 err = virtblk_setup_discard_write_zeroes(req, unmap); 250 if (err) 251 return BLK_STS_RESOURCE; 252 } 253 254 num = blk_rq_map_sg(hctx->queue, req, vbr->sg); 255 if (num) { 256 if (rq_data_dir(req) == WRITE) 257 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT); 258 else 259 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN); 260 } 261 262 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 263 err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); 264 if (err) { 265 virtqueue_kick(vblk->vqs[qid].vq); 266 /* Don't stop the queue if -ENOMEM: we may have failed to 267 * bounce the buffer due to global resource outage. 268 */ 269 if (err == -ENOSPC) 270 blk_mq_stop_hw_queue(hctx); 271 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 272 switch (err) { 273 case -ENOSPC: 274 return BLK_STS_DEV_RESOURCE; 275 case -ENOMEM: 276 return BLK_STS_RESOURCE; 277 default: 278 return BLK_STS_IOERR; 279 } 280 } 281 282 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) 283 notify = true; 284 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); 285 286 if (notify) 287 virtqueue_notify(vblk->vqs[qid].vq); 288 return BLK_STS_OK; 289 } 290 291 /* return id (s/n) string for *disk to *id_str 292 */ 293 static int virtblk_get_id(struct gendisk *disk, char *id_str) 294 { 295 struct virtio_blk *vblk = disk->private_data; 296 struct request_queue *q = vblk->disk->queue; 297 struct request *req; 298 int err; 299 300 req = blk_get_request(q, REQ_OP_DRV_IN, 0); 301 if (IS_ERR(req)) 302 return PTR_ERR(req); 303 304 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); 305 if (err) 306 goto out; 307 308 blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); 309 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); 310 out: 311 blk_put_request(req); 312 return err; 313 } 314 315 static void virtblk_get(struct virtio_blk *vblk) 316 { 317 refcount_inc(&vblk->refs); 318 } 319 320 static void virtblk_put(struct virtio_blk *vblk) 321 { 322 if (refcount_dec_and_test(&vblk->refs)) { 323 ida_simple_remove(&vd_index_ida, vblk->index); 324 mutex_destroy(&vblk->vdev_mutex); 325 kfree(vblk); 326 } 327 } 328 329 static int virtblk_open(struct block_device *bd, fmode_t mode) 330 { 331 struct virtio_blk *vblk = bd->bd_disk->private_data; 332 int ret = 0; 333 334 mutex_lock(&vblk->vdev_mutex); 335 336 if (vblk->vdev) 337 virtblk_get(vblk); 338 else 339 ret = -ENXIO; 340 341 mutex_unlock(&vblk->vdev_mutex); 342 return ret; 343 } 344 345 static void virtblk_release(struct gendisk *disk, fmode_t mode) 346 { 347 struct virtio_blk *vblk = disk->private_data; 348 349 virtblk_put(vblk); 350 } 351 352 /* We provide getgeo only to please some old bootloader/partitioning tools */ 353 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) 354 { 355 struct virtio_blk *vblk = bd->bd_disk->private_data; 356 int ret = 0; 357 358 mutex_lock(&vblk->vdev_mutex); 359 360 if (!vblk->vdev) { 361 ret = -ENXIO; 362 goto out; 363 } 364 365 /* see if the host passed in geometry config */ 366 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { 367 virtio_cread(vblk->vdev, struct virtio_blk_config, 368 geometry.cylinders, &geo->cylinders); 369 virtio_cread(vblk->vdev, struct virtio_blk_config, 370 geometry.heads, &geo->heads); 371 virtio_cread(vblk->vdev, struct virtio_blk_config, 372 geometry.sectors, &geo->sectors); 373 } else { 374 /* some standard values, similar to sd */ 375 geo->heads = 1 << 6; 376 geo->sectors = 1 << 5; 377 geo->cylinders = get_capacity(bd->bd_disk) >> 11; 378 } 379 out: 380 mutex_unlock(&vblk->vdev_mutex); 381 return ret; 382 } 383 384 static const struct block_device_operations virtblk_fops = { 385 .owner = THIS_MODULE, 386 .open = virtblk_open, 387 .release = virtblk_release, 388 .getgeo = virtblk_getgeo, 389 }; 390 391 static int index_to_minor(int index) 392 { 393 return index << PART_BITS; 394 } 395 396 static int minor_to_index(int minor) 397 { 398 return minor >> PART_BITS; 399 } 400 401 static ssize_t serial_show(struct device *dev, 402 struct device_attribute *attr, char *buf) 403 { 404 struct gendisk *disk = dev_to_disk(dev); 405 int err; 406 407 /* sysfs gives us a PAGE_SIZE buffer */ 408 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES); 409 410 buf[VIRTIO_BLK_ID_BYTES] = '\0'; 411 err = virtblk_get_id(disk, buf); 412 if (!err) 413 return strlen(buf); 414 415 if (err == -EIO) /* Unsupported? Make it empty. */ 416 return 0; 417 418 return err; 419 } 420 421 static DEVICE_ATTR_RO(serial); 422 423 /* The queue's logical block size must be set before calling this */ 424 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize) 425 { 426 struct virtio_device *vdev = vblk->vdev; 427 struct request_queue *q = vblk->disk->queue; 428 char cap_str_2[10], cap_str_10[10]; 429 unsigned long long nblocks; 430 u64 capacity; 431 432 /* Host must always specify the capacity. */ 433 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity); 434 435 /* If capacity is too big, truncate with warning. */ 436 if ((sector_t)capacity != capacity) { 437 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n", 438 (unsigned long long)capacity); 439 capacity = (sector_t)-1; 440 } 441 442 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9); 443 444 string_get_size(nblocks, queue_logical_block_size(q), 445 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); 446 string_get_size(nblocks, queue_logical_block_size(q), 447 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); 448 449 dev_notice(&vdev->dev, 450 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n", 451 vblk->disk->disk_name, 452 resize ? "new size: " : "", 453 nblocks, 454 queue_logical_block_size(q), 455 cap_str_10, 456 cap_str_2); 457 458 set_capacity_revalidate_and_notify(vblk->disk, capacity, true); 459 } 460 461 static void virtblk_config_changed_work(struct work_struct *work) 462 { 463 struct virtio_blk *vblk = 464 container_of(work, struct virtio_blk, config_work); 465 466 virtblk_update_capacity(vblk, true); 467 } 468 469 static void virtblk_config_changed(struct virtio_device *vdev) 470 { 471 struct virtio_blk *vblk = vdev->priv; 472 473 queue_work(virtblk_wq, &vblk->config_work); 474 } 475 476 static int init_vq(struct virtio_blk *vblk) 477 { 478 int err; 479 int i; 480 vq_callback_t **callbacks; 481 const char **names; 482 struct virtqueue **vqs; 483 unsigned short num_vqs; 484 struct virtio_device *vdev = vblk->vdev; 485 struct irq_affinity desc = { 0, }; 486 487 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, 488 struct virtio_blk_config, num_queues, 489 &num_vqs); 490 if (err) 491 num_vqs = 1; 492 493 num_vqs = min_t(unsigned int, nr_cpu_ids, num_vqs); 494 495 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); 496 if (!vblk->vqs) 497 return -ENOMEM; 498 499 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); 500 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); 501 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); 502 if (!names || !callbacks || !vqs) { 503 err = -ENOMEM; 504 goto out; 505 } 506 507 for (i = 0; i < num_vqs; i++) { 508 callbacks[i] = virtblk_done; 509 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i); 510 names[i] = vblk->vqs[i].name; 511 } 512 513 /* Discover virtqueues and write information to configuration. */ 514 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc); 515 if (err) 516 goto out; 517 518 for (i = 0; i < num_vqs; i++) { 519 spin_lock_init(&vblk->vqs[i].lock); 520 vblk->vqs[i].vq = vqs[i]; 521 } 522 vblk->num_vqs = num_vqs; 523 524 out: 525 kfree(vqs); 526 kfree(callbacks); 527 kfree(names); 528 if (err) 529 kfree(vblk->vqs); 530 return err; 531 } 532 533 /* 534 * Legacy naming scheme used for virtio devices. We are stuck with it for 535 * virtio blk but don't ever use it for any new driver. 536 */ 537 static int virtblk_name_format(char *prefix, int index, char *buf, int buflen) 538 { 539 const int base = 'z' - 'a' + 1; 540 char *begin = buf + strlen(prefix); 541 char *end = buf + buflen; 542 char *p; 543 int unit; 544 545 p = end - 1; 546 *p = '\0'; 547 unit = base; 548 do { 549 if (p == begin) 550 return -EINVAL; 551 *--p = 'a' + (index % unit); 552 index = (index / unit) - 1; 553 } while (index >= 0); 554 555 memmove(begin, p, end - p); 556 memcpy(buf, prefix, strlen(prefix)); 557 558 return 0; 559 } 560 561 static int virtblk_get_cache_mode(struct virtio_device *vdev) 562 { 563 u8 writeback; 564 int err; 565 566 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE, 567 struct virtio_blk_config, wce, 568 &writeback); 569 570 /* 571 * If WCE is not configurable and flush is not available, 572 * assume no writeback cache is in use. 573 */ 574 if (err) 575 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH); 576 577 return writeback; 578 } 579 580 static void virtblk_update_cache_mode(struct virtio_device *vdev) 581 { 582 u8 writeback = virtblk_get_cache_mode(vdev); 583 struct virtio_blk *vblk = vdev->priv; 584 585 blk_queue_write_cache(vblk->disk->queue, writeback, false); 586 revalidate_disk(vblk->disk); 587 } 588 589 static const char *const virtblk_cache_types[] = { 590 "write through", "write back" 591 }; 592 593 static ssize_t 594 cache_type_store(struct device *dev, struct device_attribute *attr, 595 const char *buf, size_t count) 596 { 597 struct gendisk *disk = dev_to_disk(dev); 598 struct virtio_blk *vblk = disk->private_data; 599 struct virtio_device *vdev = vblk->vdev; 600 int i; 601 602 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE)); 603 i = sysfs_match_string(virtblk_cache_types, buf); 604 if (i < 0) 605 return i; 606 607 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i); 608 virtblk_update_cache_mode(vdev); 609 return count; 610 } 611 612 static ssize_t 613 cache_type_show(struct device *dev, struct device_attribute *attr, char *buf) 614 { 615 struct gendisk *disk = dev_to_disk(dev); 616 struct virtio_blk *vblk = disk->private_data; 617 u8 writeback = virtblk_get_cache_mode(vblk->vdev); 618 619 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types)); 620 return snprintf(buf, 40, "%s\n", virtblk_cache_types[writeback]); 621 } 622 623 static DEVICE_ATTR_RW(cache_type); 624 625 static struct attribute *virtblk_attrs[] = { 626 &dev_attr_serial.attr, 627 &dev_attr_cache_type.attr, 628 NULL, 629 }; 630 631 static umode_t virtblk_attrs_are_visible(struct kobject *kobj, 632 struct attribute *a, int n) 633 { 634 struct device *dev = container_of(kobj, struct device, kobj); 635 struct gendisk *disk = dev_to_disk(dev); 636 struct virtio_blk *vblk = disk->private_data; 637 struct virtio_device *vdev = vblk->vdev; 638 639 if (a == &dev_attr_cache_type.attr && 640 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) 641 return S_IRUGO; 642 643 return a->mode; 644 } 645 646 static const struct attribute_group virtblk_attr_group = { 647 .attrs = virtblk_attrs, 648 .is_visible = virtblk_attrs_are_visible, 649 }; 650 651 static const struct attribute_group *virtblk_attr_groups[] = { 652 &virtblk_attr_group, 653 NULL, 654 }; 655 656 static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, 657 unsigned int hctx_idx, unsigned int numa_node) 658 { 659 struct virtio_blk *vblk = set->driver_data; 660 struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq); 661 662 sg_init_table(vbr->sg, vblk->sg_elems); 663 return 0; 664 } 665 666 static int virtblk_map_queues(struct blk_mq_tag_set *set) 667 { 668 struct virtio_blk *vblk = set->driver_data; 669 670 return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT], 671 vblk->vdev, 0); 672 } 673 674 static const struct blk_mq_ops virtio_mq_ops = { 675 .queue_rq = virtio_queue_rq, 676 .commit_rqs = virtio_commit_rqs, 677 .complete = virtblk_request_done, 678 .init_request = virtblk_init_request, 679 .map_queues = virtblk_map_queues, 680 }; 681 682 static unsigned int virtblk_queue_depth; 683 module_param_named(queue_depth, virtblk_queue_depth, uint, 0444); 684 685 static int virtblk_probe(struct virtio_device *vdev) 686 { 687 struct virtio_blk *vblk; 688 struct request_queue *q; 689 int err, index; 690 691 u32 v, blk_size, max_size, sg_elems, opt_io_size; 692 u16 min_io_size; 693 u8 physical_block_exp, alignment_offset; 694 695 if (!vdev->config->get) { 696 dev_err(&vdev->dev, "%s failure: config access disabled\n", 697 __func__); 698 return -EINVAL; 699 } 700 701 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS), 702 GFP_KERNEL); 703 if (err < 0) 704 goto out; 705 index = err; 706 707 /* We need to know how many segments before we allocate. */ 708 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX, 709 struct virtio_blk_config, seg_max, 710 &sg_elems); 711 712 /* We need at least one SG element, whatever they say. */ 713 if (err || !sg_elems) 714 sg_elems = 1; 715 716 /* We need an extra sg elements at head and tail. */ 717 sg_elems += 2; 718 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL); 719 if (!vblk) { 720 err = -ENOMEM; 721 goto out_free_index; 722 } 723 724 /* This reference is dropped in virtblk_remove(). */ 725 refcount_set(&vblk->refs, 1); 726 mutex_init(&vblk->vdev_mutex); 727 728 vblk->vdev = vdev; 729 vblk->sg_elems = sg_elems; 730 731 INIT_WORK(&vblk->config_work, virtblk_config_changed_work); 732 733 err = init_vq(vblk); 734 if (err) 735 goto out_free_vblk; 736 737 /* FIXME: How many partitions? How long is a piece of string? */ 738 vblk->disk = alloc_disk(1 << PART_BITS); 739 if (!vblk->disk) { 740 err = -ENOMEM; 741 goto out_free_vq; 742 } 743 744 /* Default queue sizing is to fill the ring. */ 745 if (!virtblk_queue_depth) { 746 virtblk_queue_depth = vblk->vqs[0].vq->num_free; 747 /* ... but without indirect descs, we use 2 descs per req */ 748 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) 749 virtblk_queue_depth /= 2; 750 } 751 752 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); 753 vblk->tag_set.ops = &virtio_mq_ops; 754 vblk->tag_set.queue_depth = virtblk_queue_depth; 755 vblk->tag_set.numa_node = NUMA_NO_NODE; 756 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 757 vblk->tag_set.cmd_size = 758 sizeof(struct virtblk_req) + 759 sizeof(struct scatterlist) * sg_elems; 760 vblk->tag_set.driver_data = vblk; 761 vblk->tag_set.nr_hw_queues = vblk->num_vqs; 762 763 err = blk_mq_alloc_tag_set(&vblk->tag_set); 764 if (err) 765 goto out_put_disk; 766 767 q = blk_mq_init_queue(&vblk->tag_set); 768 if (IS_ERR(q)) { 769 err = -ENOMEM; 770 goto out_free_tags; 771 } 772 vblk->disk->queue = q; 773 774 q->queuedata = vblk; 775 776 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN); 777 778 vblk->disk->major = major; 779 vblk->disk->first_minor = index_to_minor(index); 780 vblk->disk->private_data = vblk; 781 vblk->disk->fops = &virtblk_fops; 782 vblk->disk->flags |= GENHD_FL_EXT_DEVT; 783 vblk->index = index; 784 785 /* configure queue flush support */ 786 virtblk_update_cache_mode(vdev); 787 788 /* If disk is read-only in the host, the guest should obey */ 789 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO)) 790 set_disk_ro(vblk->disk, 1); 791 792 /* We can handle whatever the host told us to handle. */ 793 blk_queue_max_segments(q, vblk->sg_elems-2); 794 795 /* No real sector limit. */ 796 blk_queue_max_hw_sectors(q, -1U); 797 798 max_size = virtio_max_dma_size(vdev); 799 800 /* Host can optionally specify maximum segment size and number of 801 * segments. */ 802 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX, 803 struct virtio_blk_config, size_max, &v); 804 if (!err) 805 max_size = min(max_size, v); 806 807 blk_queue_max_segment_size(q, max_size); 808 809 /* Host can optionally specify the block size of the device */ 810 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, 811 struct virtio_blk_config, blk_size, 812 &blk_size); 813 if (!err) 814 blk_queue_logical_block_size(q, blk_size); 815 else 816 blk_size = queue_logical_block_size(q); 817 818 /* Use topology information if available */ 819 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 820 struct virtio_blk_config, physical_block_exp, 821 &physical_block_exp); 822 if (!err && physical_block_exp) 823 blk_queue_physical_block_size(q, 824 blk_size * (1 << physical_block_exp)); 825 826 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 827 struct virtio_blk_config, alignment_offset, 828 &alignment_offset); 829 if (!err && alignment_offset) 830 blk_queue_alignment_offset(q, blk_size * alignment_offset); 831 832 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 833 struct virtio_blk_config, min_io_size, 834 &min_io_size); 835 if (!err && min_io_size) 836 blk_queue_io_min(q, blk_size * min_io_size); 837 838 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY, 839 struct virtio_blk_config, opt_io_size, 840 &opt_io_size); 841 if (!err && opt_io_size) 842 blk_queue_io_opt(q, blk_size * opt_io_size); 843 844 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) { 845 q->limits.discard_granularity = blk_size; 846 847 virtio_cread(vdev, struct virtio_blk_config, 848 discard_sector_alignment, &v); 849 q->limits.discard_alignment = v ? v << SECTOR_SHIFT : 0; 850 851 virtio_cread(vdev, struct virtio_blk_config, 852 max_discard_sectors, &v); 853 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX); 854 855 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg, 856 &v); 857 blk_queue_max_discard_segments(q, 858 min_not_zero(v, 859 MAX_DISCARD_SEGMENTS)); 860 861 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 862 } 863 864 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) { 865 virtio_cread(vdev, struct virtio_blk_config, 866 max_write_zeroes_sectors, &v); 867 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX); 868 } 869 870 virtblk_update_capacity(vblk, false); 871 virtio_device_ready(vdev); 872 873 device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); 874 return 0; 875 876 out_free_tags: 877 blk_mq_free_tag_set(&vblk->tag_set); 878 out_put_disk: 879 put_disk(vblk->disk); 880 out_free_vq: 881 vdev->config->del_vqs(vdev); 882 kfree(vblk->vqs); 883 out_free_vblk: 884 kfree(vblk); 885 out_free_index: 886 ida_simple_remove(&vd_index_ida, index); 887 out: 888 return err; 889 } 890 891 static void virtblk_remove(struct virtio_device *vdev) 892 { 893 struct virtio_blk *vblk = vdev->priv; 894 895 /* Make sure no work handler is accessing the device. */ 896 flush_work(&vblk->config_work); 897 898 del_gendisk(vblk->disk); 899 blk_cleanup_queue(vblk->disk->queue); 900 901 blk_mq_free_tag_set(&vblk->tag_set); 902 903 mutex_lock(&vblk->vdev_mutex); 904 905 /* Stop all the virtqueues. */ 906 vdev->config->reset(vdev); 907 908 /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */ 909 vblk->vdev = NULL; 910 911 put_disk(vblk->disk); 912 vdev->config->del_vqs(vdev); 913 kfree(vblk->vqs); 914 915 mutex_unlock(&vblk->vdev_mutex); 916 917 virtblk_put(vblk); 918 } 919 920 #ifdef CONFIG_PM_SLEEP 921 static int virtblk_freeze(struct virtio_device *vdev) 922 { 923 struct virtio_blk *vblk = vdev->priv; 924 925 /* Ensure we don't receive any more interrupts */ 926 vdev->config->reset(vdev); 927 928 /* Make sure no work handler is accessing the device. */ 929 flush_work(&vblk->config_work); 930 931 blk_mq_quiesce_queue(vblk->disk->queue); 932 933 vdev->config->del_vqs(vdev); 934 return 0; 935 } 936 937 static int virtblk_restore(struct virtio_device *vdev) 938 { 939 struct virtio_blk *vblk = vdev->priv; 940 int ret; 941 942 ret = init_vq(vdev->priv); 943 if (ret) 944 return ret; 945 946 virtio_device_ready(vdev); 947 948 blk_mq_unquiesce_queue(vblk->disk->queue); 949 return 0; 950 } 951 #endif 952 953 static const struct virtio_device_id id_table[] = { 954 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 955 { 0 }, 956 }; 957 958 static unsigned int features_legacy[] = { 959 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 960 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 961 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 962 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 963 } 964 ; 965 static unsigned int features[] = { 966 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 967 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, 968 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 969 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, 970 }; 971 972 static struct virtio_driver virtio_blk = { 973 .feature_table = features, 974 .feature_table_size = ARRAY_SIZE(features), 975 .feature_table_legacy = features_legacy, 976 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 977 .driver.name = KBUILD_MODNAME, 978 .driver.owner = THIS_MODULE, 979 .id_table = id_table, 980 .probe = virtblk_probe, 981 .remove = virtblk_remove, 982 .config_changed = virtblk_config_changed, 983 #ifdef CONFIG_PM_SLEEP 984 .freeze = virtblk_freeze, 985 .restore = virtblk_restore, 986 #endif 987 }; 988 989 static int __init init(void) 990 { 991 int error; 992 993 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0); 994 if (!virtblk_wq) 995 return -ENOMEM; 996 997 major = register_blkdev(0, "virtblk"); 998 if (major < 0) { 999 error = major; 1000 goto out_destroy_workqueue; 1001 } 1002 1003 error = register_virtio_driver(&virtio_blk); 1004 if (error) 1005 goto out_unregister_blkdev; 1006 return 0; 1007 1008 out_unregister_blkdev: 1009 unregister_blkdev(major, "virtblk"); 1010 out_destroy_workqueue: 1011 destroy_workqueue(virtblk_wq); 1012 return error; 1013 } 1014 1015 static void __exit fini(void) 1016 { 1017 unregister_virtio_driver(&virtio_blk); 1018 unregister_blkdev(major, "virtblk"); 1019 destroy_workqueue(virtblk_wq); 1020 } 1021 module_init(init); 1022 module_exit(fini); 1023 1024 MODULE_DEVICE_TABLE(virtio, id_table); 1025 MODULE_DESCRIPTION("Virtio block driver"); 1026 MODULE_LICENSE("GPL"); 1027