1 /* 2 * blkfront.c 3 * 4 * XenLinux virtual block device driver. 5 * 6 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand 7 * Modifications by Mark A. Williamson are (c) Intel Research Cambridge 8 * Copyright (c) 2004, Christian Limpach 9 * Copyright (c) 2004, Andrew Warfield 10 * Copyright (c) 2005, Christopher Clark 11 * Copyright (c) 2005, XenSource Ltd 12 * 13 * This program is free software; you can redistribute it and/or 14 * modify it under the terms of the GNU General Public License version 2 15 * as published by the Free Software Foundation; or, when distributed 16 * separately from the Linux kernel or incorporated into other 17 * software packages, subject to the following license: 18 * 19 * Permission is hereby granted, free of charge, to any person obtaining a copy 20 * of this source file (the "Software"), to deal in the Software without 21 * restriction, including without limitation the rights to use, copy, modify, 22 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 23 * and to permit persons to whom the Software is furnished to do so, subject to 24 * the following conditions: 25 * 26 * The above copyright notice and this permission notice shall be included in 27 * all copies or substantial portions of the Software. 28 * 29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 34 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 35 * IN THE SOFTWARE. 36 */ 37 38 #include <linux/interrupt.h> 39 #include <linux/blkdev.h> 40 #include <linux/blk-mq.h> 41 #include <linux/hdreg.h> 42 #include <linux/cdrom.h> 43 #include <linux/module.h> 44 #include <linux/slab.h> 45 #include <linux/mutex.h> 46 #include <linux/scatterlist.h> 47 #include <linux/bitmap.h> 48 #include <linux/list.h> 49 #include <linux/workqueue.h> 50 51 #include <xen/xen.h> 52 #include <xen/xenbus.h> 53 #include <xen/grant_table.h> 54 #include <xen/events.h> 55 #include <xen/page.h> 56 #include <xen/platform_pci.h> 57 58 #include <xen/interface/grant_table.h> 59 #include <xen/interface/io/blkif.h> 60 #include <xen/interface/io/protocols.h> 61 62 #include <asm/xen/hypervisor.h> 63 64 /* 65 * The minimal size of segment supported by the block framework is PAGE_SIZE. 66 * When Linux is using a different page size than Xen, it may not be possible 67 * to put all the data in a single segment. 68 * This can happen when the backend doesn't support indirect descriptor and 69 * therefore the maximum amount of data that a request can carry is 70 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE = 44KB 71 * 72 * Note that we only support one extra request. So the Linux page size 73 * should be <= ( 2 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) = 74 * 88KB. 75 */ 76 #define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE) 77 78 enum blkif_state { 79 BLKIF_STATE_DISCONNECTED, 80 BLKIF_STATE_CONNECTED, 81 BLKIF_STATE_SUSPENDED, 82 }; 83 84 struct grant { 85 grant_ref_t gref; 86 struct page *page; 87 struct list_head node; 88 }; 89 90 enum blk_req_status { 91 REQ_WAITING, 92 REQ_DONE, 93 REQ_ERROR, 94 REQ_EOPNOTSUPP, 95 }; 96 97 struct blk_shadow { 98 struct blkif_request req; 99 struct request *request; 100 struct grant **grants_used; 101 struct grant **indirect_grants; 102 struct scatterlist *sg; 103 unsigned int num_sg; 104 enum blk_req_status status; 105 106 #define NO_ASSOCIATED_ID ~0UL 107 /* 108 * Id of the sibling if we ever need 2 requests when handling a 109 * block I/O request 110 */ 111 unsigned long associated_id; 112 }; 113 114 struct blkif_req { 115 blk_status_t error; 116 }; 117 118 static inline struct blkif_req *blkif_req(struct request *rq) 119 { 120 return blk_mq_rq_to_pdu(rq); 121 } 122 123 static DEFINE_MUTEX(blkfront_mutex); 124 static const struct block_device_operations xlvbd_block_fops; 125 static struct delayed_work blkfront_work; 126 static LIST_HEAD(info_list); 127 128 /* 129 * Maximum number of segments in indirect requests, the actual value used by 130 * the frontend driver is the minimum of this value and the value provided 131 * by the backend driver. 132 */ 133 134 static unsigned int xen_blkif_max_segments = 32; 135 module_param_named(max_indirect_segments, xen_blkif_max_segments, uint, 0444); 136 MODULE_PARM_DESC(max_indirect_segments, 137 "Maximum amount of segments in indirect requests (default is 32)"); 138 139 static unsigned int xen_blkif_max_queues = 4; 140 module_param_named(max_queues, xen_blkif_max_queues, uint, 0444); 141 MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk"); 142 143 /* 144 * Maximum order of pages to be used for the shared ring between front and 145 * backend, 4KB page granularity is used. 146 */ 147 static unsigned int xen_blkif_max_ring_order; 148 module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444); 149 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring"); 150 151 #define BLK_RING_SIZE(info) \ 152 __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages) 153 154 #define BLK_MAX_RING_SIZE \ 155 __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS) 156 157 /* 158 * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19 159 * characters are enough. Define to 20 to keep consistent with backend. 160 */ 161 #define RINGREF_NAME_LEN (20) 162 /* 163 * queue-%u would take 7 + 10(UINT_MAX) = 17 characters. 164 */ 165 #define QUEUE_NAME_LEN (17) 166 167 /* 168 * Per-ring info. 169 * Every blkfront device can associate with one or more blkfront_ring_info, 170 * depending on how many hardware queues/rings to be used. 171 */ 172 struct blkfront_ring_info { 173 /* Lock to protect data in every ring buffer. */ 174 spinlock_t ring_lock; 175 struct blkif_front_ring ring; 176 unsigned int ring_ref[XENBUS_MAX_RING_GRANTS]; 177 unsigned int evtchn, irq; 178 struct work_struct work; 179 struct gnttab_free_callback callback; 180 struct blk_shadow shadow[BLK_MAX_RING_SIZE]; 181 struct list_head indirect_pages; 182 struct list_head grants; 183 unsigned int persistent_gnts_c; 184 unsigned long shadow_free; 185 struct blkfront_info *dev_info; 186 }; 187 188 /* 189 * We have one of these per vbd, whether ide, scsi or 'other'. They 190 * hang in private_data off the gendisk structure. We may end up 191 * putting all kinds of interesting stuff here :-) 192 */ 193 struct blkfront_info 194 { 195 struct mutex mutex; 196 struct xenbus_device *xbdev; 197 struct gendisk *gd; 198 u16 sector_size; 199 unsigned int physical_sector_size; 200 int vdevice; 201 blkif_vdev_t handle; 202 enum blkif_state connected; 203 /* Number of pages per ring buffer. */ 204 unsigned int nr_ring_pages; 205 struct request_queue *rq; 206 unsigned int feature_flush:1; 207 unsigned int feature_fua:1; 208 unsigned int feature_discard:1; 209 unsigned int feature_secdiscard:1; 210 unsigned int feature_persistent:1; 211 unsigned int discard_granularity; 212 unsigned int discard_alignment; 213 /* Number of 4KB segments handled */ 214 unsigned int max_indirect_segments; 215 int is_ready; 216 struct blk_mq_tag_set tag_set; 217 struct blkfront_ring_info *rinfo; 218 unsigned int nr_rings; 219 /* Save uncomplete reqs and bios for migration. */ 220 struct list_head requests; 221 struct bio_list bio_list; 222 struct list_head info_list; 223 }; 224 225 static unsigned int nr_minors; 226 static unsigned long *minors; 227 static DEFINE_SPINLOCK(minor_lock); 228 229 #define GRANT_INVALID_REF 0 230 231 #define PARTS_PER_DISK 16 232 #define PARTS_PER_EXT_DISK 256 233 234 #define BLKIF_MAJOR(dev) ((dev)>>8) 235 #define BLKIF_MINOR(dev) ((dev) & 0xff) 236 237 #define EXT_SHIFT 28 238 #define EXTENDED (1<<EXT_SHIFT) 239 #define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED)) 240 #define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED)) 241 #define EMULATED_HD_DISK_MINOR_OFFSET (0) 242 #define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256) 243 #define EMULATED_SD_DISK_MINOR_OFFSET (0) 244 #define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256) 245 246 #define DEV_NAME "xvd" /* name in /dev */ 247 248 /* 249 * Grants are always the same size as a Xen page (i.e 4KB). 250 * A physical segment is always the same size as a Linux page. 251 * Number of grants per physical segment 252 */ 253 #define GRANTS_PER_PSEG (PAGE_SIZE / XEN_PAGE_SIZE) 254 255 #define GRANTS_PER_INDIRECT_FRAME \ 256 (XEN_PAGE_SIZE / sizeof(struct blkif_request_segment)) 257 258 #define INDIRECT_GREFS(_grants) \ 259 DIV_ROUND_UP(_grants, GRANTS_PER_INDIRECT_FRAME) 260 261 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo); 262 static void blkfront_gather_backend_features(struct blkfront_info *info); 263 static int negotiate_mq(struct blkfront_info *info); 264 265 static int get_id_from_freelist(struct blkfront_ring_info *rinfo) 266 { 267 unsigned long free = rinfo->shadow_free; 268 269 BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info)); 270 rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id; 271 rinfo->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */ 272 return free; 273 } 274 275 static int add_id_to_freelist(struct blkfront_ring_info *rinfo, 276 unsigned long id) 277 { 278 if (rinfo->shadow[id].req.u.rw.id != id) 279 return -EINVAL; 280 if (rinfo->shadow[id].request == NULL) 281 return -EINVAL; 282 rinfo->shadow[id].req.u.rw.id = rinfo->shadow_free; 283 rinfo->shadow[id].request = NULL; 284 rinfo->shadow_free = id; 285 return 0; 286 } 287 288 static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num) 289 { 290 struct blkfront_info *info = rinfo->dev_info; 291 struct page *granted_page; 292 struct grant *gnt_list_entry, *n; 293 int i = 0; 294 295 while (i < num) { 296 gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO); 297 if (!gnt_list_entry) 298 goto out_of_memory; 299 300 if (info->feature_persistent) { 301 granted_page = alloc_page(GFP_NOIO); 302 if (!granted_page) { 303 kfree(gnt_list_entry); 304 goto out_of_memory; 305 } 306 gnt_list_entry->page = granted_page; 307 } 308 309 gnt_list_entry->gref = GRANT_INVALID_REF; 310 list_add(&gnt_list_entry->node, &rinfo->grants); 311 i++; 312 } 313 314 return 0; 315 316 out_of_memory: 317 list_for_each_entry_safe(gnt_list_entry, n, 318 &rinfo->grants, node) { 319 list_del(&gnt_list_entry->node); 320 if (info->feature_persistent) 321 __free_page(gnt_list_entry->page); 322 kfree(gnt_list_entry); 323 i--; 324 } 325 BUG_ON(i != 0); 326 return -ENOMEM; 327 } 328 329 static struct grant *get_free_grant(struct blkfront_ring_info *rinfo) 330 { 331 struct grant *gnt_list_entry; 332 333 BUG_ON(list_empty(&rinfo->grants)); 334 gnt_list_entry = list_first_entry(&rinfo->grants, struct grant, 335 node); 336 list_del(&gnt_list_entry->node); 337 338 if (gnt_list_entry->gref != GRANT_INVALID_REF) 339 rinfo->persistent_gnts_c--; 340 341 return gnt_list_entry; 342 } 343 344 static inline void grant_foreign_access(const struct grant *gnt_list_entry, 345 const struct blkfront_info *info) 346 { 347 gnttab_page_grant_foreign_access_ref_one(gnt_list_entry->gref, 348 info->xbdev->otherend_id, 349 gnt_list_entry->page, 350 0); 351 } 352 353 static struct grant *get_grant(grant_ref_t *gref_head, 354 unsigned long gfn, 355 struct blkfront_ring_info *rinfo) 356 { 357 struct grant *gnt_list_entry = get_free_grant(rinfo); 358 struct blkfront_info *info = rinfo->dev_info; 359 360 if (gnt_list_entry->gref != GRANT_INVALID_REF) 361 return gnt_list_entry; 362 363 /* Assign a gref to this page */ 364 gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); 365 BUG_ON(gnt_list_entry->gref == -ENOSPC); 366 if (info->feature_persistent) 367 grant_foreign_access(gnt_list_entry, info); 368 else { 369 /* Grant access to the GFN passed by the caller */ 370 gnttab_grant_foreign_access_ref(gnt_list_entry->gref, 371 info->xbdev->otherend_id, 372 gfn, 0); 373 } 374 375 return gnt_list_entry; 376 } 377 378 static struct grant *get_indirect_grant(grant_ref_t *gref_head, 379 struct blkfront_ring_info *rinfo) 380 { 381 struct grant *gnt_list_entry = get_free_grant(rinfo); 382 struct blkfront_info *info = rinfo->dev_info; 383 384 if (gnt_list_entry->gref != GRANT_INVALID_REF) 385 return gnt_list_entry; 386 387 /* Assign a gref to this page */ 388 gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); 389 BUG_ON(gnt_list_entry->gref == -ENOSPC); 390 if (!info->feature_persistent) { 391 struct page *indirect_page; 392 393 /* Fetch a pre-allocated page to use for indirect grefs */ 394 BUG_ON(list_empty(&rinfo->indirect_pages)); 395 indirect_page = list_first_entry(&rinfo->indirect_pages, 396 struct page, lru); 397 list_del(&indirect_page->lru); 398 gnt_list_entry->page = indirect_page; 399 } 400 grant_foreign_access(gnt_list_entry, info); 401 402 return gnt_list_entry; 403 } 404 405 static const char *op_name(int op) 406 { 407 static const char *const names[] = { 408 [BLKIF_OP_READ] = "read", 409 [BLKIF_OP_WRITE] = "write", 410 [BLKIF_OP_WRITE_BARRIER] = "barrier", 411 [BLKIF_OP_FLUSH_DISKCACHE] = "flush", 412 [BLKIF_OP_DISCARD] = "discard" }; 413 414 if (op < 0 || op >= ARRAY_SIZE(names)) 415 return "unknown"; 416 417 if (!names[op]) 418 return "reserved"; 419 420 return names[op]; 421 } 422 static int xlbd_reserve_minors(unsigned int minor, unsigned int nr) 423 { 424 unsigned int end = minor + nr; 425 int rc; 426 427 if (end > nr_minors) { 428 unsigned long *bitmap, *old; 429 430 bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap), 431 GFP_KERNEL); 432 if (bitmap == NULL) 433 return -ENOMEM; 434 435 spin_lock(&minor_lock); 436 if (end > nr_minors) { 437 old = minors; 438 memcpy(bitmap, minors, 439 BITS_TO_LONGS(nr_minors) * sizeof(*bitmap)); 440 minors = bitmap; 441 nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG; 442 } else 443 old = bitmap; 444 spin_unlock(&minor_lock); 445 kfree(old); 446 } 447 448 spin_lock(&minor_lock); 449 if (find_next_bit(minors, end, minor) >= end) { 450 bitmap_set(minors, minor, nr); 451 rc = 0; 452 } else 453 rc = -EBUSY; 454 spin_unlock(&minor_lock); 455 456 return rc; 457 } 458 459 static void xlbd_release_minors(unsigned int minor, unsigned int nr) 460 { 461 unsigned int end = minor + nr; 462 463 BUG_ON(end > nr_minors); 464 spin_lock(&minor_lock); 465 bitmap_clear(minors, minor, nr); 466 spin_unlock(&minor_lock); 467 } 468 469 static void blkif_restart_queue_callback(void *arg) 470 { 471 struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg; 472 schedule_work(&rinfo->work); 473 } 474 475 static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg) 476 { 477 /* We don't have real geometry info, but let's at least return 478 values consistent with the size of the device */ 479 sector_t nsect = get_capacity(bd->bd_disk); 480 sector_t cylinders = nsect; 481 482 hg->heads = 0xff; 483 hg->sectors = 0x3f; 484 sector_div(cylinders, hg->heads * hg->sectors); 485 hg->cylinders = cylinders; 486 if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect) 487 hg->cylinders = 0xffff; 488 return 0; 489 } 490 491 static int blkif_ioctl(struct block_device *bdev, fmode_t mode, 492 unsigned command, unsigned long argument) 493 { 494 struct blkfront_info *info = bdev->bd_disk->private_data; 495 int i; 496 497 dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n", 498 command, (long)argument); 499 500 switch (command) { 501 case CDROMMULTISESSION: 502 dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n"); 503 for (i = 0; i < sizeof(struct cdrom_multisession); i++) 504 if (put_user(0, (char __user *)(argument + i))) 505 return -EFAULT; 506 return 0; 507 508 case CDROM_GET_CAPABILITY: { 509 struct gendisk *gd = info->gd; 510 if (gd->flags & GENHD_FL_CD) 511 return 0; 512 return -EINVAL; 513 } 514 515 default: 516 /*printk(KERN_ALERT "ioctl %08x not supported by Xen blkdev\n", 517 command);*/ 518 return -EINVAL; /* same return as native Linux */ 519 } 520 521 return 0; 522 } 523 524 static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo, 525 struct request *req, 526 struct blkif_request **ring_req) 527 { 528 unsigned long id; 529 530 *ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt); 531 rinfo->ring.req_prod_pvt++; 532 533 id = get_id_from_freelist(rinfo); 534 rinfo->shadow[id].request = req; 535 rinfo->shadow[id].status = REQ_WAITING; 536 rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID; 537 538 (*ring_req)->u.rw.id = id; 539 540 return id; 541 } 542 543 static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo) 544 { 545 struct blkfront_info *info = rinfo->dev_info; 546 struct blkif_request *ring_req; 547 unsigned long id; 548 549 /* Fill out a communications ring structure. */ 550 id = blkif_ring_get_request(rinfo, req, &ring_req); 551 552 ring_req->operation = BLKIF_OP_DISCARD; 553 ring_req->u.discard.nr_sectors = blk_rq_sectors(req); 554 ring_req->u.discard.id = id; 555 ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req); 556 if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard) 557 ring_req->u.discard.flag = BLKIF_DISCARD_SECURE; 558 else 559 ring_req->u.discard.flag = 0; 560 561 /* Keep a private copy so we can reissue requests when recovering. */ 562 rinfo->shadow[id].req = *ring_req; 563 564 return 0; 565 } 566 567 struct setup_rw_req { 568 unsigned int grant_idx; 569 struct blkif_request_segment *segments; 570 struct blkfront_ring_info *rinfo; 571 struct blkif_request *ring_req; 572 grant_ref_t gref_head; 573 unsigned int id; 574 /* Only used when persistent grant is used and it's a read request */ 575 bool need_copy; 576 unsigned int bvec_off; 577 char *bvec_data; 578 579 bool require_extra_req; 580 struct blkif_request *extra_ring_req; 581 }; 582 583 static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset, 584 unsigned int len, void *data) 585 { 586 struct setup_rw_req *setup = data; 587 int n, ref; 588 struct grant *gnt_list_entry; 589 unsigned int fsect, lsect; 590 /* Convenient aliases */ 591 unsigned int grant_idx = setup->grant_idx; 592 struct blkif_request *ring_req = setup->ring_req; 593 struct blkfront_ring_info *rinfo = setup->rinfo; 594 /* 595 * We always use the shadow of the first request to store the list 596 * of grant associated to the block I/O request. This made the 597 * completion more easy to handle even if the block I/O request is 598 * split. 599 */ 600 struct blk_shadow *shadow = &rinfo->shadow[setup->id]; 601 602 if (unlikely(setup->require_extra_req && 603 grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) { 604 /* 605 * We are using the second request, setup grant_idx 606 * to be the index of the segment array. 607 */ 608 grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST; 609 ring_req = setup->extra_ring_req; 610 } 611 612 if ((ring_req->operation == BLKIF_OP_INDIRECT) && 613 (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) { 614 if (setup->segments) 615 kunmap_atomic(setup->segments); 616 617 n = grant_idx / GRANTS_PER_INDIRECT_FRAME; 618 gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo); 619 shadow->indirect_grants[n] = gnt_list_entry; 620 setup->segments = kmap_atomic(gnt_list_entry->page); 621 ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref; 622 } 623 624 gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo); 625 ref = gnt_list_entry->gref; 626 /* 627 * All the grants are stored in the shadow of the first 628 * request. Therefore we have to use the global index. 629 */ 630 shadow->grants_used[setup->grant_idx] = gnt_list_entry; 631 632 if (setup->need_copy) { 633 void *shared_data; 634 635 shared_data = kmap_atomic(gnt_list_entry->page); 636 /* 637 * this does not wipe data stored outside the 638 * range sg->offset..sg->offset+sg->length. 639 * Therefore, blkback *could* see data from 640 * previous requests. This is OK as long as 641 * persistent grants are shared with just one 642 * domain. It may need refactoring if this 643 * changes 644 */ 645 memcpy(shared_data + offset, 646 setup->bvec_data + setup->bvec_off, 647 len); 648 649 kunmap_atomic(shared_data); 650 setup->bvec_off += len; 651 } 652 653 fsect = offset >> 9; 654 lsect = fsect + (len >> 9) - 1; 655 if (ring_req->operation != BLKIF_OP_INDIRECT) { 656 ring_req->u.rw.seg[grant_idx] = 657 (struct blkif_request_segment) { 658 .gref = ref, 659 .first_sect = fsect, 660 .last_sect = lsect }; 661 } else { 662 setup->segments[grant_idx % GRANTS_PER_INDIRECT_FRAME] = 663 (struct blkif_request_segment) { 664 .gref = ref, 665 .first_sect = fsect, 666 .last_sect = lsect }; 667 } 668 669 (setup->grant_idx)++; 670 } 671 672 static void blkif_setup_extra_req(struct blkif_request *first, 673 struct blkif_request *second) 674 { 675 uint16_t nr_segments = first->u.rw.nr_segments; 676 677 /* 678 * The second request is only present when the first request uses 679 * all its segments. It's always the continuity of the first one. 680 */ 681 first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST; 682 683 second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST; 684 second->u.rw.sector_number = first->u.rw.sector_number + 685 (BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512; 686 687 second->u.rw.handle = first->u.rw.handle; 688 second->operation = first->operation; 689 } 690 691 static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo) 692 { 693 struct blkfront_info *info = rinfo->dev_info; 694 struct blkif_request *ring_req, *extra_ring_req = NULL; 695 unsigned long id, extra_id = NO_ASSOCIATED_ID; 696 bool require_extra_req = false; 697 int i; 698 struct setup_rw_req setup = { 699 .grant_idx = 0, 700 .segments = NULL, 701 .rinfo = rinfo, 702 .need_copy = rq_data_dir(req) && info->feature_persistent, 703 }; 704 705 /* 706 * Used to store if we are able to queue the request by just using 707 * existing persistent grants, or if we have to get new grants, 708 * as there are not sufficiently many free. 709 */ 710 bool new_persistent_gnts = false; 711 struct scatterlist *sg; 712 int num_sg, max_grefs, num_grant; 713 714 max_grefs = req->nr_phys_segments * GRANTS_PER_PSEG; 715 if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST) 716 /* 717 * If we are using indirect segments we need to account 718 * for the indirect grefs used in the request. 719 */ 720 max_grefs += INDIRECT_GREFS(max_grefs); 721 722 /* Check if we have enough persistent grants to allocate a requests */ 723 if (rinfo->persistent_gnts_c < max_grefs) { 724 new_persistent_gnts = true; 725 726 if (gnttab_alloc_grant_references( 727 max_grefs - rinfo->persistent_gnts_c, 728 &setup.gref_head) < 0) { 729 gnttab_request_free_callback( 730 &rinfo->callback, 731 blkif_restart_queue_callback, 732 rinfo, 733 max_grefs - rinfo->persistent_gnts_c); 734 return 1; 735 } 736 } 737 738 /* Fill out a communications ring structure. */ 739 id = blkif_ring_get_request(rinfo, req, &ring_req); 740 741 num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg); 742 num_grant = 0; 743 /* Calculate the number of grant used */ 744 for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) 745 num_grant += gnttab_count_grant(sg->offset, sg->length); 746 747 require_extra_req = info->max_indirect_segments == 0 && 748 num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST; 749 BUG_ON(!HAS_EXTRA_REQ && require_extra_req); 750 751 rinfo->shadow[id].num_sg = num_sg; 752 if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST && 753 likely(!require_extra_req)) { 754 /* 755 * The indirect operation can only be a BLKIF_OP_READ or 756 * BLKIF_OP_WRITE 757 */ 758 BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA); 759 ring_req->operation = BLKIF_OP_INDIRECT; 760 ring_req->u.indirect.indirect_op = rq_data_dir(req) ? 761 BLKIF_OP_WRITE : BLKIF_OP_READ; 762 ring_req->u.indirect.sector_number = (blkif_sector_t)blk_rq_pos(req); 763 ring_req->u.indirect.handle = info->handle; 764 ring_req->u.indirect.nr_segments = num_grant; 765 } else { 766 ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req); 767 ring_req->u.rw.handle = info->handle; 768 ring_req->operation = rq_data_dir(req) ? 769 BLKIF_OP_WRITE : BLKIF_OP_READ; 770 if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) { 771 /* 772 * Ideally we can do an unordered flush-to-disk. 773 * In case the backend onlysupports barriers, use that. 774 * A barrier request a superset of FUA, so we can 775 * implement it the same way. (It's also a FLUSH+FUA, 776 * since it is guaranteed ordered WRT previous writes.) 777 */ 778 if (info->feature_flush && info->feature_fua) 779 ring_req->operation = 780 BLKIF_OP_WRITE_BARRIER; 781 else if (info->feature_flush) 782 ring_req->operation = 783 BLKIF_OP_FLUSH_DISKCACHE; 784 else 785 ring_req->operation = 0; 786 } 787 ring_req->u.rw.nr_segments = num_grant; 788 if (unlikely(require_extra_req)) { 789 extra_id = blkif_ring_get_request(rinfo, req, 790 &extra_ring_req); 791 /* 792 * Only the first request contains the scatter-gather 793 * list. 794 */ 795 rinfo->shadow[extra_id].num_sg = 0; 796 797 blkif_setup_extra_req(ring_req, extra_ring_req); 798 799 /* Link the 2 requests together */ 800 rinfo->shadow[extra_id].associated_id = id; 801 rinfo->shadow[id].associated_id = extra_id; 802 } 803 } 804 805 setup.ring_req = ring_req; 806 setup.id = id; 807 808 setup.require_extra_req = require_extra_req; 809 if (unlikely(require_extra_req)) 810 setup.extra_ring_req = extra_ring_req; 811 812 for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) { 813 BUG_ON(sg->offset + sg->length > PAGE_SIZE); 814 815 if (setup.need_copy) { 816 setup.bvec_off = sg->offset; 817 setup.bvec_data = kmap_atomic(sg_page(sg)); 818 } 819 820 gnttab_foreach_grant_in_range(sg_page(sg), 821 sg->offset, 822 sg->length, 823 blkif_setup_rw_req_grant, 824 &setup); 825 826 if (setup.need_copy) 827 kunmap_atomic(setup.bvec_data); 828 } 829 if (setup.segments) 830 kunmap_atomic(setup.segments); 831 832 /* Keep a private copy so we can reissue requests when recovering. */ 833 rinfo->shadow[id].req = *ring_req; 834 if (unlikely(require_extra_req)) 835 rinfo->shadow[extra_id].req = *extra_ring_req; 836 837 if (new_persistent_gnts) 838 gnttab_free_grant_references(setup.gref_head); 839 840 return 0; 841 } 842 843 /* 844 * Generate a Xen blkfront IO request from a blk layer request. Reads 845 * and writes are handled as expected. 846 * 847 * @req: a request struct 848 */ 849 static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo) 850 { 851 if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED)) 852 return 1; 853 854 if (unlikely(req_op(req) == REQ_OP_DISCARD || 855 req_op(req) == REQ_OP_SECURE_ERASE)) 856 return blkif_queue_discard_req(req, rinfo); 857 else 858 return blkif_queue_rw_req(req, rinfo); 859 } 860 861 static inline void flush_requests(struct blkfront_ring_info *rinfo) 862 { 863 int notify; 864 865 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify); 866 867 if (notify) 868 notify_remote_via_irq(rinfo->irq); 869 } 870 871 static inline bool blkif_request_flush_invalid(struct request *req, 872 struct blkfront_info *info) 873 { 874 return (blk_rq_is_passthrough(req) || 875 ((req_op(req) == REQ_OP_FLUSH) && 876 !info->feature_flush) || 877 ((req->cmd_flags & REQ_FUA) && 878 !info->feature_fua)); 879 } 880 881 static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx, 882 const struct blk_mq_queue_data *qd) 883 { 884 unsigned long flags; 885 int qid = hctx->queue_num; 886 struct blkfront_info *info = hctx->queue->queuedata; 887 struct blkfront_ring_info *rinfo = NULL; 888 889 BUG_ON(info->nr_rings <= qid); 890 rinfo = &info->rinfo[qid]; 891 blk_mq_start_request(qd->rq); 892 spin_lock_irqsave(&rinfo->ring_lock, flags); 893 if (RING_FULL(&rinfo->ring)) 894 goto out_busy; 895 896 if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info)) 897 goto out_err; 898 899 if (blkif_queue_request(qd->rq, rinfo)) 900 goto out_busy; 901 902 flush_requests(rinfo); 903 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 904 return BLK_STS_OK; 905 906 out_err: 907 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 908 return BLK_STS_IOERR; 909 910 out_busy: 911 blk_mq_stop_hw_queue(hctx); 912 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 913 return BLK_STS_DEV_RESOURCE; 914 } 915 916 static void blkif_complete_rq(struct request *rq) 917 { 918 blk_mq_end_request(rq, blkif_req(rq)->error); 919 } 920 921 static const struct blk_mq_ops blkfront_mq_ops = { 922 .queue_rq = blkif_queue_rq, 923 .complete = blkif_complete_rq, 924 }; 925 926 static void blkif_set_queue_limits(struct blkfront_info *info) 927 { 928 struct request_queue *rq = info->rq; 929 struct gendisk *gd = info->gd; 930 unsigned int segments = info->max_indirect_segments ? : 931 BLKIF_MAX_SEGMENTS_PER_REQUEST; 932 933 blk_queue_flag_set(QUEUE_FLAG_VIRT, rq); 934 935 if (info->feature_discard) { 936 blk_queue_flag_set(QUEUE_FLAG_DISCARD, rq); 937 blk_queue_max_discard_sectors(rq, get_capacity(gd)); 938 rq->limits.discard_granularity = info->discard_granularity; 939 rq->limits.discard_alignment = info->discard_alignment; 940 if (info->feature_secdiscard) 941 blk_queue_flag_set(QUEUE_FLAG_SECERASE, rq); 942 } 943 944 /* Hard sector size and max sectors impersonate the equiv. hardware. */ 945 blk_queue_logical_block_size(rq, info->sector_size); 946 blk_queue_physical_block_size(rq, info->physical_sector_size); 947 blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512); 948 949 /* Each segment in a request is up to an aligned page in size. */ 950 blk_queue_segment_boundary(rq, PAGE_SIZE - 1); 951 blk_queue_max_segment_size(rq, PAGE_SIZE); 952 953 /* Ensure a merged request will fit in a single I/O ring slot. */ 954 blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG); 955 956 /* Make sure buffer addresses are sector-aligned. */ 957 blk_queue_dma_alignment(rq, 511); 958 } 959 960 static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, 961 unsigned int physical_sector_size) 962 { 963 struct request_queue *rq; 964 struct blkfront_info *info = gd->private_data; 965 966 memset(&info->tag_set, 0, sizeof(info->tag_set)); 967 info->tag_set.ops = &blkfront_mq_ops; 968 info->tag_set.nr_hw_queues = info->nr_rings; 969 if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) { 970 /* 971 * When indirect descriptior is not supported, the I/O request 972 * will be split between multiple request in the ring. 973 * To avoid problems when sending the request, divide by 974 * 2 the depth of the queue. 975 */ 976 info->tag_set.queue_depth = BLK_RING_SIZE(info) / 2; 977 } else 978 info->tag_set.queue_depth = BLK_RING_SIZE(info); 979 info->tag_set.numa_node = NUMA_NO_NODE; 980 info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 981 info->tag_set.cmd_size = sizeof(struct blkif_req); 982 info->tag_set.driver_data = info; 983 984 if (blk_mq_alloc_tag_set(&info->tag_set)) 985 return -EINVAL; 986 rq = blk_mq_init_queue(&info->tag_set); 987 if (IS_ERR(rq)) { 988 blk_mq_free_tag_set(&info->tag_set); 989 return PTR_ERR(rq); 990 } 991 992 rq->queuedata = info; 993 info->rq = gd->queue = rq; 994 info->gd = gd; 995 info->sector_size = sector_size; 996 info->physical_sector_size = physical_sector_size; 997 blkif_set_queue_limits(info); 998 999 return 0; 1000 } 1001 1002 static const char *flush_info(struct blkfront_info *info) 1003 { 1004 if (info->feature_flush && info->feature_fua) 1005 return "barrier: enabled;"; 1006 else if (info->feature_flush) 1007 return "flush diskcache: enabled;"; 1008 else 1009 return "barrier or flush: disabled;"; 1010 } 1011 1012 static void xlvbd_flush(struct blkfront_info *info) 1013 { 1014 blk_queue_write_cache(info->rq, info->feature_flush ? true : false, 1015 info->feature_fua ? true : false); 1016 pr_info("blkfront: %s: %s %s %s %s %s\n", 1017 info->gd->disk_name, flush_info(info), 1018 "persistent grants:", info->feature_persistent ? 1019 "enabled;" : "disabled;", "indirect descriptors:", 1020 info->max_indirect_segments ? "enabled;" : "disabled;"); 1021 } 1022 1023 static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset) 1024 { 1025 int major; 1026 major = BLKIF_MAJOR(vdevice); 1027 *minor = BLKIF_MINOR(vdevice); 1028 switch (major) { 1029 case XEN_IDE0_MAJOR: 1030 *offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET; 1031 *minor = ((*minor / 64) * PARTS_PER_DISK) + 1032 EMULATED_HD_DISK_MINOR_OFFSET; 1033 break; 1034 case XEN_IDE1_MAJOR: 1035 *offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET; 1036 *minor = (((*minor / 64) + 2) * PARTS_PER_DISK) + 1037 EMULATED_HD_DISK_MINOR_OFFSET; 1038 break; 1039 case XEN_SCSI_DISK0_MAJOR: 1040 *offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET; 1041 *minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET; 1042 break; 1043 case XEN_SCSI_DISK1_MAJOR: 1044 case XEN_SCSI_DISK2_MAJOR: 1045 case XEN_SCSI_DISK3_MAJOR: 1046 case XEN_SCSI_DISK4_MAJOR: 1047 case XEN_SCSI_DISK5_MAJOR: 1048 case XEN_SCSI_DISK6_MAJOR: 1049 case XEN_SCSI_DISK7_MAJOR: 1050 *offset = (*minor / PARTS_PER_DISK) + 1051 ((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) + 1052 EMULATED_SD_DISK_NAME_OFFSET; 1053 *minor = *minor + 1054 ((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) + 1055 EMULATED_SD_DISK_MINOR_OFFSET; 1056 break; 1057 case XEN_SCSI_DISK8_MAJOR: 1058 case XEN_SCSI_DISK9_MAJOR: 1059 case XEN_SCSI_DISK10_MAJOR: 1060 case XEN_SCSI_DISK11_MAJOR: 1061 case XEN_SCSI_DISK12_MAJOR: 1062 case XEN_SCSI_DISK13_MAJOR: 1063 case XEN_SCSI_DISK14_MAJOR: 1064 case XEN_SCSI_DISK15_MAJOR: 1065 *offset = (*minor / PARTS_PER_DISK) + 1066 ((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) + 1067 EMULATED_SD_DISK_NAME_OFFSET; 1068 *minor = *minor + 1069 ((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) + 1070 EMULATED_SD_DISK_MINOR_OFFSET; 1071 break; 1072 case XENVBD_MAJOR: 1073 *offset = *minor / PARTS_PER_DISK; 1074 break; 1075 default: 1076 printk(KERN_WARNING "blkfront: your disk configuration is " 1077 "incorrect, please use an xvd device instead\n"); 1078 return -ENODEV; 1079 } 1080 return 0; 1081 } 1082 1083 static char *encode_disk_name(char *ptr, unsigned int n) 1084 { 1085 if (n >= 26) 1086 ptr = encode_disk_name(ptr, n / 26 - 1); 1087 *ptr = 'a' + n % 26; 1088 return ptr + 1; 1089 } 1090 1091 static int xlvbd_alloc_gendisk(blkif_sector_t capacity, 1092 struct blkfront_info *info, 1093 u16 vdisk_info, u16 sector_size, 1094 unsigned int physical_sector_size) 1095 { 1096 struct gendisk *gd; 1097 int nr_minors = 1; 1098 int err; 1099 unsigned int offset; 1100 int minor; 1101 int nr_parts; 1102 char *ptr; 1103 1104 BUG_ON(info->gd != NULL); 1105 BUG_ON(info->rq != NULL); 1106 1107 if ((info->vdevice>>EXT_SHIFT) > 1) { 1108 /* this is above the extended range; something is wrong */ 1109 printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice); 1110 return -ENODEV; 1111 } 1112 1113 if (!VDEV_IS_EXTENDED(info->vdevice)) { 1114 err = xen_translate_vdev(info->vdevice, &minor, &offset); 1115 if (err) 1116 return err; 1117 nr_parts = PARTS_PER_DISK; 1118 } else { 1119 minor = BLKIF_MINOR_EXT(info->vdevice); 1120 nr_parts = PARTS_PER_EXT_DISK; 1121 offset = minor / nr_parts; 1122 if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4) 1123 printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with " 1124 "emulated IDE disks,\n\t choose an xvd device name" 1125 "from xvde on\n", info->vdevice); 1126 } 1127 if (minor >> MINORBITS) { 1128 pr_warn("blkfront: %#x's minor (%#x) out of range; ignoring\n", 1129 info->vdevice, minor); 1130 return -ENODEV; 1131 } 1132 1133 if ((minor % nr_parts) == 0) 1134 nr_minors = nr_parts; 1135 1136 err = xlbd_reserve_minors(minor, nr_minors); 1137 if (err) 1138 goto out; 1139 err = -ENODEV; 1140 1141 gd = alloc_disk(nr_minors); 1142 if (gd == NULL) 1143 goto release; 1144 1145 strcpy(gd->disk_name, DEV_NAME); 1146 ptr = encode_disk_name(gd->disk_name + sizeof(DEV_NAME) - 1, offset); 1147 BUG_ON(ptr >= gd->disk_name + DISK_NAME_LEN); 1148 if (nr_minors > 1) 1149 *ptr = 0; 1150 else 1151 snprintf(ptr, gd->disk_name + DISK_NAME_LEN - ptr, 1152 "%d", minor & (nr_parts - 1)); 1153 1154 gd->major = XENVBD_MAJOR; 1155 gd->first_minor = minor; 1156 gd->fops = &xlvbd_block_fops; 1157 gd->private_data = info; 1158 set_capacity(gd, capacity); 1159 1160 if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size)) { 1161 del_gendisk(gd); 1162 goto release; 1163 } 1164 1165 xlvbd_flush(info); 1166 1167 if (vdisk_info & VDISK_READONLY) 1168 set_disk_ro(gd, 1); 1169 1170 if (vdisk_info & VDISK_REMOVABLE) 1171 gd->flags |= GENHD_FL_REMOVABLE; 1172 1173 if (vdisk_info & VDISK_CDROM) 1174 gd->flags |= GENHD_FL_CD; 1175 1176 return 0; 1177 1178 release: 1179 xlbd_release_minors(minor, nr_minors); 1180 out: 1181 return err; 1182 } 1183 1184 static void xlvbd_release_gendisk(struct blkfront_info *info) 1185 { 1186 unsigned int minor, nr_minors, i; 1187 1188 if (info->rq == NULL) 1189 return; 1190 1191 /* No more blkif_request(). */ 1192 blk_mq_stop_hw_queues(info->rq); 1193 1194 for (i = 0; i < info->nr_rings; i++) { 1195 struct blkfront_ring_info *rinfo = &info->rinfo[i]; 1196 1197 /* No more gnttab callback work. */ 1198 gnttab_cancel_free_callback(&rinfo->callback); 1199 1200 /* Flush gnttab callback work. Must be done with no locks held. */ 1201 flush_work(&rinfo->work); 1202 } 1203 1204 del_gendisk(info->gd); 1205 1206 minor = info->gd->first_minor; 1207 nr_minors = info->gd->minors; 1208 xlbd_release_minors(minor, nr_minors); 1209 1210 blk_cleanup_queue(info->rq); 1211 blk_mq_free_tag_set(&info->tag_set); 1212 info->rq = NULL; 1213 1214 put_disk(info->gd); 1215 info->gd = NULL; 1216 } 1217 1218 /* Already hold rinfo->ring_lock. */ 1219 static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo) 1220 { 1221 if (!RING_FULL(&rinfo->ring)) 1222 blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true); 1223 } 1224 1225 static void kick_pending_request_queues(struct blkfront_ring_info *rinfo) 1226 { 1227 unsigned long flags; 1228 1229 spin_lock_irqsave(&rinfo->ring_lock, flags); 1230 kick_pending_request_queues_locked(rinfo); 1231 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 1232 } 1233 1234 static void blkif_restart_queue(struct work_struct *work) 1235 { 1236 struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work); 1237 1238 if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED) 1239 kick_pending_request_queues(rinfo); 1240 } 1241 1242 static void blkif_free_ring(struct blkfront_ring_info *rinfo) 1243 { 1244 struct grant *persistent_gnt, *n; 1245 struct blkfront_info *info = rinfo->dev_info; 1246 int i, j, segs; 1247 1248 /* 1249 * Remove indirect pages, this only happens when using indirect 1250 * descriptors but not persistent grants 1251 */ 1252 if (!list_empty(&rinfo->indirect_pages)) { 1253 struct page *indirect_page, *n; 1254 1255 BUG_ON(info->feature_persistent); 1256 list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) { 1257 list_del(&indirect_page->lru); 1258 __free_page(indirect_page); 1259 } 1260 } 1261 1262 /* Remove all persistent grants. */ 1263 if (!list_empty(&rinfo->grants)) { 1264 list_for_each_entry_safe(persistent_gnt, n, 1265 &rinfo->grants, node) { 1266 list_del(&persistent_gnt->node); 1267 if (persistent_gnt->gref != GRANT_INVALID_REF) { 1268 gnttab_end_foreign_access(persistent_gnt->gref, 1269 0, 0UL); 1270 rinfo->persistent_gnts_c--; 1271 } 1272 if (info->feature_persistent) 1273 __free_page(persistent_gnt->page); 1274 kfree(persistent_gnt); 1275 } 1276 } 1277 BUG_ON(rinfo->persistent_gnts_c != 0); 1278 1279 for (i = 0; i < BLK_RING_SIZE(info); i++) { 1280 /* 1281 * Clear persistent grants present in requests already 1282 * on the shared ring 1283 */ 1284 if (!rinfo->shadow[i].request) 1285 goto free_shadow; 1286 1287 segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ? 1288 rinfo->shadow[i].req.u.indirect.nr_segments : 1289 rinfo->shadow[i].req.u.rw.nr_segments; 1290 for (j = 0; j < segs; j++) { 1291 persistent_gnt = rinfo->shadow[i].grants_used[j]; 1292 gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL); 1293 if (info->feature_persistent) 1294 __free_page(persistent_gnt->page); 1295 kfree(persistent_gnt); 1296 } 1297 1298 if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT) 1299 /* 1300 * If this is not an indirect operation don't try to 1301 * free indirect segments 1302 */ 1303 goto free_shadow; 1304 1305 for (j = 0; j < INDIRECT_GREFS(segs); j++) { 1306 persistent_gnt = rinfo->shadow[i].indirect_grants[j]; 1307 gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL); 1308 __free_page(persistent_gnt->page); 1309 kfree(persistent_gnt); 1310 } 1311 1312 free_shadow: 1313 kfree(rinfo->shadow[i].grants_used); 1314 rinfo->shadow[i].grants_used = NULL; 1315 kfree(rinfo->shadow[i].indirect_grants); 1316 rinfo->shadow[i].indirect_grants = NULL; 1317 kfree(rinfo->shadow[i].sg); 1318 rinfo->shadow[i].sg = NULL; 1319 } 1320 1321 /* No more gnttab callback work. */ 1322 gnttab_cancel_free_callback(&rinfo->callback); 1323 1324 /* Flush gnttab callback work. Must be done with no locks held. */ 1325 flush_work(&rinfo->work); 1326 1327 /* Free resources associated with old device channel. */ 1328 for (i = 0; i < info->nr_ring_pages; i++) { 1329 if (rinfo->ring_ref[i] != GRANT_INVALID_REF) { 1330 gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0); 1331 rinfo->ring_ref[i] = GRANT_INVALID_REF; 1332 } 1333 } 1334 free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE)); 1335 rinfo->ring.sring = NULL; 1336 1337 if (rinfo->irq) 1338 unbind_from_irqhandler(rinfo->irq, rinfo); 1339 rinfo->evtchn = rinfo->irq = 0; 1340 } 1341 1342 static void blkif_free(struct blkfront_info *info, int suspend) 1343 { 1344 unsigned int i; 1345 1346 /* Prevent new requests being issued until we fix things up. */ 1347 info->connected = suspend ? 1348 BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED; 1349 /* No more blkif_request(). */ 1350 if (info->rq) 1351 blk_mq_stop_hw_queues(info->rq); 1352 1353 for (i = 0; i < info->nr_rings; i++) 1354 blkif_free_ring(&info->rinfo[i]); 1355 1356 kfree(info->rinfo); 1357 info->rinfo = NULL; 1358 info->nr_rings = 0; 1359 } 1360 1361 struct copy_from_grant { 1362 const struct blk_shadow *s; 1363 unsigned int grant_idx; 1364 unsigned int bvec_offset; 1365 char *bvec_data; 1366 }; 1367 1368 static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset, 1369 unsigned int len, void *data) 1370 { 1371 struct copy_from_grant *info = data; 1372 char *shared_data; 1373 /* Convenient aliases */ 1374 const struct blk_shadow *s = info->s; 1375 1376 shared_data = kmap_atomic(s->grants_used[info->grant_idx]->page); 1377 1378 memcpy(info->bvec_data + info->bvec_offset, 1379 shared_data + offset, len); 1380 1381 info->bvec_offset += len; 1382 info->grant_idx++; 1383 1384 kunmap_atomic(shared_data); 1385 } 1386 1387 static enum blk_req_status blkif_rsp_to_req_status(int rsp) 1388 { 1389 switch (rsp) 1390 { 1391 case BLKIF_RSP_OKAY: 1392 return REQ_DONE; 1393 case BLKIF_RSP_EOPNOTSUPP: 1394 return REQ_EOPNOTSUPP; 1395 case BLKIF_RSP_ERROR: 1396 /* Fallthrough. */ 1397 default: 1398 return REQ_ERROR; 1399 } 1400 } 1401 1402 /* 1403 * Get the final status of the block request based on two ring response 1404 */ 1405 static int blkif_get_final_status(enum blk_req_status s1, 1406 enum blk_req_status s2) 1407 { 1408 BUG_ON(s1 == REQ_WAITING); 1409 BUG_ON(s2 == REQ_WAITING); 1410 1411 if (s1 == REQ_ERROR || s2 == REQ_ERROR) 1412 return BLKIF_RSP_ERROR; 1413 else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP) 1414 return BLKIF_RSP_EOPNOTSUPP; 1415 return BLKIF_RSP_OKAY; 1416 } 1417 1418 static bool blkif_completion(unsigned long *id, 1419 struct blkfront_ring_info *rinfo, 1420 struct blkif_response *bret) 1421 { 1422 int i = 0; 1423 struct scatterlist *sg; 1424 int num_sg, num_grant; 1425 struct blkfront_info *info = rinfo->dev_info; 1426 struct blk_shadow *s = &rinfo->shadow[*id]; 1427 struct copy_from_grant data = { 1428 .grant_idx = 0, 1429 }; 1430 1431 num_grant = s->req.operation == BLKIF_OP_INDIRECT ? 1432 s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments; 1433 1434 /* The I/O request may be split in two. */ 1435 if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) { 1436 struct blk_shadow *s2 = &rinfo->shadow[s->associated_id]; 1437 1438 /* Keep the status of the current response in shadow. */ 1439 s->status = blkif_rsp_to_req_status(bret->status); 1440 1441 /* Wait the second response if not yet here. */ 1442 if (s2->status == REQ_WAITING) 1443 return false; 1444 1445 bret->status = blkif_get_final_status(s->status, 1446 s2->status); 1447 1448 /* 1449 * All the grants is stored in the first shadow in order 1450 * to make the completion code simpler. 1451 */ 1452 num_grant += s2->req.u.rw.nr_segments; 1453 1454 /* 1455 * The two responses may not come in order. Only the 1456 * first request will store the scatter-gather list. 1457 */ 1458 if (s2->num_sg != 0) { 1459 /* Update "id" with the ID of the first response. */ 1460 *id = s->associated_id; 1461 s = s2; 1462 } 1463 1464 /* 1465 * We don't need anymore the second request, so recycling 1466 * it now. 1467 */ 1468 if (add_id_to_freelist(rinfo, s->associated_id)) 1469 WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n", 1470 info->gd->disk_name, s->associated_id); 1471 } 1472 1473 data.s = s; 1474 num_sg = s->num_sg; 1475 1476 if (bret->operation == BLKIF_OP_READ && info->feature_persistent) { 1477 for_each_sg(s->sg, sg, num_sg, i) { 1478 BUG_ON(sg->offset + sg->length > PAGE_SIZE); 1479 1480 data.bvec_offset = sg->offset; 1481 data.bvec_data = kmap_atomic(sg_page(sg)); 1482 1483 gnttab_foreach_grant_in_range(sg_page(sg), 1484 sg->offset, 1485 sg->length, 1486 blkif_copy_from_grant, 1487 &data); 1488 1489 kunmap_atomic(data.bvec_data); 1490 } 1491 } 1492 /* Add the persistent grant into the list of free grants */ 1493 for (i = 0; i < num_grant; i++) { 1494 if (gnttab_query_foreign_access(s->grants_used[i]->gref)) { 1495 /* 1496 * If the grant is still mapped by the backend (the 1497 * backend has chosen to make this grant persistent) 1498 * we add it at the head of the list, so it will be 1499 * reused first. 1500 */ 1501 if (!info->feature_persistent) 1502 pr_alert_ratelimited("backed has not unmapped grant: %u\n", 1503 s->grants_used[i]->gref); 1504 list_add(&s->grants_used[i]->node, &rinfo->grants); 1505 rinfo->persistent_gnts_c++; 1506 } else { 1507 /* 1508 * If the grant is not mapped by the backend we end the 1509 * foreign access and add it to the tail of the list, 1510 * so it will not be picked again unless we run out of 1511 * persistent grants. 1512 */ 1513 gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL); 1514 s->grants_used[i]->gref = GRANT_INVALID_REF; 1515 list_add_tail(&s->grants_used[i]->node, &rinfo->grants); 1516 } 1517 } 1518 if (s->req.operation == BLKIF_OP_INDIRECT) { 1519 for (i = 0; i < INDIRECT_GREFS(num_grant); i++) { 1520 if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) { 1521 if (!info->feature_persistent) 1522 pr_alert_ratelimited("backed has not unmapped grant: %u\n", 1523 s->indirect_grants[i]->gref); 1524 list_add(&s->indirect_grants[i]->node, &rinfo->grants); 1525 rinfo->persistent_gnts_c++; 1526 } else { 1527 struct page *indirect_page; 1528 1529 gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL); 1530 /* 1531 * Add the used indirect page back to the list of 1532 * available pages for indirect grefs. 1533 */ 1534 if (!info->feature_persistent) { 1535 indirect_page = s->indirect_grants[i]->page; 1536 list_add(&indirect_page->lru, &rinfo->indirect_pages); 1537 } 1538 s->indirect_grants[i]->gref = GRANT_INVALID_REF; 1539 list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants); 1540 } 1541 } 1542 } 1543 1544 return true; 1545 } 1546 1547 static irqreturn_t blkif_interrupt(int irq, void *dev_id) 1548 { 1549 struct request *req; 1550 struct blkif_response *bret; 1551 RING_IDX i, rp; 1552 unsigned long flags; 1553 struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id; 1554 struct blkfront_info *info = rinfo->dev_info; 1555 1556 if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) 1557 return IRQ_HANDLED; 1558 1559 spin_lock_irqsave(&rinfo->ring_lock, flags); 1560 again: 1561 rp = rinfo->ring.sring->rsp_prod; 1562 rmb(); /* Ensure we see queued responses up to 'rp'. */ 1563 1564 for (i = rinfo->ring.rsp_cons; i != rp; i++) { 1565 unsigned long id; 1566 1567 bret = RING_GET_RESPONSE(&rinfo->ring, i); 1568 id = bret->id; 1569 /* 1570 * The backend has messed up and given us an id that we would 1571 * never have given to it (we stamp it up to BLK_RING_SIZE - 1572 * look in get_id_from_freelist. 1573 */ 1574 if (id >= BLK_RING_SIZE(info)) { 1575 WARN(1, "%s: response to %s has incorrect id (%ld)\n", 1576 info->gd->disk_name, op_name(bret->operation), id); 1577 /* We can't safely get the 'struct request' as 1578 * the id is busted. */ 1579 continue; 1580 } 1581 req = rinfo->shadow[id].request; 1582 1583 if (bret->operation != BLKIF_OP_DISCARD) { 1584 /* 1585 * We may need to wait for an extra response if the 1586 * I/O request is split in 2 1587 */ 1588 if (!blkif_completion(&id, rinfo, bret)) 1589 continue; 1590 } 1591 1592 if (add_id_to_freelist(rinfo, id)) { 1593 WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n", 1594 info->gd->disk_name, op_name(bret->operation), id); 1595 continue; 1596 } 1597 1598 if (bret->status == BLKIF_RSP_OKAY) 1599 blkif_req(req)->error = BLK_STS_OK; 1600 else 1601 blkif_req(req)->error = BLK_STS_IOERR; 1602 1603 switch (bret->operation) { 1604 case BLKIF_OP_DISCARD: 1605 if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { 1606 struct request_queue *rq = info->rq; 1607 printk(KERN_WARNING "blkfront: %s: %s op failed\n", 1608 info->gd->disk_name, op_name(bret->operation)); 1609 blkif_req(req)->error = BLK_STS_NOTSUPP; 1610 info->feature_discard = 0; 1611 info->feature_secdiscard = 0; 1612 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, rq); 1613 blk_queue_flag_clear(QUEUE_FLAG_SECERASE, rq); 1614 } 1615 break; 1616 case BLKIF_OP_FLUSH_DISKCACHE: 1617 case BLKIF_OP_WRITE_BARRIER: 1618 if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { 1619 printk(KERN_WARNING "blkfront: %s: %s op failed\n", 1620 info->gd->disk_name, op_name(bret->operation)); 1621 blkif_req(req)->error = BLK_STS_NOTSUPP; 1622 } 1623 if (unlikely(bret->status == BLKIF_RSP_ERROR && 1624 rinfo->shadow[id].req.u.rw.nr_segments == 0)) { 1625 printk(KERN_WARNING "blkfront: %s: empty %s op failed\n", 1626 info->gd->disk_name, op_name(bret->operation)); 1627 blkif_req(req)->error = BLK_STS_NOTSUPP; 1628 } 1629 if (unlikely(blkif_req(req)->error)) { 1630 if (blkif_req(req)->error == BLK_STS_NOTSUPP) 1631 blkif_req(req)->error = BLK_STS_OK; 1632 info->feature_fua = 0; 1633 info->feature_flush = 0; 1634 xlvbd_flush(info); 1635 } 1636 /* fall through */ 1637 case BLKIF_OP_READ: 1638 case BLKIF_OP_WRITE: 1639 if (unlikely(bret->status != BLKIF_RSP_OKAY)) 1640 dev_dbg(&info->xbdev->dev, "Bad return from blkdev data " 1641 "request: %x\n", bret->status); 1642 1643 break; 1644 default: 1645 BUG(); 1646 } 1647 1648 blk_mq_complete_request(req); 1649 } 1650 1651 rinfo->ring.rsp_cons = i; 1652 1653 if (i != rinfo->ring.req_prod_pvt) { 1654 int more_to_do; 1655 RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do); 1656 if (more_to_do) 1657 goto again; 1658 } else 1659 rinfo->ring.sring->rsp_event = i + 1; 1660 1661 kick_pending_request_queues_locked(rinfo); 1662 1663 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 1664 1665 return IRQ_HANDLED; 1666 } 1667 1668 1669 static int setup_blkring(struct xenbus_device *dev, 1670 struct blkfront_ring_info *rinfo) 1671 { 1672 struct blkif_sring *sring; 1673 int err, i; 1674 struct blkfront_info *info = rinfo->dev_info; 1675 unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE; 1676 grant_ref_t gref[XENBUS_MAX_RING_GRANTS]; 1677 1678 for (i = 0; i < info->nr_ring_pages; i++) 1679 rinfo->ring_ref[i] = GRANT_INVALID_REF; 1680 1681 sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH, 1682 get_order(ring_size)); 1683 if (!sring) { 1684 xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring"); 1685 return -ENOMEM; 1686 } 1687 SHARED_RING_INIT(sring); 1688 FRONT_RING_INIT(&rinfo->ring, sring, ring_size); 1689 1690 err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref); 1691 if (err < 0) { 1692 free_pages((unsigned long)sring, get_order(ring_size)); 1693 rinfo->ring.sring = NULL; 1694 goto fail; 1695 } 1696 for (i = 0; i < info->nr_ring_pages; i++) 1697 rinfo->ring_ref[i] = gref[i]; 1698 1699 err = xenbus_alloc_evtchn(dev, &rinfo->evtchn); 1700 if (err) 1701 goto fail; 1702 1703 err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0, 1704 "blkif", rinfo); 1705 if (err <= 0) { 1706 xenbus_dev_fatal(dev, err, 1707 "bind_evtchn_to_irqhandler failed"); 1708 goto fail; 1709 } 1710 rinfo->irq = err; 1711 1712 return 0; 1713 fail: 1714 blkif_free(info, 0); 1715 return err; 1716 } 1717 1718 /* 1719 * Write out per-ring/queue nodes including ring-ref and event-channel, and each 1720 * ring buffer may have multi pages depending on ->nr_ring_pages. 1721 */ 1722 static int write_per_ring_nodes(struct xenbus_transaction xbt, 1723 struct blkfront_ring_info *rinfo, const char *dir) 1724 { 1725 int err; 1726 unsigned int i; 1727 const char *message = NULL; 1728 struct blkfront_info *info = rinfo->dev_info; 1729 1730 if (info->nr_ring_pages == 1) { 1731 err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]); 1732 if (err) { 1733 message = "writing ring-ref"; 1734 goto abort_transaction; 1735 } 1736 } else { 1737 for (i = 0; i < info->nr_ring_pages; i++) { 1738 char ring_ref_name[RINGREF_NAME_LEN]; 1739 1740 snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i); 1741 err = xenbus_printf(xbt, dir, ring_ref_name, 1742 "%u", rinfo->ring_ref[i]); 1743 if (err) { 1744 message = "writing ring-ref"; 1745 goto abort_transaction; 1746 } 1747 } 1748 } 1749 1750 err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn); 1751 if (err) { 1752 message = "writing event-channel"; 1753 goto abort_transaction; 1754 } 1755 1756 return 0; 1757 1758 abort_transaction: 1759 xenbus_transaction_end(xbt, 1); 1760 if (message) 1761 xenbus_dev_fatal(info->xbdev, err, "%s", message); 1762 1763 return err; 1764 } 1765 1766 static void free_info(struct blkfront_info *info) 1767 { 1768 list_del(&info->info_list); 1769 kfree(info); 1770 } 1771 1772 /* Common code used when first setting up, and when resuming. */ 1773 static int talk_to_blkback(struct xenbus_device *dev, 1774 struct blkfront_info *info) 1775 { 1776 const char *message = NULL; 1777 struct xenbus_transaction xbt; 1778 int err; 1779 unsigned int i, max_page_order; 1780 unsigned int ring_page_order; 1781 1782 if (!info) 1783 return -ENODEV; 1784 1785 max_page_order = xenbus_read_unsigned(info->xbdev->otherend, 1786 "max-ring-page-order", 0); 1787 ring_page_order = min(xen_blkif_max_ring_order, max_page_order); 1788 info->nr_ring_pages = 1 << ring_page_order; 1789 1790 err = negotiate_mq(info); 1791 if (err) 1792 goto destroy_blkring; 1793 1794 for (i = 0; i < info->nr_rings; i++) { 1795 struct blkfront_ring_info *rinfo = &info->rinfo[i]; 1796 1797 /* Create shared ring, alloc event channel. */ 1798 err = setup_blkring(dev, rinfo); 1799 if (err) 1800 goto destroy_blkring; 1801 } 1802 1803 again: 1804 err = xenbus_transaction_start(&xbt); 1805 if (err) { 1806 xenbus_dev_fatal(dev, err, "starting transaction"); 1807 goto destroy_blkring; 1808 } 1809 1810 if (info->nr_ring_pages > 1) { 1811 err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u", 1812 ring_page_order); 1813 if (err) { 1814 message = "writing ring-page-order"; 1815 goto abort_transaction; 1816 } 1817 } 1818 1819 /* We already got the number of queues/rings in _probe */ 1820 if (info->nr_rings == 1) { 1821 err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename); 1822 if (err) 1823 goto destroy_blkring; 1824 } else { 1825 char *path; 1826 size_t pathsize; 1827 1828 err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u", 1829 info->nr_rings); 1830 if (err) { 1831 message = "writing multi-queue-num-queues"; 1832 goto abort_transaction; 1833 } 1834 1835 pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN; 1836 path = kmalloc(pathsize, GFP_KERNEL); 1837 if (!path) { 1838 err = -ENOMEM; 1839 message = "ENOMEM while writing ring references"; 1840 goto abort_transaction; 1841 } 1842 1843 for (i = 0; i < info->nr_rings; i++) { 1844 memset(path, 0, pathsize); 1845 snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i); 1846 err = write_per_ring_nodes(xbt, &info->rinfo[i], path); 1847 if (err) { 1848 kfree(path); 1849 goto destroy_blkring; 1850 } 1851 } 1852 kfree(path); 1853 } 1854 err = xenbus_printf(xbt, dev->nodename, "protocol", "%s", 1855 XEN_IO_PROTO_ABI_NATIVE); 1856 if (err) { 1857 message = "writing protocol"; 1858 goto abort_transaction; 1859 } 1860 err = xenbus_printf(xbt, dev->nodename, 1861 "feature-persistent", "%u", 1); 1862 if (err) 1863 dev_warn(&dev->dev, 1864 "writing persistent grants feature to xenbus"); 1865 1866 err = xenbus_transaction_end(xbt, 0); 1867 if (err) { 1868 if (err == -EAGAIN) 1869 goto again; 1870 xenbus_dev_fatal(dev, err, "completing transaction"); 1871 goto destroy_blkring; 1872 } 1873 1874 for (i = 0; i < info->nr_rings; i++) { 1875 unsigned int j; 1876 struct blkfront_ring_info *rinfo = &info->rinfo[i]; 1877 1878 for (j = 0; j < BLK_RING_SIZE(info); j++) 1879 rinfo->shadow[j].req.u.rw.id = j + 1; 1880 rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff; 1881 } 1882 xenbus_switch_state(dev, XenbusStateInitialised); 1883 1884 return 0; 1885 1886 abort_transaction: 1887 xenbus_transaction_end(xbt, 1); 1888 if (message) 1889 xenbus_dev_fatal(dev, err, "%s", message); 1890 destroy_blkring: 1891 blkif_free(info, 0); 1892 1893 mutex_lock(&blkfront_mutex); 1894 free_info(info); 1895 mutex_unlock(&blkfront_mutex); 1896 1897 dev_set_drvdata(&dev->dev, NULL); 1898 1899 return err; 1900 } 1901 1902 static int negotiate_mq(struct blkfront_info *info) 1903 { 1904 unsigned int backend_max_queues; 1905 unsigned int i; 1906 1907 BUG_ON(info->nr_rings); 1908 1909 /* Check if backend supports multiple queues. */ 1910 backend_max_queues = xenbus_read_unsigned(info->xbdev->otherend, 1911 "multi-queue-max-queues", 1); 1912 info->nr_rings = min(backend_max_queues, xen_blkif_max_queues); 1913 /* We need at least one ring. */ 1914 if (!info->nr_rings) 1915 info->nr_rings = 1; 1916 1917 info->rinfo = kcalloc(info->nr_rings, 1918 sizeof(struct blkfront_ring_info), 1919 GFP_KERNEL); 1920 if (!info->rinfo) { 1921 xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure"); 1922 info->nr_rings = 0; 1923 return -ENOMEM; 1924 } 1925 1926 for (i = 0; i < info->nr_rings; i++) { 1927 struct blkfront_ring_info *rinfo; 1928 1929 rinfo = &info->rinfo[i]; 1930 INIT_LIST_HEAD(&rinfo->indirect_pages); 1931 INIT_LIST_HEAD(&rinfo->grants); 1932 rinfo->dev_info = info; 1933 INIT_WORK(&rinfo->work, blkif_restart_queue); 1934 spin_lock_init(&rinfo->ring_lock); 1935 } 1936 return 0; 1937 } 1938 /** 1939 * Entry point to this code when a new device is created. Allocate the basic 1940 * structures and the ring buffer for communication with the backend, and 1941 * inform the backend of the appropriate details for those. Switch to 1942 * Initialised state. 1943 */ 1944 static int blkfront_probe(struct xenbus_device *dev, 1945 const struct xenbus_device_id *id) 1946 { 1947 int err, vdevice; 1948 struct blkfront_info *info; 1949 1950 /* FIXME: Use dynamic device id if this is not set. */ 1951 err = xenbus_scanf(XBT_NIL, dev->nodename, 1952 "virtual-device", "%i", &vdevice); 1953 if (err != 1) { 1954 /* go looking in the extended area instead */ 1955 err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext", 1956 "%i", &vdevice); 1957 if (err != 1) { 1958 xenbus_dev_fatal(dev, err, "reading virtual-device"); 1959 return err; 1960 } 1961 } 1962 1963 if (xen_hvm_domain()) { 1964 char *type; 1965 int len; 1966 /* no unplug has been done: do not hook devices != xen vbds */ 1967 if (xen_has_pv_and_legacy_disk_devices()) { 1968 int major; 1969 1970 if (!VDEV_IS_EXTENDED(vdevice)) 1971 major = BLKIF_MAJOR(vdevice); 1972 else 1973 major = XENVBD_MAJOR; 1974 1975 if (major != XENVBD_MAJOR) { 1976 printk(KERN_INFO 1977 "%s: HVM does not support vbd %d as xen block device\n", 1978 __func__, vdevice); 1979 return -ENODEV; 1980 } 1981 } 1982 /* do not create a PV cdrom device if we are an HVM guest */ 1983 type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len); 1984 if (IS_ERR(type)) 1985 return -ENODEV; 1986 if (strncmp(type, "cdrom", 5) == 0) { 1987 kfree(type); 1988 return -ENODEV; 1989 } 1990 kfree(type); 1991 } 1992 info = kzalloc(sizeof(*info), GFP_KERNEL); 1993 if (!info) { 1994 xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure"); 1995 return -ENOMEM; 1996 } 1997 1998 info->xbdev = dev; 1999 2000 mutex_init(&info->mutex); 2001 info->vdevice = vdevice; 2002 info->connected = BLKIF_STATE_DISCONNECTED; 2003 2004 /* Front end dir is a number, which is used as the id. */ 2005 info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0); 2006 dev_set_drvdata(&dev->dev, info); 2007 2008 mutex_lock(&blkfront_mutex); 2009 list_add(&info->info_list, &info_list); 2010 mutex_unlock(&blkfront_mutex); 2011 2012 return 0; 2013 } 2014 2015 static int blkif_recover(struct blkfront_info *info) 2016 { 2017 unsigned int r_index; 2018 struct request *req, *n; 2019 int rc; 2020 struct bio *bio; 2021 unsigned int segs; 2022 2023 blkfront_gather_backend_features(info); 2024 /* Reset limits changed by blk_mq_update_nr_hw_queues(). */ 2025 blkif_set_queue_limits(info); 2026 segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST; 2027 blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG); 2028 2029 for (r_index = 0; r_index < info->nr_rings; r_index++) { 2030 struct blkfront_ring_info *rinfo = &info->rinfo[r_index]; 2031 2032 rc = blkfront_setup_indirect(rinfo); 2033 if (rc) 2034 return rc; 2035 } 2036 xenbus_switch_state(info->xbdev, XenbusStateConnected); 2037 2038 /* Now safe for us to use the shared ring */ 2039 info->connected = BLKIF_STATE_CONNECTED; 2040 2041 for (r_index = 0; r_index < info->nr_rings; r_index++) { 2042 struct blkfront_ring_info *rinfo; 2043 2044 rinfo = &info->rinfo[r_index]; 2045 /* Kick any other new requests queued since we resumed */ 2046 kick_pending_request_queues(rinfo); 2047 } 2048 2049 list_for_each_entry_safe(req, n, &info->requests, queuelist) { 2050 /* Requeue pending requests (flush or discard) */ 2051 list_del_init(&req->queuelist); 2052 BUG_ON(req->nr_phys_segments > segs); 2053 blk_mq_requeue_request(req, false); 2054 } 2055 blk_mq_start_stopped_hw_queues(info->rq, true); 2056 blk_mq_kick_requeue_list(info->rq); 2057 2058 while ((bio = bio_list_pop(&info->bio_list)) != NULL) { 2059 /* Traverse the list of pending bios and re-queue them */ 2060 submit_bio(bio); 2061 } 2062 2063 return 0; 2064 } 2065 2066 /** 2067 * We are reconnecting to the backend, due to a suspend/resume, or a backend 2068 * driver restart. We tear down our blkif structure and recreate it, but 2069 * leave the device-layer structures intact so that this is transparent to the 2070 * rest of the kernel. 2071 */ 2072 static int blkfront_resume(struct xenbus_device *dev) 2073 { 2074 struct blkfront_info *info = dev_get_drvdata(&dev->dev); 2075 int err = 0; 2076 unsigned int i, j; 2077 2078 dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename); 2079 2080 bio_list_init(&info->bio_list); 2081 INIT_LIST_HEAD(&info->requests); 2082 for (i = 0; i < info->nr_rings; i++) { 2083 struct blkfront_ring_info *rinfo = &info->rinfo[i]; 2084 struct bio_list merge_bio; 2085 struct blk_shadow *shadow = rinfo->shadow; 2086 2087 for (j = 0; j < BLK_RING_SIZE(info); j++) { 2088 /* Not in use? */ 2089 if (!shadow[j].request) 2090 continue; 2091 2092 /* 2093 * Get the bios in the request so we can re-queue them. 2094 */ 2095 if (req_op(shadow[j].request) == REQ_OP_FLUSH || 2096 req_op(shadow[j].request) == REQ_OP_DISCARD || 2097 req_op(shadow[j].request) == REQ_OP_SECURE_ERASE || 2098 shadow[j].request->cmd_flags & REQ_FUA) { 2099 /* 2100 * Flush operations don't contain bios, so 2101 * we need to requeue the whole request 2102 * 2103 * XXX: but this doesn't make any sense for a 2104 * write with the FUA flag set.. 2105 */ 2106 list_add(&shadow[j].request->queuelist, &info->requests); 2107 continue; 2108 } 2109 merge_bio.head = shadow[j].request->bio; 2110 merge_bio.tail = shadow[j].request->biotail; 2111 bio_list_merge(&info->bio_list, &merge_bio); 2112 shadow[j].request->bio = NULL; 2113 blk_mq_end_request(shadow[j].request, BLK_STS_OK); 2114 } 2115 } 2116 2117 blkif_free(info, info->connected == BLKIF_STATE_CONNECTED); 2118 2119 err = talk_to_blkback(dev, info); 2120 if (!err) 2121 blk_mq_update_nr_hw_queues(&info->tag_set, info->nr_rings); 2122 2123 /* 2124 * We have to wait for the backend to switch to 2125 * connected state, since we want to read which 2126 * features it supports. 2127 */ 2128 2129 return err; 2130 } 2131 2132 static void blkfront_closing(struct blkfront_info *info) 2133 { 2134 struct xenbus_device *xbdev = info->xbdev; 2135 struct block_device *bdev = NULL; 2136 2137 mutex_lock(&info->mutex); 2138 2139 if (xbdev->state == XenbusStateClosing) { 2140 mutex_unlock(&info->mutex); 2141 return; 2142 } 2143 2144 if (info->gd) 2145 bdev = bdget_disk(info->gd, 0); 2146 2147 mutex_unlock(&info->mutex); 2148 2149 if (!bdev) { 2150 xenbus_frontend_closed(xbdev); 2151 return; 2152 } 2153 2154 mutex_lock(&bdev->bd_mutex); 2155 2156 if (bdev->bd_openers) { 2157 xenbus_dev_error(xbdev, -EBUSY, 2158 "Device in use; refusing to close"); 2159 xenbus_switch_state(xbdev, XenbusStateClosing); 2160 } else { 2161 xlvbd_release_gendisk(info); 2162 xenbus_frontend_closed(xbdev); 2163 } 2164 2165 mutex_unlock(&bdev->bd_mutex); 2166 bdput(bdev); 2167 } 2168 2169 static void blkfront_setup_discard(struct blkfront_info *info) 2170 { 2171 int err; 2172 unsigned int discard_granularity; 2173 unsigned int discard_alignment; 2174 2175 info->feature_discard = 1; 2176 err = xenbus_gather(XBT_NIL, info->xbdev->otherend, 2177 "discard-granularity", "%u", &discard_granularity, 2178 "discard-alignment", "%u", &discard_alignment, 2179 NULL); 2180 if (!err) { 2181 info->discard_granularity = discard_granularity; 2182 info->discard_alignment = discard_alignment; 2183 } 2184 info->feature_secdiscard = 2185 !!xenbus_read_unsigned(info->xbdev->otherend, "discard-secure", 2186 0); 2187 } 2188 2189 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo) 2190 { 2191 unsigned int psegs, grants; 2192 int err, i; 2193 struct blkfront_info *info = rinfo->dev_info; 2194 2195 if (info->max_indirect_segments == 0) { 2196 if (!HAS_EXTRA_REQ) 2197 grants = BLKIF_MAX_SEGMENTS_PER_REQUEST; 2198 else { 2199 /* 2200 * When an extra req is required, the maximum 2201 * grants supported is related to the size of the 2202 * Linux block segment. 2203 */ 2204 grants = GRANTS_PER_PSEG; 2205 } 2206 } 2207 else 2208 grants = info->max_indirect_segments; 2209 psegs = DIV_ROUND_UP(grants, GRANTS_PER_PSEG); 2210 2211 err = fill_grant_buffer(rinfo, 2212 (grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info)); 2213 if (err) 2214 goto out_of_memory; 2215 2216 if (!info->feature_persistent && info->max_indirect_segments) { 2217 /* 2218 * We are using indirect descriptors but not persistent 2219 * grants, we need to allocate a set of pages that can be 2220 * used for mapping indirect grefs 2221 */ 2222 int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info); 2223 2224 BUG_ON(!list_empty(&rinfo->indirect_pages)); 2225 for (i = 0; i < num; i++) { 2226 struct page *indirect_page = alloc_page(GFP_NOIO); 2227 if (!indirect_page) 2228 goto out_of_memory; 2229 list_add(&indirect_page->lru, &rinfo->indirect_pages); 2230 } 2231 } 2232 2233 for (i = 0; i < BLK_RING_SIZE(info); i++) { 2234 rinfo->shadow[i].grants_used = 2235 kcalloc(grants, 2236 sizeof(rinfo->shadow[i].grants_used[0]), 2237 GFP_NOIO); 2238 rinfo->shadow[i].sg = kcalloc(psegs, 2239 sizeof(rinfo->shadow[i].sg[0]), 2240 GFP_NOIO); 2241 if (info->max_indirect_segments) 2242 rinfo->shadow[i].indirect_grants = 2243 kcalloc(INDIRECT_GREFS(grants), 2244 sizeof(rinfo->shadow[i].indirect_grants[0]), 2245 GFP_NOIO); 2246 if ((rinfo->shadow[i].grants_used == NULL) || 2247 (rinfo->shadow[i].sg == NULL) || 2248 (info->max_indirect_segments && 2249 (rinfo->shadow[i].indirect_grants == NULL))) 2250 goto out_of_memory; 2251 sg_init_table(rinfo->shadow[i].sg, psegs); 2252 } 2253 2254 2255 return 0; 2256 2257 out_of_memory: 2258 for (i = 0; i < BLK_RING_SIZE(info); i++) { 2259 kfree(rinfo->shadow[i].grants_used); 2260 rinfo->shadow[i].grants_used = NULL; 2261 kfree(rinfo->shadow[i].sg); 2262 rinfo->shadow[i].sg = NULL; 2263 kfree(rinfo->shadow[i].indirect_grants); 2264 rinfo->shadow[i].indirect_grants = NULL; 2265 } 2266 if (!list_empty(&rinfo->indirect_pages)) { 2267 struct page *indirect_page, *n; 2268 list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) { 2269 list_del(&indirect_page->lru); 2270 __free_page(indirect_page); 2271 } 2272 } 2273 return -ENOMEM; 2274 } 2275 2276 /* 2277 * Gather all backend feature-* 2278 */ 2279 static void blkfront_gather_backend_features(struct blkfront_info *info) 2280 { 2281 unsigned int indirect_segments; 2282 2283 info->feature_flush = 0; 2284 info->feature_fua = 0; 2285 2286 /* 2287 * If there's no "feature-barrier" defined, then it means 2288 * we're dealing with a very old backend which writes 2289 * synchronously; nothing to do. 2290 * 2291 * If there are barriers, then we use flush. 2292 */ 2293 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-barrier", 0)) { 2294 info->feature_flush = 1; 2295 info->feature_fua = 1; 2296 } 2297 2298 /* 2299 * And if there is "feature-flush-cache" use that above 2300 * barriers. 2301 */ 2302 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-flush-cache", 2303 0)) { 2304 info->feature_flush = 1; 2305 info->feature_fua = 0; 2306 } 2307 2308 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0)) 2309 blkfront_setup_discard(info); 2310 2311 info->feature_persistent = 2312 !!xenbus_read_unsigned(info->xbdev->otherend, 2313 "feature-persistent", 0); 2314 2315 indirect_segments = xenbus_read_unsigned(info->xbdev->otherend, 2316 "feature-max-indirect-segments", 0); 2317 if (indirect_segments > xen_blkif_max_segments) 2318 indirect_segments = xen_blkif_max_segments; 2319 if (indirect_segments <= BLKIF_MAX_SEGMENTS_PER_REQUEST) 2320 indirect_segments = 0; 2321 info->max_indirect_segments = indirect_segments; 2322 2323 if (info->feature_persistent) { 2324 mutex_lock(&blkfront_mutex); 2325 schedule_delayed_work(&blkfront_work, HZ * 10); 2326 mutex_unlock(&blkfront_mutex); 2327 } 2328 } 2329 2330 /* 2331 * Invoked when the backend is finally 'ready' (and has told produced 2332 * the details about the physical device - #sectors, size, etc). 2333 */ 2334 static void blkfront_connect(struct blkfront_info *info) 2335 { 2336 unsigned long long sectors; 2337 unsigned long sector_size; 2338 unsigned int physical_sector_size; 2339 unsigned int binfo; 2340 char *envp[] = { "RESIZE=1", NULL }; 2341 int err, i; 2342 2343 switch (info->connected) { 2344 case BLKIF_STATE_CONNECTED: 2345 /* 2346 * Potentially, the back-end may be signalling 2347 * a capacity change; update the capacity. 2348 */ 2349 err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, 2350 "sectors", "%Lu", §ors); 2351 if (XENBUS_EXIST_ERR(err)) 2352 return; 2353 printk(KERN_INFO "Setting capacity to %Lu\n", 2354 sectors); 2355 set_capacity(info->gd, sectors); 2356 revalidate_disk(info->gd); 2357 kobject_uevent_env(&disk_to_dev(info->gd)->kobj, 2358 KOBJ_CHANGE, envp); 2359 2360 return; 2361 case BLKIF_STATE_SUSPENDED: 2362 /* 2363 * If we are recovering from suspension, we need to wait 2364 * for the backend to announce it's features before 2365 * reconnecting, at least we need to know if the backend 2366 * supports indirect descriptors, and how many. 2367 */ 2368 blkif_recover(info); 2369 return; 2370 2371 default: 2372 break; 2373 } 2374 2375 dev_dbg(&info->xbdev->dev, "%s:%s.\n", 2376 __func__, info->xbdev->otherend); 2377 2378 err = xenbus_gather(XBT_NIL, info->xbdev->otherend, 2379 "sectors", "%llu", §ors, 2380 "info", "%u", &binfo, 2381 "sector-size", "%lu", §or_size, 2382 NULL); 2383 if (err) { 2384 xenbus_dev_fatal(info->xbdev, err, 2385 "reading backend fields at %s", 2386 info->xbdev->otherend); 2387 return; 2388 } 2389 2390 /* 2391 * physcial-sector-size is a newer field, so old backends may not 2392 * provide this. Assume physical sector size to be the same as 2393 * sector_size in that case. 2394 */ 2395 physical_sector_size = xenbus_read_unsigned(info->xbdev->otherend, 2396 "physical-sector-size", 2397 sector_size); 2398 blkfront_gather_backend_features(info); 2399 for (i = 0; i < info->nr_rings; i++) { 2400 err = blkfront_setup_indirect(&info->rinfo[i]); 2401 if (err) { 2402 xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s", 2403 info->xbdev->otherend); 2404 blkif_free(info, 0); 2405 break; 2406 } 2407 } 2408 2409 err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size, 2410 physical_sector_size); 2411 if (err) { 2412 xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s", 2413 info->xbdev->otherend); 2414 goto fail; 2415 } 2416 2417 xenbus_switch_state(info->xbdev, XenbusStateConnected); 2418 2419 /* Kick pending requests. */ 2420 info->connected = BLKIF_STATE_CONNECTED; 2421 for (i = 0; i < info->nr_rings; i++) 2422 kick_pending_request_queues(&info->rinfo[i]); 2423 2424 device_add_disk(&info->xbdev->dev, info->gd, NULL); 2425 2426 info->is_ready = 1; 2427 return; 2428 2429 fail: 2430 blkif_free(info, 0); 2431 return; 2432 } 2433 2434 /** 2435 * Callback received when the backend's state changes. 2436 */ 2437 static void blkback_changed(struct xenbus_device *dev, 2438 enum xenbus_state backend_state) 2439 { 2440 struct blkfront_info *info = dev_get_drvdata(&dev->dev); 2441 2442 dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state); 2443 2444 switch (backend_state) { 2445 case XenbusStateInitWait: 2446 if (dev->state != XenbusStateInitialising) 2447 break; 2448 if (talk_to_blkback(dev, info)) 2449 break; 2450 case XenbusStateInitialising: 2451 case XenbusStateInitialised: 2452 case XenbusStateReconfiguring: 2453 case XenbusStateReconfigured: 2454 case XenbusStateUnknown: 2455 break; 2456 2457 case XenbusStateConnected: 2458 /* 2459 * talk_to_blkback sets state to XenbusStateInitialised 2460 * and blkfront_connect sets it to XenbusStateConnected 2461 * (if connection went OK). 2462 * 2463 * If the backend (or toolstack) decides to poke at backend 2464 * state (and re-trigger the watch by setting the state repeatedly 2465 * to XenbusStateConnected (4)) we need to deal with this. 2466 * This is allowed as this is used to communicate to the guest 2467 * that the size of disk has changed! 2468 */ 2469 if ((dev->state != XenbusStateInitialised) && 2470 (dev->state != XenbusStateConnected)) { 2471 if (talk_to_blkback(dev, info)) 2472 break; 2473 } 2474 2475 blkfront_connect(info); 2476 break; 2477 2478 case XenbusStateClosed: 2479 if (dev->state == XenbusStateClosed) 2480 break; 2481 /* fall through */ 2482 case XenbusStateClosing: 2483 if (info) 2484 blkfront_closing(info); 2485 break; 2486 } 2487 } 2488 2489 static int blkfront_remove(struct xenbus_device *xbdev) 2490 { 2491 struct blkfront_info *info = dev_get_drvdata(&xbdev->dev); 2492 struct block_device *bdev = NULL; 2493 struct gendisk *disk; 2494 2495 dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename); 2496 2497 if (!info) 2498 return 0; 2499 2500 blkif_free(info, 0); 2501 2502 mutex_lock(&info->mutex); 2503 2504 disk = info->gd; 2505 if (disk) 2506 bdev = bdget_disk(disk, 0); 2507 2508 info->xbdev = NULL; 2509 mutex_unlock(&info->mutex); 2510 2511 if (!bdev) { 2512 mutex_lock(&blkfront_mutex); 2513 free_info(info); 2514 mutex_unlock(&blkfront_mutex); 2515 return 0; 2516 } 2517 2518 /* 2519 * The xbdev was removed before we reached the Closed 2520 * state. See if it's safe to remove the disk. If the bdev 2521 * isn't closed yet, we let release take care of it. 2522 */ 2523 2524 mutex_lock(&bdev->bd_mutex); 2525 info = disk->private_data; 2526 2527 dev_warn(disk_to_dev(disk), 2528 "%s was hot-unplugged, %d stale handles\n", 2529 xbdev->nodename, bdev->bd_openers); 2530 2531 if (info && !bdev->bd_openers) { 2532 xlvbd_release_gendisk(info); 2533 disk->private_data = NULL; 2534 mutex_lock(&blkfront_mutex); 2535 free_info(info); 2536 mutex_unlock(&blkfront_mutex); 2537 } 2538 2539 mutex_unlock(&bdev->bd_mutex); 2540 bdput(bdev); 2541 2542 return 0; 2543 } 2544 2545 static int blkfront_is_ready(struct xenbus_device *dev) 2546 { 2547 struct blkfront_info *info = dev_get_drvdata(&dev->dev); 2548 2549 return info->is_ready && info->xbdev; 2550 } 2551 2552 static int blkif_open(struct block_device *bdev, fmode_t mode) 2553 { 2554 struct gendisk *disk = bdev->bd_disk; 2555 struct blkfront_info *info; 2556 int err = 0; 2557 2558 mutex_lock(&blkfront_mutex); 2559 2560 info = disk->private_data; 2561 if (!info) { 2562 /* xbdev gone */ 2563 err = -ERESTARTSYS; 2564 goto out; 2565 } 2566 2567 mutex_lock(&info->mutex); 2568 2569 if (!info->gd) 2570 /* xbdev is closed */ 2571 err = -ERESTARTSYS; 2572 2573 mutex_unlock(&info->mutex); 2574 2575 out: 2576 mutex_unlock(&blkfront_mutex); 2577 return err; 2578 } 2579 2580 static void blkif_release(struct gendisk *disk, fmode_t mode) 2581 { 2582 struct blkfront_info *info = disk->private_data; 2583 struct block_device *bdev; 2584 struct xenbus_device *xbdev; 2585 2586 mutex_lock(&blkfront_mutex); 2587 2588 bdev = bdget_disk(disk, 0); 2589 2590 if (!bdev) { 2591 WARN(1, "Block device %s yanked out from us!\n", disk->disk_name); 2592 goto out_mutex; 2593 } 2594 if (bdev->bd_openers) 2595 goto out; 2596 2597 /* 2598 * Check if we have been instructed to close. We will have 2599 * deferred this request, because the bdev was still open. 2600 */ 2601 2602 mutex_lock(&info->mutex); 2603 xbdev = info->xbdev; 2604 2605 if (xbdev && xbdev->state == XenbusStateClosing) { 2606 /* pending switch to state closed */ 2607 dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n"); 2608 xlvbd_release_gendisk(info); 2609 xenbus_frontend_closed(info->xbdev); 2610 } 2611 2612 mutex_unlock(&info->mutex); 2613 2614 if (!xbdev) { 2615 /* sudden device removal */ 2616 dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n"); 2617 xlvbd_release_gendisk(info); 2618 disk->private_data = NULL; 2619 free_info(info); 2620 } 2621 2622 out: 2623 bdput(bdev); 2624 out_mutex: 2625 mutex_unlock(&blkfront_mutex); 2626 } 2627 2628 static const struct block_device_operations xlvbd_block_fops = 2629 { 2630 .owner = THIS_MODULE, 2631 .open = blkif_open, 2632 .release = blkif_release, 2633 .getgeo = blkif_getgeo, 2634 .ioctl = blkif_ioctl, 2635 }; 2636 2637 2638 static const struct xenbus_device_id blkfront_ids[] = { 2639 { "vbd" }, 2640 { "" } 2641 }; 2642 2643 static struct xenbus_driver blkfront_driver = { 2644 .ids = blkfront_ids, 2645 .probe = blkfront_probe, 2646 .remove = blkfront_remove, 2647 .resume = blkfront_resume, 2648 .otherend_changed = blkback_changed, 2649 .is_ready = blkfront_is_ready, 2650 }; 2651 2652 static void purge_persistent_grants(struct blkfront_info *info) 2653 { 2654 unsigned int i; 2655 unsigned long flags; 2656 2657 for (i = 0; i < info->nr_rings; i++) { 2658 struct blkfront_ring_info *rinfo = &info->rinfo[i]; 2659 struct grant *gnt_list_entry, *tmp; 2660 2661 spin_lock_irqsave(&rinfo->ring_lock, flags); 2662 2663 if (rinfo->persistent_gnts_c == 0) { 2664 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 2665 continue; 2666 } 2667 2668 list_for_each_entry_safe(gnt_list_entry, tmp, &rinfo->grants, 2669 node) { 2670 if (gnt_list_entry->gref == GRANT_INVALID_REF || 2671 gnttab_query_foreign_access(gnt_list_entry->gref)) 2672 continue; 2673 2674 list_del(&gnt_list_entry->node); 2675 gnttab_end_foreign_access(gnt_list_entry->gref, 0, 0UL); 2676 rinfo->persistent_gnts_c--; 2677 gnt_list_entry->gref = GRANT_INVALID_REF; 2678 list_add_tail(&gnt_list_entry->node, &rinfo->grants); 2679 } 2680 2681 spin_unlock_irqrestore(&rinfo->ring_lock, flags); 2682 } 2683 } 2684 2685 static void blkfront_delay_work(struct work_struct *work) 2686 { 2687 struct blkfront_info *info; 2688 bool need_schedule_work = false; 2689 2690 mutex_lock(&blkfront_mutex); 2691 2692 list_for_each_entry(info, &info_list, info_list) { 2693 if (info->feature_persistent) { 2694 need_schedule_work = true; 2695 mutex_lock(&info->mutex); 2696 purge_persistent_grants(info); 2697 mutex_unlock(&info->mutex); 2698 } 2699 } 2700 2701 if (need_schedule_work) 2702 schedule_delayed_work(&blkfront_work, HZ * 10); 2703 2704 mutex_unlock(&blkfront_mutex); 2705 } 2706 2707 static int __init xlblk_init(void) 2708 { 2709 int ret; 2710 int nr_cpus = num_online_cpus(); 2711 2712 if (!xen_domain()) 2713 return -ENODEV; 2714 2715 if (!xen_has_pv_disk_devices()) 2716 return -ENODEV; 2717 2718 if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) { 2719 pr_warn("xen_blk: can't get major %d with name %s\n", 2720 XENVBD_MAJOR, DEV_NAME); 2721 return -ENODEV; 2722 } 2723 2724 if (xen_blkif_max_segments < BLKIF_MAX_SEGMENTS_PER_REQUEST) 2725 xen_blkif_max_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST; 2726 2727 if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) { 2728 pr_info("Invalid max_ring_order (%d), will use default max: %d.\n", 2729 xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER); 2730 xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER; 2731 } 2732 2733 if (xen_blkif_max_queues > nr_cpus) { 2734 pr_info("Invalid max_queues (%d), will use default max: %d.\n", 2735 xen_blkif_max_queues, nr_cpus); 2736 xen_blkif_max_queues = nr_cpus; 2737 } 2738 2739 INIT_DELAYED_WORK(&blkfront_work, blkfront_delay_work); 2740 2741 ret = xenbus_register_frontend(&blkfront_driver); 2742 if (ret) { 2743 unregister_blkdev(XENVBD_MAJOR, DEV_NAME); 2744 return ret; 2745 } 2746 2747 return 0; 2748 } 2749 module_init(xlblk_init); 2750 2751 2752 static void __exit xlblk_exit(void) 2753 { 2754 cancel_delayed_work_sync(&blkfront_work); 2755 2756 xenbus_unregister_driver(&blkfront_driver); 2757 unregister_blkdev(XENVBD_MAJOR, DEV_NAME); 2758 kfree(minors); 2759 } 2760 module_exit(xlblk_exit); 2761 2762 MODULE_DESCRIPTION("Xen virtual block device frontend"); 2763 MODULE_LICENSE("GPL"); 2764 MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR); 2765 MODULE_ALIAS("xen:vbd"); 2766 MODULE_ALIAS("xenblk"); 2767