1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * virtio-fs: Virtio Filesystem 4 * Copyright (C) 2018 Red Hat, Inc. 5 */ 6 7 #include <linux/fs.h> 8 #include <linux/dax.h> 9 #include <linux/pci.h> 10 #include <linux/pfn_t.h> 11 #include <linux/memremap.h> 12 #include <linux/module.h> 13 #include <linux/virtio.h> 14 #include <linux/virtio_fs.h> 15 #include <linux/delay.h> 16 #include <linux/fs_context.h> 17 #include <linux/fs_parser.h> 18 #include <linux/highmem.h> 19 #include <linux/uio.h> 20 #include "fuse_i.h" 21 22 /* Used to help calculate the FUSE connection's max_pages limit for a request's 23 * size. Parts of the struct fuse_req are sliced into scattergather lists in 24 * addition to the pages used, so this can help account for that overhead. 25 */ 26 #define FUSE_HEADER_OVERHEAD 4 27 28 /* List of virtio-fs device instances and a lock for the list. Also provides 29 * mutual exclusion in device removal and mounting path 30 */ 31 static DEFINE_MUTEX(virtio_fs_mutex); 32 static LIST_HEAD(virtio_fs_instances); 33 34 enum { 35 VQ_HIPRIO, 36 VQ_REQUEST 37 }; 38 39 #define VQ_NAME_LEN 24 40 41 /* Per-virtqueue state */ 42 struct virtio_fs_vq { 43 spinlock_t lock; 44 struct virtqueue *vq; /* protected by ->lock */ 45 struct work_struct done_work; 46 struct list_head queued_reqs; 47 struct list_head end_reqs; /* End these requests */ 48 struct delayed_work dispatch_work; 49 struct fuse_dev *fud; 50 bool connected; 51 long in_flight; 52 struct completion in_flight_zero; /* No inflight requests */ 53 char name[VQ_NAME_LEN]; 54 } ____cacheline_aligned_in_smp; 55 56 /* A virtio-fs device instance */ 57 struct virtio_fs { 58 struct kref refcount; 59 struct list_head list; /* on virtio_fs_instances */ 60 char *tag; 61 struct virtio_fs_vq *vqs; 62 unsigned int nvqs; /* number of virtqueues */ 63 unsigned int num_request_queues; /* number of request queues */ 64 struct dax_device *dax_dev; 65 66 /* DAX memory window where file contents are mapped */ 67 void *window_kaddr; 68 phys_addr_t window_phys_addr; 69 size_t window_len; 70 }; 71 72 struct virtio_fs_forget_req { 73 struct fuse_in_header ih; 74 struct fuse_forget_in arg; 75 }; 76 77 struct virtio_fs_forget { 78 /* This request can be temporarily queued on virt queue */ 79 struct list_head list; 80 struct virtio_fs_forget_req req; 81 }; 82 83 struct virtio_fs_req_work { 84 struct fuse_req *req; 85 struct virtio_fs_vq *fsvq; 86 struct work_struct done_work; 87 }; 88 89 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 90 struct fuse_req *req, bool in_flight); 91 92 static const struct constant_table dax_param_enums[] = { 93 {"always", FUSE_DAX_ALWAYS }, 94 {"never", FUSE_DAX_NEVER }, 95 {"inode", FUSE_DAX_INODE_USER }, 96 {} 97 }; 98 99 enum { 100 OPT_DAX, 101 OPT_DAX_ENUM, 102 }; 103 104 static const struct fs_parameter_spec virtio_fs_parameters[] = { 105 fsparam_flag("dax", OPT_DAX), 106 fsparam_enum("dax", OPT_DAX_ENUM, dax_param_enums), 107 {} 108 }; 109 110 static int virtio_fs_parse_param(struct fs_context *fsc, 111 struct fs_parameter *param) 112 { 113 struct fs_parse_result result; 114 struct fuse_fs_context *ctx = fsc->fs_private; 115 int opt; 116 117 opt = fs_parse(fsc, virtio_fs_parameters, param, &result); 118 if (opt < 0) 119 return opt; 120 121 switch (opt) { 122 case OPT_DAX: 123 ctx->dax_mode = FUSE_DAX_ALWAYS; 124 break; 125 case OPT_DAX_ENUM: 126 ctx->dax_mode = result.uint_32; 127 break; 128 default: 129 return -EINVAL; 130 } 131 132 return 0; 133 } 134 135 static void virtio_fs_free_fsc(struct fs_context *fsc) 136 { 137 struct fuse_fs_context *ctx = fsc->fs_private; 138 139 kfree(ctx); 140 } 141 142 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 143 { 144 struct virtio_fs *fs = vq->vdev->priv; 145 146 return &fs->vqs[vq->index]; 147 } 148 149 /* Should be called with fsvq->lock held. */ 150 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 151 { 152 fsvq->in_flight++; 153 } 154 155 /* Should be called with fsvq->lock held. */ 156 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 157 { 158 WARN_ON(fsvq->in_flight <= 0); 159 fsvq->in_flight--; 160 if (!fsvq->in_flight) 161 complete(&fsvq->in_flight_zero); 162 } 163 164 static void release_virtio_fs_obj(struct kref *ref) 165 { 166 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 167 168 kfree(vfs->vqs); 169 kfree(vfs); 170 } 171 172 /* Make sure virtiofs_mutex is held */ 173 static void virtio_fs_put(struct virtio_fs *fs) 174 { 175 kref_put(&fs->refcount, release_virtio_fs_obj); 176 } 177 178 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 179 { 180 struct virtio_fs *vfs = fiq->priv; 181 182 mutex_lock(&virtio_fs_mutex); 183 virtio_fs_put(vfs); 184 mutex_unlock(&virtio_fs_mutex); 185 } 186 187 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 188 { 189 WARN_ON(fsvq->in_flight < 0); 190 191 /* Wait for in flight requests to finish.*/ 192 spin_lock(&fsvq->lock); 193 if (fsvq->in_flight) { 194 /* We are holding virtio_fs_mutex. There should not be any 195 * waiters waiting for completion. 196 */ 197 reinit_completion(&fsvq->in_flight_zero); 198 spin_unlock(&fsvq->lock); 199 wait_for_completion(&fsvq->in_flight_zero); 200 } else { 201 spin_unlock(&fsvq->lock); 202 } 203 204 flush_work(&fsvq->done_work); 205 flush_delayed_work(&fsvq->dispatch_work); 206 } 207 208 static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) 209 { 210 struct virtio_fs_vq *fsvq; 211 int i; 212 213 for (i = 0; i < fs->nvqs; i++) { 214 fsvq = &fs->vqs[i]; 215 virtio_fs_drain_queue(fsvq); 216 } 217 } 218 219 static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 220 { 221 /* Provides mutual exclusion between ->remove and ->kill_sb 222 * paths. We don't want both of these draining queue at the 223 * same time. Current completion logic reinits completion 224 * and that means there should not be any other thread 225 * doing reinit or waiting for completion already. 226 */ 227 mutex_lock(&virtio_fs_mutex); 228 virtio_fs_drain_all_queues_locked(fs); 229 mutex_unlock(&virtio_fs_mutex); 230 } 231 232 static void virtio_fs_start_all_queues(struct virtio_fs *fs) 233 { 234 struct virtio_fs_vq *fsvq; 235 int i; 236 237 for (i = 0; i < fs->nvqs; i++) { 238 fsvq = &fs->vqs[i]; 239 spin_lock(&fsvq->lock); 240 fsvq->connected = true; 241 spin_unlock(&fsvq->lock); 242 } 243 } 244 245 /* Add a new instance to the list or return -EEXIST if tag name exists*/ 246 static int virtio_fs_add_instance(struct virtio_fs *fs) 247 { 248 struct virtio_fs *fs2; 249 bool duplicate = false; 250 251 mutex_lock(&virtio_fs_mutex); 252 253 list_for_each_entry(fs2, &virtio_fs_instances, list) { 254 if (strcmp(fs->tag, fs2->tag) == 0) 255 duplicate = true; 256 } 257 258 if (!duplicate) 259 list_add_tail(&fs->list, &virtio_fs_instances); 260 261 mutex_unlock(&virtio_fs_mutex); 262 263 if (duplicate) 264 return -EEXIST; 265 return 0; 266 } 267 268 /* Return the virtio_fs with a given tag, or NULL */ 269 static struct virtio_fs *virtio_fs_find_instance(const char *tag) 270 { 271 struct virtio_fs *fs; 272 273 mutex_lock(&virtio_fs_mutex); 274 275 list_for_each_entry(fs, &virtio_fs_instances, list) { 276 if (strcmp(fs->tag, tag) == 0) { 277 kref_get(&fs->refcount); 278 goto found; 279 } 280 } 281 282 fs = NULL; /* not found */ 283 284 found: 285 mutex_unlock(&virtio_fs_mutex); 286 287 return fs; 288 } 289 290 static void virtio_fs_free_devs(struct virtio_fs *fs) 291 { 292 unsigned int i; 293 294 for (i = 0; i < fs->nvqs; i++) { 295 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 296 297 if (!fsvq->fud) 298 continue; 299 300 fuse_dev_free(fsvq->fud); 301 fsvq->fud = NULL; 302 } 303 } 304 305 /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 306 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 307 { 308 char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 309 char *end; 310 size_t len; 311 312 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 313 &tag_buf, sizeof(tag_buf)); 314 end = memchr(tag_buf, '\0', sizeof(tag_buf)); 315 if (end == tag_buf) 316 return -EINVAL; /* empty tag */ 317 if (!end) 318 end = &tag_buf[sizeof(tag_buf)]; 319 320 len = end - tag_buf; 321 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 322 if (!fs->tag) 323 return -ENOMEM; 324 memcpy(fs->tag, tag_buf, len); 325 fs->tag[len] = '\0'; 326 return 0; 327 } 328 329 /* Work function for hiprio completion */ 330 static void virtio_fs_hiprio_done_work(struct work_struct *work) 331 { 332 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 333 done_work); 334 struct virtqueue *vq = fsvq->vq; 335 336 /* Free completed FUSE_FORGET requests */ 337 spin_lock(&fsvq->lock); 338 do { 339 unsigned int len; 340 void *req; 341 342 virtqueue_disable_cb(vq); 343 344 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 345 kfree(req); 346 dec_in_flight_req(fsvq); 347 } 348 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 349 spin_unlock(&fsvq->lock); 350 } 351 352 static void virtio_fs_request_dispatch_work(struct work_struct *work) 353 { 354 struct fuse_req *req; 355 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 356 dispatch_work.work); 357 int ret; 358 359 pr_debug("virtio-fs: worker %s called.\n", __func__); 360 while (1) { 361 spin_lock(&fsvq->lock); 362 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 363 list); 364 if (!req) { 365 spin_unlock(&fsvq->lock); 366 break; 367 } 368 369 list_del_init(&req->list); 370 spin_unlock(&fsvq->lock); 371 fuse_request_end(req); 372 } 373 374 /* Dispatch pending requests */ 375 while (1) { 376 spin_lock(&fsvq->lock); 377 req = list_first_entry_or_null(&fsvq->queued_reqs, 378 struct fuse_req, list); 379 if (!req) { 380 spin_unlock(&fsvq->lock); 381 return; 382 } 383 list_del_init(&req->list); 384 spin_unlock(&fsvq->lock); 385 386 ret = virtio_fs_enqueue_req(fsvq, req, true); 387 if (ret < 0) { 388 if (ret == -ENOMEM || ret == -ENOSPC) { 389 spin_lock(&fsvq->lock); 390 list_add_tail(&req->list, &fsvq->queued_reqs); 391 schedule_delayed_work(&fsvq->dispatch_work, 392 msecs_to_jiffies(1)); 393 spin_unlock(&fsvq->lock); 394 return; 395 } 396 req->out.h.error = ret; 397 spin_lock(&fsvq->lock); 398 dec_in_flight_req(fsvq); 399 spin_unlock(&fsvq->lock); 400 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 401 ret); 402 fuse_request_end(req); 403 } 404 } 405 } 406 407 /* 408 * Returns 1 if queue is full and sender should wait a bit before sending 409 * next request, 0 otherwise. 410 */ 411 static int send_forget_request(struct virtio_fs_vq *fsvq, 412 struct virtio_fs_forget *forget, 413 bool in_flight) 414 { 415 struct scatterlist sg; 416 struct virtqueue *vq; 417 int ret = 0; 418 bool notify; 419 struct virtio_fs_forget_req *req = &forget->req; 420 421 spin_lock(&fsvq->lock); 422 if (!fsvq->connected) { 423 if (in_flight) 424 dec_in_flight_req(fsvq); 425 kfree(forget); 426 goto out; 427 } 428 429 sg_init_one(&sg, req, sizeof(*req)); 430 vq = fsvq->vq; 431 dev_dbg(&vq->vdev->dev, "%s\n", __func__); 432 433 ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 434 if (ret < 0) { 435 if (ret == -ENOMEM || ret == -ENOSPC) { 436 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 437 ret); 438 list_add_tail(&forget->list, &fsvq->queued_reqs); 439 schedule_delayed_work(&fsvq->dispatch_work, 440 msecs_to_jiffies(1)); 441 if (!in_flight) 442 inc_in_flight_req(fsvq); 443 /* Queue is full */ 444 ret = 1; 445 } else { 446 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 447 ret); 448 kfree(forget); 449 if (in_flight) 450 dec_in_flight_req(fsvq); 451 } 452 goto out; 453 } 454 455 if (!in_flight) 456 inc_in_flight_req(fsvq); 457 notify = virtqueue_kick_prepare(vq); 458 spin_unlock(&fsvq->lock); 459 460 if (notify) 461 virtqueue_notify(vq); 462 return ret; 463 out: 464 spin_unlock(&fsvq->lock); 465 return ret; 466 } 467 468 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 469 { 470 struct virtio_fs_forget *forget; 471 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 472 dispatch_work.work); 473 pr_debug("virtio-fs: worker %s called.\n", __func__); 474 while (1) { 475 spin_lock(&fsvq->lock); 476 forget = list_first_entry_or_null(&fsvq->queued_reqs, 477 struct virtio_fs_forget, list); 478 if (!forget) { 479 spin_unlock(&fsvq->lock); 480 return; 481 } 482 483 list_del(&forget->list); 484 spin_unlock(&fsvq->lock); 485 if (send_forget_request(fsvq, forget, true)) 486 return; 487 } 488 } 489 490 /* Allocate and copy args into req->argbuf */ 491 static int copy_args_to_argbuf(struct fuse_req *req) 492 { 493 struct fuse_args *args = req->args; 494 unsigned int offset = 0; 495 unsigned int num_in; 496 unsigned int num_out; 497 unsigned int len; 498 unsigned int i; 499 500 num_in = args->in_numargs - args->in_pages; 501 num_out = args->out_numargs - args->out_pages; 502 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 503 fuse_len_args(num_out, args->out_args); 504 505 req->argbuf = kmalloc(len, GFP_ATOMIC); 506 if (!req->argbuf) 507 return -ENOMEM; 508 509 for (i = 0; i < num_in; i++) { 510 memcpy(req->argbuf + offset, 511 args->in_args[i].value, 512 args->in_args[i].size); 513 offset += args->in_args[i].size; 514 } 515 516 return 0; 517 } 518 519 /* Copy args out of and free req->argbuf */ 520 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 521 { 522 unsigned int remaining; 523 unsigned int offset; 524 unsigned int num_in; 525 unsigned int num_out; 526 unsigned int i; 527 528 remaining = req->out.h.len - sizeof(req->out.h); 529 num_in = args->in_numargs - args->in_pages; 530 num_out = args->out_numargs - args->out_pages; 531 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 532 533 for (i = 0; i < num_out; i++) { 534 unsigned int argsize = args->out_args[i].size; 535 536 if (args->out_argvar && 537 i == args->out_numargs - 1 && 538 argsize > remaining) { 539 argsize = remaining; 540 } 541 542 memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 543 offset += argsize; 544 545 if (i != args->out_numargs - 1) 546 remaining -= argsize; 547 } 548 549 /* Store the actual size of the variable-length arg */ 550 if (args->out_argvar) 551 args->out_args[args->out_numargs - 1].size = remaining; 552 553 kfree(req->argbuf); 554 req->argbuf = NULL; 555 } 556 557 /* Work function for request completion */ 558 static void virtio_fs_request_complete(struct fuse_req *req, 559 struct virtio_fs_vq *fsvq) 560 { 561 struct fuse_pqueue *fpq = &fsvq->fud->pq; 562 struct fuse_args *args; 563 struct fuse_args_pages *ap; 564 unsigned int len, i, thislen; 565 struct page *page; 566 567 /* 568 * TODO verify that server properly follows FUSE protocol 569 * (oh.uniq, oh.len) 570 */ 571 args = req->args; 572 copy_args_from_argbuf(args, req); 573 574 if (args->out_pages && args->page_zeroing) { 575 len = args->out_args[args->out_numargs - 1].size; 576 ap = container_of(args, typeof(*ap), args); 577 for (i = 0; i < ap->num_pages; i++) { 578 thislen = ap->descs[i].length; 579 if (len < thislen) { 580 WARN_ON(ap->descs[i].offset); 581 page = ap->pages[i]; 582 zero_user_segment(page, len, thislen); 583 len = 0; 584 } else { 585 len -= thislen; 586 } 587 } 588 } 589 590 spin_lock(&fpq->lock); 591 clear_bit(FR_SENT, &req->flags); 592 spin_unlock(&fpq->lock); 593 594 fuse_request_end(req); 595 spin_lock(&fsvq->lock); 596 dec_in_flight_req(fsvq); 597 spin_unlock(&fsvq->lock); 598 } 599 600 static void virtio_fs_complete_req_work(struct work_struct *work) 601 { 602 struct virtio_fs_req_work *w = 603 container_of(work, typeof(*w), done_work); 604 605 virtio_fs_request_complete(w->req, w->fsvq); 606 kfree(w); 607 } 608 609 static void virtio_fs_requests_done_work(struct work_struct *work) 610 { 611 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 612 done_work); 613 struct fuse_pqueue *fpq = &fsvq->fud->pq; 614 struct virtqueue *vq = fsvq->vq; 615 struct fuse_req *req; 616 struct fuse_req *next; 617 unsigned int len; 618 LIST_HEAD(reqs); 619 620 /* Collect completed requests off the virtqueue */ 621 spin_lock(&fsvq->lock); 622 do { 623 virtqueue_disable_cb(vq); 624 625 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 626 spin_lock(&fpq->lock); 627 list_move_tail(&req->list, &reqs); 628 spin_unlock(&fpq->lock); 629 } 630 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 631 spin_unlock(&fsvq->lock); 632 633 /* End requests */ 634 list_for_each_entry_safe(req, next, &reqs, list) { 635 list_del_init(&req->list); 636 637 /* blocking async request completes in a worker context */ 638 if (req->args->may_block) { 639 struct virtio_fs_req_work *w; 640 641 w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL); 642 INIT_WORK(&w->done_work, virtio_fs_complete_req_work); 643 w->fsvq = fsvq; 644 w->req = req; 645 schedule_work(&w->done_work); 646 } else { 647 virtio_fs_request_complete(req, fsvq); 648 } 649 } 650 } 651 652 /* Virtqueue interrupt handler */ 653 static void virtio_fs_vq_done(struct virtqueue *vq) 654 { 655 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 656 657 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 658 659 schedule_work(&fsvq->done_work); 660 } 661 662 static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, 663 int vq_type) 664 { 665 strscpy(fsvq->name, name, VQ_NAME_LEN); 666 spin_lock_init(&fsvq->lock); 667 INIT_LIST_HEAD(&fsvq->queued_reqs); 668 INIT_LIST_HEAD(&fsvq->end_reqs); 669 init_completion(&fsvq->in_flight_zero); 670 671 if (vq_type == VQ_REQUEST) { 672 INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); 673 INIT_DELAYED_WORK(&fsvq->dispatch_work, 674 virtio_fs_request_dispatch_work); 675 } else { 676 INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); 677 INIT_DELAYED_WORK(&fsvq->dispatch_work, 678 virtio_fs_hiprio_dispatch_work); 679 } 680 } 681 682 /* Initialize virtqueues */ 683 static int virtio_fs_setup_vqs(struct virtio_device *vdev, 684 struct virtio_fs *fs) 685 { 686 struct virtqueue **vqs; 687 vq_callback_t **callbacks; 688 const char **names; 689 unsigned int i; 690 int ret = 0; 691 692 virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, 693 &fs->num_request_queues); 694 if (fs->num_request_queues == 0) 695 return -EINVAL; 696 697 fs->nvqs = VQ_REQUEST + fs->num_request_queues; 698 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 699 if (!fs->vqs) 700 return -ENOMEM; 701 702 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 703 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 704 GFP_KERNEL); 705 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 706 if (!vqs || !callbacks || !names) { 707 ret = -ENOMEM; 708 goto out; 709 } 710 711 /* Initialize the hiprio/forget request virtqueue */ 712 callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 713 virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); 714 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 715 716 /* Initialize the requests virtqueues */ 717 for (i = VQ_REQUEST; i < fs->nvqs; i++) { 718 char vq_name[VQ_NAME_LEN]; 719 720 snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); 721 virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); 722 callbacks[i] = virtio_fs_vq_done; 723 names[i] = fs->vqs[i].name; 724 } 725 726 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 727 if (ret < 0) 728 goto out; 729 730 for (i = 0; i < fs->nvqs; i++) 731 fs->vqs[i].vq = vqs[i]; 732 733 virtio_fs_start_all_queues(fs); 734 out: 735 kfree(names); 736 kfree(callbacks); 737 kfree(vqs); 738 if (ret) 739 kfree(fs->vqs); 740 return ret; 741 } 742 743 /* Free virtqueues (device must already be reset) */ 744 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev) 745 { 746 vdev->config->del_vqs(vdev); 747 } 748 749 /* Map a window offset to a page frame number. The window offset will have 750 * been produced by .iomap_begin(), which maps a file offset to a window 751 * offset. 752 */ 753 static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 754 long nr_pages, enum dax_access_mode mode, 755 void **kaddr, pfn_t *pfn) 756 { 757 struct virtio_fs *fs = dax_get_private(dax_dev); 758 phys_addr_t offset = PFN_PHYS(pgoff); 759 size_t max_nr_pages = fs->window_len / PAGE_SIZE - pgoff; 760 761 if (kaddr) 762 *kaddr = fs->window_kaddr + offset; 763 if (pfn) 764 *pfn = phys_to_pfn_t(fs->window_phys_addr + offset, 765 PFN_DEV | PFN_MAP); 766 return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; 767 } 768 769 static int virtio_fs_zero_page_range(struct dax_device *dax_dev, 770 pgoff_t pgoff, size_t nr_pages) 771 { 772 long rc; 773 void *kaddr; 774 775 rc = dax_direct_access(dax_dev, pgoff, nr_pages, DAX_ACCESS, &kaddr, 776 NULL); 777 if (rc < 0) 778 return rc; 779 memset(kaddr, 0, nr_pages << PAGE_SHIFT); 780 dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); 781 return 0; 782 } 783 784 static const struct dax_operations virtio_fs_dax_ops = { 785 .direct_access = virtio_fs_direct_access, 786 .zero_page_range = virtio_fs_zero_page_range, 787 }; 788 789 static void virtio_fs_cleanup_dax(void *data) 790 { 791 struct dax_device *dax_dev = data; 792 793 kill_dax(dax_dev); 794 put_dax(dax_dev); 795 } 796 797 static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) 798 { 799 struct virtio_shm_region cache_reg; 800 struct dev_pagemap *pgmap; 801 bool have_cache; 802 803 if (!IS_ENABLED(CONFIG_FUSE_DAX)) 804 return 0; 805 806 /* Get cache region */ 807 have_cache = virtio_get_shm_region(vdev, &cache_reg, 808 (u8)VIRTIO_FS_SHMCAP_ID_CACHE); 809 if (!have_cache) { 810 dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); 811 return 0; 812 } 813 814 if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, 815 dev_name(&vdev->dev))) { 816 dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", 817 cache_reg.addr, cache_reg.len); 818 return -EBUSY; 819 } 820 821 dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, 822 cache_reg.addr); 823 824 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); 825 if (!pgmap) 826 return -ENOMEM; 827 828 pgmap->type = MEMORY_DEVICE_FS_DAX; 829 830 /* Ideally we would directly use the PCI BAR resource but 831 * devm_memremap_pages() wants its own copy in pgmap. So 832 * initialize a struct resource from scratch (only the start 833 * and end fields will be used). 834 */ 835 pgmap->range = (struct range) { 836 .start = (phys_addr_t) cache_reg.addr, 837 .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, 838 }; 839 pgmap->nr_range = 1; 840 841 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); 842 if (IS_ERR(fs->window_kaddr)) 843 return PTR_ERR(fs->window_kaddr); 844 845 fs->window_phys_addr = (phys_addr_t) cache_reg.addr; 846 fs->window_len = (phys_addr_t) cache_reg.len; 847 848 dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", 849 __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); 850 851 fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops); 852 if (IS_ERR(fs->dax_dev)) 853 return PTR_ERR(fs->dax_dev); 854 855 return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, 856 fs->dax_dev); 857 } 858 859 static int virtio_fs_probe(struct virtio_device *vdev) 860 { 861 struct virtio_fs *fs; 862 int ret; 863 864 fs = kzalloc(sizeof(*fs), GFP_KERNEL); 865 if (!fs) 866 return -ENOMEM; 867 kref_init(&fs->refcount); 868 vdev->priv = fs; 869 870 ret = virtio_fs_read_tag(vdev, fs); 871 if (ret < 0) 872 goto out; 873 874 ret = virtio_fs_setup_vqs(vdev, fs); 875 if (ret < 0) 876 goto out; 877 878 /* TODO vq affinity */ 879 880 ret = virtio_fs_setup_dax(vdev, fs); 881 if (ret < 0) 882 goto out_vqs; 883 884 /* Bring the device online in case the filesystem is mounted and 885 * requests need to be sent before we return. 886 */ 887 virtio_device_ready(vdev); 888 889 ret = virtio_fs_add_instance(fs); 890 if (ret < 0) 891 goto out_vqs; 892 893 return 0; 894 895 out_vqs: 896 virtio_reset_device(vdev); 897 virtio_fs_cleanup_vqs(vdev); 898 kfree(fs->vqs); 899 900 out: 901 vdev->priv = NULL; 902 kfree(fs); 903 return ret; 904 } 905 906 static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 907 { 908 struct virtio_fs_vq *fsvq; 909 int i; 910 911 for (i = 0; i < fs->nvqs; i++) { 912 fsvq = &fs->vqs[i]; 913 spin_lock(&fsvq->lock); 914 fsvq->connected = false; 915 spin_unlock(&fsvq->lock); 916 } 917 } 918 919 static void virtio_fs_remove(struct virtio_device *vdev) 920 { 921 struct virtio_fs *fs = vdev->priv; 922 923 mutex_lock(&virtio_fs_mutex); 924 /* This device is going away. No one should get new reference */ 925 list_del_init(&fs->list); 926 virtio_fs_stop_all_queues(fs); 927 virtio_fs_drain_all_queues_locked(fs); 928 virtio_reset_device(vdev); 929 virtio_fs_cleanup_vqs(vdev); 930 931 vdev->priv = NULL; 932 /* Put device reference on virtio_fs object */ 933 virtio_fs_put(fs); 934 mutex_unlock(&virtio_fs_mutex); 935 } 936 937 #ifdef CONFIG_PM_SLEEP 938 static int virtio_fs_freeze(struct virtio_device *vdev) 939 { 940 /* TODO need to save state here */ 941 pr_warn("virtio-fs: suspend/resume not yet supported\n"); 942 return -EOPNOTSUPP; 943 } 944 945 static int virtio_fs_restore(struct virtio_device *vdev) 946 { 947 /* TODO need to restore state here */ 948 return 0; 949 } 950 #endif /* CONFIG_PM_SLEEP */ 951 952 static const struct virtio_device_id id_table[] = { 953 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 954 {}, 955 }; 956 957 static const unsigned int feature_table[] = {}; 958 959 static struct virtio_driver virtio_fs_driver = { 960 .driver.name = KBUILD_MODNAME, 961 .driver.owner = THIS_MODULE, 962 .id_table = id_table, 963 .feature_table = feature_table, 964 .feature_table_size = ARRAY_SIZE(feature_table), 965 .probe = virtio_fs_probe, 966 .remove = virtio_fs_remove, 967 #ifdef CONFIG_PM_SLEEP 968 .freeze = virtio_fs_freeze, 969 .restore = virtio_fs_restore, 970 #endif 971 }; 972 973 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 974 __releases(fiq->lock) 975 { 976 struct fuse_forget_link *link; 977 struct virtio_fs_forget *forget; 978 struct virtio_fs_forget_req *req; 979 struct virtio_fs *fs; 980 struct virtio_fs_vq *fsvq; 981 u64 unique; 982 983 link = fuse_dequeue_forget(fiq, 1, NULL); 984 unique = fuse_get_unique(fiq); 985 986 fs = fiq->priv; 987 fsvq = &fs->vqs[VQ_HIPRIO]; 988 spin_unlock(&fiq->lock); 989 990 /* Allocate a buffer for the request */ 991 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 992 req = &forget->req; 993 994 req->ih = (struct fuse_in_header){ 995 .opcode = FUSE_FORGET, 996 .nodeid = link->forget_one.nodeid, 997 .unique = unique, 998 .len = sizeof(*req), 999 }; 1000 req->arg = (struct fuse_forget_in){ 1001 .nlookup = link->forget_one.nlookup, 1002 }; 1003 1004 send_forget_request(fsvq, forget, false); 1005 kfree(link); 1006 } 1007 1008 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 1009 __releases(fiq->lock) 1010 { 1011 /* 1012 * TODO interrupts. 1013 * 1014 * Normal fs operations on a local filesystems aren't interruptible. 1015 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 1016 * with shared lock between host and guest. 1017 */ 1018 spin_unlock(&fiq->lock); 1019 } 1020 1021 /* Count number of scatter-gather elements required */ 1022 static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, 1023 unsigned int num_pages, 1024 unsigned int total_len) 1025 { 1026 unsigned int i; 1027 unsigned int this_len; 1028 1029 for (i = 0; i < num_pages && total_len; i++) { 1030 this_len = min(page_descs[i].length, total_len); 1031 total_len -= this_len; 1032 } 1033 1034 return i; 1035 } 1036 1037 /* Return the number of scatter-gather list elements required */ 1038 static unsigned int sg_count_fuse_req(struct fuse_req *req) 1039 { 1040 struct fuse_args *args = req->args; 1041 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 1042 unsigned int size, total_sgs = 1 /* fuse_in_header */; 1043 1044 if (args->in_numargs - args->in_pages) 1045 total_sgs += 1; 1046 1047 if (args->in_pages) { 1048 size = args->in_args[args->in_numargs - 1].size; 1049 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1050 size); 1051 } 1052 1053 if (!test_bit(FR_ISREPLY, &req->flags)) 1054 return total_sgs; 1055 1056 total_sgs += 1 /* fuse_out_header */; 1057 1058 if (args->out_numargs - args->out_pages) 1059 total_sgs += 1; 1060 1061 if (args->out_pages) { 1062 size = args->out_args[args->out_numargs - 1].size; 1063 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1064 size); 1065 } 1066 1067 return total_sgs; 1068 } 1069 1070 /* Add pages to scatter-gather list and return number of elements used */ 1071 static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 1072 struct page **pages, 1073 struct fuse_page_desc *page_descs, 1074 unsigned int num_pages, 1075 unsigned int total_len) 1076 { 1077 unsigned int i; 1078 unsigned int this_len; 1079 1080 for (i = 0; i < num_pages && total_len; i++) { 1081 sg_init_table(&sg[i], 1); 1082 this_len = min(page_descs[i].length, total_len); 1083 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 1084 total_len -= this_len; 1085 } 1086 1087 return i; 1088 } 1089 1090 /* Add args to scatter-gather list and return number of elements used */ 1091 static unsigned int sg_init_fuse_args(struct scatterlist *sg, 1092 struct fuse_req *req, 1093 struct fuse_arg *args, 1094 unsigned int numargs, 1095 bool argpages, 1096 void *argbuf, 1097 unsigned int *len_used) 1098 { 1099 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 1100 unsigned int total_sgs = 0; 1101 unsigned int len; 1102 1103 len = fuse_len_args(numargs - argpages, args); 1104 if (len) 1105 sg_init_one(&sg[total_sgs++], argbuf, len); 1106 1107 if (argpages) 1108 total_sgs += sg_init_fuse_pages(&sg[total_sgs], 1109 ap->pages, ap->descs, 1110 ap->num_pages, 1111 args[numargs - 1].size); 1112 1113 if (len_used) 1114 *len_used = len; 1115 1116 return total_sgs; 1117 } 1118 1119 /* Add a request to a virtqueue and kick the device */ 1120 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 1121 struct fuse_req *req, bool in_flight) 1122 { 1123 /* requests need at least 4 elements */ 1124 struct scatterlist *stack_sgs[6]; 1125 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 1126 struct scatterlist **sgs = stack_sgs; 1127 struct scatterlist *sg = stack_sg; 1128 struct virtqueue *vq; 1129 struct fuse_args *args = req->args; 1130 unsigned int argbuf_used = 0; 1131 unsigned int out_sgs = 0; 1132 unsigned int in_sgs = 0; 1133 unsigned int total_sgs; 1134 unsigned int i; 1135 int ret; 1136 bool notify; 1137 struct fuse_pqueue *fpq; 1138 1139 /* Does the sglist fit on the stack? */ 1140 total_sgs = sg_count_fuse_req(req); 1141 if (total_sgs > ARRAY_SIZE(stack_sgs)) { 1142 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 1143 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 1144 if (!sgs || !sg) { 1145 ret = -ENOMEM; 1146 goto out; 1147 } 1148 } 1149 1150 /* Use a bounce buffer since stack args cannot be mapped */ 1151 ret = copy_args_to_argbuf(req); 1152 if (ret < 0) 1153 goto out; 1154 1155 /* Request elements */ 1156 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 1157 out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 1158 (struct fuse_arg *)args->in_args, 1159 args->in_numargs, args->in_pages, 1160 req->argbuf, &argbuf_used); 1161 1162 /* Reply elements */ 1163 if (test_bit(FR_ISREPLY, &req->flags)) { 1164 sg_init_one(&sg[out_sgs + in_sgs++], 1165 &req->out.h, sizeof(req->out.h)); 1166 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 1167 args->out_args, args->out_numargs, 1168 args->out_pages, 1169 req->argbuf + argbuf_used, NULL); 1170 } 1171 1172 WARN_ON(out_sgs + in_sgs != total_sgs); 1173 1174 for (i = 0; i < total_sgs; i++) 1175 sgs[i] = &sg[i]; 1176 1177 spin_lock(&fsvq->lock); 1178 1179 if (!fsvq->connected) { 1180 spin_unlock(&fsvq->lock); 1181 ret = -ENOTCONN; 1182 goto out; 1183 } 1184 1185 vq = fsvq->vq; 1186 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 1187 if (ret < 0) { 1188 spin_unlock(&fsvq->lock); 1189 goto out; 1190 } 1191 1192 /* Request successfully sent. */ 1193 fpq = &fsvq->fud->pq; 1194 spin_lock(&fpq->lock); 1195 list_add_tail(&req->list, fpq->processing); 1196 spin_unlock(&fpq->lock); 1197 set_bit(FR_SENT, &req->flags); 1198 /* matches barrier in request_wait_answer() */ 1199 smp_mb__after_atomic(); 1200 1201 if (!in_flight) 1202 inc_in_flight_req(fsvq); 1203 notify = virtqueue_kick_prepare(vq); 1204 1205 spin_unlock(&fsvq->lock); 1206 1207 if (notify) 1208 virtqueue_notify(vq); 1209 1210 out: 1211 if (ret < 0 && req->argbuf) { 1212 kfree(req->argbuf); 1213 req->argbuf = NULL; 1214 } 1215 if (sgs != stack_sgs) { 1216 kfree(sgs); 1217 kfree(sg); 1218 } 1219 1220 return ret; 1221 } 1222 1223 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 1224 __releases(fiq->lock) 1225 { 1226 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 1227 struct virtio_fs *fs; 1228 struct fuse_req *req; 1229 struct virtio_fs_vq *fsvq; 1230 int ret; 1231 1232 WARN_ON(list_empty(&fiq->pending)); 1233 req = list_last_entry(&fiq->pending, struct fuse_req, list); 1234 clear_bit(FR_PENDING, &req->flags); 1235 list_del_init(&req->list); 1236 WARN_ON(!list_empty(&fiq->pending)); 1237 spin_unlock(&fiq->lock); 1238 1239 fs = fiq->priv; 1240 1241 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 1242 __func__, req->in.h.opcode, req->in.h.unique, 1243 req->in.h.nodeid, req->in.h.len, 1244 fuse_len_args(req->args->out_numargs, req->args->out_args)); 1245 1246 fsvq = &fs->vqs[queue_id]; 1247 ret = virtio_fs_enqueue_req(fsvq, req, false); 1248 if (ret < 0) { 1249 if (ret == -ENOMEM || ret == -ENOSPC) { 1250 /* 1251 * Virtqueue full. Retry submission from worker 1252 * context as we might be holding fc->bg_lock. 1253 */ 1254 spin_lock(&fsvq->lock); 1255 list_add_tail(&req->list, &fsvq->queued_reqs); 1256 inc_in_flight_req(fsvq); 1257 schedule_delayed_work(&fsvq->dispatch_work, 1258 msecs_to_jiffies(1)); 1259 spin_unlock(&fsvq->lock); 1260 return; 1261 } 1262 req->out.h.error = ret; 1263 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 1264 1265 /* Can't end request in submission context. Use a worker */ 1266 spin_lock(&fsvq->lock); 1267 list_add_tail(&req->list, &fsvq->end_reqs); 1268 schedule_delayed_work(&fsvq->dispatch_work, 0); 1269 spin_unlock(&fsvq->lock); 1270 return; 1271 } 1272 } 1273 1274 static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1275 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1276 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1277 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1278 .release = virtio_fs_fiq_release, 1279 }; 1280 1281 static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx) 1282 { 1283 ctx->rootmode = S_IFDIR; 1284 ctx->default_permissions = 1; 1285 ctx->allow_other = 1; 1286 ctx->max_read = UINT_MAX; 1287 ctx->blksize = 512; 1288 ctx->destroy = true; 1289 ctx->no_control = true; 1290 ctx->no_force_umount = true; 1291 } 1292 1293 static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) 1294 { 1295 struct fuse_mount *fm = get_fuse_mount_super(sb); 1296 struct fuse_conn *fc = fm->fc; 1297 struct virtio_fs *fs = fc->iq.priv; 1298 struct fuse_fs_context *ctx = fsc->fs_private; 1299 unsigned int i; 1300 int err; 1301 1302 virtio_fs_ctx_set_defaults(ctx); 1303 mutex_lock(&virtio_fs_mutex); 1304 1305 /* After holding mutex, make sure virtiofs device is still there. 1306 * Though we are holding a reference to it, drive ->remove might 1307 * still have cleaned up virtual queues. In that case bail out. 1308 */ 1309 err = -EINVAL; 1310 if (list_empty(&fs->list)) { 1311 pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1312 goto err; 1313 } 1314 1315 err = -ENOMEM; 1316 /* Allocate fuse_dev for hiprio and notification queues */ 1317 for (i = 0; i < fs->nvqs; i++) { 1318 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1319 1320 fsvq->fud = fuse_dev_alloc(); 1321 if (!fsvq->fud) 1322 goto err_free_fuse_devs; 1323 } 1324 1325 /* virtiofs allocates and installs its own fuse devices */ 1326 ctx->fudptr = NULL; 1327 if (ctx->dax_mode != FUSE_DAX_NEVER) { 1328 if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) { 1329 err = -EINVAL; 1330 pr_err("virtio-fs: dax can't be enabled as filesystem" 1331 " device does not support it.\n"); 1332 goto err_free_fuse_devs; 1333 } 1334 ctx->dax_dev = fs->dax_dev; 1335 } 1336 err = fuse_fill_super_common(sb, ctx); 1337 if (err < 0) 1338 goto err_free_fuse_devs; 1339 1340 for (i = 0; i < fs->nvqs; i++) { 1341 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1342 1343 fuse_dev_install(fsvq->fud, fc); 1344 } 1345 1346 /* Previous unmount will stop all queues. Start these again */ 1347 virtio_fs_start_all_queues(fs); 1348 fuse_send_init(fm); 1349 mutex_unlock(&virtio_fs_mutex); 1350 return 0; 1351 1352 err_free_fuse_devs: 1353 virtio_fs_free_devs(fs); 1354 err: 1355 mutex_unlock(&virtio_fs_mutex); 1356 return err; 1357 } 1358 1359 static void virtio_fs_conn_destroy(struct fuse_mount *fm) 1360 { 1361 struct fuse_conn *fc = fm->fc; 1362 struct virtio_fs *vfs = fc->iq.priv; 1363 struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; 1364 1365 /* Stop dax worker. Soon evict_inodes() will be called which 1366 * will free all memory ranges belonging to all inodes. 1367 */ 1368 if (IS_ENABLED(CONFIG_FUSE_DAX)) 1369 fuse_dax_cancel_work(fc); 1370 1371 /* Stop forget queue. Soon destroy will be sent */ 1372 spin_lock(&fsvq->lock); 1373 fsvq->connected = false; 1374 spin_unlock(&fsvq->lock); 1375 virtio_fs_drain_all_queues(vfs); 1376 1377 fuse_conn_destroy(fm); 1378 1379 /* fuse_conn_destroy() must have sent destroy. Stop all queues 1380 * and drain one more time and free fuse devices. Freeing fuse 1381 * devices will drop their reference on fuse_conn and that in 1382 * turn will drop its reference on virtio_fs object. 1383 */ 1384 virtio_fs_stop_all_queues(vfs); 1385 virtio_fs_drain_all_queues(vfs); 1386 virtio_fs_free_devs(vfs); 1387 } 1388 1389 static void virtio_kill_sb(struct super_block *sb) 1390 { 1391 struct fuse_mount *fm = get_fuse_mount_super(sb); 1392 bool last; 1393 1394 /* If mount failed, we can still be called without any fc */ 1395 if (sb->s_root) { 1396 last = fuse_mount_remove(fm); 1397 if (last) 1398 virtio_fs_conn_destroy(fm); 1399 } 1400 kill_anon_super(sb); 1401 fuse_mount_destroy(fm); 1402 } 1403 1404 static int virtio_fs_test_super(struct super_block *sb, 1405 struct fs_context *fsc) 1406 { 1407 struct fuse_mount *fsc_fm = fsc->s_fs_info; 1408 struct fuse_mount *sb_fm = get_fuse_mount_super(sb); 1409 1410 return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; 1411 } 1412 1413 static int virtio_fs_get_tree(struct fs_context *fsc) 1414 { 1415 struct virtio_fs *fs; 1416 struct super_block *sb; 1417 struct fuse_conn *fc = NULL; 1418 struct fuse_mount *fm; 1419 unsigned int virtqueue_size; 1420 int err = -EIO; 1421 1422 /* This gets a reference on virtio_fs object. This ptr gets installed 1423 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1424 * to drop the reference to this object. 1425 */ 1426 fs = virtio_fs_find_instance(fsc->source); 1427 if (!fs) { 1428 pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1429 return -EINVAL; 1430 } 1431 1432 virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq); 1433 if (WARN_ON(virtqueue_size <= FUSE_HEADER_OVERHEAD)) 1434 goto out_err; 1435 1436 err = -ENOMEM; 1437 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1438 if (!fc) 1439 goto out_err; 1440 1441 fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL); 1442 if (!fm) 1443 goto out_err; 1444 1445 fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); 1446 fc->release = fuse_free_conn; 1447 fc->delete_stale = true; 1448 fc->auto_submounts = true; 1449 fc->sync_fs = true; 1450 1451 /* Tell FUSE to split requests that exceed the virtqueue's size */ 1452 fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, 1453 virtqueue_size - FUSE_HEADER_OVERHEAD); 1454 1455 fsc->s_fs_info = fm; 1456 sb = sget_fc(fsc, virtio_fs_test_super, set_anon_super_fc); 1457 if (fsc->s_fs_info) 1458 fuse_mount_destroy(fm); 1459 if (IS_ERR(sb)) 1460 return PTR_ERR(sb); 1461 1462 if (!sb->s_root) { 1463 err = virtio_fs_fill_super(sb, fsc); 1464 if (err) { 1465 deactivate_locked_super(sb); 1466 return err; 1467 } 1468 1469 sb->s_flags |= SB_ACTIVE; 1470 } 1471 1472 WARN_ON(fsc->root); 1473 fsc->root = dget(sb->s_root); 1474 return 0; 1475 1476 out_err: 1477 kfree(fc); 1478 mutex_lock(&virtio_fs_mutex); 1479 virtio_fs_put(fs); 1480 mutex_unlock(&virtio_fs_mutex); 1481 return err; 1482 } 1483 1484 static const struct fs_context_operations virtio_fs_context_ops = { 1485 .free = virtio_fs_free_fsc, 1486 .parse_param = virtio_fs_parse_param, 1487 .get_tree = virtio_fs_get_tree, 1488 }; 1489 1490 static int virtio_fs_init_fs_context(struct fs_context *fsc) 1491 { 1492 struct fuse_fs_context *ctx; 1493 1494 if (fsc->purpose == FS_CONTEXT_FOR_SUBMOUNT) 1495 return fuse_init_fs_context_submount(fsc); 1496 1497 ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL); 1498 if (!ctx) 1499 return -ENOMEM; 1500 fsc->fs_private = ctx; 1501 fsc->ops = &virtio_fs_context_ops; 1502 return 0; 1503 } 1504 1505 static struct file_system_type virtio_fs_type = { 1506 .owner = THIS_MODULE, 1507 .name = "virtiofs", 1508 .init_fs_context = virtio_fs_init_fs_context, 1509 .kill_sb = virtio_kill_sb, 1510 }; 1511 1512 static int __init virtio_fs_init(void) 1513 { 1514 int ret; 1515 1516 ret = register_virtio_driver(&virtio_fs_driver); 1517 if (ret < 0) 1518 return ret; 1519 1520 ret = register_filesystem(&virtio_fs_type); 1521 if (ret < 0) { 1522 unregister_virtio_driver(&virtio_fs_driver); 1523 return ret; 1524 } 1525 1526 return 0; 1527 } 1528 module_init(virtio_fs_init); 1529 1530 static void __exit virtio_fs_exit(void) 1531 { 1532 unregister_filesystem(&virtio_fs_type); 1533 unregister_virtio_driver(&virtio_fs_driver); 1534 } 1535 module_exit(virtio_fs_exit); 1536 1537 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1538 MODULE_DESCRIPTION("Virtio Filesystem"); 1539 MODULE_LICENSE("GPL"); 1540 MODULE_ALIAS_FS(KBUILD_MODNAME); 1541 MODULE_DEVICE_TABLE(virtio, id_table); 1542