1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * virtio-fs: Virtio Filesystem 4 * Copyright (C) 2018 Red Hat, Inc. 5 */ 6 7 #include <linux/fs.h> 8 #include <linux/dax.h> 9 #include <linux/pci.h> 10 #include <linux/pfn_t.h> 11 #include <linux/memremap.h> 12 #include <linux/module.h> 13 #include <linux/virtio.h> 14 #include <linux/virtio_fs.h> 15 #include <linux/delay.h> 16 #include <linux/fs_context.h> 17 #include <linux/fs_parser.h> 18 #include <linux/highmem.h> 19 #include <linux/uio.h> 20 #include "fuse_i.h" 21 22 /* Used to help calculate the FUSE connection's max_pages limit for a request's 23 * size. Parts of the struct fuse_req are sliced into scattergather lists in 24 * addition to the pages used, so this can help account for that overhead. 25 */ 26 #define FUSE_HEADER_OVERHEAD 4 27 28 /* List of virtio-fs device instances and a lock for the list. Also provides 29 * mutual exclusion in device removal and mounting path 30 */ 31 static DEFINE_MUTEX(virtio_fs_mutex); 32 static LIST_HEAD(virtio_fs_instances); 33 34 enum { 35 VQ_HIPRIO, 36 VQ_REQUEST 37 }; 38 39 #define VQ_NAME_LEN 24 40 41 /* Per-virtqueue state */ 42 struct virtio_fs_vq { 43 spinlock_t lock; 44 struct virtqueue *vq; /* protected by ->lock */ 45 struct work_struct done_work; 46 struct list_head queued_reqs; 47 struct list_head end_reqs; /* End these requests */ 48 struct delayed_work dispatch_work; 49 struct fuse_dev *fud; 50 bool connected; 51 long in_flight; 52 struct completion in_flight_zero; /* No inflight requests */ 53 char name[VQ_NAME_LEN]; 54 } ____cacheline_aligned_in_smp; 55 56 /* A virtio-fs device instance */ 57 struct virtio_fs { 58 struct kref refcount; 59 struct list_head list; /* on virtio_fs_instances */ 60 char *tag; 61 struct virtio_fs_vq *vqs; 62 unsigned int nvqs; /* number of virtqueues */ 63 unsigned int num_request_queues; /* number of request queues */ 64 struct dax_device *dax_dev; 65 66 /* DAX memory window where file contents are mapped */ 67 void *window_kaddr; 68 phys_addr_t window_phys_addr; 69 size_t window_len; 70 }; 71 72 struct virtio_fs_forget_req { 73 struct fuse_in_header ih; 74 struct fuse_forget_in arg; 75 }; 76 77 struct virtio_fs_forget { 78 /* This request can be temporarily queued on virt queue */ 79 struct list_head list; 80 struct virtio_fs_forget_req req; 81 }; 82 83 struct virtio_fs_req_work { 84 struct fuse_req *req; 85 struct virtio_fs_vq *fsvq; 86 struct work_struct done_work; 87 }; 88 89 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 90 struct fuse_req *req, bool in_flight); 91 92 static const struct constant_table dax_param_enums[] = { 93 {"always", FUSE_DAX_ALWAYS }, 94 {"never", FUSE_DAX_NEVER }, 95 {"inode", FUSE_DAX_INODE_USER }, 96 {} 97 }; 98 99 enum { 100 OPT_DAX, 101 OPT_DAX_ENUM, 102 }; 103 104 static const struct fs_parameter_spec virtio_fs_parameters[] = { 105 fsparam_flag("dax", OPT_DAX), 106 fsparam_enum("dax", OPT_DAX_ENUM, dax_param_enums), 107 {} 108 }; 109 110 static int virtio_fs_parse_param(struct fs_context *fsc, 111 struct fs_parameter *param) 112 { 113 struct fs_parse_result result; 114 struct fuse_fs_context *ctx = fsc->fs_private; 115 int opt; 116 117 opt = fs_parse(fsc, virtio_fs_parameters, param, &result); 118 if (opt < 0) 119 return opt; 120 121 switch (opt) { 122 case OPT_DAX: 123 ctx->dax_mode = FUSE_DAX_ALWAYS; 124 break; 125 case OPT_DAX_ENUM: 126 ctx->dax_mode = result.uint_32; 127 break; 128 default: 129 return -EINVAL; 130 } 131 132 return 0; 133 } 134 135 static void virtio_fs_free_fsc(struct fs_context *fsc) 136 { 137 struct fuse_fs_context *ctx = fsc->fs_private; 138 139 kfree(ctx); 140 } 141 142 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 143 { 144 struct virtio_fs *fs = vq->vdev->priv; 145 146 return &fs->vqs[vq->index]; 147 } 148 149 /* Should be called with fsvq->lock held. */ 150 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 151 { 152 fsvq->in_flight++; 153 } 154 155 /* Should be called with fsvq->lock held. */ 156 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 157 { 158 WARN_ON(fsvq->in_flight <= 0); 159 fsvq->in_flight--; 160 if (!fsvq->in_flight) 161 complete(&fsvq->in_flight_zero); 162 } 163 164 static void release_virtio_fs_obj(struct kref *ref) 165 { 166 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 167 168 kfree(vfs->vqs); 169 kfree(vfs); 170 } 171 172 /* Make sure virtiofs_mutex is held */ 173 static void virtio_fs_put(struct virtio_fs *fs) 174 { 175 kref_put(&fs->refcount, release_virtio_fs_obj); 176 } 177 178 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 179 { 180 struct virtio_fs *vfs = fiq->priv; 181 182 mutex_lock(&virtio_fs_mutex); 183 virtio_fs_put(vfs); 184 mutex_unlock(&virtio_fs_mutex); 185 } 186 187 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 188 { 189 WARN_ON(fsvq->in_flight < 0); 190 191 /* Wait for in flight requests to finish.*/ 192 spin_lock(&fsvq->lock); 193 if (fsvq->in_flight) { 194 /* We are holding virtio_fs_mutex. There should not be any 195 * waiters waiting for completion. 196 */ 197 reinit_completion(&fsvq->in_flight_zero); 198 spin_unlock(&fsvq->lock); 199 wait_for_completion(&fsvq->in_flight_zero); 200 } else { 201 spin_unlock(&fsvq->lock); 202 } 203 204 flush_work(&fsvq->done_work); 205 flush_delayed_work(&fsvq->dispatch_work); 206 } 207 208 static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) 209 { 210 struct virtio_fs_vq *fsvq; 211 int i; 212 213 for (i = 0; i < fs->nvqs; i++) { 214 fsvq = &fs->vqs[i]; 215 virtio_fs_drain_queue(fsvq); 216 } 217 } 218 219 static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 220 { 221 /* Provides mutual exclusion between ->remove and ->kill_sb 222 * paths. We don't want both of these draining queue at the 223 * same time. Current completion logic reinits completion 224 * and that means there should not be any other thread 225 * doing reinit or waiting for completion already. 226 */ 227 mutex_lock(&virtio_fs_mutex); 228 virtio_fs_drain_all_queues_locked(fs); 229 mutex_unlock(&virtio_fs_mutex); 230 } 231 232 static void virtio_fs_start_all_queues(struct virtio_fs *fs) 233 { 234 struct virtio_fs_vq *fsvq; 235 int i; 236 237 for (i = 0; i < fs->nvqs; i++) { 238 fsvq = &fs->vqs[i]; 239 spin_lock(&fsvq->lock); 240 fsvq->connected = true; 241 spin_unlock(&fsvq->lock); 242 } 243 } 244 245 /* Add a new instance to the list or return -EEXIST if tag name exists*/ 246 static int virtio_fs_add_instance(struct virtio_fs *fs) 247 { 248 struct virtio_fs *fs2; 249 bool duplicate = false; 250 251 mutex_lock(&virtio_fs_mutex); 252 253 list_for_each_entry(fs2, &virtio_fs_instances, list) { 254 if (strcmp(fs->tag, fs2->tag) == 0) 255 duplicate = true; 256 } 257 258 if (!duplicate) 259 list_add_tail(&fs->list, &virtio_fs_instances); 260 261 mutex_unlock(&virtio_fs_mutex); 262 263 if (duplicate) 264 return -EEXIST; 265 return 0; 266 } 267 268 /* Return the virtio_fs with a given tag, or NULL */ 269 static struct virtio_fs *virtio_fs_find_instance(const char *tag) 270 { 271 struct virtio_fs *fs; 272 273 mutex_lock(&virtio_fs_mutex); 274 275 list_for_each_entry(fs, &virtio_fs_instances, list) { 276 if (strcmp(fs->tag, tag) == 0) { 277 kref_get(&fs->refcount); 278 goto found; 279 } 280 } 281 282 fs = NULL; /* not found */ 283 284 found: 285 mutex_unlock(&virtio_fs_mutex); 286 287 return fs; 288 } 289 290 static void virtio_fs_free_devs(struct virtio_fs *fs) 291 { 292 unsigned int i; 293 294 for (i = 0; i < fs->nvqs; i++) { 295 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 296 297 if (!fsvq->fud) 298 continue; 299 300 fuse_dev_free(fsvq->fud); 301 fsvq->fud = NULL; 302 } 303 } 304 305 /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 306 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 307 { 308 char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 309 char *end; 310 size_t len; 311 312 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 313 &tag_buf, sizeof(tag_buf)); 314 end = memchr(tag_buf, '\0', sizeof(tag_buf)); 315 if (end == tag_buf) 316 return -EINVAL; /* empty tag */ 317 if (!end) 318 end = &tag_buf[sizeof(tag_buf)]; 319 320 len = end - tag_buf; 321 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 322 if (!fs->tag) 323 return -ENOMEM; 324 memcpy(fs->tag, tag_buf, len); 325 fs->tag[len] = '\0'; 326 return 0; 327 } 328 329 /* Work function for hiprio completion */ 330 static void virtio_fs_hiprio_done_work(struct work_struct *work) 331 { 332 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 333 done_work); 334 struct virtqueue *vq = fsvq->vq; 335 336 /* Free completed FUSE_FORGET requests */ 337 spin_lock(&fsvq->lock); 338 do { 339 unsigned int len; 340 void *req; 341 342 virtqueue_disable_cb(vq); 343 344 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 345 kfree(req); 346 dec_in_flight_req(fsvq); 347 } 348 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 349 spin_unlock(&fsvq->lock); 350 } 351 352 static void virtio_fs_request_dispatch_work(struct work_struct *work) 353 { 354 struct fuse_req *req; 355 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 356 dispatch_work.work); 357 int ret; 358 359 pr_debug("virtio-fs: worker %s called.\n", __func__); 360 while (1) { 361 spin_lock(&fsvq->lock); 362 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 363 list); 364 if (!req) { 365 spin_unlock(&fsvq->lock); 366 break; 367 } 368 369 list_del_init(&req->list); 370 spin_unlock(&fsvq->lock); 371 fuse_request_end(req); 372 } 373 374 /* Dispatch pending requests */ 375 while (1) { 376 spin_lock(&fsvq->lock); 377 req = list_first_entry_or_null(&fsvq->queued_reqs, 378 struct fuse_req, list); 379 if (!req) { 380 spin_unlock(&fsvq->lock); 381 return; 382 } 383 list_del_init(&req->list); 384 spin_unlock(&fsvq->lock); 385 386 ret = virtio_fs_enqueue_req(fsvq, req, true); 387 if (ret < 0) { 388 if (ret == -ENOMEM || ret == -ENOSPC) { 389 spin_lock(&fsvq->lock); 390 list_add_tail(&req->list, &fsvq->queued_reqs); 391 schedule_delayed_work(&fsvq->dispatch_work, 392 msecs_to_jiffies(1)); 393 spin_unlock(&fsvq->lock); 394 return; 395 } 396 req->out.h.error = ret; 397 spin_lock(&fsvq->lock); 398 dec_in_flight_req(fsvq); 399 spin_unlock(&fsvq->lock); 400 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 401 ret); 402 fuse_request_end(req); 403 } 404 } 405 } 406 407 /* 408 * Returns 1 if queue is full and sender should wait a bit before sending 409 * next request, 0 otherwise. 410 */ 411 static int send_forget_request(struct virtio_fs_vq *fsvq, 412 struct virtio_fs_forget *forget, 413 bool in_flight) 414 { 415 struct scatterlist sg; 416 struct virtqueue *vq; 417 int ret = 0; 418 bool notify; 419 struct virtio_fs_forget_req *req = &forget->req; 420 421 spin_lock(&fsvq->lock); 422 if (!fsvq->connected) { 423 if (in_flight) 424 dec_in_flight_req(fsvq); 425 kfree(forget); 426 goto out; 427 } 428 429 sg_init_one(&sg, req, sizeof(*req)); 430 vq = fsvq->vq; 431 dev_dbg(&vq->vdev->dev, "%s\n", __func__); 432 433 ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 434 if (ret < 0) { 435 if (ret == -ENOMEM || ret == -ENOSPC) { 436 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 437 ret); 438 list_add_tail(&forget->list, &fsvq->queued_reqs); 439 schedule_delayed_work(&fsvq->dispatch_work, 440 msecs_to_jiffies(1)); 441 if (!in_flight) 442 inc_in_flight_req(fsvq); 443 /* Queue is full */ 444 ret = 1; 445 } else { 446 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 447 ret); 448 kfree(forget); 449 if (in_flight) 450 dec_in_flight_req(fsvq); 451 } 452 goto out; 453 } 454 455 if (!in_flight) 456 inc_in_flight_req(fsvq); 457 notify = virtqueue_kick_prepare(vq); 458 spin_unlock(&fsvq->lock); 459 460 if (notify) 461 virtqueue_notify(vq); 462 return ret; 463 out: 464 spin_unlock(&fsvq->lock); 465 return ret; 466 } 467 468 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 469 { 470 struct virtio_fs_forget *forget; 471 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 472 dispatch_work.work); 473 pr_debug("virtio-fs: worker %s called.\n", __func__); 474 while (1) { 475 spin_lock(&fsvq->lock); 476 forget = list_first_entry_or_null(&fsvq->queued_reqs, 477 struct virtio_fs_forget, list); 478 if (!forget) { 479 spin_unlock(&fsvq->lock); 480 return; 481 } 482 483 list_del(&forget->list); 484 spin_unlock(&fsvq->lock); 485 if (send_forget_request(fsvq, forget, true)) 486 return; 487 } 488 } 489 490 /* Allocate and copy args into req->argbuf */ 491 static int copy_args_to_argbuf(struct fuse_req *req) 492 { 493 struct fuse_args *args = req->args; 494 unsigned int offset = 0; 495 unsigned int num_in; 496 unsigned int num_out; 497 unsigned int len; 498 unsigned int i; 499 500 num_in = args->in_numargs - args->in_pages; 501 num_out = args->out_numargs - args->out_pages; 502 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 503 fuse_len_args(num_out, args->out_args); 504 505 req->argbuf = kmalloc(len, GFP_ATOMIC); 506 if (!req->argbuf) 507 return -ENOMEM; 508 509 for (i = 0; i < num_in; i++) { 510 memcpy(req->argbuf + offset, 511 args->in_args[i].value, 512 args->in_args[i].size); 513 offset += args->in_args[i].size; 514 } 515 516 return 0; 517 } 518 519 /* Copy args out of and free req->argbuf */ 520 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 521 { 522 unsigned int remaining; 523 unsigned int offset; 524 unsigned int num_in; 525 unsigned int num_out; 526 unsigned int i; 527 528 remaining = req->out.h.len - sizeof(req->out.h); 529 num_in = args->in_numargs - args->in_pages; 530 num_out = args->out_numargs - args->out_pages; 531 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 532 533 for (i = 0; i < num_out; i++) { 534 unsigned int argsize = args->out_args[i].size; 535 536 if (args->out_argvar && 537 i == args->out_numargs - 1 && 538 argsize > remaining) { 539 argsize = remaining; 540 } 541 542 memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 543 offset += argsize; 544 545 if (i != args->out_numargs - 1) 546 remaining -= argsize; 547 } 548 549 /* Store the actual size of the variable-length arg */ 550 if (args->out_argvar) 551 args->out_args[args->out_numargs - 1].size = remaining; 552 553 kfree(req->argbuf); 554 req->argbuf = NULL; 555 } 556 557 /* Work function for request completion */ 558 static void virtio_fs_request_complete(struct fuse_req *req, 559 struct virtio_fs_vq *fsvq) 560 { 561 struct fuse_pqueue *fpq = &fsvq->fud->pq; 562 struct fuse_args *args; 563 struct fuse_args_pages *ap; 564 unsigned int len, i, thislen; 565 struct page *page; 566 567 /* 568 * TODO verify that server properly follows FUSE protocol 569 * (oh.uniq, oh.len) 570 */ 571 args = req->args; 572 copy_args_from_argbuf(args, req); 573 574 if (args->out_pages && args->page_zeroing) { 575 len = args->out_args[args->out_numargs - 1].size; 576 ap = container_of(args, typeof(*ap), args); 577 for (i = 0; i < ap->num_pages; i++) { 578 thislen = ap->descs[i].length; 579 if (len < thislen) { 580 WARN_ON(ap->descs[i].offset); 581 page = ap->pages[i]; 582 zero_user_segment(page, len, thislen); 583 len = 0; 584 } else { 585 len -= thislen; 586 } 587 } 588 } 589 590 spin_lock(&fpq->lock); 591 clear_bit(FR_SENT, &req->flags); 592 spin_unlock(&fpq->lock); 593 594 fuse_request_end(req); 595 spin_lock(&fsvq->lock); 596 dec_in_flight_req(fsvq); 597 spin_unlock(&fsvq->lock); 598 } 599 600 static void virtio_fs_complete_req_work(struct work_struct *work) 601 { 602 struct virtio_fs_req_work *w = 603 container_of(work, typeof(*w), done_work); 604 605 virtio_fs_request_complete(w->req, w->fsvq); 606 kfree(w); 607 } 608 609 static void virtio_fs_requests_done_work(struct work_struct *work) 610 { 611 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 612 done_work); 613 struct fuse_pqueue *fpq = &fsvq->fud->pq; 614 struct virtqueue *vq = fsvq->vq; 615 struct fuse_req *req; 616 struct fuse_req *next; 617 unsigned int len; 618 LIST_HEAD(reqs); 619 620 /* Collect completed requests off the virtqueue */ 621 spin_lock(&fsvq->lock); 622 do { 623 virtqueue_disable_cb(vq); 624 625 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 626 spin_lock(&fpq->lock); 627 list_move_tail(&req->list, &reqs); 628 spin_unlock(&fpq->lock); 629 } 630 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 631 spin_unlock(&fsvq->lock); 632 633 /* End requests */ 634 list_for_each_entry_safe(req, next, &reqs, list) { 635 list_del_init(&req->list); 636 637 /* blocking async request completes in a worker context */ 638 if (req->args->may_block) { 639 struct virtio_fs_req_work *w; 640 641 w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL); 642 INIT_WORK(&w->done_work, virtio_fs_complete_req_work); 643 w->fsvq = fsvq; 644 w->req = req; 645 schedule_work(&w->done_work); 646 } else { 647 virtio_fs_request_complete(req, fsvq); 648 } 649 } 650 } 651 652 /* Virtqueue interrupt handler */ 653 static void virtio_fs_vq_done(struct virtqueue *vq) 654 { 655 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 656 657 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 658 659 schedule_work(&fsvq->done_work); 660 } 661 662 static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, 663 int vq_type) 664 { 665 strscpy(fsvq->name, name, VQ_NAME_LEN); 666 spin_lock_init(&fsvq->lock); 667 INIT_LIST_HEAD(&fsvq->queued_reqs); 668 INIT_LIST_HEAD(&fsvq->end_reqs); 669 init_completion(&fsvq->in_flight_zero); 670 671 if (vq_type == VQ_REQUEST) { 672 INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); 673 INIT_DELAYED_WORK(&fsvq->dispatch_work, 674 virtio_fs_request_dispatch_work); 675 } else { 676 INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); 677 INIT_DELAYED_WORK(&fsvq->dispatch_work, 678 virtio_fs_hiprio_dispatch_work); 679 } 680 } 681 682 /* Initialize virtqueues */ 683 static int virtio_fs_setup_vqs(struct virtio_device *vdev, 684 struct virtio_fs *fs) 685 { 686 struct virtqueue **vqs; 687 vq_callback_t **callbacks; 688 const char **names; 689 unsigned int i; 690 int ret = 0; 691 692 virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, 693 &fs->num_request_queues); 694 if (fs->num_request_queues == 0) 695 return -EINVAL; 696 697 fs->nvqs = VQ_REQUEST + fs->num_request_queues; 698 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 699 if (!fs->vqs) 700 return -ENOMEM; 701 702 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 703 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 704 GFP_KERNEL); 705 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 706 if (!vqs || !callbacks || !names) { 707 ret = -ENOMEM; 708 goto out; 709 } 710 711 /* Initialize the hiprio/forget request virtqueue */ 712 callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 713 virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); 714 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 715 716 /* Initialize the requests virtqueues */ 717 for (i = VQ_REQUEST; i < fs->nvqs; i++) { 718 char vq_name[VQ_NAME_LEN]; 719 720 snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); 721 virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); 722 callbacks[i] = virtio_fs_vq_done; 723 names[i] = fs->vqs[i].name; 724 } 725 726 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 727 if (ret < 0) 728 goto out; 729 730 for (i = 0; i < fs->nvqs; i++) 731 fs->vqs[i].vq = vqs[i]; 732 733 virtio_fs_start_all_queues(fs); 734 out: 735 kfree(names); 736 kfree(callbacks); 737 kfree(vqs); 738 if (ret) 739 kfree(fs->vqs); 740 return ret; 741 } 742 743 /* Free virtqueues (device must already be reset) */ 744 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 745 struct virtio_fs *fs) 746 { 747 vdev->config->del_vqs(vdev); 748 } 749 750 /* Map a window offset to a page frame number. The window offset will have 751 * been produced by .iomap_begin(), which maps a file offset to a window 752 * offset. 753 */ 754 static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 755 long nr_pages, void **kaddr, pfn_t *pfn) 756 { 757 struct virtio_fs *fs = dax_get_private(dax_dev); 758 phys_addr_t offset = PFN_PHYS(pgoff); 759 size_t max_nr_pages = fs->window_len/PAGE_SIZE - pgoff; 760 761 if (kaddr) 762 *kaddr = fs->window_kaddr + offset; 763 if (pfn) 764 *pfn = phys_to_pfn_t(fs->window_phys_addr + offset, 765 PFN_DEV | PFN_MAP); 766 return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; 767 } 768 769 static int virtio_fs_zero_page_range(struct dax_device *dax_dev, 770 pgoff_t pgoff, size_t nr_pages) 771 { 772 long rc; 773 void *kaddr; 774 775 rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL); 776 if (rc < 0) 777 return rc; 778 memset(kaddr, 0, nr_pages << PAGE_SHIFT); 779 dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); 780 return 0; 781 } 782 783 static const struct dax_operations virtio_fs_dax_ops = { 784 .direct_access = virtio_fs_direct_access, 785 .zero_page_range = virtio_fs_zero_page_range, 786 }; 787 788 static void virtio_fs_cleanup_dax(void *data) 789 { 790 struct dax_device *dax_dev = data; 791 792 kill_dax(dax_dev); 793 put_dax(dax_dev); 794 } 795 796 static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) 797 { 798 struct virtio_shm_region cache_reg; 799 struct dev_pagemap *pgmap; 800 bool have_cache; 801 802 if (!IS_ENABLED(CONFIG_FUSE_DAX)) 803 return 0; 804 805 /* Get cache region */ 806 have_cache = virtio_get_shm_region(vdev, &cache_reg, 807 (u8)VIRTIO_FS_SHMCAP_ID_CACHE); 808 if (!have_cache) { 809 dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); 810 return 0; 811 } 812 813 if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, 814 dev_name(&vdev->dev))) { 815 dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", 816 cache_reg.addr, cache_reg.len); 817 return -EBUSY; 818 } 819 820 dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, 821 cache_reg.addr); 822 823 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); 824 if (!pgmap) 825 return -ENOMEM; 826 827 pgmap->type = MEMORY_DEVICE_FS_DAX; 828 829 /* Ideally we would directly use the PCI BAR resource but 830 * devm_memremap_pages() wants its own copy in pgmap. So 831 * initialize a struct resource from scratch (only the start 832 * and end fields will be used). 833 */ 834 pgmap->range = (struct range) { 835 .start = (phys_addr_t) cache_reg.addr, 836 .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, 837 }; 838 pgmap->nr_range = 1; 839 840 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); 841 if (IS_ERR(fs->window_kaddr)) 842 return PTR_ERR(fs->window_kaddr); 843 844 fs->window_phys_addr = (phys_addr_t) cache_reg.addr; 845 fs->window_len = (phys_addr_t) cache_reg.len; 846 847 dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", 848 __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); 849 850 fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops); 851 if (IS_ERR(fs->dax_dev)) 852 return PTR_ERR(fs->dax_dev); 853 854 return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, 855 fs->dax_dev); 856 } 857 858 static int virtio_fs_probe(struct virtio_device *vdev) 859 { 860 struct virtio_fs *fs; 861 int ret; 862 863 fs = kzalloc(sizeof(*fs), GFP_KERNEL); 864 if (!fs) 865 return -ENOMEM; 866 kref_init(&fs->refcount); 867 vdev->priv = fs; 868 869 ret = virtio_fs_read_tag(vdev, fs); 870 if (ret < 0) 871 goto out; 872 873 ret = virtio_fs_setup_vqs(vdev, fs); 874 if (ret < 0) 875 goto out; 876 877 /* TODO vq affinity */ 878 879 ret = virtio_fs_setup_dax(vdev, fs); 880 if (ret < 0) 881 goto out_vqs; 882 883 /* Bring the device online in case the filesystem is mounted and 884 * requests need to be sent before we return. 885 */ 886 virtio_device_ready(vdev); 887 888 ret = virtio_fs_add_instance(fs); 889 if (ret < 0) 890 goto out_vqs; 891 892 return 0; 893 894 out_vqs: 895 virtio_reset_device(vdev); 896 virtio_fs_cleanup_vqs(vdev, fs); 897 kfree(fs->vqs); 898 899 out: 900 vdev->priv = NULL; 901 kfree(fs); 902 return ret; 903 } 904 905 static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 906 { 907 struct virtio_fs_vq *fsvq; 908 int i; 909 910 for (i = 0; i < fs->nvqs; i++) { 911 fsvq = &fs->vqs[i]; 912 spin_lock(&fsvq->lock); 913 fsvq->connected = false; 914 spin_unlock(&fsvq->lock); 915 } 916 } 917 918 static void virtio_fs_remove(struct virtio_device *vdev) 919 { 920 struct virtio_fs *fs = vdev->priv; 921 922 mutex_lock(&virtio_fs_mutex); 923 /* This device is going away. No one should get new reference */ 924 list_del_init(&fs->list); 925 virtio_fs_stop_all_queues(fs); 926 virtio_fs_drain_all_queues_locked(fs); 927 virtio_reset_device(vdev); 928 virtio_fs_cleanup_vqs(vdev, fs); 929 930 vdev->priv = NULL; 931 /* Put device reference on virtio_fs object */ 932 virtio_fs_put(fs); 933 mutex_unlock(&virtio_fs_mutex); 934 } 935 936 #ifdef CONFIG_PM_SLEEP 937 static int virtio_fs_freeze(struct virtio_device *vdev) 938 { 939 /* TODO need to save state here */ 940 pr_warn("virtio-fs: suspend/resume not yet supported\n"); 941 return -EOPNOTSUPP; 942 } 943 944 static int virtio_fs_restore(struct virtio_device *vdev) 945 { 946 /* TODO need to restore state here */ 947 return 0; 948 } 949 #endif /* CONFIG_PM_SLEEP */ 950 951 static const struct virtio_device_id id_table[] = { 952 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 953 {}, 954 }; 955 956 static const unsigned int feature_table[] = {}; 957 958 static struct virtio_driver virtio_fs_driver = { 959 .driver.name = KBUILD_MODNAME, 960 .driver.owner = THIS_MODULE, 961 .id_table = id_table, 962 .feature_table = feature_table, 963 .feature_table_size = ARRAY_SIZE(feature_table), 964 .probe = virtio_fs_probe, 965 .remove = virtio_fs_remove, 966 #ifdef CONFIG_PM_SLEEP 967 .freeze = virtio_fs_freeze, 968 .restore = virtio_fs_restore, 969 #endif 970 }; 971 972 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 973 __releases(fiq->lock) 974 { 975 struct fuse_forget_link *link; 976 struct virtio_fs_forget *forget; 977 struct virtio_fs_forget_req *req; 978 struct virtio_fs *fs; 979 struct virtio_fs_vq *fsvq; 980 u64 unique; 981 982 link = fuse_dequeue_forget(fiq, 1, NULL); 983 unique = fuse_get_unique(fiq); 984 985 fs = fiq->priv; 986 fsvq = &fs->vqs[VQ_HIPRIO]; 987 spin_unlock(&fiq->lock); 988 989 /* Allocate a buffer for the request */ 990 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 991 req = &forget->req; 992 993 req->ih = (struct fuse_in_header){ 994 .opcode = FUSE_FORGET, 995 .nodeid = link->forget_one.nodeid, 996 .unique = unique, 997 .len = sizeof(*req), 998 }; 999 req->arg = (struct fuse_forget_in){ 1000 .nlookup = link->forget_one.nlookup, 1001 }; 1002 1003 send_forget_request(fsvq, forget, false); 1004 kfree(link); 1005 } 1006 1007 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 1008 __releases(fiq->lock) 1009 { 1010 /* 1011 * TODO interrupts. 1012 * 1013 * Normal fs operations on a local filesystems aren't interruptible. 1014 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 1015 * with shared lock between host and guest. 1016 */ 1017 spin_unlock(&fiq->lock); 1018 } 1019 1020 /* Count number of scatter-gather elements required */ 1021 static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, 1022 unsigned int num_pages, 1023 unsigned int total_len) 1024 { 1025 unsigned int i; 1026 unsigned int this_len; 1027 1028 for (i = 0; i < num_pages && total_len; i++) { 1029 this_len = min(page_descs[i].length, total_len); 1030 total_len -= this_len; 1031 } 1032 1033 return i; 1034 } 1035 1036 /* Return the number of scatter-gather list elements required */ 1037 static unsigned int sg_count_fuse_req(struct fuse_req *req) 1038 { 1039 struct fuse_args *args = req->args; 1040 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 1041 unsigned int size, total_sgs = 1 /* fuse_in_header */; 1042 1043 if (args->in_numargs - args->in_pages) 1044 total_sgs += 1; 1045 1046 if (args->in_pages) { 1047 size = args->in_args[args->in_numargs - 1].size; 1048 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1049 size); 1050 } 1051 1052 if (!test_bit(FR_ISREPLY, &req->flags)) 1053 return total_sgs; 1054 1055 total_sgs += 1 /* fuse_out_header */; 1056 1057 if (args->out_numargs - args->out_pages) 1058 total_sgs += 1; 1059 1060 if (args->out_pages) { 1061 size = args->out_args[args->out_numargs - 1].size; 1062 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1063 size); 1064 } 1065 1066 return total_sgs; 1067 } 1068 1069 /* Add pages to scatter-gather list and return number of elements used */ 1070 static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 1071 struct page **pages, 1072 struct fuse_page_desc *page_descs, 1073 unsigned int num_pages, 1074 unsigned int total_len) 1075 { 1076 unsigned int i; 1077 unsigned int this_len; 1078 1079 for (i = 0; i < num_pages && total_len; i++) { 1080 sg_init_table(&sg[i], 1); 1081 this_len = min(page_descs[i].length, total_len); 1082 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 1083 total_len -= this_len; 1084 } 1085 1086 return i; 1087 } 1088 1089 /* Add args to scatter-gather list and return number of elements used */ 1090 static unsigned int sg_init_fuse_args(struct scatterlist *sg, 1091 struct fuse_req *req, 1092 struct fuse_arg *args, 1093 unsigned int numargs, 1094 bool argpages, 1095 void *argbuf, 1096 unsigned int *len_used) 1097 { 1098 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 1099 unsigned int total_sgs = 0; 1100 unsigned int len; 1101 1102 len = fuse_len_args(numargs - argpages, args); 1103 if (len) 1104 sg_init_one(&sg[total_sgs++], argbuf, len); 1105 1106 if (argpages) 1107 total_sgs += sg_init_fuse_pages(&sg[total_sgs], 1108 ap->pages, ap->descs, 1109 ap->num_pages, 1110 args[numargs - 1].size); 1111 1112 if (len_used) 1113 *len_used = len; 1114 1115 return total_sgs; 1116 } 1117 1118 /* Add a request to a virtqueue and kick the device */ 1119 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 1120 struct fuse_req *req, bool in_flight) 1121 { 1122 /* requests need at least 4 elements */ 1123 struct scatterlist *stack_sgs[6]; 1124 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 1125 struct scatterlist **sgs = stack_sgs; 1126 struct scatterlist *sg = stack_sg; 1127 struct virtqueue *vq; 1128 struct fuse_args *args = req->args; 1129 unsigned int argbuf_used = 0; 1130 unsigned int out_sgs = 0; 1131 unsigned int in_sgs = 0; 1132 unsigned int total_sgs; 1133 unsigned int i; 1134 int ret; 1135 bool notify; 1136 struct fuse_pqueue *fpq; 1137 1138 /* Does the sglist fit on the stack? */ 1139 total_sgs = sg_count_fuse_req(req); 1140 if (total_sgs > ARRAY_SIZE(stack_sgs)) { 1141 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 1142 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 1143 if (!sgs || !sg) { 1144 ret = -ENOMEM; 1145 goto out; 1146 } 1147 } 1148 1149 /* Use a bounce buffer since stack args cannot be mapped */ 1150 ret = copy_args_to_argbuf(req); 1151 if (ret < 0) 1152 goto out; 1153 1154 /* Request elements */ 1155 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 1156 out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 1157 (struct fuse_arg *)args->in_args, 1158 args->in_numargs, args->in_pages, 1159 req->argbuf, &argbuf_used); 1160 1161 /* Reply elements */ 1162 if (test_bit(FR_ISREPLY, &req->flags)) { 1163 sg_init_one(&sg[out_sgs + in_sgs++], 1164 &req->out.h, sizeof(req->out.h)); 1165 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 1166 args->out_args, args->out_numargs, 1167 args->out_pages, 1168 req->argbuf + argbuf_used, NULL); 1169 } 1170 1171 WARN_ON(out_sgs + in_sgs != total_sgs); 1172 1173 for (i = 0; i < total_sgs; i++) 1174 sgs[i] = &sg[i]; 1175 1176 spin_lock(&fsvq->lock); 1177 1178 if (!fsvq->connected) { 1179 spin_unlock(&fsvq->lock); 1180 ret = -ENOTCONN; 1181 goto out; 1182 } 1183 1184 vq = fsvq->vq; 1185 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 1186 if (ret < 0) { 1187 spin_unlock(&fsvq->lock); 1188 goto out; 1189 } 1190 1191 /* Request successfully sent. */ 1192 fpq = &fsvq->fud->pq; 1193 spin_lock(&fpq->lock); 1194 list_add_tail(&req->list, fpq->processing); 1195 spin_unlock(&fpq->lock); 1196 set_bit(FR_SENT, &req->flags); 1197 /* matches barrier in request_wait_answer() */ 1198 smp_mb__after_atomic(); 1199 1200 if (!in_flight) 1201 inc_in_flight_req(fsvq); 1202 notify = virtqueue_kick_prepare(vq); 1203 1204 spin_unlock(&fsvq->lock); 1205 1206 if (notify) 1207 virtqueue_notify(vq); 1208 1209 out: 1210 if (ret < 0 && req->argbuf) { 1211 kfree(req->argbuf); 1212 req->argbuf = NULL; 1213 } 1214 if (sgs != stack_sgs) { 1215 kfree(sgs); 1216 kfree(sg); 1217 } 1218 1219 return ret; 1220 } 1221 1222 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 1223 __releases(fiq->lock) 1224 { 1225 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 1226 struct virtio_fs *fs; 1227 struct fuse_req *req; 1228 struct virtio_fs_vq *fsvq; 1229 int ret; 1230 1231 WARN_ON(list_empty(&fiq->pending)); 1232 req = list_last_entry(&fiq->pending, struct fuse_req, list); 1233 clear_bit(FR_PENDING, &req->flags); 1234 list_del_init(&req->list); 1235 WARN_ON(!list_empty(&fiq->pending)); 1236 spin_unlock(&fiq->lock); 1237 1238 fs = fiq->priv; 1239 1240 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 1241 __func__, req->in.h.opcode, req->in.h.unique, 1242 req->in.h.nodeid, req->in.h.len, 1243 fuse_len_args(req->args->out_numargs, req->args->out_args)); 1244 1245 fsvq = &fs->vqs[queue_id]; 1246 ret = virtio_fs_enqueue_req(fsvq, req, false); 1247 if (ret < 0) { 1248 if (ret == -ENOMEM || ret == -ENOSPC) { 1249 /* 1250 * Virtqueue full. Retry submission from worker 1251 * context as we might be holding fc->bg_lock. 1252 */ 1253 spin_lock(&fsvq->lock); 1254 list_add_tail(&req->list, &fsvq->queued_reqs); 1255 inc_in_flight_req(fsvq); 1256 schedule_delayed_work(&fsvq->dispatch_work, 1257 msecs_to_jiffies(1)); 1258 spin_unlock(&fsvq->lock); 1259 return; 1260 } 1261 req->out.h.error = ret; 1262 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 1263 1264 /* Can't end request in submission context. Use a worker */ 1265 spin_lock(&fsvq->lock); 1266 list_add_tail(&req->list, &fsvq->end_reqs); 1267 schedule_delayed_work(&fsvq->dispatch_work, 0); 1268 spin_unlock(&fsvq->lock); 1269 return; 1270 } 1271 } 1272 1273 static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1274 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1275 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1276 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1277 .release = virtio_fs_fiq_release, 1278 }; 1279 1280 static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx) 1281 { 1282 ctx->rootmode = S_IFDIR; 1283 ctx->default_permissions = 1; 1284 ctx->allow_other = 1; 1285 ctx->max_read = UINT_MAX; 1286 ctx->blksize = 512; 1287 ctx->destroy = true; 1288 ctx->no_control = true; 1289 ctx->no_force_umount = true; 1290 } 1291 1292 static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) 1293 { 1294 struct fuse_mount *fm = get_fuse_mount_super(sb); 1295 struct fuse_conn *fc = fm->fc; 1296 struct virtio_fs *fs = fc->iq.priv; 1297 struct fuse_fs_context *ctx = fsc->fs_private; 1298 unsigned int i; 1299 int err; 1300 1301 virtio_fs_ctx_set_defaults(ctx); 1302 mutex_lock(&virtio_fs_mutex); 1303 1304 /* After holding mutex, make sure virtiofs device is still there. 1305 * Though we are holding a reference to it, drive ->remove might 1306 * still have cleaned up virtual queues. In that case bail out. 1307 */ 1308 err = -EINVAL; 1309 if (list_empty(&fs->list)) { 1310 pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1311 goto err; 1312 } 1313 1314 err = -ENOMEM; 1315 /* Allocate fuse_dev for hiprio and notification queues */ 1316 for (i = 0; i < fs->nvqs; i++) { 1317 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1318 1319 fsvq->fud = fuse_dev_alloc(); 1320 if (!fsvq->fud) 1321 goto err_free_fuse_devs; 1322 } 1323 1324 /* virtiofs allocates and installs its own fuse devices */ 1325 ctx->fudptr = NULL; 1326 if (ctx->dax_mode != FUSE_DAX_NEVER) { 1327 if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) { 1328 err = -EINVAL; 1329 pr_err("virtio-fs: dax can't be enabled as filesystem" 1330 " device does not support it.\n"); 1331 goto err_free_fuse_devs; 1332 } 1333 ctx->dax_dev = fs->dax_dev; 1334 } 1335 err = fuse_fill_super_common(sb, ctx); 1336 if (err < 0) 1337 goto err_free_fuse_devs; 1338 1339 for (i = 0; i < fs->nvqs; i++) { 1340 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1341 1342 fuse_dev_install(fsvq->fud, fc); 1343 } 1344 1345 /* Previous unmount will stop all queues. Start these again */ 1346 virtio_fs_start_all_queues(fs); 1347 fuse_send_init(fm); 1348 mutex_unlock(&virtio_fs_mutex); 1349 return 0; 1350 1351 err_free_fuse_devs: 1352 virtio_fs_free_devs(fs); 1353 err: 1354 mutex_unlock(&virtio_fs_mutex); 1355 return err; 1356 } 1357 1358 static void virtio_fs_conn_destroy(struct fuse_mount *fm) 1359 { 1360 struct fuse_conn *fc = fm->fc; 1361 struct virtio_fs *vfs = fc->iq.priv; 1362 struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; 1363 1364 /* Stop dax worker. Soon evict_inodes() will be called which 1365 * will free all memory ranges belonging to all inodes. 1366 */ 1367 if (IS_ENABLED(CONFIG_FUSE_DAX)) 1368 fuse_dax_cancel_work(fc); 1369 1370 /* Stop forget queue. Soon destroy will be sent */ 1371 spin_lock(&fsvq->lock); 1372 fsvq->connected = false; 1373 spin_unlock(&fsvq->lock); 1374 virtio_fs_drain_all_queues(vfs); 1375 1376 fuse_conn_destroy(fm); 1377 1378 /* fuse_conn_destroy() must have sent destroy. Stop all queues 1379 * and drain one more time and free fuse devices. Freeing fuse 1380 * devices will drop their reference on fuse_conn and that in 1381 * turn will drop its reference on virtio_fs object. 1382 */ 1383 virtio_fs_stop_all_queues(vfs); 1384 virtio_fs_drain_all_queues(vfs); 1385 virtio_fs_free_devs(vfs); 1386 } 1387 1388 static void virtio_kill_sb(struct super_block *sb) 1389 { 1390 struct fuse_mount *fm = get_fuse_mount_super(sb); 1391 bool last; 1392 1393 /* If mount failed, we can still be called without any fc */ 1394 if (sb->s_root) { 1395 last = fuse_mount_remove(fm); 1396 if (last) 1397 virtio_fs_conn_destroy(fm); 1398 } 1399 kill_anon_super(sb); 1400 fuse_mount_destroy(fm); 1401 } 1402 1403 static int virtio_fs_test_super(struct super_block *sb, 1404 struct fs_context *fsc) 1405 { 1406 struct fuse_mount *fsc_fm = fsc->s_fs_info; 1407 struct fuse_mount *sb_fm = get_fuse_mount_super(sb); 1408 1409 return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; 1410 } 1411 1412 static int virtio_fs_get_tree(struct fs_context *fsc) 1413 { 1414 struct virtio_fs *fs; 1415 struct super_block *sb; 1416 struct fuse_conn *fc = NULL; 1417 struct fuse_mount *fm; 1418 unsigned int virtqueue_size; 1419 int err = -EIO; 1420 1421 /* This gets a reference on virtio_fs object. This ptr gets installed 1422 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1423 * to drop the reference to this object. 1424 */ 1425 fs = virtio_fs_find_instance(fsc->source); 1426 if (!fs) { 1427 pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1428 return -EINVAL; 1429 } 1430 1431 virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq); 1432 if (WARN_ON(virtqueue_size <= FUSE_HEADER_OVERHEAD)) 1433 goto out_err; 1434 1435 err = -ENOMEM; 1436 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1437 if (!fc) 1438 goto out_err; 1439 1440 fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL); 1441 if (!fm) 1442 goto out_err; 1443 1444 fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); 1445 fc->release = fuse_free_conn; 1446 fc->delete_stale = true; 1447 fc->auto_submounts = true; 1448 fc->sync_fs = true; 1449 1450 /* Tell FUSE to split requests that exceed the virtqueue's size */ 1451 fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, 1452 virtqueue_size - FUSE_HEADER_OVERHEAD); 1453 1454 fsc->s_fs_info = fm; 1455 sb = sget_fc(fsc, virtio_fs_test_super, set_anon_super_fc); 1456 if (fsc->s_fs_info) 1457 fuse_mount_destroy(fm); 1458 if (IS_ERR(sb)) 1459 return PTR_ERR(sb); 1460 1461 if (!sb->s_root) { 1462 err = virtio_fs_fill_super(sb, fsc); 1463 if (err) { 1464 deactivate_locked_super(sb); 1465 return err; 1466 } 1467 1468 sb->s_flags |= SB_ACTIVE; 1469 } 1470 1471 WARN_ON(fsc->root); 1472 fsc->root = dget(sb->s_root); 1473 return 0; 1474 1475 out_err: 1476 kfree(fc); 1477 mutex_lock(&virtio_fs_mutex); 1478 virtio_fs_put(fs); 1479 mutex_unlock(&virtio_fs_mutex); 1480 return err; 1481 } 1482 1483 static const struct fs_context_operations virtio_fs_context_ops = { 1484 .free = virtio_fs_free_fsc, 1485 .parse_param = virtio_fs_parse_param, 1486 .get_tree = virtio_fs_get_tree, 1487 }; 1488 1489 static int virtio_fs_init_fs_context(struct fs_context *fsc) 1490 { 1491 struct fuse_fs_context *ctx; 1492 1493 if (fsc->purpose == FS_CONTEXT_FOR_SUBMOUNT) 1494 return fuse_init_fs_context_submount(fsc); 1495 1496 ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL); 1497 if (!ctx) 1498 return -ENOMEM; 1499 fsc->fs_private = ctx; 1500 fsc->ops = &virtio_fs_context_ops; 1501 return 0; 1502 } 1503 1504 static struct file_system_type virtio_fs_type = { 1505 .owner = THIS_MODULE, 1506 .name = "virtiofs", 1507 .init_fs_context = virtio_fs_init_fs_context, 1508 .kill_sb = virtio_kill_sb, 1509 }; 1510 1511 static int __init virtio_fs_init(void) 1512 { 1513 int ret; 1514 1515 ret = register_virtio_driver(&virtio_fs_driver); 1516 if (ret < 0) 1517 return ret; 1518 1519 ret = register_filesystem(&virtio_fs_type); 1520 if (ret < 0) { 1521 unregister_virtio_driver(&virtio_fs_driver); 1522 return ret; 1523 } 1524 1525 return 0; 1526 } 1527 module_init(virtio_fs_init); 1528 1529 static void __exit virtio_fs_exit(void) 1530 { 1531 unregister_filesystem(&virtio_fs_type); 1532 unregister_virtio_driver(&virtio_fs_driver); 1533 } 1534 module_exit(virtio_fs_exit); 1535 1536 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1537 MODULE_DESCRIPTION("Virtio Filesystem"); 1538 MODULE_LICENSE("GPL"); 1539 MODULE_ALIAS_FS(KBUILD_MODNAME); 1540 MODULE_DEVICE_TABLE(virtio, id_table); 1541