Lines Matching +full:cs +full:- +full:out
3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
45 return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout); in fuse_request_expired()
62 * - check the fiq pending list
63 * - check the bg queue
64 * - check the fpq io and processing lists
69 * between lists, re-sent requests at the head of the pending list having a
78 struct fuse_iqueue *fiq = &fc->iq; in fuse_check_timeout()
83 if (!atomic_read(&fc->num_waiting)) in fuse_check_timeout()
84 goto out; in fuse_check_timeout()
86 spin_lock(&fiq->lock); in fuse_check_timeout()
87 expired = fuse_request_expired(fc, &fiq->pending); in fuse_check_timeout()
88 spin_unlock(&fiq->lock); in fuse_check_timeout()
92 spin_lock(&fc->bg_lock); in fuse_check_timeout()
93 expired = fuse_request_expired(fc, &fc->bg_queue); in fuse_check_timeout()
94 spin_unlock(&fc->bg_lock); in fuse_check_timeout()
98 spin_lock(&fc->lock); in fuse_check_timeout()
99 if (!fc->connected) { in fuse_check_timeout()
100 spin_unlock(&fc->lock); in fuse_check_timeout()
103 list_for_each_entry(fud, &fc->devices, entry) { in fuse_check_timeout()
104 fpq = &fud->pq; in fuse_check_timeout()
105 spin_lock(&fpq->lock); in fuse_check_timeout()
106 if (fuse_request_expired(fc, &fpq->io) || in fuse_check_timeout()
107 fuse_fpq_processing_expired(fc, fpq->processing)) { in fuse_check_timeout()
108 spin_unlock(&fpq->lock); in fuse_check_timeout()
109 spin_unlock(&fc->lock); in fuse_check_timeout()
113 spin_unlock(&fpq->lock); in fuse_check_timeout()
115 spin_unlock(&fc->lock); in fuse_check_timeout()
120 out: in fuse_check_timeout()
121 queue_delayed_work(system_percpu_wq, &fc->timeout.work, in fuse_check_timeout()
131 INIT_LIST_HEAD(&req->list); in fuse_request_init()
132 INIT_LIST_HEAD(&req->intr_entry); in fuse_request_init()
133 init_waitqueue_head(&req->waitq); in fuse_request_init()
134 refcount_set(&req->count, 1); in fuse_request_init()
135 __set_bit(FR_PENDING, &req->flags); in fuse_request_init()
136 req->fm = fm; in fuse_request_init()
137 req->create_time = jiffies; in fuse_request_init()
156 refcount_inc(&req->count); in __fuse_get_request()
162 refcount_dec(&req->count); in __fuse_put_request()
169 fc->initialized = 1; in fuse_set_initialized()
174 return !fc->initialized || (for_background && fc->blocked) || in fuse_block_alloc()
175 (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); in fuse_block_alloc()
181 * lockess check of fc->connected is okay, because atomic_dec_and_test() in fuse_drop_waiting()
183 * to ensure no wake-up is missed. in fuse_drop_waiting()
185 if (atomic_dec_and_test(&fc->num_waiting) && in fuse_drop_waiting()
186 !READ_ONCE(fc->connected)) { in fuse_drop_waiting()
188 wake_up_all(&fc->blocked_waitq); in fuse_drop_waiting()
198 struct fuse_conn *fc = fm->fc; in fuse_get_req()
200 bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); in fuse_get_req()
205 atomic_inc(&fc->num_waiting); in fuse_get_req()
208 err = -EINTR; in fuse_get_req()
209 if (wait_event_state_exclusive(fc->blocked_waitq, in fuse_get_req()
212 goto out; in fuse_get_req()
217 err = -ENOTCONN; in fuse_get_req()
218 if (!fc->connected) in fuse_get_req()
219 goto out; in fuse_get_req()
221 err = -ECONNREFUSED; in fuse_get_req()
222 if (fc->conn_error) in fuse_get_req()
223 goto out; in fuse_get_req()
226 err = -ENOMEM; in fuse_get_req()
229 wake_up(&fc->blocked_waitq); in fuse_get_req()
230 goto out; in fuse_get_req()
233 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); in fuse_get_req()
235 __set_bit(FR_WAITING, &req->flags); in fuse_get_req()
237 __set_bit(FR_BACKGROUND, &req->flags); in fuse_get_req()
246 * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID. in fuse_get_req()
248 fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns); in fuse_get_req()
249 fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns); in fuse_get_req()
250 req->in.h.uid = from_kuid(fc->user_ns, fsuid); in fuse_get_req()
251 req->in.h.gid = from_kgid(fc->user_ns, fsgid); in fuse_get_req()
253 if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) || in fuse_get_req()
254 req->in.h.gid == ((gid_t)-1))) { in fuse_get_req()
256 return ERR_PTR(-EOVERFLOW); in fuse_get_req()
261 out: in fuse_get_req()
268 struct fuse_conn *fc = req->fm->fc; in fuse_put_request()
270 if (refcount_dec_and_test(&req->count)) { in fuse_put_request()
271 if (test_bit(FR_BACKGROUND, &req->flags)) { in fuse_put_request()
276 spin_lock(&fc->bg_lock); in fuse_put_request()
277 if (!fc->blocked) in fuse_put_request()
278 wake_up(&fc->blocked_waitq); in fuse_put_request()
279 spin_unlock(&fc->bg_lock); in fuse_put_request()
282 if (test_bit(FR_WAITING, &req->flags)) { in fuse_put_request()
283 __clear_bit(FR_WAITING, &req->flags); in fuse_put_request()
305 fiq->reqctr += FUSE_REQ_ID_STEP; in fuse_get_unique_locked()
306 return fiq->reqctr; in fuse_get_unique_locked()
313 spin_lock(&fiq->lock); in fuse_get_unique()
315 spin_unlock(&fiq->lock); in fuse_get_unique()
328 * A new request is available, wake fiq->waitq
331 __releases(fiq->lock) in fuse_dev_wake_and_unlock()
333 wake_up(&fiq->waitq); in fuse_dev_wake_and_unlock()
334 kill_fasync(&fiq->fasync, SIGIO, POLL_IN); in fuse_dev_wake_and_unlock()
335 spin_unlock(&fiq->lock); in fuse_dev_wake_and_unlock()
341 spin_lock(&fiq->lock); in fuse_dev_queue_forget()
342 if (fiq->connected) { in fuse_dev_queue_forget()
343 fiq->forget_list_tail->next = forget; in fuse_dev_queue_forget()
344 fiq->forget_list_tail = forget; in fuse_dev_queue_forget()
348 spin_unlock(&fiq->lock); in fuse_dev_queue_forget()
354 spin_lock(&fiq->lock); in fuse_dev_queue_interrupt()
355 if (list_empty(&req->intr_entry)) { in fuse_dev_queue_interrupt()
356 list_add_tail(&req->intr_entry, &fiq->interrupts); in fuse_dev_queue_interrupt()
362 if (test_bit(FR_FINISHED, &req->flags)) { in fuse_dev_queue_interrupt()
363 list_del_init(&req->intr_entry); in fuse_dev_queue_interrupt()
364 spin_unlock(&fiq->lock); in fuse_dev_queue_interrupt()
369 spin_unlock(&fiq->lock); in fuse_dev_queue_interrupt()
376 if (req->in.h.opcode != FUSE_NOTIFY_REPLY) in fuse_request_assign_unique_locked()
377 req->in.h.unique = fuse_get_unique_locked(fiq); in fuse_request_assign_unique_locked()
386 if (req->in.h.opcode != FUSE_NOTIFY_REPLY) in fuse_request_assign_unique()
387 req->in.h.unique = fuse_get_unique(fiq); in fuse_request_assign_unique()
396 spin_lock(&fiq->lock); in fuse_dev_queue_req()
397 if (fiq->connected) { in fuse_dev_queue_req()
399 list_add_tail(&req->list, &fiq->pending); in fuse_dev_queue_req()
402 spin_unlock(&fiq->lock); in fuse_dev_queue_req()
403 req->out.h.error = -ENOTCONN; in fuse_dev_queue_req()
404 clear_bit(FR_PENDING, &req->flags); in fuse_dev_queue_req()
418 req->in.h.len = sizeof(struct fuse_in_header) + in fuse_send_one()
419 fuse_len_args(req->args->in_numargs, in fuse_send_one()
420 (struct fuse_arg *) req->args->in_args); in fuse_send_one()
421 fiq->ops->send_req(fiq, req); in fuse_send_one()
427 struct fuse_iqueue *fiq = &fc->iq; in fuse_queue_forget()
429 forget->forget_one.nodeid = nodeid; in fuse_queue_forget()
430 forget->forget_one.nlookup = nlookup; in fuse_queue_forget()
432 fiq->ops->send_forget(fiq, forget); in fuse_queue_forget()
437 struct fuse_iqueue *fiq = &fc->iq; in flush_bg_queue()
439 while (fc->active_background < fc->max_background && in flush_bg_queue()
440 !list_empty(&fc->bg_queue)) { in flush_bg_queue()
443 req = list_first_entry(&fc->bg_queue, struct fuse_req, list); in flush_bg_queue()
444 list_del(&req->list); in flush_bg_queue()
445 fc->active_background++; in flush_bg_queue()
460 struct fuse_mount *fm = req->fm; in fuse_request_end()
461 struct fuse_conn *fc = fm->fc; in fuse_request_end()
462 struct fuse_iqueue *fiq = &fc->iq; in fuse_request_end()
464 if (test_and_set_bit(FR_FINISHED, &req->flags)) in fuse_request_end()
473 if (test_bit(FR_INTERRUPTED, &req->flags)) { in fuse_request_end()
474 spin_lock(&fiq->lock); in fuse_request_end()
475 list_del_init(&req->intr_entry); in fuse_request_end()
476 spin_unlock(&fiq->lock); in fuse_request_end()
478 WARN_ON(test_bit(FR_PENDING, &req->flags)); in fuse_request_end()
479 WARN_ON(test_bit(FR_SENT, &req->flags)); in fuse_request_end()
480 if (test_bit(FR_BACKGROUND, &req->flags)) { in fuse_request_end()
481 spin_lock(&fc->bg_lock); in fuse_request_end()
482 clear_bit(FR_BACKGROUND, &req->flags); in fuse_request_end()
483 if (fc->num_background == fc->max_background) { in fuse_request_end()
484 fc->blocked = 0; in fuse_request_end()
485 wake_up(&fc->blocked_waitq); in fuse_request_end()
486 } else if (!fc->blocked) { in fuse_request_end()
490 * fc->blocked with waiters with the wake_up() call in fuse_request_end()
493 if (waitqueue_active(&fc->blocked_waitq)) in fuse_request_end()
494 wake_up(&fc->blocked_waitq); in fuse_request_end()
497 fc->num_background--; in fuse_request_end()
498 fc->active_background--; in fuse_request_end()
500 spin_unlock(&fc->bg_lock); in fuse_request_end()
503 wake_up(&req->waitq); in fuse_request_end()
506 if (test_bit(FR_ASYNC, &req->flags)) in fuse_request_end()
507 req->args->end(fm, req->args, req->out.h.error); in fuse_request_end()
515 struct fuse_iqueue *fiq = &req->fm->fc->iq; in queue_interrupt()
518 if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) in queue_interrupt()
519 return -EINVAL; in queue_interrupt()
521 fiq->ops->send_interrupt(fiq, req); in queue_interrupt()
529 if (test_bit(FR_PENDING, &req->flags)) { in fuse_remove_pending_req()
534 list_del(&req->list); in fuse_remove_pending_req()
537 req->out.h.error = -EINTR; in fuse_remove_pending_req()
546 struct fuse_conn *fc = req->fm->fc; in request_wait_answer()
547 struct fuse_iqueue *fiq = &fc->iq; in request_wait_answer()
550 if (!fc->no_interrupt) { in request_wait_answer()
552 err = wait_event_interruptible(req->waitq, in request_wait_answer()
553 test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
557 set_bit(FR_INTERRUPTED, &req->flags); in request_wait_answer()
560 if (test_bit(FR_SENT, &req->flags)) in request_wait_answer()
564 if (!test_bit(FR_FORCE, &req->flags)) { in request_wait_answer()
568 err = wait_event_killable(req->waitq, in request_wait_answer()
569 test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
573 if (test_bit(FR_URING, &req->flags)) in request_wait_answer()
576 removed = fuse_remove_pending_req(req, &fiq->lock); in request_wait_answer()
583 * Wait it out. in request_wait_answer()
585 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
590 struct fuse_iqueue *fiq = &req->fm->fc->iq; in __fuse_request_send()
592 BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); in __fuse_request_send()
606 if (fc->minor < 4 && args->opcode == FUSE_STATFS) in fuse_adjust_compat()
607 args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE; in fuse_adjust_compat()
609 if (fc->minor < 9) { in fuse_adjust_compat()
610 switch (args->opcode) { in fuse_adjust_compat()
617 args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE; in fuse_adjust_compat()
621 args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE; in fuse_adjust_compat()
625 if (fc->minor < 12) { in fuse_adjust_compat()
626 switch (args->opcode) { in fuse_adjust_compat()
628 args->in_args[0].size = sizeof(struct fuse_open_in); in fuse_adjust_compat()
631 args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE; in fuse_adjust_compat()
639 struct fuse_conn *fc = req->fm->fc; in fuse_force_creds()
641 if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) { in fuse_force_creds()
642 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid()); in fuse_force_creds()
643 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid()); in fuse_force_creds()
645 req->in.h.uid = FUSE_INVALID_UIDGID; in fuse_force_creds()
646 req->in.h.gid = FUSE_INVALID_UIDGID; in fuse_force_creds()
649 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); in fuse_force_creds()
654 req->in.h.opcode = args->opcode; in fuse_args_to_req()
655 req->in.h.nodeid = args->nodeid; in fuse_args_to_req()
656 req->args = args; in fuse_args_to_req()
657 if (args->is_ext) in fuse_args_to_req()
658 req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; in fuse_args_to_req()
659 if (args->end) in fuse_args_to_req()
660 __set_bit(FR_ASYNC, &req->flags); in fuse_args_to_req()
667 struct fuse_conn *fc = fm->fc; in __fuse_simple_request()
671 if (args->force) { in __fuse_simple_request()
672 atomic_inc(&fc->num_waiting); in __fuse_simple_request()
675 if (!args->nocreds) in __fuse_simple_request()
678 __set_bit(FR_WAITING, &req->flags); in __fuse_simple_request()
679 __set_bit(FR_FORCE, &req->flags); in __fuse_simple_request()
681 WARN_ON(args->nocreds); in __fuse_simple_request()
687 /* Needs to be done after fuse_get_req() so that fc->minor is valid */ in __fuse_simple_request()
691 if (!args->noreply) in __fuse_simple_request()
692 __set_bit(FR_ISREPLY, &req->flags); in __fuse_simple_request()
694 ret = req->out.h.error; in __fuse_simple_request()
695 if (!ret && args->out_argvar) { in __fuse_simple_request()
696 BUG_ON(args->out_numargs == 0); in __fuse_simple_request()
697 ret = args->out_args[args->out_numargs - 1].size; in __fuse_simple_request()
708 struct fuse_iqueue *fiq = &fc->iq; in fuse_request_queue_background_uring()
710 req->in.h.len = sizeof(struct fuse_in_header) + in fuse_request_queue_background_uring()
711 fuse_len_args(req->args->in_numargs, in fuse_request_queue_background_uring()
712 (struct fuse_arg *) req->args->in_args); in fuse_request_queue_background_uring()
724 struct fuse_mount *fm = req->fm; in fuse_request_queue_background()
725 struct fuse_conn *fc = fm->fc; in fuse_request_queue_background()
728 WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); in fuse_request_queue_background()
729 if (!test_bit(FR_WAITING, &req->flags)) { in fuse_request_queue_background()
730 __set_bit(FR_WAITING, &req->flags); in fuse_request_queue_background()
731 atomic_inc(&fc->num_waiting); in fuse_request_queue_background()
733 __set_bit(FR_ISREPLY, &req->flags); in fuse_request_queue_background()
740 spin_lock(&fc->bg_lock); in fuse_request_queue_background()
741 if (likely(fc->connected)) { in fuse_request_queue_background()
742 fc->num_background++; in fuse_request_queue_background()
743 if (fc->num_background == fc->max_background) in fuse_request_queue_background()
744 fc->blocked = 1; in fuse_request_queue_background()
745 list_add_tail(&req->list, &fc->bg_queue); in fuse_request_queue_background()
749 spin_unlock(&fc->bg_lock); in fuse_request_queue_background()
759 if (args->force) { in fuse_simple_background()
760 WARN_ON(!args->nocreds); in fuse_simple_background()
763 return -ENOMEM; in fuse_simple_background()
764 __set_bit(FR_BACKGROUND, &req->flags); in fuse_simple_background()
766 WARN_ON(args->nocreds); in fuse_simple_background()
776 return -ENOTCONN; in fuse_simple_background()
787 struct fuse_iqueue *fiq = &fm->fc->iq; in fuse_simple_notify_reply()
793 __clear_bit(FR_ISREPLY, &req->flags); in fuse_simple_notify_reply()
794 req->in.h.unique = unique; in fuse_simple_notify_reply()
805 * anything that could cause a page-fault. If the request was already
806 * aborted bail out.
812 spin_lock(&req->waitq.lock); in lock_request()
813 if (test_bit(FR_ABORTED, &req->flags)) in lock_request()
814 err = -ENOENT; in lock_request()
816 set_bit(FR_LOCKED, &req->flags); in lock_request()
817 spin_unlock(&req->waitq.lock); in lock_request()
830 spin_lock(&req->waitq.lock); in unlock_request()
831 if (test_bit(FR_ABORTED, &req->flags)) in unlock_request()
832 err = -ENOENT; in unlock_request()
834 clear_bit(FR_LOCKED, &req->flags); in unlock_request()
835 spin_unlock(&req->waitq.lock); in unlock_request()
840 void fuse_copy_init(struct fuse_copy_state *cs, bool write, in fuse_copy_init() argument
843 memset(cs, 0, sizeof(*cs)); in fuse_copy_init()
844 cs->write = write; in fuse_copy_init()
845 cs->iter = iter; in fuse_copy_init()
849 static void fuse_copy_finish(struct fuse_copy_state *cs) in fuse_copy_finish() argument
851 if (cs->currbuf) { in fuse_copy_finish()
852 struct pipe_buffer *buf = cs->currbuf; in fuse_copy_finish()
854 if (cs->write) in fuse_copy_finish()
855 buf->len = PAGE_SIZE - cs->len; in fuse_copy_finish()
856 cs->currbuf = NULL; in fuse_copy_finish()
857 } else if (cs->pg) { in fuse_copy_finish()
858 if (cs->write) { in fuse_copy_finish()
859 flush_dcache_page(cs->pg); in fuse_copy_finish()
860 set_page_dirty_lock(cs->pg); in fuse_copy_finish()
862 put_page(cs->pg); in fuse_copy_finish()
864 cs->pg = NULL; in fuse_copy_finish()
871 static int fuse_copy_fill(struct fuse_copy_state *cs) in fuse_copy_fill() argument
876 err = unlock_request(cs->req); in fuse_copy_fill()
880 fuse_copy_finish(cs); in fuse_copy_fill()
881 if (cs->pipebufs) { in fuse_copy_fill()
882 struct pipe_buffer *buf = cs->pipebufs; in fuse_copy_fill()
884 if (!cs->write) { in fuse_copy_fill()
885 err = pipe_buf_confirm(cs->pipe, buf); in fuse_copy_fill()
889 BUG_ON(!cs->nr_segs); in fuse_copy_fill()
890 cs->currbuf = buf; in fuse_copy_fill()
891 cs->pg = buf->page; in fuse_copy_fill()
892 cs->offset = buf->offset; in fuse_copy_fill()
893 cs->len = buf->len; in fuse_copy_fill()
894 cs->pipebufs++; in fuse_copy_fill()
895 cs->nr_segs--; in fuse_copy_fill()
897 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_copy_fill()
898 return -EIO; in fuse_copy_fill()
902 return -ENOMEM; in fuse_copy_fill()
904 buf->page = page; in fuse_copy_fill()
905 buf->offset = 0; in fuse_copy_fill()
906 buf->len = 0; in fuse_copy_fill()
908 cs->currbuf = buf; in fuse_copy_fill()
909 cs->pg = page; in fuse_copy_fill()
910 cs->offset = 0; in fuse_copy_fill()
911 cs->len = PAGE_SIZE; in fuse_copy_fill()
912 cs->pipebufs++; in fuse_copy_fill()
913 cs->nr_segs++; in fuse_copy_fill()
917 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); in fuse_copy_fill()
921 cs->len = err; in fuse_copy_fill()
922 cs->offset = off; in fuse_copy_fill()
923 cs->pg = page; in fuse_copy_fill()
926 return lock_request(cs->req); in fuse_copy_fill()
930 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) in fuse_copy_do() argument
932 unsigned ncpy = min(*size, cs->len); in fuse_copy_do()
934 void *pgaddr = kmap_local_page(cs->pg); in fuse_copy_do()
935 void *buf = pgaddr + cs->offset; in fuse_copy_do()
937 if (cs->write) in fuse_copy_do()
945 *size -= ncpy; in fuse_copy_do()
946 cs->len -= ncpy; in fuse_copy_do()
947 cs->offset += ncpy; in fuse_copy_do()
948 if (cs->is_uring) in fuse_copy_do()
949 cs->ring.copied_sz += ncpy; in fuse_copy_do()
957 folio->mapping != NULL || in fuse_check_folio()
958 (folio->flags.f & PAGE_FLAGS_CHECK_AT_PREP & in fuse_check_folio()
967 dump_page(&folio->page, "fuse: trying to steal weird page"); in fuse_check_folio()
979 static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop) in fuse_try_move_folio() argument
984 struct pipe_buffer *buf = cs->pipebufs; in fuse_try_move_folio()
987 err = unlock_request(cs->req); in fuse_try_move_folio()
991 fuse_copy_finish(cs); in fuse_try_move_folio()
993 err = pipe_buf_confirm(cs->pipe, buf); in fuse_try_move_folio()
997 BUG_ON(!cs->nr_segs); in fuse_try_move_folio()
998 cs->currbuf = buf; in fuse_try_move_folio()
999 cs->len = buf->len; in fuse_try_move_folio()
1000 cs->pipebufs++; in fuse_try_move_folio()
1001 cs->nr_segs--; in fuse_try_move_folio()
1003 if (cs->len != folio_size(oldfolio)) in fuse_try_move_folio()
1006 if (!pipe_buf_try_steal(cs->pipe, buf)) in fuse_try_move_folio()
1009 newfolio = page_folio(buf->page); in fuse_try_move_folio()
1035 if (!(buf->flags & PIPE_BUF_FLAG_LRU)) in fuse_try_move_folio()
1042 pipe_buf_release(cs->pipe, buf); in fuse_try_move_folio()
1045 spin_lock(&cs->req->waitq.lock); in fuse_try_move_folio()
1046 if (test_bit(FR_ABORTED, &cs->req->flags)) in fuse_try_move_folio()
1047 err = -ENOENT; in fuse_try_move_folio()
1050 spin_unlock(&cs->req->waitq.lock); in fuse_try_move_folio()
1059 /* Drop ref for ap->pages[] array */ in fuse_try_move_folio()
1061 cs->len = 0; in fuse_try_move_folio()
1072 cs->pg = buf->page; in fuse_try_move_folio()
1073 cs->offset = buf->offset; in fuse_try_move_folio()
1075 err = lock_request(cs->req); in fuse_try_move_folio()
1082 static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, in fuse_ref_folio() argument
1088 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_ref_folio()
1089 return -EIO; in fuse_ref_folio()
1092 err = unlock_request(cs->req); in fuse_ref_folio()
1098 fuse_copy_finish(cs); in fuse_ref_folio()
1100 buf = cs->pipebufs; in fuse_ref_folio()
1101 buf->page = &folio->page; in fuse_ref_folio()
1102 buf->offset = offset; in fuse_ref_folio()
1103 buf->len = count; in fuse_ref_folio()
1105 cs->pipebufs++; in fuse_ref_folio()
1106 cs->nr_segs++; in fuse_ref_folio()
1107 cs->len = 0; in fuse_ref_folio()
1116 static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, in fuse_copy_folio() argument
1130 if (cs->write && cs->pipebufs && folio) { in fuse_copy_folio()
1135 if (cs->req->args->user_pages) { in fuse_copy_folio()
1136 err = fuse_copy_fill(cs); in fuse_copy_folio()
1140 return fuse_ref_folio(cs, folio, offset, count); in fuse_copy_folio()
1142 } else if (!cs->len) { in fuse_copy_folio()
1143 if (cs->move_folios && folio && in fuse_copy_folio()
1145 err = fuse_try_move_folio(cs, foliop); in fuse_copy_folio()
1149 err = fuse_copy_fill(cs); in fuse_copy_folio()
1160 if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset)) in fuse_copy_folio()
1161 copy = PAGE_SIZE - offset_in_page(offset); in fuse_copy_folio()
1163 bytes_copied = fuse_copy_do(cs, &buf, ©); in fuse_copy_folio()
1166 count -= bytes_copied; in fuse_copy_folio()
1168 offset += fuse_copy_do(cs, NULL, &count); in fuse_copy_folio()
1170 if (folio && !cs->write) in fuse_copy_folio()
1176 static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes, in fuse_copy_folios() argument
1180 struct fuse_req *req = cs->req; in fuse_copy_folios()
1181 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); in fuse_copy_folios()
1183 for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { in fuse_copy_folios()
1185 unsigned int offset = ap->descs[i].offset; in fuse_copy_folios()
1186 unsigned int count = min(nbytes, ap->descs[i].length); in fuse_copy_folios()
1188 err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing); in fuse_copy_folios()
1192 nbytes -= count; in fuse_copy_folios()
1198 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) in fuse_copy_one() argument
1201 if (!cs->len) { in fuse_copy_one()
1202 int err = fuse_copy_fill(cs); in fuse_copy_one()
1206 fuse_copy_do(cs, &val, &size); in fuse_copy_one()
1212 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, in fuse_copy_args() argument
1221 if (i == numargs - 1 && argpages) in fuse_copy_args()
1222 err = fuse_copy_folios(cs, arg->size, zeroing); in fuse_copy_args()
1224 err = fuse_copy_one(cs, arg->value, arg->size); in fuse_copy_args()
1231 return fiq->forget_list_head.next != NULL; in forget_pending()
1236 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) || in request_pending()
1246 * Called with fiq->lock held, releases it
1249 struct fuse_copy_state *cs, in fuse_read_interrupt() argument
1251 __releases(fiq->lock) in fuse_read_interrupt()
1258 list_del_init(&req->intr_entry); in fuse_read_interrupt()
1263 ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT); in fuse_read_interrupt()
1264 arg.unique = req->in.h.unique; in fuse_read_interrupt()
1266 spin_unlock(&fiq->lock); in fuse_read_interrupt()
1268 return -EINVAL; in fuse_read_interrupt()
1270 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_interrupt()
1272 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_interrupt()
1273 fuse_copy_finish(cs); in fuse_read_interrupt()
1282 struct fuse_forget_link *head = fiq->forget_list_head.next; in fuse_dequeue_forget()
1287 newhead = &(*newhead)->next; in fuse_dequeue_forget()
1289 fiq->forget_list_head.next = *newhead; in fuse_dequeue_forget()
1291 if (fiq->forget_list_head.next == NULL) in fuse_dequeue_forget()
1292 fiq->forget_list_tail = &fiq->forget_list_head; in fuse_dequeue_forget()
1301 struct fuse_copy_state *cs, in fuse_read_single_forget() argument
1303 __releases(fiq->lock) in fuse_read_single_forget()
1308 .nlookup = forget->forget_one.nlookup, in fuse_read_single_forget()
1312 .nodeid = forget->forget_one.nodeid, in fuse_read_single_forget()
1317 spin_unlock(&fiq->lock); in fuse_read_single_forget()
1320 return -EINVAL; in fuse_read_single_forget()
1322 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_single_forget()
1324 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_single_forget()
1325 fuse_copy_finish(cs); in fuse_read_single_forget()
1334 struct fuse_copy_state *cs, size_t nbytes) in fuse_read_batch_forget() argument
1335 __releases(fiq->lock) in fuse_read_batch_forget()
1349 spin_unlock(&fiq->lock); in fuse_read_batch_forget()
1350 return -EINVAL; in fuse_read_batch_forget()
1353 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one); in fuse_read_batch_forget()
1355 spin_unlock(&fiq->lock); in fuse_read_batch_forget()
1359 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_batch_forget()
1361 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_batch_forget()
1367 err = fuse_copy_one(cs, &forget->forget_one, in fuse_read_batch_forget()
1368 sizeof(forget->forget_one)); in fuse_read_batch_forget()
1370 head = forget->next; in fuse_read_batch_forget()
1374 fuse_copy_finish(cs); in fuse_read_batch_forget()
1383 struct fuse_copy_state *cs, in fuse_read_forget() argument
1385 __releases(fiq->lock) in fuse_read_forget()
1387 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL) in fuse_read_forget()
1388 return fuse_read_single_forget(fiq, cs, nbytes); in fuse_read_forget()
1390 return fuse_read_batch_forget(fiq, cs, nbytes); in fuse_read_forget()
1403 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_read() argument
1406 struct fuse_conn *fc = fud->fc; in fuse_dev_do_read()
1407 struct fuse_iqueue *fiq = &fc->iq; in fuse_dev_do_read()
1408 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_do_read()
1415 * Require sane minimum read buffer - that has capacity for fixed part in fuse_dev_do_read()
1429 fc->max_write)) in fuse_dev_do_read()
1430 return -EINVAL; in fuse_dev_do_read()
1434 spin_lock(&fiq->lock); in fuse_dev_do_read()
1435 if (!fiq->connected || request_pending(fiq)) in fuse_dev_do_read()
1437 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1439 if (file->f_flags & O_NONBLOCK) in fuse_dev_do_read()
1440 return -EAGAIN; in fuse_dev_do_read()
1441 err = wait_event_interruptible_exclusive(fiq->waitq, in fuse_dev_do_read()
1442 !fiq->connected || request_pending(fiq)); in fuse_dev_do_read()
1447 if (!fiq->connected) { in fuse_dev_do_read()
1448 err = fc->aborted ? -ECONNABORTED : -ENODEV; in fuse_dev_do_read()
1452 if (!list_empty(&fiq->interrupts)) { in fuse_dev_do_read()
1453 req = list_entry(fiq->interrupts.next, struct fuse_req, in fuse_dev_do_read()
1455 return fuse_read_interrupt(fiq, cs, nbytes, req); in fuse_dev_do_read()
1459 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0) in fuse_dev_do_read()
1460 return fuse_read_forget(fc, fiq, cs, nbytes); in fuse_dev_do_read()
1462 if (fiq->forget_batch <= -8) in fuse_dev_do_read()
1463 fiq->forget_batch = 16; in fuse_dev_do_read()
1466 req = list_entry(fiq->pending.next, struct fuse_req, list); in fuse_dev_do_read()
1467 clear_bit(FR_PENDING, &req->flags); in fuse_dev_do_read()
1468 list_del_init(&req->list); in fuse_dev_do_read()
1469 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1471 args = req->args; in fuse_dev_do_read()
1472 reqsize = req->in.h.len; in fuse_dev_do_read()
1476 req->out.h.error = -EIO; in fuse_dev_do_read()
1478 if (args->opcode == FUSE_SETXATTR) in fuse_dev_do_read()
1479 req->out.h.error = -E2BIG; in fuse_dev_do_read()
1483 spin_lock(&fpq->lock); in fuse_dev_do_read()
1485 * Must not put request on fpq->io queue after having been shut down by in fuse_dev_do_read()
1488 if (!fpq->connected) { in fuse_dev_do_read()
1489 req->out.h.error = err = -ECONNABORTED; in fuse_dev_do_read()
1493 list_add(&req->list, &fpq->io); in fuse_dev_do_read()
1494 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1495 cs->req = req; in fuse_dev_do_read()
1496 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); in fuse_dev_do_read()
1498 err = fuse_copy_args(cs, args->in_numargs, args->in_pages, in fuse_dev_do_read()
1499 (struct fuse_arg *) args->in_args, 0); in fuse_dev_do_read()
1500 fuse_copy_finish(cs); in fuse_dev_do_read()
1501 spin_lock(&fpq->lock); in fuse_dev_do_read()
1502 clear_bit(FR_LOCKED, &req->flags); in fuse_dev_do_read()
1503 if (!fpq->connected) { in fuse_dev_do_read()
1504 err = fc->aborted ? -ECONNABORTED : -ENODEV; in fuse_dev_do_read()
1508 req->out.h.error = -EIO; in fuse_dev_do_read()
1511 if (!test_bit(FR_ISREPLY, &req->flags)) { in fuse_dev_do_read()
1515 hash = fuse_req_hash(req->in.h.unique); in fuse_dev_do_read()
1516 list_move_tail(&req->list, &fpq->processing[hash]); in fuse_dev_do_read()
1518 set_bit(FR_SENT, &req->flags); in fuse_dev_do_read()
1519 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1522 if (test_bit(FR_INTERRUPTED, &req->flags)) in fuse_dev_do_read()
1529 if (!test_bit(FR_PRIVATE, &req->flags)) in fuse_dev_do_read()
1530 list_del_init(&req->list); in fuse_dev_do_read()
1531 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1536 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1547 file->private_data = NULL; in fuse_dev_open()
1560 READ_ONCE(file->private_data) != FUSE_DEV_SYNC_INIT); in fuse_get_dev()
1566 return ERR_PTR(-EPERM); in fuse_get_dev()
1573 struct fuse_copy_state cs; in fuse_dev_read() local
1574 struct file *file = iocb->ki_filp; in fuse_dev_read()
1581 return -EINVAL; in fuse_dev_read()
1583 fuse_copy_init(&cs, true, to); in fuse_dev_read()
1585 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to)); in fuse_dev_read()
1595 struct fuse_copy_state cs; in fuse_dev_splice_read() local
1601 bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer), in fuse_dev_splice_read()
1604 return -ENOMEM; in fuse_dev_splice_read()
1606 fuse_copy_init(&cs, true, NULL); in fuse_dev_splice_read()
1607 cs.pipebufs = bufs; in fuse_dev_splice_read()
1608 cs.pipe = pipe; in fuse_dev_splice_read()
1609 ret = fuse_dev_do_read(fud, in, &cs, len); in fuse_dev_splice_read()
1611 goto out; in fuse_dev_splice_read()
1613 if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) { in fuse_dev_splice_read()
1614 ret = -EIO; in fuse_dev_splice_read()
1615 goto out; in fuse_dev_splice_read()
1618 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) { in fuse_dev_splice_read()
1620 * Need to be careful about this. Having buf->ops in module in fuse_dev_splice_read()
1631 out: in fuse_dev_splice_read()
1632 for (; page_nr < cs.nr_segs; page_nr++) in fuse_dev_splice_read()
1640 struct fuse_copy_state *cs) in fuse_notify_poll() argument
1646 return -EINVAL; in fuse_notify_poll()
1648 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_poll()
1652 fuse_copy_finish(cs); in fuse_notify_poll()
1657 struct fuse_copy_state *cs) in fuse_notify_inval_inode() argument
1663 return -EINVAL; in fuse_notify_inval_inode()
1665 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_inode()
1668 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1670 down_read(&fc->killsb); in fuse_notify_inval_inode()
1673 up_read(&fc->killsb); in fuse_notify_inval_inode()
1678 struct fuse_copy_state *cs) in fuse_notify_inval_entry() argument
1686 return -EINVAL; in fuse_notify_inval_entry()
1688 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_entry()
1692 if (outarg.namelen > fc->name_max) in fuse_notify_inval_entry()
1693 return -ENAMETOOLONG; in fuse_notify_inval_entry()
1695 err = -EINVAL; in fuse_notify_inval_entry()
1697 return -EINVAL; in fuse_notify_inval_entry()
1701 return -ENOMEM; in fuse_notify_inval_entry()
1705 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_inval_entry()
1708 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1711 down_read(&fc->killsb); in fuse_notify_inval_entry()
1713 up_read(&fc->killsb); in fuse_notify_inval_entry()
1720 struct fuse_copy_state *cs) in fuse_notify_delete() argument
1728 return -EINVAL; in fuse_notify_delete()
1730 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_delete()
1734 if (outarg.namelen > fc->name_max) in fuse_notify_delete()
1735 return -ENAMETOOLONG; in fuse_notify_delete()
1738 return -EINVAL; in fuse_notify_delete()
1742 return -ENOMEM; in fuse_notify_delete()
1746 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_delete()
1749 fuse_copy_finish(cs); in fuse_notify_delete()
1752 down_read(&fc->killsb); in fuse_notify_delete()
1754 up_read(&fc->killsb); in fuse_notify_delete()
1761 struct fuse_copy_state *cs) in fuse_notify_store() argument
1775 return -EINVAL; in fuse_notify_store()
1777 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_store()
1781 if (size - sizeof(outarg) != outarg.size) in fuse_notify_store()
1782 return -EINVAL; in fuse_notify_store()
1786 down_read(&fc->killsb); in fuse_notify_store()
1788 err = -ENOENT; in fuse_notify_store()
1793 mapping = inode->i_mapping; in fuse_notify_store()
1815 folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; in fuse_notify_store()
1816 nr_bytes = min_t(unsigned, num, folio_size(folio) - folio_offset); in fuse_notify_store()
1817 nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; in fuse_notify_store()
1819 err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); in fuse_notify_store()
1831 num -= nr_bytes; in fuse_notify_store()
1841 up_read(&fc->killsb); in fuse_notify_store()
1856 release_pages(ra->ap.folios, ra->ap.num_folios); in fuse_retrieve_end()
1864 struct address_space *mapping = inode->i_mapping; in fuse_retrieve()
1871 struct fuse_conn *fc = fm->fc; in fuse_retrieve()
1877 offset = outarg->offset & ~PAGE_MASK; in fuse_retrieve()
1880 num = min(outarg->size, fc->max_write); in fuse_retrieve()
1881 if (outarg->offset > file_size) in fuse_retrieve()
1883 else if (outarg->offset + num > file_size) in fuse_retrieve()
1884 num = file_size - outarg->offset; in fuse_retrieve()
1886 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; in fuse_retrieve()
1887 num_pages = min(num_pages, fc->max_pages); in fuse_retrieve()
1890 args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); in fuse_retrieve()
1894 return -ENOMEM; in fuse_retrieve()
1896 ap = &ra->ap; in fuse_retrieve()
1897 ap->folios = (void *) (ra + 1); in fuse_retrieve()
1898 ap->descs = (void *) (ap->folios + num_pages); in fuse_retrieve()
1900 args = &ap->args; in fuse_retrieve()
1901 args->nodeid = outarg->nodeid; in fuse_retrieve()
1902 args->opcode = FUSE_NOTIFY_REPLY; in fuse_retrieve()
1903 args->in_numargs = 3; in fuse_retrieve()
1904 args->in_pages = true; in fuse_retrieve()
1905 args->end = fuse_retrieve_end; in fuse_retrieve()
1907 index = outarg->offset >> PAGE_SHIFT; in fuse_retrieve()
1909 while (num && ap->num_folios < num_pages) { in fuse_retrieve()
1919 folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; in fuse_retrieve()
1920 nr_bytes = min(folio_size(folio) - folio_offset, num); in fuse_retrieve()
1921 nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; in fuse_retrieve()
1923 ap->folios[ap->num_folios] = folio; in fuse_retrieve()
1924 ap->descs[ap->num_folios].offset = folio_offset; in fuse_retrieve()
1925 ap->descs[ap->num_folios].length = nr_bytes; in fuse_retrieve()
1926 ap->num_folios++; in fuse_retrieve()
1929 num -= nr_bytes; in fuse_retrieve()
1933 ra->inarg.offset = outarg->offset; in fuse_retrieve()
1934 ra->inarg.size = total_len; in fuse_retrieve()
1936 args->in_args[1].size = sizeof(ra->inarg); in fuse_retrieve()
1937 args->in_args[1].value = &ra->inarg; in fuse_retrieve()
1938 args->in_args[2].size = total_len; in fuse_retrieve()
1940 err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); in fuse_retrieve()
1948 struct fuse_copy_state *cs) in fuse_notify_retrieve() argument
1957 return -EINVAL; in fuse_notify_retrieve()
1959 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_retrieve()
1963 fuse_copy_finish(cs); in fuse_notify_retrieve()
1965 down_read(&fc->killsb); in fuse_notify_retrieve()
1966 err = -ENOENT; in fuse_notify_retrieve()
1974 up_read(&fc->killsb); in fuse_notify_retrieve()
1990 * non-idempotent requests.
1996 struct fuse_iqueue *fiq = &fc->iq; in fuse_resend()
2000 spin_lock(&fc->lock); in fuse_resend()
2001 if (!fc->connected) { in fuse_resend()
2002 spin_unlock(&fc->lock); in fuse_resend()
2006 list_for_each_entry(fud, &fc->devices, entry) { in fuse_resend()
2007 struct fuse_pqueue *fpq = &fud->pq; in fuse_resend()
2009 spin_lock(&fpq->lock); in fuse_resend()
2011 list_splice_tail_init(&fpq->processing[i], &to_queue); in fuse_resend()
2012 spin_unlock(&fpq->lock); in fuse_resend()
2014 spin_unlock(&fc->lock); in fuse_resend()
2017 set_bit(FR_PENDING, &req->flags); in fuse_resend()
2018 clear_bit(FR_SENT, &req->flags); in fuse_resend()
2020 req->in.h.unique |= FUSE_UNIQUE_RESEND; in fuse_resend()
2023 spin_lock(&fiq->lock); in fuse_resend()
2024 if (!fiq->connected) { in fuse_resend()
2025 spin_unlock(&fiq->lock); in fuse_resend()
2027 clear_bit(FR_PENDING, &req->flags); in fuse_resend()
2032 list_splice(&to_queue, &fiq->pending); in fuse_resend()
2050 atomic_inc(&fc->epoch); in fuse_notify_inc_epoch()
2056 struct fuse_copy_state *cs) in fuse_notify_prune() argument
2065 return -ENOMEM; in fuse_notify_prune()
2068 return -EINVAL; in fuse_notify_prune()
2070 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_prune()
2074 if (size - sizeof(outarg) != outarg.count * sizeof(u64)) in fuse_notify_prune()
2075 return -EINVAL; in fuse_notify_prune()
2077 for (; outarg.count; outarg.count -= num) { in fuse_notify_prune()
2079 err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); in fuse_notify_prune()
2083 scoped_guard(rwsem_read, &fc->killsb) { in fuse_notify_prune()
2092 unsigned int size, struct fuse_copy_state *cs) in fuse_notify() argument
2095 cs->move_folios = false; in fuse_notify()
2099 return fuse_notify_poll(fc, size, cs); in fuse_notify()
2102 return fuse_notify_inval_inode(fc, size, cs); in fuse_notify()
2105 return fuse_notify_inval_entry(fc, size, cs); in fuse_notify()
2108 return fuse_notify_store(fc, size, cs); in fuse_notify()
2111 return fuse_notify_retrieve(fc, size, cs); in fuse_notify()
2114 return fuse_notify_delete(fc, size, cs); in fuse_notify()
2123 return fuse_notify_prune(fc, size, cs); in fuse_notify()
2126 return -EINVAL; in fuse_notify()
2136 list_for_each_entry(req, &fpq->processing[hash], list) { in fuse_request_find()
2137 if (req->in.h.unique == unique) in fuse_request_find()
2143 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, in fuse_copy_out_args() argument
2150 * Uring has all headers separated from args - args is payload only in fuse_copy_out_args()
2152 if (!cs->is_uring) in fuse_copy_out_args()
2155 reqsize += fuse_len_args(args->out_numargs, args->out_args); in fuse_copy_out_args()
2157 if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar)) in fuse_copy_out_args()
2158 return -EINVAL; in fuse_copy_out_args()
2160 struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1]; in fuse_copy_out_args()
2161 unsigned diffsize = reqsize - nbytes; in fuse_copy_out_args()
2163 if (diffsize > lastarg->size) in fuse_copy_out_args()
2164 return -EINVAL; in fuse_copy_out_args()
2165 lastarg->size -= diffsize; in fuse_copy_out_args()
2167 return fuse_copy_args(cs, args->out_numargs, args->out_pages, in fuse_copy_out_args()
2168 args->out_args, args->page_zeroing); in fuse_copy_out_args()
2179 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_write() argument
2182 struct fuse_conn *fc = fud->fc; in fuse_dev_do_write()
2183 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_do_write()
2187 err = -EINVAL; in fuse_dev_do_write()
2189 goto out; in fuse_dev_do_write()
2191 err = fuse_copy_one(cs, &oh, sizeof(oh)); in fuse_dev_do_write()
2195 err = -EINVAL; in fuse_dev_do_write()
2204 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); in fuse_dev_do_write()
2208 err = -EINVAL; in fuse_dev_do_write()
2209 if (oh.error <= -512 || oh.error > 0) in fuse_dev_do_write()
2212 spin_lock(&fpq->lock); in fuse_dev_do_write()
2214 if (fpq->connected) in fuse_dev_do_write()
2217 err = -ENOENT; in fuse_dev_do_write()
2219 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2226 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2230 err = -EINVAL; in fuse_dev_do_write()
2231 else if (oh.error == -ENOSYS) in fuse_dev_do_write()
2232 fc->no_interrupt = 1; in fuse_dev_do_write()
2233 else if (oh.error == -EAGAIN) in fuse_dev_do_write()
2241 clear_bit(FR_SENT, &req->flags); in fuse_dev_do_write()
2242 list_move(&req->list, &fpq->io); in fuse_dev_do_write()
2243 req->out.h = oh; in fuse_dev_do_write()
2244 set_bit(FR_LOCKED, &req->flags); in fuse_dev_do_write()
2245 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2246 cs->req = req; in fuse_dev_do_write()
2247 if (!req->args->page_replace) in fuse_dev_do_write()
2248 cs->move_folios = false; in fuse_dev_do_write()
2251 err = nbytes != sizeof(oh) ? -EINVAL : 0; in fuse_dev_do_write()
2253 err = fuse_copy_out_args(cs, req->args, nbytes); in fuse_dev_do_write()
2254 fuse_copy_finish(cs); in fuse_dev_do_write()
2256 spin_lock(&fpq->lock); in fuse_dev_do_write()
2257 clear_bit(FR_LOCKED, &req->flags); in fuse_dev_do_write()
2258 if (!fpq->connected) in fuse_dev_do_write()
2259 err = -ENOENT; in fuse_dev_do_write()
2261 req->out.h.error = -EIO; in fuse_dev_do_write()
2262 if (!test_bit(FR_PRIVATE, &req->flags)) in fuse_dev_do_write()
2263 list_del_init(&req->list); in fuse_dev_do_write()
2264 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2267 out: in fuse_dev_do_write()
2271 fuse_copy_finish(cs); in fuse_dev_do_write()
2272 goto out; in fuse_dev_do_write()
2277 struct fuse_copy_state cs; in fuse_dev_write() local
2278 struct fuse_dev *fud = __fuse_get_dev(iocb->ki_filp); in fuse_dev_write()
2281 return -EPERM; in fuse_dev_write()
2284 return -EINVAL; in fuse_dev_write()
2286 fuse_copy_init(&cs, false, from); in fuse_dev_write()
2288 return fuse_dev_do_write(fud, &cs, iov_iter_count(from)); in fuse_dev_write()
2292 struct file *out, loff_t *ppos, in fuse_dev_splice_write() argument
2299 struct fuse_copy_state cs; in fuse_dev_splice_write() local
2300 struct fuse_dev *fud = __fuse_get_dev(out); in fuse_dev_splice_write()
2305 return -EPERM; in fuse_dev_splice_write()
2309 head = pipe->head; in fuse_dev_splice_write()
2310 tail = pipe->tail; in fuse_dev_splice_write()
2316 return -ENOMEM; in fuse_dev_splice_write()
2322 rem += pipe_buf(pipe, idx)->len; in fuse_dev_splice_write()
2324 ret = -EINVAL; in fuse_dev_splice_write()
2339 if (rem >= ibuf->len) { in fuse_dev_splice_write()
2341 ibuf->ops = NULL; in fuse_dev_splice_write()
2343 pipe->tail = tail; in fuse_dev_splice_write()
2349 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; in fuse_dev_splice_write()
2350 obuf->len = rem; in fuse_dev_splice_write()
2351 ibuf->offset += obuf->len; in fuse_dev_splice_write()
2352 ibuf->len -= obuf->len; in fuse_dev_splice_write()
2355 rem -= obuf->len; in fuse_dev_splice_write()
2359 fuse_copy_init(&cs, false, NULL); in fuse_dev_splice_write()
2360 cs.pipebufs = bufs; in fuse_dev_splice_write()
2361 cs.nr_segs = nbuf; in fuse_dev_splice_write()
2362 cs.pipe = pipe; in fuse_dev_splice_write()
2365 cs.move_folios = true; in fuse_dev_splice_write()
2367 ret = fuse_dev_do_write(fud, &cs, len); in fuse_dev_splice_write()
2374 if (buf->ops) in fuse_dev_splice_write()
2392 fiq = &fud->fc->iq; in fuse_dev_poll()
2393 poll_wait(file, &fiq->waitq, wait); in fuse_dev_poll()
2395 spin_lock(&fiq->lock); in fuse_dev_poll()
2396 if (!fiq->connected) in fuse_dev_poll()
2400 spin_unlock(&fiq->lock); in fuse_dev_poll()
2410 req = list_entry(head->next, struct fuse_req, list); in fuse_dev_end_requests()
2411 req->out.h.error = -ECONNABORTED; in fuse_dev_end_requests()
2412 clear_bit(FR_SENT, &req->flags); in fuse_dev_end_requests()
2413 list_del_init(&req->list); in fuse_dev_end_requests()
2422 p = rb_first(&fc->polled_files); in end_polls()
2427 wake_up_interruptible_all(&ff->poll_wait); in end_polls()
2444 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2453 struct fuse_iqueue *fiq = &fc->iq; in fuse_abort_conn()
2455 spin_lock(&fc->lock); in fuse_abort_conn()
2456 if (fc->connected) { in fuse_abort_conn()
2462 if (fc->timeout.req_timeout) in fuse_abort_conn()
2463 cancel_delayed_work(&fc->timeout.work); in fuse_abort_conn()
2465 /* Background queuing checks fc->connected under bg_lock */ in fuse_abort_conn()
2466 spin_lock(&fc->bg_lock); in fuse_abort_conn()
2467 fc->connected = 0; in fuse_abort_conn()
2468 spin_unlock(&fc->bg_lock); in fuse_abort_conn()
2471 list_for_each_entry(fud, &fc->devices, entry) { in fuse_abort_conn()
2472 struct fuse_pqueue *fpq = &fud->pq; in fuse_abort_conn()
2474 spin_lock(&fpq->lock); in fuse_abort_conn()
2475 fpq->connected = 0; in fuse_abort_conn()
2476 list_for_each_entry_safe(req, next, &fpq->io, list) { in fuse_abort_conn()
2477 req->out.h.error = -ECONNABORTED; in fuse_abort_conn()
2478 spin_lock(&req->waitq.lock); in fuse_abort_conn()
2479 set_bit(FR_ABORTED, &req->flags); in fuse_abort_conn()
2480 if (!test_bit(FR_LOCKED, &req->flags)) { in fuse_abort_conn()
2481 set_bit(FR_PRIVATE, &req->flags); in fuse_abort_conn()
2483 list_move(&req->list, &to_end); in fuse_abort_conn()
2485 spin_unlock(&req->waitq.lock); in fuse_abort_conn()
2488 list_splice_tail_init(&fpq->processing[i], in fuse_abort_conn()
2490 spin_unlock(&fpq->lock); in fuse_abort_conn()
2492 spin_lock(&fc->bg_lock); in fuse_abort_conn()
2493 fc->blocked = 0; in fuse_abort_conn()
2494 fc->max_background = UINT_MAX; in fuse_abort_conn()
2496 spin_unlock(&fc->bg_lock); in fuse_abort_conn()
2498 spin_lock(&fiq->lock); in fuse_abort_conn()
2499 fiq->connected = 0; in fuse_abort_conn()
2500 list_for_each_entry(req, &fiq->pending, list) in fuse_abort_conn()
2501 clear_bit(FR_PENDING, &req->flags); in fuse_abort_conn()
2502 list_splice_tail_init(&fiq->pending, &to_end); in fuse_abort_conn()
2505 wake_up_all(&fiq->waitq); in fuse_abort_conn()
2506 spin_unlock(&fiq->lock); in fuse_abort_conn()
2507 kill_fasync(&fiq->fasync, SIGIO, POLL_IN); in fuse_abort_conn()
2509 wake_up_all(&fc->blocked_waitq); in fuse_abort_conn()
2510 spin_unlock(&fc->lock); in fuse_abort_conn()
2515 * fc->lock must not be taken to avoid conflicts with io-uring in fuse_abort_conn()
2520 spin_unlock(&fc->lock); in fuse_abort_conn()
2529 wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); in fuse_wait_aborted()
2539 struct fuse_conn *fc = fud->fc; in fuse_dev_release()
2540 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_release()
2544 spin_lock(&fpq->lock); in fuse_dev_release()
2545 WARN_ON(!list_empty(&fpq->io)); in fuse_dev_release()
2547 list_splice_init(&fpq->processing[i], &to_end); in fuse_dev_release()
2548 spin_unlock(&fpq->lock); in fuse_dev_release()
2553 if (atomic_dec_and_test(&fc->dev_count)) { in fuse_dev_release()
2554 WARN_ON(fc->iq.fasync != NULL); in fuse_dev_release()
2570 /* No locking - fasync_helper does its own locking */ in fuse_dev_fasync()
2571 return fasync_helper(fd, file, on, &fud->fc->iq.fasync); in fuse_dev_fasync()
2579 return -EINVAL; in fuse_device_clone()
2583 return -ENOMEM; in fuse_device_clone()
2585 new->private_data = fud; in fuse_device_clone()
2586 atomic_inc(&fc->dev_count); in fuse_device_clone()
2598 return -EFAULT; in fuse_dev_ioctl_clone()
2602 return -EINVAL; in fuse_dev_ioctl_clone()
2605 * Check against file->f_op because CUSE in fuse_dev_ioctl_clone()
2608 if (fd_file(f)->f_op == file->f_op) in fuse_dev_ioctl_clone()
2611 res = -EINVAL; in fuse_dev_ioctl_clone()
2614 res = fuse_device_clone(fud->fc, file); in fuse_dev_ioctl_clone()
2631 return -EOPNOTSUPP; in fuse_dev_ioctl_backing_open()
2634 return -EFAULT; in fuse_dev_ioctl_backing_open()
2636 return fuse_backing_open(fud->fc, &map); in fuse_dev_ioctl_backing_open()
2648 return -EOPNOTSUPP; in fuse_dev_ioctl_backing_close()
2651 return -EFAULT; in fuse_dev_ioctl_backing_close()
2653 return fuse_backing_close(fud->fc, backing_id); in fuse_dev_ioctl_backing_close()
2658 int err = -EINVAL; in fuse_dev_ioctl_sync_init()
2662 WRITE_ONCE(file->private_data, FUSE_DEV_SYNC_INIT); in fuse_dev_ioctl_sync_init()
2688 return -ENOTTY; in fuse_dev_ioctl()
2699 seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev); in fuse_dev_show_fdinfo()
2732 int err = -ENOMEM; in fuse_dev_init()
2737 goto out; in fuse_dev_init()
2747 out: in fuse_dev_init()