Lines Matching +full:t +full:- +full:head
21 #include <linux/backing-dev.h>
42 #include <linux/percpu-refcount.h>
59 unsigned head; /* Written to by userland or under ring_lock member
73 * Plugging is meant to work with larger batches of IOs. If we don't
74 * have more than the below, then don't bother setting up a plug.
114 * The real limit is nr_events - 1, which will be larger (see
131 * signals when all in-flight requests are done
182 struct wait_queue_head *head; member
194 * access the file pointer through any of the sub-structs,
221 /*------ sysctl variables----*/
225 /*----end sysctl variables---*/
229 .procname = "aio-nr",
236 .procname = "aio-max-nr",
263 struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); in aio_private_file()
267 inode->i_mapping->a_ops = &aio_ctx_aops; in aio_private_file()
268 inode->i_mapping->i_private_data = ctx; in aio_private_file()
269 inode->i_size = PAGE_SIZE * nr_pages; in aio_private_file()
281 return -ENOMEM; in aio_init_fs_context()
282 fc->s_iflags |= SB_I_NOEXEC; in aio_init_fs_context()
310 struct file *aio_ring_file = ctx->aio_ring_file; in put_aio_ring_file()
317 i_mapping = aio_ring_file->f_mapping; in put_aio_ring_file()
318 spin_lock(&i_mapping->i_private_lock); in put_aio_ring_file()
319 i_mapping->i_private_data = NULL; in put_aio_ring_file()
320 ctx->aio_ring_file = NULL; in put_aio_ring_file()
321 spin_unlock(&i_mapping->i_private_lock); in put_aio_ring_file()
336 for (i = 0; i < ctx->nr_pages; i++) { in aio_free_ring()
337 struct folio *folio = ctx->ring_folios[i]; in aio_free_ring()
342 pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, in aio_free_ring()
344 ctx->ring_folios[i] = NULL; in aio_free_ring()
348 if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) { in aio_free_ring()
349 kfree(ctx->ring_folios); in aio_free_ring()
350 ctx->ring_folios = NULL; in aio_free_ring()
356 struct file *file = vma->vm_file; in aio_ring_mremap()
357 struct mm_struct *mm = vma->vm_mm; in aio_ring_mremap()
359 int i, res = -EINVAL; in aio_ring_mremap()
361 spin_lock(&mm->ioctx_lock); in aio_ring_mremap()
363 table = rcu_dereference(mm->ioctx_table); in aio_ring_mremap()
367 for (i = 0; i < table->nr; i++) { in aio_ring_mremap()
370 ctx = rcu_dereference(table->table[i]); in aio_ring_mremap()
371 if (ctx && ctx->aio_ring_file == file) { in aio_ring_mremap()
372 if (!atomic_read(&ctx->dead)) { in aio_ring_mremap()
373 ctx->user_id = ctx->mmap_base = vma->vm_start; in aio_ring_mremap()
382 spin_unlock(&mm->ioctx_lock); in aio_ring_mremap()
397 desc->vm_flags |= VM_DONTEXPAND; in aio_ring_mmap_prepare()
398 desc->vm_ops = &aio_ring_vm_ops; in aio_ring_mmap_prepare()
415 /* mapping->i_private_lock here protects against the kioctx teardown. */ in aio_migrate_folio()
416 spin_lock(&mapping->i_private_lock); in aio_migrate_folio()
417 ctx = mapping->i_private_data; in aio_migrate_folio()
419 rc = -EINVAL; in aio_migrate_folio()
424 * to the ring's head, and prevents page migration from mucking in in aio_migrate_folio()
427 if (!mutex_trylock(&ctx->ring_lock)) { in aio_migrate_folio()
428 rc = -EAGAIN; in aio_migrate_folio()
432 idx = src->index; in aio_migrate_folio()
433 if (idx < (pgoff_t)ctx->nr_pages) { in aio_migrate_folio()
434 /* Make sure the old folio hasn't already been changed */ in aio_migrate_folio()
435 if (ctx->ring_folios[idx] != src) in aio_migrate_folio()
436 rc = -EAGAIN; in aio_migrate_folio()
438 rc = -EINVAL; in aio_migrate_folio()
457 spin_lock_irqsave(&ctx->completion_lock, flags); in aio_migrate_folio()
460 BUG_ON(ctx->ring_folios[idx] != src); in aio_migrate_folio()
461 ctx->ring_folios[idx] = dst; in aio_migrate_folio()
462 spin_unlock_irqrestore(&ctx->completion_lock, flags); in aio_migrate_folio()
468 mutex_unlock(&ctx->ring_lock); in aio_migrate_folio()
470 spin_unlock(&mapping->i_private_lock); in aio_migrate_folio()
485 struct mm_struct *mm = current->mm; in aio_setup_ring()
491 /* Compensate for the ring buffer's head/tail overlap entry */ in aio_setup_ring()
499 return -EINVAL; in aio_setup_ring()
503 ctx->aio_ring_file = NULL; in aio_setup_ring()
504 return -ENOMEM; in aio_setup_ring()
507 ctx->aio_ring_file = file; in aio_setup_ring()
508 nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) in aio_setup_ring()
511 ctx->ring_folios = ctx->internal_folios; in aio_setup_ring()
513 ctx->ring_folios = kcalloc(nr_pages, sizeof(struct folio *), in aio_setup_ring()
515 if (!ctx->ring_folios) { in aio_setup_ring()
517 return -ENOMEM; in aio_setup_ring()
524 folio = __filemap_get_folio(file->f_mapping, i, in aio_setup_ring()
530 pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, in aio_setup_ring()
534 ctx->ring_folios[i] = folio; in aio_setup_ring()
536 ctx->nr_pages = i; in aio_setup_ring()
540 return -ENOMEM; in aio_setup_ring()
543 ctx->mmap_size = nr_pages * PAGE_SIZE; in aio_setup_ring()
544 pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); in aio_setup_ring()
547 ctx->mmap_size = 0; in aio_setup_ring()
549 return -EINTR; in aio_setup_ring()
552 ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, in aio_setup_ring()
556 if (IS_ERR((void *)ctx->mmap_base)) { in aio_setup_ring()
557 ctx->mmap_size = 0; in aio_setup_ring()
559 return -ENOMEM; in aio_setup_ring()
562 pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); in aio_setup_ring()
564 ctx->user_id = ctx->mmap_base; in aio_setup_ring()
565 ctx->nr_events = nr_events; /* trusted copy */ in aio_setup_ring()
567 ring = folio_address(ctx->ring_folios[0]); in aio_setup_ring()
568 ring->nr = nr_events; /* user copy */ in aio_setup_ring()
569 ring->id = ~0U; in aio_setup_ring()
570 ring->head = ring->tail = 0; in aio_setup_ring()
571 ring->magic = AIO_RING_MAGIC; in aio_setup_ring()
572 ring->compat_features = AIO_RING_COMPAT_FEATURES; in aio_setup_ring()
573 ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; in aio_setup_ring()
574 ring->header_length = sizeof(struct aio_ring); in aio_setup_ring()
575 flush_dcache_folio(ctx->ring_folios[0]); in aio_setup_ring()
581 #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
582 #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
591 * kiocb didn't come from aio or is neither a read nor a write, hence in kiocb_set_cancel_fn()
594 if (!(iocb->ki_flags & IOCB_AIO_RW)) in kiocb_set_cancel_fn()
599 if (WARN_ON_ONCE(!list_empty(&req->ki_list))) in kiocb_set_cancel_fn()
602 ctx = req->ki_ctx; in kiocb_set_cancel_fn()
604 spin_lock_irqsave(&ctx->ctx_lock, flags); in kiocb_set_cancel_fn()
605 list_add_tail(&req->ki_list, &ctx->active_reqs); in kiocb_set_cancel_fn()
606 req->ki_cancel = cancel; in kiocb_set_cancel_fn()
607 spin_unlock_irqrestore(&ctx->ctx_lock, flags); in kiocb_set_cancel_fn()
623 free_percpu(ctx->cpu); in free_ioctx()
624 percpu_ref_exit(&ctx->reqs); in free_ioctx()
625 percpu_ref_exit(&ctx->users); in free_ioctx()
633 /* At this point we know that there are no any in-flight requests */ in free_ioctx_reqs()
634 if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) in free_ioctx_reqs()
635 complete(&ctx->rq_wait->comp); in free_ioctx_reqs()
637 /* Synchronize against RCU protected table->table[] dereferences */ in free_ioctx_reqs()
638 INIT_RCU_WORK(&ctx->free_rwork, free_ioctx); in free_ioctx_reqs()
639 queue_rcu_work(system_percpu_wq, &ctx->free_rwork); in free_ioctx_reqs()
644 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
652 spin_lock_irq(&ctx->ctx_lock); in free_ioctx_users()
654 while (!list_empty(&ctx->active_reqs)) { in free_ioctx_users()
655 req = list_first_entry(&ctx->active_reqs, in free_ioctx_users()
657 req->ki_cancel(&req->rw); in free_ioctx_users()
658 list_del_init(&req->ki_list); in free_ioctx_users()
661 spin_unlock_irq(&ctx->ctx_lock); in free_ioctx_users()
663 percpu_ref_kill(&ctx->reqs); in free_ioctx_users()
664 percpu_ref_put(&ctx->reqs); in free_ioctx_users()
673 spin_lock(&mm->ioctx_lock); in ioctx_add_table()
674 table = rcu_dereference_raw(mm->ioctx_table); in ioctx_add_table()
678 for (i = 0; i < table->nr; i++) in ioctx_add_table()
679 if (!rcu_access_pointer(table->table[i])) { in ioctx_add_table()
680 ctx->id = i; in ioctx_add_table()
681 rcu_assign_pointer(table->table[i], ctx); in ioctx_add_table()
682 spin_unlock(&mm->ioctx_lock); in ioctx_add_table()
686 * changes ring_folios by ->ring_lock. in ioctx_add_table()
688 ring = folio_address(ctx->ring_folios[0]); in ioctx_add_table()
689 ring->id = ctx->id; in ioctx_add_table()
693 new_nr = (table ? table->nr : 1) * 4; in ioctx_add_table()
694 spin_unlock(&mm->ioctx_lock); in ioctx_add_table()
698 return -ENOMEM; in ioctx_add_table()
700 table->nr = new_nr; in ioctx_add_table()
702 spin_lock(&mm->ioctx_lock); in ioctx_add_table()
703 old = rcu_dereference_raw(mm->ioctx_table); in ioctx_add_table()
706 rcu_assign_pointer(mm->ioctx_table, table); in ioctx_add_table()
707 } else if (table->nr > old->nr) { in ioctx_add_table()
708 memcpy(table->table, old->table, in ioctx_add_table()
709 old->nr * sizeof(struct kioctx *)); in ioctx_add_table()
711 rcu_assign_pointer(mm->ioctx_table, table); in ioctx_add_table()
723 if (WARN_ON(aio_nr - nr > aio_nr)) in aio_nr_sub()
726 aio_nr -= nr; in aio_nr_sub()
735 struct mm_struct *mm = current->mm; in ioctx_alloc()
737 int err = -ENOMEM; in ioctx_alloc()
740 * Store the original nr_events -- what userspace passed to io_setup(), in ioctx_alloc()
741 * for counting against the global limit -- before it changes. in ioctx_alloc()
752 * counters at a time, so make sure that isn't 0: in ioctx_alloc()
760 return ERR_PTR(-EINVAL); in ioctx_alloc()
764 return ERR_PTR(-EAGAIN); in ioctx_alloc()
768 return ERR_PTR(-ENOMEM); in ioctx_alloc()
770 ctx->max_reqs = max_reqs; in ioctx_alloc()
772 spin_lock_init(&ctx->ctx_lock); in ioctx_alloc()
773 spin_lock_init(&ctx->completion_lock); in ioctx_alloc()
774 mutex_init(&ctx->ring_lock); in ioctx_alloc()
777 mutex_lock(&ctx->ring_lock); in ioctx_alloc()
778 init_waitqueue_head(&ctx->wait); in ioctx_alloc()
780 INIT_LIST_HEAD(&ctx->active_reqs); in ioctx_alloc()
782 if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL)) in ioctx_alloc()
785 if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL)) in ioctx_alloc()
788 ctx->cpu = alloc_percpu(struct kioctx_cpu); in ioctx_alloc()
789 if (!ctx->cpu) in ioctx_alloc()
796 atomic_set(&ctx->reqs_available, ctx->nr_events - 1); in ioctx_alloc()
797 ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4); in ioctx_alloc()
798 if (ctx->req_batch < 1) in ioctx_alloc()
799 ctx->req_batch = 1; in ioctx_alloc()
803 if (aio_nr + ctx->max_reqs > aio_max_nr || in ioctx_alloc()
804 aio_nr + ctx->max_reqs < aio_nr) { in ioctx_alloc()
806 err = -EAGAIN; in ioctx_alloc()
809 aio_nr += ctx->max_reqs; in ioctx_alloc()
812 percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */ in ioctx_alloc()
813 percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */ in ioctx_alloc()
820 mutex_unlock(&ctx->ring_lock); in ioctx_alloc()
823 ctx, ctx->user_id, mm, ctx->nr_events); in ioctx_alloc()
827 aio_nr_sub(ctx->max_reqs); in ioctx_alloc()
829 atomic_set(&ctx->dead, 1); in ioctx_alloc()
830 if (ctx->mmap_size) in ioctx_alloc()
831 vm_munmap(ctx->mmap_base, ctx->mmap_size); in ioctx_alloc()
834 mutex_unlock(&ctx->ring_lock); in ioctx_alloc()
835 free_percpu(ctx->cpu); in ioctx_alloc()
836 percpu_ref_exit(&ctx->reqs); in ioctx_alloc()
837 percpu_ref_exit(&ctx->users); in ioctx_alloc()
853 spin_lock(&mm->ioctx_lock); in kill_ioctx()
854 if (atomic_xchg(&ctx->dead, 1)) { in kill_ioctx()
855 spin_unlock(&mm->ioctx_lock); in kill_ioctx()
856 return -EINVAL; in kill_ioctx()
859 table = rcu_dereference_raw(mm->ioctx_table); in kill_ioctx()
860 WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id])); in kill_ioctx()
861 RCU_INIT_POINTER(table->table[ctx->id], NULL); in kill_ioctx()
862 spin_unlock(&mm->ioctx_lock); in kill_ioctx()
865 wake_up_all(&ctx->wait); in kill_ioctx()
869 * the outstanding kiocbs have finished - but by then io_destroy in kill_ioctx()
871 * -EAGAIN with no ioctxs actually in use (as far as userspace in kill_ioctx()
874 aio_nr_sub(ctx->max_reqs); in kill_ioctx()
876 if (ctx->mmap_size) in kill_ioctx()
877 vm_munmap(ctx->mmap_base, ctx->mmap_size); in kill_ioctx()
879 ctx->rq_wait = wait; in kill_ioctx()
880 percpu_ref_kill(&ctx->users); in kill_ioctx()
894 struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table); in exit_aio()
901 atomic_set(&wait.count, table->nr); in exit_aio()
905 for (i = 0; i < table->nr; ++i) { in exit_aio()
907 rcu_dereference_protected(table->table[i], true); in exit_aio()
915 * We don't need to bother with munmap() here - exit_mmap(mm) in exit_aio()
916 * is coming and it'll unmap everything. And we simply can't, in exit_aio()
917 * this is not necessarily our ->mm. in exit_aio()
918 * Since kill_ioctx() uses non-zero ->mmap_size as indicator in exit_aio()
921 ctx->mmap_size = 0; in exit_aio()
930 RCU_INIT_POINTER(mm->ioctx_table, NULL); in exit_aio()
940 kcpu = this_cpu_ptr(ctx->cpu); in put_reqs_available()
941 kcpu->reqs_available += nr; in put_reqs_available()
943 while (kcpu->reqs_available >= ctx->req_batch * 2) { in put_reqs_available()
944 kcpu->reqs_available -= ctx->req_batch; in put_reqs_available()
945 atomic_add(ctx->req_batch, &ctx->reqs_available); in put_reqs_available()
958 kcpu = this_cpu_ptr(ctx->cpu); in __get_reqs_available()
959 if (!kcpu->reqs_available) { in __get_reqs_available()
960 int avail = atomic_read(&ctx->reqs_available); in __get_reqs_available()
963 if (avail < ctx->req_batch) in __get_reqs_available()
965 } while (!atomic_try_cmpxchg(&ctx->reqs_available, in __get_reqs_available()
966 &avail, avail - ctx->req_batch)); in __get_reqs_available()
968 kcpu->reqs_available += ctx->req_batch; in __get_reqs_available()
972 kcpu->reqs_available--; in __get_reqs_available()
983 * called holding ctx->completion_lock.
985 static void refill_reqs_available(struct kioctx *ctx, unsigned head, in refill_reqs_available() argument
990 /* Clamp head since userland can write to it. */ in refill_reqs_available()
991 head %= ctx->nr_events; in refill_reqs_available()
992 if (head <= tail) in refill_reqs_available()
993 events_in_ring = tail - head; in refill_reqs_available()
995 events_in_ring = ctx->nr_events - (head - tail); in refill_reqs_available()
997 completed = ctx->completed_events; in refill_reqs_available()
999 completed -= events_in_ring; in refill_reqs_available()
1006 ctx->completed_events -= completed; in refill_reqs_available()
1016 spin_lock_irq(&ctx->completion_lock); in user_refill_reqs_available()
1017 if (ctx->completed_events) { in user_refill_reqs_available()
1019 unsigned head; in user_refill_reqs_available() local
1021 /* Access of ring->head may race with aio_read_events_ring() in user_refill_reqs_available()
1024 * part is that head cannot pass tail since we prevent in user_refill_reqs_available()
1026 * ctx->completion_lock. Even if head is invalid, the check in user_refill_reqs_available()
1027 * against ctx->completed_events below will make sure we do the in user_refill_reqs_available()
1030 ring = folio_address(ctx->ring_folios[0]); in user_refill_reqs_available()
1031 head = ring->head; in user_refill_reqs_available()
1033 refill_reqs_available(ctx, head, ctx->tail); in user_refill_reqs_available()
1036 spin_unlock_irq(&ctx->completion_lock); in user_refill_reqs_available()
1051 * The refcount is initialized to 2 - one for the async op completion,
1067 percpu_ref_get(&ctx->reqs); in aio_get_req()
1068 req->ki_ctx = ctx; in aio_get_req()
1069 INIT_LIST_HEAD(&req->ki_list); in aio_get_req()
1070 refcount_set(&req->ki_refcnt, 2); in aio_get_req()
1071 req->ki_eventfd = NULL; in aio_get_req()
1078 struct mm_struct *mm = current->mm; in lookup_ioctx()
1083 if (get_user(id, &ring->id)) in lookup_ioctx()
1087 table = rcu_dereference(mm->ioctx_table); in lookup_ioctx()
1089 if (!table || id >= table->nr) in lookup_ioctx()
1092 id = array_index_nospec(id, table->nr); in lookup_ioctx()
1093 ctx = rcu_dereference(table->table[id]); in lookup_ioctx()
1094 if (ctx && ctx->user_id == ctx_id) { in lookup_ioctx()
1095 if (percpu_ref_tryget_live(&ctx->users)) in lookup_ioctx()
1105 if (iocb->ki_eventfd) in iocb_destroy()
1106 eventfd_ctx_put(iocb->ki_eventfd); in iocb_destroy()
1107 if (iocb->ki_filp) in iocb_destroy()
1108 fput(iocb->ki_filp); in iocb_destroy()
1109 percpu_ref_put(&iocb->ki_ctx->reqs); in iocb_destroy()
1123 struct kioctx *ctx = iocb->ki_ctx; in aio_complete()
1126 unsigned tail, pos, head, avail; in aio_complete() local
1131 * ctx->completion_lock to prevent other code from messing with the tail in aio_complete()
1134 spin_lock_irqsave(&ctx->completion_lock, flags); in aio_complete()
1136 tail = ctx->tail; in aio_complete()
1139 if (++tail >= ctx->nr_events) in aio_complete()
1142 ev_page = folio_address(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); in aio_complete()
1145 *event = iocb->ki_res; in aio_complete()
1147 flush_dcache_folio(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); in aio_complete()
1150 (void __user *)(unsigned long)iocb->ki_res.obj, in aio_complete()
1151 iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2); in aio_complete()
1158 ctx->tail = tail; in aio_complete()
1160 ring = folio_address(ctx->ring_folios[0]); in aio_complete()
1161 head = ring->head; in aio_complete()
1162 ring->tail = tail; in aio_complete()
1163 flush_dcache_folio(ctx->ring_folios[0]); in aio_complete()
1165 ctx->completed_events++; in aio_complete()
1166 if (ctx->completed_events > 1) in aio_complete()
1167 refill_reqs_available(ctx, head, tail); in aio_complete()
1169 avail = tail > head in aio_complete()
1170 ? tail - head in aio_complete()
1171 : tail + ctx->nr_events - head; in aio_complete()
1172 spin_unlock_irqrestore(&ctx->completion_lock, flags); in aio_complete()
1181 if (iocb->ki_eventfd) in aio_complete()
1182 eventfd_signal(iocb->ki_eventfd); in aio_complete()
1192 if (waitqueue_active(&ctx->wait)) { in aio_complete()
1196 spin_lock_irqsave(&ctx->wait.lock, flags); in aio_complete()
1197 list_for_each_entry_safe(curr, next, &ctx->wait.head, w.entry) in aio_complete()
1198 if (avail >= curr->min_nr) { in aio_complete()
1199 wake_up_process(curr->w.private); in aio_complete()
1200 list_del_init_careful(&curr->w.entry); in aio_complete()
1202 spin_unlock_irqrestore(&ctx->wait.lock, flags); in aio_complete()
1208 if (refcount_dec_and_test(&iocb->ki_refcnt)) { in iocb_put()
1222 unsigned head, tail, pos; in aio_read_events_ring() local
1229 * and repeat. This should be rare enough that it doesn't cause in aio_read_events_ring()
1233 mutex_lock(&ctx->ring_lock); in aio_read_events_ring()
1235 /* Access to ->ring_folios here is protected by ctx->ring_lock. */ in aio_read_events_ring()
1236 ring = folio_address(ctx->ring_folios[0]); in aio_read_events_ring()
1237 head = ring->head; in aio_read_events_ring()
1238 tail = ring->tail; in aio_read_events_ring()
1246 pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events); in aio_read_events_ring()
1248 if (head == tail) in aio_read_events_ring()
1251 head %= ctx->nr_events; in aio_read_events_ring()
1252 tail %= ctx->nr_events; in aio_read_events_ring()
1259 avail = (head <= tail ? tail : ctx->nr_events) - head; in aio_read_events_ring()
1260 if (head == tail) in aio_read_events_ring()
1263 pos = head + AIO_EVENTS_OFFSET; in aio_read_events_ring()
1264 folio = ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]; in aio_read_events_ring()
1267 avail = min(avail, nr - ret); in aio_read_events_ring()
1268 avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); in aio_read_events_ring()
1275 ret = -EFAULT; in aio_read_events_ring()
1280 head += avail; in aio_read_events_ring()
1281 head %= ctx->nr_events; in aio_read_events_ring()
1284 ring = folio_address(ctx->ring_folios[0]); in aio_read_events_ring()
1285 ring->head = head; in aio_read_events_ring()
1286 flush_dcache_folio(ctx->ring_folios[0]); in aio_read_events_ring()
1288 pr_debug("%li h%u t%u\n", ret, head, tail); in aio_read_events_ring()
1290 mutex_unlock(&ctx->ring_lock); in aio_read_events_ring()
1298 long ret = aio_read_events_ring(ctx, event + *i, nr - *i); in aio_read_events()
1303 if (unlikely(atomic_read(&ctx->dead))) in aio_read_events()
1304 ret = -EINVAL; in aio_read_events()
1316 struct hrtimer_sleeper t; in read_events() local
1321 * Note that aio_read_events() is being called as the conditional - i.e. in read_events()
1328 * This should be ok, provided it doesn't flip the state back to in read_events()
1329 * TASK_RUNNING and return 0 too much - that causes us to spin. That in read_events()
1338 hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in read_events()
1340 hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns); in read_events()
1341 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL); in read_events()
1349 w.min_nr = min_nr - ret; in read_events()
1351 ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE); in read_events()
1352 if (!ret2 && !t.task) in read_events()
1353 ret2 = -ETIME; in read_events()
1362 finish_wait(&ctx->wait, &w.w); in read_events()
1363 hrtimer_cancel(&t.timer); in read_events()
1364 destroy_hrtimer_on_stack(&t.timer); in read_events()
1374 * handle. May fail with -EINVAL if *ctxp is not initialized,
1376 * with -EAGAIN if the specified nr_events exceeds the user's limit
1377 * of available events. May fail with -ENOMEM if insufficient kernel
1378 * resources are available. May fail with -EFAULT if an invalid
1379 * pointer is passed for ctxp. Will fail with -ENOSYS if not
1392 ret = -EINVAL; in SYSCALL_DEFINE2()
1402 ret = put_user(ioctx->user_id, ctxp); in SYSCALL_DEFINE2()
1404 kill_ioctx(current->mm, ioctx, NULL); in SYSCALL_DEFINE2()
1405 percpu_ref_put(&ioctx->users); in SYSCALL_DEFINE2()
1423 ret = -EINVAL; in COMPAT_SYSCALL_DEFINE2()
1434 ret = put_user((u32)ioctx->user_id, ctx32p); in COMPAT_SYSCALL_DEFINE2()
1436 kill_ioctx(current->mm, ioctx, NULL); in COMPAT_SYSCALL_DEFINE2()
1437 percpu_ref_put(&ioctx->users); in COMPAT_SYSCALL_DEFINE2()
1447 * AIOs and block on completion. Will fail with -ENOSYS if not
1448 * implemented. May fail with -EINVAL if the context pointed to
1462 * in a thread-safe way. If we try to set it here then we have in SYSCALL_DEFINE1()
1465 ret = kill_ioctx(current->mm, ioctx, &wait); in SYSCALL_DEFINE1()
1466 percpu_ref_put(&ioctx->users); in SYSCALL_DEFINE1()
1469 * keep using user-space buffers even if user thinks the context in SYSCALL_DEFINE1()
1478 return -EINVAL; in SYSCALL_DEFINE1()
1483 struct kioctx *ctx = iocb->ki_ctx; in aio_remove_iocb()
1486 spin_lock_irqsave(&ctx->ctx_lock, flags); in aio_remove_iocb()
1487 list_del(&iocb->ki_list); in aio_remove_iocb()
1488 spin_unlock_irqrestore(&ctx->ctx_lock, flags); in aio_remove_iocb()
1495 if (!list_empty_careful(&iocb->ki_list)) in aio_complete_rw()
1498 if (kiocb->ki_flags & IOCB_WRITE) { in aio_complete_rw()
1499 struct inode *inode = file_inode(kiocb->ki_filp); in aio_complete_rw()
1501 if (S_ISREG(inode->i_mode)) in aio_complete_rw()
1505 iocb->ki_res.res = res; in aio_complete_rw()
1506 iocb->ki_res.res2 = 0; in aio_complete_rw()
1514 req->ki_write_stream = 0; in aio_prep_rw()
1515 req->ki_complete = aio_complete_rw; in aio_prep_rw()
1516 req->private = NULL; in aio_prep_rw()
1517 req->ki_pos = iocb->aio_offset; in aio_prep_rw()
1518 req->ki_flags = req->ki_filp->f_iocb_flags | IOCB_AIO_RW; in aio_prep_rw()
1519 if (iocb->aio_flags & IOCB_FLAG_RESFD) in aio_prep_rw()
1520 req->ki_flags |= IOCB_EVENTFD; in aio_prep_rw()
1521 if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { in aio_prep_rw()
1527 ret = ioprio_check_cap(iocb->aio_reqprio); in aio_prep_rw()
1533 req->ki_ioprio = iocb->aio_reqprio; in aio_prep_rw()
1535 req->ki_ioprio = get_current_ioprio(); in aio_prep_rw()
1537 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags, rw_type); in aio_prep_rw()
1541 req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ in aio_prep_rw()
1549 void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; in aio_setup_rw()
1550 size_t len = iocb->aio_nbytes; in aio_setup_rw()
1564 case -EIOCBQUEUED: in aio_rw_done()
1566 case -ERESTARTSYS: in aio_rw_done()
1567 case -ERESTARTNOINTR: in aio_rw_done()
1568 case -ERESTARTNOHAND: in aio_rw_done()
1569 case -ERESTART_RESTARTBLOCK: in aio_rw_done()
1574 ret = -EINTR; in aio_rw_done()
1577 req->ki_complete(req, ret); in aio_rw_done()
1592 file = req->ki_filp; in aio_read()
1593 if (unlikely(!(file->f_mode & FMODE_READ))) in aio_read()
1594 return -EBADF; in aio_read()
1595 if (unlikely(!file->f_op->read_iter)) in aio_read()
1596 return -EINVAL; in aio_read()
1601 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); in aio_read()
1603 aio_rw_done(req, file->f_op->read_iter(req, &iter)); in aio_read()
1619 file = req->ki_filp; in aio_write()
1621 if (unlikely(!(file->f_mode & FMODE_WRITE))) in aio_write()
1622 return -EBADF; in aio_write()
1623 if (unlikely(!file->f_op->write_iter)) in aio_write()
1624 return -EINVAL; in aio_write()
1629 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); in aio_write()
1631 if (S_ISREG(file_inode(file)->i_mode)) in aio_write()
1633 req->ki_flags |= IOCB_WRITE; in aio_write()
1634 aio_rw_done(req, file->f_op->write_iter(req, &iter)); in aio_write()
1643 const struct cred *old_cred = override_creds(iocb->fsync.creds); in aio_fsync_work()
1645 iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync); in aio_fsync_work()
1647 put_cred(iocb->fsync.creds); in aio_fsync_work()
1654 if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || in aio_fsync()
1655 iocb->aio_rw_flags)) in aio_fsync()
1656 return -EINVAL; in aio_fsync()
1658 if (unlikely(!req->file->f_op->fsync)) in aio_fsync()
1659 return -EINVAL; in aio_fsync()
1661 req->creds = prepare_creds(); in aio_fsync()
1662 if (!req->creds) in aio_fsync()
1663 return -ENOMEM; in aio_fsync()
1665 req->datasync = datasync; in aio_fsync()
1666 INIT_WORK(&req->work, aio_fsync_work); in aio_fsync()
1667 schedule_work(&req->work); in aio_fsync()
1681 * case where the ->poll() provider decides to free its waitqueue early.
1683 * Returns true on success, meaning that req->head->lock was locked, req->wait
1684 * is on req->head, and an RCU read lock was taken. Returns false if the
1689 wait_queue_head_t *head; in poll_iocb_lock_wq() local
1697 * all users of wake_up_pollfree() will RCU-delay the actual free. If in poll_iocb_lock_wq()
1699 * non-NULL, we can then lock it without the memory being freed out from in poll_iocb_lock_wq()
1707 head = smp_load_acquire(&req->head); in poll_iocb_lock_wq()
1708 if (head) { in poll_iocb_lock_wq()
1709 spin_lock(&head->lock); in poll_iocb_lock_wq()
1710 if (!list_empty(&req->wait.entry)) in poll_iocb_lock_wq()
1712 spin_unlock(&head->lock); in poll_iocb_lock_wq()
1720 spin_unlock(&req->head->lock); in poll_iocb_unlock_wq()
1728 struct poll_table_struct pt = { ._key = req->events }; in aio_poll_complete_work()
1729 struct kioctx *ctx = iocb->ki_ctx; in aio_poll_complete_work()
1732 if (!READ_ONCE(req->cancelled)) in aio_poll_complete_work()
1733 mask = vfs_poll(req->file, &pt) & req->events; in aio_poll_complete_work()
1736 * Note that ->ki_cancel callers also delete iocb from active_reqs after in aio_poll_complete_work()
1737 * calling ->ki_cancel. We need the ctx_lock roundtrip here to in aio_poll_complete_work()
1742 spin_lock_irq(&ctx->ctx_lock); in aio_poll_complete_work()
1744 if (!mask && !READ_ONCE(req->cancelled)) { in aio_poll_complete_work()
1746 * The request isn't actually ready to be completed yet. in aio_poll_complete_work()
1749 if (req->work_need_resched) { in aio_poll_complete_work()
1750 schedule_work(&req->work); in aio_poll_complete_work()
1751 req->work_need_resched = false; in aio_poll_complete_work()
1753 req->work_scheduled = false; in aio_poll_complete_work()
1756 spin_unlock_irq(&ctx->ctx_lock); in aio_poll_complete_work()
1759 list_del_init(&req->wait.entry); in aio_poll_complete_work()
1762 list_del_init(&iocb->ki_list); in aio_poll_complete_work()
1763 iocb->ki_res.res = mangle_poll(mask); in aio_poll_complete_work()
1764 spin_unlock_irq(&ctx->ctx_lock); in aio_poll_complete_work()
1773 struct poll_iocb *req = &aiocb->poll; in aio_poll_cancel()
1776 WRITE_ONCE(req->cancelled, true); in aio_poll_cancel()
1777 if (!req->work_scheduled) { in aio_poll_cancel()
1778 schedule_work(&aiocb->poll.work); in aio_poll_cancel()
1779 req->work_scheduled = true; in aio_poll_cancel()
1782 } /* else, the request was force-cancelled by POLLFREE already */ in aio_poll_cancel()
1796 if (mask && !(mask & req->events)) in aio_poll_wake()
1804 * the events, so inline completion isn't possible. in aio_poll_wake()
1812 if (mask && !req->work_scheduled && in aio_poll_wake()
1813 spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) { in aio_poll_wake()
1814 struct kioctx *ctx = iocb->ki_ctx; in aio_poll_wake()
1816 list_del_init(&req->wait.entry); in aio_poll_wake()
1817 list_del(&iocb->ki_list); in aio_poll_wake()
1818 iocb->ki_res.res = mangle_poll(mask); in aio_poll_wake()
1819 if (iocb->ki_eventfd && !eventfd_signal_allowed()) { in aio_poll_wake()
1821 INIT_WORK(&req->work, aio_poll_put_work); in aio_poll_wake()
1822 schedule_work(&req->work); in aio_poll_wake()
1824 spin_unlock_irqrestore(&ctx->ctx_lock, flags); in aio_poll_wake()
1832 * Don't remove the request from the waitqueue here, as it might in aio_poll_wake()
1833 * not actually be complete yet (we won't know until vfs_poll() in aio_poll_wake()
1837 if (req->work_scheduled) { in aio_poll_wake()
1838 req->work_need_resched = true; in aio_poll_wake()
1840 schedule_work(&req->work); in aio_poll_wake()
1841 req->work_scheduled = true; in aio_poll_wake()
1845 * If the waitqueue is being freed early but we can't complete in aio_poll_wake()
1851 * cancelled, to potentially skip an unneeded call to ->poll(). in aio_poll_wake()
1854 WRITE_ONCE(req->cancelled, true); in aio_poll_wake()
1855 list_del_init(&req->wait.entry); in aio_poll_wake()
1859 * as req->head is NULL'ed out, the request can be in aio_poll_wake()
1863 smp_store_release(&req->head, NULL); in aio_poll_wake()
1877 aio_poll_queue_proc(struct file *file, struct wait_queue_head *head, in aio_poll_queue_proc() argument
1883 if (unlikely(pt->queued)) { in aio_poll_queue_proc()
1884 pt->error = -EINVAL; in aio_poll_queue_proc()
1888 pt->queued = true; in aio_poll_queue_proc()
1889 pt->error = 0; in aio_poll_queue_proc()
1890 pt->iocb->poll.head = head; in aio_poll_queue_proc()
1891 add_wait_queue(head, &pt->iocb->poll.wait); in aio_poll_queue_proc()
1896 struct kioctx *ctx = aiocb->ki_ctx; in aio_poll()
1897 struct poll_iocb *req = &aiocb->poll; in aio_poll()
1903 if ((u16)iocb->aio_buf != iocb->aio_buf) in aio_poll()
1904 return -EINVAL; in aio_poll()
1906 if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) in aio_poll()
1907 return -EINVAL; in aio_poll()
1909 INIT_WORK(&req->work, aio_poll_complete_work); in aio_poll()
1910 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; in aio_poll()
1912 req->head = NULL; in aio_poll()
1913 req->cancelled = false; in aio_poll()
1914 req->work_scheduled = false; in aio_poll()
1915 req->work_need_resched = false; in aio_poll()
1918 apt.pt._key = req->events; in aio_poll()
1921 apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ in aio_poll()
1924 INIT_LIST_HEAD(&req->wait.entry); in aio_poll()
1925 init_waitqueue_func_entry(&req->wait, aio_poll_wake); in aio_poll()
1927 mask = vfs_poll(req->file, &apt.pt) & req->events; in aio_poll()
1928 spin_lock_irq(&ctx->ctx_lock); in aio_poll()
1932 if (!on_queue || req->work_scheduled) { in aio_poll()
1944 list_del_init(&req->wait.entry); in aio_poll()
1947 WRITE_ONCE(req->cancelled, true); in aio_poll()
1953 list_add_tail(&aiocb->ki_list, &ctx->active_reqs); in aio_poll()
1954 aiocb->ki_cancel = aio_poll_cancel; in aio_poll()
1960 aiocb->ki_res.res = mangle_poll(mask); in aio_poll()
1963 spin_unlock_irq(&ctx->ctx_lock); in aio_poll()
1973 req->ki_filp = fget(iocb->aio_fildes); in __io_submit_one()
1974 if (unlikely(!req->ki_filp)) in __io_submit_one()
1975 return -EBADF; in __io_submit_one()
1977 if (iocb->aio_flags & IOCB_FLAG_RESFD) { in __io_submit_one()
1985 eventfd = eventfd_ctx_fdget(iocb->aio_resfd); in __io_submit_one()
1989 req->ki_eventfd = eventfd; in __io_submit_one()
1992 if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { in __io_submit_one()
1994 return -EFAULT; in __io_submit_one()
1997 req->ki_res.obj = (u64)(unsigned long)user_iocb; in __io_submit_one()
1998 req->ki_res.data = iocb->aio_data; in __io_submit_one()
1999 req->ki_res.res = 0; in __io_submit_one()
2000 req->ki_res.res2 = 0; in __io_submit_one()
2002 switch (iocb->aio_lio_opcode) { in __io_submit_one()
2004 return aio_read(&req->rw, iocb, false, compat); in __io_submit_one()
2006 return aio_write(&req->rw, iocb, false, compat); in __io_submit_one()
2008 return aio_read(&req->rw, iocb, true, compat); in __io_submit_one()
2010 return aio_write(&req->rw, iocb, true, compat); in __io_submit_one()
2012 return aio_fsync(&req->fsync, iocb, false); in __io_submit_one()
2014 return aio_fsync(&req->fsync, iocb, true); in __io_submit_one()
2018 pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); in __io_submit_one()
2019 return -EINVAL; in __io_submit_one()
2031 return -EFAULT; in io_submit_one()
2036 return -EINVAL; in io_submit_one()
2046 return -EINVAL; in io_submit_one()
2051 return -EAGAIN; in io_submit_one()
2060 * arranged for that to be done asynchronously. Anything non-zero in io_submit_one()
2072 * the number of iocbs queued. May return -EINVAL if the aio_context
2076 * -EFAULT if any of the data structures point to invalid data. May
2077 * fail with -EBADF if the file descriptor specified in the first
2078 * iocb is invalid. May fail with -EAGAIN if insufficient resources
2080 * fail with -ENOSYS if not implemented.
2091 return -EINVAL; in SYSCALL_DEFINE3()
2096 return -EINVAL; in SYSCALL_DEFINE3()
2099 if (nr > ctx->nr_events) in SYSCALL_DEFINE3()
2100 nr = ctx->nr_events; in SYSCALL_DEFINE3()
2108 ret = -EFAULT; in SYSCALL_DEFINE3()
2119 percpu_ref_put(&ctx->users); in SYSCALL_DEFINE3()
2133 return -EINVAL; in COMPAT_SYSCALL_DEFINE3()
2138 return -EINVAL; in COMPAT_SYSCALL_DEFINE3()
2141 if (nr > ctx->nr_events) in COMPAT_SYSCALL_DEFINE3()
2142 nr = ctx->nr_events; in COMPAT_SYSCALL_DEFINE3()
2150 ret = -EFAULT; in COMPAT_SYSCALL_DEFINE3()
2161 percpu_ref_put(&ctx->users); in COMPAT_SYSCALL_DEFINE3()
2171 * -EFAULT if any of the data structures pointed to are invalid.
2172 * May fail with -EINVAL if aio_context specified by ctx_id is
2173 * invalid. May fail with -EAGAIN if the iocb specified was not
2174 * cancelled. Will fail with -ENOSYS if not implemented.
2181 int ret = -EINVAL; in SYSCALL_DEFINE3()
2185 if (unlikely(get_user(key, &iocb->aio_key))) in SYSCALL_DEFINE3()
2186 return -EFAULT; in SYSCALL_DEFINE3()
2188 return -EINVAL; in SYSCALL_DEFINE3()
2192 return -EINVAL; in SYSCALL_DEFINE3()
2194 spin_lock_irq(&ctx->ctx_lock); in SYSCALL_DEFINE3()
2195 list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { in SYSCALL_DEFINE3()
2196 if (kiocb->ki_res.obj == obj) { in SYSCALL_DEFINE3()
2197 ret = kiocb->ki_cancel(&kiocb->rw); in SYSCALL_DEFINE3()
2198 list_del_init(&kiocb->ki_list); in SYSCALL_DEFINE3()
2202 spin_unlock_irq(&ctx->ctx_lock); in SYSCALL_DEFINE3()
2206 * The result argument is no longer used - the io_event is in SYSCALL_DEFINE3()
2207 * always delivered via the ring buffer. -EINPROGRESS indicates in SYSCALL_DEFINE3()
2210 ret = -EINPROGRESS; in SYSCALL_DEFINE3()
2213 percpu_ref_put(&ctx->users); in SYSCALL_DEFINE3()
2226 long ret = -EINVAL; in do_io_getevents()
2231 percpu_ref_put(&ioctx->users); in do_io_getevents()
2241 * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
2242 * out of range, if timeout is out of range. May fail with -EFAULT
2247 * timeout is relative. Will fail with -ENOSYS if not implemented.
2261 return -EFAULT; in SYSCALL_DEFINE5()
2265 ret = -EINTR; in SYSCALL_DEFINE5()
2290 return -EFAULT; in SYSCALL_DEFINE6()
2293 return -EFAULT; in SYSCALL_DEFINE6()
2304 ret = -ERESTARTNOHAND; in SYSCALL_DEFINE6()
2325 return -EFAULT; in SYSCALL_DEFINE6()
2328 return -EFAULT; in SYSCALL_DEFINE6()
2340 ret = -ERESTARTNOHAND; in SYSCALL_DEFINE6()
2355 struct timespec64 t; in SYSCALL_DEFINE5() local
2358 if (timeout && get_old_timespec32(&t, timeout)) in SYSCALL_DEFINE5()
2359 return -EFAULT; in SYSCALL_DEFINE5()
2361 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); in SYSCALL_DEFINE5()
2363 ret = -EINTR; in SYSCALL_DEFINE5()
2387 struct timespec64 t; in COMPAT_SYSCALL_DEFINE6() local
2391 if (timeout && get_old_timespec32(&t, timeout)) in COMPAT_SYSCALL_DEFINE6()
2392 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2395 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2401 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); in COMPAT_SYSCALL_DEFINE6()
2406 ret = -ERESTARTNOHAND; in COMPAT_SYSCALL_DEFINE6()
2422 struct timespec64 t; in COMPAT_SYSCALL_DEFINE6() local
2426 if (timeout && get_timespec64(&t, timeout)) in COMPAT_SYSCALL_DEFINE6()
2427 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2430 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2436 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); in COMPAT_SYSCALL_DEFINE6()
2441 ret = -ERESTARTNOHAND; in COMPAT_SYSCALL_DEFINE6()