Lines Matching full:ep
57 * ep->mtx, re-queueing items in level-triggered mode.
70 * > ep->mtx (per-eventpoll; sleepable; serializes most ops)
71 * > ep->lock (per-eventpoll; IRQ-safe spinlock)
74 * nested inside ep->mtx, outside ep->lock)
77 * - ep->lock is a spinlock because ep_poll_callback() is called from
78 * wake_up() which may run in hard-IRQ context. All ep->lock
80 * - ep->mtx is a sleepable mutex because the event delivery loop
87 * - Per-ep ep->mtx is preferred for scalability elsewhere. Events
91 * ep->mtx on both: outer first, target second. Since cycles are
92 * forbidden the set of live ep->mtx holds is always a strict chain,
102 * rbr - ep->mtx
103 * ovflist, rdllist - ep->lock (IRQ-safe)
104 * wq - ep->lock for queue mutation
106 * refs - file->f_lock for adds; ep->mtx for removes;
107 * RCU for readers (hlist_del_rcu + kfree_rcu(ep))
108 * ws - ep->mtx
115 * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr).
118 * rdllink, next - ep->lock
119 * ffd, ep - immutable after ep_insert()
120 * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead
126 * event - ep->mtx for writes; lockless read in
134 * Readiness is tracked in two lists under ep->lock:
139 * without ep->lock.
141 * Encoded in ep->ovflist:
163 * rollback. Caller holds ep->mtx.
174 * A and B both hold ep->mtx serially. B walks the rbtree with
176 * B and C both take ep->mtx; the loser sees fewer entries or an
186 * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock),
191 * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in
260 /* Link on the owning eventpoll's ready list (ep->rdllist). */
264 * Link on the owning eventpoll's scan-overflow list (ep->ovflist),
278 struct eventpoll *ep; member
385 * walks upward from the outer ep, both bounded by EP_MAX_NESTS.
393 * skip this machinery entirely and take only ep->mtx.
397 * a check and again at the end. ep->gen caches
398 * the value under which ep was last visited by
403 * "ep->gen == loop_check_gen" trigger in
487 * The caller owns a reference to @file or holds the ep->mtx for the in list_file()
560 /* True iff @epi is on its owning ep's ready list. */
584 /* True iff @ep is between ep_enter_scan() and ep_exit_scan(). */
585 static inline bool ep_is_scanning(struct eventpoll *ep) in ep_is_scanning() argument
587 return READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; in ep_is_scanning()
591 static inline void ep_enter_scan(struct eventpoll *ep) in ep_enter_scan() argument
593 WRITE_ONCE(ep->ovflist, NULL); in ep_enter_scan()
597 static inline void ep_exit_scan(struct eventpoll *ep) in ep_exit_scan() argument
599 WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); in ep_exit_scan()
602 /* True iff @epi is currently linked on its ep's ovflist. */
614 /* True iff @ep has ready events that epoll_wait() might harvest. */
615 static inline bool ep_events_available(struct eventpoll *ep) in ep_events_available() argument
617 unsigned int seq = read_seqcount_begin(&ep->seq); in ep_events_available()
619 return !list_empty_careful(&ep->rdllist) || ep_is_scanning(ep) || in ep_events_available()
620 read_seqcount_retry(&ep->seq, seq); in ep_events_available()
626 * from the epoll instance ep is preferred, but if it is not set fallback to
630 * @ep: Pointer to the eventpoll context.
635 struct eventpoll *ep) in busy_loop_ep_timeout() argument
637 unsigned long bp_usec = READ_ONCE(ep->busy_poll_usecs); in busy_loop_ep_timeout()
649 static bool ep_busy_loop_on(struct eventpoll *ep) in ep_busy_loop_on() argument
651 return !!READ_ONCE(ep->busy_poll_usecs) || in ep_busy_loop_on()
652 READ_ONCE(ep->prefer_busy_poll) || in ep_busy_loop_on()
658 struct eventpoll *ep = p; in ep_busy_loop_end() local
660 return ep_events_available(ep) || busy_loop_ep_timeout(start_time, ep); in ep_busy_loop_end()
669 static bool ep_busy_loop(struct eventpoll *ep) in ep_busy_loop() argument
671 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_busy_loop()
672 u16 budget = READ_ONCE(ep->busy_poll_budget); in ep_busy_loop()
673 bool prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); in ep_busy_loop()
678 if (napi_id_valid(napi_id) && ep_busy_loop_on(ep)) { in ep_busy_loop()
680 ep, prefer_busy_poll, budget); in ep_busy_loop()
681 if (ep_events_available(ep)) in ep_busy_loop()
690 ep->napi_id = 0; in ep_busy_loop()
701 struct eventpoll *ep = epi->ep; in ep_set_busy_poll_napi_id() local
706 if (!ep_busy_loop_on(ep)) in ep_set_busy_poll_napi_id()
723 if (!napi_id_valid(napi_id) || napi_id == ep->napi_id) in ep_set_busy_poll_napi_id()
727 ep->napi_id = napi_id; in ep_set_busy_poll_napi_id()
733 struct eventpoll *ep = file->private_data; in ep_eventpoll_bp_ioctl() local
756 WRITE_ONCE(ep->busy_poll_usecs, epoll_params.busy_poll_usecs); in ep_eventpoll_bp_ioctl()
757 WRITE_ONCE(ep->busy_poll_budget, epoll_params.busy_poll_budget); in ep_eventpoll_bp_ioctl()
758 WRITE_ONCE(ep->prefer_busy_poll, epoll_params.prefer_busy_poll); in ep_eventpoll_bp_ioctl()
762 epoll_params.busy_poll_usecs = READ_ONCE(ep->busy_poll_usecs); in ep_eventpoll_bp_ioctl()
763 epoll_params.busy_poll_budget = READ_ONCE(ep->busy_poll_budget); in ep_eventpoll_bp_ioctl()
764 epoll_params.prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); in ep_eventpoll_bp_ioctl()
773 static void ep_suspend_napi_irqs(struct eventpoll *ep) in ep_suspend_napi_irqs() argument
775 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_suspend_napi_irqs()
777 if (napi_id_valid(napi_id) && READ_ONCE(ep->prefer_busy_poll)) in ep_suspend_napi_irqs()
781 static void ep_resume_napi_irqs(struct eventpoll *ep) in ep_resume_napi_irqs() argument
783 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_resume_napi_irqs()
785 if (napi_id_valid(napi_id) && READ_ONCE(ep->prefer_busy_poll)) in ep_resume_napi_irqs()
791 static inline bool ep_busy_loop(struct eventpoll *ep) in ep_busy_loop() argument
806 static void ep_suspend_napi_irqs(struct eventpoll *ep) in ep_suspend_napi_irqs() argument
810 static void ep_resume_napi_irqs(struct eventpoll *ep) in ep_resume_napi_irqs() argument
843 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, in ep_poll_safewake() argument
853 * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can in ep_poll_safewake()
873 spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); in ep_poll_safewake()
874 ep->nests = nests + 1; in ep_poll_safewake()
875 wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags); in ep_poll_safewake()
876 ep->nests = 0; in ep_poll_safewake()
877 spin_unlock_irqrestore(&ep->poll_wait.lock, flags); in ep_poll_safewake()
882 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, in ep_poll_safewake() argument
885 wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags); in ep_poll_safewake()
916 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
928 /* call only when ep->mtx is held */
931 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
934 /* call only when ep->mtx is held */
948 /* call when ep->mtx cannot be held (ep_poll_callback) */
962 * ep->mutex needs to be held because we could be hit by
965 static void ep_start_scan(struct eventpoll *ep, struct list_head *scan_batch) in ep_start_scan() argument
969 * empty list. Also, set ep->ovflist to NULL so that events in ep_start_scan()
971 * have the poll callback to queue directly on ep->rdllist, in ep_start_scan()
976 spin_lock_irq(&ep->lock); in ep_start_scan()
977 write_seqcount_begin(&ep->seq); in ep_start_scan()
979 list_splice_init(&ep->rdllist, scan_batch); in ep_start_scan()
980 ep_enter_scan(ep); in ep_start_scan()
982 write_seqcount_end(&ep->seq); in ep_start_scan()
983 spin_unlock_irq(&ep->lock); in ep_start_scan()
986 static void ep_done_scan(struct eventpoll *ep, in ep_done_scan() argument
991 spin_lock_irq(&ep->lock); in ep_done_scan()
997 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; ) { in ep_done_scan()
1009 list_add(&epi->rdllink, &ep->rdllist); in ep_done_scan()
1014 write_seqcount_begin(&ep->seq); in ep_done_scan()
1016 /* Back out of scan mode; callbacks target ep->rdllist again. */ in ep_done_scan()
1017 ep_exit_scan(ep); in ep_done_scan()
1022 list_splice(scan_batch, &ep->rdllist); in ep_done_scan()
1024 write_seqcount_end(&ep->seq); in ep_done_scan()
1026 __pm_relax(ep->ws); in ep_done_scan()
1028 if (!list_empty(&ep->rdllist)) { in ep_done_scan()
1029 if (waitqueue_active(&ep->wq)) in ep_done_scan()
1030 wake_up(&ep->wq); in ep_done_scan()
1033 spin_unlock_irq(&ep->lock); in ep_done_scan()
1036 static void ep_get(struct eventpoll *ep) in ep_get() argument
1038 refcount_inc(&ep->refcount); in ep_get()
1042 * Drop a reference to @ep; returns true iff it was the last, in which
1045 static bool ep_put(struct eventpoll *ep) in ep_put() argument
1047 if (!refcount_dec_and_test(&ep->refcount)) in ep_put()
1050 WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root)); in ep_put()
1054 static void ep_free(struct eventpoll *ep) in ep_free() argument
1056 ep_resume_napi_irqs(ep); in ep_free()
1057 mutex_destroy(&ep->mtx); in ep_free()
1058 free_uid(ep->user); in ep_free()
1059 wakeup_source_unregister(ep->ws); in ep_free()
1060 /* ep_get_upwards_depth_proc() may still hold epi->ep under RCU */ in ep_free()
1061 kfree_rcu(ep, rcu); in ep_free()
1070 * because the caller holds ep->mtx and eventpoll_release_file() blocks
1096 static void ep_remove_file(struct eventpoll *ep, struct epitem *epi, in ep_remove_file() argument
1102 lockdep_assert_held(&ep->mtx); in ep_remove_file()
1127 static void ep_remove_epi(struct eventpoll *ep, struct epitem *epi) in ep_remove_epi() argument
1129 lockdep_assert_held(&ep->mtx); in ep_remove_epi()
1131 rb_erase_cached(&epi->rbn, &ep->rbr); in ep_remove_epi()
1133 spin_lock_irq(&ep->lock); in ep_remove_epi()
1136 spin_unlock_irq(&ep->lock); in ep_remove_epi()
1143 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make in ep_remove_epi()
1148 percpu_counter_dec(&ep->user->epoll_watches); in ep_remove_epi()
1152 * ep_remove variant for callers owing an additional reference to the ep
1154 static void ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
1159 lockdep_assert_held(&ep->mtx); in ep_remove()
1161 ep_unregister_pollwait(ep, epi); in ep_remove()
1172 ep_remove_file(ep, epi, file); in ep_remove()
1173 ep_remove_epi(ep, epi); in ep_remove()
1174 WARN_ON_ONCE(ep_put(ep)); in ep_remove()
1182 * on this ep. Must strictly precede ep_drain_tree() -- fusing the
1186 static void ep_drain_pollwaits(struct eventpoll *ep) in ep_drain_pollwaits() argument
1191 lockdep_assert_held(&ep->mtx); in ep_drain_pollwaits()
1193 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_drain_pollwaits()
1196 ep_unregister_pollwait(ep, epi); in ep_drain_pollwaits()
1209 * file serializes with us via ep->mtx; ep_remove() transparently
1213 static void ep_drain_tree(struct eventpoll *ep) in ep_drain_tree() argument
1218 lockdep_assert_held(&ep->mtx); in ep_drain_tree()
1220 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) { in ep_drain_tree()
1223 ep_remove(ep, epi); in ep_drain_tree()
1232 * Two passes under ep->mtx: first ep_drain_pollwaits() quiesces
1234 * ep->refcount is kept > 0 across the walk by the ep file's own
1238 static void ep_clear_and_put(struct eventpoll *ep) in ep_clear_and_put() argument
1240 /* Release any threads blocked in poll-on-ep. */ in ep_clear_and_put()
1241 if (waitqueue_active(&ep->poll_wait)) in ep_clear_and_put()
1242 ep_poll_safewake(ep, NULL, 0); in ep_clear_and_put()
1244 mutex_lock(&ep->mtx); in ep_clear_and_put()
1245 ep_drain_pollwaits(ep); in ep_clear_and_put()
1246 ep_drain_tree(ep); in ep_clear_and_put()
1247 mutex_unlock(&ep->mtx); in ep_clear_and_put()
1249 if (ep_put(ep)) in ep_clear_and_put()
1250 ep_free(ep); in ep_clear_and_put()
1276 struct eventpoll *ep = file->private_data; in ep_eventpoll_release() local
1278 if (ep) in ep_eventpoll_release()
1279 ep_clear_and_put(ep); in ep_eventpoll_release()
1288 struct eventpoll *ep = file->private_data; in __ep_eventpoll_poll() local
1297 poll_wait(file, &ep->poll_wait, wait); in __ep_eventpoll_poll()
1303 mutex_lock_nested(&ep->mtx, depth); in __ep_eventpoll_poll()
1304 ep_start_scan(ep, &scan_batch); in __ep_eventpoll_poll()
1319 ep_done_scan(ep, &scan_batch); in __ep_eventpoll_poll()
1320 mutex_unlock(&ep->mtx); in __ep_eventpoll_poll()
1326 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
1359 struct eventpoll *ep = f->private_data; in ep_show_fdinfo() local
1362 mutex_lock(&ep->mtx); in ep_show_fdinfo()
1363 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_show_fdinfo()
1376 mutex_unlock(&ep->mtx); in ep_show_fdinfo()
1399 struct eventpoll *ep; in eventpoll_release_file() local
1415 * ep access is safe as we still own a reference to the ep in eventpoll_release_file()
1418 ep = epi->ep; in eventpoll_release_file()
1419 mutex_lock(&ep->mtx); in eventpoll_release_file()
1421 ep_unregister_pollwait(ep, epi); in eventpoll_release_file()
1423 ep_remove_file(ep, epi, file); in eventpoll_release_file()
1424 ep_remove_epi(ep, epi); in eventpoll_release_file()
1426 mutex_unlock(&ep->mtx); in eventpoll_release_file()
1428 if (ep_put(ep)) in eventpoll_release_file()
1429 ep_free(ep); in eventpoll_release_file()
1437 struct eventpoll *ep; in ep_alloc() local
1439 ep = kzalloc_obj(*ep); in ep_alloc()
1440 if (unlikely(!ep)) in ep_alloc()
1443 mutex_init(&ep->mtx); in ep_alloc()
1444 spin_lock_init(&ep->lock); in ep_alloc()
1445 seqcount_spinlock_init(&ep->seq, &ep->lock); in ep_alloc()
1446 init_waitqueue_head(&ep->wq); in ep_alloc()
1447 init_waitqueue_head(&ep->poll_wait); in ep_alloc()
1448 INIT_LIST_HEAD(&ep->rdllist); in ep_alloc()
1449 ep->rbr = RB_ROOT_CACHED; in ep_alloc()
1450 ep->ovflist = EP_UNACTIVE_PTR; /* not scanning */ in ep_alloc()
1451 ep->user = get_current_user(); in ep_alloc()
1452 refcount_set(&ep->refcount, 1); in ep_alloc()
1454 *pep = ep; in ep_alloc()
1464 static struct epitem *ep_find(struct eventpoll *ep, struct epoll_key *tf) in ep_find() argument
1470 for (rbp = ep->rbr.rb_root.rb_node; rbp; ) { in ep_find()
1495 struct eventpoll *ep = epi->ep; in ep_poll_callback() local
1500 spin_lock_irqsave(&ep->lock, flags); in ep_poll_callback()
1526 * chained in ep->ovflist and requeued later on. in ep_poll_callback()
1528 if (ep_is_scanning(ep)) { in ep_poll_callback()
1530 epi->ovflist_next = READ_ONCE(ep->ovflist); in ep_poll_callback()
1531 WRITE_ONCE(ep->ovflist, epi); in ep_poll_callback()
1536 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_poll_callback()
1544 if (waitqueue_active(&ep->wq)) { in ep_poll_callback()
1562 wake_up_sync(&ep->wq); in ep_poll_callback()
1564 wake_up(&ep->wq); in ep_poll_callback()
1566 if (waitqueue_active(&ep->poll_wait)) in ep_poll_callback()
1570 spin_unlock_irqrestore(&ep->lock, flags); in ep_poll_callback()
1574 ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); in ep_poll_callback()
1596 * protects ep / epi / wait. in ep_poll_callback()
1636 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1639 struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL; in ep_rbtree_insert()
1654 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1702 struct hlist_head *refs = &epi->ep->refs; in reverse_path_check_proc()
1744 if (!epi->ep->ws) { in ep_create_wakeup_source()
1745 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1746 if (!epi->ep->ws) in ep_create_wakeup_source()
1781 struct eventpoll *ep = NULL; in ep_attach_file() local
1784 ep = file->private_data; in ep_attach_file()
1786 if (ep) { in ep_attach_file()
1787 head = &ep->refs; in ep_attach_file()
1821 static struct epitem *ep_alloc_epitem(struct eventpoll *ep, in ep_alloc_epitem() argument
1827 if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, in ep_alloc_epitem()
1830 percpu_counter_inc(&ep->user->epoll_watches); in ep_alloc_epitem()
1834 percpu_counter_dec(&ep->user->epoll_watches); in ep_alloc_epitem()
1839 epi->ep = ep; in ep_alloc_epitem()
1848 * Install @epi into its target file's f_ep hlist and into @ep's rbtree,
1849 * taking one additional reference on @ep for the lifetime of the item.
1853 * with the target side. RB tree ops are protected by @ep->mtx, which
1859 * unwind; that cannot drop @ep's refcount to zero because the ep file
1862 static int ep_register_epitem(struct ep_ctl_ctx *ctx, struct eventpoll *ep, in ep_register_epitem() argument
1877 percpu_counter_dec(&ep->user->epoll_watches); in ep_register_epitem()
1884 ep_rbtree_insert(ep, epi); in ep_register_epitem()
1889 ep_get(ep); in ep_register_epitem()
1896 static int ep_insert(struct ep_ctl_ctx *ctx, struct eventpoll *ep, in ep_insert() argument
1911 epi = ep_alloc_epitem(ep, event, tf); in ep_insert()
1915 error = ep_register_epitem(ctx, ep, epi, tep, full_check); in ep_insert()
1921 ep_remove(ep, epi); in ep_insert()
1928 ep_remove(ep, epi); in ep_insert()
1948 ep_remove(ep, epi); in ep_insert()
1953 spin_lock_irq(&ep->lock); in ep_insert()
1958 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1961 if (waitqueue_active(&ep->wq)) in ep_insert()
1962 wake_up(&ep->wq); in ep_insert()
1963 if (waitqueue_active(&ep->poll_wait)) in ep_insert()
1967 spin_unlock_irq(&ep->lock); in ep_insert()
1971 ep_poll_safewake(ep, NULL, 0); in ep_insert()
1980 static int ep_modify(struct eventpoll *ep, struct epitem *epi, in ep_modify() argument
2010 * We need this because we did not take ep->lock while in ep_modify()
2012 * ep->lock). in ep_modify()
2031 spin_lock_irq(&ep->lock); in ep_modify()
2033 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
2037 if (waitqueue_active(&ep->wq)) in ep_modify()
2038 wake_up(&ep->wq); in ep_modify()
2039 if (waitqueue_active(&ep->poll_wait)) in ep_modify()
2042 spin_unlock_irq(&ep->lock); in ep_modify()
2047 ep_poll_safewake(ep, NULL, 0); in ep_modify()
2063 * Caller holds ep->mtx and the scan is active.
2065 static int ep_deliver_event(struct eventpoll *ep, struct epitem *epi, in ep_deliver_event() argument
2075 * Activate ep->ws before deactivating epi->ws to prevent in ep_deliver_event()
2083 __pm_stay_awake(ep->ws); in ep_deliver_event()
2090 * Re-poll under ep->mtx so userspace cannot change the item in ep_deliver_event()
2119 * by ep->mtx, and the poll callback queues to ovflist in ep_deliver_event()
2122 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_deliver_event()
2128 static int ep_send_events(struct eventpoll *ep, in ep_send_events() argument
2146 mutex_lock(&ep->mtx); in ep_send_events()
2147 ep_start_scan(ep, &scan_batch); in ep_send_events()
2151 * scan_batch; items cannot vanish while we hold ep->mtx. in ep_send_events()
2159 delivered = ep_deliver_event(ep, epi, &pt, &events, &scan_batch); in ep_send_events()
2168 ep_done_scan(ep, &scan_batch); in ep_send_events()
2169 mutex_unlock(&ep->mtx); in ep_send_events()
2214 static int ep_try_send_events(struct eventpoll *ep, in ep_try_send_events() argument
2224 res = ep_send_events(ep, events, maxevents); in ep_try_send_events()
2226 ep_suspend_napi_irqs(ep); in ep_try_send_events()
2242 * @ep: Pointer to the eventpoll context.
2255 static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, in ep_poll() argument
2285 eavail = ep_events_available(ep); in ep_poll()
2289 res = ep_try_send_events(ep, events, maxevents); in ep_poll()
2297 eavail = ep_busy_loop(ep); in ep_poll()
2312 * explicitly, thus ep->lock is not taken, which halts the in ep_poll()
2324 spin_lock_irq(&ep->lock); in ep_poll()
2339 eavail = ep_events_available(ep); in ep_poll()
2341 __add_wait_queue_exclusive(&ep->wq, &wait); in ep_poll()
2343 spin_unlock_irq(&ep->lock); in ep_poll()
2362 spin_lock_irq(&ep->lock); in ep_poll()
2372 __remove_wait_queue(&ep->wq, &wait); in ep_poll()
2373 spin_unlock_irq(&ep->lock); in ep_poll()
2379 * ep_loop_check_proc - verify that adding an epoll file @ep inside another
2381 * determine the depth of the subtree starting at @ep
2384 * @ep: the &struct eventpoll to be currently checked.
2391 struct eventpoll *ep, int depth) in ep_loop_check_proc() argument
2397 if (ep->gen == loop_check_gen) in ep_loop_check_proc()
2398 return ep->loop_check_depth; in ep_loop_check_proc()
2400 mutex_lock_nested(&ep->mtx, depth + 1); in ep_loop_check_proc()
2401 ep->gen = loop_check_gen; in ep_loop_check_proc()
2402 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_loop_check_proc()
2426 ep->loop_check_depth = result; in ep_loop_check_proc()
2427 mutex_unlock(&ep->mtx); in ep_loop_check_proc()
2432 /* ep_get_upwards_depth_proc - determine depth of @ep when traversed upwards */
2433 static int ep_get_upwards_depth_proc(struct eventpoll *ep, int depth) in ep_get_upwards_depth_proc() argument
2438 if (ep->gen == loop_check_gen) in ep_get_upwards_depth_proc()
2439 return ep->loop_check_depth; in ep_get_upwards_depth_proc()
2440 hlist_for_each_entry_rcu(epi, &ep->refs, fllink) in ep_get_upwards_depth_proc()
2441 result = max(result, ep_get_upwards_depth_proc(epi->ep, depth + 1) + 1); in ep_get_upwards_depth_proc()
2442 ep->gen = loop_check_gen; in ep_get_upwards_depth_proc()
2443 ep->loop_check_depth = result; in ep_get_upwards_depth_proc()
2449 * into another epoll file (represented by @ep) does not create
2453 * @ep: Pointer to the epoll we are inserting into.
2459 static int ep_loop_check(struct ep_ctl_ctx *ctx, struct eventpoll *ep, in ep_loop_check() argument
2464 ctx->inserting_into = ep; in ep_loop_check()
2467 * to loop up to @ep. in ep_loop_check()
2472 /* Check how far up we can go from @ep. */ in ep_loop_check()
2474 upwards_depth = ep_get_upwards_depth_proc(ep, 0); in ep_loop_check()
2497 struct eventpoll *ep; in do_epoll_create() local
2507 error = ep_alloc(&ep); in do_epoll_create()
2515 anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep, in do_epoll_create()
2518 ep_clear_and_put(ep); in do_epoll_create()
2521 ep->file = fd_prepare_file(fdf); in do_epoll_create()
2561 * Acquire the locks required for do_epoll_ctl() on @ep for @op.
2563 * Always takes ep->mtx. For EPOLL_CTL_ADD, additionally runs the
2565 * change: @ep is already watched (epfile->f_ep non-NULL), @ep was
2566 * recently loop-checked (ep->gen == loop_check_gen), or @tfile is
2571 * 0 success; ep->mtx held.
2572 * 1 success; ep->mtx held AND the full check ran under
2577 * The caller releases what was taken with ep_ctl_unlock(ep, ret).
2583 static int ep_ctl_lock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, int op, in ep_ctl_lock() argument
2589 error = epoll_mutex_lock(&ep->mtx, nonblock); in ep_ctl_lock()
2595 if (!READ_ONCE(epfile->f_ep) && ep->gen != loop_check_gen && in ep_ctl_lock()
2599 /* Full check needed: drop ep->mtx so we can take epnested_mutex. */ in ep_ctl_lock()
2600 mutex_unlock(&ep->mtx); in ep_ctl_lock()
2609 if (ep_loop_check(ctx, ep, tep) != 0) { in ep_ctl_lock()
2615 error = epoll_mutex_lock(&ep->mtx, nonblock); in ep_ctl_lock()
2628 static void ep_ctl_unlock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, in ep_ctl_unlock() argument
2631 mutex_unlock(&ep->mtx); in ep_ctl_unlock()
2644 struct eventpoll *ep; in do_epoll_ctl_file() local
2678 ep = f->private_data; in do_epoll_ctl_file()
2680 full_check = ep_ctl_lock(&ctx, ep, op, f, tf->file, nonblock); in do_epoll_ctl_file()
2685 * Look the target up in ep's RB tree. We hold ep->mtx, so the in do_epoll_ctl_file()
2688 epi = ep_find(ep, tf); in do_epoll_ctl_file()
2695 error = ep_insert(&ctx, ep, epds, tf, full_check); in do_epoll_ctl_file()
2705 ep_remove(ep, epi); in do_epoll_ctl_file()
2715 error = ep_modify(ep, epi, epds); in do_epoll_ctl_file()
2722 ep_ctl_unlock(&ctx, ep, full_check); in do_epoll_ctl_file()
2786 struct eventpoll *ep; in epoll_sendevents() local
2793 ep = file->private_data; in epoll_sendevents()
2798 if (ep_events_available(ep)) in epoll_sendevents()
2799 return ep_try_send_events(ep, events, maxevents); in epoll_sendevents()
2810 struct eventpoll *ep; in do_epoll_wait() local
2826 ep = fd_file(f)->private_data; in do_epoll_wait()
2829 return ep_poll(ep, events, maxevents, to); in do_epoll_wait()
2896 static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff) in ep_find_tfd() argument
2901 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_find_tfd()
2919 struct eventpoll *ep; in get_epoll_tfile_raw_ptr() local
2925 ep = file->private_data; in get_epoll_tfile_raw_ptr()
2927 mutex_lock(&ep->mtx); in get_epoll_tfile_raw_ptr()
2928 epi = ep_find_tfd(ep, tfd, toff); in get_epoll_tfile_raw_ptr()
2933 mutex_unlock(&ep->mtx); in get_epoll_tfile_raw_ptr()