| /linux/kernel/locking/ |
| H A D | ww_mutex.h | 313 struct ww_acquire_ctx *ww_ctx, struct wake_q_head *wake_q) in __ww_mutex_die() argument
328 wake_q_add(wake_q, waiter->task); in __ww_mutex_die()
344 struct wake_q_head *wake_q) in __ww_mutex_wound() argument
387 wake_q_add(wake_q, owner); in __ww_mutex_wound()
409 struct wake_q_head *wake_q) in __ww_mutex_check_waiters() argument
422 if (__ww_mutex_die(lock, cur, ww_ctx, wake_q) || in __ww_mutex_check_waiters()
423 __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx, wake_q)) in __ww_mutex_check_waiters()
435 DEFINE_WAKE_Q(wake_q); in ww_mutex_set_context_fastpath()
468 __ww_mutex_check_waiters(&lock->base, ctx, &wake_q); in ww_mutex_set_context_fastpath()
471 wake_up_q(&wake_q); in ww_mutex_set_context_fastpath()
556 __ww_mutex_add_waiter(struct MUTEX_WAITER * waiter,struct MUTEX * lock,struct ww_acquire_ctx * ww_ctx,struct wake_q_head * wake_q) __ww_mutex_add_waiter() argument
[all...] |
| H A D | rtmutex.c | 24 #include <linux/sched/wake_q.h>
39 struct wake_q_head *wake_q) in __ww_mutex_add_waiter() argument
46 struct wake_q_head *wake_q) in __ww_mutex_check_waiters() argument
550 /* RT mutex specific wake_q wrappers */ in rt_mutex_wake_q_add_task()
1217 struct wake_q_head *wake_q) in task_blocks_on_rt_mutex()
1263 res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q); in task_blocks_on_rt_mutex()
1307 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in task_blocks_on_rt_mutex()
1621 * @wake_q: wake_q of tasks to wake when we drop the lock->wait_lock in rt_mutex_slowlock_block()
1630 struct wake_q_head *wake_q) in rt_mutex_slowlock_block()
1214 task_blocks_on_rt_mutex(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter,struct task_struct * task,struct ww_acquire_ctx * ww_ctx,enum rtmutex_chainwalk chwalk,struct wake_q_head * wake_q) task_blocks_on_rt_mutex() argument
1625 rt_mutex_slowlock_block(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state,struct hrtimer_sleeper * timeout,struct rt_mutex_waiter * waiter,struct wake_q_head * wake_q) rt_mutex_slowlock_block() argument
1715 __rt_mutex_slowlock(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state,enum rtmutex_chainwalk chwalk,struct rt_mutex_waiter * waiter,struct wake_q_head * wake_q) __rt_mutex_slowlock() argument
1773 __rt_mutex_slowlock_locked(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state,struct wake_q_head * wake_q) __rt_mutex_slowlock_locked() argument
1852 rtlock_slowlock_locked(struct rt_mutex_base * lock,struct wake_q_head * wake_q) rtlock_slowlock_locked() argument
[all...] |
| H A D | mutex.c | 25 #include <linux/sched/wake_q.h>
614 DEFINE_WAKE_Q(wake_q); in __mutex_lock_common()
666 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); in __mutex_lock_common()
686 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q); in __mutex_lock_common()
724 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
783 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); in __mutex_lock_common()
798 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
809 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in __mutex_lock_common()
|
| H A D | rtmutex_api.c | 320 * @wake_q: the wake_q to wake tasks after we release the wait_lock in __rt_mutex_start_proxy_lock()
338 struct wake_q_head *wake_q) in __rt_mutex_start_proxy_lock()
350 RT_MUTEX_FULL_CHAINWALK, wake_q);
389 DEFINE_WAKE_Q(wake_q);
392 ret = __rt_mutex_start_proxy_lock(lock, waiter, task, &wake_q);
397 wake_up_q(&wake_q); in rt_mutex_wait_proxy_lock()
314 __rt_mutex_start_proxy_lock(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter,struct task_struct * task,struct wake_q_head * wake_q) __rt_mutex_start_proxy_lock() argument
|
| /linux/ipc/ |
| H A D | sem.c | 87 #include <linux/sched/wake_q.h>
237 * The wakeup is handled using the wake_q infrastructure. wake_q wakeups may
787 struct wake_q_head *wake_q) in wake_up_sem_queue_prepare() argument
796 wake_q_add_safe(wake_q, sleeper); in wake_up_sem_queue_prepare()
844 * @wake_q: lockless wake-queue head.
850 * The tasks that must be woken up are added to @wake_q. The return code
855 struct wake_q_head *wake_q) in wake_const_ops() argument
874 wake_up_sem_queue_prepare(q, error, wake_q); in wake_const_ops()
887 * @wake_q
894 do_smart_wakeup_zero(struct sem_array * sma,struct sembuf * sops,int nsops,struct wake_q_head * wake_q) do_smart_wakeup_zero() argument
949 update_queue(struct sem_array * sma,int semnum,struct wake_q_head * wake_q) update_queue() argument
1030 do_smart_update(struct sem_array * sma,struct sembuf * sops,int nsops,int otime,struct wake_q_head * wake_q) do_smart_update() argument
[all...] |
| H A D | msg.c | 213 struct wake_q_head *wake_q, bool kill) in ss_wakeup() argument
245 wake_q_add(wake_q, mss->tsk); in ss_wakeup()
250 struct wake_q_head *wake_q) in expunge_all() argument
261 wake_q_add_safe(wake_q, r_tsk); in expunge_all()
279 DEFINE_WAKE_Q(wake_q); in freeque()
281 expunge_all(msq, -EIDRM, &wake_q); in freeque()
282 ss_wakeup(msq, &wake_q, true); in freeque()
285 wake_up_q(&wake_q); in freeque()
432 DEFINE_WAKE_Q(wake_q); in msgctl_down()
452 expunge_all(msq, -EAGAIN, &wake_q); in msgctl_down()
[all …]
|
| H A D | mqueue.c | 993 static inline void __pipelined_op(struct wake_q_head *wake_q, in __pipelined_op() argument
1004 wake_q_add_safe(wake_q, task); in __pipelined_op()
1010 static inline void pipelined_send(struct wake_q_head *wake_q, in pipelined_send() argument
1016 __pipelined_op(wake_q, info, receiver); in pipelined_send()
1021 static inline void pipelined_receive(struct wake_q_head *wake_q, in pipelined_receive() argument
1034 __pipelined_op(wake_q, info, sender); in pipelined_receive()
1049 DEFINE_WAKE_Q(wake_q); in do_mq_timedsend()
1123 pipelined_send(&wake_q, info, msg_ptr, receiver); in do_mq_timedsend()
1135 wake_up_q(&wake_q); in do_mq_timedsend()
1209 DEFINE_WAKE_Q(wake_q); in do_mq_timedreceive()
[all …]
|
| /linux/kernel/futex/ |
| H A D | requeue.c | 390 DEFINE_WAKE_Q(wake_q); in futex_requeue()
615 this->wake(&wake_q, this); in futex_requeue()
703 wake_up_q(&wake_q); in futex_requeue()
|
| H A D | pi.c | 929 DEFINE_WAKE_Q(wake_q); in futex_lock_pi()
1042 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q); in futex_lock_pi()
1043 raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q); in futex_lock_pi()
|
| /linux/io_uring/ |
| H A D | futex.c | 157 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) in io_futex_wakev_fn() argument
210 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) in io_futex_wake_fn() argument
|
| /linux/include/linux/ |
| H A D | sched.h | 1241 struct wake_q_node wake_q;
1233 struct wake_q_node wake_q; global() member
|
| /linux/kernel/sched/ |
| H A D | core.c | 64 #include <linux/sched/wake_q.h>
1093 struct wake_q_node *node = &task->wake_q; in __wake_q_add()
1096 * Atomically grab the task, if ->wake_q is !nil already it means in __wake_q_add()
1147 * wake_q to do the right thing depending whether or not the @task is already in wake_up_q()
1163 task = container_of(node, struct task_struct, wake_q);
1166 WRITE_ONCE(task->wake_q.next, NULL);
|
| /linux/kernel/ |
| H A D | fork.c | 979 tsk->wake_q.next = NULL; in dup_task_struct()
|