11696a8beSPeter Zijlstra /* 21696a8beSPeter Zijlstra * RT-Mutexes: simple blocking mutual exclusion locks with PI support 31696a8beSPeter Zijlstra * 41696a8beSPeter Zijlstra * started by Ingo Molnar and Thomas Gleixner. 51696a8beSPeter Zijlstra * 61696a8beSPeter Zijlstra * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 71696a8beSPeter Zijlstra * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com> 81696a8beSPeter Zijlstra * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt 91696a8beSPeter Zijlstra * Copyright (C) 2006 Esben Nielsen 101696a8beSPeter Zijlstra * 11214e0aedSDavidlohr Bueso * See Documentation/locking/rt-mutex-design.txt for details. 121696a8beSPeter Zijlstra */ 131696a8beSPeter Zijlstra #include <linux/spinlock.h> 141696a8beSPeter Zijlstra #include <linux/export.h> 15174cd4b1SIngo Molnar #include <linux/sched/signal.h> 161696a8beSPeter Zijlstra #include <linux/sched/rt.h> 17fb00aca4SPeter Zijlstra #include <linux/sched/deadline.h> 1884f001e1SIngo Molnar #include <linux/sched/wake_q.h> 19b17b0153SIngo Molnar #include <linux/sched/debug.h> 201696a8beSPeter Zijlstra #include <linux/timer.h> 211696a8beSPeter Zijlstra 221696a8beSPeter Zijlstra #include "rtmutex_common.h" 231696a8beSPeter Zijlstra 241696a8beSPeter Zijlstra /* 251696a8beSPeter Zijlstra * lock->owner state tracking: 261696a8beSPeter Zijlstra * 271696a8beSPeter Zijlstra * lock->owner holds the task_struct pointer of the owner. Bit 0 281696a8beSPeter Zijlstra * is used to keep track of the "lock has waiters" state. 291696a8beSPeter Zijlstra * 301696a8beSPeter Zijlstra * owner bit0 311696a8beSPeter Zijlstra * NULL 0 lock is free (fast acquire possible) 321696a8beSPeter Zijlstra * NULL 1 lock is free and has waiters and the top waiter 331696a8beSPeter Zijlstra * is going to take the lock* 341696a8beSPeter Zijlstra * taskpointer 0 lock is held (fast release possible) 351696a8beSPeter Zijlstra * taskpointer 1 lock is held and has waiters** 361696a8beSPeter Zijlstra * 371696a8beSPeter Zijlstra * The fast atomic compare exchange based acquire and release is only 381696a8beSPeter Zijlstra * possible when bit 0 of lock->owner is 0. 391696a8beSPeter Zijlstra * 401696a8beSPeter Zijlstra * (*) It also can be a transitional state when grabbing the lock 411696a8beSPeter Zijlstra * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock, 421696a8beSPeter Zijlstra * we need to set the bit0 before looking at the lock, and the owner may be 431696a8beSPeter Zijlstra * NULL in this small time, hence this can be a transitional state. 441696a8beSPeter Zijlstra * 451696a8beSPeter Zijlstra * (**) There is a small time when bit 0 is set but there are no 461696a8beSPeter Zijlstra * waiters. This can happen when grabbing the lock in the slow path. 471696a8beSPeter Zijlstra * To prevent a cmpxchg of the owner releasing the lock, we need to 481696a8beSPeter Zijlstra * set this bit before looking at the lock. 491696a8beSPeter Zijlstra */ 501696a8beSPeter Zijlstra 511696a8beSPeter Zijlstra static void 521696a8beSPeter Zijlstra rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) 531696a8beSPeter Zijlstra { 541696a8beSPeter Zijlstra unsigned long val = (unsigned long)owner; 551696a8beSPeter Zijlstra 561696a8beSPeter Zijlstra if (rt_mutex_has_waiters(lock)) 571696a8beSPeter Zijlstra val |= RT_MUTEX_HAS_WAITERS; 581696a8beSPeter Zijlstra 591696a8beSPeter Zijlstra lock->owner = (struct task_struct *)val; 601696a8beSPeter Zijlstra } 611696a8beSPeter Zijlstra 621696a8beSPeter Zijlstra static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) 631696a8beSPeter Zijlstra { 641696a8beSPeter Zijlstra lock->owner = (struct task_struct *) 651696a8beSPeter Zijlstra ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); 661696a8beSPeter Zijlstra } 671696a8beSPeter Zijlstra 681696a8beSPeter Zijlstra static void fixup_rt_mutex_waiters(struct rt_mutex *lock) 691696a8beSPeter Zijlstra { 70dbb26055SThomas Gleixner unsigned long owner, *p = (unsigned long *) &lock->owner; 71dbb26055SThomas Gleixner 72dbb26055SThomas Gleixner if (rt_mutex_has_waiters(lock)) 73dbb26055SThomas Gleixner return; 74dbb26055SThomas Gleixner 75dbb26055SThomas Gleixner /* 76dbb26055SThomas Gleixner * The rbtree has no waiters enqueued, now make sure that the 77dbb26055SThomas Gleixner * lock->owner still has the waiters bit set, otherwise the 78dbb26055SThomas Gleixner * following can happen: 79dbb26055SThomas Gleixner * 80dbb26055SThomas Gleixner * CPU 0 CPU 1 CPU2 81dbb26055SThomas Gleixner * l->owner=T1 82dbb26055SThomas Gleixner * rt_mutex_lock(l) 83dbb26055SThomas Gleixner * lock(l->lock) 84dbb26055SThomas Gleixner * l->owner = T1 | HAS_WAITERS; 85dbb26055SThomas Gleixner * enqueue(T2) 86dbb26055SThomas Gleixner * boost() 87dbb26055SThomas Gleixner * unlock(l->lock) 88dbb26055SThomas Gleixner * block() 89dbb26055SThomas Gleixner * 90dbb26055SThomas Gleixner * rt_mutex_lock(l) 91dbb26055SThomas Gleixner * lock(l->lock) 92dbb26055SThomas Gleixner * l->owner = T1 | HAS_WAITERS; 93dbb26055SThomas Gleixner * enqueue(T3) 94dbb26055SThomas Gleixner * boost() 95dbb26055SThomas Gleixner * unlock(l->lock) 96dbb26055SThomas Gleixner * block() 97dbb26055SThomas Gleixner * signal(->T2) signal(->T3) 98dbb26055SThomas Gleixner * lock(l->lock) 99dbb26055SThomas Gleixner * dequeue(T2) 100dbb26055SThomas Gleixner * deboost() 101dbb26055SThomas Gleixner * unlock(l->lock) 102dbb26055SThomas Gleixner * lock(l->lock) 103dbb26055SThomas Gleixner * dequeue(T3) 104dbb26055SThomas Gleixner * ==> wait list is empty 105dbb26055SThomas Gleixner * deboost() 106dbb26055SThomas Gleixner * unlock(l->lock) 107dbb26055SThomas Gleixner * lock(l->lock) 108dbb26055SThomas Gleixner * fixup_rt_mutex_waiters() 109dbb26055SThomas Gleixner * if (wait_list_empty(l) { 110dbb26055SThomas Gleixner * l->owner = owner 111dbb26055SThomas Gleixner * owner = l->owner & ~HAS_WAITERS; 112dbb26055SThomas Gleixner * ==> l->owner = T1 113dbb26055SThomas Gleixner * } 114dbb26055SThomas Gleixner * lock(l->lock) 115dbb26055SThomas Gleixner * rt_mutex_unlock(l) fixup_rt_mutex_waiters() 116dbb26055SThomas Gleixner * if (wait_list_empty(l) { 117dbb26055SThomas Gleixner * owner = l->owner & ~HAS_WAITERS; 118dbb26055SThomas Gleixner * cmpxchg(l->owner, T1, NULL) 119dbb26055SThomas Gleixner * ===> Success (l->owner = NULL) 120dbb26055SThomas Gleixner * 121dbb26055SThomas Gleixner * l->owner = owner 122dbb26055SThomas Gleixner * ==> l->owner = T1 123dbb26055SThomas Gleixner * } 124dbb26055SThomas Gleixner * 125dbb26055SThomas Gleixner * With the check for the waiter bit in place T3 on CPU2 will not 126dbb26055SThomas Gleixner * overwrite. All tasks fiddling with the waiters bit are 127dbb26055SThomas Gleixner * serialized by l->lock, so nothing else can modify the waiters 128dbb26055SThomas Gleixner * bit. If the bit is set then nothing can change l->owner either 129dbb26055SThomas Gleixner * so the simple RMW is safe. The cmpxchg() will simply fail if it 130dbb26055SThomas Gleixner * happens in the middle of the RMW because the waiters bit is 131dbb26055SThomas Gleixner * still set. 132dbb26055SThomas Gleixner */ 133dbb26055SThomas Gleixner owner = READ_ONCE(*p); 134dbb26055SThomas Gleixner if (owner & RT_MUTEX_HAS_WAITERS) 135dbb26055SThomas Gleixner WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS); 1361696a8beSPeter Zijlstra } 1371696a8beSPeter Zijlstra 1381696a8beSPeter Zijlstra /* 139cede8841SSebastian Andrzej Siewior * We can speed up the acquire/release, if there's no debugging state to be 140cede8841SSebastian Andrzej Siewior * set up. 1411696a8beSPeter Zijlstra */ 142cede8841SSebastian Andrzej Siewior #ifndef CONFIG_DEBUG_RT_MUTEXES 143700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_relaxed(l,c,n) (cmpxchg_relaxed(&l->owner, c, n) == c) 144700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_acquire(l,c,n) (cmpxchg_acquire(&l->owner, c, n) == c) 145700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_release(l,c,n) (cmpxchg_release(&l->owner, c, n) == c) 146700318d1SDavidlohr Bueso 147700318d1SDavidlohr Bueso /* 148700318d1SDavidlohr Bueso * Callers must hold the ->wait_lock -- which is the whole purpose as we force 149700318d1SDavidlohr Bueso * all future threads that attempt to [Rmw] the lock to the slowpath. As such 150700318d1SDavidlohr Bueso * relaxed semantics suffice. 151700318d1SDavidlohr Bueso */ 1521696a8beSPeter Zijlstra static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) 1531696a8beSPeter Zijlstra { 1541696a8beSPeter Zijlstra unsigned long owner, *p = (unsigned long *) &lock->owner; 1551696a8beSPeter Zijlstra 1561696a8beSPeter Zijlstra do { 1571696a8beSPeter Zijlstra owner = *p; 158700318d1SDavidlohr Bueso } while (cmpxchg_relaxed(p, owner, 159700318d1SDavidlohr Bueso owner | RT_MUTEX_HAS_WAITERS) != owner); 1601696a8beSPeter Zijlstra } 16127e35715SThomas Gleixner 16227e35715SThomas Gleixner /* 16327e35715SThomas Gleixner * Safe fastpath aware unlock: 16427e35715SThomas Gleixner * 1) Clear the waiters bit 16527e35715SThomas Gleixner * 2) Drop lock->wait_lock 16627e35715SThomas Gleixner * 3) Try to unlock the lock with cmpxchg 16727e35715SThomas Gleixner */ 168b4abf910SThomas Gleixner static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, 169b4abf910SThomas Gleixner unsigned long flags) 17027e35715SThomas Gleixner __releases(lock->wait_lock) 17127e35715SThomas Gleixner { 17227e35715SThomas Gleixner struct task_struct *owner = rt_mutex_owner(lock); 17327e35715SThomas Gleixner 17427e35715SThomas Gleixner clear_rt_mutex_waiters(lock); 175b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 17627e35715SThomas Gleixner /* 17727e35715SThomas Gleixner * If a new waiter comes in between the unlock and the cmpxchg 17827e35715SThomas Gleixner * we have two situations: 17927e35715SThomas Gleixner * 18027e35715SThomas Gleixner * unlock(wait_lock); 18127e35715SThomas Gleixner * lock(wait_lock); 18227e35715SThomas Gleixner * cmpxchg(p, owner, 0) == owner 18327e35715SThomas Gleixner * mark_rt_mutex_waiters(lock); 18427e35715SThomas Gleixner * acquire(lock); 18527e35715SThomas Gleixner * or: 18627e35715SThomas Gleixner * 18727e35715SThomas Gleixner * unlock(wait_lock); 18827e35715SThomas Gleixner * lock(wait_lock); 18927e35715SThomas Gleixner * mark_rt_mutex_waiters(lock); 19027e35715SThomas Gleixner * 19127e35715SThomas Gleixner * cmpxchg(p, owner, 0) != owner 19227e35715SThomas Gleixner * enqueue_waiter(); 19327e35715SThomas Gleixner * unlock(wait_lock); 19427e35715SThomas Gleixner * lock(wait_lock); 19527e35715SThomas Gleixner * wake waiter(); 19627e35715SThomas Gleixner * unlock(wait_lock); 19727e35715SThomas Gleixner * lock(wait_lock); 19827e35715SThomas Gleixner * acquire(lock); 19927e35715SThomas Gleixner */ 200700318d1SDavidlohr Bueso return rt_mutex_cmpxchg_release(lock, owner, NULL); 20127e35715SThomas Gleixner } 20227e35715SThomas Gleixner 2031696a8beSPeter Zijlstra #else 204700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_relaxed(l,c,n) (0) 205700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_acquire(l,c,n) (0) 206700318d1SDavidlohr Bueso # define rt_mutex_cmpxchg_release(l,c,n) (0) 207700318d1SDavidlohr Bueso 2081696a8beSPeter Zijlstra static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) 2091696a8beSPeter Zijlstra { 2101696a8beSPeter Zijlstra lock->owner = (struct task_struct *) 2111696a8beSPeter Zijlstra ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); 2121696a8beSPeter Zijlstra } 21327e35715SThomas Gleixner 21427e35715SThomas Gleixner /* 21527e35715SThomas Gleixner * Simple slow path only version: lock->owner is protected by lock->wait_lock. 21627e35715SThomas Gleixner */ 217b4abf910SThomas Gleixner static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, 218b4abf910SThomas Gleixner unsigned long flags) 21927e35715SThomas Gleixner __releases(lock->wait_lock) 22027e35715SThomas Gleixner { 22127e35715SThomas Gleixner lock->owner = NULL; 222b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 22327e35715SThomas Gleixner return true; 22427e35715SThomas Gleixner } 2251696a8beSPeter Zijlstra #endif 2261696a8beSPeter Zijlstra 227fb00aca4SPeter Zijlstra static inline int 228fb00aca4SPeter Zijlstra rt_mutex_waiter_less(struct rt_mutex_waiter *left, 229fb00aca4SPeter Zijlstra struct rt_mutex_waiter *right) 230fb00aca4SPeter Zijlstra { 2312d3d891dSDario Faggioli if (left->prio < right->prio) 232fb00aca4SPeter Zijlstra return 1; 233fb00aca4SPeter Zijlstra 2341696a8beSPeter Zijlstra /* 2352d3d891dSDario Faggioli * If both waiters have dl_prio(), we check the deadlines of the 2362d3d891dSDario Faggioli * associated tasks. 2372d3d891dSDario Faggioli * If left waiter has a dl_prio(), and we didn't return 1 above, 2382d3d891dSDario Faggioli * then right waiter has a dl_prio() too. 239fb00aca4SPeter Zijlstra */ 2402d3d891dSDario Faggioli if (dl_prio(left->prio)) 241*e0aad5b4SPeter Zijlstra return dl_time_before(left->deadline, right->deadline); 242fb00aca4SPeter Zijlstra 243fb00aca4SPeter Zijlstra return 0; 244fb00aca4SPeter Zijlstra } 245fb00aca4SPeter Zijlstra 246fb00aca4SPeter Zijlstra static void 247fb00aca4SPeter Zijlstra rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) 248fb00aca4SPeter Zijlstra { 249fb00aca4SPeter Zijlstra struct rb_node **link = &lock->waiters.rb_node; 250fb00aca4SPeter Zijlstra struct rb_node *parent = NULL; 251fb00aca4SPeter Zijlstra struct rt_mutex_waiter *entry; 252fb00aca4SPeter Zijlstra int leftmost = 1; 253fb00aca4SPeter Zijlstra 254fb00aca4SPeter Zijlstra while (*link) { 255fb00aca4SPeter Zijlstra parent = *link; 256fb00aca4SPeter Zijlstra entry = rb_entry(parent, struct rt_mutex_waiter, tree_entry); 257fb00aca4SPeter Zijlstra if (rt_mutex_waiter_less(waiter, entry)) { 258fb00aca4SPeter Zijlstra link = &parent->rb_left; 259fb00aca4SPeter Zijlstra } else { 260fb00aca4SPeter Zijlstra link = &parent->rb_right; 261fb00aca4SPeter Zijlstra leftmost = 0; 262fb00aca4SPeter Zijlstra } 263fb00aca4SPeter Zijlstra } 264fb00aca4SPeter Zijlstra 265fb00aca4SPeter Zijlstra if (leftmost) 266fb00aca4SPeter Zijlstra lock->waiters_leftmost = &waiter->tree_entry; 267fb00aca4SPeter Zijlstra 268fb00aca4SPeter Zijlstra rb_link_node(&waiter->tree_entry, parent, link); 269fb00aca4SPeter Zijlstra rb_insert_color(&waiter->tree_entry, &lock->waiters); 270fb00aca4SPeter Zijlstra } 271fb00aca4SPeter Zijlstra 272fb00aca4SPeter Zijlstra static void 273fb00aca4SPeter Zijlstra rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) 274fb00aca4SPeter Zijlstra { 275fb00aca4SPeter Zijlstra if (RB_EMPTY_NODE(&waiter->tree_entry)) 276fb00aca4SPeter Zijlstra return; 277fb00aca4SPeter Zijlstra 278fb00aca4SPeter Zijlstra if (lock->waiters_leftmost == &waiter->tree_entry) 279fb00aca4SPeter Zijlstra lock->waiters_leftmost = rb_next(&waiter->tree_entry); 280fb00aca4SPeter Zijlstra 281fb00aca4SPeter Zijlstra rb_erase(&waiter->tree_entry, &lock->waiters); 282fb00aca4SPeter Zijlstra RB_CLEAR_NODE(&waiter->tree_entry); 283fb00aca4SPeter Zijlstra } 284fb00aca4SPeter Zijlstra 285fb00aca4SPeter Zijlstra static void 286fb00aca4SPeter Zijlstra rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter) 287fb00aca4SPeter Zijlstra { 288fb00aca4SPeter Zijlstra struct rb_node **link = &task->pi_waiters.rb_node; 289fb00aca4SPeter Zijlstra struct rb_node *parent = NULL; 290fb00aca4SPeter Zijlstra struct rt_mutex_waiter *entry; 291fb00aca4SPeter Zijlstra int leftmost = 1; 292fb00aca4SPeter Zijlstra 293fb00aca4SPeter Zijlstra while (*link) { 294fb00aca4SPeter Zijlstra parent = *link; 295fb00aca4SPeter Zijlstra entry = rb_entry(parent, struct rt_mutex_waiter, pi_tree_entry); 296fb00aca4SPeter Zijlstra if (rt_mutex_waiter_less(waiter, entry)) { 297fb00aca4SPeter Zijlstra link = &parent->rb_left; 298fb00aca4SPeter Zijlstra } else { 299fb00aca4SPeter Zijlstra link = &parent->rb_right; 300fb00aca4SPeter Zijlstra leftmost = 0; 301fb00aca4SPeter Zijlstra } 302fb00aca4SPeter Zijlstra } 303fb00aca4SPeter Zijlstra 304fb00aca4SPeter Zijlstra if (leftmost) 305fb00aca4SPeter Zijlstra task->pi_waiters_leftmost = &waiter->pi_tree_entry; 306fb00aca4SPeter Zijlstra 307fb00aca4SPeter Zijlstra rb_link_node(&waiter->pi_tree_entry, parent, link); 308fb00aca4SPeter Zijlstra rb_insert_color(&waiter->pi_tree_entry, &task->pi_waiters); 309fb00aca4SPeter Zijlstra } 310fb00aca4SPeter Zijlstra 311fb00aca4SPeter Zijlstra static void 312fb00aca4SPeter Zijlstra rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter) 313fb00aca4SPeter Zijlstra { 314fb00aca4SPeter Zijlstra if (RB_EMPTY_NODE(&waiter->pi_tree_entry)) 315fb00aca4SPeter Zijlstra return; 316fb00aca4SPeter Zijlstra 317fb00aca4SPeter Zijlstra if (task->pi_waiters_leftmost == &waiter->pi_tree_entry) 318fb00aca4SPeter Zijlstra task->pi_waiters_leftmost = rb_next(&waiter->pi_tree_entry); 319fb00aca4SPeter Zijlstra 320fb00aca4SPeter Zijlstra rb_erase(&waiter->pi_tree_entry, &task->pi_waiters); 321fb00aca4SPeter Zijlstra RB_CLEAR_NODE(&waiter->pi_tree_entry); 322fb00aca4SPeter Zijlstra } 323fb00aca4SPeter Zijlstra 324acd58620SPeter Zijlstra static void rt_mutex_adjust_prio(struct task_struct *p) 325e96a7705SXunlei Pang { 326acd58620SPeter Zijlstra struct task_struct *pi_task = NULL; 327e96a7705SXunlei Pang 328acd58620SPeter Zijlstra lockdep_assert_held(&p->pi_lock); 329e96a7705SXunlei Pang 330acd58620SPeter Zijlstra if (task_has_pi_waiters(p)) 331acd58620SPeter Zijlstra pi_task = task_top_pi_waiter(p)->task; 3321696a8beSPeter Zijlstra 333acd58620SPeter Zijlstra rt_mutex_setprio(p, pi_task); 3341696a8beSPeter Zijlstra } 3351696a8beSPeter Zijlstra 3361696a8beSPeter Zijlstra /* 3378930ed80SThomas Gleixner * Deadlock detection is conditional: 3388930ed80SThomas Gleixner * 3398930ed80SThomas Gleixner * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted 3408930ed80SThomas Gleixner * if the detect argument is == RT_MUTEX_FULL_CHAINWALK. 3418930ed80SThomas Gleixner * 3428930ed80SThomas Gleixner * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always 3438930ed80SThomas Gleixner * conducted independent of the detect argument. 3448930ed80SThomas Gleixner * 3458930ed80SThomas Gleixner * If the waiter argument is NULL this indicates the deboost path and 3468930ed80SThomas Gleixner * deadlock detection is disabled independent of the detect argument 3478930ed80SThomas Gleixner * and the config settings. 3488930ed80SThomas Gleixner */ 3498930ed80SThomas Gleixner static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter, 3508930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk) 3518930ed80SThomas Gleixner { 3528930ed80SThomas Gleixner /* 3538930ed80SThomas Gleixner * This is just a wrapper function for the following call, 3548930ed80SThomas Gleixner * because debug_rt_mutex_detect_deadlock() smells like a magic 3558930ed80SThomas Gleixner * debug feature and I wanted to keep the cond function in the 3568930ed80SThomas Gleixner * main source file along with the comments instead of having 3578930ed80SThomas Gleixner * two of the same in the headers. 3588930ed80SThomas Gleixner */ 3598930ed80SThomas Gleixner return debug_rt_mutex_detect_deadlock(waiter, chwalk); 3608930ed80SThomas Gleixner } 3618930ed80SThomas Gleixner 3628930ed80SThomas Gleixner /* 3631696a8beSPeter Zijlstra * Max number of times we'll walk the boosting chain: 3641696a8beSPeter Zijlstra */ 3651696a8beSPeter Zijlstra int max_lock_depth = 1024; 3661696a8beSPeter Zijlstra 36782084984SThomas Gleixner static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) 36882084984SThomas Gleixner { 36982084984SThomas Gleixner return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; 37082084984SThomas Gleixner } 37182084984SThomas Gleixner 3721696a8beSPeter Zijlstra /* 3731696a8beSPeter Zijlstra * Adjust the priority chain. Also used for deadlock detection. 3741696a8beSPeter Zijlstra * Decreases task's usage by one - may thus free the task. 3751696a8beSPeter Zijlstra * 37682084984SThomas Gleixner * @task: the task owning the mutex (owner) for which a chain walk is 37782084984SThomas Gleixner * probably needed 378e6beaa36STom(JeHyeon) Yeon * @chwalk: do we have to carry out deadlock detection? 3791696a8beSPeter Zijlstra * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck 3801696a8beSPeter Zijlstra * things for a task that has just got its priority adjusted, and 3811696a8beSPeter Zijlstra * is waiting on a mutex) 38282084984SThomas Gleixner * @next_lock: the mutex on which the owner of @orig_lock was blocked before 38382084984SThomas Gleixner * we dropped its pi_lock. Is never dereferenced, only used for 38482084984SThomas Gleixner * comparison to detect lock chain changes. 3851696a8beSPeter Zijlstra * @orig_waiter: rt_mutex_waiter struct for the task that has just donated 3861696a8beSPeter Zijlstra * its priority to the mutex owner (can be NULL in the case 3871696a8beSPeter Zijlstra * depicted above or if the top waiter is gone away and we are 3881696a8beSPeter Zijlstra * actually deboosting the owner) 3891696a8beSPeter Zijlstra * @top_task: the current top waiter 3901696a8beSPeter Zijlstra * 3911696a8beSPeter Zijlstra * Returns 0 or -EDEADLK. 3923eb65aeaSThomas Gleixner * 3933eb65aeaSThomas Gleixner * Chain walk basics and protection scope 3943eb65aeaSThomas Gleixner * 3953eb65aeaSThomas Gleixner * [R] refcount on task 3963eb65aeaSThomas Gleixner * [P] task->pi_lock held 3973eb65aeaSThomas Gleixner * [L] rtmutex->wait_lock held 3983eb65aeaSThomas Gleixner * 3993eb65aeaSThomas Gleixner * Step Description Protected by 4003eb65aeaSThomas Gleixner * function arguments: 4013eb65aeaSThomas Gleixner * @task [R] 4023eb65aeaSThomas Gleixner * @orig_lock if != NULL @top_task is blocked on it 4033eb65aeaSThomas Gleixner * @next_lock Unprotected. Cannot be 4043eb65aeaSThomas Gleixner * dereferenced. Only used for 4053eb65aeaSThomas Gleixner * comparison. 4063eb65aeaSThomas Gleixner * @orig_waiter if != NULL @top_task is blocked on it 4073eb65aeaSThomas Gleixner * @top_task current, or in case of proxy 4083eb65aeaSThomas Gleixner * locking protected by calling 4093eb65aeaSThomas Gleixner * code 4103eb65aeaSThomas Gleixner * again: 4113eb65aeaSThomas Gleixner * loop_sanity_check(); 4123eb65aeaSThomas Gleixner * retry: 4133eb65aeaSThomas Gleixner * [1] lock(task->pi_lock); [R] acquire [P] 4143eb65aeaSThomas Gleixner * [2] waiter = task->pi_blocked_on; [P] 4153eb65aeaSThomas Gleixner * [3] check_exit_conditions_1(); [P] 4163eb65aeaSThomas Gleixner * [4] lock = waiter->lock; [P] 4173eb65aeaSThomas Gleixner * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L] 4183eb65aeaSThomas Gleixner * unlock(task->pi_lock); release [P] 4193eb65aeaSThomas Gleixner * goto retry; 4203eb65aeaSThomas Gleixner * } 4213eb65aeaSThomas Gleixner * [6] check_exit_conditions_2(); [P] + [L] 4223eb65aeaSThomas Gleixner * [7] requeue_lock_waiter(lock, waiter); [P] + [L] 4233eb65aeaSThomas Gleixner * [8] unlock(task->pi_lock); release [P] 4243eb65aeaSThomas Gleixner * put_task_struct(task); release [R] 4253eb65aeaSThomas Gleixner * [9] check_exit_conditions_3(); [L] 4263eb65aeaSThomas Gleixner * [10] task = owner(lock); [L] 4273eb65aeaSThomas Gleixner * get_task_struct(task); [L] acquire [R] 4283eb65aeaSThomas Gleixner * lock(task->pi_lock); [L] acquire [P] 4293eb65aeaSThomas Gleixner * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L] 4303eb65aeaSThomas Gleixner * [12] check_exit_conditions_4(); [P] + [L] 4313eb65aeaSThomas Gleixner * [13] unlock(task->pi_lock); release [P] 4323eb65aeaSThomas Gleixner * unlock(lock->wait_lock); release [L] 4333eb65aeaSThomas Gleixner * goto again; 4341696a8beSPeter Zijlstra */ 4351696a8beSPeter Zijlstra static int rt_mutex_adjust_prio_chain(struct task_struct *task, 4368930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk, 4371696a8beSPeter Zijlstra struct rt_mutex *orig_lock, 43882084984SThomas Gleixner struct rt_mutex *next_lock, 4391696a8beSPeter Zijlstra struct rt_mutex_waiter *orig_waiter, 4401696a8beSPeter Zijlstra struct task_struct *top_task) 4411696a8beSPeter Zijlstra { 4421696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter; 443a57594a1SThomas Gleixner struct rt_mutex_waiter *prerequeue_top_waiter; 4448930ed80SThomas Gleixner int ret = 0, depth = 0; 445a57594a1SThomas Gleixner struct rt_mutex *lock; 4468930ed80SThomas Gleixner bool detect_deadlock; 44767792e2cSThomas Gleixner bool requeue = true; 4481696a8beSPeter Zijlstra 4498930ed80SThomas Gleixner detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk); 4501696a8beSPeter Zijlstra 4511696a8beSPeter Zijlstra /* 4521696a8beSPeter Zijlstra * The (de)boosting is a step by step approach with a lot of 4531696a8beSPeter Zijlstra * pitfalls. We want this to be preemptible and we want hold a 4541696a8beSPeter Zijlstra * maximum of two locks per step. So we have to check 4551696a8beSPeter Zijlstra * carefully whether things change under us. 4561696a8beSPeter Zijlstra */ 4571696a8beSPeter Zijlstra again: 4583eb65aeaSThomas Gleixner /* 4593eb65aeaSThomas Gleixner * We limit the lock chain length for each invocation. 4603eb65aeaSThomas Gleixner */ 4611696a8beSPeter Zijlstra if (++depth > max_lock_depth) { 4621696a8beSPeter Zijlstra static int prev_max; 4631696a8beSPeter Zijlstra 4641696a8beSPeter Zijlstra /* 4651696a8beSPeter Zijlstra * Print this only once. If the admin changes the limit, 4661696a8beSPeter Zijlstra * print a new message when reaching the limit again. 4671696a8beSPeter Zijlstra */ 4681696a8beSPeter Zijlstra if (prev_max != max_lock_depth) { 4691696a8beSPeter Zijlstra prev_max = max_lock_depth; 4701696a8beSPeter Zijlstra printk(KERN_WARNING "Maximum lock depth %d reached " 4711696a8beSPeter Zijlstra "task: %s (%d)\n", max_lock_depth, 4721696a8beSPeter Zijlstra top_task->comm, task_pid_nr(top_task)); 4731696a8beSPeter Zijlstra } 4741696a8beSPeter Zijlstra put_task_struct(task); 4751696a8beSPeter Zijlstra 4763d5c9340SThomas Gleixner return -EDEADLK; 4771696a8beSPeter Zijlstra } 4783eb65aeaSThomas Gleixner 4793eb65aeaSThomas Gleixner /* 4803eb65aeaSThomas Gleixner * We are fully preemptible here and only hold the refcount on 4813eb65aeaSThomas Gleixner * @task. So everything can have changed under us since the 4823eb65aeaSThomas Gleixner * caller or our own code below (goto retry/again) dropped all 4833eb65aeaSThomas Gleixner * locks. 4843eb65aeaSThomas Gleixner */ 4851696a8beSPeter Zijlstra retry: 4861696a8beSPeter Zijlstra /* 4873eb65aeaSThomas Gleixner * [1] Task cannot go away as we did a get_task() before ! 4881696a8beSPeter Zijlstra */ 489b4abf910SThomas Gleixner raw_spin_lock_irq(&task->pi_lock); 4901696a8beSPeter Zijlstra 4913eb65aeaSThomas Gleixner /* 4923eb65aeaSThomas Gleixner * [2] Get the waiter on which @task is blocked on. 4933eb65aeaSThomas Gleixner */ 4941696a8beSPeter Zijlstra waiter = task->pi_blocked_on; 4953eb65aeaSThomas Gleixner 4963eb65aeaSThomas Gleixner /* 4973eb65aeaSThomas Gleixner * [3] check_exit_conditions_1() protected by task->pi_lock. 4983eb65aeaSThomas Gleixner */ 4993eb65aeaSThomas Gleixner 5001696a8beSPeter Zijlstra /* 5011696a8beSPeter Zijlstra * Check whether the end of the boosting chain has been 5021696a8beSPeter Zijlstra * reached or the state of the chain has changed while we 5031696a8beSPeter Zijlstra * dropped the locks. 5041696a8beSPeter Zijlstra */ 5051696a8beSPeter Zijlstra if (!waiter) 5061696a8beSPeter Zijlstra goto out_unlock_pi; 5071696a8beSPeter Zijlstra 5081696a8beSPeter Zijlstra /* 5091696a8beSPeter Zijlstra * Check the orig_waiter state. After we dropped the locks, 5101696a8beSPeter Zijlstra * the previous owner of the lock might have released the lock. 5111696a8beSPeter Zijlstra */ 5121696a8beSPeter Zijlstra if (orig_waiter && !rt_mutex_owner(orig_lock)) 5131696a8beSPeter Zijlstra goto out_unlock_pi; 5141696a8beSPeter Zijlstra 5151696a8beSPeter Zijlstra /* 51682084984SThomas Gleixner * We dropped all locks after taking a refcount on @task, so 51782084984SThomas Gleixner * the task might have moved on in the lock chain or even left 51882084984SThomas Gleixner * the chain completely and blocks now on an unrelated lock or 51982084984SThomas Gleixner * on @orig_lock. 52082084984SThomas Gleixner * 52182084984SThomas Gleixner * We stored the lock on which @task was blocked in @next_lock, 52282084984SThomas Gleixner * so we can detect the chain change. 52382084984SThomas Gleixner */ 52482084984SThomas Gleixner if (next_lock != waiter->lock) 52582084984SThomas Gleixner goto out_unlock_pi; 52682084984SThomas Gleixner 52782084984SThomas Gleixner /* 5281696a8beSPeter Zijlstra * Drop out, when the task has no waiters. Note, 5291696a8beSPeter Zijlstra * top_waiter can be NULL, when we are in the deboosting 5301696a8beSPeter Zijlstra * mode! 5311696a8beSPeter Zijlstra */ 532397335f0SThomas Gleixner if (top_waiter) { 533397335f0SThomas Gleixner if (!task_has_pi_waiters(task)) 5341696a8beSPeter Zijlstra goto out_unlock_pi; 535397335f0SThomas Gleixner /* 536397335f0SThomas Gleixner * If deadlock detection is off, we stop here if we 53767792e2cSThomas Gleixner * are not the top pi waiter of the task. If deadlock 53867792e2cSThomas Gleixner * detection is enabled we continue, but stop the 53967792e2cSThomas Gleixner * requeueing in the chain walk. 540397335f0SThomas Gleixner */ 54167792e2cSThomas Gleixner if (top_waiter != task_top_pi_waiter(task)) { 54267792e2cSThomas Gleixner if (!detect_deadlock) 543397335f0SThomas Gleixner goto out_unlock_pi; 54467792e2cSThomas Gleixner else 54567792e2cSThomas Gleixner requeue = false; 54667792e2cSThomas Gleixner } 547397335f0SThomas Gleixner } 5481696a8beSPeter Zijlstra 5491696a8beSPeter Zijlstra /* 55067792e2cSThomas Gleixner * If the waiter priority is the same as the task priority 55167792e2cSThomas Gleixner * then there is no further priority adjustment necessary. If 55267792e2cSThomas Gleixner * deadlock detection is off, we stop the chain walk. If its 55367792e2cSThomas Gleixner * enabled we continue, but stop the requeueing in the chain 55467792e2cSThomas Gleixner * walk. 5551696a8beSPeter Zijlstra */ 55685e2d4f9SXunlei Pang if (waiter->prio == task->prio && !dl_task(task)) { 55767792e2cSThomas Gleixner if (!detect_deadlock) 5581696a8beSPeter Zijlstra goto out_unlock_pi; 55967792e2cSThomas Gleixner else 56067792e2cSThomas Gleixner requeue = false; 56167792e2cSThomas Gleixner } 5621696a8beSPeter Zijlstra 5633eb65aeaSThomas Gleixner /* 5643eb65aeaSThomas Gleixner * [4] Get the next lock 5653eb65aeaSThomas Gleixner */ 5661696a8beSPeter Zijlstra lock = waiter->lock; 5673eb65aeaSThomas Gleixner /* 5683eb65aeaSThomas Gleixner * [5] We need to trylock here as we are holding task->pi_lock, 5693eb65aeaSThomas Gleixner * which is the reverse lock order versus the other rtmutex 5703eb65aeaSThomas Gleixner * operations. 5713eb65aeaSThomas Gleixner */ 5721696a8beSPeter Zijlstra if (!raw_spin_trylock(&lock->wait_lock)) { 573b4abf910SThomas Gleixner raw_spin_unlock_irq(&task->pi_lock); 5741696a8beSPeter Zijlstra cpu_relax(); 5751696a8beSPeter Zijlstra goto retry; 5761696a8beSPeter Zijlstra } 5771696a8beSPeter Zijlstra 578397335f0SThomas Gleixner /* 5793eb65aeaSThomas Gleixner * [6] check_exit_conditions_2() protected by task->pi_lock and 5803eb65aeaSThomas Gleixner * lock->wait_lock. 5813eb65aeaSThomas Gleixner * 582397335f0SThomas Gleixner * Deadlock detection. If the lock is the same as the original 583397335f0SThomas Gleixner * lock which caused us to walk the lock chain or if the 584397335f0SThomas Gleixner * current lock is owned by the task which initiated the chain 585397335f0SThomas Gleixner * walk, we detected a deadlock. 586397335f0SThomas Gleixner */ 5871696a8beSPeter Zijlstra if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { 5888930ed80SThomas Gleixner debug_rt_mutex_deadlock(chwalk, orig_waiter, lock); 5891696a8beSPeter Zijlstra raw_spin_unlock(&lock->wait_lock); 5903d5c9340SThomas Gleixner ret = -EDEADLK; 5911696a8beSPeter Zijlstra goto out_unlock_pi; 5921696a8beSPeter Zijlstra } 5931696a8beSPeter Zijlstra 594a57594a1SThomas Gleixner /* 59567792e2cSThomas Gleixner * If we just follow the lock chain for deadlock detection, no 59667792e2cSThomas Gleixner * need to do all the requeue operations. To avoid a truckload 59767792e2cSThomas Gleixner * of conditionals around the various places below, just do the 59867792e2cSThomas Gleixner * minimum chain walk checks. 59967792e2cSThomas Gleixner */ 60067792e2cSThomas Gleixner if (!requeue) { 60167792e2cSThomas Gleixner /* 60267792e2cSThomas Gleixner * No requeue[7] here. Just release @task [8] 60367792e2cSThomas Gleixner */ 604b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 60567792e2cSThomas Gleixner put_task_struct(task); 60667792e2cSThomas Gleixner 60767792e2cSThomas Gleixner /* 60867792e2cSThomas Gleixner * [9] check_exit_conditions_3 protected by lock->wait_lock. 60967792e2cSThomas Gleixner * If there is no owner of the lock, end of chain. 61067792e2cSThomas Gleixner */ 61167792e2cSThomas Gleixner if (!rt_mutex_owner(lock)) { 612b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 61367792e2cSThomas Gleixner return 0; 61467792e2cSThomas Gleixner } 61567792e2cSThomas Gleixner 61667792e2cSThomas Gleixner /* [10] Grab the next task, i.e. owner of @lock */ 61767792e2cSThomas Gleixner task = rt_mutex_owner(lock); 61867792e2cSThomas Gleixner get_task_struct(task); 619b4abf910SThomas Gleixner raw_spin_lock(&task->pi_lock); 62067792e2cSThomas Gleixner 62167792e2cSThomas Gleixner /* 62267792e2cSThomas Gleixner * No requeue [11] here. We just do deadlock detection. 62367792e2cSThomas Gleixner * 62467792e2cSThomas Gleixner * [12] Store whether owner is blocked 62567792e2cSThomas Gleixner * itself. Decision is made after dropping the locks 62667792e2cSThomas Gleixner */ 62767792e2cSThomas Gleixner next_lock = task_blocked_on_lock(task); 62867792e2cSThomas Gleixner /* 62967792e2cSThomas Gleixner * Get the top waiter for the next iteration 63067792e2cSThomas Gleixner */ 63167792e2cSThomas Gleixner top_waiter = rt_mutex_top_waiter(lock); 63267792e2cSThomas Gleixner 63367792e2cSThomas Gleixner /* [13] Drop locks */ 634b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 635b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 63667792e2cSThomas Gleixner 63767792e2cSThomas Gleixner /* If owner is not blocked, end of chain. */ 63867792e2cSThomas Gleixner if (!next_lock) 63967792e2cSThomas Gleixner goto out_put_task; 64067792e2cSThomas Gleixner goto again; 64167792e2cSThomas Gleixner } 64267792e2cSThomas Gleixner 64367792e2cSThomas Gleixner /* 644a57594a1SThomas Gleixner * Store the current top waiter before doing the requeue 645a57594a1SThomas Gleixner * operation on @lock. We need it for the boost/deboost 646a57594a1SThomas Gleixner * decision below. 647a57594a1SThomas Gleixner */ 648a57594a1SThomas Gleixner prerequeue_top_waiter = rt_mutex_top_waiter(lock); 6491696a8beSPeter Zijlstra 6509f40a51aSDavidlohr Bueso /* [7] Requeue the waiter in the lock waiter tree. */ 651fb00aca4SPeter Zijlstra rt_mutex_dequeue(lock, waiter); 652*e0aad5b4SPeter Zijlstra 653*e0aad5b4SPeter Zijlstra /* 654*e0aad5b4SPeter Zijlstra * Update the waiter prio fields now that we're dequeued. 655*e0aad5b4SPeter Zijlstra * 656*e0aad5b4SPeter Zijlstra * These values can have changed through either: 657*e0aad5b4SPeter Zijlstra * 658*e0aad5b4SPeter Zijlstra * sys_sched_set_scheduler() / sys_sched_setattr() 659*e0aad5b4SPeter Zijlstra * 660*e0aad5b4SPeter Zijlstra * or 661*e0aad5b4SPeter Zijlstra * 662*e0aad5b4SPeter Zijlstra * DL CBS enforcement advancing the effective deadline. 663*e0aad5b4SPeter Zijlstra * 664*e0aad5b4SPeter Zijlstra * Even though pi_waiters also uses these fields, and that tree is only 665*e0aad5b4SPeter Zijlstra * updated in [11], we can do this here, since we hold [L], which 666*e0aad5b4SPeter Zijlstra * serializes all pi_waiters access and rb_erase() does not care about 667*e0aad5b4SPeter Zijlstra * the values of the node being removed. 668*e0aad5b4SPeter Zijlstra */ 6692d3d891dSDario Faggioli waiter->prio = task->prio; 670*e0aad5b4SPeter Zijlstra waiter->deadline = task->dl.deadline; 671*e0aad5b4SPeter Zijlstra 672fb00aca4SPeter Zijlstra rt_mutex_enqueue(lock, waiter); 6731696a8beSPeter Zijlstra 6743eb65aeaSThomas Gleixner /* [8] Release the task */ 675b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 6762ffa5a5cSThomas Gleixner put_task_struct(task); 6772ffa5a5cSThomas Gleixner 678a57594a1SThomas Gleixner /* 6793eb65aeaSThomas Gleixner * [9] check_exit_conditions_3 protected by lock->wait_lock. 6803eb65aeaSThomas Gleixner * 681a57594a1SThomas Gleixner * We must abort the chain walk if there is no lock owner even 682a57594a1SThomas Gleixner * in the dead lock detection case, as we have nothing to 683a57594a1SThomas Gleixner * follow here. This is the end of the chain we are walking. 684a57594a1SThomas Gleixner */ 6851696a8beSPeter Zijlstra if (!rt_mutex_owner(lock)) { 6861696a8beSPeter Zijlstra /* 6873eb65aeaSThomas Gleixner * If the requeue [7] above changed the top waiter, 6883eb65aeaSThomas Gleixner * then we need to wake the new top waiter up to try 6893eb65aeaSThomas Gleixner * to get the lock. 6901696a8beSPeter Zijlstra */ 691a57594a1SThomas Gleixner if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) 6921696a8beSPeter Zijlstra wake_up_process(rt_mutex_top_waiter(lock)->task); 693b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 6942ffa5a5cSThomas Gleixner return 0; 6951696a8beSPeter Zijlstra } 6961696a8beSPeter Zijlstra 6973eb65aeaSThomas Gleixner /* [10] Grab the next task, i.e. the owner of @lock */ 6981696a8beSPeter Zijlstra task = rt_mutex_owner(lock); 6991696a8beSPeter Zijlstra get_task_struct(task); 700b4abf910SThomas Gleixner raw_spin_lock(&task->pi_lock); 7011696a8beSPeter Zijlstra 7023eb65aeaSThomas Gleixner /* [11] requeue the pi waiters if necessary */ 7031696a8beSPeter Zijlstra if (waiter == rt_mutex_top_waiter(lock)) { 704a57594a1SThomas Gleixner /* 705a57594a1SThomas Gleixner * The waiter became the new top (highest priority) 706a57594a1SThomas Gleixner * waiter on the lock. Replace the previous top waiter 7079f40a51aSDavidlohr Bueso * in the owner tasks pi waiters tree with this waiter 708a57594a1SThomas Gleixner * and adjust the priority of the owner. 709a57594a1SThomas Gleixner */ 710a57594a1SThomas Gleixner rt_mutex_dequeue_pi(task, prerequeue_top_waiter); 711fb00aca4SPeter Zijlstra rt_mutex_enqueue_pi(task, waiter); 712acd58620SPeter Zijlstra rt_mutex_adjust_prio(task); 7131696a8beSPeter Zijlstra 714a57594a1SThomas Gleixner } else if (prerequeue_top_waiter == waiter) { 715a57594a1SThomas Gleixner /* 716a57594a1SThomas Gleixner * The waiter was the top waiter on the lock, but is 717a57594a1SThomas Gleixner * no longer the top prority waiter. Replace waiter in 7189f40a51aSDavidlohr Bueso * the owner tasks pi waiters tree with the new top 719a57594a1SThomas Gleixner * (highest priority) waiter and adjust the priority 720a57594a1SThomas Gleixner * of the owner. 721a57594a1SThomas Gleixner * The new top waiter is stored in @waiter so that 722a57594a1SThomas Gleixner * @waiter == @top_waiter evaluates to true below and 723a57594a1SThomas Gleixner * we continue to deboost the rest of the chain. 724a57594a1SThomas Gleixner */ 725fb00aca4SPeter Zijlstra rt_mutex_dequeue_pi(task, waiter); 7261696a8beSPeter Zijlstra waiter = rt_mutex_top_waiter(lock); 727fb00aca4SPeter Zijlstra rt_mutex_enqueue_pi(task, waiter); 728acd58620SPeter Zijlstra rt_mutex_adjust_prio(task); 729a57594a1SThomas Gleixner } else { 730a57594a1SThomas Gleixner /* 731a57594a1SThomas Gleixner * Nothing changed. No need to do any priority 732a57594a1SThomas Gleixner * adjustment. 733a57594a1SThomas Gleixner */ 7341696a8beSPeter Zijlstra } 7351696a8beSPeter Zijlstra 73682084984SThomas Gleixner /* 7373eb65aeaSThomas Gleixner * [12] check_exit_conditions_4() protected by task->pi_lock 7383eb65aeaSThomas Gleixner * and lock->wait_lock. The actual decisions are made after we 7393eb65aeaSThomas Gleixner * dropped the locks. 7403eb65aeaSThomas Gleixner * 74182084984SThomas Gleixner * Check whether the task which owns the current lock is pi 74282084984SThomas Gleixner * blocked itself. If yes we store a pointer to the lock for 74382084984SThomas Gleixner * the lock chain change detection above. After we dropped 74482084984SThomas Gleixner * task->pi_lock next_lock cannot be dereferenced anymore. 74582084984SThomas Gleixner */ 74682084984SThomas Gleixner next_lock = task_blocked_on_lock(task); 747a57594a1SThomas Gleixner /* 748a57594a1SThomas Gleixner * Store the top waiter of @lock for the end of chain walk 749a57594a1SThomas Gleixner * decision below. 750a57594a1SThomas Gleixner */ 7511696a8beSPeter Zijlstra top_waiter = rt_mutex_top_waiter(lock); 7523eb65aeaSThomas Gleixner 7533eb65aeaSThomas Gleixner /* [13] Drop the locks */ 754b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 755b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 7561696a8beSPeter Zijlstra 75782084984SThomas Gleixner /* 7583eb65aeaSThomas Gleixner * Make the actual exit decisions [12], based on the stored 7593eb65aeaSThomas Gleixner * values. 7603eb65aeaSThomas Gleixner * 76182084984SThomas Gleixner * We reached the end of the lock chain. Stop right here. No 76282084984SThomas Gleixner * point to go back just to figure that out. 76382084984SThomas Gleixner */ 76482084984SThomas Gleixner if (!next_lock) 76582084984SThomas Gleixner goto out_put_task; 76682084984SThomas Gleixner 767a57594a1SThomas Gleixner /* 768a57594a1SThomas Gleixner * If the current waiter is not the top waiter on the lock, 769a57594a1SThomas Gleixner * then we can stop the chain walk here if we are not in full 770a57594a1SThomas Gleixner * deadlock detection mode. 771a57594a1SThomas Gleixner */ 7721696a8beSPeter Zijlstra if (!detect_deadlock && waiter != top_waiter) 7731696a8beSPeter Zijlstra goto out_put_task; 7741696a8beSPeter Zijlstra 7751696a8beSPeter Zijlstra goto again; 7761696a8beSPeter Zijlstra 7771696a8beSPeter Zijlstra out_unlock_pi: 778b4abf910SThomas Gleixner raw_spin_unlock_irq(&task->pi_lock); 7791696a8beSPeter Zijlstra out_put_task: 7801696a8beSPeter Zijlstra put_task_struct(task); 7811696a8beSPeter Zijlstra 7821696a8beSPeter Zijlstra return ret; 7831696a8beSPeter Zijlstra } 7841696a8beSPeter Zijlstra 7851696a8beSPeter Zijlstra /* 7861696a8beSPeter Zijlstra * Try to take an rt-mutex 7871696a8beSPeter Zijlstra * 788b4abf910SThomas Gleixner * Must be called with lock->wait_lock held and interrupts disabled 7891696a8beSPeter Zijlstra * 790358c331fSThomas Gleixner * @lock: The lock to be acquired. 791358c331fSThomas Gleixner * @task: The task which wants to acquire the lock 7929f40a51aSDavidlohr Bueso * @waiter: The waiter that is queued to the lock's wait tree if the 793358c331fSThomas Gleixner * callsite called task_blocked_on_lock(), otherwise NULL 7941696a8beSPeter Zijlstra */ 7951696a8beSPeter Zijlstra static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, 7961696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter) 7971696a8beSPeter Zijlstra { 798*e0aad5b4SPeter Zijlstra lockdep_assert_held(&lock->wait_lock); 799*e0aad5b4SPeter Zijlstra 8001696a8beSPeter Zijlstra /* 801358c331fSThomas Gleixner * Before testing whether we can acquire @lock, we set the 802358c331fSThomas Gleixner * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all 803358c331fSThomas Gleixner * other tasks which try to modify @lock into the slow path 804358c331fSThomas Gleixner * and they serialize on @lock->wait_lock. 8051696a8beSPeter Zijlstra * 806358c331fSThomas Gleixner * The RT_MUTEX_HAS_WAITERS bit can have a transitional state 807358c331fSThomas Gleixner * as explained at the top of this file if and only if: 8081696a8beSPeter Zijlstra * 809358c331fSThomas Gleixner * - There is a lock owner. The caller must fixup the 810358c331fSThomas Gleixner * transient state if it does a trylock or leaves the lock 811358c331fSThomas Gleixner * function due to a signal or timeout. 812358c331fSThomas Gleixner * 813358c331fSThomas Gleixner * - @task acquires the lock and there are no other 814358c331fSThomas Gleixner * waiters. This is undone in rt_mutex_set_owner(@task) at 815358c331fSThomas Gleixner * the end of this function. 8161696a8beSPeter Zijlstra */ 8171696a8beSPeter Zijlstra mark_rt_mutex_waiters(lock); 8181696a8beSPeter Zijlstra 819358c331fSThomas Gleixner /* 820358c331fSThomas Gleixner * If @lock has an owner, give up. 821358c331fSThomas Gleixner */ 8221696a8beSPeter Zijlstra if (rt_mutex_owner(lock)) 8231696a8beSPeter Zijlstra return 0; 8241696a8beSPeter Zijlstra 8251696a8beSPeter Zijlstra /* 826358c331fSThomas Gleixner * If @waiter != NULL, @task has already enqueued the waiter 8279f40a51aSDavidlohr Bueso * into @lock waiter tree. If @waiter == NULL then this is a 828358c331fSThomas Gleixner * trylock attempt. 829358c331fSThomas Gleixner */ 830358c331fSThomas Gleixner if (waiter) { 831358c331fSThomas Gleixner /* 832358c331fSThomas Gleixner * If waiter is not the highest priority waiter of 833358c331fSThomas Gleixner * @lock, give up. 834358c331fSThomas Gleixner */ 835358c331fSThomas Gleixner if (waiter != rt_mutex_top_waiter(lock)) 836358c331fSThomas Gleixner return 0; 837358c331fSThomas Gleixner 838358c331fSThomas Gleixner /* 839358c331fSThomas Gleixner * We can acquire the lock. Remove the waiter from the 8409f40a51aSDavidlohr Bueso * lock waiters tree. 841358c331fSThomas Gleixner */ 842358c331fSThomas Gleixner rt_mutex_dequeue(lock, waiter); 843358c331fSThomas Gleixner 844358c331fSThomas Gleixner } else { 845358c331fSThomas Gleixner /* 846358c331fSThomas Gleixner * If the lock has waiters already we check whether @task is 847358c331fSThomas Gleixner * eligible to take over the lock. 848358c331fSThomas Gleixner * 849358c331fSThomas Gleixner * If there are no other waiters, @task can acquire 850358c331fSThomas Gleixner * the lock. @task->pi_blocked_on is NULL, so it does 851358c331fSThomas Gleixner * not need to be dequeued. 8521696a8beSPeter Zijlstra */ 8531696a8beSPeter Zijlstra if (rt_mutex_has_waiters(lock)) { 854358c331fSThomas Gleixner /* 855358c331fSThomas Gleixner * If @task->prio is greater than or equal to 856358c331fSThomas Gleixner * the top waiter priority (kernel view), 857358c331fSThomas Gleixner * @task lost. 858358c331fSThomas Gleixner */ 859358c331fSThomas Gleixner if (task->prio >= rt_mutex_top_waiter(lock)->prio) 8601696a8beSPeter Zijlstra return 0; 861358c331fSThomas Gleixner 862358c331fSThomas Gleixner /* 863358c331fSThomas Gleixner * The current top waiter stays enqueued. We 864358c331fSThomas Gleixner * don't have to change anything in the lock 865358c331fSThomas Gleixner * waiters order. 866358c331fSThomas Gleixner */ 867358c331fSThomas Gleixner } else { 868358c331fSThomas Gleixner /* 869358c331fSThomas Gleixner * No waiters. Take the lock without the 870358c331fSThomas Gleixner * pi_lock dance.@task->pi_blocked_on is NULL 871358c331fSThomas Gleixner * and we have no waiters to enqueue in @task 8729f40a51aSDavidlohr Bueso * pi waiters tree. 873358c331fSThomas Gleixner */ 874358c331fSThomas Gleixner goto takeit; 8751696a8beSPeter Zijlstra } 8761696a8beSPeter Zijlstra } 8771696a8beSPeter Zijlstra 8781696a8beSPeter Zijlstra /* 879358c331fSThomas Gleixner * Clear @task->pi_blocked_on. Requires protection by 880358c331fSThomas Gleixner * @task->pi_lock. Redundant operation for the @waiter == NULL 881358c331fSThomas Gleixner * case, but conditionals are more expensive than a redundant 882358c331fSThomas Gleixner * store. 8831696a8beSPeter Zijlstra */ 884b4abf910SThomas Gleixner raw_spin_lock(&task->pi_lock); 885358c331fSThomas Gleixner task->pi_blocked_on = NULL; 886358c331fSThomas Gleixner /* 887358c331fSThomas Gleixner * Finish the lock acquisition. @task is the new owner. If 888358c331fSThomas Gleixner * other waiters exist we have to insert the highest priority 8899f40a51aSDavidlohr Bueso * waiter into @task->pi_waiters tree. 890358c331fSThomas Gleixner */ 891358c331fSThomas Gleixner if (rt_mutex_has_waiters(lock)) 892358c331fSThomas Gleixner rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); 893b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 8941696a8beSPeter Zijlstra 895358c331fSThomas Gleixner takeit: 8961696a8beSPeter Zijlstra /* We got the lock. */ 8971696a8beSPeter Zijlstra debug_rt_mutex_lock(lock); 8981696a8beSPeter Zijlstra 899358c331fSThomas Gleixner /* 900358c331fSThomas Gleixner * This either preserves the RT_MUTEX_HAS_WAITERS bit if there 901358c331fSThomas Gleixner * are still waiters or clears it. 902358c331fSThomas Gleixner */ 9031696a8beSPeter Zijlstra rt_mutex_set_owner(lock, task); 9041696a8beSPeter Zijlstra 9051696a8beSPeter Zijlstra return 1; 9061696a8beSPeter Zijlstra } 9071696a8beSPeter Zijlstra 9081696a8beSPeter Zijlstra /* 9091696a8beSPeter Zijlstra * Task blocks on lock. 9101696a8beSPeter Zijlstra * 9111696a8beSPeter Zijlstra * Prepare waiter and propagate pi chain 9121696a8beSPeter Zijlstra * 913b4abf910SThomas Gleixner * This must be called with lock->wait_lock held and interrupts disabled 9141696a8beSPeter Zijlstra */ 9151696a8beSPeter Zijlstra static int task_blocks_on_rt_mutex(struct rt_mutex *lock, 9161696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter, 9171696a8beSPeter Zijlstra struct task_struct *task, 9188930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk) 9191696a8beSPeter Zijlstra { 9201696a8beSPeter Zijlstra struct task_struct *owner = rt_mutex_owner(lock); 9211696a8beSPeter Zijlstra struct rt_mutex_waiter *top_waiter = waiter; 92282084984SThomas Gleixner struct rt_mutex *next_lock; 9231696a8beSPeter Zijlstra int chain_walk = 0, res; 9241696a8beSPeter Zijlstra 925*e0aad5b4SPeter Zijlstra lockdep_assert_held(&lock->wait_lock); 926*e0aad5b4SPeter Zijlstra 927397335f0SThomas Gleixner /* 928397335f0SThomas Gleixner * Early deadlock detection. We really don't want the task to 929397335f0SThomas Gleixner * enqueue on itself just to untangle the mess later. It's not 930397335f0SThomas Gleixner * only an optimization. We drop the locks, so another waiter 931397335f0SThomas Gleixner * can come in before the chain walk detects the deadlock. So 932397335f0SThomas Gleixner * the other will detect the deadlock and return -EDEADLOCK, 933397335f0SThomas Gleixner * which is wrong, as the other waiter is not in a deadlock 934397335f0SThomas Gleixner * situation. 935397335f0SThomas Gleixner */ 9363d5c9340SThomas Gleixner if (owner == task) 937397335f0SThomas Gleixner return -EDEADLK; 938397335f0SThomas Gleixner 939b4abf910SThomas Gleixner raw_spin_lock(&task->pi_lock); 940acd58620SPeter Zijlstra rt_mutex_adjust_prio(task); 9411696a8beSPeter Zijlstra waiter->task = task; 9421696a8beSPeter Zijlstra waiter->lock = lock; 9432d3d891dSDario Faggioli waiter->prio = task->prio; 944*e0aad5b4SPeter Zijlstra waiter->deadline = task->dl.deadline; 9451696a8beSPeter Zijlstra 9461696a8beSPeter Zijlstra /* Get the top priority waiter on the lock */ 9471696a8beSPeter Zijlstra if (rt_mutex_has_waiters(lock)) 9481696a8beSPeter Zijlstra top_waiter = rt_mutex_top_waiter(lock); 949fb00aca4SPeter Zijlstra rt_mutex_enqueue(lock, waiter); 9501696a8beSPeter Zijlstra 9511696a8beSPeter Zijlstra task->pi_blocked_on = waiter; 9521696a8beSPeter Zijlstra 953b4abf910SThomas Gleixner raw_spin_unlock(&task->pi_lock); 9541696a8beSPeter Zijlstra 9551696a8beSPeter Zijlstra if (!owner) 9561696a8beSPeter Zijlstra return 0; 9571696a8beSPeter Zijlstra 958b4abf910SThomas Gleixner raw_spin_lock(&owner->pi_lock); 95982084984SThomas Gleixner if (waiter == rt_mutex_top_waiter(lock)) { 960fb00aca4SPeter Zijlstra rt_mutex_dequeue_pi(owner, top_waiter); 961fb00aca4SPeter Zijlstra rt_mutex_enqueue_pi(owner, waiter); 9621696a8beSPeter Zijlstra 963acd58620SPeter Zijlstra rt_mutex_adjust_prio(owner); 9641696a8beSPeter Zijlstra if (owner->pi_blocked_on) 9651696a8beSPeter Zijlstra chain_walk = 1; 9668930ed80SThomas Gleixner } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) { 9671696a8beSPeter Zijlstra chain_walk = 1; 96882084984SThomas Gleixner } 9691696a8beSPeter Zijlstra 97082084984SThomas Gleixner /* Store the lock on which owner is blocked or NULL */ 97182084984SThomas Gleixner next_lock = task_blocked_on_lock(owner); 97282084984SThomas Gleixner 973b4abf910SThomas Gleixner raw_spin_unlock(&owner->pi_lock); 97482084984SThomas Gleixner /* 97582084984SThomas Gleixner * Even if full deadlock detection is on, if the owner is not 97682084984SThomas Gleixner * blocked itself, we can avoid finding this out in the chain 97782084984SThomas Gleixner * walk. 97882084984SThomas Gleixner */ 97982084984SThomas Gleixner if (!chain_walk || !next_lock) 9801696a8beSPeter Zijlstra return 0; 9811696a8beSPeter Zijlstra 9821696a8beSPeter Zijlstra /* 9831696a8beSPeter Zijlstra * The owner can't disappear while holding a lock, 9841696a8beSPeter Zijlstra * so the owner struct is protected by wait_lock. 9851696a8beSPeter Zijlstra * Gets dropped in rt_mutex_adjust_prio_chain()! 9861696a8beSPeter Zijlstra */ 9871696a8beSPeter Zijlstra get_task_struct(owner); 9881696a8beSPeter Zijlstra 989b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 9901696a8beSPeter Zijlstra 9918930ed80SThomas Gleixner res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, 99282084984SThomas Gleixner next_lock, waiter, task); 9931696a8beSPeter Zijlstra 994b4abf910SThomas Gleixner raw_spin_lock_irq(&lock->wait_lock); 9951696a8beSPeter Zijlstra 9961696a8beSPeter Zijlstra return res; 9971696a8beSPeter Zijlstra } 9981696a8beSPeter Zijlstra 9991696a8beSPeter Zijlstra /* 10009f40a51aSDavidlohr Bueso * Remove the top waiter from the current tasks pi waiter tree and 100145ab4effSDavidlohr Bueso * queue it up. 10021696a8beSPeter Zijlstra * 1003b4abf910SThomas Gleixner * Called with lock->wait_lock held and interrupts disabled. 10041696a8beSPeter Zijlstra */ 100545ab4effSDavidlohr Bueso static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, 100645ab4effSDavidlohr Bueso struct rt_mutex *lock) 10071696a8beSPeter Zijlstra { 10081696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter; 10091696a8beSPeter Zijlstra 1010b4abf910SThomas Gleixner raw_spin_lock(¤t->pi_lock); 10111696a8beSPeter Zijlstra 10121696a8beSPeter Zijlstra waiter = rt_mutex_top_waiter(lock); 10131696a8beSPeter Zijlstra 10141696a8beSPeter Zijlstra /* 1015acd58620SPeter Zijlstra * Remove it from current->pi_waiters and deboost. 1016acd58620SPeter Zijlstra * 1017acd58620SPeter Zijlstra * We must in fact deboost here in order to ensure we call 1018acd58620SPeter Zijlstra * rt_mutex_setprio() to update p->pi_top_task before the 1019acd58620SPeter Zijlstra * task unblocks. 10201696a8beSPeter Zijlstra */ 1021fb00aca4SPeter Zijlstra rt_mutex_dequeue_pi(current, waiter); 1022acd58620SPeter Zijlstra rt_mutex_adjust_prio(current); 10231696a8beSPeter Zijlstra 102427e35715SThomas Gleixner /* 102527e35715SThomas Gleixner * As we are waking up the top waiter, and the waiter stays 102627e35715SThomas Gleixner * queued on the lock until it gets the lock, this lock 102727e35715SThomas Gleixner * obviously has waiters. Just set the bit here and this has 102827e35715SThomas Gleixner * the added benefit of forcing all new tasks into the 102927e35715SThomas Gleixner * slow path making sure no task of lower priority than 103027e35715SThomas Gleixner * the top waiter can steal this lock. 103127e35715SThomas Gleixner */ 103227e35715SThomas Gleixner lock->owner = (void *) RT_MUTEX_HAS_WAITERS; 10331696a8beSPeter Zijlstra 1034acd58620SPeter Zijlstra /* 1035acd58620SPeter Zijlstra * We deboosted before waking the top waiter task such that we don't 1036acd58620SPeter Zijlstra * run two tasks with the 'same' priority (and ensure the 1037acd58620SPeter Zijlstra * p->pi_top_task pointer points to a blocked task). This however can 1038acd58620SPeter Zijlstra * lead to priority inversion if we would get preempted after the 1039acd58620SPeter Zijlstra * deboost but before waking our donor task, hence the preempt_disable() 1040acd58620SPeter Zijlstra * before unlock. 1041acd58620SPeter Zijlstra * 1042acd58620SPeter Zijlstra * Pairs with preempt_enable() in rt_mutex_postunlock(); 1043acd58620SPeter Zijlstra */ 1044acd58620SPeter Zijlstra preempt_disable(); 104545ab4effSDavidlohr Bueso wake_q_add(wake_q, waiter->task); 1046acd58620SPeter Zijlstra raw_spin_unlock(¤t->pi_lock); 10471696a8beSPeter Zijlstra } 10481696a8beSPeter Zijlstra 10491696a8beSPeter Zijlstra /* 10501696a8beSPeter Zijlstra * Remove a waiter from a lock and give up 10511696a8beSPeter Zijlstra * 1052b4abf910SThomas Gleixner * Must be called with lock->wait_lock held and interrupts disabled. I must 10531696a8beSPeter Zijlstra * have just failed to try_to_take_rt_mutex(). 10541696a8beSPeter Zijlstra */ 10551696a8beSPeter Zijlstra static void remove_waiter(struct rt_mutex *lock, 10561696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter) 10571696a8beSPeter Zijlstra { 10581ca7b860SThomas Gleixner bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); 10591696a8beSPeter Zijlstra struct task_struct *owner = rt_mutex_owner(lock); 10601ca7b860SThomas Gleixner struct rt_mutex *next_lock; 10611696a8beSPeter Zijlstra 1062*e0aad5b4SPeter Zijlstra lockdep_assert_held(&lock->wait_lock); 1063*e0aad5b4SPeter Zijlstra 1064b4abf910SThomas Gleixner raw_spin_lock(¤t->pi_lock); 1065fb00aca4SPeter Zijlstra rt_mutex_dequeue(lock, waiter); 10661696a8beSPeter Zijlstra current->pi_blocked_on = NULL; 1067b4abf910SThomas Gleixner raw_spin_unlock(¤t->pi_lock); 10681696a8beSPeter Zijlstra 10691ca7b860SThomas Gleixner /* 10701ca7b860SThomas Gleixner * Only update priority if the waiter was the highest priority 10711ca7b860SThomas Gleixner * waiter of the lock and there is an owner to update. 10721ca7b860SThomas Gleixner */ 10731ca7b860SThomas Gleixner if (!owner || !is_top_waiter) 10741696a8beSPeter Zijlstra return; 10751696a8beSPeter Zijlstra 1076b4abf910SThomas Gleixner raw_spin_lock(&owner->pi_lock); 10771696a8beSPeter Zijlstra 1078fb00aca4SPeter Zijlstra rt_mutex_dequeue_pi(owner, waiter); 10791696a8beSPeter Zijlstra 10801ca7b860SThomas Gleixner if (rt_mutex_has_waiters(lock)) 10811ca7b860SThomas Gleixner rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock)); 10821696a8beSPeter Zijlstra 1083acd58620SPeter Zijlstra rt_mutex_adjust_prio(owner); 10841696a8beSPeter Zijlstra 108582084984SThomas Gleixner /* Store the lock on which owner is blocked or NULL */ 108682084984SThomas Gleixner next_lock = task_blocked_on_lock(owner); 10871696a8beSPeter Zijlstra 1088b4abf910SThomas Gleixner raw_spin_unlock(&owner->pi_lock); 10891696a8beSPeter Zijlstra 10901ca7b860SThomas Gleixner /* 10911ca7b860SThomas Gleixner * Don't walk the chain, if the owner task is not blocked 10921ca7b860SThomas Gleixner * itself. 10931ca7b860SThomas Gleixner */ 109482084984SThomas Gleixner if (!next_lock) 10951696a8beSPeter Zijlstra return; 10961696a8beSPeter Zijlstra 10971696a8beSPeter Zijlstra /* gets dropped in rt_mutex_adjust_prio_chain()! */ 10981696a8beSPeter Zijlstra get_task_struct(owner); 10991696a8beSPeter Zijlstra 1100b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 11011696a8beSPeter Zijlstra 11028930ed80SThomas Gleixner rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, 11038930ed80SThomas Gleixner next_lock, NULL, current); 11041696a8beSPeter Zijlstra 1105b4abf910SThomas Gleixner raw_spin_lock_irq(&lock->wait_lock); 11061696a8beSPeter Zijlstra } 11071696a8beSPeter Zijlstra 11081696a8beSPeter Zijlstra /* 11091696a8beSPeter Zijlstra * Recheck the pi chain, in case we got a priority setting 11101696a8beSPeter Zijlstra * 11111696a8beSPeter Zijlstra * Called from sched_setscheduler 11121696a8beSPeter Zijlstra */ 11131696a8beSPeter Zijlstra void rt_mutex_adjust_pi(struct task_struct *task) 11141696a8beSPeter Zijlstra { 11151696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter; 111682084984SThomas Gleixner struct rt_mutex *next_lock; 11171696a8beSPeter Zijlstra unsigned long flags; 11181696a8beSPeter Zijlstra 11191696a8beSPeter Zijlstra raw_spin_lock_irqsave(&task->pi_lock, flags); 11201696a8beSPeter Zijlstra 11211696a8beSPeter Zijlstra waiter = task->pi_blocked_on; 1122acd58620SPeter Zijlstra if (!waiter || (waiter->prio == task->prio && !dl_prio(task->prio))) { 11231696a8beSPeter Zijlstra raw_spin_unlock_irqrestore(&task->pi_lock, flags); 11241696a8beSPeter Zijlstra return; 11251696a8beSPeter Zijlstra } 112682084984SThomas Gleixner next_lock = waiter->lock; 11271696a8beSPeter Zijlstra raw_spin_unlock_irqrestore(&task->pi_lock, flags); 11281696a8beSPeter Zijlstra 11291696a8beSPeter Zijlstra /* gets dropped in rt_mutex_adjust_prio_chain()! */ 11301696a8beSPeter Zijlstra get_task_struct(task); 113182084984SThomas Gleixner 11328930ed80SThomas Gleixner rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, 11338930ed80SThomas Gleixner next_lock, NULL, task); 11341696a8beSPeter Zijlstra } 11351696a8beSPeter Zijlstra 113650809358SPeter Zijlstra void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter) 113750809358SPeter Zijlstra { 113850809358SPeter Zijlstra debug_rt_mutex_init_waiter(waiter); 113950809358SPeter Zijlstra RB_CLEAR_NODE(&waiter->pi_tree_entry); 114050809358SPeter Zijlstra RB_CLEAR_NODE(&waiter->tree_entry); 114150809358SPeter Zijlstra waiter->task = NULL; 114250809358SPeter Zijlstra } 114350809358SPeter Zijlstra 11441696a8beSPeter Zijlstra /** 11451696a8beSPeter Zijlstra * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop 11461696a8beSPeter Zijlstra * @lock: the rt_mutex to take 11471696a8beSPeter Zijlstra * @state: the state the task should block in (TASK_INTERRUPTIBLE 11481696a8beSPeter Zijlstra * or TASK_UNINTERRUPTIBLE) 11491696a8beSPeter Zijlstra * @timeout: the pre-initialized and started timer, or NULL for none 11501696a8beSPeter Zijlstra * @waiter: the pre-initialized rt_mutex_waiter 11511696a8beSPeter Zijlstra * 1152b4abf910SThomas Gleixner * Must be called with lock->wait_lock held and interrupts disabled 11531696a8beSPeter Zijlstra */ 11541696a8beSPeter Zijlstra static int __sched 11551696a8beSPeter Zijlstra __rt_mutex_slowlock(struct rt_mutex *lock, int state, 11561696a8beSPeter Zijlstra struct hrtimer_sleeper *timeout, 11571696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter) 11581696a8beSPeter Zijlstra { 11591696a8beSPeter Zijlstra int ret = 0; 11601696a8beSPeter Zijlstra 11611696a8beSPeter Zijlstra for (;;) { 11621696a8beSPeter Zijlstra /* Try to acquire the lock: */ 11631696a8beSPeter Zijlstra if (try_to_take_rt_mutex(lock, current, waiter)) 11641696a8beSPeter Zijlstra break; 11651696a8beSPeter Zijlstra 11661696a8beSPeter Zijlstra /* 11671696a8beSPeter Zijlstra * TASK_INTERRUPTIBLE checks for signals and 11681696a8beSPeter Zijlstra * timeout. Ignored otherwise. 11691696a8beSPeter Zijlstra */ 11704009f4b3SSteven Rostedt (VMware) if (likely(state == TASK_INTERRUPTIBLE)) { 11711696a8beSPeter Zijlstra /* Signal pending? */ 11721696a8beSPeter Zijlstra if (signal_pending(current)) 11731696a8beSPeter Zijlstra ret = -EINTR; 11741696a8beSPeter Zijlstra if (timeout && !timeout->task) 11751696a8beSPeter Zijlstra ret = -ETIMEDOUT; 11761696a8beSPeter Zijlstra if (ret) 11771696a8beSPeter Zijlstra break; 11781696a8beSPeter Zijlstra } 11791696a8beSPeter Zijlstra 1180b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 11811696a8beSPeter Zijlstra 11821696a8beSPeter Zijlstra debug_rt_mutex_print_deadlock(waiter); 11831696a8beSPeter Zijlstra 11841b0b7c17SDavidlohr Bueso schedule(); 11851696a8beSPeter Zijlstra 1186b4abf910SThomas Gleixner raw_spin_lock_irq(&lock->wait_lock); 11871696a8beSPeter Zijlstra set_current_state(state); 11881696a8beSPeter Zijlstra } 11891696a8beSPeter Zijlstra 1190afffc6c1SDavidlohr Bueso __set_current_state(TASK_RUNNING); 11911696a8beSPeter Zijlstra return ret; 11921696a8beSPeter Zijlstra } 11931696a8beSPeter Zijlstra 11943d5c9340SThomas Gleixner static void rt_mutex_handle_deadlock(int res, int detect_deadlock, 11953d5c9340SThomas Gleixner struct rt_mutex_waiter *w) 11963d5c9340SThomas Gleixner { 11973d5c9340SThomas Gleixner /* 11983d5c9340SThomas Gleixner * If the result is not -EDEADLOCK or the caller requested 11993d5c9340SThomas Gleixner * deadlock detection, nothing to do here. 12003d5c9340SThomas Gleixner */ 12013d5c9340SThomas Gleixner if (res != -EDEADLOCK || detect_deadlock) 12023d5c9340SThomas Gleixner return; 12033d5c9340SThomas Gleixner 12043d5c9340SThomas Gleixner /* 12053d5c9340SThomas Gleixner * Yell lowdly and stop the task right here. 12063d5c9340SThomas Gleixner */ 12073d5c9340SThomas Gleixner rt_mutex_print_deadlock(w); 12083d5c9340SThomas Gleixner while (1) { 12093d5c9340SThomas Gleixner set_current_state(TASK_INTERRUPTIBLE); 12103d5c9340SThomas Gleixner schedule(); 12113d5c9340SThomas Gleixner } 12123d5c9340SThomas Gleixner } 12133d5c9340SThomas Gleixner 12141696a8beSPeter Zijlstra /* 12151696a8beSPeter Zijlstra * Slow path lock function: 12161696a8beSPeter Zijlstra */ 12171696a8beSPeter Zijlstra static int __sched 12181696a8beSPeter Zijlstra rt_mutex_slowlock(struct rt_mutex *lock, int state, 12191696a8beSPeter Zijlstra struct hrtimer_sleeper *timeout, 12208930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk) 12211696a8beSPeter Zijlstra { 12221696a8beSPeter Zijlstra struct rt_mutex_waiter waiter; 1223b4abf910SThomas Gleixner unsigned long flags; 12241696a8beSPeter Zijlstra int ret = 0; 12251696a8beSPeter Zijlstra 122650809358SPeter Zijlstra rt_mutex_init_waiter(&waiter); 12271696a8beSPeter Zijlstra 1228b4abf910SThomas Gleixner /* 1229b4abf910SThomas Gleixner * Technically we could use raw_spin_[un]lock_irq() here, but this can 1230b4abf910SThomas Gleixner * be called in early boot if the cmpxchg() fast path is disabled 1231b4abf910SThomas Gleixner * (debug, no architecture support). In this case we will acquire the 1232b4abf910SThomas Gleixner * rtmutex with lock->wait_lock held. But we cannot unconditionally 1233b4abf910SThomas Gleixner * enable interrupts in that early boot case. So we need to use the 1234b4abf910SThomas Gleixner * irqsave/restore variants. 1235b4abf910SThomas Gleixner */ 1236b4abf910SThomas Gleixner raw_spin_lock_irqsave(&lock->wait_lock, flags); 12371696a8beSPeter Zijlstra 12381696a8beSPeter Zijlstra /* Try to acquire the lock again: */ 12391696a8beSPeter Zijlstra if (try_to_take_rt_mutex(lock, current, NULL)) { 1240b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 12411696a8beSPeter Zijlstra return 0; 12421696a8beSPeter Zijlstra } 12431696a8beSPeter Zijlstra 12441696a8beSPeter Zijlstra set_current_state(state); 12451696a8beSPeter Zijlstra 12461696a8beSPeter Zijlstra /* Setup the timer, when timeout != NULL */ 1247ccdd92c1SThomas Gleixner if (unlikely(timeout)) 12481696a8beSPeter Zijlstra hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS); 12491696a8beSPeter Zijlstra 12508930ed80SThomas Gleixner ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk); 12511696a8beSPeter Zijlstra 12521696a8beSPeter Zijlstra if (likely(!ret)) 1253afffc6c1SDavidlohr Bueso /* sleep on the mutex */ 12541696a8beSPeter Zijlstra ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); 12551696a8beSPeter Zijlstra 12563d5c9340SThomas Gleixner if (unlikely(ret)) { 12579d3e2d02SSebastian Andrzej Siewior __set_current_state(TASK_RUNNING); 12588d1e5a1aSSebastian Andrzej Siewior if (rt_mutex_has_waiters(lock)) 12591696a8beSPeter Zijlstra remove_waiter(lock, &waiter); 12608930ed80SThomas Gleixner rt_mutex_handle_deadlock(ret, chwalk, &waiter); 12613d5c9340SThomas Gleixner } 12621696a8beSPeter Zijlstra 12631696a8beSPeter Zijlstra /* 12641696a8beSPeter Zijlstra * try_to_take_rt_mutex() sets the waiter bit 12651696a8beSPeter Zijlstra * unconditionally. We might have to fix that up. 12661696a8beSPeter Zijlstra */ 12671696a8beSPeter Zijlstra fixup_rt_mutex_waiters(lock); 12681696a8beSPeter Zijlstra 1269b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 12701696a8beSPeter Zijlstra 12711696a8beSPeter Zijlstra /* Remove pending timer: */ 12721696a8beSPeter Zijlstra if (unlikely(timeout)) 12731696a8beSPeter Zijlstra hrtimer_cancel(&timeout->timer); 12741696a8beSPeter Zijlstra 12751696a8beSPeter Zijlstra debug_rt_mutex_free_waiter(&waiter); 12761696a8beSPeter Zijlstra 12771696a8beSPeter Zijlstra return ret; 12781696a8beSPeter Zijlstra } 12791696a8beSPeter Zijlstra 12801696a8beSPeter Zijlstra /* 12811696a8beSPeter Zijlstra * Slow path try-lock function: 12821696a8beSPeter Zijlstra */ 128388f2b4c1SThomas Gleixner static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) 12841696a8beSPeter Zijlstra { 1285b4abf910SThomas Gleixner unsigned long flags; 128688f2b4c1SThomas Gleixner int ret; 12871696a8beSPeter Zijlstra 128888f2b4c1SThomas Gleixner /* 128988f2b4c1SThomas Gleixner * If the lock already has an owner we fail to get the lock. 129088f2b4c1SThomas Gleixner * This can be done without taking the @lock->wait_lock as 129188f2b4c1SThomas Gleixner * it is only being read, and this is a trylock anyway. 129288f2b4c1SThomas Gleixner */ 129388f2b4c1SThomas Gleixner if (rt_mutex_owner(lock)) 129488f2b4c1SThomas Gleixner return 0; 129588f2b4c1SThomas Gleixner 129688f2b4c1SThomas Gleixner /* 1297b4abf910SThomas Gleixner * The mutex has currently no owner. Lock the wait lock and try to 1298b4abf910SThomas Gleixner * acquire the lock. We use irqsave here to support early boot calls. 129988f2b4c1SThomas Gleixner */ 1300b4abf910SThomas Gleixner raw_spin_lock_irqsave(&lock->wait_lock, flags); 13011696a8beSPeter Zijlstra 13021696a8beSPeter Zijlstra ret = try_to_take_rt_mutex(lock, current, NULL); 130388f2b4c1SThomas Gleixner 13041696a8beSPeter Zijlstra /* 130588f2b4c1SThomas Gleixner * try_to_take_rt_mutex() sets the lock waiters bit 130688f2b4c1SThomas Gleixner * unconditionally. Clean this up. 13071696a8beSPeter Zijlstra */ 13081696a8beSPeter Zijlstra fixup_rt_mutex_waiters(lock); 13091696a8beSPeter Zijlstra 1310b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 13111696a8beSPeter Zijlstra 13121696a8beSPeter Zijlstra return ret; 13131696a8beSPeter Zijlstra } 13141696a8beSPeter Zijlstra 13151696a8beSPeter Zijlstra /* 1316802ab58dSSebastian Andrzej Siewior * Slow path to release a rt-mutex. 1317aa2bfe55SPeter Zijlstra * 1318aa2bfe55SPeter Zijlstra * Return whether the current task needs to call rt_mutex_postunlock(). 13191696a8beSPeter Zijlstra */ 1320802ab58dSSebastian Andrzej Siewior static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, 1321802ab58dSSebastian Andrzej Siewior struct wake_q_head *wake_q) 13221696a8beSPeter Zijlstra { 1323b4abf910SThomas Gleixner unsigned long flags; 1324b4abf910SThomas Gleixner 1325b4abf910SThomas Gleixner /* irqsave required to support early boot calls */ 1326b4abf910SThomas Gleixner raw_spin_lock_irqsave(&lock->wait_lock, flags); 13271696a8beSPeter Zijlstra 13281696a8beSPeter Zijlstra debug_rt_mutex_unlock(lock); 13291696a8beSPeter Zijlstra 133027e35715SThomas Gleixner /* 133127e35715SThomas Gleixner * We must be careful here if the fast path is enabled. If we 133227e35715SThomas Gleixner * have no waiters queued we cannot set owner to NULL here 133327e35715SThomas Gleixner * because of: 133427e35715SThomas Gleixner * 133527e35715SThomas Gleixner * foo->lock->owner = NULL; 133627e35715SThomas Gleixner * rtmutex_lock(foo->lock); <- fast path 133727e35715SThomas Gleixner * free = atomic_dec_and_test(foo->refcnt); 133827e35715SThomas Gleixner * rtmutex_unlock(foo->lock); <- fast path 133927e35715SThomas Gleixner * if (free) 134027e35715SThomas Gleixner * kfree(foo); 134127e35715SThomas Gleixner * raw_spin_unlock(foo->lock->wait_lock); 134227e35715SThomas Gleixner * 134327e35715SThomas Gleixner * So for the fastpath enabled kernel: 134427e35715SThomas Gleixner * 134527e35715SThomas Gleixner * Nothing can set the waiters bit as long as we hold 134627e35715SThomas Gleixner * lock->wait_lock. So we do the following sequence: 134727e35715SThomas Gleixner * 134827e35715SThomas Gleixner * owner = rt_mutex_owner(lock); 134927e35715SThomas Gleixner * clear_rt_mutex_waiters(lock); 135027e35715SThomas Gleixner * raw_spin_unlock(&lock->wait_lock); 135127e35715SThomas Gleixner * if (cmpxchg(&lock->owner, owner, 0) == owner) 135227e35715SThomas Gleixner * return; 135327e35715SThomas Gleixner * goto retry; 135427e35715SThomas Gleixner * 135527e35715SThomas Gleixner * The fastpath disabled variant is simple as all access to 135627e35715SThomas Gleixner * lock->owner is serialized by lock->wait_lock: 135727e35715SThomas Gleixner * 135827e35715SThomas Gleixner * lock->owner = NULL; 135927e35715SThomas Gleixner * raw_spin_unlock(&lock->wait_lock); 136027e35715SThomas Gleixner */ 136127e35715SThomas Gleixner while (!rt_mutex_has_waiters(lock)) { 136227e35715SThomas Gleixner /* Drops lock->wait_lock ! */ 1363b4abf910SThomas Gleixner if (unlock_rt_mutex_safe(lock, flags) == true) 1364802ab58dSSebastian Andrzej Siewior return false; 136527e35715SThomas Gleixner /* Relock the rtmutex and try again */ 1366b4abf910SThomas Gleixner raw_spin_lock_irqsave(&lock->wait_lock, flags); 13671696a8beSPeter Zijlstra } 13681696a8beSPeter Zijlstra 136927e35715SThomas Gleixner /* 137027e35715SThomas Gleixner * The wakeup next waiter path does not suffer from the above 137127e35715SThomas Gleixner * race. See the comments there. 137245ab4effSDavidlohr Bueso * 137345ab4effSDavidlohr Bueso * Queue the next waiter for wakeup once we release the wait_lock. 137427e35715SThomas Gleixner */ 1375802ab58dSSebastian Andrzej Siewior mark_wakeup_next_waiter(wake_q, lock); 1376b4abf910SThomas Gleixner raw_spin_unlock_irqrestore(&lock->wait_lock, flags); 13771696a8beSPeter Zijlstra 1378aa2bfe55SPeter Zijlstra return true; /* call rt_mutex_postunlock() */ 13791696a8beSPeter Zijlstra } 13801696a8beSPeter Zijlstra 13811696a8beSPeter Zijlstra /* 13821696a8beSPeter Zijlstra * debug aware fast / slowpath lock,trylock,unlock 13831696a8beSPeter Zijlstra * 13841696a8beSPeter Zijlstra * The atomic acquire/release ops are compiled away, when either the 13851696a8beSPeter Zijlstra * architecture does not support cmpxchg or when debugging is enabled. 13861696a8beSPeter Zijlstra */ 13871696a8beSPeter Zijlstra static inline int 13881696a8beSPeter Zijlstra rt_mutex_fastlock(struct rt_mutex *lock, int state, 13891696a8beSPeter Zijlstra int (*slowfn)(struct rt_mutex *lock, int state, 13901696a8beSPeter Zijlstra struct hrtimer_sleeper *timeout, 13918930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk)) 13921696a8beSPeter Zijlstra { 1393fffa954fSPeter Zijlstra if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) 13941696a8beSPeter Zijlstra return 0; 1395fffa954fSPeter Zijlstra 13968930ed80SThomas Gleixner return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); 13971696a8beSPeter Zijlstra } 13981696a8beSPeter Zijlstra 13991696a8beSPeter Zijlstra static inline int 14001696a8beSPeter Zijlstra rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, 14018930ed80SThomas Gleixner struct hrtimer_sleeper *timeout, 14028930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk, 14031696a8beSPeter Zijlstra int (*slowfn)(struct rt_mutex *lock, int state, 14041696a8beSPeter Zijlstra struct hrtimer_sleeper *timeout, 14058930ed80SThomas Gleixner enum rtmutex_chainwalk chwalk)) 14061696a8beSPeter Zijlstra { 14078930ed80SThomas Gleixner if (chwalk == RT_MUTEX_MIN_CHAINWALK && 1408fffa954fSPeter Zijlstra likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) 14091696a8beSPeter Zijlstra return 0; 1410fffa954fSPeter Zijlstra 14118930ed80SThomas Gleixner return slowfn(lock, state, timeout, chwalk); 14121696a8beSPeter Zijlstra } 14131696a8beSPeter Zijlstra 14141696a8beSPeter Zijlstra static inline int 14151696a8beSPeter Zijlstra rt_mutex_fasttrylock(struct rt_mutex *lock, 14161696a8beSPeter Zijlstra int (*slowfn)(struct rt_mutex *lock)) 14171696a8beSPeter Zijlstra { 1418fffa954fSPeter Zijlstra if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) 14191696a8beSPeter Zijlstra return 1; 1420fffa954fSPeter Zijlstra 14211696a8beSPeter Zijlstra return slowfn(lock); 14221696a8beSPeter Zijlstra } 14231696a8beSPeter Zijlstra 14242a1c6029SXunlei Pang /* 1425aa2bfe55SPeter Zijlstra * Performs the wakeup of the the top-waiter and re-enables preemption. 14262a1c6029SXunlei Pang */ 1427aa2bfe55SPeter Zijlstra void rt_mutex_postunlock(struct wake_q_head *wake_q) 14282a1c6029SXunlei Pang { 14292a1c6029SXunlei Pang wake_up_q(wake_q); 14302a1c6029SXunlei Pang 14312a1c6029SXunlei Pang /* Pairs with preempt_disable() in rt_mutex_slowunlock() */ 14322a1c6029SXunlei Pang preempt_enable(); 14332a1c6029SXunlei Pang } 14342a1c6029SXunlei Pang 14351696a8beSPeter Zijlstra static inline void 14361696a8beSPeter Zijlstra rt_mutex_fastunlock(struct rt_mutex *lock, 1437802ab58dSSebastian Andrzej Siewior bool (*slowfn)(struct rt_mutex *lock, 1438802ab58dSSebastian Andrzej Siewior struct wake_q_head *wqh)) 14391696a8beSPeter Zijlstra { 1440194a6b5bSWaiman Long DEFINE_WAKE_Q(wake_q); 1441802ab58dSSebastian Andrzej Siewior 1442fffa954fSPeter Zijlstra if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) 1443fffa954fSPeter Zijlstra return; 1444802ab58dSSebastian Andrzej Siewior 1445aa2bfe55SPeter Zijlstra if (slowfn(lock, &wake_q)) 1446aa2bfe55SPeter Zijlstra rt_mutex_postunlock(&wake_q); 1447802ab58dSSebastian Andrzej Siewior } 14481696a8beSPeter Zijlstra 14491696a8beSPeter Zijlstra /** 14501696a8beSPeter Zijlstra * rt_mutex_lock - lock a rt_mutex 14511696a8beSPeter Zijlstra * 14521696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 14531696a8beSPeter Zijlstra */ 14541696a8beSPeter Zijlstra void __sched rt_mutex_lock(struct rt_mutex *lock) 14551696a8beSPeter Zijlstra { 14561696a8beSPeter Zijlstra might_sleep(); 14571696a8beSPeter Zijlstra 1458c051b21fSThomas Gleixner rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); 14591696a8beSPeter Zijlstra } 14601696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_lock); 14611696a8beSPeter Zijlstra 14621696a8beSPeter Zijlstra /** 14631696a8beSPeter Zijlstra * rt_mutex_lock_interruptible - lock a rt_mutex interruptible 14641696a8beSPeter Zijlstra * 14651696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 14661696a8beSPeter Zijlstra * 14671696a8beSPeter Zijlstra * Returns: 14681696a8beSPeter Zijlstra * 0 on success 14691696a8beSPeter Zijlstra * -EINTR when interrupted by a signal 14701696a8beSPeter Zijlstra */ 1471c051b21fSThomas Gleixner int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) 14721696a8beSPeter Zijlstra { 14731696a8beSPeter Zijlstra might_sleep(); 14741696a8beSPeter Zijlstra 1475c051b21fSThomas Gleixner return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); 14761696a8beSPeter Zijlstra } 14771696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); 14781696a8beSPeter Zijlstra 1479c051b21fSThomas Gleixner /* 14805293c2efSPeter Zijlstra * Futex variant, must not use fastpath. 14815293c2efSPeter Zijlstra */ 14825293c2efSPeter Zijlstra int __sched rt_mutex_futex_trylock(struct rt_mutex *lock) 14835293c2efSPeter Zijlstra { 14845293c2efSPeter Zijlstra return rt_mutex_slowtrylock(lock); 1485c051b21fSThomas Gleixner } 1486c051b21fSThomas Gleixner 14871696a8beSPeter Zijlstra /** 14881696a8beSPeter Zijlstra * rt_mutex_timed_lock - lock a rt_mutex interruptible 14891696a8beSPeter Zijlstra * the timeout structure is provided 14901696a8beSPeter Zijlstra * by the caller 14911696a8beSPeter Zijlstra * 14921696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 14931696a8beSPeter Zijlstra * @timeout: timeout structure or NULL (no timeout) 14941696a8beSPeter Zijlstra * 14951696a8beSPeter Zijlstra * Returns: 14961696a8beSPeter Zijlstra * 0 on success 14971696a8beSPeter Zijlstra * -EINTR when interrupted by a signal 14981696a8beSPeter Zijlstra * -ETIMEDOUT when the timeout expired 14991696a8beSPeter Zijlstra */ 15001696a8beSPeter Zijlstra int 1501c051b21fSThomas Gleixner rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout) 15021696a8beSPeter Zijlstra { 15031696a8beSPeter Zijlstra might_sleep(); 15041696a8beSPeter Zijlstra 15058930ed80SThomas Gleixner return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 15068930ed80SThomas Gleixner RT_MUTEX_MIN_CHAINWALK, 1507c051b21fSThomas Gleixner rt_mutex_slowlock); 15081696a8beSPeter Zijlstra } 15091696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); 15101696a8beSPeter Zijlstra 15111696a8beSPeter Zijlstra /** 15121696a8beSPeter Zijlstra * rt_mutex_trylock - try to lock a rt_mutex 15131696a8beSPeter Zijlstra * 15141696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 15151696a8beSPeter Zijlstra * 15166ce47fd9SThomas Gleixner * This function can only be called in thread context. It's safe to 15176ce47fd9SThomas Gleixner * call it from atomic regions, but not from hard interrupt or soft 15186ce47fd9SThomas Gleixner * interrupt context. 15196ce47fd9SThomas Gleixner * 15201696a8beSPeter Zijlstra * Returns 1 on success and 0 on contention 15211696a8beSPeter Zijlstra */ 15221696a8beSPeter Zijlstra int __sched rt_mutex_trylock(struct rt_mutex *lock) 15231696a8beSPeter Zijlstra { 1524a461d587SSebastian Andrzej Siewior if (WARN_ON_ONCE(in_irq() || in_nmi() || in_serving_softirq())) 15256ce47fd9SThomas Gleixner return 0; 15266ce47fd9SThomas Gleixner 15271696a8beSPeter Zijlstra return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); 15281696a8beSPeter Zijlstra } 15291696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_trylock); 15301696a8beSPeter Zijlstra 15311696a8beSPeter Zijlstra /** 15321696a8beSPeter Zijlstra * rt_mutex_unlock - unlock a rt_mutex 15331696a8beSPeter Zijlstra * 15341696a8beSPeter Zijlstra * @lock: the rt_mutex to be unlocked 15351696a8beSPeter Zijlstra */ 15361696a8beSPeter Zijlstra void __sched rt_mutex_unlock(struct rt_mutex *lock) 15371696a8beSPeter Zijlstra { 15381696a8beSPeter Zijlstra rt_mutex_fastunlock(lock, rt_mutex_slowunlock); 15391696a8beSPeter Zijlstra } 15401696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_unlock); 15411696a8beSPeter Zijlstra 15421696a8beSPeter Zijlstra /** 15435293c2efSPeter Zijlstra * Futex variant, that since futex variants do not use the fast-path, can be 15445293c2efSPeter Zijlstra * simple and will not need to retry. 1545802ab58dSSebastian Andrzej Siewior */ 15465293c2efSPeter Zijlstra bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock, 15475293c2efSPeter Zijlstra struct wake_q_head *wake_q) 1548802ab58dSSebastian Andrzej Siewior { 15495293c2efSPeter Zijlstra lockdep_assert_held(&lock->wait_lock); 1550fffa954fSPeter Zijlstra 15515293c2efSPeter Zijlstra debug_rt_mutex_unlock(lock); 15525293c2efSPeter Zijlstra 15535293c2efSPeter Zijlstra if (!rt_mutex_has_waiters(lock)) { 15545293c2efSPeter Zijlstra lock->owner = NULL; 15555293c2efSPeter Zijlstra return false; /* done */ 15565293c2efSPeter Zijlstra } 15575293c2efSPeter Zijlstra 15585293c2efSPeter Zijlstra mark_wakeup_next_waiter(wake_q, lock); 15592a1c6029SXunlei Pang /* 15602a1c6029SXunlei Pang * We've already deboosted, retain preempt_disabled when dropping 15612a1c6029SXunlei Pang * the wait_lock to avoid inversion until the wakeup. Matched 15622a1c6029SXunlei Pang * by rt_mutex_postunlock(); 15632a1c6029SXunlei Pang */ 15642a1c6029SXunlei Pang preempt_disable(); 15652a1c6029SXunlei Pang 1566aa2bfe55SPeter Zijlstra return true; /* call postunlock() */ 15675293c2efSPeter Zijlstra } 15685293c2efSPeter Zijlstra 15695293c2efSPeter Zijlstra void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) 15705293c2efSPeter Zijlstra { 15715293c2efSPeter Zijlstra DEFINE_WAKE_Q(wake_q); 1572aa2bfe55SPeter Zijlstra bool postunlock; 15735293c2efSPeter Zijlstra 15745293c2efSPeter Zijlstra raw_spin_lock_irq(&lock->wait_lock); 1575aa2bfe55SPeter Zijlstra postunlock = __rt_mutex_futex_unlock(lock, &wake_q); 15765293c2efSPeter Zijlstra raw_spin_unlock_irq(&lock->wait_lock); 15775293c2efSPeter Zijlstra 1578aa2bfe55SPeter Zijlstra if (postunlock) 1579aa2bfe55SPeter Zijlstra rt_mutex_postunlock(&wake_q); 1580802ab58dSSebastian Andrzej Siewior } 1581802ab58dSSebastian Andrzej Siewior 1582802ab58dSSebastian Andrzej Siewior /** 15831696a8beSPeter Zijlstra * rt_mutex_destroy - mark a mutex unusable 15841696a8beSPeter Zijlstra * @lock: the mutex to be destroyed 15851696a8beSPeter Zijlstra * 15861696a8beSPeter Zijlstra * This function marks the mutex uninitialized, and any subsequent 15871696a8beSPeter Zijlstra * use of the mutex is forbidden. The mutex must not be locked when 15881696a8beSPeter Zijlstra * this function is called. 15891696a8beSPeter Zijlstra */ 15901696a8beSPeter Zijlstra void rt_mutex_destroy(struct rt_mutex *lock) 15911696a8beSPeter Zijlstra { 15921696a8beSPeter Zijlstra WARN_ON(rt_mutex_is_locked(lock)); 15931696a8beSPeter Zijlstra #ifdef CONFIG_DEBUG_RT_MUTEXES 15941696a8beSPeter Zijlstra lock->magic = NULL; 15951696a8beSPeter Zijlstra #endif 15961696a8beSPeter Zijlstra } 15971696a8beSPeter Zijlstra 15981696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(rt_mutex_destroy); 15991696a8beSPeter Zijlstra 16001696a8beSPeter Zijlstra /** 16011696a8beSPeter Zijlstra * __rt_mutex_init - initialize the rt lock 16021696a8beSPeter Zijlstra * 16031696a8beSPeter Zijlstra * @lock: the rt lock to be initialized 16041696a8beSPeter Zijlstra * 16051696a8beSPeter Zijlstra * Initialize the rt lock to unlocked state. 16061696a8beSPeter Zijlstra * 16071696a8beSPeter Zijlstra * Initializing of a locked rt lock is not allowed 16081696a8beSPeter Zijlstra */ 16091696a8beSPeter Zijlstra void __rt_mutex_init(struct rt_mutex *lock, const char *name) 16101696a8beSPeter Zijlstra { 16111696a8beSPeter Zijlstra lock->owner = NULL; 16121696a8beSPeter Zijlstra raw_spin_lock_init(&lock->wait_lock); 1613fb00aca4SPeter Zijlstra lock->waiters = RB_ROOT; 1614fb00aca4SPeter Zijlstra lock->waiters_leftmost = NULL; 16151696a8beSPeter Zijlstra 16161696a8beSPeter Zijlstra debug_rt_mutex_init(lock, name); 16171696a8beSPeter Zijlstra } 16181696a8beSPeter Zijlstra EXPORT_SYMBOL_GPL(__rt_mutex_init); 16191696a8beSPeter Zijlstra 16201696a8beSPeter Zijlstra /** 16211696a8beSPeter Zijlstra * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a 16221696a8beSPeter Zijlstra * proxy owner 16231696a8beSPeter Zijlstra * 16241696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 16251696a8beSPeter Zijlstra * @proxy_owner:the task to set as owner 16261696a8beSPeter Zijlstra * 16271696a8beSPeter Zijlstra * No locking. Caller has to do serializing itself 162884d82ec5SThomas Gleixner * 162984d82ec5SThomas Gleixner * Special API call for PI-futex support. This initializes the rtmutex and 163084d82ec5SThomas Gleixner * assigns it to @proxy_owner. Concurrent operations on the rtmutex are not 163184d82ec5SThomas Gleixner * possible at this point because the pi_state which contains the rtmutex 163284d82ec5SThomas Gleixner * is not yet visible to other tasks. 16331696a8beSPeter Zijlstra */ 16341696a8beSPeter Zijlstra void rt_mutex_init_proxy_locked(struct rt_mutex *lock, 16351696a8beSPeter Zijlstra struct task_struct *proxy_owner) 16361696a8beSPeter Zijlstra { 16371696a8beSPeter Zijlstra __rt_mutex_init(lock, NULL); 16381696a8beSPeter Zijlstra debug_rt_mutex_proxy_lock(lock, proxy_owner); 16391696a8beSPeter Zijlstra rt_mutex_set_owner(lock, proxy_owner); 16401696a8beSPeter Zijlstra } 16411696a8beSPeter Zijlstra 16421696a8beSPeter Zijlstra /** 16431696a8beSPeter Zijlstra * rt_mutex_proxy_unlock - release a lock on behalf of owner 16441696a8beSPeter Zijlstra * 16451696a8beSPeter Zijlstra * @lock: the rt_mutex to be locked 16461696a8beSPeter Zijlstra * 16471696a8beSPeter Zijlstra * No locking. Caller has to do serializing itself 164884d82ec5SThomas Gleixner * 164984d82ec5SThomas Gleixner * Special API call for PI-futex support. This merrily cleans up the rtmutex 165084d82ec5SThomas Gleixner * (debugging) state. Concurrent operations on this rt_mutex are not 165184d82ec5SThomas Gleixner * possible because it belongs to the pi_state which is about to be freed 165284d82ec5SThomas Gleixner * and it is not longer visible to other tasks. 16531696a8beSPeter Zijlstra */ 16541696a8beSPeter Zijlstra void rt_mutex_proxy_unlock(struct rt_mutex *lock, 16551696a8beSPeter Zijlstra struct task_struct *proxy_owner) 16561696a8beSPeter Zijlstra { 16571696a8beSPeter Zijlstra debug_rt_mutex_proxy_unlock(lock); 16581696a8beSPeter Zijlstra rt_mutex_set_owner(lock, NULL); 16591696a8beSPeter Zijlstra } 16601696a8beSPeter Zijlstra 166156222b21SPeter Zijlstra int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, 166256222b21SPeter Zijlstra struct rt_mutex_waiter *waiter, 166356222b21SPeter Zijlstra struct task_struct *task) 166456222b21SPeter Zijlstra { 166556222b21SPeter Zijlstra int ret; 166656222b21SPeter Zijlstra 166756222b21SPeter Zijlstra if (try_to_take_rt_mutex(lock, task, NULL)) 166856222b21SPeter Zijlstra return 1; 166956222b21SPeter Zijlstra 167056222b21SPeter Zijlstra /* We enforce deadlock detection for futexes */ 167156222b21SPeter Zijlstra ret = task_blocks_on_rt_mutex(lock, waiter, task, 167256222b21SPeter Zijlstra RT_MUTEX_FULL_CHAINWALK); 167356222b21SPeter Zijlstra 167456222b21SPeter Zijlstra if (ret && !rt_mutex_owner(lock)) { 167556222b21SPeter Zijlstra /* 167656222b21SPeter Zijlstra * Reset the return value. We might have 167756222b21SPeter Zijlstra * returned with -EDEADLK and the owner 167856222b21SPeter Zijlstra * released the lock while we were walking the 167956222b21SPeter Zijlstra * pi chain. Let the waiter sort it out. 168056222b21SPeter Zijlstra */ 168156222b21SPeter Zijlstra ret = 0; 168256222b21SPeter Zijlstra } 168356222b21SPeter Zijlstra 168456222b21SPeter Zijlstra if (unlikely(ret)) 168556222b21SPeter Zijlstra remove_waiter(lock, waiter); 168656222b21SPeter Zijlstra 168756222b21SPeter Zijlstra debug_rt_mutex_print_deadlock(waiter); 168856222b21SPeter Zijlstra 168956222b21SPeter Zijlstra return ret; 169056222b21SPeter Zijlstra } 169156222b21SPeter Zijlstra 16921696a8beSPeter Zijlstra /** 16931696a8beSPeter Zijlstra * rt_mutex_start_proxy_lock() - Start lock acquisition for another task 16941696a8beSPeter Zijlstra * @lock: the rt_mutex to take 16951696a8beSPeter Zijlstra * @waiter: the pre-initialized rt_mutex_waiter 16961696a8beSPeter Zijlstra * @task: the task to prepare 16971696a8beSPeter Zijlstra * 16981696a8beSPeter Zijlstra * Returns: 16991696a8beSPeter Zijlstra * 0 - task blocked on lock 17001696a8beSPeter Zijlstra * 1 - acquired the lock for task, caller should wake it up 17011696a8beSPeter Zijlstra * <0 - error 17021696a8beSPeter Zijlstra * 17031696a8beSPeter Zijlstra * Special API call for FUTEX_REQUEUE_PI support. 17041696a8beSPeter Zijlstra */ 17051696a8beSPeter Zijlstra int rt_mutex_start_proxy_lock(struct rt_mutex *lock, 17061696a8beSPeter Zijlstra struct rt_mutex_waiter *waiter, 1707c051b21fSThomas Gleixner struct task_struct *task) 17081696a8beSPeter Zijlstra { 17091696a8beSPeter Zijlstra int ret; 17101696a8beSPeter Zijlstra 1711b4abf910SThomas Gleixner raw_spin_lock_irq(&lock->wait_lock); 171256222b21SPeter Zijlstra ret = __rt_mutex_start_proxy_lock(lock, waiter, task); 1713b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 17141696a8beSPeter Zijlstra 17151696a8beSPeter Zijlstra return ret; 17161696a8beSPeter Zijlstra } 17171696a8beSPeter Zijlstra 17181696a8beSPeter Zijlstra /** 17191696a8beSPeter Zijlstra * rt_mutex_next_owner - return the next owner of the lock 17201696a8beSPeter Zijlstra * 17211696a8beSPeter Zijlstra * @lock: the rt lock query 17221696a8beSPeter Zijlstra * 17231696a8beSPeter Zijlstra * Returns the next owner of the lock or NULL 17241696a8beSPeter Zijlstra * 17251696a8beSPeter Zijlstra * Caller has to serialize against other accessors to the lock 17261696a8beSPeter Zijlstra * itself. 17271696a8beSPeter Zijlstra * 17281696a8beSPeter Zijlstra * Special API call for PI-futex support 17291696a8beSPeter Zijlstra */ 17301696a8beSPeter Zijlstra struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock) 17311696a8beSPeter Zijlstra { 17321696a8beSPeter Zijlstra if (!rt_mutex_has_waiters(lock)) 17331696a8beSPeter Zijlstra return NULL; 17341696a8beSPeter Zijlstra 17351696a8beSPeter Zijlstra return rt_mutex_top_waiter(lock)->task; 17361696a8beSPeter Zijlstra } 17371696a8beSPeter Zijlstra 17381696a8beSPeter Zijlstra /** 173938d589f2SPeter Zijlstra * rt_mutex_wait_proxy_lock() - Wait for lock acquisition 17401696a8beSPeter Zijlstra * @lock: the rt_mutex we were woken on 17411696a8beSPeter Zijlstra * @to: the timeout, null if none. hrtimer should already have 17421696a8beSPeter Zijlstra * been started. 17431696a8beSPeter Zijlstra * @waiter: the pre-initialized rt_mutex_waiter 17441696a8beSPeter Zijlstra * 174538d589f2SPeter Zijlstra * Wait for the the lock acquisition started on our behalf by 174638d589f2SPeter Zijlstra * rt_mutex_start_proxy_lock(). Upon failure, the caller must call 174738d589f2SPeter Zijlstra * rt_mutex_cleanup_proxy_lock(). 17481696a8beSPeter Zijlstra * 17491696a8beSPeter Zijlstra * Returns: 17501696a8beSPeter Zijlstra * 0 - success 1751c051b21fSThomas Gleixner * <0 - error, one of -EINTR, -ETIMEDOUT 17521696a8beSPeter Zijlstra * 175338d589f2SPeter Zijlstra * Special API call for PI-futex support 17541696a8beSPeter Zijlstra */ 175538d589f2SPeter Zijlstra int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, 17561696a8beSPeter Zijlstra struct hrtimer_sleeper *to, 1757c051b21fSThomas Gleixner struct rt_mutex_waiter *waiter) 17581696a8beSPeter Zijlstra { 17591696a8beSPeter Zijlstra int ret; 17601696a8beSPeter Zijlstra 1761b4abf910SThomas Gleixner raw_spin_lock_irq(&lock->wait_lock); 17621696a8beSPeter Zijlstra 17631696a8beSPeter Zijlstra set_current_state(TASK_INTERRUPTIBLE); 17641696a8beSPeter Zijlstra 1765afffc6c1SDavidlohr Bueso /* sleep on the mutex */ 17661696a8beSPeter Zijlstra ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); 17671696a8beSPeter Zijlstra 1768b4abf910SThomas Gleixner raw_spin_unlock_irq(&lock->wait_lock); 17691696a8beSPeter Zijlstra 17701696a8beSPeter Zijlstra return ret; 17711696a8beSPeter Zijlstra } 177238d589f2SPeter Zijlstra 177338d589f2SPeter Zijlstra /** 177438d589f2SPeter Zijlstra * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition 177538d589f2SPeter Zijlstra * @lock: the rt_mutex we were woken on 177638d589f2SPeter Zijlstra * @waiter: the pre-initialized rt_mutex_waiter 177738d589f2SPeter Zijlstra * 177838d589f2SPeter Zijlstra * Attempt to clean up after a failed rt_mutex_wait_proxy_lock(). 177938d589f2SPeter Zijlstra * 178038d589f2SPeter Zijlstra * Unless we acquired the lock; we're still enqueued on the wait-list and can 178138d589f2SPeter Zijlstra * in fact still be granted ownership until we're removed. Therefore we can 178238d589f2SPeter Zijlstra * find we are in fact the owner and must disregard the 178338d589f2SPeter Zijlstra * rt_mutex_wait_proxy_lock() failure. 178438d589f2SPeter Zijlstra * 178538d589f2SPeter Zijlstra * Returns: 178638d589f2SPeter Zijlstra * true - did the cleanup, we done. 178738d589f2SPeter Zijlstra * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned, 178838d589f2SPeter Zijlstra * caller should disregards its return value. 178938d589f2SPeter Zijlstra * 179038d589f2SPeter Zijlstra * Special API call for PI-futex support 179138d589f2SPeter Zijlstra */ 179238d589f2SPeter Zijlstra bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, 179338d589f2SPeter Zijlstra struct rt_mutex_waiter *waiter) 179438d589f2SPeter Zijlstra { 179538d589f2SPeter Zijlstra bool cleanup = false; 179638d589f2SPeter Zijlstra 179738d589f2SPeter Zijlstra raw_spin_lock_irq(&lock->wait_lock); 179838d589f2SPeter Zijlstra /* 179938d589f2SPeter Zijlstra * Unless we're the owner; we're still enqueued on the wait_list. 180038d589f2SPeter Zijlstra * So check if we became owner, if not, take us off the wait_list. 180138d589f2SPeter Zijlstra */ 180238d589f2SPeter Zijlstra if (rt_mutex_owner(lock) != current) { 180338d589f2SPeter Zijlstra remove_waiter(lock, waiter); 180438d589f2SPeter Zijlstra fixup_rt_mutex_waiters(lock); 180538d589f2SPeter Zijlstra cleanup = true; 180638d589f2SPeter Zijlstra } 1807cfafcd11SPeter Zijlstra 1808cfafcd11SPeter Zijlstra /* 1809cfafcd11SPeter Zijlstra * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might 1810cfafcd11SPeter Zijlstra * have to fix that up. 1811cfafcd11SPeter Zijlstra */ 1812cfafcd11SPeter Zijlstra fixup_rt_mutex_waiters(lock); 1813cfafcd11SPeter Zijlstra 181438d589f2SPeter Zijlstra raw_spin_unlock_irq(&lock->wait_lock); 181538d589f2SPeter Zijlstra 181638d589f2SPeter Zijlstra return cleanup; 181738d589f2SPeter Zijlstra } 1818