1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021 Facebook 3 */ 4 5 #ifndef __MMAP_UNLOCK_WORK_H__ 6 #define __MMAP_UNLOCK_WORK_H__ 7 #include <linux/atomic.h> 8 #include <linux/err.h> 9 #include <linux/irq_work.h> 10 11 /* irq_work to run mmap_read_unlock() in irq_work */ 12 struct mmap_unlock_irq_work { 13 struct irq_work irq_work; 14 struct mm_struct *mm; 15 atomic_t active; 16 }; 17 18 DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); 19 20 /* 21 * We cannot do mmap_read_unlock() when the irq is disabled, because of 22 * risk to deadlock with rq_lock. To look up vma when the irqs are 23 * disabled, we need to run mmap_read_unlock() in irq_work. We use a 24 * percpu variable to do the irq_work. The active flag reserves the slot 25 * before mmap_read_trylock() and until the irq_work callback consumes mm. 26 */ 27 static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) 28 { 29 struct mmap_unlock_irq_work *work; 30 31 if (!irqs_disabled()) 32 return NULL; 33 34 /* 35 * PREEMPT_RT does not allow to trylock mmap sem in interrupt 36 * disabled context. Force the fallback code. 37 */ 38 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 39 return ERR_PTR(-EBUSY); 40 41 work = this_cpu_ptr(&mmap_unlock_work); 42 if (irq_work_is_busy(&work->irq_work) || 43 atomic_cmpxchg_acquire(&work->active, 0, 1)) 44 return ERR_PTR(-EBUSY); 45 46 return work; 47 } 48 49 static inline void 50 bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) 51 { 52 if (work) 53 atomic_set_release(&work->active, 0); 54 } 55 56 static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) 57 { 58 if (!work) { 59 mmap_read_unlock(mm); 60 } else { 61 work->mm = mm; 62 63 /* The lock will be released once we're out of interrupt 64 * context. Tell lockdep that we've released it now so 65 * it doesn't complain that we forgot to release it. 66 */ 67 rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_); 68 irq_work_queue(&work->irq_work); 69 } 70 } 71 72 #endif /* __MMAP_UNLOCK_WORK_H__ */ 73