1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2022-2023 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_trans_resv.h" 11 #include "xfs_mount.h" 12 #include "xfs_ag.h" 13 #include "xfs_trace.h" 14 15 /* 16 * Use a static key here to reduce the overhead of xfs_drain_rele. If the 17 * compiler supports jump labels, the static branch will be replaced by a nop 18 * sled when there are no xfs_drain_wait callers. Online fsck is currently 19 * the only caller, so this is a reasonable tradeoff. 20 * 21 * Note: Patching the kernel code requires taking the cpu hotplug lock. Other 22 * parts of the kernel allocate memory with that lock held, which means that 23 * XFS callers cannot hold any locks that might be used by memory reclaim or 24 * writeback when calling the static_branch_{inc,dec} functions. 25 */ 26 static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate); 27 28 void 29 xfs_drain_wait_disable(void) 30 { 31 static_branch_dec(&xfs_drain_waiter_gate); 32 } 33 34 void 35 xfs_drain_wait_enable(void) 36 { 37 static_branch_inc(&xfs_drain_waiter_gate); 38 } 39 40 void 41 xfs_defer_drain_init( 42 struct xfs_defer_drain *dr) 43 { 44 atomic_set(&dr->dr_count, 0); 45 init_waitqueue_head(&dr->dr_waiters); 46 } 47 48 void 49 xfs_defer_drain_free(struct xfs_defer_drain *dr) 50 { 51 ASSERT(atomic_read(&dr->dr_count) == 0); 52 } 53 54 /* Increase the pending intent count. */ 55 static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr) 56 { 57 atomic_inc(&dr->dr_count); 58 } 59 60 static inline bool has_waiters(struct wait_queue_head *wq_head) 61 { 62 /* 63 * This memory barrier is paired with the one in set_current_state on 64 * the waiting side. 65 */ 66 smp_mb__after_atomic(); 67 return waitqueue_active(wq_head); 68 } 69 70 /* Decrease the pending intent count, and wake any waiters, if appropriate. */ 71 static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr) 72 { 73 if (atomic_dec_and_test(&dr->dr_count) && 74 static_branch_unlikely(&xfs_drain_waiter_gate) && 75 has_waiters(&dr->dr_waiters)) 76 wake_up(&dr->dr_waiters); 77 } 78 79 /* Are there intents pending? */ 80 static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr) 81 { 82 return atomic_read(&dr->dr_count) > 0; 83 } 84 85 /* 86 * Wait for the pending intent count for a drain to hit zero. 87 * 88 * Callers must not hold any locks that would prevent intents from being 89 * finished. 90 */ 91 static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr) 92 { 93 return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr)); 94 } 95 96 /* 97 * Get a passive reference to the group that contains a fsbno and declare an 98 * intent to update its metadata. 99 * 100 * Other threads that need exclusive access can decide to back off if they see 101 * declared intentions. 102 */ 103 struct xfs_group * 104 xfs_group_intent_get( 105 struct xfs_mount *mp, 106 xfs_fsblock_t fsbno, 107 enum xfs_group_type type) 108 { 109 struct xfs_group *xg; 110 111 xg = xfs_group_get_by_fsb(mp, fsbno, type); 112 if (!xg) 113 return NULL; 114 trace_xfs_group_intent_hold(xg, __return_address); 115 xfs_defer_drain_grab(&xg->xg_intents_drain); 116 return xg; 117 } 118 119 /* 120 * Release our intent to update this groups metadata, and then release our 121 * passive ref to it. 122 */ 123 void 124 xfs_group_intent_put( 125 struct xfs_group *xg) 126 { 127 trace_xfs_group_intent_rele(xg, __return_address); 128 xfs_defer_drain_rele(&xg->xg_intents_drain); 129 xfs_group_put(xg); 130 } 131 132 /* 133 * Wait for the intent update count for this AG to hit zero. 134 * Callers must not hold any AG header buffers. 135 */ 136 int 137 xfs_group_intent_drain( 138 struct xfs_group *xg) 139 { 140 trace_xfs_group_wait_intents(xg, __return_address); 141 return xfs_defer_drain_wait(&xg->xg_intents_drain); 142 } 143 144 /* 145 * Has anyone declared an intent to update this group? 146 */ 147 bool 148 xfs_group_intent_busy( 149 struct xfs_group *xg) 150 { 151 return xfs_defer_drain_busy(&xg->xg_intents_drain); 152 } 153