xref: /linux/fs/btrfs/locking.c (revision ebf68996de0ab250c5d520eb2291ab65643e9a1e) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2008 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/pagemap.h>
8 #include <linux/spinlock.h>
9 #include <linux/page-flags.h>
10 #include <asm/bug.h>
11 #include "ctree.h"
12 #include "extent_io.h"
13 #include "locking.h"
14 
15 #ifdef CONFIG_BTRFS_DEBUG
16 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
17 {
18 	WARN_ON(atomic_read(&eb->spinning_writers));
19 	atomic_inc(&eb->spinning_writers);
20 }
21 
22 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
23 {
24 	WARN_ON(atomic_read(&eb->spinning_writers) != 1);
25 	atomic_dec(&eb->spinning_writers);
26 }
27 
28 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
29 {
30 	WARN_ON(atomic_read(&eb->spinning_writers));
31 }
32 
33 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
34 {
35 	atomic_inc(&eb->spinning_readers);
36 }
37 
38 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
39 {
40 	WARN_ON(atomic_read(&eb->spinning_readers) == 0);
41 	atomic_dec(&eb->spinning_readers);
42 }
43 
44 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
45 {
46 	atomic_inc(&eb->read_locks);
47 }
48 
49 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
50 {
51 	atomic_dec(&eb->read_locks);
52 }
53 
54 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
55 {
56 	BUG_ON(!atomic_read(&eb->read_locks));
57 }
58 
59 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)
60 {
61 	atomic_inc(&eb->write_locks);
62 }
63 
64 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)
65 {
66 	atomic_dec(&eb->write_locks);
67 }
68 
69 void btrfs_assert_tree_locked(struct extent_buffer *eb)
70 {
71 	BUG_ON(!atomic_read(&eb->write_locks));
72 }
73 
74 #else
75 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
76 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
77 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
78 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
79 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
80 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
81 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
82 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
83 void btrfs_assert_tree_locked(struct extent_buffer *eb) { }
84 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }
85 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }
86 #endif
87 
88 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
89 {
90 	trace_btrfs_set_lock_blocking_read(eb);
91 	/*
92 	 * No lock is required.  The lock owner may change if we have a read
93 	 * lock, but it won't change to or away from us.  If we have the write
94 	 * lock, we are the owner and it'll never change.
95 	 */
96 	if (eb->lock_nested && current->pid == eb->lock_owner)
97 		return;
98 	btrfs_assert_tree_read_locked(eb);
99 	atomic_inc(&eb->blocking_readers);
100 	btrfs_assert_spinning_readers_put(eb);
101 	read_unlock(&eb->lock);
102 }
103 
104 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
105 {
106 	trace_btrfs_set_lock_blocking_write(eb);
107 	/*
108 	 * No lock is required.  The lock owner may change if we have a read
109 	 * lock, but it won't change to or away from us.  If we have the write
110 	 * lock, we are the owner and it'll never change.
111 	 */
112 	if (eb->lock_nested && current->pid == eb->lock_owner)
113 		return;
114 	if (atomic_read(&eb->blocking_writers) == 0) {
115 		btrfs_assert_spinning_writers_put(eb);
116 		btrfs_assert_tree_locked(eb);
117 		atomic_inc(&eb->blocking_writers);
118 		write_unlock(&eb->lock);
119 	}
120 }
121 
122 void btrfs_clear_lock_blocking_read(struct extent_buffer *eb)
123 {
124 	trace_btrfs_clear_lock_blocking_read(eb);
125 	/*
126 	 * No lock is required.  The lock owner may change if we have a read
127 	 * lock, but it won't change to or away from us.  If we have the write
128 	 * lock, we are the owner and it'll never change.
129 	 */
130 	if (eb->lock_nested && current->pid == eb->lock_owner)
131 		return;
132 	BUG_ON(atomic_read(&eb->blocking_readers) == 0);
133 	read_lock(&eb->lock);
134 	btrfs_assert_spinning_readers_get(eb);
135 	/* atomic_dec_and_test implies a barrier */
136 	if (atomic_dec_and_test(&eb->blocking_readers))
137 		cond_wake_up_nomb(&eb->read_lock_wq);
138 }
139 
140 void btrfs_clear_lock_blocking_write(struct extent_buffer *eb)
141 {
142 	trace_btrfs_clear_lock_blocking_write(eb);
143 	/*
144 	 * no lock is required.  The lock owner may change if
145 	 * we have a read lock, but it won't change to or away
146 	 * from us.  If we have the write lock, we are the owner
147 	 * and it'll never change.
148 	 */
149 	if (eb->lock_nested && current->pid == eb->lock_owner)
150 		return;
151 	BUG_ON(atomic_read(&eb->blocking_writers) != 1);
152 	write_lock(&eb->lock);
153 	btrfs_assert_spinning_writers_get(eb);
154 	/* atomic_dec_and_test implies a barrier */
155 	if (atomic_dec_and_test(&eb->blocking_writers))
156 		cond_wake_up_nomb(&eb->write_lock_wq);
157 }
158 
159 /*
160  * take a spinning read lock.  This will wait for any blocking
161  * writers
162  */
163 void btrfs_tree_read_lock(struct extent_buffer *eb)
164 {
165 	u64 start_ns = 0;
166 
167 	if (trace_btrfs_tree_read_lock_enabled())
168 		start_ns = ktime_get_ns();
169 again:
170 	BUG_ON(!atomic_read(&eb->blocking_writers) &&
171 	       current->pid == eb->lock_owner);
172 
173 	read_lock(&eb->lock);
174 	if (atomic_read(&eb->blocking_writers) &&
175 	    current->pid == eb->lock_owner) {
176 		/*
177 		 * This extent is already write-locked by our thread. We allow
178 		 * an additional read lock to be added because it's for the same
179 		 * thread. btrfs_find_all_roots() depends on this as it may be
180 		 * called on a partly (write-)locked tree.
181 		 */
182 		BUG_ON(eb->lock_nested);
183 		eb->lock_nested = true;
184 		read_unlock(&eb->lock);
185 		trace_btrfs_tree_read_lock(eb, start_ns);
186 		return;
187 	}
188 	if (atomic_read(&eb->blocking_writers)) {
189 		read_unlock(&eb->lock);
190 		wait_event(eb->write_lock_wq,
191 			   atomic_read(&eb->blocking_writers) == 0);
192 		goto again;
193 	}
194 	btrfs_assert_tree_read_locks_get(eb);
195 	btrfs_assert_spinning_readers_get(eb);
196 	trace_btrfs_tree_read_lock(eb, start_ns);
197 }
198 
199 /*
200  * take a spinning read lock.
201  * returns 1 if we get the read lock and 0 if we don't
202  * this won't wait for blocking writers
203  */
204 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
205 {
206 	if (atomic_read(&eb->blocking_writers))
207 		return 0;
208 
209 	read_lock(&eb->lock);
210 	if (atomic_read(&eb->blocking_writers)) {
211 		read_unlock(&eb->lock);
212 		return 0;
213 	}
214 	btrfs_assert_tree_read_locks_get(eb);
215 	btrfs_assert_spinning_readers_get(eb);
216 	trace_btrfs_tree_read_lock_atomic(eb);
217 	return 1;
218 }
219 
220 /*
221  * returns 1 if we get the read lock and 0 if we don't
222  * this won't wait for blocking writers
223  */
224 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
225 {
226 	if (atomic_read(&eb->blocking_writers))
227 		return 0;
228 
229 	if (!read_trylock(&eb->lock))
230 		return 0;
231 
232 	if (atomic_read(&eb->blocking_writers)) {
233 		read_unlock(&eb->lock);
234 		return 0;
235 	}
236 	btrfs_assert_tree_read_locks_get(eb);
237 	btrfs_assert_spinning_readers_get(eb);
238 	trace_btrfs_try_tree_read_lock(eb);
239 	return 1;
240 }
241 
242 /*
243  * returns 1 if we get the read lock and 0 if we don't
244  * this won't wait for blocking writers or readers
245  */
246 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
247 {
248 	if (atomic_read(&eb->blocking_writers) ||
249 	    atomic_read(&eb->blocking_readers))
250 		return 0;
251 
252 	write_lock(&eb->lock);
253 	if (atomic_read(&eb->blocking_writers) ||
254 	    atomic_read(&eb->blocking_readers)) {
255 		write_unlock(&eb->lock);
256 		return 0;
257 	}
258 	btrfs_assert_tree_write_locks_get(eb);
259 	btrfs_assert_spinning_writers_get(eb);
260 	eb->lock_owner = current->pid;
261 	trace_btrfs_try_tree_write_lock(eb);
262 	return 1;
263 }
264 
265 /*
266  * drop a spinning read lock
267  */
268 void btrfs_tree_read_unlock(struct extent_buffer *eb)
269 {
270 	trace_btrfs_tree_read_unlock(eb);
271 	/*
272 	 * if we're nested, we have the write lock.  No new locking
273 	 * is needed as long as we are the lock owner.
274 	 * The write unlock will do a barrier for us, and the lock_nested
275 	 * field only matters to the lock owner.
276 	 */
277 	if (eb->lock_nested && current->pid == eb->lock_owner) {
278 		eb->lock_nested = false;
279 		return;
280 	}
281 	btrfs_assert_tree_read_locked(eb);
282 	btrfs_assert_spinning_readers_put(eb);
283 	btrfs_assert_tree_read_locks_put(eb);
284 	read_unlock(&eb->lock);
285 }
286 
287 /*
288  * drop a blocking read lock
289  */
290 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
291 {
292 	trace_btrfs_tree_read_unlock_blocking(eb);
293 	/*
294 	 * if we're nested, we have the write lock.  No new locking
295 	 * is needed as long as we are the lock owner.
296 	 * The write unlock will do a barrier for us, and the lock_nested
297 	 * field only matters to the lock owner.
298 	 */
299 	if (eb->lock_nested && current->pid == eb->lock_owner) {
300 		eb->lock_nested = false;
301 		return;
302 	}
303 	btrfs_assert_tree_read_locked(eb);
304 	WARN_ON(atomic_read(&eb->blocking_readers) == 0);
305 	/* atomic_dec_and_test implies a barrier */
306 	if (atomic_dec_and_test(&eb->blocking_readers))
307 		cond_wake_up_nomb(&eb->read_lock_wq);
308 	btrfs_assert_tree_read_locks_put(eb);
309 }
310 
311 /*
312  * take a spinning write lock.  This will wait for both
313  * blocking readers or writers
314  */
315 void btrfs_tree_lock(struct extent_buffer *eb)
316 {
317 	u64 start_ns = 0;
318 
319 	if (trace_btrfs_tree_lock_enabled())
320 		start_ns = ktime_get_ns();
321 
322 	WARN_ON(eb->lock_owner == current->pid);
323 again:
324 	wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
325 	wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
326 	write_lock(&eb->lock);
327 	if (atomic_read(&eb->blocking_readers) ||
328 	    atomic_read(&eb->blocking_writers)) {
329 		write_unlock(&eb->lock);
330 		goto again;
331 	}
332 	btrfs_assert_spinning_writers_get(eb);
333 	btrfs_assert_tree_write_locks_get(eb);
334 	eb->lock_owner = current->pid;
335 	trace_btrfs_tree_lock(eb, start_ns);
336 }
337 
338 /*
339  * drop a spinning or a blocking write lock.
340  */
341 void btrfs_tree_unlock(struct extent_buffer *eb)
342 {
343 	int blockers = atomic_read(&eb->blocking_writers);
344 
345 	BUG_ON(blockers > 1);
346 
347 	btrfs_assert_tree_locked(eb);
348 	trace_btrfs_tree_unlock(eb);
349 	eb->lock_owner = 0;
350 	btrfs_assert_tree_write_locks_put(eb);
351 
352 	if (blockers) {
353 		btrfs_assert_no_spinning_writers(eb);
354 		atomic_dec(&eb->blocking_writers);
355 		/* Use the lighter barrier after atomic */
356 		smp_mb__after_atomic();
357 		cond_wake_up_nomb(&eb->write_lock_wq);
358 	} else {
359 		btrfs_assert_spinning_writers_put(eb);
360 		write_unlock(&eb->lock);
361 	}
362 }
363