xref: /linux/fs/xfs/libxfs/xfs_defer.c (revision 9208c05f9fdfd927ea160b97dfef3c379049fff2)
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (C) 2016 Oracle.  All Rights Reserved.
4  * Author: Darrick J. Wong <darrick.wong@oracle.com>
5  */
6 #include "xfs.h"
7 #include "xfs_fs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_defer.h"
14 #include "xfs_trans.h"
15 #include "xfs_trans_priv.h"
16 #include "xfs_buf_item.h"
17 #include "xfs_inode.h"
18 #include "xfs_inode_item.h"
19 #include "xfs_trace.h"
20 #include "xfs_icache.h"
21 #include "xfs_log.h"
22 #include "xfs_log_priv.h"
23 #include "xfs_rmap.h"
24 #include "xfs_refcount.h"
25 #include "xfs_bmap.h"
26 #include "xfs_alloc.h"
27 #include "xfs_buf.h"
28 #include "xfs_da_format.h"
29 #include "xfs_da_btree.h"
30 #include "xfs_attr.h"
31 #include "xfs_exchmaps.h"
32 
33 static struct kmem_cache	*xfs_defer_pending_cache;
34 
35 /*
36  * Deferred Operations in XFS
37  *
38  * Due to the way locking rules work in XFS, certain transactions (block
39  * mapping and unmapping, typically) have permanent reservations so that
40  * we can roll the transaction to adhere to AG locking order rules and
41  * to unlock buffers between metadata updates.  Prior to rmap/reflink,
42  * the mapping code had a mechanism to perform these deferrals for
43  * extents that were going to be freed; this code makes that facility
44  * more generic.
45  *
46  * When adding the reverse mapping and reflink features, it became
47  * necessary to perform complex remapping multi-transactions to comply
48  * with AG locking order rules, and to be able to spread a single
49  * refcount update operation (an operation on an n-block extent can
50  * update as many as n records!) among multiple transactions.  XFS can
51  * roll a transaction to facilitate this, but using this facility
52  * requires us to log "intent" items in case log recovery needs to
53  * redo the operation, and to log "done" items to indicate that redo
54  * is not necessary.
55  *
56  * Deferred work is tracked in xfs_defer_pending items.  Each pending
57  * item tracks one type of deferred work.  Incoming work items (which
58  * have not yet had an intent logged) are attached to a pending item
59  * on the dop_intake list, where they wait for the caller to finish
60  * the deferred operations.
61  *
62  * Finishing a set of deferred operations is an involved process.  To
63  * start, we define "rolling a deferred-op transaction" as follows:
64  *
65  * > For each xfs_defer_pending item on the dop_intake list,
66  *   - Sort the work items in AG order.  XFS locking
67  *     order rules require us to lock buffers in AG order.
68  *   - Create a log intent item for that type.
69  *   - Attach it to the pending item.
70  *   - Move the pending item from the dop_intake list to the
71  *     dop_pending list.
72  * > Roll the transaction.
73  *
74  * NOTE: To avoid exceeding the transaction reservation, we limit the
75  * number of items that we attach to a given xfs_defer_pending.
76  *
77  * The actual finishing process looks like this:
78  *
79  * > For each xfs_defer_pending in the dop_pending list,
80  *   - Roll the deferred-op transaction as above.
81  *   - Create a log done item for that type, and attach it to the
82  *     log intent item.
83  *   - For each work item attached to the log intent item,
84  *     * Perform the described action.
85  *     * Attach the work item to the log done item.
86  *     * If the result of doing the work was -EAGAIN, ->finish work
87  *       wants a new transaction.  See the "Requesting a Fresh
88  *       Transaction while Finishing Deferred Work" section below for
89  *       details.
90  *
91  * The key here is that we must log an intent item for all pending
92  * work items every time we roll the transaction, and that we must log
93  * a done item as soon as the work is completed.  With this mechanism
94  * we can perform complex remapping operations, chaining intent items
95  * as needed.
96  *
97  * Requesting a Fresh Transaction while Finishing Deferred Work
98  *
99  * If ->finish_item decides that it needs a fresh transaction to
100  * finish the work, it must ask its caller (xfs_defer_finish) for a
101  * continuation.  The most likely cause of this circumstance are the
102  * refcount adjust functions deciding that they've logged enough items
103  * to be at risk of exceeding the transaction reservation.
104  *
105  * To get a fresh transaction, we want to log the existing log done
106  * item to prevent the log intent item from replaying, immediately log
107  * a new log intent item with the unfinished work items, roll the
108  * transaction, and re-call ->finish_item wherever it left off.  The
109  * log done item and the new log intent item must be in the same
110  * transaction or atomicity cannot be guaranteed; defer_finish ensures
111  * that this happens.
112  *
113  * This requires some coordination between ->finish_item and
114  * defer_finish.  Upon deciding to request a new transaction,
115  * ->finish_item should update the current work item to reflect the
116  * unfinished work.  Next, it should reset the log done item's list
117  * count to the number of items finished, and return -EAGAIN.
118  * defer_finish sees the -EAGAIN, logs the new log intent item
119  * with the remaining work items, and leaves the xfs_defer_pending
120  * item at the head of the dop_work queue.  Then it rolls the
121  * transaction and picks up processing where it left off.  It is
122  * required that ->finish_item must be careful to leave enough
123  * transaction reservation to fit the new log intent item.
124  *
125  * This is an example of remapping the extent (E, E+B) into file X at
126  * offset A and dealing with the extent (C, C+B) already being mapped
127  * there:
128  * +-------------------------------------------------+
129  * | Unmap file X startblock C offset A length B     | t0
130  * | Intent to reduce refcount for extent (C, B)     |
131  * | Intent to remove rmap (X, C, A, B)              |
132  * | Intent to free extent (D, 1) (bmbt block)       |
133  * | Intent to map (X, A, B) at startblock E         |
134  * +-------------------------------------------------+
135  * | Map file X startblock E offset A length B       | t1
136  * | Done mapping (X, E, A, B)                       |
137  * | Intent to increase refcount for extent (E, B)   |
138  * | Intent to add rmap (X, E, A, B)                 |
139  * +-------------------------------------------------+
140  * | Reduce refcount for extent (C, B)               | t2
141  * | Done reducing refcount for extent (C, 9)        |
142  * | Intent to reduce refcount for extent (C+9, B-9) |
143  * | (ran out of space after 9 refcount updates)     |
144  * +-------------------------------------------------+
145  * | Reduce refcount for extent (C+9, B+9)           | t3
146  * | Done reducing refcount for extent (C+9, B-9)    |
147  * | Increase refcount for extent (E, B)             |
148  * | Done increasing refcount for extent (E, B)      |
149  * | Intent to free extent (C, B)                    |
150  * | Intent to free extent (F, 1) (refcountbt block) |
151  * | Intent to remove rmap (F, 1, REFC)              |
152  * +-------------------------------------------------+
153  * | Remove rmap (X, C, A, B)                        | t4
154  * | Done removing rmap (X, C, A, B)                 |
155  * | Add rmap (X, E, A, B)                           |
156  * | Done adding rmap (X, E, A, B)                   |
157  * | Remove rmap (F, 1, REFC)                        |
158  * | Done removing rmap (F, 1, REFC)                 |
159  * +-------------------------------------------------+
160  * | Free extent (C, B)                              | t5
161  * | Done freeing extent (C, B)                      |
162  * | Free extent (D, 1)                              |
163  * | Done freeing extent (D, 1)                      |
164  * | Free extent (F, 1)                              |
165  * | Done freeing extent (F, 1)                      |
166  * +-------------------------------------------------+
167  *
168  * If we should crash before t2 commits, log recovery replays
169  * the following intent items:
170  *
171  * - Intent to reduce refcount for extent (C, B)
172  * - Intent to remove rmap (X, C, A, B)
173  * - Intent to free extent (D, 1) (bmbt block)
174  * - Intent to increase refcount for extent (E, B)
175  * - Intent to add rmap (X, E, A, B)
176  *
177  * In the process of recovering, it should also generate and take care
178  * of these intent items:
179  *
180  * - Intent to free extent (C, B)
181  * - Intent to free extent (F, 1) (refcountbt block)
182  * - Intent to remove rmap (F, 1, REFC)
183  *
184  * Note that the continuation requested between t2 and t3 is likely to
185  * reoccur.
186  */
187 STATIC struct xfs_log_item *
188 xfs_defer_barrier_create_intent(
189 	struct xfs_trans		*tp,
190 	struct list_head		*items,
191 	unsigned int			count,
192 	bool				sort)
193 {
194 	return NULL;
195 }
196 
197 STATIC void
198 xfs_defer_barrier_abort_intent(
199 	struct xfs_log_item		*intent)
200 {
201 	/* empty */
202 }
203 
204 STATIC struct xfs_log_item *
205 xfs_defer_barrier_create_done(
206 	struct xfs_trans		*tp,
207 	struct xfs_log_item		*intent,
208 	unsigned int			count)
209 {
210 	return NULL;
211 }
212 
213 STATIC int
214 xfs_defer_barrier_finish_item(
215 	struct xfs_trans		*tp,
216 	struct xfs_log_item		*done,
217 	struct list_head		*item,
218 	struct xfs_btree_cur		**state)
219 {
220 	ASSERT(0);
221 	return -EFSCORRUPTED;
222 }
223 
224 STATIC void
225 xfs_defer_barrier_cancel_item(
226 	struct list_head		*item)
227 {
228 	ASSERT(0);
229 }
230 
231 static const struct xfs_defer_op_type xfs_barrier_defer_type = {
232 	.max_items	= 1,
233 	.create_intent	= xfs_defer_barrier_create_intent,
234 	.abort_intent	= xfs_defer_barrier_abort_intent,
235 	.create_done	= xfs_defer_barrier_create_done,
236 	.finish_item	= xfs_defer_barrier_finish_item,
237 	.cancel_item	= xfs_defer_barrier_cancel_item,
238 };
239 
240 /* Create a log intent done item for a log intent item. */
241 static inline void
242 xfs_defer_create_done(
243 	struct xfs_trans		*tp,
244 	struct xfs_defer_pending	*dfp)
245 {
246 	struct xfs_log_item		*lip;
247 
248 	/* If there is no log intent item, there can be no log done item. */
249 	if (!dfp->dfp_intent)
250 		return;
251 
252 	/*
253 	 * Mark the transaction dirty, even on error. This ensures the
254 	 * transaction is aborted, which:
255 	 *
256 	 * 1.) releases the log intent item and frees the log done item
257 	 * 2.) shuts down the filesystem
258 	 */
259 	tp->t_flags |= XFS_TRANS_DIRTY;
260 	lip = dfp->dfp_ops->create_done(tp, dfp->dfp_intent, dfp->dfp_count);
261 	if (!lip)
262 		return;
263 
264 	tp->t_flags |= XFS_TRANS_HAS_INTENT_DONE;
265 	xfs_trans_add_item(tp, lip);
266 	set_bit(XFS_LI_DIRTY, &lip->li_flags);
267 	dfp->dfp_done = lip;
268 }
269 
270 /*
271  * Ensure there's a log intent item associated with this deferred work item if
272  * the operation must be restarted on crash.  Returns 1 if there's a log item;
273  * 0 if there isn't; or a negative errno.
274  */
275 static int
276 xfs_defer_create_intent(
277 	struct xfs_trans		*tp,
278 	struct xfs_defer_pending	*dfp,
279 	bool				sort)
280 {
281 	struct xfs_log_item		*lip;
282 
283 	if (dfp->dfp_intent)
284 		return 1;
285 
286 	lip = dfp->dfp_ops->create_intent(tp, &dfp->dfp_work, dfp->dfp_count,
287 			sort);
288 	if (!lip)
289 		return 0;
290 	if (IS_ERR(lip))
291 		return PTR_ERR(lip);
292 
293 	tp->t_flags |= XFS_TRANS_DIRTY;
294 	xfs_trans_add_item(tp, lip);
295 	set_bit(XFS_LI_DIRTY, &lip->li_flags);
296 	dfp->dfp_intent = lip;
297 	return 1;
298 }
299 
300 /*
301  * For each pending item in the intake list, log its intent item and the
302  * associated extents, then add the entire intake list to the end of
303  * the pending list.
304  *
305  * Returns 1 if at least one log item was associated with the deferred work;
306  * 0 if there are no log items; or a negative errno.
307  */
308 static int
309 xfs_defer_create_intents(
310 	struct xfs_trans		*tp)
311 {
312 	struct xfs_defer_pending	*dfp;
313 	int				ret = 0;
314 
315 	list_for_each_entry(dfp, &tp->t_dfops, dfp_list) {
316 		int			ret2;
317 
318 		trace_xfs_defer_create_intent(tp->t_mountp, dfp);
319 		ret2 = xfs_defer_create_intent(tp, dfp, true);
320 		if (ret2 < 0)
321 			return ret2;
322 		ret |= ret2;
323 	}
324 	return ret;
325 }
326 
327 static inline void
328 xfs_defer_pending_abort(
329 	struct xfs_mount		*mp,
330 	struct xfs_defer_pending	*dfp)
331 {
332 	trace_xfs_defer_pending_abort(mp, dfp);
333 
334 	if (dfp->dfp_intent && !dfp->dfp_done) {
335 		dfp->dfp_ops->abort_intent(dfp->dfp_intent);
336 		dfp->dfp_intent = NULL;
337 	}
338 }
339 
340 static inline void
341 xfs_defer_pending_cancel_work(
342 	struct xfs_mount		*mp,
343 	struct xfs_defer_pending	*dfp)
344 {
345 	struct list_head		*pwi;
346 	struct list_head		*n;
347 
348 	trace_xfs_defer_cancel_list(mp, dfp);
349 
350 	list_del(&dfp->dfp_list);
351 	list_for_each_safe(pwi, n, &dfp->dfp_work) {
352 		list_del(pwi);
353 		dfp->dfp_count--;
354 		trace_xfs_defer_cancel_item(mp, dfp, pwi);
355 		dfp->dfp_ops->cancel_item(pwi);
356 	}
357 	ASSERT(dfp->dfp_count == 0);
358 	kmem_cache_free(xfs_defer_pending_cache, dfp);
359 }
360 
361 STATIC void
362 xfs_defer_pending_abort_list(
363 	struct xfs_mount		*mp,
364 	struct list_head		*dop_list)
365 {
366 	struct xfs_defer_pending	*dfp;
367 
368 	/* Abort intent items that don't have a done item. */
369 	list_for_each_entry(dfp, dop_list, dfp_list)
370 		xfs_defer_pending_abort(mp, dfp);
371 }
372 
373 /* Abort all the intents that were committed. */
374 STATIC void
375 xfs_defer_trans_abort(
376 	struct xfs_trans		*tp,
377 	struct list_head		*dop_pending)
378 {
379 	trace_xfs_defer_trans_abort(tp, _RET_IP_);
380 	xfs_defer_pending_abort_list(tp->t_mountp, dop_pending);
381 }
382 
383 /*
384  * Capture resources that the caller said not to release ("held") when the
385  * transaction commits.  Caller is responsible for zero-initializing @dres.
386  */
387 static int
388 xfs_defer_save_resources(
389 	struct xfs_defer_resources	*dres,
390 	struct xfs_trans		*tp)
391 {
392 	struct xfs_buf_log_item		*bli;
393 	struct xfs_inode_log_item	*ili;
394 	struct xfs_log_item		*lip;
395 
396 	BUILD_BUG_ON(NBBY * sizeof(dres->dr_ordered) < XFS_DEFER_OPS_NR_BUFS);
397 
398 	list_for_each_entry(lip, &tp->t_items, li_trans) {
399 		switch (lip->li_type) {
400 		case XFS_LI_BUF:
401 			bli = container_of(lip, struct xfs_buf_log_item,
402 					   bli_item);
403 			if (bli->bli_flags & XFS_BLI_HOLD) {
404 				if (dres->dr_bufs >= XFS_DEFER_OPS_NR_BUFS) {
405 					ASSERT(0);
406 					return -EFSCORRUPTED;
407 				}
408 				if (bli->bli_flags & XFS_BLI_ORDERED)
409 					dres->dr_ordered |=
410 							(1U << dres->dr_bufs);
411 				else
412 					xfs_trans_dirty_buf(tp, bli->bli_buf);
413 				dres->dr_bp[dres->dr_bufs++] = bli->bli_buf;
414 			}
415 			break;
416 		case XFS_LI_INODE:
417 			ili = container_of(lip, struct xfs_inode_log_item,
418 					   ili_item);
419 			if (ili->ili_lock_flags == 0) {
420 				if (dres->dr_inos >= XFS_DEFER_OPS_NR_INODES) {
421 					ASSERT(0);
422 					return -EFSCORRUPTED;
423 				}
424 				xfs_trans_log_inode(tp, ili->ili_inode,
425 						    XFS_ILOG_CORE);
426 				dres->dr_ip[dres->dr_inos++] = ili->ili_inode;
427 			}
428 			break;
429 		default:
430 			break;
431 		}
432 	}
433 
434 	return 0;
435 }
436 
437 /* Attach the held resources to the transaction. */
438 static void
439 xfs_defer_restore_resources(
440 	struct xfs_trans		*tp,
441 	struct xfs_defer_resources	*dres)
442 {
443 	unsigned short			i;
444 
445 	/* Rejoin the joined inodes. */
446 	for (i = 0; i < dres->dr_inos; i++)
447 		xfs_trans_ijoin(tp, dres->dr_ip[i], 0);
448 
449 	/* Rejoin the buffers and dirty them so the log moves forward. */
450 	for (i = 0; i < dres->dr_bufs; i++) {
451 		xfs_trans_bjoin(tp, dres->dr_bp[i]);
452 		if (dres->dr_ordered & (1U << i))
453 			xfs_trans_ordered_buf(tp, dres->dr_bp[i]);
454 		xfs_trans_bhold(tp, dres->dr_bp[i]);
455 	}
456 }
457 
458 /* Roll a transaction so we can do some deferred op processing. */
459 STATIC int
460 xfs_defer_trans_roll(
461 	struct xfs_trans		**tpp)
462 {
463 	struct xfs_defer_resources	dres = { };
464 	int				error;
465 
466 	error = xfs_defer_save_resources(&dres, *tpp);
467 	if (error)
468 		return error;
469 
470 	trace_xfs_defer_trans_roll(*tpp, _RET_IP_);
471 
472 	/*
473 	 * Roll the transaction.  Rolling always given a new transaction (even
474 	 * if committing the old one fails!) to hand back to the caller, so we
475 	 * join the held resources to the new transaction so that we always
476 	 * return with the held resources joined to @tpp, no matter what
477 	 * happened.
478 	 */
479 	error = xfs_trans_roll(tpp);
480 
481 	xfs_defer_restore_resources(*tpp, &dres);
482 
483 	if (error)
484 		trace_xfs_defer_trans_roll_error(*tpp, error);
485 	return error;
486 }
487 
488 /*
489  * Free up any items left in the list.
490  */
491 static void
492 xfs_defer_cancel_list(
493 	struct xfs_mount		*mp,
494 	struct list_head		*dop_list)
495 {
496 	struct xfs_defer_pending	*dfp;
497 	struct xfs_defer_pending	*pli;
498 
499 	/*
500 	 * Free the pending items.  Caller should already have arranged
501 	 * for the intent items to be released.
502 	 */
503 	list_for_each_entry_safe(dfp, pli, dop_list, dfp_list)
504 		xfs_defer_pending_cancel_work(mp, dfp);
505 }
506 
507 static inline void
508 xfs_defer_relog_intent(
509 	struct xfs_trans		*tp,
510 	struct xfs_defer_pending	*dfp)
511 {
512 	struct xfs_log_item		*lip;
513 
514 	xfs_defer_create_done(tp, dfp);
515 
516 	lip = dfp->dfp_ops->relog_intent(tp, dfp->dfp_intent, dfp->dfp_done);
517 	if (lip) {
518 		xfs_trans_add_item(tp, lip);
519 		set_bit(XFS_LI_DIRTY, &lip->li_flags);
520 	}
521 	dfp->dfp_done = NULL;
522 	dfp->dfp_intent = lip;
523 }
524 
525 /*
526  * Prevent a log intent item from pinning the tail of the log by logging a
527  * done item to release the intent item; and then log a new intent item.
528  * The caller should provide a fresh transaction and roll it after we're done.
529  */
530 static void
531 xfs_defer_relog(
532 	struct xfs_trans		**tpp,
533 	struct list_head		*dfops)
534 {
535 	struct xlog			*log = (*tpp)->t_mountp->m_log;
536 	struct xfs_defer_pending	*dfp;
537 	xfs_lsn_t			threshold_lsn = NULLCOMMITLSN;
538 
539 
540 	ASSERT((*tpp)->t_flags & XFS_TRANS_PERM_LOG_RES);
541 
542 	list_for_each_entry(dfp, dfops, dfp_list) {
543 		/*
544 		 * If the log intent item for this deferred op is not a part of
545 		 * the current log checkpoint, relog the intent item to keep
546 		 * the log tail moving forward.  We're ok with this being racy
547 		 * because an incorrect decision means we'll be a little slower
548 		 * at pushing the tail.
549 		 */
550 		if (dfp->dfp_intent == NULL ||
551 		    xfs_log_item_in_current_chkpt(dfp->dfp_intent))
552 			continue;
553 
554 		/*
555 		 * Figure out where we need the tail to be in order to maintain
556 		 * the minimum required free space in the log.  Only sample
557 		 * the log threshold once per call.
558 		 */
559 		if (threshold_lsn == NULLCOMMITLSN) {
560 			threshold_lsn = xfs_ail_get_push_target(log->l_ailp);
561 			if (threshold_lsn == NULLCOMMITLSN)
562 				break;
563 		}
564 		if (XFS_LSN_CMP(dfp->dfp_intent->li_lsn, threshold_lsn) >= 0)
565 			continue;
566 
567 		trace_xfs_defer_relog_intent((*tpp)->t_mountp, dfp);
568 		XFS_STATS_INC((*tpp)->t_mountp, defer_relog);
569 
570 		xfs_defer_relog_intent(*tpp, dfp);
571 	}
572 }
573 
574 /*
575  * Log an intent-done item for the first pending intent, and finish the work
576  * items.
577  */
578 int
579 xfs_defer_finish_one(
580 	struct xfs_trans		*tp,
581 	struct xfs_defer_pending	*dfp)
582 {
583 	const struct xfs_defer_op_type	*ops = dfp->dfp_ops;
584 	struct xfs_btree_cur		*state = NULL;
585 	struct list_head		*li, *n;
586 	int				error;
587 
588 	trace_xfs_defer_pending_finish(tp->t_mountp, dfp);
589 
590 	xfs_defer_create_done(tp, dfp);
591 	list_for_each_safe(li, n, &dfp->dfp_work) {
592 		list_del(li);
593 		dfp->dfp_count--;
594 		trace_xfs_defer_finish_item(tp->t_mountp, dfp, li);
595 		error = ops->finish_item(tp, dfp->dfp_done, li, &state);
596 		if (error == -EAGAIN) {
597 			int		ret;
598 
599 			/*
600 			 * Caller wants a fresh transaction; put the work item
601 			 * back on the list and log a new log intent item to
602 			 * replace the old one.  See "Requesting a Fresh
603 			 * Transaction while Finishing Deferred Work" above.
604 			 */
605 			list_add(li, &dfp->dfp_work);
606 			dfp->dfp_count++;
607 			dfp->dfp_done = NULL;
608 			dfp->dfp_intent = NULL;
609 			ret = xfs_defer_create_intent(tp, dfp, false);
610 			if (ret < 0)
611 				error = ret;
612 		}
613 
614 		if (error)
615 			goto out;
616 	}
617 
618 	/* Done with the dfp, free it. */
619 	list_del(&dfp->dfp_list);
620 	kmem_cache_free(xfs_defer_pending_cache, dfp);
621 out:
622 	if (ops->finish_cleanup)
623 		ops->finish_cleanup(tp, state, error);
624 	return error;
625 }
626 
627 /* Move all paused deferred work from @tp to @paused_list. */
628 static void
629 xfs_defer_isolate_paused(
630 	struct xfs_trans		*tp,
631 	struct list_head		*paused_list)
632 {
633 	struct xfs_defer_pending	*dfp;
634 	struct xfs_defer_pending	*pli;
635 
636 	list_for_each_entry_safe(dfp, pli, &tp->t_dfops, dfp_list) {
637 		if (!(dfp->dfp_flags & XFS_DEFER_PAUSED))
638 			continue;
639 
640 		list_move_tail(&dfp->dfp_list, paused_list);
641 		trace_xfs_defer_isolate_paused(tp->t_mountp, dfp);
642 	}
643 }
644 
645 /*
646  * Finish all the pending work.  This involves logging intent items for
647  * any work items that wandered in since the last transaction roll (if
648  * one has even happened), rolling the transaction, and finishing the
649  * work items in the first item on the logged-and-pending list.
650  *
651  * If an inode is provided, relog it to the new transaction.
652  */
653 int
654 xfs_defer_finish_noroll(
655 	struct xfs_trans		**tp)
656 {
657 	struct xfs_defer_pending	*dfp = NULL;
658 	int				error = 0;
659 	LIST_HEAD(dop_pending);
660 	LIST_HEAD(dop_paused);
661 
662 	ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES);
663 
664 	trace_xfs_defer_finish(*tp, _RET_IP_);
665 
666 	/* Until we run out of pending work to finish... */
667 	while (!list_empty(&dop_pending) || !list_empty(&(*tp)->t_dfops)) {
668 		/*
669 		 * Deferred items that are created in the process of finishing
670 		 * other deferred work items should be queued at the head of
671 		 * the pending list, which puts them ahead of the deferred work
672 		 * that was created by the caller.  This keeps the number of
673 		 * pending work items to a minimum, which decreases the amount
674 		 * of time that any one intent item can stick around in memory,
675 		 * pinning the log tail.
676 		 */
677 		int has_intents = xfs_defer_create_intents(*tp);
678 
679 		xfs_defer_isolate_paused(*tp, &dop_paused);
680 
681 		list_splice_init(&(*tp)->t_dfops, &dop_pending);
682 
683 		if (has_intents < 0) {
684 			error = has_intents;
685 			goto out_shutdown;
686 		}
687 		if (has_intents || dfp) {
688 			error = xfs_defer_trans_roll(tp);
689 			if (error)
690 				goto out_shutdown;
691 
692 			/* Relog intent items to keep the log moving. */
693 			xfs_defer_relog(tp, &dop_pending);
694 			xfs_defer_relog(tp, &dop_paused);
695 
696 			if ((*tp)->t_flags & XFS_TRANS_DIRTY) {
697 				error = xfs_defer_trans_roll(tp);
698 				if (error)
699 					goto out_shutdown;
700 			}
701 		}
702 
703 		dfp = list_first_entry_or_null(&dop_pending,
704 				struct xfs_defer_pending, dfp_list);
705 		if (!dfp)
706 			break;
707 		error = xfs_defer_finish_one(*tp, dfp);
708 		if (error && error != -EAGAIN)
709 			goto out_shutdown;
710 	}
711 
712 	/* Requeue the paused items in the outgoing transaction. */
713 	list_splice_tail_init(&dop_paused, &(*tp)->t_dfops);
714 
715 	trace_xfs_defer_finish_done(*tp, _RET_IP_);
716 	return 0;
717 
718 out_shutdown:
719 	list_splice_tail_init(&dop_paused, &dop_pending);
720 	xfs_defer_trans_abort(*tp, &dop_pending);
721 	xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE);
722 	trace_xfs_defer_finish_error(*tp, error);
723 	xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending);
724 	xfs_defer_cancel(*tp);
725 	return error;
726 }
727 
728 int
729 xfs_defer_finish(
730 	struct xfs_trans	**tp)
731 {
732 #ifdef DEBUG
733 	struct xfs_defer_pending *dfp;
734 #endif
735 	int			error;
736 
737 	/*
738 	 * Finish and roll the transaction once more to avoid returning to the
739 	 * caller with a dirty transaction.
740 	 */
741 	error = xfs_defer_finish_noroll(tp);
742 	if (error)
743 		return error;
744 	if ((*tp)->t_flags & XFS_TRANS_DIRTY) {
745 		error = xfs_defer_trans_roll(tp);
746 		if (error) {
747 			xfs_force_shutdown((*tp)->t_mountp,
748 					   SHUTDOWN_CORRUPT_INCORE);
749 			return error;
750 		}
751 	}
752 
753 	/* Reset LOWMODE now that we've finished all the dfops. */
754 #ifdef DEBUG
755 	list_for_each_entry(dfp, &(*tp)->t_dfops, dfp_list)
756 		ASSERT(dfp->dfp_flags & XFS_DEFER_PAUSED);
757 #endif
758 	(*tp)->t_flags &= ~XFS_TRANS_LOWMODE;
759 	return 0;
760 }
761 
762 void
763 xfs_defer_cancel(
764 	struct xfs_trans	*tp)
765 {
766 	struct xfs_mount	*mp = tp->t_mountp;
767 
768 	trace_xfs_defer_cancel(tp, _RET_IP_);
769 	xfs_defer_trans_abort(tp, &tp->t_dfops);
770 	xfs_defer_cancel_list(mp, &tp->t_dfops);
771 }
772 
773 /*
774  * Return the last pending work item attached to this transaction if it matches
775  * the deferred op type.
776  */
777 static inline struct xfs_defer_pending *
778 xfs_defer_find_last(
779 	struct xfs_trans		*tp,
780 	const struct xfs_defer_op_type	*ops)
781 {
782 	struct xfs_defer_pending	*dfp = NULL;
783 
784 	/* No dfops at all? */
785 	if (list_empty(&tp->t_dfops))
786 		return NULL;
787 
788 	dfp = list_last_entry(&tp->t_dfops, struct xfs_defer_pending,
789 			dfp_list);
790 
791 	/* Wrong type? */
792 	if (dfp->dfp_ops != ops)
793 		return NULL;
794 	return dfp;
795 }
796 
797 /*
798  * Decide if we can add a deferred work item to the last dfops item attached
799  * to the transaction.
800  */
801 static inline bool
802 xfs_defer_can_append(
803 	struct xfs_defer_pending	*dfp,
804 	const struct xfs_defer_op_type	*ops)
805 {
806 	/* Already logged? */
807 	if (dfp->dfp_intent)
808 		return false;
809 
810 	/* Paused items cannot absorb more work */
811 	if (dfp->dfp_flags & XFS_DEFER_PAUSED)
812 		return NULL;
813 
814 	/* Already full? */
815 	if (ops->max_items && dfp->dfp_count >= ops->max_items)
816 		return false;
817 
818 	return true;
819 }
820 
821 /* Create a new pending item at the end of the transaction list. */
822 static inline struct xfs_defer_pending *
823 xfs_defer_alloc(
824 	struct list_head		*dfops,
825 	const struct xfs_defer_op_type	*ops)
826 {
827 	struct xfs_defer_pending	*dfp;
828 
829 	dfp = kmem_cache_zalloc(xfs_defer_pending_cache,
830 			GFP_KERNEL | __GFP_NOFAIL);
831 	dfp->dfp_ops = ops;
832 	INIT_LIST_HEAD(&dfp->dfp_work);
833 	list_add_tail(&dfp->dfp_list, dfops);
834 
835 	return dfp;
836 }
837 
838 /* Add an item for later deferred processing. */
839 struct xfs_defer_pending *
840 xfs_defer_add(
841 	struct xfs_trans		*tp,
842 	struct list_head		*li,
843 	const struct xfs_defer_op_type	*ops)
844 {
845 	struct xfs_defer_pending	*dfp = NULL;
846 
847 	ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES);
848 
849 	if (!ops->finish_item) {
850 		ASSERT(ops->finish_item != NULL);
851 		xfs_force_shutdown(tp->t_mountp, SHUTDOWN_CORRUPT_INCORE);
852 		return NULL;
853 	}
854 
855 	dfp = xfs_defer_find_last(tp, ops);
856 	if (!dfp || !xfs_defer_can_append(dfp, ops))
857 		dfp = xfs_defer_alloc(&tp->t_dfops, ops);
858 
859 	xfs_defer_add_item(dfp, li);
860 	trace_xfs_defer_add_item(tp->t_mountp, dfp, li);
861 	return dfp;
862 }
863 
864 /*
865  * Add a defer ops barrier to force two otherwise adjacent deferred work items
866  * to be tracked separately and have separate log items.
867  */
868 void
869 xfs_defer_add_barrier(
870 	struct xfs_trans		*tp)
871 {
872 	struct xfs_defer_pending	*dfp;
873 
874 	ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES);
875 
876 	/* If the last defer op added was a barrier, we're done. */
877 	dfp = xfs_defer_find_last(tp, &xfs_barrier_defer_type);
878 	if (dfp)
879 		return;
880 
881 	xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type);
882 
883 	trace_xfs_defer_add_item(tp->t_mountp, dfp, NULL);
884 }
885 
886 /*
887  * Create a pending deferred work item to replay the recovered intent item
888  * and add it to the list.
889  */
890 void
891 xfs_defer_start_recovery(
892 	struct xfs_log_item		*lip,
893 	struct list_head		*r_dfops,
894 	const struct xfs_defer_op_type	*ops)
895 {
896 	struct xfs_defer_pending	*dfp = xfs_defer_alloc(r_dfops, ops);
897 
898 	dfp->dfp_intent = lip;
899 }
900 
901 /*
902  * Cancel a deferred work item created to recover a log intent item.  @dfp
903  * will be freed after this function returns.
904  */
905 void
906 xfs_defer_cancel_recovery(
907 	struct xfs_mount		*mp,
908 	struct xfs_defer_pending	*dfp)
909 {
910 	xfs_defer_pending_abort(mp, dfp);
911 	xfs_defer_pending_cancel_work(mp, dfp);
912 }
913 
914 /* Replay the deferred work item created from a recovered log intent item. */
915 int
916 xfs_defer_finish_recovery(
917 	struct xfs_mount		*mp,
918 	struct xfs_defer_pending	*dfp,
919 	struct list_head		*capture_list)
920 {
921 	const struct xfs_defer_op_type	*ops = dfp->dfp_ops;
922 	int				error;
923 
924 	/* dfp is freed by recover_work and must not be accessed afterwards */
925 	error = ops->recover_work(dfp, capture_list);
926 	if (error)
927 		trace_xlog_intent_recovery_failed(mp, ops, error);
928 	return error;
929 }
930 
931 /*
932  * Move deferred ops from one transaction to another and reset the source to
933  * initial state. This is primarily used to carry state forward across
934  * transaction rolls with pending dfops.
935  */
936 void
937 xfs_defer_move(
938 	struct xfs_trans	*dtp,
939 	struct xfs_trans	*stp)
940 {
941 	list_splice_init(&stp->t_dfops, &dtp->t_dfops);
942 
943 	/*
944 	 * Low free space mode was historically controlled by a dfops field.
945 	 * This meant that low mode state potentially carried across multiple
946 	 * transaction rolls. Transfer low mode on a dfops move to preserve
947 	 * that behavior.
948 	 */
949 	dtp->t_flags |= (stp->t_flags & XFS_TRANS_LOWMODE);
950 	stp->t_flags &= ~XFS_TRANS_LOWMODE;
951 }
952 
953 /*
954  * Prepare a chain of fresh deferred ops work items to be completed later.  Log
955  * recovery requires the ability to put off until later the actual finishing
956  * work so that it can process unfinished items recovered from the log in
957  * correct order.
958  *
959  * Create and log intent items for all the work that we're capturing so that we
960  * can be assured that the items will get replayed if the system goes down
961  * before log recovery gets a chance to finish the work it put off.  The entire
962  * deferred ops state is transferred to the capture structure and the
963  * transaction is then ready for the caller to commit it.  If there are no
964  * intent items to capture, this function returns NULL.
965  *
966  * If capture_ip is not NULL, the capture structure will obtain an extra
967  * reference to the inode.
968  */
969 static struct xfs_defer_capture *
970 xfs_defer_ops_capture(
971 	struct xfs_trans		*tp)
972 {
973 	struct xfs_defer_capture	*dfc;
974 	unsigned short			i;
975 	int				error;
976 
977 	if (list_empty(&tp->t_dfops))
978 		return NULL;
979 
980 	error = xfs_defer_create_intents(tp);
981 	if (error < 0)
982 		return ERR_PTR(error);
983 
984 	/* Create an object to capture the defer ops. */
985 	dfc = kzalloc(sizeof(*dfc), GFP_KERNEL | __GFP_NOFAIL);
986 	INIT_LIST_HEAD(&dfc->dfc_list);
987 	INIT_LIST_HEAD(&dfc->dfc_dfops);
988 
989 	/* Move the dfops chain and transaction state to the capture struct. */
990 	list_splice_init(&tp->t_dfops, &dfc->dfc_dfops);
991 	dfc->dfc_tpflags = tp->t_flags & XFS_TRANS_LOWMODE;
992 	tp->t_flags &= ~XFS_TRANS_LOWMODE;
993 
994 	/* Capture the remaining block reservations along with the dfops. */
995 	dfc->dfc_blkres = tp->t_blk_res - tp->t_blk_res_used;
996 	dfc->dfc_rtxres = tp->t_rtx_res - tp->t_rtx_res_used;
997 
998 	/* Preserve the log reservation size. */
999 	dfc->dfc_logres = tp->t_log_res;
1000 
1001 	error = xfs_defer_save_resources(&dfc->dfc_held, tp);
1002 	if (error) {
1003 		/*
1004 		 * Resource capture should never fail, but if it does, we
1005 		 * still have to shut down the log and release things
1006 		 * properly.
1007 		 */
1008 		xfs_force_shutdown(tp->t_mountp, SHUTDOWN_CORRUPT_INCORE);
1009 	}
1010 
1011 	/*
1012 	 * Grab extra references to the inodes and buffers because callers are
1013 	 * expected to release their held references after we commit the
1014 	 * transaction.
1015 	 */
1016 	for (i = 0; i < dfc->dfc_held.dr_inos; i++) {
1017 		xfs_assert_ilocked(dfc->dfc_held.dr_ip[i], XFS_ILOCK_EXCL);
1018 		ihold(VFS_I(dfc->dfc_held.dr_ip[i]));
1019 	}
1020 
1021 	for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
1022 		xfs_buf_hold(dfc->dfc_held.dr_bp[i]);
1023 
1024 	return dfc;
1025 }
1026 
1027 /* Release all resources that we used to capture deferred ops. */
1028 void
1029 xfs_defer_ops_capture_abort(
1030 	struct xfs_mount		*mp,
1031 	struct xfs_defer_capture	*dfc)
1032 {
1033 	unsigned short			i;
1034 
1035 	xfs_defer_pending_abort_list(mp, &dfc->dfc_dfops);
1036 	xfs_defer_cancel_list(mp, &dfc->dfc_dfops);
1037 
1038 	for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
1039 		xfs_buf_relse(dfc->dfc_held.dr_bp[i]);
1040 
1041 	for (i = 0; i < dfc->dfc_held.dr_inos; i++)
1042 		xfs_irele(dfc->dfc_held.dr_ip[i]);
1043 
1044 	kfree(dfc);
1045 }
1046 
1047 /*
1048  * Capture any deferred ops and commit the transaction.  This is the last step
1049  * needed to finish a log intent item that we recovered from the log.  If any
1050  * of the deferred ops operate on an inode, the caller must pass in that inode
1051  * so that the reference can be transferred to the capture structure.  The
1052  * caller must hold ILOCK_EXCL on the inode, and must unlock it before calling
1053  * xfs_defer_ops_continue.
1054  */
1055 int
1056 xfs_defer_ops_capture_and_commit(
1057 	struct xfs_trans		*tp,
1058 	struct list_head		*capture_list)
1059 {
1060 	struct xfs_mount		*mp = tp->t_mountp;
1061 	struct xfs_defer_capture	*dfc;
1062 	int				error;
1063 
1064 	/* If we don't capture anything, commit transaction and exit. */
1065 	dfc = xfs_defer_ops_capture(tp);
1066 	if (IS_ERR(dfc)) {
1067 		xfs_trans_cancel(tp);
1068 		return PTR_ERR(dfc);
1069 	}
1070 	if (!dfc)
1071 		return xfs_trans_commit(tp);
1072 
1073 	/* Commit the transaction and add the capture structure to the list. */
1074 	error = xfs_trans_commit(tp);
1075 	if (error) {
1076 		xfs_defer_ops_capture_abort(mp, dfc);
1077 		return error;
1078 	}
1079 
1080 	list_add_tail(&dfc->dfc_list, capture_list);
1081 	return 0;
1082 }
1083 
1084 /*
1085  * Attach a chain of captured deferred ops to a new transaction and free the
1086  * capture structure.  If an inode was captured, it will be passed back to the
1087  * caller with ILOCK_EXCL held and joined to the transaction with lockflags==0.
1088  * The caller now owns the inode reference.
1089  */
1090 void
1091 xfs_defer_ops_continue(
1092 	struct xfs_defer_capture	*dfc,
1093 	struct xfs_trans		*tp,
1094 	struct xfs_defer_resources	*dres)
1095 {
1096 	unsigned int			i;
1097 
1098 	ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES);
1099 	ASSERT(!(tp->t_flags & XFS_TRANS_DIRTY));
1100 
1101 	/* Lock the captured resources to the new transaction. */
1102 	if (dfc->dfc_held.dr_inos > 2) {
1103 		xfs_sort_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos);
1104 		xfs_lock_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos,
1105 				XFS_ILOCK_EXCL);
1106 	} else if (dfc->dfc_held.dr_inos == 2)
1107 		xfs_lock_two_inodes(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL,
1108 				    dfc->dfc_held.dr_ip[1], XFS_ILOCK_EXCL);
1109 	else if (dfc->dfc_held.dr_inos == 1)
1110 		xfs_ilock(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL);
1111 
1112 	for (i = 0; i < dfc->dfc_held.dr_bufs; i++)
1113 		xfs_buf_lock(dfc->dfc_held.dr_bp[i]);
1114 
1115 	/* Join the captured resources to the new transaction. */
1116 	xfs_defer_restore_resources(tp, &dfc->dfc_held);
1117 	memcpy(dres, &dfc->dfc_held, sizeof(struct xfs_defer_resources));
1118 	dres->dr_bufs = 0;
1119 
1120 	/* Move captured dfops chain and state to the transaction. */
1121 	list_splice_init(&dfc->dfc_dfops, &tp->t_dfops);
1122 	tp->t_flags |= dfc->dfc_tpflags;
1123 
1124 	kfree(dfc);
1125 }
1126 
1127 /* Release the resources captured and continued during recovery. */
1128 void
1129 xfs_defer_resources_rele(
1130 	struct xfs_defer_resources	*dres)
1131 {
1132 	unsigned short			i;
1133 
1134 	for (i = 0; i < dres->dr_inos; i++) {
1135 		xfs_iunlock(dres->dr_ip[i], XFS_ILOCK_EXCL);
1136 		xfs_irele(dres->dr_ip[i]);
1137 		dres->dr_ip[i] = NULL;
1138 	}
1139 
1140 	for (i = 0; i < dres->dr_bufs; i++) {
1141 		xfs_buf_relse(dres->dr_bp[i]);
1142 		dres->dr_bp[i] = NULL;
1143 	}
1144 
1145 	dres->dr_inos = 0;
1146 	dres->dr_bufs = 0;
1147 	dres->dr_ordered = 0;
1148 }
1149 
1150 static inline int __init
1151 xfs_defer_init_cache(void)
1152 {
1153 	xfs_defer_pending_cache = kmem_cache_create("xfs_defer_pending",
1154 			sizeof(struct xfs_defer_pending),
1155 			0, 0, NULL);
1156 
1157 	return xfs_defer_pending_cache != NULL ? 0 : -ENOMEM;
1158 }
1159 
1160 static inline void
1161 xfs_defer_destroy_cache(void)
1162 {
1163 	kmem_cache_destroy(xfs_defer_pending_cache);
1164 	xfs_defer_pending_cache = NULL;
1165 }
1166 
1167 /* Set up caches for deferred work items. */
1168 int __init
1169 xfs_defer_init_item_caches(void)
1170 {
1171 	int				error;
1172 
1173 	error = xfs_defer_init_cache();
1174 	if (error)
1175 		return error;
1176 	error = xfs_rmap_intent_init_cache();
1177 	if (error)
1178 		goto err;
1179 	error = xfs_refcount_intent_init_cache();
1180 	if (error)
1181 		goto err;
1182 	error = xfs_bmap_intent_init_cache();
1183 	if (error)
1184 		goto err;
1185 	error = xfs_extfree_intent_init_cache();
1186 	if (error)
1187 		goto err;
1188 	error = xfs_attr_intent_init_cache();
1189 	if (error)
1190 		goto err;
1191 	error = xfs_exchmaps_intent_init_cache();
1192 	if (error)
1193 		goto err;
1194 
1195 	return 0;
1196 err:
1197 	xfs_defer_destroy_item_caches();
1198 	return error;
1199 }
1200 
1201 /* Destroy all the deferred work item caches, if they've been allocated. */
1202 void
1203 xfs_defer_destroy_item_caches(void)
1204 {
1205 	xfs_exchmaps_intent_destroy_cache();
1206 	xfs_attr_intent_destroy_cache();
1207 	xfs_extfree_intent_destroy_cache();
1208 	xfs_bmap_intent_destroy_cache();
1209 	xfs_refcount_intent_destroy_cache();
1210 	xfs_rmap_intent_destroy_cache();
1211 	xfs_defer_destroy_cache();
1212 }
1213 
1214 /*
1215  * Mark a deferred work item so that it will be requeued indefinitely without
1216  * being finished.  Caller must ensure there are no data dependencies on this
1217  * work item in the meantime.
1218  */
1219 void
1220 xfs_defer_item_pause(
1221 	struct xfs_trans		*tp,
1222 	struct xfs_defer_pending	*dfp)
1223 {
1224 	ASSERT(!(dfp->dfp_flags & XFS_DEFER_PAUSED));
1225 
1226 	dfp->dfp_flags |= XFS_DEFER_PAUSED;
1227 
1228 	trace_xfs_defer_item_pause(tp->t_mountp, dfp);
1229 }
1230 
1231 /*
1232  * Release a paused deferred work item so that it will be finished during the
1233  * next transaction roll.
1234  */
1235 void
1236 xfs_defer_item_unpause(
1237 	struct xfs_trans		*tp,
1238 	struct xfs_defer_pending	*dfp)
1239 {
1240 	ASSERT(dfp->dfp_flags & XFS_DEFER_PAUSED);
1241 
1242 	dfp->dfp_flags &= ~XFS_DEFER_PAUSED;
1243 
1244 	trace_xfs_defer_item_unpause(tp->t_mountp, dfp);
1245 }
1246