1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
36 	__must_hold(&ailp->ail_lock)  in xfs_ail_check()
46 	if (list_empty(&ailp->ail_head))  in xfs_ail_check()
52 	in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags);  in xfs_ail_check()
53 	prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail);  in xfs_ail_check()
54 	if (&prev_lip->li_ail != &ailp->ail_head)  in xfs_ail_check()
55 		prev_lsn = prev_lip->li_lsn;  in xfs_ail_check()
56 	next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail);  in xfs_ail_check()
57 	if (&next_lip->li_ail != &ailp->ail_head)  in xfs_ail_check()
58 		next_lsn = next_lip->li_lsn;  in xfs_ail_check()
59 	lsn = lip->li_lsn;  in xfs_ail_check()
66 	spin_unlock(&ailp->ail_lock);  in xfs_ail_check()
70 	spin_lock(&ailp->ail_lock);  in xfs_ail_check()
84 	if (list_empty(&ailp->ail_head))  in xfs_ail_max()
87 	return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail);  in xfs_ail_max()
99 	if (lip->li_ail.next == &ailp->ail_head)  in xfs_ail_next()
102 	return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail);  in xfs_ail_next()
120 		return lip->li_lsn;  in __xfs_ail_min_lsn()
130 	spin_lock(&ailp->ail_lock);  in xfs_ail_min_lsn()
132 	spin_unlock(&ailp->ail_lock);  in xfs_ail_min_lsn()
149 	cur->item = NULL;  in xfs_trans_ail_cursor_init()
150 	list_add_tail(&cur->list, &ailp->ail_cursors);  in xfs_trans_ail_cursor_init()
162 	struct xfs_log_item	*lip = cur->item;  in xfs_trans_ail_cursor_next()
167 		cur->item = xfs_ail_next(ailp, lip);  in xfs_trans_ail_cursor_next()
179 	cur->item = NULL;  in xfs_trans_ail_cursor_done()
180 	list_del_init(&cur->list);  in xfs_trans_ail_cursor_done()
198 	list_for_each_entry(cur, &ailp->ail_cursors, list) {  in xfs_trans_ail_cursor_clear()
199 		if (cur->item == lip)  in xfs_trans_ail_cursor_clear()
200 			cur->item = (struct xfs_log_item *)  in xfs_trans_ail_cursor_clear()
201 					((uintptr_t)cur->item | 1);  in xfs_trans_ail_cursor_clear()
226 	list_for_each_entry(lip, &ailp->ail_head, li_ail) {  in xfs_trans_ail_cursor_first()
227 		if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)  in xfs_trans_ail_cursor_first()
234 		cur->item = xfs_ail_next(ailp, lip);  in xfs_trans_ail_cursor_first()
245 	list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {  in __xfs_trans_ail_cursor_last()
246 		if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)  in __xfs_trans_ail_cursor_last()
254  * LSN order and initialise the cursor to point to that item.  If there is no
265 	cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);  in xfs_trans_ail_cursor_last()
266 	return cur->item;  in xfs_trans_ail_cursor_last()
271  * tail of the given LSN to maintain insert order for push traversals. The
291 	lip = cur ? cur->item : NULL;  in xfs_ail_splice()
303 		cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);  in xfs_ail_splice()
312 		list_splice(list, &lip->li_ail);  in xfs_ail_splice()
314 		list_splice(list, &ailp->ail_head);  in xfs_ail_splice()
326 	list_del(&lip->li_ail);  in xfs_ail_delete()
338  * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which
339  * order we process them in - the buffer is locked, and we own the buffer list
344  * avoiding the transient zero-reference state that leads to use-after-free.
351 	struct xfs_buf		*bp = lip->li_buf;  in xfsaild_resubmit_item()
362 	list_for_each_entry(lip, &bp->b_li_list, li_bio_list)  in xfsaild_resubmit_item()
363 		clear_bit(XFS_LI_FAILED, &lip->li_flags);  in xfsaild_resubmit_item()
375 	 * pinned. This can help induce head-behind-tail conditions.  in xfsaild_push_item()
377 	if (XFS_TEST_ERROR(false, ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN))  in xfsaild_push_item()
384 	 * the on-disk log.  in xfsaild_push_item()
386 	if (!lip->li_ops->iop_push)  in xfsaild_push_item()
388 	if (test_bit(XFS_LI_FAILED, &lip->li_flags))  in xfsaild_push_item()
389 		return xfsaild_resubmit_item(lip, &ailp->ail_buf_list);  in xfsaild_push_item()
390 	return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);  in xfsaild_push_item()
403 	struct xlog		*log = ailp->ail_log;  in xfs_ail_calc_push_target()
412 	lockdep_assert_held(&ailp->ail_lock);  in xfs_ail_calc_push_target()
418 	max_lsn = lip->li_lsn;  in xfs_ail_calc_push_target()
428 	if (test_and_clear_bit(XFS_AIL_OPSTATE_PUSH_ALL, &ailp->ail_opstate))  in xfs_ail_calc_push_target()
432 	if (waitqueue_active(&ailp->ail_empty))  in xfs_ail_calc_push_target()
436 	 * Background pushing - attempt to keep 25% of the log free and if we  in xfs_ail_calc_push_target()
439 	free_bytes = log->l_logsize - xlog_lsn_sub(log, max_lsn, min_lsn);  in xfs_ail_calc_push_target()
440 	if (free_bytes >= log->l_logsize >> 2)  in xfs_ail_calc_push_target()
441 		return ailp->ail_target;  in xfs_ail_calc_push_target()
444 	target_block = BLOCK_LSN(min_lsn) + (log->l_logBBsize >> 2);  in xfs_ail_calc_push_target()
445 	if (target_block >= log->l_logBBsize) {  in xfs_ail_calc_push_target()
446 		target_block -= log->l_logBBsize;  in xfs_ail_calc_push_target()
456 	if (XFS_LSN_CMP(ailp->ail_target, target_lsn) >= 0)  in xfs_ail_calc_push_target()
457 		return ailp->ail_target;  in xfs_ail_calc_push_target()
465 	struct xfs_mount	*mp = ailp->ail_log->l_mp;  in xfsaild_push()
483 	if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&  in xfsaild_push()
484 	    (!list_empty_careful(&ailp->ail_buf_list) ||  in xfsaild_push()
486 		ailp->ail_log_flush = 0;  in xfsaild_push()
489 		xlog_cil_flush(ailp->ail_log);  in xfsaild_push()
492 	spin_lock(&ailp->ail_lock);  in xfsaild_push()
493 	WRITE_ONCE(ailp->ail_target, xfs_ail_calc_push_target(ailp));  in xfsaild_push()
494 	if (ailp->ail_target == NULLCOMMITLSN)  in xfsaild_push()
498 	lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);  in xfsaild_push()
504 	ASSERT(ailp->ail_target != NULLCOMMITLSN);  in xfsaild_push()
506 	lsn = lip->li_lsn;  in xfsaild_push()
507 	while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) {  in xfsaild_push()
510 		if (test_bit(XFS_LI_FLUSHING, &lip->li_flags))  in xfsaild_push()
524 			ailp->ail_last_pushed_lsn = lsn;  in xfsaild_push()
536 			 * re-try the flushing relatively soon if most of the  in xfsaild_push()
543 			ailp->ail_last_pushed_lsn = lsn;  in xfsaild_push()
551 			ailp->ail_log_flush++;  in xfsaild_push()
586 		if (lip->li_lsn != lsn && count > 1000)  in xfsaild_push()
588 		lsn = lip->li_lsn;  in xfsaild_push()
594 	spin_unlock(&ailp->ail_lock);  in xfsaild_push()
596 	if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))  in xfsaild_push()
597 		ailp->ail_log_flush++;  in xfsaild_push()
599 	if (!count || XFS_LSN_CMP(lsn, ailp->ail_target) >= 0) {  in xfsaild_push()
606 		ailp->ail_last_pushed_lsn = 0;  in xfsaild_push()
619 		ailp->ail_last_pushed_lsn = 0;  in xfsaild_push()
677 			ASSERT(list_empty(&ailp->ail_buf_list) ||  in xfsaild()
678 			       xlog_is_shutdown(ailp->ail_log));  in xfsaild()
679 			xfs_buf_delwri_cancel(&ailp->ail_buf_list);  in xfsaild()
684 		spin_lock(&ailp->ail_lock);  in xfsaild()
685 		if (!xfs_ail_min(ailp) && list_empty(&ailp->ail_buf_list)) {  in xfsaild()
686 			spin_unlock(&ailp->ail_lock);  in xfsaild()
691 		spin_unlock(&ailp->ail_lock);  in xfsaild()
716 	spin_lock(&ailp->ail_lock);  in xfs_ail_push_all_sync()
718 		prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);  in xfs_ail_push_all_sync()
719 		wake_up_process(ailp->ail_task);  in xfs_ail_push_all_sync()
720 		spin_unlock(&ailp->ail_lock);  in xfs_ail_push_all_sync()
722 		spin_lock(&ailp->ail_lock);  in xfs_ail_push_all_sync()
724 	spin_unlock(&ailp->ail_lock);  in xfs_ail_push_all_sync()
726 	finish_wait(&ailp->ail_empty, &wait);  in xfs_ail_push_all_sync()
733 	struct xlog		*log = ailp->ail_log;  in __xfs_ail_assign_tail_lsn()
736 	assert_spin_locked(&ailp->ail_lock);  in __xfs_ail_assign_tail_lsn()
743 		tail_lsn = ailp->ail_head_lsn;  in __xfs_ail_assign_tail_lsn()
745 	WRITE_ONCE(log->l_tail_space,  in __xfs_ail_assign_tail_lsn()
746 			xlog_lsn_sub(log, ailp->ail_head_lsn, tail_lsn));  in __xfs_ail_assign_tail_lsn()
748 	atomic64_set(&log->l_tail_lsn, tail_lsn);  in __xfs_ail_assign_tail_lsn()
752  * Callers should pass the original tail lsn so that we can detect if the tail
762 	xfs_lsn_t		old_lsn) __releases(ailp->ail_lock)  in xfs_ail_update_finish()
764 	struct xlog		*log = ailp->ail_log;  in xfs_ail_update_finish()
768 		spin_unlock(&ailp->ail_lock);  in xfs_ail_update_finish()
773 	if (list_empty(&ailp->ail_head))  in xfs_ail_update_finish()
774 		wake_up_all(&ailp->ail_empty);  in xfs_ail_update_finish()
775 	spin_unlock(&ailp->ail_lock);  in xfs_ail_update_finish()
776 	xfs_log_space_wake(log->l_mp);  in xfs_ail_update_finish()
780  * xfs_trans_ail_update - bulk AIL insertion operation.
784  * be added.  Otherwise, it will be repositioned  by removing it and re-adding
793  * To optimise the insert operation, we delete all the items from the AIL in
796  * insert operation on every item.
807 	xfs_lsn_t		lsn) __releases(ailp->ail_lock)  in xfs_trans_ail_update_bulk()
819 		if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) {  in xfs_trans_ail_update_bulk()
821 			if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)  in xfs_trans_ail_update_bulk()
824 			trace_xfs_ail_move(lip, lip->li_lsn, lsn);  in xfs_trans_ail_update_bulk()
826 				tail_lsn = lip->li_lsn;  in xfs_trans_ail_update_bulk()
832 		lip->li_lsn = lsn;  in xfs_trans_ail_update_bulk()
833 		list_add_tail(&lip->li_ail, &tmp);  in xfs_trans_ail_update_bulk()
840 	 * If this is the first insert, wake up the push daemon so it can  in xfs_trans_ail_update_bulk()
848 		wake_up_process(ailp->ail_task);  in xfs_trans_ail_update_bulk()
855 /* Insert a log item into the AIL. */
862 	spin_lock(&ailp->ail_lock);  in xfs_trans_ail_insert()
879 	xfs_lsn_t		lsn = lip->li_lsn;  in xfs_ail_delete_one()
881 	trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);  in xfs_ail_delete_one()
883 	clear_bit(XFS_LI_IN_AIL, &lip->li_flags);  in xfs_ail_delete_one()
884 	lip->li_lsn = 0;  in xfs_ail_delete_one()
896 	struct xfs_ail		*ailp = lip->li_ailp;  in xfs_trans_ail_delete()
897 	struct xlog		*log = ailp->ail_log;  in xfs_trans_ail_delete()
900 	spin_lock(&ailp->ail_lock);  in xfs_trans_ail_delete()
901 	if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) {  in xfs_trans_ail_delete()
902 		spin_unlock(&ailp->ail_lock);  in xfs_trans_ail_delete()
904 			xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE,  in xfs_trans_ail_delete()
927 		return -ENOMEM;  in xfs_trans_ail_init()
929 	ailp->ail_log = mp->m_log;  in xfs_trans_ail_init()
930 	INIT_LIST_HEAD(&ailp->ail_head);  in xfs_trans_ail_init()
931 	INIT_LIST_HEAD(&ailp->ail_cursors);  in xfs_trans_ail_init()
932 	spin_lock_init(&ailp->ail_lock);  in xfs_trans_ail_init()
933 	INIT_LIST_HEAD(&ailp->ail_buf_list);  in xfs_trans_ail_init()
934 	init_waitqueue_head(&ailp->ail_empty);  in xfs_trans_ail_init()
936 	ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",  in xfs_trans_ail_init()
937 				mp->m_super->s_id);  in xfs_trans_ail_init()
938 	if (IS_ERR(ailp->ail_task))  in xfs_trans_ail_init()
941 	mp->m_ail = ailp;  in xfs_trans_ail_init()
946 	return -ENOMEM;  in xfs_trans_ail_init()
953 	struct xfs_ail	*ailp = mp->m_ail;  in xfs_trans_ail_destroy()
955 	kthread_stop(ailp->ail_task);  in xfs_trans_ail_destroy()