Lines Matching +full:no +full:- +full:insert +full:- +full:detect
1 // SPDX-License-Identifier: GPL-2.0
27 * We don't reserve any space for the ticket - we are going to steal whatever
45 tic->t_curr_res = 0; in xlog_cil_ticket_alloc()
46 tic->t_iclog_hdrs = 0; in xlog_cil_ticket_alloc()
53 struct xlog *log = cil->xc_log; in xlog_cil_set_iclog_hdr_count()
55 atomic_set(&cil->xc_iclog_hdrs, in xlog_cil_set_iclog_hdr_count()
57 (log->l_iclog_size - log->l_iclog_hsize))); in xlog_cil_set_iclog_hdr_count()
65 * Note: for this to be used in a non-racy manner, it has to be called with
74 if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) in xlog_item_in_current_chkpt()
82 return lip->li_seq == READ_ONCE(cil->xc_current_sequence); in xlog_item_in_current_chkpt()
89 return xlog_item_in_current_chkpt(lip->li_log->l_cilp, lip); in xfs_log_item_in_current_chkpt()
93 * Unavoidable forward declaration - xlog_cil_push_work() calls
104 INIT_LIST_HEAD(&ctx->committing); in xlog_cil_ctx_alloc()
105 INIT_LIST_HEAD(&ctx->busy_extents.extent_list); in xlog_cil_ctx_alloc()
106 INIT_LIST_HEAD(&ctx->log_items); in xlog_cil_ctx_alloc()
107 INIT_LIST_HEAD(&ctx->lv_chain); in xlog_cil_ctx_alloc()
108 INIT_WORK(&ctx->push_work, xlog_cil_push_work); in xlog_cil_ctx_alloc()
116 * will be accessing or modifying the per-cpu counters.
126 for_each_cpu(cpu, &ctx->cil_pcpmask) { in xlog_cil_push_pcp_aggregate()
127 cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); in xlog_cil_push_pcp_aggregate()
129 ctx->ticket->t_curr_res += cilpcp->space_reserved; in xlog_cil_push_pcp_aggregate()
130 cilpcp->space_reserved = 0; in xlog_cil_push_pcp_aggregate()
132 if (!list_empty(&cilpcp->busy_extents)) { in xlog_cil_push_pcp_aggregate()
133 list_splice_init(&cilpcp->busy_extents, in xlog_cil_push_pcp_aggregate()
134 &ctx->busy_extents.extent_list); in xlog_cil_push_pcp_aggregate()
136 if (!list_empty(&cilpcp->log_items)) in xlog_cil_push_pcp_aggregate()
137 list_splice_init(&cilpcp->log_items, &ctx->log_items); in xlog_cil_push_pcp_aggregate()
141 * counter we use to detect when the current context is nearing in xlog_cil_push_pcp_aggregate()
144 cilpcp->space_used = 0; in xlog_cil_push_pcp_aggregate()
149 * Aggregate the CIL per-cpu space used counters into the global atomic value.
150 * This is called when the per-cpu counter aggregation will first pass the soft
163 if (!test_and_clear_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) in xlog_cil_insert_pcp_aggregate()
172 for_each_cpu(cpu, &ctx->cil_pcpmask) { in xlog_cil_insert_pcp_aggregate()
173 struct xlog_cil_pcp *cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); in xlog_cil_insert_pcp_aggregate()
175 count += xchg(&cilpcp->space_used, 0); in xlog_cil_insert_pcp_aggregate()
177 atomic_add(count, &ctx->space_used); in xlog_cil_insert_pcp_aggregate()
186 set_bit(XLOG_CIL_EMPTY, &cil->xc_flags); in xlog_cil_ctx_switch()
187 set_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags); in xlog_cil_ctx_switch()
188 ctx->sequence = ++cil->xc_current_sequence; in xlog_cil_ctx_switch()
189 ctx->cil = cil; in xlog_cil_ctx_switch()
190 cil->xc_ctx = ctx; in xlog_cil_ctx_switch()
207 log->l_cilp->xc_ctx->ticket = xlog_cil_ticket_alloc(log); in xlog_cil_init_post_recovery()
208 log->l_cilp->xc_ctx->sequence = 1; in xlog_cil_init_post_recovery()
209 xlog_cil_set_iclog_hdr_count(log->l_cilp); in xlog_cil_init_post_recovery()
274 list_for_each_entry(lip, &tp->t_items, li_trans) { in xlog_cil_alloc_shadow_bufs()
282 if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) in xlog_cil_alloc_shadow_bufs()
286 lip->li_ops->iop_size(lip, &niovecs, &nbytes); in xlog_cil_alloc_shadow_bufs()
300 * We 64-bit align the length of each iovec so that the start of in xlog_cil_alloc_shadow_bufs()
309 * Then round nbytes up to 64-bit alignment so that the initial in xlog_cil_alloc_shadow_bufs()
317 * The data buffer needs to start 64-bit aligned, so round up in xlog_cil_alloc_shadow_bufs()
324 * if we have no shadow buffer, or it is too small, we need to in xlog_cil_alloc_shadow_bufs()
327 if (!lip->li_lv_shadow || in xlog_cil_alloc_shadow_bufs()
328 buf_size > lip->li_lv_shadow->lv_size) { in xlog_cil_alloc_shadow_bufs()
332 * same reason - we don't need to zero the data area in in xlog_cil_alloc_shadow_bufs()
336 kvfree(lip->li_lv_shadow); in xlog_cil_alloc_shadow_bufs()
341 INIT_LIST_HEAD(&lv->lv_list); in xlog_cil_alloc_shadow_bufs()
342 lv->lv_item = lip; in xlog_cil_alloc_shadow_bufs()
343 lv->lv_size = buf_size; in xlog_cil_alloc_shadow_bufs()
345 lv->lv_buf_len = XFS_LOG_VEC_ORDERED; in xlog_cil_alloc_shadow_bufs()
347 lv->lv_iovecp = (struct xfs_log_iovec *)&lv[1]; in xlog_cil_alloc_shadow_bufs()
348 lip->li_lv_shadow = lv; in xlog_cil_alloc_shadow_bufs()
351 lv = lip->li_lv_shadow; in xlog_cil_alloc_shadow_bufs()
353 lv->lv_buf_len = XFS_LOG_VEC_ORDERED; in xlog_cil_alloc_shadow_bufs()
355 lv->lv_buf_len = 0; in xlog_cil_alloc_shadow_bufs()
356 lv->lv_bytes = 0; in xlog_cil_alloc_shadow_bufs()
359 /* Ensure the lv is set up according to ->iop_size */ in xlog_cil_alloc_shadow_bufs()
360 lv->lv_niovecs = niovecs; in xlog_cil_alloc_shadow_bufs()
363 lv->lv_buf = (char *)lv + xlog_cil_iovec_space(niovecs); in xlog_cil_alloc_shadow_bufs()
380 if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED) in xfs_cil_prepare_item()
381 *diff_len += lv->lv_bytes; in xfs_cil_prepare_item()
384 * If there is no old LV, this is the first time we've seen the item in in xfs_cil_prepare_item()
391 if (lv->lv_item->li_ops->iop_pin) in xfs_cil_prepare_item()
392 lv->lv_item->li_ops->iop_pin(lv->lv_item); in xfs_cil_prepare_item()
393 lv->lv_item->li_lv_shadow = NULL; in xfs_cil_prepare_item()
395 ASSERT(lv->lv_buf_len != XFS_LOG_VEC_ORDERED); in xfs_cil_prepare_item()
397 *diff_len -= old_lv->lv_bytes; in xfs_cil_prepare_item()
398 lv->lv_item->li_lv_shadow = old_lv; in xfs_cil_prepare_item()
402 lv->lv_item->li_lv = lv; in xfs_cil_prepare_item()
410 if (!lv->lv_item->li_seq) in xfs_cil_prepare_item()
411 lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence; in xfs_cil_prepare_item()
424 * dependent on the current state of the vector in the CIL - the shadow lv is
427 * lv, then simple swap it out for the shadow lv. We don't free it - that is
451 if (list_empty(&tp->t_items)) { in xlog_cil_insert_format_items()
456 list_for_each_entry(lip, &tp->t_items, li_trans) { in xlog_cil_insert_format_items()
463 if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) in xlog_cil_insert_format_items()
470 shadow = lip->li_lv_shadow; in xlog_cil_insert_format_items()
471 if (shadow->lv_buf_len == XFS_LOG_VEC_ORDERED) in xlog_cil_insert_format_items()
475 if (!shadow->lv_niovecs && !ordered) in xlog_cil_insert_format_items()
479 old_lv = lip->li_lv; in xlog_cil_insert_format_items()
480 if (lip->li_lv && shadow->lv_size <= lip->li_lv->lv_size) { in xlog_cil_insert_format_items()
482 lv = lip->li_lv; in xlog_cil_insert_format_items()
485 goto insert; in xlog_cil_insert_format_items()
491 *diff_len -= lv->lv_bytes; in xlog_cil_insert_format_items()
493 /* Ensure the lv is set up according to ->iop_size */ in xlog_cil_insert_format_items()
494 lv->lv_niovecs = shadow->lv_niovecs; in xlog_cil_insert_format_items()
497 lv->lv_buf_len = 0; in xlog_cil_insert_format_items()
498 lv->lv_bytes = 0; in xlog_cil_insert_format_items()
499 lv->lv_buf = (char *)lv + in xlog_cil_insert_format_items()
500 xlog_cil_iovec_space(lv->lv_niovecs); in xlog_cil_insert_format_items()
504 lv->lv_item = lip; in xlog_cil_insert_format_items()
507 ASSERT(lip->li_lv == NULL); in xlog_cil_insert_format_items()
508 goto insert; in xlog_cil_insert_format_items()
512 ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t))); in xlog_cil_insert_format_items()
513 lip->li_ops->iop_format(lip, lv); in xlog_cil_insert_format_items()
514 insert: in xlog_cil_insert_format_items()
529 if (waitqueue_active(&log->l_cilp->xc_push_wait)) in xlog_cil_over_hard_limit()
537 * Insert the log items into the CIL and calculate the difference in space
549 struct xfs_cil *cil = log->l_cilp; in xlog_cil_insert_items()
550 struct xfs_cil_ctx *ctx = cil->xc_ctx; in xlog_cil_insert_items()
572 len -= released_space; in xlog_cil_insert_items()
575 * Grab the per-cpu pointer for the CIL before we start any accounting. in xlog_cil_insert_items()
576 * That ensures that we are running with pre-emption disabled and so we in xlog_cil_insert_items()
581 cilpcp = this_cpu_ptr(cil->xc_pcp); in xlog_cil_insert_items()
584 if (!cpumask_test_cpu(cpu_nr, &ctx->cil_pcpmask)) in xlog_cil_insert_items()
585 cpumask_test_and_set_cpu(cpu_nr, &ctx->cil_pcpmask); in xlog_cil_insert_items()
594 if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) && in xlog_cil_insert_items()
595 test_and_clear_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) in xlog_cil_insert_items()
596 ctx_res = ctx->ticket->t_unit_res; in xlog_cil_insert_items()
613 * The cil->xc_ctx_lock provides the serialisation necessary for safely in xlog_cil_insert_items()
616 space_used = atomic_read(&ctx->space_used) + cilpcp->space_used + len; in xlog_cil_insert_items()
617 if (atomic_read(&cil->xc_iclog_hdrs) > 0 || in xlog_cil_insert_items()
619 split_res = log->l_iclog_hsize + in xlog_cil_insert_items()
622 ctx_res += split_res * (tp->t_ticket->t_iclog_hdrs - 1); in xlog_cil_insert_items()
624 ctx_res = split_res * tp->t_ticket->t_iclog_hdrs; in xlog_cil_insert_items()
625 atomic_sub(tp->t_ticket->t_iclog_hdrs, &cil->xc_iclog_hdrs); in xlog_cil_insert_items()
627 cilpcp->space_reserved += ctx_res; in xlog_cil_insert_items()
631 * percpu count into the global count if over the per-cpu threshold. in xlog_cil_insert_items()
633 if (!test_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) { in xlog_cil_insert_items()
634 atomic_add(len, &ctx->space_used); in xlog_cil_insert_items()
635 } else if (cilpcp->space_used + len > in xlog_cil_insert_items()
637 space_used = atomic_add_return(cilpcp->space_used + len, in xlog_cil_insert_items()
638 &ctx->space_used); in xlog_cil_insert_items()
639 cilpcp->space_used = 0; in xlog_cil_insert_items()
648 cilpcp->space_used += len; in xlog_cil_insert_items()
651 if (!list_empty(&tp->t_busy)) in xlog_cil_insert_items()
652 list_splice_init(&tp->t_busy, &cilpcp->busy_extents); in xlog_cil_insert_items()
656 * and insert items into the CIL if they aren't already there. in xlog_cil_insert_items()
660 order = atomic_inc_return(&ctx->order_id); in xlog_cil_insert_items()
661 list_for_each_entry(lip, &tp->t_items, li_trans) { in xlog_cil_insert_items()
663 if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) in xlog_cil_insert_items()
666 lip->li_order_id = order; in xlog_cil_insert_items()
667 if (!list_empty(&lip->li_cil)) in xlog_cil_insert_items()
669 list_add_tail(&lip->li_cil, &cilpcp->log_items); in xlog_cil_insert_items()
677 tp->t_ticket->t_curr_res -= ctx_res + len; in xlog_cil_insert_items()
678 if (WARN_ON(tp->t_ticket->t_curr_res < 0)) { in xlog_cil_insert_items()
679 xfs_warn(log->l_mp, "Transaction log reservation overrun:"); in xlog_cil_insert_items()
680 xfs_warn(log->l_mp, in xlog_cil_insert_items()
683 xfs_warn(log->l_mp, " split region headers: %d bytes", in xlog_cil_insert_items()
685 xfs_warn(log->l_mp, " ctx ticket: %d bytes", ctx_res); in xlog_cil_insert_items()
701 spin_lock(&ailp->ail_lock); in xlog_cil_ail_insert_batch()
702 /* xfs_trans_ail_update_bulk drops ailp->ail_lock */ in xlog_cil_ail_insert_batch()
708 if (lip->li_ops->iop_unpin) in xlog_cil_ail_insert_batch()
709 lip->li_ops->iop_unpin(lip, 0); in xlog_cil_ail_insert_batch()
714 * Take the checkpoint's log vector chain of items and insert the attached log
724 * +-------------+------------+----------------+
729 * start record for N+1 is no longer in the active portion of the log
744 * The AIL cursor is used to optimise the insert process. If commit_lsn is not
745 * at the end of the AIL, the insert cursor avoids the need to walk the AIL to
757 struct xfs_ail *ailp = ctx->cil->xc_log->l_ailp; in xlog_cil_ail_insert()
771 * xfs_ail_update_finish() so that tail space and space-based wakeups in xlog_cil_ail_insert()
774 ASSERT(XFS_LSN_CMP(ctx->commit_lsn, ailp->ail_head_lsn) >= 0 || in xlog_cil_ail_insert()
776 spin_lock(&ailp->ail_lock); in xlog_cil_ail_insert()
777 xfs_trans_ail_cursor_last(ailp, &cur, ctx->start_lsn); in xlog_cil_ail_insert()
778 old_head = ailp->ail_head_lsn; in xlog_cil_ail_insert()
779 ailp->ail_head_lsn = ctx->commit_lsn; in xlog_cil_ail_insert()
791 xlog_grant_return_space(ailp->ail_log, old_head, ailp->ail_head_lsn); in xlog_cil_ail_insert()
794 list_for_each_entry(lv, &ctx->lv_chain, lv_list) { in xlog_cil_ail_insert()
795 struct xfs_log_item *lip = lv->lv_item; in xlog_cil_ail_insert()
799 set_bit(XFS_LI_ABORTED, &lip->li_flags); in xlog_cil_ail_insert()
801 if (lip->li_ops->flags & XFS_ITEM_RELEASE_WHEN_COMMITTED) { in xlog_cil_ail_insert()
802 lip->li_ops->iop_release(lip); in xlog_cil_ail_insert()
806 if (lip->li_ops->iop_committed) in xlog_cil_ail_insert()
807 item_lsn = lip->li_ops->iop_committed(lip, in xlog_cil_ail_insert()
808 ctx->start_lsn); in xlog_cil_ail_insert()
810 item_lsn = ctx->start_lsn; in xlog_cil_ail_insert()
812 /* item_lsn of -1 means the item needs no further processing */ in xlog_cil_ail_insert()
813 if (XFS_LSN_CMP(item_lsn, (xfs_lsn_t)-1) == 0) in xlog_cil_ail_insert()
817 * if we are aborting the operation, no point in inserting the in xlog_cil_ail_insert()
821 ASSERT(xlog_is_shutdown(ailp->ail_log)); in xlog_cil_ail_insert()
822 if (lip->li_ops->iop_unpin) in xlog_cil_ail_insert()
823 lip->li_ops->iop_unpin(lip, 1); in xlog_cil_ail_insert()
827 if (item_lsn != ctx->start_lsn) { in xlog_cil_ail_insert()
833 * not affect the AIL cursor the bulk insert path is in xlog_cil_ail_insert()
836 spin_lock(&ailp->ail_lock); in xlog_cil_ail_insert()
837 if (XFS_LSN_CMP(item_lsn, lip->li_lsn) > 0) in xlog_cil_ail_insert()
840 spin_unlock(&ailp->ail_lock); in xlog_cil_ail_insert()
841 if (lip->li_ops->iop_unpin) in xlog_cil_ail_insert()
842 lip->li_ops->iop_unpin(lip, 0); in xlog_cil_ail_insert()
846 /* Item is a candidate for bulk AIL insert. */ in xlog_cil_ail_insert()
847 log_items[i++] = lv->lv_item; in xlog_cil_ail_insert()
850 LOG_ITEM_BATCH_SIZE, ctx->start_lsn); in xlog_cil_ail_insert()
855 /* make sure we insert the remainder! */ in xlog_cil_ail_insert()
858 ctx->start_lsn); in xlog_cil_ail_insert()
860 spin_lock(&ailp->ail_lock); in xlog_cil_ail_insert()
862 spin_unlock(&ailp->ail_lock); in xlog_cil_ail_insert()
873 list_del_init(&lv->lv_list); in xlog_cil_free_logvec()
887 struct xfs_mount *mp = ctx->cil->xc_log->l_mp; in xlog_cil_committed()
888 bool abort = xlog_is_shutdown(ctx->cil->xc_log); in xlog_cil_committed()
898 spin_lock(&ctx->cil->xc_push_lock); in xlog_cil_committed()
899 wake_up_all(&ctx->cil->xc_start_wait); in xlog_cil_committed()
900 wake_up_all(&ctx->cil->xc_commit_wait); in xlog_cil_committed()
901 spin_unlock(&ctx->cil->xc_push_lock); in xlog_cil_committed()
906 xfs_extent_busy_sort(&ctx->busy_extents.extent_list); in xlog_cil_committed()
907 xfs_extent_busy_clear(&ctx->busy_extents.extent_list, in xlog_cil_committed()
910 spin_lock(&ctx->cil->xc_push_lock); in xlog_cil_committed()
911 list_del(&ctx->committing); in xlog_cil_committed()
912 spin_unlock(&ctx->cil->xc_push_lock); in xlog_cil_committed()
914 xlog_cil_free_logvec(&ctx->lv_chain); in xlog_cil_committed()
916 if (!list_empty(&ctx->busy_extents.extent_list)) { in xlog_cil_committed()
917 ctx->busy_extents.owner = ctx; in xlog_cil_committed()
918 xfs_discard_extents(mp, &ctx->busy_extents); in xlog_cil_committed()
933 list_del(&ctx->iclog_entry); in xlog_cil_process_committed()
949 struct xfs_cil *cil = ctx->cil; in xlog_cil_set_ctx_write_state()
950 xfs_lsn_t lsn = be64_to_cpu(iclog->ic_header.h_lsn); in xlog_cil_set_ctx_write_state()
952 ASSERT(!ctx->commit_lsn); in xlog_cil_set_ctx_write_state()
953 if (!ctx->start_lsn) { in xlog_cil_set_ctx_write_state()
954 spin_lock(&cil->xc_push_lock); in xlog_cil_set_ctx_write_state()
962 ctx->start_lsn = lsn; in xlog_cil_set_ctx_write_state()
963 wake_up_all(&cil->xc_start_wait); in xlog_cil_set_ctx_write_state()
964 spin_unlock(&cil->xc_push_lock); in xlog_cil_set_ctx_write_state()
971 spin_lock(&cil->xc_log->l_icloglock); in xlog_cil_set_ctx_write_state()
972 iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; in xlog_cil_set_ctx_write_state()
973 spin_unlock(&cil->xc_log->l_icloglock); in xlog_cil_set_ctx_write_state()
982 atomic_inc(&iclog->ic_refcnt); in xlog_cil_set_ctx_write_state()
992 spin_lock(&cil->xc_log->l_icloglock); in xlog_cil_set_ctx_write_state()
993 list_add_tail(&ctx->iclog_entry, &iclog->ic_callbacks); in xlog_cil_set_ctx_write_state()
994 spin_unlock(&cil->xc_log->l_icloglock); in xlog_cil_set_ctx_write_state()
1001 spin_lock(&cil->xc_push_lock); in xlog_cil_set_ctx_write_state()
1002 ctx->commit_iclog = iclog; in xlog_cil_set_ctx_write_state()
1003 ctx->commit_lsn = lsn; in xlog_cil_set_ctx_write_state()
1004 wake_up_all(&cil->xc_commit_wait); in xlog_cil_set_ctx_write_state()
1005 spin_unlock(&cil->xc_push_lock); in xlog_cil_set_ctx_write_state()
1028 spin_lock(&cil->xc_push_lock); in xlog_cil_order_write()
1029 list_for_each_entry(ctx, &cil->xc_committing, committing) { in xlog_cil_order_write()
1035 if (xlog_is_shutdown(cil->xc_log)) { in xlog_cil_order_write()
1036 spin_unlock(&cil->xc_push_lock); in xlog_cil_order_write()
1037 return -EIO; in xlog_cil_order_write()
1044 if (ctx->sequence >= sequence) in xlog_cil_order_write()
1050 if (!ctx->start_lsn) { in xlog_cil_order_write()
1051 xlog_wait(&cil->xc_start_wait, &cil->xc_push_lock); in xlog_cil_order_write()
1056 if (!ctx->commit_lsn) { in xlog_cil_order_write()
1057 xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock); in xlog_cil_order_write()
1063 spin_unlock(&cil->xc_push_lock); in xlog_cil_order_write()
1070 * sequence order so that log recovery will always use in-order start LSNs when
1078 struct xlog *log = ctx->cil->xc_log; in xlog_cil_write_chain()
1081 error = xlog_cil_order_write(ctx->cil, ctx->sequence, _START_RECORD); in xlog_cil_write_chain()
1084 return xlog_write(log, ctx, &ctx->lv_chain, ctx->ticket, chain_len); in xlog_cil_write_chain()
1097 struct xlog *log = ctx->cil->xc_log; in xlog_cil_write_commit_record()
1100 .oh_tid = cpu_to_be32(ctx->ticket->t_tid), in xlog_cil_write_commit_record()
1117 return -EIO; in xlog_cil_write_commit_record()
1119 error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD); in xlog_cil_write_commit_record()
1124 ctx->ticket->t_curr_res -= reg.i_len; in xlog_cil_write_commit_record()
1125 error = xlog_write(log, ctx, &lv_chain, ctx->ticket, reg.i_len); in xlog_cil_write_commit_record()
1156 struct xlog_ticket *tic = ctx->ticket; in xlog_cil_build_trans_hdr()
1157 __be32 tid = cpu_to_be32(tic->t_tid); in xlog_cil_build_trans_hdr()
1162 hdr->oph[0].oh_tid = tid; in xlog_cil_build_trans_hdr()
1163 hdr->oph[0].oh_clientid = XFS_TRANSACTION; in xlog_cil_build_trans_hdr()
1164 hdr->oph[0].oh_flags = XLOG_START_TRANS; in xlog_cil_build_trans_hdr()
1167 hdr->lhdr[0].i_addr = &hdr->oph[0]; in xlog_cil_build_trans_hdr()
1168 hdr->lhdr[0].i_len = sizeof(struct xlog_op_header); in xlog_cil_build_trans_hdr()
1169 hdr->lhdr[0].i_type = XLOG_REG_TYPE_LRHEADER; in xlog_cil_build_trans_hdr()
1172 hdr->oph[1].oh_tid = tid; in xlog_cil_build_trans_hdr()
1173 hdr->oph[1].oh_clientid = XFS_TRANSACTION; in xlog_cil_build_trans_hdr()
1174 hdr->oph[1].oh_len = cpu_to_be32(sizeof(struct xfs_trans_header)); in xlog_cil_build_trans_hdr()
1177 hdr->thdr.th_magic = XFS_TRANS_HEADER_MAGIC; in xlog_cil_build_trans_hdr()
1178 hdr->thdr.th_type = XFS_TRANS_CHECKPOINT; in xlog_cil_build_trans_hdr()
1179 hdr->thdr.th_tid = tic->t_tid; in xlog_cil_build_trans_hdr()
1180 hdr->thdr.th_num_items = num_iovecs; in xlog_cil_build_trans_hdr()
1183 hdr->lhdr[1].i_addr = &hdr->oph[1]; in xlog_cil_build_trans_hdr()
1184 hdr->lhdr[1].i_len = sizeof(struct xlog_op_header) + in xlog_cil_build_trans_hdr()
1186 hdr->lhdr[1].i_type = XLOG_REG_TYPE_TRANSHDR; in xlog_cil_build_trans_hdr()
1188 lvhdr->lv_niovecs = 2; in xlog_cil_build_trans_hdr()
1189 lvhdr->lv_iovecp = &hdr->lhdr[0]; in xlog_cil_build_trans_hdr()
1190 lvhdr->lv_bytes = hdr->lhdr[0].i_len + hdr->lhdr[1].i_len; in xlog_cil_build_trans_hdr()
1192 tic->t_curr_res -= lvhdr->lv_bytes; in xlog_cil_build_trans_hdr()
1212 return l1->lv_order_id > l2->lv_order_id; in xlog_cil_order_cmp()
1231 while (!list_empty(&ctx->log_items)) { in xlog_cil_build_lv_chain()
1235 item = list_first_entry(&ctx->log_items, in xlog_cil_build_lv_chain()
1238 if (test_bit(XFS_LI_WHITEOUT, &item->li_flags)) { in xlog_cil_build_lv_chain()
1239 list_move(&item->li_cil, whiteouts); in xlog_cil_build_lv_chain()
1244 lv = item->li_lv; in xlog_cil_build_lv_chain()
1245 lv->lv_order_id = item->li_order_id; in xlog_cil_build_lv_chain()
1248 if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED) in xlog_cil_build_lv_chain()
1249 *num_bytes += lv->lv_bytes; in xlog_cil_build_lv_chain()
1250 *num_iovecs += lv->lv_niovecs; in xlog_cil_build_lv_chain()
1251 list_add_tail(&lv->lv_list, &ctx->lv_chain); in xlog_cil_build_lv_chain()
1253 list_del_init(&item->li_cil); in xlog_cil_build_lv_chain()
1254 item->li_order_id = 0; in xlog_cil_build_lv_chain()
1255 item->li_lv = NULL; in xlog_cil_build_lv_chain()
1266 list_del_init(&item->li_cil); in xlog_cil_cleanup_whiteouts()
1268 item->li_ops->iop_unpin(item, 1); in xlog_cil_cleanup_whiteouts()
1277 * flushed and we don't need to do anything - the caller will wait for it to
1299 struct xfs_cil *cil = ctx->cil; in xlog_cil_push_work()
1300 struct xlog *log = cil->xc_log; in xlog_cil_push_work()
1313 new_ctx->ticket = xlog_cil_ticket_alloc(log); in xlog_cil_push_work()
1315 down_write(&cil->xc_ctx_lock); in xlog_cil_push_work()
1317 spin_lock(&cil->xc_push_lock); in xlog_cil_push_work()
1318 push_seq = cil->xc_push_seq; in xlog_cil_push_work()
1319 ASSERT(push_seq <= ctx->sequence); in xlog_cil_push_work()
1320 push_commit_stable = cil->xc_push_commit_stable; in xlog_cil_push_work()
1321 cil->xc_push_commit_stable = false; in xlog_cil_push_work()
1324 * As we are about to switch to a new, empty CIL context, we no longer in xlog_cil_push_work()
1327 * to the new context. The ctx->xc_push_lock provides the serialisation in xlog_cil_push_work()
1331 if (waitqueue_active(&cil->xc_push_wait)) in xlog_cil_push_work()
1332 wake_up_all(&cil->xc_push_wait); in xlog_cil_push_work()
1341 if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) { in xlog_cil_push_work()
1342 cil->xc_push_seq = 0; in xlog_cil_push_work()
1343 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_work()
1349 if (push_seq < ctx->sequence) { in xlog_cil_push_work()
1350 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_work()
1357 * this push can easily detect the difference between a "push in in xlog_cil_push_work()
1364 * to detect a push that had nothing to do and therefore does not need in xlog_cil_push_work()
1378 list_add(&ctx->committing, &cil->xc_committing); in xlog_cil_push_work()
1379 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_work()
1386 * though - we need to synchronise with previous and future commits so in xlog_cil_push_work()
1408 spin_lock(&cil->xc_push_lock); in xlog_cil_push_work()
1410 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_work()
1411 up_write(&cil->xc_ctx_lock); in xlog_cil_push_work()
1418 list_sort(NULL, &ctx->lv_chain, xlog_cil_order_cmp); in xlog_cil_push_work()
1429 list_add(&lvhdr.lv_list, &ctx->lv_chain); in xlog_cil_push_work()
1452 ticket = ctx->ticket; in xlog_cil_push_work()
1457 * checks for this - ACTIVE can be either a past completed iclog or a in xlog_cil_push_work()
1466 spin_lock(&log->l_icloglock); in xlog_cil_push_work()
1467 if (ctx->start_lsn != ctx->commit_lsn) { in xlog_cil_push_work()
1470 plsn = be64_to_cpu(ctx->commit_iclog->ic_prev->ic_header.h_lsn); in xlog_cil_push_work()
1471 if (plsn && XFS_LSN_CMP(plsn, ctx->commit_lsn) < 0) { in xlog_cil_push_work()
1477 xlog_wait_on_iclog(ctx->commit_iclog->ic_prev); in xlog_cil_push_work()
1478 spin_lock(&log->l_icloglock); in xlog_cil_push_work()
1482 * We need to issue a pre-flush so that the ordering for this in xlog_cil_push_work()
1485 ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; in xlog_cil_push_work()
1498 ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA; in xlog_cil_push_work()
1500 ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE) in xlog_cil_push_work()
1501 xlog_state_switch_iclogs(log, ctx->commit_iclog, 0); in xlog_cil_push_work()
1502 ticket = ctx->ticket; in xlog_cil_push_work()
1503 xlog_state_release_iclog(log, ctx->commit_iclog, ticket); in xlog_cil_push_work()
1507 spin_unlock(&log->l_icloglock); in xlog_cil_push_work()
1514 up_write(&cil->xc_ctx_lock); in xlog_cil_push_work()
1515 xfs_log_ticket_put(new_ctx->ticket); in xlog_cil_push_work()
1523 if (!ctx->commit_iclog) { in xlog_cil_push_work()
1524 xfs_log_ticket_ungrant(log, ctx->ticket); in xlog_cil_push_work()
1529 spin_lock(&log->l_icloglock); in xlog_cil_push_work()
1530 ticket = ctx->ticket; in xlog_cil_push_work()
1531 xlog_state_release_iclog(log, ctx->commit_iclog, ticket); in xlog_cil_push_work()
1533 spin_unlock(&log->l_icloglock); in xlog_cil_push_work()
1549 struct xfs_cil *cil = log->l_cilp; in xlog_cil_push_background()
1550 int space_used = atomic_read(&cil->xc_ctx->space_used); in xlog_cil_push_background()
1556 ASSERT(!test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)); in xlog_cil_push_background()
1560 * - we haven't used up all the space available yet; or in xlog_cil_push_background()
1561 * - we've already queued up a push; and in xlog_cil_push_background()
1562 * - we're not over the hard limit; and in xlog_cil_push_background()
1563 * - nothing has been over the hard limit. in xlog_cil_push_background()
1568 (cil->xc_push_seq == cil->xc_current_sequence && in xlog_cil_push_background()
1570 !waitqueue_active(&cil->xc_push_wait))) { in xlog_cil_push_background()
1571 up_read(&cil->xc_ctx_lock); in xlog_cil_push_background()
1575 spin_lock(&cil->xc_push_lock); in xlog_cil_push_background()
1576 if (cil->xc_push_seq < cil->xc_current_sequence) { in xlog_cil_push_background()
1577 cil->xc_push_seq = cil->xc_current_sequence; in xlog_cil_push_background()
1578 queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work); in xlog_cil_push_background()
1587 up_read(&cil->xc_ctx_lock); in xlog_cil_push_background()
1596 * The ctx->xc_push_lock provides the serialisation necessary for safely in xlog_cil_push_background()
1600 trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket); in xlog_cil_push_background()
1601 ASSERT(space_used < log->l_logsize); in xlog_cil_push_background()
1602 xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock); in xlog_cil_push_background()
1606 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_background()
1624 * which defeats the purpose of having an async, non-blocking CIL force
1634 struct xfs_cil *cil = log->l_cilp; in xlog_cil_push_now()
1639 ASSERT(push_seq && push_seq <= cil->xc_current_sequence); in xlog_cil_push_now()
1643 flush_workqueue(cil->xc_push_wq); in xlog_cil_push_now()
1645 spin_lock(&cil->xc_push_lock); in xlog_cil_push_now()
1654 cil->xc_push_commit_stable = async; in xlog_cil_push_now()
1658 * there's no more work that we need to do. in xlog_cil_push_now()
1660 if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) || in xlog_cil_push_now()
1661 push_seq <= cil->xc_push_seq) { in xlog_cil_push_now()
1662 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_now()
1666 cil->xc_push_seq = push_seq; in xlog_cil_push_now()
1667 queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work); in xlog_cil_push_now()
1668 spin_unlock(&cil->xc_push_lock); in xlog_cil_push_now()
1675 struct xfs_cil *cil = log->l_cilp; in xlog_cil_empty()
1678 spin_lock(&cil->xc_push_lock); in xlog_cil_empty()
1679 if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) in xlog_cil_empty()
1681 spin_unlock(&cil->xc_push_lock); in xlog_cil_empty()
1703 list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) { in xlog_cil_process_intents()
1704 if (!(lip->li_ops->flags & XFS_ITEM_INTENT_DONE)) in xlog_cil_process_intents()
1707 ilip = lip->li_ops->iop_intent(lip); in xlog_cil_process_intents()
1710 set_bit(XFS_LI_WHITEOUT, &ilip->li_flags); in xlog_cil_process_intents()
1712 len += ilip->li_lv->lv_bytes; in xlog_cil_process_intents()
1713 kvfree(ilip->li_lv); in xlog_cil_process_intents()
1714 ilip->li_lv = NULL; in xlog_cil_process_intents()
1717 lip->li_ops->iop_release(lip); in xlog_cil_process_intents()
1742 struct xfs_cil *cil = log->l_cilp; in xlog_cil_commit()
1754 down_read(&cil->xc_ctx_lock); in xlog_cil_commit()
1756 if (tp->t_flags & XFS_TRANS_HAS_INTENT_DONE) in xlog_cil_commit()
1762 xfs_log_ticket_regrant(log, tp->t_ticket); in xlog_cil_commit()
1764 xfs_log_ticket_ungrant(log, tp->t_ticket); in xlog_cil_commit()
1765 tp->t_ticket = NULL; in xlog_cil_commit()
1780 list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) { in xlog_cil_commit()
1782 if (lip->li_ops->iop_committing) in xlog_cil_commit()
1783 lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence); in xlog_cil_commit()
1786 *commit_seq = cil->xc_ctx->sequence; in xlog_cil_commit()
1788 /* xlog_cil_push_background() releases cil->xc_ctx_lock */ in xlog_cil_commit()
1795 * but otherwise is no different to a push done from a log force.
1801 xfs_csn_t seq = log->l_cilp->xc_current_sequence; in xlog_cil_flush()
1803 trace_xfs_log_force(log->l_mp, seq, _RET_IP_); in xlog_cil_flush()
1810 if (test_bit(XLOG_CIL_EMPTY, &log->l_cilp->xc_flags)) in xlog_cil_flush()
1811 xfs_log_force(log->l_mp, 0); in xlog_cil_flush()
1829 struct xfs_cil *cil = log->l_cilp; in xlog_cil_force_seq()
1833 ASSERT(sequence <= cil->xc_current_sequence); in xlog_cil_force_seq()
1836 sequence = cil->xc_current_sequence; in xlog_cil_force_seq()
1837 trace_xfs_log_force(log->l_mp, sequence, _RET_IP_); in xlog_cil_force_seq()
1842 * so no need to deal with it here. in xlog_cil_force_seq()
1853 spin_lock(&cil->xc_push_lock); in xlog_cil_force_seq()
1854 list_for_each_entry(ctx, &cil->xc_committing, committing) { in xlog_cil_force_seq()
1862 if (ctx->sequence > sequence) in xlog_cil_force_seq()
1864 if (!ctx->commit_lsn) { in xlog_cil_force_seq()
1869 XFS_STATS_INC(log->l_mp, xs_log_force_sleep); in xlog_cil_force_seq()
1870 xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock); in xlog_cil_force_seq()
1873 if (ctx->sequence != sequence) in xlog_cil_force_seq()
1876 commit_lsn = ctx->commit_lsn; in xlog_cil_force_seq()
1894 if (sequence == cil->xc_current_sequence && in xlog_cil_force_seq()
1895 !test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) { in xlog_cil_force_seq()
1896 spin_unlock(&cil->xc_push_lock); in xlog_cil_force_seq()
1900 spin_unlock(&cil->xc_push_lock); in xlog_cil_force_seq()
1911 spin_unlock(&cil->xc_push_lock); in xlog_cil_force_seq()
1929 return -ENOMEM; in xlog_cil_init()
1934 cil->xc_push_wq = alloc_workqueue("xfs-cil/%s", in xlog_cil_init()
1936 4, log->l_mp->m_super->s_id); in xlog_cil_init()
1937 if (!cil->xc_push_wq) in xlog_cil_init()
1940 cil->xc_log = log; in xlog_cil_init()
1941 cil->xc_pcp = alloc_percpu(struct xlog_cil_pcp); in xlog_cil_init()
1942 if (!cil->xc_pcp) in xlog_cil_init()
1946 cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); in xlog_cil_init()
1947 INIT_LIST_HEAD(&cilpcp->busy_extents); in xlog_cil_init()
1948 INIT_LIST_HEAD(&cilpcp->log_items); in xlog_cil_init()
1951 INIT_LIST_HEAD(&cil->xc_committing); in xlog_cil_init()
1952 spin_lock_init(&cil->xc_push_lock); in xlog_cil_init()
1953 init_waitqueue_head(&cil->xc_push_wait); in xlog_cil_init()
1954 init_rwsem(&cil->xc_ctx_lock); in xlog_cil_init()
1955 init_waitqueue_head(&cil->xc_start_wait); in xlog_cil_init()
1956 init_waitqueue_head(&cil->xc_commit_wait); in xlog_cil_init()
1957 log->l_cilp = cil; in xlog_cil_init()
1964 destroy_workqueue(cil->xc_push_wq); in xlog_cil_init()
1967 return -ENOMEM; in xlog_cil_init()
1974 struct xfs_cil *cil = log->l_cilp; in xlog_cil_destroy()
1976 if (cil->xc_ctx) { in xlog_cil_destroy()
1977 if (cil->xc_ctx->ticket) in xlog_cil_destroy()
1978 xfs_log_ticket_put(cil->xc_ctx->ticket); in xlog_cil_destroy()
1979 kfree(cil->xc_ctx); in xlog_cil_destroy()
1982 ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)); in xlog_cil_destroy()
1983 free_percpu(cil->xc_pcp); in xlog_cil_destroy()
1984 destroy_workqueue(cil->xc_push_wq); in xlog_cil_destroy()