1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. 4 * Copyright (c) 2008 Dave Chinner 5 * All Rights Reserved. 6 */ 7 #include "xfs.h" 8 #include "xfs_fs.h" 9 #include "xfs_shared.h" 10 #include "xfs_format.h" 11 #include "xfs_log_format.h" 12 #include "xfs_trans_resv.h" 13 #include "xfs_mount.h" 14 #include "xfs_trans.h" 15 #include "xfs_trans_priv.h" 16 #include "xfs_trace.h" 17 #include "xfs_errortag.h" 18 #include "xfs_error.h" 19 #include "xfs_log.h" 20 #include "xfs_log_priv.h" 21 22 #ifdef DEBUG 23 /* 24 * Check that the list is sorted as it should be. 25 * 26 * Called with the ail lock held, but we don't want to assert fail with it 27 * held otherwise we'll lock everything up and won't be able to debug the 28 * cause. Hence we sample and check the state under the AIL lock and return if 29 * everything is fine, otherwise we drop the lock and run the ASSERT checks. 30 * Asserts may not be fatal, so pick the lock back up and continue onwards. 31 */ 32 STATIC void 33 xfs_ail_check( 34 struct xfs_ail *ailp, 35 struct xfs_log_item *lip) 36 __must_hold(&ailp->ail_lock) 37 { 38 struct xfs_log_item *prev_lip; 39 struct xfs_log_item *next_lip; 40 xfs_lsn_t prev_lsn = NULLCOMMITLSN; 41 xfs_lsn_t next_lsn = NULLCOMMITLSN; 42 xfs_lsn_t lsn; 43 bool in_ail; 44 45 46 if (list_empty(&ailp->ail_head)) 47 return; 48 49 /* 50 * Sample then check the next and previous entries are valid. 51 */ 52 in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags); 53 prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail); 54 if (&prev_lip->li_ail != &ailp->ail_head) 55 prev_lsn = prev_lip->li_lsn; 56 next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail); 57 if (&next_lip->li_ail != &ailp->ail_head) 58 next_lsn = next_lip->li_lsn; 59 lsn = lip->li_lsn; 60 61 if (in_ail && 62 (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) && 63 (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0)) 64 return; 65 66 spin_unlock(&ailp->ail_lock); 67 ASSERT(in_ail); 68 ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0); 69 ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0); 70 spin_lock(&ailp->ail_lock); 71 } 72 #else /* !DEBUG */ 73 #define xfs_ail_check(a,l) 74 #endif /* DEBUG */ 75 76 /* 77 * Return a pointer to the last item in the AIL. If the AIL is empty, then 78 * return NULL. 79 */ 80 static struct xfs_log_item * 81 xfs_ail_max( 82 struct xfs_ail *ailp) 83 { 84 if (list_empty(&ailp->ail_head)) 85 return NULL; 86 87 return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail); 88 } 89 90 /* 91 * Return a pointer to the item which follows the given item in the AIL. If 92 * the given item is the last item in the list, then return NULL. 93 */ 94 static struct xfs_log_item * 95 xfs_ail_next( 96 struct xfs_ail *ailp, 97 struct xfs_log_item *lip) 98 { 99 if (lip->li_ail.next == &ailp->ail_head) 100 return NULL; 101 102 return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail); 103 } 104 105 /* 106 * This is called by the log manager code to determine the LSN of the tail of 107 * the log. This is exactly the LSN of the first item in the AIL. If the AIL 108 * is empty, then this function returns 0. 109 * 110 * We need the AIL lock in order to get a coherent read of the lsn of the last 111 * item in the AIL. 112 */ 113 static xfs_lsn_t 114 __xfs_ail_min_lsn( 115 struct xfs_ail *ailp) 116 { 117 struct xfs_log_item *lip = xfs_ail_min(ailp); 118 119 if (lip) 120 return lip->li_lsn; 121 return 0; 122 } 123 124 xfs_lsn_t 125 xfs_ail_min_lsn( 126 struct xfs_ail *ailp) 127 { 128 xfs_lsn_t lsn; 129 130 spin_lock(&ailp->ail_lock); 131 lsn = __xfs_ail_min_lsn(ailp); 132 spin_unlock(&ailp->ail_lock); 133 134 return lsn; 135 } 136 137 /* 138 * Return the maximum lsn held in the AIL, or zero if the AIL is empty. 139 */ 140 static xfs_lsn_t 141 xfs_ail_max_lsn( 142 struct xfs_ail *ailp) 143 { 144 xfs_lsn_t lsn = 0; 145 struct xfs_log_item *lip; 146 147 spin_lock(&ailp->ail_lock); 148 lip = xfs_ail_max(ailp); 149 if (lip) 150 lsn = lip->li_lsn; 151 spin_unlock(&ailp->ail_lock); 152 153 return lsn; 154 } 155 156 /* 157 * The cursor keeps track of where our current traversal is up to by tracking 158 * the next item in the list for us. However, for this to be safe, removing an 159 * object from the AIL needs to invalidate any cursor that points to it. hence 160 * the traversal cursor needs to be linked to the struct xfs_ail so that 161 * deletion can search all the active cursors for invalidation. 162 */ 163 STATIC void 164 xfs_trans_ail_cursor_init( 165 struct xfs_ail *ailp, 166 struct xfs_ail_cursor *cur) 167 { 168 cur->item = NULL; 169 list_add_tail(&cur->list, &ailp->ail_cursors); 170 } 171 172 /* 173 * Get the next item in the traversal and advance the cursor. If the cursor 174 * was invalidated (indicated by a lip of 1), restart the traversal. 175 */ 176 struct xfs_log_item * 177 xfs_trans_ail_cursor_next( 178 struct xfs_ail *ailp, 179 struct xfs_ail_cursor *cur) 180 { 181 struct xfs_log_item *lip = cur->item; 182 183 if ((uintptr_t)lip & 1) 184 lip = xfs_ail_min(ailp); 185 if (lip) 186 cur->item = xfs_ail_next(ailp, lip); 187 return lip; 188 } 189 190 /* 191 * When the traversal is complete, we need to remove the cursor from the list 192 * of traversing cursors. 193 */ 194 void 195 xfs_trans_ail_cursor_done( 196 struct xfs_ail_cursor *cur) 197 { 198 cur->item = NULL; 199 list_del_init(&cur->list); 200 } 201 202 /* 203 * Invalidate any cursor that is pointing to this item. This is called when an 204 * item is removed from the AIL. Any cursor pointing to this object is now 205 * invalid and the traversal needs to be terminated so it doesn't reference a 206 * freed object. We set the low bit of the cursor item pointer so we can 207 * distinguish between an invalidation and the end of the list when getting the 208 * next item from the cursor. 209 */ 210 STATIC void 211 xfs_trans_ail_cursor_clear( 212 struct xfs_ail *ailp, 213 struct xfs_log_item *lip) 214 { 215 struct xfs_ail_cursor *cur; 216 217 list_for_each_entry(cur, &ailp->ail_cursors, list) { 218 if (cur->item == lip) 219 cur->item = (struct xfs_log_item *) 220 ((uintptr_t)cur->item | 1); 221 } 222 } 223 224 /* 225 * Find the first item in the AIL with the given @lsn by searching in ascending 226 * LSN order and initialise the cursor to point to the next item for a 227 * ascending traversal. Pass a @lsn of zero to initialise the cursor to the 228 * first item in the AIL. Returns NULL if the list is empty. 229 */ 230 struct xfs_log_item * 231 xfs_trans_ail_cursor_first( 232 struct xfs_ail *ailp, 233 struct xfs_ail_cursor *cur, 234 xfs_lsn_t lsn) 235 { 236 struct xfs_log_item *lip; 237 238 xfs_trans_ail_cursor_init(ailp, cur); 239 240 if (lsn == 0) { 241 lip = xfs_ail_min(ailp); 242 goto out; 243 } 244 245 list_for_each_entry(lip, &ailp->ail_head, li_ail) { 246 if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0) 247 goto out; 248 } 249 return NULL; 250 251 out: 252 if (lip) 253 cur->item = xfs_ail_next(ailp, lip); 254 return lip; 255 } 256 257 static struct xfs_log_item * 258 __xfs_trans_ail_cursor_last( 259 struct xfs_ail *ailp, 260 xfs_lsn_t lsn) 261 { 262 struct xfs_log_item *lip; 263 264 list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) { 265 if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0) 266 return lip; 267 } 268 return NULL; 269 } 270 271 /* 272 * Find the last item in the AIL with the given @lsn by searching in descending 273 * LSN order and initialise the cursor to point to that item. If there is no 274 * item with the value of @lsn, then it sets the cursor to the last item with an 275 * LSN lower than @lsn. Returns NULL if the list is empty. 276 */ 277 struct xfs_log_item * 278 xfs_trans_ail_cursor_last( 279 struct xfs_ail *ailp, 280 struct xfs_ail_cursor *cur, 281 xfs_lsn_t lsn) 282 { 283 xfs_trans_ail_cursor_init(ailp, cur); 284 cur->item = __xfs_trans_ail_cursor_last(ailp, lsn); 285 return cur->item; 286 } 287 288 /* 289 * Splice the log item list into the AIL at the given LSN. We splice to the 290 * tail of the given LSN to maintain insert order for push traversals. The 291 * cursor is optional, allowing repeated updates to the same LSN to avoid 292 * repeated traversals. This should not be called with an empty list. 293 */ 294 static void 295 xfs_ail_splice( 296 struct xfs_ail *ailp, 297 struct xfs_ail_cursor *cur, 298 struct list_head *list, 299 xfs_lsn_t lsn) 300 { 301 struct xfs_log_item *lip; 302 303 ASSERT(!list_empty(list)); 304 305 /* 306 * Use the cursor to determine the insertion point if one is 307 * provided. If not, or if the one we got is not valid, 308 * find the place in the AIL where the items belong. 309 */ 310 lip = cur ? cur->item : NULL; 311 if (!lip || (uintptr_t)lip & 1) 312 lip = __xfs_trans_ail_cursor_last(ailp, lsn); 313 314 /* 315 * If a cursor is provided, we know we're processing the AIL 316 * in lsn order, and future items to be spliced in will 317 * follow the last one being inserted now. Update the 318 * cursor to point to that last item, now while we have a 319 * reliable pointer to it. 320 */ 321 if (cur) 322 cur->item = list_entry(list->prev, struct xfs_log_item, li_ail); 323 324 /* 325 * Finally perform the splice. Unless the AIL was empty, 326 * lip points to the item in the AIL _after_ which the new 327 * items should go. If lip is null the AIL was empty, so 328 * the new items go at the head of the AIL. 329 */ 330 if (lip) 331 list_splice(list, &lip->li_ail); 332 else 333 list_splice(list, &ailp->ail_head); 334 } 335 336 /* 337 * Delete the given item from the AIL. Return a pointer to the item. 338 */ 339 static void 340 xfs_ail_delete( 341 struct xfs_ail *ailp, 342 struct xfs_log_item *lip) 343 { 344 xfs_ail_check(ailp, lip); 345 list_del(&lip->li_ail); 346 xfs_trans_ail_cursor_clear(ailp, lip); 347 } 348 349 /* 350 * Requeue a failed buffer for writeback. 351 * 352 * We clear the log item failed state here as well, but we have to be careful 353 * about reference counts because the only active reference counts on the buffer 354 * may be the failed log items. Hence if we clear the log item failed state 355 * before queuing the buffer for IO we can release all active references to 356 * the buffer and free it, leading to use after free problems in 357 * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which 358 * order we process them in - the buffer is locked, and we own the buffer list 359 * so nothing on them is going to change while we are performing this action. 360 * 361 * Hence we can safely queue the buffer for IO before we clear the failed log 362 * item state, therefore always having an active reference to the buffer and 363 * avoiding the transient zero-reference state that leads to use-after-free. 364 */ 365 static inline int 366 xfsaild_resubmit_item( 367 struct xfs_log_item *lip, 368 struct list_head *buffer_list) 369 { 370 struct xfs_buf *bp = lip->li_buf; 371 372 if (!xfs_buf_trylock(bp)) 373 return XFS_ITEM_LOCKED; 374 375 if (!xfs_buf_delwri_queue(bp, buffer_list)) { 376 xfs_buf_unlock(bp); 377 return XFS_ITEM_FLUSHING; 378 } 379 380 /* protected by ail_lock */ 381 list_for_each_entry(lip, &bp->b_li_list, li_bio_list) { 382 if (bp->b_flags & _XBF_INODES) 383 clear_bit(XFS_LI_FAILED, &lip->li_flags); 384 else 385 xfs_clear_li_failed(lip); 386 } 387 388 xfs_buf_unlock(bp); 389 return XFS_ITEM_SUCCESS; 390 } 391 392 static inline uint 393 xfsaild_push_item( 394 struct xfs_ail *ailp, 395 struct xfs_log_item *lip) 396 { 397 /* 398 * If log item pinning is enabled, skip the push and track the item as 399 * pinned. This can help induce head-behind-tail conditions. 400 */ 401 if (XFS_TEST_ERROR(false, ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN)) 402 return XFS_ITEM_PINNED; 403 404 /* 405 * Consider the item pinned if a push callback is not defined so the 406 * caller will force the log. This should only happen for intent items 407 * as they are unpinned once the associated done item is committed to 408 * the on-disk log. 409 */ 410 if (!lip->li_ops->iop_push) 411 return XFS_ITEM_PINNED; 412 if (test_bit(XFS_LI_FAILED, &lip->li_flags)) 413 return xfsaild_resubmit_item(lip, &ailp->ail_buf_list); 414 return lip->li_ops->iop_push(lip, &ailp->ail_buf_list); 415 } 416 417 static long 418 xfsaild_push( 419 struct xfs_ail *ailp) 420 { 421 struct xfs_mount *mp = ailp->ail_log->l_mp; 422 struct xfs_ail_cursor cur; 423 struct xfs_log_item *lip; 424 xfs_lsn_t lsn; 425 xfs_lsn_t target = NULLCOMMITLSN; 426 long tout; 427 int stuck = 0; 428 int flushing = 0; 429 int count = 0; 430 431 /* 432 * If we encountered pinned items or did not finish writing out all 433 * buffers the last time we ran, force a background CIL push to get the 434 * items unpinned in the near future. We do not wait on the CIL push as 435 * that could stall us for seconds if there is enough background IO 436 * load. Stalling for that long when the tail of the log is pinned and 437 * needs flushing will hard stop the transaction subsystem when log 438 * space runs out. 439 */ 440 if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 && 441 (!list_empty_careful(&ailp->ail_buf_list) || 442 xfs_ail_min_lsn(ailp))) { 443 ailp->ail_log_flush = 0; 444 445 XFS_STATS_INC(mp, xs_push_ail_flush); 446 xlog_cil_flush(ailp->ail_log); 447 } 448 449 spin_lock(&ailp->ail_lock); 450 451 /* 452 * If we have a sync push waiter, we always have to push till the AIL is 453 * empty. Update the target to point to the end of the AIL so that 454 * capture updates that occur after the sync push waiter has gone to 455 * sleep. 456 */ 457 if (waitqueue_active(&ailp->ail_empty)) { 458 lip = xfs_ail_max(ailp); 459 if (lip) 460 target = lip->li_lsn; 461 } else { 462 /* barrier matches the ail_target update in xfs_ail_push() */ 463 smp_rmb(); 464 target = ailp->ail_target; 465 ailp->ail_target_prev = target; 466 } 467 468 /* we're done if the AIL is empty or our push has reached the end */ 469 lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn); 470 if (!lip) 471 goto out_done; 472 473 XFS_STATS_INC(mp, xs_push_ail); 474 475 ASSERT(target != NULLCOMMITLSN); 476 477 lsn = lip->li_lsn; 478 while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) { 479 int lock_result; 480 481 /* 482 * Note that iop_push may unlock and reacquire the AIL lock. We 483 * rely on the AIL cursor implementation to be able to deal with 484 * the dropped lock. 485 */ 486 lock_result = xfsaild_push_item(ailp, lip); 487 switch (lock_result) { 488 case XFS_ITEM_SUCCESS: 489 XFS_STATS_INC(mp, xs_push_ail_success); 490 trace_xfs_ail_push(lip); 491 492 ailp->ail_last_pushed_lsn = lsn; 493 break; 494 495 case XFS_ITEM_FLUSHING: 496 /* 497 * The item or its backing buffer is already being 498 * flushed. The typical reason for that is that an 499 * inode buffer is locked because we already pushed the 500 * updates to it as part of inode clustering. 501 * 502 * We do not want to stop flushing just because lots 503 * of items are already being flushed, but we need to 504 * re-try the flushing relatively soon if most of the 505 * AIL is being flushed. 506 */ 507 XFS_STATS_INC(mp, xs_push_ail_flushing); 508 trace_xfs_ail_flushing(lip); 509 510 flushing++; 511 ailp->ail_last_pushed_lsn = lsn; 512 break; 513 514 case XFS_ITEM_PINNED: 515 XFS_STATS_INC(mp, xs_push_ail_pinned); 516 trace_xfs_ail_pinned(lip); 517 518 stuck++; 519 ailp->ail_log_flush++; 520 break; 521 case XFS_ITEM_LOCKED: 522 XFS_STATS_INC(mp, xs_push_ail_locked); 523 trace_xfs_ail_locked(lip); 524 525 stuck++; 526 break; 527 default: 528 ASSERT(0); 529 break; 530 } 531 532 count++; 533 534 /* 535 * Are there too many items we can't do anything with? 536 * 537 * If we are skipping too many items because we can't flush 538 * them or they are already being flushed, we back off and 539 * given them time to complete whatever operation is being 540 * done. i.e. remove pressure from the AIL while we can't make 541 * progress so traversals don't slow down further inserts and 542 * removals to/from the AIL. 543 * 544 * The value of 100 is an arbitrary magic number based on 545 * observation. 546 */ 547 if (stuck > 100) 548 break; 549 550 lip = xfs_trans_ail_cursor_next(ailp, &cur); 551 if (lip == NULL) 552 break; 553 lsn = lip->li_lsn; 554 } 555 556 out_done: 557 xfs_trans_ail_cursor_done(&cur); 558 spin_unlock(&ailp->ail_lock); 559 560 if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list)) 561 ailp->ail_log_flush++; 562 563 if (!count || XFS_LSN_CMP(lsn, target) >= 0) { 564 /* 565 * We reached the target or the AIL is empty, so wait a bit 566 * longer for I/O to complete and remove pushed items from the 567 * AIL before we start the next scan from the start of the AIL. 568 */ 569 tout = 50; 570 ailp->ail_last_pushed_lsn = 0; 571 } else if (((stuck + flushing) * 100) / count > 90) { 572 /* 573 * Either there is a lot of contention on the AIL or we are 574 * stuck due to operations in progress. "Stuck" in this case 575 * is defined as >90% of the items we tried to push were stuck. 576 * 577 * Backoff a bit more to allow some I/O to complete before 578 * restarting from the start of the AIL. This prevents us from 579 * spinning on the same items, and if they are pinned will all 580 * the restart to issue a log force to unpin the stuck items. 581 */ 582 tout = 20; 583 ailp->ail_last_pushed_lsn = 0; 584 } else { 585 /* 586 * Assume we have more work to do in a short while. 587 */ 588 tout = 10; 589 } 590 591 return tout; 592 } 593 594 static int 595 xfsaild( 596 void *data) 597 { 598 struct xfs_ail *ailp = data; 599 long tout = 0; /* milliseconds */ 600 unsigned int noreclaim_flag; 601 602 noreclaim_flag = memalloc_noreclaim_save(); 603 set_freezable(); 604 605 while (1) { 606 if (tout && tout <= 20) 607 set_current_state(TASK_KILLABLE|TASK_FREEZABLE); 608 else 609 set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); 610 611 /* 612 * Check kthread_should_stop() after we set the task state to 613 * guarantee that we either see the stop bit and exit or the 614 * task state is reset to runnable such that it's not scheduled 615 * out indefinitely and detects the stop bit at next iteration. 616 * A memory barrier is included in above task state set to 617 * serialize again kthread_stop(). 618 */ 619 if (kthread_should_stop()) { 620 __set_current_state(TASK_RUNNING); 621 622 /* 623 * The caller forces out the AIL before stopping the 624 * thread in the common case, which means the delwri 625 * queue is drained. In the shutdown case, the queue may 626 * still hold relogged buffers that haven't been 627 * submitted because they were pinned since added to the 628 * queue. 629 * 630 * Log I/O error processing stales the underlying buffer 631 * and clears the delwri state, expecting the buf to be 632 * removed on the next submission attempt. That won't 633 * happen if we're shutting down, so this is the last 634 * opportunity to release such buffers from the queue. 635 */ 636 ASSERT(list_empty(&ailp->ail_buf_list) || 637 xlog_is_shutdown(ailp->ail_log)); 638 xfs_buf_delwri_cancel(&ailp->ail_buf_list); 639 break; 640 } 641 642 spin_lock(&ailp->ail_lock); 643 644 /* 645 * Idle if the AIL is empty and we are not racing with a target 646 * update. We check the AIL after we set the task to a sleep 647 * state to guarantee that we either catch an ail_target update 648 * or that a wake_up resets the state to TASK_RUNNING. 649 * Otherwise, we run the risk of sleeping indefinitely. 650 * 651 * The barrier matches the ail_target update in xfs_ail_push(). 652 */ 653 smp_rmb(); 654 if (!xfs_ail_min(ailp) && 655 ailp->ail_target == ailp->ail_target_prev && 656 list_empty(&ailp->ail_buf_list)) { 657 spin_unlock(&ailp->ail_lock); 658 schedule(); 659 tout = 0; 660 continue; 661 } 662 spin_unlock(&ailp->ail_lock); 663 664 if (tout) 665 schedule_timeout(msecs_to_jiffies(tout)); 666 667 __set_current_state(TASK_RUNNING); 668 669 try_to_freeze(); 670 671 tout = xfsaild_push(ailp); 672 } 673 674 memalloc_noreclaim_restore(noreclaim_flag); 675 return 0; 676 } 677 678 /* 679 * This routine is called to move the tail of the AIL forward. It does this by 680 * trying to flush items in the AIL whose lsns are below the given 681 * threshold_lsn. 682 * 683 * The push is run asynchronously in a workqueue, which means the caller needs 684 * to handle waiting on the async flush for space to become available. 685 * We don't want to interrupt any push that is in progress, hence we only queue 686 * work if we set the pushing bit appropriately. 687 * 688 * We do this unlocked - we only need to know whether there is anything in the 689 * AIL at the time we are called. We don't need to access the contents of 690 * any of the objects, so the lock is not needed. 691 */ 692 void 693 xfs_ail_push( 694 struct xfs_ail *ailp, 695 xfs_lsn_t threshold_lsn) 696 { 697 struct xfs_log_item *lip; 698 699 lip = xfs_ail_min(ailp); 700 if (!lip || xlog_is_shutdown(ailp->ail_log) || 701 XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0) 702 return; 703 704 /* 705 * Ensure that the new target is noticed in push code before it clears 706 * the XFS_AIL_PUSHING_BIT. 707 */ 708 smp_wmb(); 709 xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn); 710 smp_wmb(); 711 712 wake_up_process(ailp->ail_task); 713 } 714 715 /* 716 * Push out all items in the AIL immediately 717 */ 718 void 719 xfs_ail_push_all( 720 struct xfs_ail *ailp) 721 { 722 xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp); 723 724 if (threshold_lsn) 725 xfs_ail_push(ailp, threshold_lsn); 726 } 727 728 /* 729 * Push out all items in the AIL immediately and wait until the AIL is empty. 730 */ 731 void 732 xfs_ail_push_all_sync( 733 struct xfs_ail *ailp) 734 { 735 DEFINE_WAIT(wait); 736 737 spin_lock(&ailp->ail_lock); 738 while (xfs_ail_max(ailp) != NULL) { 739 prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE); 740 wake_up_process(ailp->ail_task); 741 spin_unlock(&ailp->ail_lock); 742 schedule(); 743 spin_lock(&ailp->ail_lock); 744 } 745 spin_unlock(&ailp->ail_lock); 746 747 finish_wait(&ailp->ail_empty, &wait); 748 } 749 750 void 751 xfs_ail_update_finish( 752 struct xfs_ail *ailp, 753 xfs_lsn_t old_lsn) __releases(ailp->ail_lock) 754 { 755 struct xlog *log = ailp->ail_log; 756 757 /* if the tail lsn hasn't changed, don't do updates or wakeups. */ 758 if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) { 759 spin_unlock(&ailp->ail_lock); 760 return; 761 } 762 763 if (!xlog_is_shutdown(log)) 764 xlog_assign_tail_lsn_locked(log->l_mp); 765 766 if (list_empty(&ailp->ail_head)) 767 wake_up_all(&ailp->ail_empty); 768 spin_unlock(&ailp->ail_lock); 769 xfs_log_space_wake(log->l_mp); 770 } 771 772 /* 773 * xfs_trans_ail_update - bulk AIL insertion operation. 774 * 775 * @xfs_trans_ail_update takes an array of log items that all need to be 776 * positioned at the same LSN in the AIL. If an item is not in the AIL, it will 777 * be added. Otherwise, it will be repositioned by removing it and re-adding 778 * it to the AIL. If we move the first item in the AIL, update the log tail to 779 * match the new minimum LSN in the AIL. 780 * 781 * This function takes the AIL lock once to execute the update operations on 782 * all the items in the array, and as such should not be called with the AIL 783 * lock held. As a result, once we have the AIL lock, we need to check each log 784 * item LSN to confirm it needs to be moved forward in the AIL. 785 * 786 * To optimise the insert operation, we delete all the items from the AIL in 787 * the first pass, moving them into a temporary list, then splice the temporary 788 * list into the correct position in the AIL. This avoids needing to do an 789 * insert operation on every item. 790 * 791 * This function must be called with the AIL lock held. The lock is dropped 792 * before returning. 793 */ 794 void 795 xfs_trans_ail_update_bulk( 796 struct xfs_ail *ailp, 797 struct xfs_ail_cursor *cur, 798 struct xfs_log_item **log_items, 799 int nr_items, 800 xfs_lsn_t lsn) __releases(ailp->ail_lock) 801 { 802 struct xfs_log_item *mlip; 803 xfs_lsn_t tail_lsn = 0; 804 int i; 805 LIST_HEAD(tmp); 806 807 ASSERT(nr_items > 0); /* Not required, but true. */ 808 mlip = xfs_ail_min(ailp); 809 810 for (i = 0; i < nr_items; i++) { 811 struct xfs_log_item *lip = log_items[i]; 812 if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) { 813 /* check if we really need to move the item */ 814 if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0) 815 continue; 816 817 trace_xfs_ail_move(lip, lip->li_lsn, lsn); 818 if (mlip == lip && !tail_lsn) 819 tail_lsn = lip->li_lsn; 820 821 xfs_ail_delete(ailp, lip); 822 } else { 823 trace_xfs_ail_insert(lip, 0, lsn); 824 } 825 lip->li_lsn = lsn; 826 list_add_tail(&lip->li_ail, &tmp); 827 } 828 829 if (!list_empty(&tmp)) 830 xfs_ail_splice(ailp, cur, &tmp, lsn); 831 832 xfs_ail_update_finish(ailp, tail_lsn); 833 } 834 835 /* Insert a log item into the AIL. */ 836 void 837 xfs_trans_ail_insert( 838 struct xfs_ail *ailp, 839 struct xfs_log_item *lip, 840 xfs_lsn_t lsn) 841 { 842 spin_lock(&ailp->ail_lock); 843 xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); 844 } 845 846 /* 847 * Delete one log item from the AIL. 848 * 849 * If this item was at the tail of the AIL, return the LSN of the log item so 850 * that we can use it to check if the LSN of the tail of the log has moved 851 * when finishing up the AIL delete process in xfs_ail_update_finish(). 852 */ 853 xfs_lsn_t 854 xfs_ail_delete_one( 855 struct xfs_ail *ailp, 856 struct xfs_log_item *lip) 857 { 858 struct xfs_log_item *mlip = xfs_ail_min(ailp); 859 xfs_lsn_t lsn = lip->li_lsn; 860 861 trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn); 862 xfs_ail_delete(ailp, lip); 863 clear_bit(XFS_LI_IN_AIL, &lip->li_flags); 864 lip->li_lsn = 0; 865 866 if (mlip == lip) 867 return lsn; 868 return 0; 869 } 870 871 void 872 xfs_trans_ail_delete( 873 struct xfs_log_item *lip, 874 int shutdown_type) 875 { 876 struct xfs_ail *ailp = lip->li_ailp; 877 struct xlog *log = ailp->ail_log; 878 xfs_lsn_t tail_lsn; 879 880 spin_lock(&ailp->ail_lock); 881 if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { 882 spin_unlock(&ailp->ail_lock); 883 if (shutdown_type && !xlog_is_shutdown(log)) { 884 xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE, 885 "%s: attempting to delete a log item that is not in the AIL", 886 __func__); 887 xlog_force_shutdown(log, shutdown_type); 888 } 889 return; 890 } 891 892 /* xfs_ail_update_finish() drops the AIL lock */ 893 xfs_clear_li_failed(lip); 894 tail_lsn = xfs_ail_delete_one(ailp, lip); 895 xfs_ail_update_finish(ailp, tail_lsn); 896 } 897 898 int 899 xfs_trans_ail_init( 900 xfs_mount_t *mp) 901 { 902 struct xfs_ail *ailp; 903 904 ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL); 905 if (!ailp) 906 return -ENOMEM; 907 908 ailp->ail_log = mp->m_log; 909 INIT_LIST_HEAD(&ailp->ail_head); 910 INIT_LIST_HEAD(&ailp->ail_cursors); 911 spin_lock_init(&ailp->ail_lock); 912 INIT_LIST_HEAD(&ailp->ail_buf_list); 913 init_waitqueue_head(&ailp->ail_empty); 914 915 ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s", 916 mp->m_super->s_id); 917 if (IS_ERR(ailp->ail_task)) 918 goto out_free_ailp; 919 920 mp->m_ail = ailp; 921 return 0; 922 923 out_free_ailp: 924 kmem_free(ailp); 925 return -ENOMEM; 926 } 927 928 void 929 xfs_trans_ail_destroy( 930 xfs_mount_t *mp) 931 { 932 struct xfs_ail *ailp = mp->m_ail; 933 934 kthread_stop(ailp->ail_task); 935 kmem_free(ailp); 936 } 937