1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_fs.h" 20 #include "xfs_types.h" 21 #include "xfs_bit.h" 22 #include "xfs_log.h" 23 #include "xfs_inum.h" 24 #include "xfs_trans.h" 25 #include "xfs_sb.h" 26 #include "xfs_ag.h" 27 #include "xfs_dmapi.h" 28 #include "xfs_mount.h" 29 #include "xfs_buf_item.h" 30 #include "xfs_trans_priv.h" 31 #include "xfs_error.h" 32 33 34 kmem_zone_t *xfs_buf_item_zone; 35 36 #ifdef XFS_TRANS_DEBUG 37 /* 38 * This function uses an alternate strategy for tracking the bytes 39 * that the user requests to be logged. This can then be used 40 * in conjunction with the bli_orig array in the buf log item to 41 * catch bugs in our callers' code. 42 * 43 * We also double check the bits set in xfs_buf_item_log using a 44 * simple algorithm to check that every byte is accounted for. 45 */ 46 STATIC void 47 xfs_buf_item_log_debug( 48 xfs_buf_log_item_t *bip, 49 uint first, 50 uint last) 51 { 52 uint x; 53 uint byte; 54 uint nbytes; 55 uint chunk_num; 56 uint word_num; 57 uint bit_num; 58 uint bit_set; 59 uint *wordp; 60 61 ASSERT(bip->bli_logged != NULL); 62 byte = first; 63 nbytes = last - first + 1; 64 bfset(bip->bli_logged, first, nbytes); 65 for (x = 0; x < nbytes; x++) { 66 chunk_num = byte >> XFS_BLI_SHIFT; 67 word_num = chunk_num >> BIT_TO_WORD_SHIFT; 68 bit_num = chunk_num & (NBWORD - 1); 69 wordp = &(bip->bli_format.blf_data_map[word_num]); 70 bit_set = *wordp & (1 << bit_num); 71 ASSERT(bit_set); 72 byte++; 73 } 74 } 75 76 /* 77 * This function is called when we flush something into a buffer without 78 * logging it. This happens for things like inodes which are logged 79 * separately from the buffer. 80 */ 81 void 82 xfs_buf_item_flush_log_debug( 83 xfs_buf_t *bp, 84 uint first, 85 uint last) 86 { 87 xfs_buf_log_item_t *bip; 88 uint nbytes; 89 90 bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*); 91 if ((bip == NULL) || (bip->bli_item.li_type != XFS_LI_BUF)) { 92 return; 93 } 94 95 ASSERT(bip->bli_logged != NULL); 96 nbytes = last - first + 1; 97 bfset(bip->bli_logged, first, nbytes); 98 } 99 100 /* 101 * This function is called to verify that our callers have logged 102 * all the bytes that they changed. 103 * 104 * It does this by comparing the original copy of the buffer stored in 105 * the buf log item's bli_orig array to the current copy of the buffer 106 * and ensuring that all bytes which mismatch are set in the bli_logged 107 * array of the buf log item. 108 */ 109 STATIC void 110 xfs_buf_item_log_check( 111 xfs_buf_log_item_t *bip) 112 { 113 char *orig; 114 char *buffer; 115 int x; 116 xfs_buf_t *bp; 117 118 ASSERT(bip->bli_orig != NULL); 119 ASSERT(bip->bli_logged != NULL); 120 121 bp = bip->bli_buf; 122 ASSERT(XFS_BUF_COUNT(bp) > 0); 123 ASSERT(XFS_BUF_PTR(bp) != NULL); 124 orig = bip->bli_orig; 125 buffer = XFS_BUF_PTR(bp); 126 for (x = 0; x < XFS_BUF_COUNT(bp); x++) { 127 if (orig[x] != buffer[x] && !btst(bip->bli_logged, x)) 128 cmn_err(CE_PANIC, 129 "xfs_buf_item_log_check bip %x buffer %x orig %x index %d", 130 bip, bp, orig, x); 131 } 132 } 133 #else 134 #define xfs_buf_item_log_debug(x,y,z) 135 #define xfs_buf_item_log_check(x) 136 #endif 137 138 STATIC void xfs_buf_error_relse(xfs_buf_t *bp); 139 STATIC void xfs_buf_do_callbacks(xfs_buf_t *bp, xfs_log_item_t *lip); 140 141 /* 142 * This returns the number of log iovecs needed to log the 143 * given buf log item. 144 * 145 * It calculates this as 1 iovec for the buf log format structure 146 * and 1 for each stretch of non-contiguous chunks to be logged. 147 * Contiguous chunks are logged in a single iovec. 148 * 149 * If the XFS_BLI_STALE flag has been set, then log nothing. 150 */ 151 STATIC uint 152 xfs_buf_item_size( 153 xfs_buf_log_item_t *bip) 154 { 155 uint nvecs; 156 int next_bit; 157 int last_bit; 158 xfs_buf_t *bp; 159 160 ASSERT(atomic_read(&bip->bli_refcount) > 0); 161 if (bip->bli_flags & XFS_BLI_STALE) { 162 /* 163 * The buffer is stale, so all we need to log 164 * is the buf log format structure with the 165 * cancel flag in it. 166 */ 167 xfs_buf_item_trace("SIZE STALE", bip); 168 ASSERT(bip->bli_format.blf_flags & XFS_BLI_CANCEL); 169 return 1; 170 } 171 172 bp = bip->bli_buf; 173 ASSERT(bip->bli_flags & XFS_BLI_LOGGED); 174 nvecs = 1; 175 last_bit = xfs_next_bit(bip->bli_format.blf_data_map, 176 bip->bli_format.blf_map_size, 0); 177 ASSERT(last_bit != -1); 178 nvecs++; 179 while (last_bit != -1) { 180 /* 181 * This takes the bit number to start looking from and 182 * returns the next set bit from there. It returns -1 183 * if there are no more bits set or the start bit is 184 * beyond the end of the bitmap. 185 */ 186 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 187 bip->bli_format.blf_map_size, 188 last_bit + 1); 189 /* 190 * If we run out of bits, leave the loop, 191 * else if we find a new set of bits bump the number of vecs, 192 * else keep scanning the current set of bits. 193 */ 194 if (next_bit == -1) { 195 last_bit = -1; 196 } else if (next_bit != last_bit + 1) { 197 last_bit = next_bit; 198 nvecs++; 199 } else if (xfs_buf_offset(bp, next_bit * XFS_BLI_CHUNK) != 200 (xfs_buf_offset(bp, last_bit * XFS_BLI_CHUNK) + 201 XFS_BLI_CHUNK)) { 202 last_bit = next_bit; 203 nvecs++; 204 } else { 205 last_bit++; 206 } 207 } 208 209 xfs_buf_item_trace("SIZE NORM", bip); 210 return nvecs; 211 } 212 213 /* 214 * This is called to fill in the vector of log iovecs for the 215 * given log buf item. It fills the first entry with a buf log 216 * format structure, and the rest point to contiguous chunks 217 * within the buffer. 218 */ 219 STATIC void 220 xfs_buf_item_format( 221 xfs_buf_log_item_t *bip, 222 xfs_log_iovec_t *log_vector) 223 { 224 uint base_size; 225 uint nvecs; 226 xfs_log_iovec_t *vecp; 227 xfs_buf_t *bp; 228 int first_bit; 229 int last_bit; 230 int next_bit; 231 uint nbits; 232 uint buffer_offset; 233 234 ASSERT(atomic_read(&bip->bli_refcount) > 0); 235 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 236 (bip->bli_flags & XFS_BLI_STALE)); 237 bp = bip->bli_buf; 238 vecp = log_vector; 239 240 /* 241 * The size of the base structure is the size of the 242 * declared structure plus the space for the extra words 243 * of the bitmap. We subtract one from the map size, because 244 * the first element of the bitmap is accounted for in the 245 * size of the base structure. 246 */ 247 base_size = 248 (uint)(sizeof(xfs_buf_log_format_t) + 249 ((bip->bli_format.blf_map_size - 1) * sizeof(uint))); 250 vecp->i_addr = (xfs_caddr_t)&bip->bli_format; 251 vecp->i_len = base_size; 252 XLOG_VEC_SET_TYPE(vecp, XLOG_REG_TYPE_BFORMAT); 253 vecp++; 254 nvecs = 1; 255 256 if (bip->bli_flags & XFS_BLI_STALE) { 257 /* 258 * The buffer is stale, so all we need to log 259 * is the buf log format structure with the 260 * cancel flag in it. 261 */ 262 xfs_buf_item_trace("FORMAT STALE", bip); 263 ASSERT(bip->bli_format.blf_flags & XFS_BLI_CANCEL); 264 bip->bli_format.blf_size = nvecs; 265 return; 266 } 267 268 /* 269 * Fill in an iovec for each set of contiguous chunks. 270 */ 271 first_bit = xfs_next_bit(bip->bli_format.blf_data_map, 272 bip->bli_format.blf_map_size, 0); 273 ASSERT(first_bit != -1); 274 last_bit = first_bit; 275 nbits = 1; 276 for (;;) { 277 /* 278 * This takes the bit number to start looking from and 279 * returns the next set bit from there. It returns -1 280 * if there are no more bits set or the start bit is 281 * beyond the end of the bitmap. 282 */ 283 next_bit = xfs_next_bit(bip->bli_format.blf_data_map, 284 bip->bli_format.blf_map_size, 285 (uint)last_bit + 1); 286 /* 287 * If we run out of bits fill in the last iovec and get 288 * out of the loop. 289 * Else if we start a new set of bits then fill in the 290 * iovec for the series we were looking at and start 291 * counting the bits in the new one. 292 * Else we're still in the same set of bits so just 293 * keep counting and scanning. 294 */ 295 if (next_bit == -1) { 296 buffer_offset = first_bit * XFS_BLI_CHUNK; 297 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 298 vecp->i_len = nbits * XFS_BLI_CHUNK; 299 XLOG_VEC_SET_TYPE(vecp, XLOG_REG_TYPE_BCHUNK); 300 nvecs++; 301 break; 302 } else if (next_bit != last_bit + 1) { 303 buffer_offset = first_bit * XFS_BLI_CHUNK; 304 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 305 vecp->i_len = nbits * XFS_BLI_CHUNK; 306 XLOG_VEC_SET_TYPE(vecp, XLOG_REG_TYPE_BCHUNK); 307 nvecs++; 308 vecp++; 309 first_bit = next_bit; 310 last_bit = next_bit; 311 nbits = 1; 312 } else if (xfs_buf_offset(bp, next_bit << XFS_BLI_SHIFT) != 313 (xfs_buf_offset(bp, last_bit << XFS_BLI_SHIFT) + 314 XFS_BLI_CHUNK)) { 315 buffer_offset = first_bit * XFS_BLI_CHUNK; 316 vecp->i_addr = xfs_buf_offset(bp, buffer_offset); 317 vecp->i_len = nbits * XFS_BLI_CHUNK; 318 XLOG_VEC_SET_TYPE(vecp, XLOG_REG_TYPE_BCHUNK); 319 /* You would think we need to bump the nvecs here too, but we do not 320 * this number is used by recovery, and it gets confused by the boundary 321 * split here 322 * nvecs++; 323 */ 324 vecp++; 325 first_bit = next_bit; 326 last_bit = next_bit; 327 nbits = 1; 328 } else { 329 last_bit++; 330 nbits++; 331 } 332 } 333 bip->bli_format.blf_size = nvecs; 334 335 /* 336 * Check to make sure everything is consistent. 337 */ 338 xfs_buf_item_trace("FORMAT NORM", bip); 339 xfs_buf_item_log_check(bip); 340 } 341 342 /* 343 * This is called to pin the buffer associated with the buf log 344 * item in memory so it cannot be written out. Simply call bpin() 345 * on the buffer to do this. 346 */ 347 STATIC void 348 xfs_buf_item_pin( 349 xfs_buf_log_item_t *bip) 350 { 351 xfs_buf_t *bp; 352 353 bp = bip->bli_buf; 354 ASSERT(XFS_BUF_ISBUSY(bp)); 355 ASSERT(atomic_read(&bip->bli_refcount) > 0); 356 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || 357 (bip->bli_flags & XFS_BLI_STALE)); 358 xfs_buf_item_trace("PIN", bip); 359 xfs_buftrace("XFS_PIN", bp); 360 xfs_bpin(bp); 361 } 362 363 364 /* 365 * This is called to unpin the buffer associated with the buf log 366 * item which was previously pinned with a call to xfs_buf_item_pin(). 367 * Just call bunpin() on the buffer to do this. 368 * 369 * Also drop the reference to the buf item for the current transaction. 370 * If the XFS_BLI_STALE flag is set and we are the last reference, 371 * then free up the buf log item and unlock the buffer. 372 */ 373 STATIC void 374 xfs_buf_item_unpin( 375 xfs_buf_log_item_t *bip, 376 int stale) 377 { 378 xfs_mount_t *mp; 379 xfs_buf_t *bp; 380 int freed; 381 SPLDECL(s); 382 383 bp = bip->bli_buf; 384 ASSERT(bp != NULL); 385 ASSERT(XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t *) == bip); 386 ASSERT(atomic_read(&bip->bli_refcount) > 0); 387 xfs_buf_item_trace("UNPIN", bip); 388 xfs_buftrace("XFS_UNPIN", bp); 389 390 freed = atomic_dec_and_test(&bip->bli_refcount); 391 mp = bip->bli_item.li_mountp; 392 xfs_bunpin(bp); 393 if (freed && stale) { 394 ASSERT(bip->bli_flags & XFS_BLI_STALE); 395 ASSERT(XFS_BUF_VALUSEMA(bp) <= 0); 396 ASSERT(!(XFS_BUF_ISDELAYWRITE(bp))); 397 ASSERT(XFS_BUF_ISSTALE(bp)); 398 ASSERT(bip->bli_format.blf_flags & XFS_BLI_CANCEL); 399 xfs_buf_item_trace("UNPIN STALE", bip); 400 xfs_buftrace("XFS_UNPIN STALE", bp); 401 /* 402 * If we get called here because of an IO error, we may 403 * or may not have the item on the AIL. xfs_trans_delete_ail() 404 * will take care of that situation. 405 * xfs_trans_delete_ail() drops the AIL lock. 406 */ 407 if (bip->bli_flags & XFS_BLI_STALE_INODE) { 408 xfs_buf_do_callbacks(bp, (xfs_log_item_t *)bip); 409 XFS_BUF_SET_FSPRIVATE(bp, NULL); 410 XFS_BUF_CLR_IODONE_FUNC(bp); 411 } else { 412 AIL_LOCK(mp,s); 413 xfs_trans_delete_ail(mp, (xfs_log_item_t *)bip, s); 414 xfs_buf_item_relse(bp); 415 ASSERT(XFS_BUF_FSPRIVATE(bp, void *) == NULL); 416 } 417 xfs_buf_relse(bp); 418 } 419 } 420 421 /* 422 * this is called from uncommit in the forced-shutdown path. 423 * we need to check to see if the reference count on the log item 424 * is going to drop to zero. If so, unpin will free the log item 425 * so we need to free the item's descriptor (that points to the item) 426 * in the transaction. 427 */ 428 STATIC void 429 xfs_buf_item_unpin_remove( 430 xfs_buf_log_item_t *bip, 431 xfs_trans_t *tp) 432 { 433 xfs_buf_t *bp; 434 xfs_log_item_desc_t *lidp; 435 int stale = 0; 436 437 bp = bip->bli_buf; 438 /* 439 * will xfs_buf_item_unpin() call xfs_buf_item_relse()? 440 */ 441 if ((atomic_read(&bip->bli_refcount) == 1) && 442 (bip->bli_flags & XFS_BLI_STALE)) { 443 ASSERT(XFS_BUF_VALUSEMA(bip->bli_buf) <= 0); 444 xfs_buf_item_trace("UNPIN REMOVE", bip); 445 xfs_buftrace("XFS_UNPIN_REMOVE", bp); 446 /* 447 * yes -- clear the xaction descriptor in-use flag 448 * and free the chunk if required. We can safely 449 * do some work here and then call buf_item_unpin 450 * to do the rest because if the if is true, then 451 * we are holding the buffer locked so no one else 452 * will be able to bump up the refcount. 453 */ 454 lidp = xfs_trans_find_item(tp, (xfs_log_item_t *) bip); 455 stale = lidp->lid_flags & XFS_LID_BUF_STALE; 456 xfs_trans_free_item(tp, lidp); 457 /* 458 * Since the transaction no longer refers to the buffer, 459 * the buffer should no longer refer to the transaction. 460 */ 461 XFS_BUF_SET_FSPRIVATE2(bp, NULL); 462 } 463 464 xfs_buf_item_unpin(bip, stale); 465 466 return; 467 } 468 469 /* 470 * This is called to attempt to lock the buffer associated with this 471 * buf log item. Don't sleep on the buffer lock. If we can't get 472 * the lock right away, return 0. If we can get the lock, pull the 473 * buffer from the free list, mark it busy, and return 1. 474 */ 475 STATIC uint 476 xfs_buf_item_trylock( 477 xfs_buf_log_item_t *bip) 478 { 479 xfs_buf_t *bp; 480 481 bp = bip->bli_buf; 482 483 if (XFS_BUF_ISPINNED(bp)) { 484 return XFS_ITEM_PINNED; 485 } 486 487 if (!XFS_BUF_CPSEMA(bp)) { 488 return XFS_ITEM_LOCKED; 489 } 490 491 /* 492 * Remove the buffer from the free list. Only do this 493 * if it's on the free list. Private buffers like the 494 * superblock buffer are not. 495 */ 496 XFS_BUF_HOLD(bp); 497 498 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 499 xfs_buf_item_trace("TRYLOCK SUCCESS", bip); 500 return XFS_ITEM_SUCCESS; 501 } 502 503 /* 504 * Release the buffer associated with the buf log item. 505 * If there is no dirty logged data associated with the 506 * buffer recorded in the buf log item, then free the 507 * buf log item and remove the reference to it in the 508 * buffer. 509 * 510 * This call ignores the recursion count. It is only called 511 * when the buffer should REALLY be unlocked, regardless 512 * of the recursion count. 513 * 514 * If the XFS_BLI_HOLD flag is set in the buf log item, then 515 * free the log item if necessary but do not unlock the buffer. 516 * This is for support of xfs_trans_bhold(). Make sure the 517 * XFS_BLI_HOLD field is cleared if we don't free the item. 518 */ 519 STATIC void 520 xfs_buf_item_unlock( 521 xfs_buf_log_item_t *bip) 522 { 523 int aborted; 524 xfs_buf_t *bp; 525 uint hold; 526 527 bp = bip->bli_buf; 528 xfs_buftrace("XFS_UNLOCK", bp); 529 530 /* 531 * Clear the buffer's association with this transaction. 532 */ 533 XFS_BUF_SET_FSPRIVATE2(bp, NULL); 534 535 /* 536 * If this is a transaction abort, don't return early. 537 * Instead, allow the brelse to happen. 538 * Normally it would be done for stale (cancelled) buffers 539 * at unpin time, but we'll never go through the pin/unpin 540 * cycle if we abort inside commit. 541 */ 542 aborted = (bip->bli_item.li_flags & XFS_LI_ABORTED) != 0; 543 544 /* 545 * If the buf item is marked stale, then don't do anything. 546 * We'll unlock the buffer and free the buf item when the 547 * buffer is unpinned for the last time. 548 */ 549 if (bip->bli_flags & XFS_BLI_STALE) { 550 bip->bli_flags &= ~XFS_BLI_LOGGED; 551 xfs_buf_item_trace("UNLOCK STALE", bip); 552 ASSERT(bip->bli_format.blf_flags & XFS_BLI_CANCEL); 553 if (!aborted) 554 return; 555 } 556 557 /* 558 * Drop the transaction's reference to the log item if 559 * it was not logged as part of the transaction. Otherwise 560 * we'll drop the reference in xfs_buf_item_unpin() when 561 * the transaction is really through with the buffer. 562 */ 563 if (!(bip->bli_flags & XFS_BLI_LOGGED)) { 564 atomic_dec(&bip->bli_refcount); 565 } else { 566 /* 567 * Clear the logged flag since this is per 568 * transaction state. 569 */ 570 bip->bli_flags &= ~XFS_BLI_LOGGED; 571 } 572 573 /* 574 * Before possibly freeing the buf item, determine if we should 575 * release the buffer at the end of this routine. 576 */ 577 hold = bip->bli_flags & XFS_BLI_HOLD; 578 xfs_buf_item_trace("UNLOCK", bip); 579 580 /* 581 * If the buf item isn't tracking any data, free it. 582 * Otherwise, if XFS_BLI_HOLD is set clear it. 583 */ 584 if (xfs_bitmap_empty(bip->bli_format.blf_data_map, 585 bip->bli_format.blf_map_size)) { 586 xfs_buf_item_relse(bp); 587 } else if (hold) { 588 bip->bli_flags &= ~XFS_BLI_HOLD; 589 } 590 591 /* 592 * Release the buffer if XFS_BLI_HOLD was not set. 593 */ 594 if (!hold) { 595 xfs_buf_relse(bp); 596 } 597 } 598 599 /* 600 * This is called to find out where the oldest active copy of the 601 * buf log item in the on disk log resides now that the last log 602 * write of it completed at the given lsn. 603 * We always re-log all the dirty data in a buffer, so usually the 604 * latest copy in the on disk log is the only one that matters. For 605 * those cases we simply return the given lsn. 606 * 607 * The one exception to this is for buffers full of newly allocated 608 * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF 609 * flag set, indicating that only the di_next_unlinked fields from the 610 * inodes in the buffers will be replayed during recovery. If the 611 * original newly allocated inode images have not yet been flushed 612 * when the buffer is so relogged, then we need to make sure that we 613 * keep the old images in the 'active' portion of the log. We do this 614 * by returning the original lsn of that transaction here rather than 615 * the current one. 616 */ 617 STATIC xfs_lsn_t 618 xfs_buf_item_committed( 619 xfs_buf_log_item_t *bip, 620 xfs_lsn_t lsn) 621 { 622 xfs_buf_item_trace("COMMITTED", bip); 623 if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && 624 (bip->bli_item.li_lsn != 0)) { 625 return bip->bli_item.li_lsn; 626 } 627 return (lsn); 628 } 629 630 /* 631 * This is called to asynchronously write the buffer associated with this 632 * buf log item out to disk. The buffer will already have been locked by 633 * a successful call to xfs_buf_item_trylock(). If the buffer still has 634 * B_DELWRI set, then get it going out to disk with a call to bawrite(). 635 * If not, then just release the buffer. 636 */ 637 STATIC void 638 xfs_buf_item_push( 639 xfs_buf_log_item_t *bip) 640 { 641 xfs_buf_t *bp; 642 643 ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); 644 xfs_buf_item_trace("PUSH", bip); 645 646 bp = bip->bli_buf; 647 648 if (XFS_BUF_ISDELAYWRITE(bp)) { 649 xfs_bawrite(bip->bli_item.li_mountp, bp); 650 } else { 651 xfs_buf_relse(bp); 652 } 653 } 654 655 /* ARGSUSED */ 656 STATIC void 657 xfs_buf_item_committing(xfs_buf_log_item_t *bip, xfs_lsn_t commit_lsn) 658 { 659 } 660 661 /* 662 * This is the ops vector shared by all buf log items. 663 */ 664 static struct xfs_item_ops xfs_buf_item_ops = { 665 .iop_size = (uint(*)(xfs_log_item_t*))xfs_buf_item_size, 666 .iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*)) 667 xfs_buf_item_format, 668 .iop_pin = (void(*)(xfs_log_item_t*))xfs_buf_item_pin, 669 .iop_unpin = (void(*)(xfs_log_item_t*, int))xfs_buf_item_unpin, 670 .iop_unpin_remove = (void(*)(xfs_log_item_t*, xfs_trans_t *)) 671 xfs_buf_item_unpin_remove, 672 .iop_trylock = (uint(*)(xfs_log_item_t*))xfs_buf_item_trylock, 673 .iop_unlock = (void(*)(xfs_log_item_t*))xfs_buf_item_unlock, 674 .iop_committed = (xfs_lsn_t(*)(xfs_log_item_t*, xfs_lsn_t)) 675 xfs_buf_item_committed, 676 .iop_push = (void(*)(xfs_log_item_t*))xfs_buf_item_push, 677 .iop_pushbuf = NULL, 678 .iop_committing = (void(*)(xfs_log_item_t*, xfs_lsn_t)) 679 xfs_buf_item_committing 680 }; 681 682 683 /* 684 * Allocate a new buf log item to go with the given buffer. 685 * Set the buffer's b_fsprivate field to point to the new 686 * buf log item. If there are other item's attached to the 687 * buffer (see xfs_buf_attach_iodone() below), then put the 688 * buf log item at the front. 689 */ 690 void 691 xfs_buf_item_init( 692 xfs_buf_t *bp, 693 xfs_mount_t *mp) 694 { 695 xfs_log_item_t *lip; 696 xfs_buf_log_item_t *bip; 697 int chunks; 698 int map_size; 699 700 /* 701 * Check to see if there is already a buf log item for 702 * this buffer. If there is, it is guaranteed to be 703 * the first. If we do already have one, there is 704 * nothing to do here so return. 705 */ 706 if (XFS_BUF_FSPRIVATE3(bp, xfs_mount_t *) != mp) 707 XFS_BUF_SET_FSPRIVATE3(bp, mp); 708 XFS_BUF_SET_BDSTRAT_FUNC(bp, xfs_bdstrat_cb); 709 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 710 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 711 if (lip->li_type == XFS_LI_BUF) { 712 return; 713 } 714 } 715 716 /* 717 * chunks is the number of XFS_BLI_CHUNK size pieces 718 * the buffer can be divided into. Make sure not to 719 * truncate any pieces. map_size is the size of the 720 * bitmap needed to describe the chunks of the buffer. 721 */ 722 chunks = (int)((XFS_BUF_COUNT(bp) + (XFS_BLI_CHUNK - 1)) >> XFS_BLI_SHIFT); 723 map_size = (int)((chunks + NBWORD) >> BIT_TO_WORD_SHIFT); 724 725 bip = (xfs_buf_log_item_t*)kmem_zone_zalloc(xfs_buf_item_zone, 726 KM_SLEEP); 727 bip->bli_item.li_type = XFS_LI_BUF; 728 bip->bli_item.li_ops = &xfs_buf_item_ops; 729 bip->bli_item.li_mountp = mp; 730 bip->bli_buf = bp; 731 bip->bli_format.blf_type = XFS_LI_BUF; 732 bip->bli_format.blf_blkno = (__int64_t)XFS_BUF_ADDR(bp); 733 bip->bli_format.blf_len = (ushort)BTOBB(XFS_BUF_COUNT(bp)); 734 bip->bli_format.blf_map_size = map_size; 735 #ifdef XFS_BLI_TRACE 736 bip->bli_trace = ktrace_alloc(XFS_BLI_TRACE_SIZE, KM_SLEEP); 737 #endif 738 739 #ifdef XFS_TRANS_DEBUG 740 /* 741 * Allocate the arrays for tracking what needs to be logged 742 * and what our callers request to be logged. bli_orig 743 * holds a copy of the original, clean buffer for comparison 744 * against, and bli_logged keeps a 1 bit flag per byte in 745 * the buffer to indicate which bytes the callers have asked 746 * to have logged. 747 */ 748 bip->bli_orig = (char *)kmem_alloc(XFS_BUF_COUNT(bp), KM_SLEEP); 749 memcpy(bip->bli_orig, XFS_BUF_PTR(bp), XFS_BUF_COUNT(bp)); 750 bip->bli_logged = (char *)kmem_zalloc(XFS_BUF_COUNT(bp) / NBBY, KM_SLEEP); 751 #endif 752 753 /* 754 * Put the buf item into the list of items attached to the 755 * buffer at the front. 756 */ 757 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 758 bip->bli_item.li_bio_list = 759 XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 760 } 761 XFS_BUF_SET_FSPRIVATE(bp, bip); 762 } 763 764 765 /* 766 * Mark bytes first through last inclusive as dirty in the buf 767 * item's bitmap. 768 */ 769 void 770 xfs_buf_item_log( 771 xfs_buf_log_item_t *bip, 772 uint first, 773 uint last) 774 { 775 uint first_bit; 776 uint last_bit; 777 uint bits_to_set; 778 uint bits_set; 779 uint word_num; 780 uint *wordp; 781 uint bit; 782 uint end_bit; 783 uint mask; 784 785 /* 786 * Mark the item as having some dirty data for 787 * quick reference in xfs_buf_item_dirty. 788 */ 789 bip->bli_flags |= XFS_BLI_DIRTY; 790 791 /* 792 * Convert byte offsets to bit numbers. 793 */ 794 first_bit = first >> XFS_BLI_SHIFT; 795 last_bit = last >> XFS_BLI_SHIFT; 796 797 /* 798 * Calculate the total number of bits to be set. 799 */ 800 bits_to_set = last_bit - first_bit + 1; 801 802 /* 803 * Get a pointer to the first word in the bitmap 804 * to set a bit in. 805 */ 806 word_num = first_bit >> BIT_TO_WORD_SHIFT; 807 wordp = &(bip->bli_format.blf_data_map[word_num]); 808 809 /* 810 * Calculate the starting bit in the first word. 811 */ 812 bit = first_bit & (uint)(NBWORD - 1); 813 814 /* 815 * First set any bits in the first word of our range. 816 * If it starts at bit 0 of the word, it will be 817 * set below rather than here. That is what the variable 818 * bit tells us. The variable bits_set tracks the number 819 * of bits that have been set so far. End_bit is the number 820 * of the last bit to be set in this word plus one. 821 */ 822 if (bit) { 823 end_bit = MIN(bit + bits_to_set, (uint)NBWORD); 824 mask = ((1 << (end_bit - bit)) - 1) << bit; 825 *wordp |= mask; 826 wordp++; 827 bits_set = end_bit - bit; 828 } else { 829 bits_set = 0; 830 } 831 832 /* 833 * Now set bits a whole word at a time that are between 834 * first_bit and last_bit. 835 */ 836 while ((bits_to_set - bits_set) >= NBWORD) { 837 *wordp |= 0xffffffff; 838 bits_set += NBWORD; 839 wordp++; 840 } 841 842 /* 843 * Finally, set any bits left to be set in one last partial word. 844 */ 845 end_bit = bits_to_set - bits_set; 846 if (end_bit) { 847 mask = (1 << end_bit) - 1; 848 *wordp |= mask; 849 } 850 851 xfs_buf_item_log_debug(bip, first, last); 852 } 853 854 855 /* 856 * Return 1 if the buffer has some data that has been logged (at any 857 * point, not just the current transaction) and 0 if not. 858 */ 859 uint 860 xfs_buf_item_dirty( 861 xfs_buf_log_item_t *bip) 862 { 863 return (bip->bli_flags & XFS_BLI_DIRTY); 864 } 865 866 /* 867 * This is called when the buf log item is no longer needed. It should 868 * free the buf log item associated with the given buffer and clear 869 * the buffer's pointer to the buf log item. If there are no more 870 * items in the list, clear the b_iodone field of the buffer (see 871 * xfs_buf_attach_iodone() below). 872 */ 873 void 874 xfs_buf_item_relse( 875 xfs_buf_t *bp) 876 { 877 xfs_buf_log_item_t *bip; 878 879 xfs_buftrace("XFS_RELSE", bp); 880 bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*); 881 XFS_BUF_SET_FSPRIVATE(bp, bip->bli_item.li_bio_list); 882 if ((XFS_BUF_FSPRIVATE(bp, void *) == NULL) && 883 (XFS_BUF_IODONE_FUNC(bp) != NULL)) { 884 XFS_BUF_CLR_IODONE_FUNC(bp); 885 } 886 887 #ifdef XFS_TRANS_DEBUG 888 kmem_free(bip->bli_orig, XFS_BUF_COUNT(bp)); 889 bip->bli_orig = NULL; 890 kmem_free(bip->bli_logged, XFS_BUF_COUNT(bp) / NBBY); 891 bip->bli_logged = NULL; 892 #endif /* XFS_TRANS_DEBUG */ 893 894 #ifdef XFS_BLI_TRACE 895 ktrace_free(bip->bli_trace); 896 #endif 897 kmem_zone_free(xfs_buf_item_zone, bip); 898 } 899 900 901 /* 902 * Add the given log item with its callback to the list of callbacks 903 * to be called when the buffer's I/O completes. If it is not set 904 * already, set the buffer's b_iodone() routine to be 905 * xfs_buf_iodone_callbacks() and link the log item into the list of 906 * items rooted at b_fsprivate. Items are always added as the second 907 * entry in the list if there is a first, because the buf item code 908 * assumes that the buf log item is first. 909 */ 910 void 911 xfs_buf_attach_iodone( 912 xfs_buf_t *bp, 913 void (*cb)(xfs_buf_t *, xfs_log_item_t *), 914 xfs_log_item_t *lip) 915 { 916 xfs_log_item_t *head_lip; 917 918 ASSERT(XFS_BUF_ISBUSY(bp)); 919 ASSERT(XFS_BUF_VALUSEMA(bp) <= 0); 920 921 lip->li_cb = cb; 922 if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) { 923 head_lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 924 lip->li_bio_list = head_lip->li_bio_list; 925 head_lip->li_bio_list = lip; 926 } else { 927 XFS_BUF_SET_FSPRIVATE(bp, lip); 928 } 929 930 ASSERT((XFS_BUF_IODONE_FUNC(bp) == xfs_buf_iodone_callbacks) || 931 (XFS_BUF_IODONE_FUNC(bp) == NULL)); 932 XFS_BUF_SET_IODONE_FUNC(bp, xfs_buf_iodone_callbacks); 933 } 934 935 STATIC void 936 xfs_buf_do_callbacks( 937 xfs_buf_t *bp, 938 xfs_log_item_t *lip) 939 { 940 xfs_log_item_t *nlip; 941 942 while (lip != NULL) { 943 nlip = lip->li_bio_list; 944 ASSERT(lip->li_cb != NULL); 945 /* 946 * Clear the next pointer so we don't have any 947 * confusion if the item is added to another buf. 948 * Don't touch the log item after calling its 949 * callback, because it could have freed itself. 950 */ 951 lip->li_bio_list = NULL; 952 lip->li_cb(bp, lip); 953 lip = nlip; 954 } 955 } 956 957 /* 958 * This is the iodone() function for buffers which have had callbacks 959 * attached to them by xfs_buf_attach_iodone(). It should remove each 960 * log item from the buffer's list and call the callback of each in turn. 961 * When done, the buffer's fsprivate field is set to NULL and the buffer 962 * is unlocked with a call to iodone(). 963 */ 964 void 965 xfs_buf_iodone_callbacks( 966 xfs_buf_t *bp) 967 { 968 xfs_log_item_t *lip; 969 static ulong lasttime; 970 static xfs_buftarg_t *lasttarg; 971 xfs_mount_t *mp; 972 973 ASSERT(XFS_BUF_FSPRIVATE(bp, void *) != NULL); 974 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 975 976 if (XFS_BUF_GETERROR(bp) != 0) { 977 /* 978 * If we've already decided to shutdown the filesystem 979 * because of IO errors, there's no point in giving this 980 * a retry. 981 */ 982 mp = lip->li_mountp; 983 if (XFS_FORCED_SHUTDOWN(mp)) { 984 ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp); 985 XFS_BUF_SUPER_STALE(bp); 986 xfs_buftrace("BUF_IODONE_CB", bp); 987 xfs_buf_do_callbacks(bp, lip); 988 XFS_BUF_SET_FSPRIVATE(bp, NULL); 989 XFS_BUF_CLR_IODONE_FUNC(bp); 990 991 /* 992 * XFS_SHUT flag gets set when we go thru the 993 * entire buffer cache and deliberately start 994 * throwing away delayed write buffers. 995 * Since there's no biowait done on those, 996 * we should just brelse them. 997 */ 998 if (XFS_BUF_ISSHUT(bp)) { 999 XFS_BUF_UNSHUT(bp); 1000 xfs_buf_relse(bp); 1001 } else { 1002 xfs_biodone(bp); 1003 } 1004 1005 return; 1006 } 1007 1008 if ((XFS_BUF_TARGET(bp) != lasttarg) || 1009 (time_after(jiffies, (lasttime + 5*HZ)))) { 1010 lasttime = jiffies; 1011 cmn_err(CE_ALERT, "Device %s, XFS metadata write error" 1012 " block 0x%llx in %s", 1013 XFS_BUFTARG_NAME(XFS_BUF_TARGET(bp)), 1014 (__uint64_t)XFS_BUF_ADDR(bp), mp->m_fsname); 1015 } 1016 lasttarg = XFS_BUF_TARGET(bp); 1017 1018 if (XFS_BUF_ISASYNC(bp)) { 1019 /* 1020 * If the write was asynchronous then noone will be 1021 * looking for the error. Clear the error state 1022 * and write the buffer out again delayed write. 1023 * 1024 * XXXsup This is OK, so long as we catch these 1025 * before we start the umount; we don't want these 1026 * DELWRI metadata bufs to be hanging around. 1027 */ 1028 XFS_BUF_ERROR(bp,0); /* errno of 0 unsets the flag */ 1029 1030 if (!(XFS_BUF_ISSTALE(bp))) { 1031 XFS_BUF_DELAYWRITE(bp); 1032 XFS_BUF_DONE(bp); 1033 XFS_BUF_SET_START(bp); 1034 } 1035 ASSERT(XFS_BUF_IODONE_FUNC(bp)); 1036 xfs_buftrace("BUF_IODONE ASYNC", bp); 1037 xfs_buf_relse(bp); 1038 } else { 1039 /* 1040 * If the write of the buffer was not asynchronous, 1041 * then we want to make sure to return the error 1042 * to the caller of bwrite(). Because of this we 1043 * cannot clear the B_ERROR state at this point. 1044 * Instead we install a callback function that 1045 * will be called when the buffer is released, and 1046 * that routine will clear the error state and 1047 * set the buffer to be written out again after 1048 * some delay. 1049 */ 1050 /* We actually overwrite the existing b-relse 1051 function at times, but we're gonna be shutting down 1052 anyway. */ 1053 XFS_BUF_SET_BRELSE_FUNC(bp,xfs_buf_error_relse); 1054 XFS_BUF_DONE(bp); 1055 XFS_BUF_V_IODONESEMA(bp); 1056 } 1057 return; 1058 } 1059 #ifdef XFSERRORDEBUG 1060 xfs_buftrace("XFS BUFCB NOERR", bp); 1061 #endif 1062 xfs_buf_do_callbacks(bp, lip); 1063 XFS_BUF_SET_FSPRIVATE(bp, NULL); 1064 XFS_BUF_CLR_IODONE_FUNC(bp); 1065 xfs_biodone(bp); 1066 } 1067 1068 /* 1069 * This is a callback routine attached to a buffer which gets an error 1070 * when being written out synchronously. 1071 */ 1072 STATIC void 1073 xfs_buf_error_relse( 1074 xfs_buf_t *bp) 1075 { 1076 xfs_log_item_t *lip; 1077 xfs_mount_t *mp; 1078 1079 lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *); 1080 mp = (xfs_mount_t *)lip->li_mountp; 1081 ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp); 1082 1083 XFS_BUF_STALE(bp); 1084 XFS_BUF_DONE(bp); 1085 XFS_BUF_UNDELAYWRITE(bp); 1086 XFS_BUF_ERROR(bp,0); 1087 xfs_buftrace("BUF_ERROR_RELSE", bp); 1088 if (! XFS_FORCED_SHUTDOWN(mp)) 1089 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); 1090 /* 1091 * We have to unpin the pinned buffers so do the 1092 * callbacks. 1093 */ 1094 xfs_buf_do_callbacks(bp, lip); 1095 XFS_BUF_SET_FSPRIVATE(bp, NULL); 1096 XFS_BUF_CLR_IODONE_FUNC(bp); 1097 XFS_BUF_SET_BRELSE_FUNC(bp,NULL); 1098 xfs_buf_relse(bp); 1099 } 1100 1101 1102 /* 1103 * This is the iodone() function for buffers which have been 1104 * logged. It is called when they are eventually flushed out. 1105 * It should remove the buf item from the AIL, and free the buf item. 1106 * It is called by xfs_buf_iodone_callbacks() above which will take 1107 * care of cleaning up the buffer itself. 1108 */ 1109 /* ARGSUSED */ 1110 void 1111 xfs_buf_iodone( 1112 xfs_buf_t *bp, 1113 xfs_buf_log_item_t *bip) 1114 { 1115 struct xfs_mount *mp; 1116 SPLDECL(s); 1117 1118 ASSERT(bip->bli_buf == bp); 1119 1120 mp = bip->bli_item.li_mountp; 1121 1122 /* 1123 * If we are forcibly shutting down, this may well be 1124 * off the AIL already. That's because we simulate the 1125 * log-committed callbacks to unpin these buffers. Or we may never 1126 * have put this item on AIL because of the transaction was 1127 * aborted forcibly. xfs_trans_delete_ail() takes care of these. 1128 * 1129 * Either way, AIL is useless if we're forcing a shutdown. 1130 */ 1131 AIL_LOCK(mp,s); 1132 /* 1133 * xfs_trans_delete_ail() drops the AIL lock. 1134 */ 1135 xfs_trans_delete_ail(mp, (xfs_log_item_t *)bip, s); 1136 1137 #ifdef XFS_TRANS_DEBUG 1138 kmem_free(bip->bli_orig, XFS_BUF_COUNT(bp)); 1139 bip->bli_orig = NULL; 1140 kmem_free(bip->bli_logged, XFS_BUF_COUNT(bp) / NBBY); 1141 bip->bli_logged = NULL; 1142 #endif /* XFS_TRANS_DEBUG */ 1143 1144 #ifdef XFS_BLI_TRACE 1145 ktrace_free(bip->bli_trace); 1146 #endif 1147 kmem_zone_free(xfs_buf_item_zone, bip); 1148 } 1149 1150 #if defined(XFS_BLI_TRACE) 1151 void 1152 xfs_buf_item_trace( 1153 char *id, 1154 xfs_buf_log_item_t *bip) 1155 { 1156 xfs_buf_t *bp; 1157 ASSERT(bip->bli_trace != NULL); 1158 1159 bp = bip->bli_buf; 1160 ktrace_enter(bip->bli_trace, 1161 (void *)id, 1162 (void *)bip->bli_buf, 1163 (void *)((unsigned long)bip->bli_flags), 1164 (void *)((unsigned long)bip->bli_recur), 1165 (void *)((unsigned long)atomic_read(&bip->bli_refcount)), 1166 (void *)((unsigned long) 1167 (0xFFFFFFFF & XFS_BUF_ADDR(bp) >> 32)), 1168 (void *)((unsigned long)(0xFFFFFFFF & XFS_BUF_ADDR(bp))), 1169 (void *)((unsigned long)XFS_BUF_COUNT(bp)), 1170 (void *)((unsigned long)XFS_BUF_BFLAGS(bp)), 1171 XFS_BUF_FSPRIVATE(bp, void *), 1172 XFS_BUF_FSPRIVATE2(bp, void *), 1173 (void *)(unsigned long)XFS_BUF_ISPINNED(bp), 1174 (void *)XFS_BUF_IODONE_FUNC(bp), 1175 (void *)((unsigned long)(XFS_BUF_VALUSEMA(bp))), 1176 (void *)bip->bli_item.li_desc, 1177 (void *)((unsigned long)bip->bli_item.li_flags)); 1178 } 1179 #endif /* XFS_BLI_TRACE */ 1180