1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_fs.h" 20 #include "xfs_format.h" 21 #include "xfs_log_format.h" 22 #include "xfs_trans_resv.h" 23 #include "xfs_inum.h" 24 #include "xfs_sb.h" 25 #include "xfs_ag.h" 26 #include "xfs_mount.h" 27 #include "xfs_inode.h" 28 #include "xfs_error.h" 29 #include "xfs_trans.h" 30 #include "xfs_trans_priv.h" 31 #include "xfs_inode_item.h" 32 #include "xfs_quota.h" 33 #include "xfs_trace.h" 34 #include "xfs_icache.h" 35 #include "xfs_bmap_util.h" 36 37 #include <linux/kthread.h> 38 #include <linux/freezer.h> 39 40 STATIC void __xfs_inode_clear_reclaim_tag(struct xfs_mount *mp, 41 struct xfs_perag *pag, struct xfs_inode *ip); 42 43 /* 44 * Allocate and initialise an xfs_inode. 45 */ 46 struct xfs_inode * 47 xfs_inode_alloc( 48 struct xfs_mount *mp, 49 xfs_ino_t ino) 50 { 51 struct xfs_inode *ip; 52 53 /* 54 * if this didn't occur in transactions, we could use 55 * KM_MAYFAIL and return NULL here on ENOMEM. Set the 56 * code up to do this anyway. 57 */ 58 ip = kmem_zone_alloc(xfs_inode_zone, KM_SLEEP); 59 if (!ip) 60 return NULL; 61 if (inode_init_always(mp->m_super, VFS_I(ip))) { 62 kmem_zone_free(xfs_inode_zone, ip); 63 return NULL; 64 } 65 66 ASSERT(atomic_read(&ip->i_pincount) == 0); 67 ASSERT(!spin_is_locked(&ip->i_flags_lock)); 68 ASSERT(!xfs_isiflocked(ip)); 69 ASSERT(ip->i_ino == 0); 70 71 mrlock_init(&ip->i_iolock, MRLOCK_BARRIER, "xfsio", ip->i_ino); 72 73 /* initialise the xfs inode */ 74 ip->i_ino = ino; 75 ip->i_mount = mp; 76 memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); 77 ip->i_afp = NULL; 78 memset(&ip->i_df, 0, sizeof(xfs_ifork_t)); 79 ip->i_flags = 0; 80 ip->i_delayed_blks = 0; 81 memset(&ip->i_d, 0, sizeof(xfs_icdinode_t)); 82 83 return ip; 84 } 85 86 STATIC void 87 xfs_inode_free_callback( 88 struct rcu_head *head) 89 { 90 struct inode *inode = container_of(head, struct inode, i_rcu); 91 struct xfs_inode *ip = XFS_I(inode); 92 93 kmem_zone_free(xfs_inode_zone, ip); 94 } 95 96 void 97 xfs_inode_free( 98 struct xfs_inode *ip) 99 { 100 switch (ip->i_d.di_mode & S_IFMT) { 101 case S_IFREG: 102 case S_IFDIR: 103 case S_IFLNK: 104 xfs_idestroy_fork(ip, XFS_DATA_FORK); 105 break; 106 } 107 108 if (ip->i_afp) 109 xfs_idestroy_fork(ip, XFS_ATTR_FORK); 110 111 if (ip->i_itemp) { 112 ASSERT(!(ip->i_itemp->ili_item.li_flags & XFS_LI_IN_AIL)); 113 xfs_inode_item_destroy(ip); 114 ip->i_itemp = NULL; 115 } 116 117 /* 118 * Because we use RCU freeing we need to ensure the inode always 119 * appears to be reclaimed with an invalid inode number when in the 120 * free state. The ip->i_flags_lock provides the barrier against lookup 121 * races. 122 */ 123 spin_lock(&ip->i_flags_lock); 124 ip->i_flags = XFS_IRECLAIM; 125 ip->i_ino = 0; 126 spin_unlock(&ip->i_flags_lock); 127 128 /* asserts to verify all state is correct here */ 129 ASSERT(atomic_read(&ip->i_pincount) == 0); 130 ASSERT(!xfs_isiflocked(ip)); 131 132 call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback); 133 } 134 135 /* 136 * Check the validity of the inode we just found it the cache 137 */ 138 static int 139 xfs_iget_cache_hit( 140 struct xfs_perag *pag, 141 struct xfs_inode *ip, 142 xfs_ino_t ino, 143 int flags, 144 int lock_flags) __releases(RCU) 145 { 146 struct inode *inode = VFS_I(ip); 147 struct xfs_mount *mp = ip->i_mount; 148 int error; 149 150 /* 151 * check for re-use of an inode within an RCU grace period due to the 152 * radix tree nodes not being updated yet. We monitor for this by 153 * setting the inode number to zero before freeing the inode structure. 154 * If the inode has been reallocated and set up, then the inode number 155 * will not match, so check for that, too. 156 */ 157 spin_lock(&ip->i_flags_lock); 158 if (ip->i_ino != ino) { 159 trace_xfs_iget_skip(ip); 160 XFS_STATS_INC(xs_ig_frecycle); 161 error = EAGAIN; 162 goto out_error; 163 } 164 165 166 /* 167 * If we are racing with another cache hit that is currently 168 * instantiating this inode or currently recycling it out of 169 * reclaimabe state, wait for the initialisation to complete 170 * before continuing. 171 * 172 * XXX(hch): eventually we should do something equivalent to 173 * wait_on_inode to wait for these flags to be cleared 174 * instead of polling for it. 175 */ 176 if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) { 177 trace_xfs_iget_skip(ip); 178 XFS_STATS_INC(xs_ig_frecycle); 179 error = EAGAIN; 180 goto out_error; 181 } 182 183 /* 184 * If lookup is racing with unlink return an error immediately. 185 */ 186 if (ip->i_d.di_mode == 0 && !(flags & XFS_IGET_CREATE)) { 187 error = ENOENT; 188 goto out_error; 189 } 190 191 /* 192 * If IRECLAIMABLE is set, we've torn down the VFS inode already. 193 * Need to carefully get it back into useable state. 194 */ 195 if (ip->i_flags & XFS_IRECLAIMABLE) { 196 trace_xfs_iget_reclaim(ip); 197 198 /* 199 * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode 200 * from stomping over us while we recycle the inode. We can't 201 * clear the radix tree reclaimable tag yet as it requires 202 * pag_ici_lock to be held exclusive. 203 */ 204 ip->i_flags |= XFS_IRECLAIM; 205 206 spin_unlock(&ip->i_flags_lock); 207 rcu_read_unlock(); 208 209 error = -inode_init_always(mp->m_super, inode); 210 if (error) { 211 /* 212 * Re-initializing the inode failed, and we are in deep 213 * trouble. Try to re-add it to the reclaim list. 214 */ 215 rcu_read_lock(); 216 spin_lock(&ip->i_flags_lock); 217 218 ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM); 219 ASSERT(ip->i_flags & XFS_IRECLAIMABLE); 220 trace_xfs_iget_reclaim_fail(ip); 221 goto out_error; 222 } 223 224 spin_lock(&pag->pag_ici_lock); 225 spin_lock(&ip->i_flags_lock); 226 227 /* 228 * Clear the per-lifetime state in the inode as we are now 229 * effectively a new inode and need to return to the initial 230 * state before reuse occurs. 231 */ 232 ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS; 233 ip->i_flags |= XFS_INEW; 234 __xfs_inode_clear_reclaim_tag(mp, pag, ip); 235 inode->i_state = I_NEW; 236 237 ASSERT(!rwsem_is_locked(&ip->i_iolock.mr_lock)); 238 mrlock_init(&ip->i_iolock, MRLOCK_BARRIER, "xfsio", ip->i_ino); 239 240 spin_unlock(&ip->i_flags_lock); 241 spin_unlock(&pag->pag_ici_lock); 242 } else { 243 /* If the VFS inode is being torn down, pause and try again. */ 244 if (!igrab(inode)) { 245 trace_xfs_iget_skip(ip); 246 error = EAGAIN; 247 goto out_error; 248 } 249 250 /* We've got a live one. */ 251 spin_unlock(&ip->i_flags_lock); 252 rcu_read_unlock(); 253 trace_xfs_iget_hit(ip); 254 } 255 256 if (lock_flags != 0) 257 xfs_ilock(ip, lock_flags); 258 259 xfs_iflags_clear(ip, XFS_ISTALE | XFS_IDONTCACHE); 260 XFS_STATS_INC(xs_ig_found); 261 262 return 0; 263 264 out_error: 265 spin_unlock(&ip->i_flags_lock); 266 rcu_read_unlock(); 267 return error; 268 } 269 270 271 static int 272 xfs_iget_cache_miss( 273 struct xfs_mount *mp, 274 struct xfs_perag *pag, 275 xfs_trans_t *tp, 276 xfs_ino_t ino, 277 struct xfs_inode **ipp, 278 int flags, 279 int lock_flags) 280 { 281 struct xfs_inode *ip; 282 int error; 283 xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino); 284 int iflags; 285 286 ip = xfs_inode_alloc(mp, ino); 287 if (!ip) 288 return ENOMEM; 289 290 error = xfs_iread(mp, tp, ip, flags); 291 if (error) 292 goto out_destroy; 293 294 trace_xfs_iget_miss(ip); 295 296 if ((ip->i_d.di_mode == 0) && !(flags & XFS_IGET_CREATE)) { 297 error = ENOENT; 298 goto out_destroy; 299 } 300 301 /* 302 * Preload the radix tree so we can insert safely under the 303 * write spinlock. Note that we cannot sleep inside the preload 304 * region. Since we can be called from transaction context, don't 305 * recurse into the file system. 306 */ 307 if (radix_tree_preload(GFP_NOFS)) { 308 error = EAGAIN; 309 goto out_destroy; 310 } 311 312 /* 313 * Because the inode hasn't been added to the radix-tree yet it can't 314 * be found by another thread, so we can do the non-sleeping lock here. 315 */ 316 if (lock_flags) { 317 if (!xfs_ilock_nowait(ip, lock_flags)) 318 BUG(); 319 } 320 321 /* 322 * These values must be set before inserting the inode into the radix 323 * tree as the moment it is inserted a concurrent lookup (allowed by the 324 * RCU locking mechanism) can find it and that lookup must see that this 325 * is an inode currently under construction (i.e. that XFS_INEW is set). 326 * The ip->i_flags_lock that protects the XFS_INEW flag forms the 327 * memory barrier that ensures this detection works correctly at lookup 328 * time. 329 */ 330 iflags = XFS_INEW; 331 if (flags & XFS_IGET_DONTCACHE) 332 iflags |= XFS_IDONTCACHE; 333 ip->i_udquot = NULL; 334 ip->i_gdquot = NULL; 335 ip->i_pdquot = NULL; 336 xfs_iflags_set(ip, iflags); 337 338 /* insert the new inode */ 339 spin_lock(&pag->pag_ici_lock); 340 error = radix_tree_insert(&pag->pag_ici_root, agino, ip); 341 if (unlikely(error)) { 342 WARN_ON(error != -EEXIST); 343 XFS_STATS_INC(xs_ig_dup); 344 error = EAGAIN; 345 goto out_preload_end; 346 } 347 spin_unlock(&pag->pag_ici_lock); 348 radix_tree_preload_end(); 349 350 *ipp = ip; 351 return 0; 352 353 out_preload_end: 354 spin_unlock(&pag->pag_ici_lock); 355 radix_tree_preload_end(); 356 if (lock_flags) 357 xfs_iunlock(ip, lock_flags); 358 out_destroy: 359 __destroy_inode(VFS_I(ip)); 360 xfs_inode_free(ip); 361 return error; 362 } 363 364 /* 365 * Look up an inode by number in the given file system. 366 * The inode is looked up in the cache held in each AG. 367 * If the inode is found in the cache, initialise the vfs inode 368 * if necessary. 369 * 370 * If it is not in core, read it in from the file system's device, 371 * add it to the cache and initialise the vfs inode. 372 * 373 * The inode is locked according to the value of the lock_flags parameter. 374 * This flag parameter indicates how and if the inode's IO lock and inode lock 375 * should be taken. 376 * 377 * mp -- the mount point structure for the current file system. It points 378 * to the inode hash table. 379 * tp -- a pointer to the current transaction if there is one. This is 380 * simply passed through to the xfs_iread() call. 381 * ino -- the number of the inode desired. This is the unique identifier 382 * within the file system for the inode being requested. 383 * lock_flags -- flags indicating how to lock the inode. See the comment 384 * for xfs_ilock() for a list of valid values. 385 */ 386 int 387 xfs_iget( 388 xfs_mount_t *mp, 389 xfs_trans_t *tp, 390 xfs_ino_t ino, 391 uint flags, 392 uint lock_flags, 393 xfs_inode_t **ipp) 394 { 395 xfs_inode_t *ip; 396 int error; 397 xfs_perag_t *pag; 398 xfs_agino_t agino; 399 400 /* 401 * xfs_reclaim_inode() uses the ILOCK to ensure an inode 402 * doesn't get freed while it's being referenced during a 403 * radix tree traversal here. It assumes this function 404 * aqcuires only the ILOCK (and therefore it has no need to 405 * involve the IOLOCK in this synchronization). 406 */ 407 ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0); 408 409 /* reject inode numbers outside existing AGs */ 410 if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount) 411 return EINVAL; 412 413 /* get the perag structure and ensure that it's inode capable */ 414 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino)); 415 agino = XFS_INO_TO_AGINO(mp, ino); 416 417 again: 418 error = 0; 419 rcu_read_lock(); 420 ip = radix_tree_lookup(&pag->pag_ici_root, agino); 421 422 if (ip) { 423 error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags); 424 if (error) 425 goto out_error_or_again; 426 } else { 427 rcu_read_unlock(); 428 XFS_STATS_INC(xs_ig_missed); 429 430 error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip, 431 flags, lock_flags); 432 if (error) 433 goto out_error_or_again; 434 } 435 xfs_perag_put(pag); 436 437 *ipp = ip; 438 439 /* 440 * If we have a real type for an on-disk inode, we can set ops(&unlock) 441 * now. If it's a new inode being created, xfs_ialloc will handle it. 442 */ 443 if (xfs_iflags_test(ip, XFS_INEW) && ip->i_d.di_mode != 0) 444 xfs_setup_inode(ip); 445 return 0; 446 447 out_error_or_again: 448 if (error == EAGAIN) { 449 delay(1); 450 goto again; 451 } 452 xfs_perag_put(pag); 453 return error; 454 } 455 456 /* 457 * The inode lookup is done in batches to keep the amount of lock traffic and 458 * radix tree lookups to a minimum. The batch size is a trade off between 459 * lookup reduction and stack usage. This is in the reclaim path, so we can't 460 * be too greedy. 461 */ 462 #define XFS_LOOKUP_BATCH 32 463 464 STATIC int 465 xfs_inode_ag_walk_grab( 466 struct xfs_inode *ip) 467 { 468 struct inode *inode = VFS_I(ip); 469 470 ASSERT(rcu_read_lock_held()); 471 472 /* 473 * check for stale RCU freed inode 474 * 475 * If the inode has been reallocated, it doesn't matter if it's not in 476 * the AG we are walking - we are walking for writeback, so if it 477 * passes all the "valid inode" checks and is dirty, then we'll write 478 * it back anyway. If it has been reallocated and still being 479 * initialised, the XFS_INEW check below will catch it. 480 */ 481 spin_lock(&ip->i_flags_lock); 482 if (!ip->i_ino) 483 goto out_unlock_noent; 484 485 /* avoid new or reclaimable inodes. Leave for reclaim code to flush */ 486 if (__xfs_iflags_test(ip, XFS_INEW | XFS_IRECLAIMABLE | XFS_IRECLAIM)) 487 goto out_unlock_noent; 488 spin_unlock(&ip->i_flags_lock); 489 490 /* nothing to sync during shutdown */ 491 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) 492 return EFSCORRUPTED; 493 494 /* If we can't grab the inode, it must on it's way to reclaim. */ 495 if (!igrab(inode)) 496 return ENOENT; 497 498 /* inode is valid */ 499 return 0; 500 501 out_unlock_noent: 502 spin_unlock(&ip->i_flags_lock); 503 return ENOENT; 504 } 505 506 STATIC int 507 xfs_inode_ag_walk( 508 struct xfs_mount *mp, 509 struct xfs_perag *pag, 510 int (*execute)(struct xfs_inode *ip, 511 struct xfs_perag *pag, int flags, 512 void *args), 513 int flags, 514 void *args, 515 int tag) 516 { 517 uint32_t first_index; 518 int last_error = 0; 519 int skipped; 520 int done; 521 int nr_found; 522 523 restart: 524 done = 0; 525 skipped = 0; 526 first_index = 0; 527 nr_found = 0; 528 do { 529 struct xfs_inode *batch[XFS_LOOKUP_BATCH]; 530 int error = 0; 531 int i; 532 533 rcu_read_lock(); 534 535 if (tag == -1) 536 nr_found = radix_tree_gang_lookup(&pag->pag_ici_root, 537 (void **)batch, first_index, 538 XFS_LOOKUP_BATCH); 539 else 540 nr_found = radix_tree_gang_lookup_tag( 541 &pag->pag_ici_root, 542 (void **) batch, first_index, 543 XFS_LOOKUP_BATCH, tag); 544 545 if (!nr_found) { 546 rcu_read_unlock(); 547 break; 548 } 549 550 /* 551 * Grab the inodes before we drop the lock. if we found 552 * nothing, nr == 0 and the loop will be skipped. 553 */ 554 for (i = 0; i < nr_found; i++) { 555 struct xfs_inode *ip = batch[i]; 556 557 if (done || xfs_inode_ag_walk_grab(ip)) 558 batch[i] = NULL; 559 560 /* 561 * Update the index for the next lookup. Catch 562 * overflows into the next AG range which can occur if 563 * we have inodes in the last block of the AG and we 564 * are currently pointing to the last inode. 565 * 566 * Because we may see inodes that are from the wrong AG 567 * due to RCU freeing and reallocation, only update the 568 * index if it lies in this AG. It was a race that lead 569 * us to see this inode, so another lookup from the 570 * same index will not find it again. 571 */ 572 if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno) 573 continue; 574 first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); 575 if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) 576 done = 1; 577 } 578 579 /* unlock now we've grabbed the inodes. */ 580 rcu_read_unlock(); 581 582 for (i = 0; i < nr_found; i++) { 583 if (!batch[i]) 584 continue; 585 error = execute(batch[i], pag, flags, args); 586 IRELE(batch[i]); 587 if (error == EAGAIN) { 588 skipped++; 589 continue; 590 } 591 if (error && last_error != EFSCORRUPTED) 592 last_error = error; 593 } 594 595 /* bail out if the filesystem is corrupted. */ 596 if (error == EFSCORRUPTED) 597 break; 598 599 cond_resched(); 600 601 } while (nr_found && !done); 602 603 if (skipped) { 604 delay(1); 605 goto restart; 606 } 607 return last_error; 608 } 609 610 /* 611 * Background scanning to trim post-EOF preallocated space. This is queued 612 * based on the 'speculative_prealloc_lifetime' tunable (5m by default). 613 */ 614 STATIC void 615 xfs_queue_eofblocks( 616 struct xfs_mount *mp) 617 { 618 rcu_read_lock(); 619 if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG)) 620 queue_delayed_work(mp->m_eofblocks_workqueue, 621 &mp->m_eofblocks_work, 622 msecs_to_jiffies(xfs_eofb_secs * 1000)); 623 rcu_read_unlock(); 624 } 625 626 void 627 xfs_eofblocks_worker( 628 struct work_struct *work) 629 { 630 struct xfs_mount *mp = container_of(to_delayed_work(work), 631 struct xfs_mount, m_eofblocks_work); 632 xfs_icache_free_eofblocks(mp, NULL); 633 xfs_queue_eofblocks(mp); 634 } 635 636 int 637 xfs_inode_ag_iterator( 638 struct xfs_mount *mp, 639 int (*execute)(struct xfs_inode *ip, 640 struct xfs_perag *pag, int flags, 641 void *args), 642 int flags, 643 void *args) 644 { 645 struct xfs_perag *pag; 646 int error = 0; 647 int last_error = 0; 648 xfs_agnumber_t ag; 649 650 ag = 0; 651 while ((pag = xfs_perag_get(mp, ag))) { 652 ag = pag->pag_agno + 1; 653 error = xfs_inode_ag_walk(mp, pag, execute, flags, args, -1); 654 xfs_perag_put(pag); 655 if (error) { 656 last_error = error; 657 if (error == EFSCORRUPTED) 658 break; 659 } 660 } 661 return XFS_ERROR(last_error); 662 } 663 664 int 665 xfs_inode_ag_iterator_tag( 666 struct xfs_mount *mp, 667 int (*execute)(struct xfs_inode *ip, 668 struct xfs_perag *pag, int flags, 669 void *args), 670 int flags, 671 void *args, 672 int tag) 673 { 674 struct xfs_perag *pag; 675 int error = 0; 676 int last_error = 0; 677 xfs_agnumber_t ag; 678 679 ag = 0; 680 while ((pag = xfs_perag_get_tag(mp, ag, tag))) { 681 ag = pag->pag_agno + 1; 682 error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag); 683 xfs_perag_put(pag); 684 if (error) { 685 last_error = error; 686 if (error == EFSCORRUPTED) 687 break; 688 } 689 } 690 return XFS_ERROR(last_error); 691 } 692 693 /* 694 * Queue a new inode reclaim pass if there are reclaimable inodes and there 695 * isn't a reclaim pass already in progress. By default it runs every 5s based 696 * on the xfs periodic sync default of 30s. Perhaps this should have it's own 697 * tunable, but that can be done if this method proves to be ineffective or too 698 * aggressive. 699 */ 700 static void 701 xfs_reclaim_work_queue( 702 struct xfs_mount *mp) 703 { 704 705 rcu_read_lock(); 706 if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) { 707 queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work, 708 msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10)); 709 } 710 rcu_read_unlock(); 711 } 712 713 /* 714 * This is a fast pass over the inode cache to try to get reclaim moving on as 715 * many inodes as possible in a short period of time. It kicks itself every few 716 * seconds, as well as being kicked by the inode cache shrinker when memory 717 * goes low. It scans as quickly as possible avoiding locked inodes or those 718 * already being flushed, and once done schedules a future pass. 719 */ 720 void 721 xfs_reclaim_worker( 722 struct work_struct *work) 723 { 724 struct xfs_mount *mp = container_of(to_delayed_work(work), 725 struct xfs_mount, m_reclaim_work); 726 727 xfs_reclaim_inodes(mp, SYNC_TRYLOCK); 728 xfs_reclaim_work_queue(mp); 729 } 730 731 static void 732 __xfs_inode_set_reclaim_tag( 733 struct xfs_perag *pag, 734 struct xfs_inode *ip) 735 { 736 radix_tree_tag_set(&pag->pag_ici_root, 737 XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), 738 XFS_ICI_RECLAIM_TAG); 739 740 if (!pag->pag_ici_reclaimable) { 741 /* propagate the reclaim tag up into the perag radix tree */ 742 spin_lock(&ip->i_mount->m_perag_lock); 743 radix_tree_tag_set(&ip->i_mount->m_perag_tree, 744 XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), 745 XFS_ICI_RECLAIM_TAG); 746 spin_unlock(&ip->i_mount->m_perag_lock); 747 748 /* schedule periodic background inode reclaim */ 749 xfs_reclaim_work_queue(ip->i_mount); 750 751 trace_xfs_perag_set_reclaim(ip->i_mount, pag->pag_agno, 752 -1, _RET_IP_); 753 } 754 pag->pag_ici_reclaimable++; 755 } 756 757 /* 758 * We set the inode flag atomically with the radix tree tag. 759 * Once we get tag lookups on the radix tree, this inode flag 760 * can go away. 761 */ 762 void 763 xfs_inode_set_reclaim_tag( 764 xfs_inode_t *ip) 765 { 766 struct xfs_mount *mp = ip->i_mount; 767 struct xfs_perag *pag; 768 769 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); 770 spin_lock(&pag->pag_ici_lock); 771 spin_lock(&ip->i_flags_lock); 772 __xfs_inode_set_reclaim_tag(pag, ip); 773 __xfs_iflags_set(ip, XFS_IRECLAIMABLE); 774 spin_unlock(&ip->i_flags_lock); 775 spin_unlock(&pag->pag_ici_lock); 776 xfs_perag_put(pag); 777 } 778 779 STATIC void 780 __xfs_inode_clear_reclaim( 781 xfs_perag_t *pag, 782 xfs_inode_t *ip) 783 { 784 pag->pag_ici_reclaimable--; 785 if (!pag->pag_ici_reclaimable) { 786 /* clear the reclaim tag from the perag radix tree */ 787 spin_lock(&ip->i_mount->m_perag_lock); 788 radix_tree_tag_clear(&ip->i_mount->m_perag_tree, 789 XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), 790 XFS_ICI_RECLAIM_TAG); 791 spin_unlock(&ip->i_mount->m_perag_lock); 792 trace_xfs_perag_clear_reclaim(ip->i_mount, pag->pag_agno, 793 -1, _RET_IP_); 794 } 795 } 796 797 STATIC void 798 __xfs_inode_clear_reclaim_tag( 799 xfs_mount_t *mp, 800 xfs_perag_t *pag, 801 xfs_inode_t *ip) 802 { 803 radix_tree_tag_clear(&pag->pag_ici_root, 804 XFS_INO_TO_AGINO(mp, ip->i_ino), XFS_ICI_RECLAIM_TAG); 805 __xfs_inode_clear_reclaim(pag, ip); 806 } 807 808 /* 809 * Grab the inode for reclaim exclusively. 810 * Return 0 if we grabbed it, non-zero otherwise. 811 */ 812 STATIC int 813 xfs_reclaim_inode_grab( 814 struct xfs_inode *ip, 815 int flags) 816 { 817 ASSERT(rcu_read_lock_held()); 818 819 /* quick check for stale RCU freed inode */ 820 if (!ip->i_ino) 821 return 1; 822 823 /* 824 * If we are asked for non-blocking operation, do unlocked checks to 825 * see if the inode already is being flushed or in reclaim to avoid 826 * lock traffic. 827 */ 828 if ((flags & SYNC_TRYLOCK) && 829 __xfs_iflags_test(ip, XFS_IFLOCK | XFS_IRECLAIM)) 830 return 1; 831 832 /* 833 * The radix tree lock here protects a thread in xfs_iget from racing 834 * with us starting reclaim on the inode. Once we have the 835 * XFS_IRECLAIM flag set it will not touch us. 836 * 837 * Due to RCU lookup, we may find inodes that have been freed and only 838 * have XFS_IRECLAIM set. Indeed, we may see reallocated inodes that 839 * aren't candidates for reclaim at all, so we must check the 840 * XFS_IRECLAIMABLE is set first before proceeding to reclaim. 841 */ 842 spin_lock(&ip->i_flags_lock); 843 if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) || 844 __xfs_iflags_test(ip, XFS_IRECLAIM)) { 845 /* not a reclaim candidate. */ 846 spin_unlock(&ip->i_flags_lock); 847 return 1; 848 } 849 __xfs_iflags_set(ip, XFS_IRECLAIM); 850 spin_unlock(&ip->i_flags_lock); 851 return 0; 852 } 853 854 /* 855 * Inodes in different states need to be treated differently. The following 856 * table lists the inode states and the reclaim actions necessary: 857 * 858 * inode state iflush ret required action 859 * --------------- ---------- --------------- 860 * bad - reclaim 861 * shutdown EIO unpin and reclaim 862 * clean, unpinned 0 reclaim 863 * stale, unpinned 0 reclaim 864 * clean, pinned(*) 0 requeue 865 * stale, pinned EAGAIN requeue 866 * dirty, async - requeue 867 * dirty, sync 0 reclaim 868 * 869 * (*) dgc: I don't think the clean, pinned state is possible but it gets 870 * handled anyway given the order of checks implemented. 871 * 872 * Also, because we get the flush lock first, we know that any inode that has 873 * been flushed delwri has had the flush completed by the time we check that 874 * the inode is clean. 875 * 876 * Note that because the inode is flushed delayed write by AIL pushing, the 877 * flush lock may already be held here and waiting on it can result in very 878 * long latencies. Hence for sync reclaims, where we wait on the flush lock, 879 * the caller should push the AIL first before trying to reclaim inodes to 880 * minimise the amount of time spent waiting. For background relaim, we only 881 * bother to reclaim clean inodes anyway. 882 * 883 * Hence the order of actions after gaining the locks should be: 884 * bad => reclaim 885 * shutdown => unpin and reclaim 886 * pinned, async => requeue 887 * pinned, sync => unpin 888 * stale => reclaim 889 * clean => reclaim 890 * dirty, async => requeue 891 * dirty, sync => flush, wait and reclaim 892 */ 893 STATIC int 894 xfs_reclaim_inode( 895 struct xfs_inode *ip, 896 struct xfs_perag *pag, 897 int sync_mode) 898 { 899 struct xfs_buf *bp = NULL; 900 int error; 901 902 restart: 903 error = 0; 904 xfs_ilock(ip, XFS_ILOCK_EXCL); 905 if (!xfs_iflock_nowait(ip)) { 906 if (!(sync_mode & SYNC_WAIT)) 907 goto out; 908 xfs_iflock(ip); 909 } 910 911 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { 912 xfs_iunpin_wait(ip); 913 xfs_iflush_abort(ip, false); 914 goto reclaim; 915 } 916 if (xfs_ipincount(ip)) { 917 if (!(sync_mode & SYNC_WAIT)) 918 goto out_ifunlock; 919 xfs_iunpin_wait(ip); 920 } 921 if (xfs_iflags_test(ip, XFS_ISTALE)) 922 goto reclaim; 923 if (xfs_inode_clean(ip)) 924 goto reclaim; 925 926 /* 927 * Never flush out dirty data during non-blocking reclaim, as it would 928 * just contend with AIL pushing trying to do the same job. 929 */ 930 if (!(sync_mode & SYNC_WAIT)) 931 goto out_ifunlock; 932 933 /* 934 * Now we have an inode that needs flushing. 935 * 936 * Note that xfs_iflush will never block on the inode buffer lock, as 937 * xfs_ifree_cluster() can lock the inode buffer before it locks the 938 * ip->i_lock, and we are doing the exact opposite here. As a result, 939 * doing a blocking xfs_imap_to_bp() to get the cluster buffer would 940 * result in an ABBA deadlock with xfs_ifree_cluster(). 941 * 942 * As xfs_ifree_cluser() must gather all inodes that are active in the 943 * cache to mark them stale, if we hit this case we don't actually want 944 * to do IO here - we want the inode marked stale so we can simply 945 * reclaim it. Hence if we get an EAGAIN error here, just unlock the 946 * inode, back off and try again. Hopefully the next pass through will 947 * see the stale flag set on the inode. 948 */ 949 error = xfs_iflush(ip, &bp); 950 if (error == EAGAIN) { 951 xfs_iunlock(ip, XFS_ILOCK_EXCL); 952 /* backoff longer than in xfs_ifree_cluster */ 953 delay(2); 954 goto restart; 955 } 956 957 if (!error) { 958 error = xfs_bwrite(bp); 959 xfs_buf_relse(bp); 960 } 961 962 xfs_iflock(ip); 963 reclaim: 964 xfs_ifunlock(ip); 965 xfs_iunlock(ip, XFS_ILOCK_EXCL); 966 967 XFS_STATS_INC(xs_ig_reclaims); 968 /* 969 * Remove the inode from the per-AG radix tree. 970 * 971 * Because radix_tree_delete won't complain even if the item was never 972 * added to the tree assert that it's been there before to catch 973 * problems with the inode life time early on. 974 */ 975 spin_lock(&pag->pag_ici_lock); 976 if (!radix_tree_delete(&pag->pag_ici_root, 977 XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino))) 978 ASSERT(0); 979 __xfs_inode_clear_reclaim(pag, ip); 980 spin_unlock(&pag->pag_ici_lock); 981 982 /* 983 * Here we do an (almost) spurious inode lock in order to coordinate 984 * with inode cache radix tree lookups. This is because the lookup 985 * can reference the inodes in the cache without taking references. 986 * 987 * We make that OK here by ensuring that we wait until the inode is 988 * unlocked after the lookup before we go ahead and free it. 989 */ 990 xfs_ilock(ip, XFS_ILOCK_EXCL); 991 xfs_qm_dqdetach(ip); 992 xfs_iunlock(ip, XFS_ILOCK_EXCL); 993 994 xfs_inode_free(ip); 995 return error; 996 997 out_ifunlock: 998 xfs_ifunlock(ip); 999 out: 1000 xfs_iflags_clear(ip, XFS_IRECLAIM); 1001 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1002 /* 1003 * We could return EAGAIN here to make reclaim rescan the inode tree in 1004 * a short while. However, this just burns CPU time scanning the tree 1005 * waiting for IO to complete and the reclaim work never goes back to 1006 * the idle state. Instead, return 0 to let the next scheduled 1007 * background reclaim attempt to reclaim the inode again. 1008 */ 1009 return 0; 1010 } 1011 1012 /* 1013 * Walk the AGs and reclaim the inodes in them. Even if the filesystem is 1014 * corrupted, we still want to try to reclaim all the inodes. If we don't, 1015 * then a shut down during filesystem unmount reclaim walk leak all the 1016 * unreclaimed inodes. 1017 */ 1018 STATIC int 1019 xfs_reclaim_inodes_ag( 1020 struct xfs_mount *mp, 1021 int flags, 1022 int *nr_to_scan) 1023 { 1024 struct xfs_perag *pag; 1025 int error = 0; 1026 int last_error = 0; 1027 xfs_agnumber_t ag; 1028 int trylock = flags & SYNC_TRYLOCK; 1029 int skipped; 1030 1031 restart: 1032 ag = 0; 1033 skipped = 0; 1034 while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) { 1035 unsigned long first_index = 0; 1036 int done = 0; 1037 int nr_found = 0; 1038 1039 ag = pag->pag_agno + 1; 1040 1041 if (trylock) { 1042 if (!mutex_trylock(&pag->pag_ici_reclaim_lock)) { 1043 skipped++; 1044 xfs_perag_put(pag); 1045 continue; 1046 } 1047 first_index = pag->pag_ici_reclaim_cursor; 1048 } else 1049 mutex_lock(&pag->pag_ici_reclaim_lock); 1050 1051 do { 1052 struct xfs_inode *batch[XFS_LOOKUP_BATCH]; 1053 int i; 1054 1055 rcu_read_lock(); 1056 nr_found = radix_tree_gang_lookup_tag( 1057 &pag->pag_ici_root, 1058 (void **)batch, first_index, 1059 XFS_LOOKUP_BATCH, 1060 XFS_ICI_RECLAIM_TAG); 1061 if (!nr_found) { 1062 done = 1; 1063 rcu_read_unlock(); 1064 break; 1065 } 1066 1067 /* 1068 * Grab the inodes before we drop the lock. if we found 1069 * nothing, nr == 0 and the loop will be skipped. 1070 */ 1071 for (i = 0; i < nr_found; i++) { 1072 struct xfs_inode *ip = batch[i]; 1073 1074 if (done || xfs_reclaim_inode_grab(ip, flags)) 1075 batch[i] = NULL; 1076 1077 /* 1078 * Update the index for the next lookup. Catch 1079 * overflows into the next AG range which can 1080 * occur if we have inodes in the last block of 1081 * the AG and we are currently pointing to the 1082 * last inode. 1083 * 1084 * Because we may see inodes that are from the 1085 * wrong AG due to RCU freeing and 1086 * reallocation, only update the index if it 1087 * lies in this AG. It was a race that lead us 1088 * to see this inode, so another lookup from 1089 * the same index will not find it again. 1090 */ 1091 if (XFS_INO_TO_AGNO(mp, ip->i_ino) != 1092 pag->pag_agno) 1093 continue; 1094 first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); 1095 if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) 1096 done = 1; 1097 } 1098 1099 /* unlock now we've grabbed the inodes. */ 1100 rcu_read_unlock(); 1101 1102 for (i = 0; i < nr_found; i++) { 1103 if (!batch[i]) 1104 continue; 1105 error = xfs_reclaim_inode(batch[i], pag, flags); 1106 if (error && last_error != EFSCORRUPTED) 1107 last_error = error; 1108 } 1109 1110 *nr_to_scan -= XFS_LOOKUP_BATCH; 1111 1112 cond_resched(); 1113 1114 } while (nr_found && !done && *nr_to_scan > 0); 1115 1116 if (trylock && !done) 1117 pag->pag_ici_reclaim_cursor = first_index; 1118 else 1119 pag->pag_ici_reclaim_cursor = 0; 1120 mutex_unlock(&pag->pag_ici_reclaim_lock); 1121 xfs_perag_put(pag); 1122 } 1123 1124 /* 1125 * if we skipped any AG, and we still have scan count remaining, do 1126 * another pass this time using blocking reclaim semantics (i.e 1127 * waiting on the reclaim locks and ignoring the reclaim cursors). This 1128 * ensure that when we get more reclaimers than AGs we block rather 1129 * than spin trying to execute reclaim. 1130 */ 1131 if (skipped && (flags & SYNC_WAIT) && *nr_to_scan > 0) { 1132 trylock = 0; 1133 goto restart; 1134 } 1135 return XFS_ERROR(last_error); 1136 } 1137 1138 int 1139 xfs_reclaim_inodes( 1140 xfs_mount_t *mp, 1141 int mode) 1142 { 1143 int nr_to_scan = INT_MAX; 1144 1145 return xfs_reclaim_inodes_ag(mp, mode, &nr_to_scan); 1146 } 1147 1148 /* 1149 * Scan a certain number of inodes for reclaim. 1150 * 1151 * When called we make sure that there is a background (fast) inode reclaim in 1152 * progress, while we will throttle the speed of reclaim via doing synchronous 1153 * reclaim of inodes. That means if we come across dirty inodes, we wait for 1154 * them to be cleaned, which we hope will not be very long due to the 1155 * background walker having already kicked the IO off on those dirty inodes. 1156 */ 1157 long 1158 xfs_reclaim_inodes_nr( 1159 struct xfs_mount *mp, 1160 int nr_to_scan) 1161 { 1162 /* kick background reclaimer and push the AIL */ 1163 xfs_reclaim_work_queue(mp); 1164 xfs_ail_push_all(mp->m_ail); 1165 1166 return xfs_reclaim_inodes_ag(mp, SYNC_TRYLOCK | SYNC_WAIT, &nr_to_scan); 1167 } 1168 1169 /* 1170 * Return the number of reclaimable inodes in the filesystem for 1171 * the shrinker to determine how much to reclaim. 1172 */ 1173 int 1174 xfs_reclaim_inodes_count( 1175 struct xfs_mount *mp) 1176 { 1177 struct xfs_perag *pag; 1178 xfs_agnumber_t ag = 0; 1179 int reclaimable = 0; 1180 1181 while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) { 1182 ag = pag->pag_agno + 1; 1183 reclaimable += pag->pag_ici_reclaimable; 1184 xfs_perag_put(pag); 1185 } 1186 return reclaimable; 1187 } 1188 1189 STATIC int 1190 xfs_inode_match_id( 1191 struct xfs_inode *ip, 1192 struct xfs_eofblocks *eofb) 1193 { 1194 if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) && 1195 !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid)) 1196 return 0; 1197 1198 if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) && 1199 !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid)) 1200 return 0; 1201 1202 if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) && 1203 xfs_get_projid(ip) != eofb->eof_prid) 1204 return 0; 1205 1206 return 1; 1207 } 1208 1209 STATIC int 1210 xfs_inode_free_eofblocks( 1211 struct xfs_inode *ip, 1212 struct xfs_perag *pag, 1213 int flags, 1214 void *args) 1215 { 1216 int ret; 1217 struct xfs_eofblocks *eofb = args; 1218 1219 if (!xfs_can_free_eofblocks(ip, false)) { 1220 /* inode could be preallocated or append-only */ 1221 trace_xfs_inode_free_eofblocks_invalid(ip); 1222 xfs_inode_clear_eofblocks_tag(ip); 1223 return 0; 1224 } 1225 1226 /* 1227 * If the mapping is dirty the operation can block and wait for some 1228 * time. Unless we are waiting, skip it. 1229 */ 1230 if (!(flags & SYNC_WAIT) && 1231 mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY)) 1232 return 0; 1233 1234 if (eofb) { 1235 if (!xfs_inode_match_id(ip, eofb)) 1236 return 0; 1237 1238 /* skip the inode if the file size is too small */ 1239 if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE && 1240 XFS_ISIZE(ip) < eofb->eof_min_file_size) 1241 return 0; 1242 } 1243 1244 ret = xfs_free_eofblocks(ip->i_mount, ip, true); 1245 1246 /* don't revisit the inode if we're not waiting */ 1247 if (ret == EAGAIN && !(flags & SYNC_WAIT)) 1248 ret = 0; 1249 1250 return ret; 1251 } 1252 1253 int 1254 xfs_icache_free_eofblocks( 1255 struct xfs_mount *mp, 1256 struct xfs_eofblocks *eofb) 1257 { 1258 int flags = SYNC_TRYLOCK; 1259 1260 if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC)) 1261 flags = SYNC_WAIT; 1262 1263 return xfs_inode_ag_iterator_tag(mp, xfs_inode_free_eofblocks, flags, 1264 eofb, XFS_ICI_EOFBLOCKS_TAG); 1265 } 1266 1267 void 1268 xfs_inode_set_eofblocks_tag( 1269 xfs_inode_t *ip) 1270 { 1271 struct xfs_mount *mp = ip->i_mount; 1272 struct xfs_perag *pag; 1273 int tagged; 1274 1275 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); 1276 spin_lock(&pag->pag_ici_lock); 1277 trace_xfs_inode_set_eofblocks_tag(ip); 1278 1279 tagged = radix_tree_tagged(&pag->pag_ici_root, 1280 XFS_ICI_EOFBLOCKS_TAG); 1281 radix_tree_tag_set(&pag->pag_ici_root, 1282 XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), 1283 XFS_ICI_EOFBLOCKS_TAG); 1284 if (!tagged) { 1285 /* propagate the eofblocks tag up into the perag radix tree */ 1286 spin_lock(&ip->i_mount->m_perag_lock); 1287 radix_tree_tag_set(&ip->i_mount->m_perag_tree, 1288 XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), 1289 XFS_ICI_EOFBLOCKS_TAG); 1290 spin_unlock(&ip->i_mount->m_perag_lock); 1291 1292 /* kick off background trimming */ 1293 xfs_queue_eofblocks(ip->i_mount); 1294 1295 trace_xfs_perag_set_eofblocks(ip->i_mount, pag->pag_agno, 1296 -1, _RET_IP_); 1297 } 1298 1299 spin_unlock(&pag->pag_ici_lock); 1300 xfs_perag_put(pag); 1301 } 1302 1303 void 1304 xfs_inode_clear_eofblocks_tag( 1305 xfs_inode_t *ip) 1306 { 1307 struct xfs_mount *mp = ip->i_mount; 1308 struct xfs_perag *pag; 1309 1310 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); 1311 spin_lock(&pag->pag_ici_lock); 1312 trace_xfs_inode_clear_eofblocks_tag(ip); 1313 1314 radix_tree_tag_clear(&pag->pag_ici_root, 1315 XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), 1316 XFS_ICI_EOFBLOCKS_TAG); 1317 if (!radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_EOFBLOCKS_TAG)) { 1318 /* clear the eofblocks tag from the perag radix tree */ 1319 spin_lock(&ip->i_mount->m_perag_lock); 1320 radix_tree_tag_clear(&ip->i_mount->m_perag_tree, 1321 XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), 1322 XFS_ICI_EOFBLOCKS_TAG); 1323 spin_unlock(&ip->i_mount->m_perag_lock); 1324 trace_xfs_perag_clear_eofblocks(ip->i_mount, pag->pag_agno, 1325 -1, _RET_IP_); 1326 } 1327 1328 spin_unlock(&pag->pag_ici_lock); 1329 xfs_perag_put(pag); 1330 } 1331 1332