1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * (C) 1997 Linus Torvalds 4 * (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation) 5 */ 6 #include <linux/export.h> 7 #include <linux/fs.h> 8 #include <linux/filelock.h> 9 #include <linux/mm.h> 10 #include <linux/backing-dev.h> 11 #include <linux/hash.h> 12 #include <linux/swap.h> 13 #include <linux/security.h> 14 #include <linux/cdev.h> 15 #include <linux/memblock.h> 16 #include <linux/fsnotify.h> 17 #include <linux/fsverity.h> 18 #include <linux/mount.h> 19 #include <linux/posix_acl.h> 20 #include <linux/ratelimit.h> 21 #include <linux/list_lru.h> 22 #include <linux/iversion.h> 23 #include <linux/rw_hint.h> 24 #include <linux/seq_file.h> 25 #include <linux/debugfs.h> 26 #include <trace/events/writeback.h> 27 #define CREATE_TRACE_POINTS 28 #include <trace/events/timestamp.h> 29 30 #include "internal.h" 31 32 /* 33 * Inode locking rules: 34 * 35 * inode->i_lock protects: 36 * inode->i_state, inode->i_hash, __iget(), inode->i_io_list 37 * Inode LRU list locks protect: 38 * inode->i_sb->s_inode_lru, inode->i_lru 39 * inode->i_sb->s_inode_list_lock protects: 40 * inode->i_sb->s_inodes, inode->i_sb_list 41 * bdi->wb.list_lock protects: 42 * bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list 43 * inode_hash_lock protects: 44 * inode_hashtable, inode->i_hash 45 * 46 * Lock ordering: 47 * 48 * inode->i_sb->s_inode_list_lock 49 * inode->i_lock 50 * Inode LRU list locks 51 * 52 * bdi->wb.list_lock 53 * inode->i_lock 54 * 55 * inode_hash_lock 56 * inode->i_lock 57 */ 58 59 static unsigned int i_hash_mask __ro_after_init; 60 static unsigned int i_hash_shift __ro_after_init; 61 static struct hlist_head *inode_hashtable __ro_after_init; 62 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock); 63 64 /* 65 * Empty aops. Can be used for the cases where the user does not 66 * define any of the address_space operations. 67 */ 68 const struct address_space_operations empty_aops = { 69 }; 70 EXPORT_SYMBOL(empty_aops); 71 72 static DEFINE_PER_CPU(unsigned long, nr_inodes); 73 static DEFINE_PER_CPU(unsigned long, nr_unused); 74 75 static struct kmem_cache *inode_cachep __ro_after_init; 76 77 static long get_nr_inodes(void) 78 { 79 int i; 80 long sum = 0; 81 for_each_possible_cpu(i) 82 sum += per_cpu(nr_inodes, i); 83 return sum < 0 ? 0 : sum; 84 } 85 86 static inline long get_nr_inodes_unused(void) 87 { 88 int i; 89 long sum = 0; 90 for_each_possible_cpu(i) 91 sum += per_cpu(nr_unused, i); 92 return sum < 0 ? 0 : sum; 93 } 94 95 long get_nr_dirty_inodes(void) 96 { 97 /* not actually dirty inodes, but a wild approximation */ 98 long nr_dirty = get_nr_inodes() - get_nr_inodes_unused(); 99 return nr_dirty > 0 ? nr_dirty : 0; 100 } 101 102 #ifdef CONFIG_DEBUG_FS 103 static DEFINE_PER_CPU(long, mg_ctime_updates); 104 static DEFINE_PER_CPU(long, mg_fine_stamps); 105 static DEFINE_PER_CPU(long, mg_ctime_swaps); 106 107 static unsigned long get_mg_ctime_updates(void) 108 { 109 unsigned long sum = 0; 110 int i; 111 112 for_each_possible_cpu(i) 113 sum += data_race(per_cpu(mg_ctime_updates, i)); 114 return sum; 115 } 116 117 static unsigned long get_mg_fine_stamps(void) 118 { 119 unsigned long sum = 0; 120 int i; 121 122 for_each_possible_cpu(i) 123 sum += data_race(per_cpu(mg_fine_stamps, i)); 124 return sum; 125 } 126 127 static unsigned long get_mg_ctime_swaps(void) 128 { 129 unsigned long sum = 0; 130 int i; 131 132 for_each_possible_cpu(i) 133 sum += data_race(per_cpu(mg_ctime_swaps, i)); 134 return sum; 135 } 136 137 #define mgtime_counter_inc(__var) this_cpu_inc(__var) 138 139 static int mgts_show(struct seq_file *s, void *p) 140 { 141 unsigned long ctime_updates = get_mg_ctime_updates(); 142 unsigned long ctime_swaps = get_mg_ctime_swaps(); 143 unsigned long fine_stamps = get_mg_fine_stamps(); 144 unsigned long floor_swaps = timekeeping_get_mg_floor_swaps(); 145 146 seq_printf(s, "%lu %lu %lu %lu\n", 147 ctime_updates, ctime_swaps, fine_stamps, floor_swaps); 148 return 0; 149 } 150 151 DEFINE_SHOW_ATTRIBUTE(mgts); 152 153 static int __init mg_debugfs_init(void) 154 { 155 debugfs_create_file("multigrain_timestamps", S_IFREG | S_IRUGO, NULL, NULL, &mgts_fops); 156 return 0; 157 } 158 late_initcall(mg_debugfs_init); 159 160 #else /* ! CONFIG_DEBUG_FS */ 161 162 #define mgtime_counter_inc(__var) do { } while (0) 163 164 #endif /* CONFIG_DEBUG_FS */ 165 166 /* 167 * Handle nr_inode sysctl 168 */ 169 #ifdef CONFIG_SYSCTL 170 /* 171 * Statistics gathering.. 172 */ 173 static struct inodes_stat_t inodes_stat; 174 175 static int proc_nr_inodes(const struct ctl_table *table, int write, void *buffer, 176 size_t *lenp, loff_t *ppos) 177 { 178 inodes_stat.nr_inodes = get_nr_inodes(); 179 inodes_stat.nr_unused = get_nr_inodes_unused(); 180 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 181 } 182 183 static const struct ctl_table inodes_sysctls[] = { 184 { 185 .procname = "inode-nr", 186 .data = &inodes_stat, 187 .maxlen = 2*sizeof(long), 188 .mode = 0444, 189 .proc_handler = proc_nr_inodes, 190 }, 191 { 192 .procname = "inode-state", 193 .data = &inodes_stat, 194 .maxlen = 7*sizeof(long), 195 .mode = 0444, 196 .proc_handler = proc_nr_inodes, 197 }, 198 }; 199 200 static int __init init_fs_inode_sysctls(void) 201 { 202 register_sysctl_init("fs", inodes_sysctls); 203 return 0; 204 } 205 early_initcall(init_fs_inode_sysctls); 206 #endif 207 208 static int no_open(struct inode *inode, struct file *file) 209 { 210 return -ENXIO; 211 } 212 213 /** 214 * inode_init_always_gfp - perform inode structure initialisation 215 * @sb: superblock inode belongs to 216 * @inode: inode to initialise 217 * @gfp: allocation flags 218 * 219 * These are initializations that need to be done on every inode 220 * allocation as the fields are not initialised by slab allocation. 221 * If there are additional allocations required @gfp is used. 222 */ 223 int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp) 224 { 225 static const struct inode_operations empty_iops; 226 static const struct file_operations no_open_fops = {.open = no_open}; 227 struct address_space *const mapping = &inode->i_data; 228 229 inode->i_sb = sb; 230 inode->i_blkbits = sb->s_blocksize_bits; 231 inode->i_flags = 0; 232 inode_state_assign_raw(inode, 0); 233 atomic64_set(&inode->i_sequence, 0); 234 atomic_set(&inode->i_count, 1); 235 inode->i_op = &empty_iops; 236 inode->i_fop = &no_open_fops; 237 inode->i_ino = 0; 238 inode->__i_nlink = 1; 239 inode->i_opflags = 0; 240 if (sb->s_xattr) 241 inode->i_opflags |= IOP_XATTR; 242 if (sb->s_type->fs_flags & FS_MGTIME) 243 inode->i_opflags |= IOP_MGTIME; 244 i_uid_write(inode, 0); 245 i_gid_write(inode, 0); 246 atomic_set(&inode->i_writecount, 0); 247 inode->i_size = 0; 248 inode->i_write_hint = WRITE_LIFE_NOT_SET; 249 inode->i_blocks = 0; 250 inode->i_bytes = 0; 251 inode->i_generation = 0; 252 inode->i_pipe = NULL; 253 inode->i_cdev = NULL; 254 inode->i_link = NULL; 255 inode->i_dir_seq = 0; 256 inode->i_rdev = 0; 257 inode->dirtied_when = 0; 258 259 #ifdef CONFIG_CGROUP_WRITEBACK 260 inode->i_wb_frn_winner = 0; 261 inode->i_wb_frn_avg_time = 0; 262 inode->i_wb_frn_history = 0; 263 #endif 264 265 spin_lock_init(&inode->i_lock); 266 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key); 267 268 init_rwsem(&inode->i_rwsem); 269 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key); 270 271 atomic_set(&inode->i_dio_count, 0); 272 273 mapping->a_ops = &empty_aops; 274 mapping->host = inode; 275 mapping->flags = 0; 276 mapping->wb_err = 0; 277 atomic_set(&mapping->i_mmap_writable, 0); 278 mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE); 279 mapping->writeback_index = 0; 280 init_rwsem(&mapping->invalidate_lock); 281 lockdep_set_class_and_name(&mapping->invalidate_lock, 282 &sb->s_type->invalidate_lock_key, 283 "mapping.invalidate_lock"); 284 if (sb->s_iflags & SB_I_STABLE_WRITES) 285 mapping_set_stable_writes(mapping); 286 inode->i_private = NULL; 287 inode->i_mapping = mapping; 288 INIT_HLIST_HEAD(&inode->i_dentry); /* buggered by rcu freeing */ 289 #ifdef CONFIG_FS_POSIX_ACL 290 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED; 291 #endif 292 293 #ifdef CONFIG_FSNOTIFY 294 inode->i_fsnotify_mask = 0; 295 #endif 296 inode->i_flctx = NULL; 297 298 if (unlikely(security_inode_alloc(inode, gfp))) 299 return -ENOMEM; 300 301 this_cpu_inc(nr_inodes); 302 303 return 0; 304 } 305 EXPORT_SYMBOL(inode_init_always_gfp); 306 307 void free_inode_nonrcu(struct inode *inode) 308 { 309 kmem_cache_free(inode_cachep, inode); 310 } 311 EXPORT_SYMBOL(free_inode_nonrcu); 312 313 static void i_callback(struct rcu_head *head) 314 { 315 struct inode *inode = container_of(head, struct inode, i_rcu); 316 if (inode->free_inode) 317 inode->free_inode(inode); 318 else 319 free_inode_nonrcu(inode); 320 } 321 322 /** 323 * alloc_inode - obtain an inode 324 * @sb: superblock 325 * 326 * Allocates a new inode for given superblock. 327 * Inode wont be chained in superblock s_inodes list 328 * This means : 329 * - fs can't be unmount 330 * - quotas, fsnotify, writeback can't work 331 */ 332 struct inode *alloc_inode(struct super_block *sb) 333 { 334 const struct super_operations *ops = sb->s_op; 335 struct inode *inode; 336 337 if (ops->alloc_inode) 338 inode = ops->alloc_inode(sb); 339 else 340 inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL); 341 342 if (!inode) 343 return NULL; 344 345 if (unlikely(inode_init_always(sb, inode))) { 346 if (ops->destroy_inode) { 347 ops->destroy_inode(inode); 348 if (!ops->free_inode) 349 return NULL; 350 } 351 inode->free_inode = ops->free_inode; 352 i_callback(&inode->i_rcu); 353 return NULL; 354 } 355 356 return inode; 357 } 358 359 void __destroy_inode(struct inode *inode) 360 { 361 inode_detach_wb(inode); 362 security_inode_free(inode); 363 fsnotify_inode_delete(inode); 364 locks_free_lock_context(inode); 365 if (!inode->i_nlink) { 366 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0); 367 atomic_long_dec(&inode->i_sb->s_remove_count); 368 } 369 370 #ifdef CONFIG_FS_POSIX_ACL 371 if (inode->i_acl && !is_uncached_acl(inode->i_acl)) 372 posix_acl_release(inode->i_acl); 373 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl)) 374 posix_acl_release(inode->i_default_acl); 375 #endif 376 this_cpu_dec(nr_inodes); 377 } 378 EXPORT_SYMBOL(__destroy_inode); 379 380 static void destroy_inode(struct inode *inode) 381 { 382 const struct super_operations *ops = inode->i_sb->s_op; 383 384 BUG_ON(!list_empty(&inode->i_lru)); 385 __destroy_inode(inode); 386 if (ops->destroy_inode) { 387 ops->destroy_inode(inode); 388 if (!ops->free_inode) 389 return; 390 } 391 inode->free_inode = ops->free_inode; 392 call_rcu(&inode->i_rcu, i_callback); 393 } 394 395 /** 396 * drop_nlink - directly drop an inode's link count 397 * @inode: inode 398 * 399 * This is a low-level filesystem helper to replace any 400 * direct filesystem manipulation of i_nlink. In cases 401 * where we are attempting to track writes to the 402 * filesystem, a decrement to zero means an imminent 403 * write when the file is truncated and actually unlinked 404 * on the filesystem. 405 */ 406 void drop_nlink(struct inode *inode) 407 { 408 WARN_ON(inode->i_nlink == 0); 409 inode->__i_nlink--; 410 if (!inode->i_nlink) 411 atomic_long_inc(&inode->i_sb->s_remove_count); 412 } 413 EXPORT_SYMBOL(drop_nlink); 414 415 /** 416 * clear_nlink - directly zero an inode's link count 417 * @inode: inode 418 * 419 * This is a low-level filesystem helper to replace any 420 * direct filesystem manipulation of i_nlink. See 421 * drop_nlink() for why we care about i_nlink hitting zero. 422 */ 423 void clear_nlink(struct inode *inode) 424 { 425 if (inode->i_nlink) { 426 inode->__i_nlink = 0; 427 atomic_long_inc(&inode->i_sb->s_remove_count); 428 } 429 } 430 EXPORT_SYMBOL(clear_nlink); 431 432 /** 433 * set_nlink - directly set an inode's link count 434 * @inode: inode 435 * @nlink: new nlink (should be non-zero) 436 * 437 * This is a low-level filesystem helper to replace any 438 * direct filesystem manipulation of i_nlink. 439 */ 440 void set_nlink(struct inode *inode, unsigned int nlink) 441 { 442 if (!nlink) { 443 clear_nlink(inode); 444 } else { 445 /* Yes, some filesystems do change nlink from zero to one */ 446 if (inode->i_nlink == 0) 447 atomic_long_dec(&inode->i_sb->s_remove_count); 448 449 inode->__i_nlink = nlink; 450 } 451 } 452 EXPORT_SYMBOL(set_nlink); 453 454 /** 455 * inc_nlink - directly increment an inode's link count 456 * @inode: inode 457 * 458 * This is a low-level filesystem helper to replace any 459 * direct filesystem manipulation of i_nlink. Currently, 460 * it is only here for parity with dec_nlink(). 461 */ 462 void inc_nlink(struct inode *inode) 463 { 464 if (unlikely(inode->i_nlink == 0)) { 465 WARN_ON(!(inode_state_read_once(inode) & I_LINKABLE)); 466 atomic_long_dec(&inode->i_sb->s_remove_count); 467 } 468 469 inode->__i_nlink++; 470 } 471 EXPORT_SYMBOL(inc_nlink); 472 473 static void __address_space_init_once(struct address_space *mapping) 474 { 475 xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT); 476 init_rwsem(&mapping->i_mmap_rwsem); 477 spin_lock_init(&mapping->i_private_lock); 478 mapping->i_mmap = RB_ROOT_CACHED; 479 } 480 481 void address_space_init_once(struct address_space *mapping) 482 { 483 memset(mapping, 0, sizeof(*mapping)); 484 __address_space_init_once(mapping); 485 } 486 EXPORT_SYMBOL(address_space_init_once); 487 488 /* 489 * These are initializations that only need to be done 490 * once, because the fields are idempotent across use 491 * of the inode, so let the slab aware of that. 492 */ 493 void inode_init_once(struct inode *inode) 494 { 495 memset(inode, 0, sizeof(*inode)); 496 INIT_HLIST_NODE(&inode->i_hash); 497 INIT_LIST_HEAD(&inode->i_devices); 498 INIT_LIST_HEAD(&inode->i_io_list); 499 INIT_LIST_HEAD(&inode->i_wb_list); 500 INIT_LIST_HEAD(&inode->i_lru); 501 INIT_LIST_HEAD(&inode->i_sb_list); 502 __address_space_init_once(&inode->i_data); 503 i_size_ordered_init(inode); 504 } 505 EXPORT_SYMBOL(inode_init_once); 506 507 static void init_once(void *foo) 508 { 509 struct inode *inode = (struct inode *) foo; 510 511 inode_init_once(inode); 512 } 513 514 struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe, 515 struct inode *inode, u32 bit) 516 { 517 void *bit_address; 518 519 bit_address = inode_state_wait_address(inode, bit); 520 init_wait_var_entry(wqe, bit_address, 0); 521 return __var_waitqueue(bit_address); 522 } 523 EXPORT_SYMBOL(inode_bit_waitqueue); 524 525 void wait_on_new_inode(struct inode *inode) 526 { 527 struct wait_bit_queue_entry wqe; 528 struct wait_queue_head *wq_head; 529 530 spin_lock(&inode->i_lock); 531 if (!(inode_state_read(inode) & I_NEW)) { 532 spin_unlock(&inode->i_lock); 533 return; 534 } 535 536 wq_head = inode_bit_waitqueue(&wqe, inode, __I_NEW); 537 for (;;) { 538 prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); 539 if (!(inode_state_read(inode) & I_NEW)) 540 break; 541 spin_unlock(&inode->i_lock); 542 schedule(); 543 spin_lock(&inode->i_lock); 544 } 545 finish_wait(wq_head, &wqe.wq_entry); 546 WARN_ON(inode_state_read(inode) & I_NEW); 547 spin_unlock(&inode->i_lock); 548 } 549 EXPORT_SYMBOL(wait_on_new_inode); 550 551 static void __inode_lru_list_add(struct inode *inode, bool rotate) 552 { 553 lockdep_assert_held(&inode->i_lock); 554 555 if (inode_state_read(inode) & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE)) 556 return; 557 if (icount_read(inode)) 558 return; 559 if (!(inode->i_sb->s_flags & SB_ACTIVE)) 560 return; 561 if (!mapping_shrinkable(&inode->i_data)) 562 return; 563 564 if (list_lru_add_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) 565 this_cpu_inc(nr_unused); 566 else if (rotate) 567 inode_state_set(inode, I_REFERENCED); 568 } 569 570 /* 571 * Add inode to LRU if needed (inode is unused and clean). 572 */ 573 void inode_lru_list_add(struct inode *inode) 574 { 575 __inode_lru_list_add(inode, false); 576 } 577 578 static void inode_lru_list_del(struct inode *inode) 579 { 580 if (list_empty(&inode->i_lru)) 581 return; 582 583 if (list_lru_del_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) 584 this_cpu_dec(nr_unused); 585 } 586 587 static void inode_pin_lru_isolating(struct inode *inode) 588 { 589 lockdep_assert_held(&inode->i_lock); 590 WARN_ON(inode_state_read(inode) & (I_LRU_ISOLATING | I_FREEING | I_WILL_FREE)); 591 inode_state_set(inode, I_LRU_ISOLATING); 592 } 593 594 static void inode_unpin_lru_isolating(struct inode *inode) 595 { 596 lockdep_assert_held(&inode->i_lock); 597 WARN_ON(!(inode_state_read(inode) & I_LRU_ISOLATING)); 598 inode_state_clear(inode, I_LRU_ISOLATING); 599 /* Called with inode->i_lock which ensures memory ordering. */ 600 inode_wake_up_bit(inode, __I_LRU_ISOLATING); 601 } 602 603 static void inode_wait_for_lru_isolating(struct inode *inode) 604 { 605 struct wait_bit_queue_entry wqe; 606 struct wait_queue_head *wq_head; 607 608 lockdep_assert_held(&inode->i_lock); 609 if (!(inode_state_read(inode) & I_LRU_ISOLATING)) 610 return; 611 612 wq_head = inode_bit_waitqueue(&wqe, inode, __I_LRU_ISOLATING); 613 for (;;) { 614 prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); 615 /* 616 * Checking I_LRU_ISOLATING with inode->i_lock guarantees 617 * memory ordering. 618 */ 619 if (!(inode_state_read(inode) & I_LRU_ISOLATING)) 620 break; 621 spin_unlock(&inode->i_lock); 622 schedule(); 623 spin_lock(&inode->i_lock); 624 } 625 finish_wait(wq_head, &wqe.wq_entry); 626 WARN_ON(inode_state_read(inode) & I_LRU_ISOLATING); 627 } 628 629 /** 630 * inode_sb_list_add - add inode to the superblock list of inodes 631 * @inode: inode to add 632 */ 633 void inode_sb_list_add(struct inode *inode) 634 { 635 struct super_block *sb = inode->i_sb; 636 637 spin_lock(&sb->s_inode_list_lock); 638 list_add(&inode->i_sb_list, &sb->s_inodes); 639 spin_unlock(&sb->s_inode_list_lock); 640 } 641 EXPORT_SYMBOL_GPL(inode_sb_list_add); 642 643 static inline void inode_sb_list_del(struct inode *inode) 644 { 645 struct super_block *sb = inode->i_sb; 646 647 if (!list_empty(&inode->i_sb_list)) { 648 spin_lock(&sb->s_inode_list_lock); 649 list_del_init(&inode->i_sb_list); 650 spin_unlock(&sb->s_inode_list_lock); 651 } 652 } 653 654 static unsigned long hash(struct super_block *sb, u64 hashval) 655 { 656 unsigned long tmp; 657 658 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / 659 L1_CACHE_BYTES; 660 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift); 661 return tmp & i_hash_mask; 662 } 663 664 /** 665 * __insert_inode_hash - hash an inode 666 * @inode: unhashed inode 667 * @hashval: u64 value used to locate this object in the 668 * inode_hashtable. 669 * 670 * Add an inode to the inode hash for this superblock. 671 */ 672 void __insert_inode_hash(struct inode *inode, u64 hashval) 673 { 674 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval); 675 676 spin_lock(&inode_hash_lock); 677 spin_lock(&inode->i_lock); 678 hlist_add_head_rcu(&inode->i_hash, b); 679 spin_unlock(&inode->i_lock); 680 spin_unlock(&inode_hash_lock); 681 } 682 EXPORT_SYMBOL(__insert_inode_hash); 683 684 /** 685 * __remove_inode_hash - remove an inode from the hash 686 * @inode: inode to unhash 687 * 688 * Remove an inode from the superblock. 689 */ 690 void __remove_inode_hash(struct inode *inode) 691 { 692 spin_lock(&inode_hash_lock); 693 spin_lock(&inode->i_lock); 694 hlist_del_init_rcu(&inode->i_hash); 695 spin_unlock(&inode->i_lock); 696 spin_unlock(&inode_hash_lock); 697 } 698 EXPORT_SYMBOL(__remove_inode_hash); 699 700 void dump_mapping(const struct address_space *mapping) 701 { 702 struct inode *host; 703 const struct address_space_operations *a_ops; 704 struct hlist_node *dentry_first; 705 struct dentry *dentry_ptr; 706 struct dentry dentry; 707 char fname[64] = {}; 708 u64 ino; 709 710 /* 711 * If mapping is an invalid pointer, we don't want to crash 712 * accessing it, so probe everything depending on it carefully. 713 */ 714 if (get_kernel_nofault(host, &mapping->host) || 715 get_kernel_nofault(a_ops, &mapping->a_ops)) { 716 pr_warn("invalid mapping:%px\n", mapping); 717 return; 718 } 719 720 if (!host) { 721 pr_warn("aops:%ps\n", a_ops); 722 return; 723 } 724 725 if (get_kernel_nofault(dentry_first, &host->i_dentry.first) || 726 get_kernel_nofault(ino, &host->i_ino)) { 727 pr_warn("aops:%ps invalid inode:%px\n", a_ops, host); 728 return; 729 } 730 731 if (!dentry_first) { 732 pr_warn("aops:%ps ino:%llx\n", a_ops, ino); 733 return; 734 } 735 736 dentry_ptr = container_of(dentry_first, struct dentry, d_alias); 737 if (get_kernel_nofault(dentry, dentry_ptr) || 738 !dentry.d_parent || !dentry.d_name.name) { 739 pr_warn("aops:%ps ino:%llx invalid dentry:%px\n", 740 a_ops, ino, dentry_ptr); 741 return; 742 } 743 744 if (strncpy_from_kernel_nofault(fname, dentry.d_name.name, 63) < 0) 745 strscpy(fname, "<invalid>"); 746 /* 747 * Even if strncpy_from_kernel_nofault() succeeded, 748 * the fname could be unreliable 749 */ 750 pr_warn("aops:%ps ino:%llx dentry name(?):\"%s\"\n", 751 a_ops, ino, fname); 752 } 753 754 void clear_inode(struct inode *inode) 755 { 756 /* 757 * Only IS_VERITY() inodes can have verity info, so start by checking 758 * for IS_VERITY() (which is faster than retrieving the pointer to the 759 * verity info). This minimizes overhead for non-verity inodes. 760 */ 761 if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) 762 fsverity_cleanup_inode(inode); 763 764 /* 765 * We have to cycle the i_pages lock here because reclaim 766 * can be in the process of removing the last page (in 767 * __filemap_remove_folio()) and we must not free the mapping 768 * under it. We also remove nodes which are empty; these 769 * can occur in two different ways. The first is that radix 770 * tree expansion can fail partway and the second is that THP 771 * collapse_file() can allocate some temporary nodes and not 772 * clean them up. 773 */ 774 xa_destroy(&inode->i_data.i_pages); 775 776 BUG_ON(inode->i_data.nrpages); 777 BUG_ON(!(inode_state_read_once(inode) & I_FREEING)); 778 BUG_ON(inode_state_read_once(inode) & I_CLEAR); 779 BUG_ON(!list_empty(&inode->i_wb_list)); 780 /* don't need i_lock here, no concurrent mods to i_state */ 781 inode_state_assign_raw(inode, I_FREEING | I_CLEAR); 782 } 783 EXPORT_SYMBOL(clear_inode); 784 785 /* 786 * Free the inode passed in, removing it from the lists it is still connected 787 * to. We remove any pages still attached to the inode and wait for any IO that 788 * is still in progress before finally destroying the inode. 789 * 790 * An inode must already be marked I_FREEING so that we avoid the inode being 791 * moved back onto lists if we race with other code that manipulates the lists 792 * (e.g. writeback_single_inode). The caller is responsible for setting this. 793 * 794 * An inode must already be removed from the LRU list before being evicted from 795 * the cache. This should occur atomically with setting the I_FREEING state 796 * flag, so no inodes here should ever be on the LRU when being evicted. 797 */ 798 static void evict(struct inode *inode) 799 { 800 const struct super_operations *op = inode->i_sb->s_op; 801 802 BUG_ON(!(inode_state_read_once(inode) & I_FREEING)); 803 BUG_ON(!list_empty(&inode->i_lru)); 804 805 inode_io_list_del(inode); 806 inode_sb_list_del(inode); 807 808 spin_lock(&inode->i_lock); 809 inode_wait_for_lru_isolating(inode); 810 811 /* 812 * Wait for flusher thread to be done with the inode so that filesystem 813 * does not start destroying it while writeback is still running. Since 814 * the inode has I_FREEING set, flusher thread won't start new work on 815 * the inode. We just have to wait for running writeback to finish. 816 */ 817 inode_wait_for_writeback(inode); 818 spin_unlock(&inode->i_lock); 819 820 /* 821 * fsync() can't find what I_METADATA_WRITEBACK tracks anymore once the 822 * inode is gone, so write it out unless the inode is being deleted. 823 */ 824 if (inode->i_nlink && 825 (inode_state_read_once(inode) & I_METADATA_WRITEBACK)) 826 sync_inode_metadata(inode, 1); 827 828 if (op->evict_inode) { 829 op->evict_inode(inode); 830 } else { 831 truncate_inode_pages_final(&inode->i_data); 832 clear_inode(inode); 833 } 834 if (S_ISCHR(inode->i_mode) && inode->i_cdev) 835 cd_forget(inode); 836 837 remove_inode_hash(inode); 838 839 /* 840 * Wake up waiters in __wait_on_freeing_inode(). 841 * 842 * It is an invariant that any thread we need to wake up is already 843 * accounted for before remove_inode_hash() acquires ->i_lock -- both 844 * sides take the lock and sleep is aborted if the inode is found 845 * unhashed. Thus either the sleeper wins and goes off CPU, or removal 846 * wins and the sleeper aborts after testing with the lock. 847 * 848 * This also means we don't need any fences for the call below. 849 */ 850 inode_wake_up_bit(inode, __I_NEW); 851 BUG_ON(inode_state_read_once(inode) != (I_FREEING | I_CLEAR)); 852 853 destroy_inode(inode); 854 } 855 856 /* 857 * dispose_list - dispose of the contents of a local list 858 * @head: the head of the list to free 859 * 860 * Dispose-list gets a local list with local inodes in it, so it doesn't 861 * need to worry about list corruption and SMP locks. 862 */ 863 static void dispose_list(struct list_head *head) 864 { 865 while (!list_empty(head)) { 866 struct inode *inode; 867 868 inode = list_first_entry(head, struct inode, i_lru); 869 list_del_init(&inode->i_lru); 870 871 evict(inode); 872 cond_resched(); 873 } 874 } 875 876 /** 877 * evict_inodes - evict all evictable inodes for a superblock 878 * @sb: superblock to operate on 879 * 880 * Make sure that no inodes with zero refcount are retained. This is 881 * called by superblock shutdown after having SB_ACTIVE flag removed, 882 * so any inode reaching zero refcount during or after that call will 883 * be immediately evicted. 884 */ 885 void evict_inodes(struct super_block *sb) 886 { 887 struct inode *inode; 888 LIST_HEAD(dispose); 889 890 spin_lock(&sb->s_inode_list_lock); 891 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { 892 if (icount_read_once(inode)) 893 continue; 894 895 spin_lock(&inode->i_lock); 896 if (icount_read(inode)) { 897 spin_unlock(&inode->i_lock); 898 continue; 899 } 900 if (inode_state_read(inode) & (I_NEW | I_FREEING | I_WILL_FREE)) { 901 spin_unlock(&inode->i_lock); 902 continue; 903 } 904 905 inode_state_set(inode, I_FREEING); 906 inode_lru_list_del(inode); 907 spin_unlock(&inode->i_lock); 908 909 /* 910 * Keep this inode out of dispose so it stays on s_inodes while 911 * the list lock is dropped. I_FREEING prevents new references 912 * and leaves eviction to us, so we can resume the walk from it. 913 */ 914 if (need_resched()) { 915 spin_unlock(&sb->s_inode_list_lock); 916 cond_resched(); 917 dispose_list(&dispose); 918 spin_lock(&sb->s_inode_list_lock); 919 } 920 list_add(&inode->i_lru, &dispose); 921 } 922 spin_unlock(&sb->s_inode_list_lock); 923 924 dispose_list(&dispose); 925 } 926 EXPORT_SYMBOL_GPL(evict_inodes); 927 928 /* 929 * Isolate the inode from the LRU in preparation for freeing it. 930 * 931 * If the inode has the I_REFERENCED flag set, then it means that it has been 932 * used recently - the flag is set in iput_final(). When we encounter such an 933 * inode, clear the flag and move it to the back of the LRU so it gets another 934 * pass through the LRU before it gets reclaimed. This is necessary because of 935 * the fact we are doing lazy LRU updates to minimise lock contention so the 936 * LRU does not have strict ordering. Hence we don't want to reclaim inodes 937 * with this flag set because they are the inodes that are out of order. 938 */ 939 static enum lru_status inode_lru_isolate(struct list_head *item, 940 struct list_lru_one *lru, void *arg) 941 { 942 struct list_head *freeable = arg; 943 struct inode *inode = container_of(item, struct inode, i_lru); 944 945 /* 946 * We are inverting the lru lock/inode->i_lock here, so use a 947 * trylock. If we fail to get the lock, just skip it. 948 */ 949 if (!spin_trylock(&inode->i_lock)) 950 return LRU_SKIP; 951 952 /* Another walker is dropping its page cache, leave it to that one */ 953 if (inode_state_read(inode) & I_LRU_ISOLATING) { 954 spin_unlock(&inode->i_lock); 955 return LRU_SKIP; 956 } 957 958 /* 959 * Inodes can get referenced, redirtied, or repopulated while 960 * they're already on the LRU, and this can make them 961 * unreclaimable for a while. Remove them lazily here; iput, 962 * sync, or the last page cache deletion will requeue them. 963 * Nothing requeues them for I_METADATA_WRITEBACK, evict() writes 964 * that out instead. 965 */ 966 if (icount_read(inode) || 967 (inode_state_read(inode) & 968 ~(I_REFERENCED | I_METADATA_WRITEBACK)) || 969 !mapping_shrinkable(&inode->i_data)) { 970 list_lru_isolate(lru, &inode->i_lru); 971 spin_unlock(&inode->i_lock); 972 this_cpu_dec(nr_unused); 973 return LRU_REMOVED; 974 } 975 976 /* Recently referenced inodes get one more pass */ 977 if (inode_state_read(inode) & I_REFERENCED) { 978 inode_state_clear(inode, I_REFERENCED); 979 spin_unlock(&inode->i_lock); 980 return LRU_ROTATE; 981 } 982 983 /* 984 * On highmem systems, mapping_shrinkable() permits dropping 985 * page cache in order to free up struct inodes: lowmem might 986 * be under pressure before the cache inside the highmem zone. 987 */ 988 if (!mapping_empty(&inode->i_data)) { 989 unsigned long reap; 990 991 inode_pin_lru_isolating(inode); 992 spin_unlock(&inode->i_lock); 993 spin_unlock(&lru->lock); 994 reap = invalidate_mapping_pages(&inode->i_data, 0, -1); 995 if (current_is_kswapd()) 996 __count_vm_events(KSWAPD_INODESTEAL, reap); 997 else 998 __count_vm_events(PGINODESTEAL, reap); 999 mm_account_reclaimed_pages(reap); 1000 1001 spin_lock(&inode->i_lock); 1002 /* The walk restarts here: don't retry leftovers right away */ 1003 if (!mapping_empty(&inode->i_data)) 1004 inode_state_set(inode, I_REFERENCED); 1005 inode_unpin_lru_isolating(inode); 1006 spin_unlock(&inode->i_lock); 1007 return LRU_RETRY; 1008 } 1009 1010 WARN_ON(inode_state_read(inode) & I_NEW); 1011 inode_state_set(inode, I_FREEING); 1012 list_lru_isolate_move(lru, &inode->i_lru, freeable); 1013 spin_unlock(&inode->i_lock); 1014 1015 this_cpu_dec(nr_unused); 1016 return LRU_REMOVED; 1017 } 1018 1019 /* 1020 * Walk the superblock inode LRU for freeable inodes and attempt to free them. 1021 * This is called from the superblock shrinker function with a number of inodes 1022 * to trim from the LRU. Inodes to be freed are moved to a temporary list and 1023 * then are freed outside inode_lock by dispose_list(). 1024 */ 1025 long prune_icache_sb(struct super_block *sb, struct shrink_control *sc) 1026 { 1027 LIST_HEAD(freeable); 1028 long freed; 1029 1030 freed = list_lru_shrink_walk(&sb->s_inode_lru, sc, 1031 inode_lru_isolate, &freeable); 1032 dispose_list(&freeable); 1033 return freed; 1034 } 1035 1036 static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked); 1037 static bool igrab_from_hash(struct inode *inode); 1038 1039 /* 1040 * Called with the inode lock held. 1041 */ 1042 static struct inode *find_inode(struct super_block *sb, 1043 struct hlist_head *head, 1044 int (*test)(struct inode *, void *), 1045 void *data, bool hash_locked, 1046 bool *isnew) 1047 { 1048 struct inode *inode = NULL; 1049 1050 if (hash_locked) 1051 lockdep_assert_held(&inode_hash_lock); 1052 else 1053 lockdep_assert_not_held(&inode_hash_lock); 1054 1055 rcu_read_lock(); 1056 repeat: 1057 hlist_for_each_entry_rcu(inode, head, i_hash) { 1058 if (inode->i_sb != sb) 1059 continue; 1060 if (!test(inode, data)) 1061 continue; 1062 if (igrab_from_hash(inode)) { 1063 rcu_read_unlock(); 1064 *isnew = false; 1065 return inode; 1066 } 1067 spin_lock(&inode->i_lock); 1068 if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { 1069 __wait_on_freeing_inode(inode, hash_locked, true); 1070 goto repeat; 1071 } 1072 if (unlikely(inode_state_read(inode) & I_CREATING)) { 1073 spin_unlock(&inode->i_lock); 1074 rcu_read_unlock(); 1075 return ERR_PTR(-ESTALE); 1076 } 1077 __iget(inode); 1078 *isnew = !!(inode_state_read(inode) & I_NEW); 1079 spin_unlock(&inode->i_lock); 1080 rcu_read_unlock(); 1081 return inode; 1082 } 1083 rcu_read_unlock(); 1084 return NULL; 1085 } 1086 1087 /* 1088 * find_inode_fast is the fast path version of find_inode, see the comment at 1089 * iget_locked for details. 1090 */ 1091 static struct inode *find_inode_fast(struct super_block *sb, 1092 struct hlist_head *head, u64 ino, 1093 bool hash_locked, bool *isnew) 1094 { 1095 struct inode *inode = NULL; 1096 1097 if (hash_locked) 1098 lockdep_assert_held(&inode_hash_lock); 1099 else 1100 lockdep_assert_not_held(&inode_hash_lock); 1101 1102 rcu_read_lock(); 1103 repeat: 1104 hlist_for_each_entry_rcu(inode, head, i_hash) { 1105 if (inode->i_ino != ino) 1106 continue; 1107 if (inode->i_sb != sb) 1108 continue; 1109 if (igrab_from_hash(inode)) { 1110 rcu_read_unlock(); 1111 *isnew = false; 1112 return inode; 1113 } 1114 spin_lock(&inode->i_lock); 1115 if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { 1116 __wait_on_freeing_inode(inode, hash_locked, true); 1117 goto repeat; 1118 } 1119 if (unlikely(inode_state_read(inode) & I_CREATING)) { 1120 spin_unlock(&inode->i_lock); 1121 rcu_read_unlock(); 1122 return ERR_PTR(-ESTALE); 1123 } 1124 __iget(inode); 1125 *isnew = !!(inode_state_read(inode) & I_NEW); 1126 spin_unlock(&inode->i_lock); 1127 rcu_read_unlock(); 1128 return inode; 1129 } 1130 rcu_read_unlock(); 1131 return NULL; 1132 } 1133 1134 /* 1135 * Each cpu owns a range of LAST_INO_BATCH numbers. 1136 * 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations, 1137 * to renew the exhausted range. 1138 * 1139 * This does not significantly increase overflow rate because every CPU can 1140 * consume at most LAST_INO_BATCH-1 unused inode numbers. So there is 1141 * NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the 1142 * 2^32 range, and is a worst-case. Even a 50% wastage would only increase 1143 * overflow rate by 2x, which does not seem too significant. 1144 * 1145 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW 1146 * error if st_ino won't fit in target struct field. Use 32bit counter 1147 * here to attempt to avoid that. 1148 */ 1149 #define LAST_INO_BATCH 1024 1150 static DEFINE_PER_CPU(unsigned int, last_ino); 1151 1152 unsigned int get_next_ino(void) 1153 { 1154 unsigned int *p = &get_cpu_var(last_ino); 1155 unsigned int res = *p; 1156 1157 #ifdef CONFIG_SMP 1158 if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) { 1159 static atomic_t shared_last_ino; 1160 int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino); 1161 1162 res = next - LAST_INO_BATCH; 1163 } 1164 #endif 1165 1166 res++; 1167 /* get_next_ino should not provide a 0 inode number */ 1168 if (unlikely(!res)) 1169 res++; 1170 *p = res; 1171 put_cpu_var(last_ino); 1172 return res; 1173 } 1174 EXPORT_SYMBOL(get_next_ino); 1175 1176 /** 1177 * new_inode - obtain an inode 1178 * @sb: superblock 1179 * 1180 * Allocates a new inode for given superblock. The default gfp_mask 1181 * for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE. 1182 * If HIGHMEM pages are unsuitable or it is known that pages allocated 1183 * for the page cache are not reclaimable or migratable, 1184 * mapping_set_gfp_mask() must be called with suitable flags on the 1185 * newly created inode's mapping 1186 * 1187 */ 1188 struct inode *new_inode(struct super_block *sb) 1189 { 1190 struct inode *inode; 1191 1192 inode = alloc_inode(sb); 1193 if (inode) 1194 inode_sb_list_add(inode); 1195 return inode; 1196 } 1197 EXPORT_SYMBOL(new_inode); 1198 1199 #ifdef CONFIG_DEBUG_LOCK_ALLOC 1200 void lockdep_annotate_inode_mutex_key(struct inode *inode) 1201 { 1202 if (S_ISDIR(inode->i_mode)) { 1203 struct file_system_type *type = inode->i_sb->s_type; 1204 1205 /* Set new key only if filesystem hasn't already changed it */ 1206 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) { 1207 /* 1208 * ensure nobody is actually holding i_rwsem 1209 */ 1210 init_rwsem(&inode->i_rwsem); 1211 lockdep_set_class(&inode->i_rwsem, 1212 &type->i_mutex_dir_key); 1213 } 1214 } 1215 } 1216 EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key); 1217 #endif 1218 1219 /** 1220 * unlock_new_inode - clear the I_NEW state and wake up any waiters 1221 * @inode: new inode to unlock 1222 * 1223 * Called when the inode is fully initialised to clear the new state of the 1224 * inode and wake up anyone waiting for the inode to finish initialisation. 1225 */ 1226 void unlock_new_inode(struct inode *inode) 1227 { 1228 lockdep_annotate_inode_mutex_key(inode); 1229 spin_lock(&inode->i_lock); 1230 WARN_ON(!(inode_state_read(inode) & I_NEW)); 1231 /* 1232 * Paired with igrab_from_hash() 1233 */ 1234 smp_wmb(); 1235 inode_state_clear(inode, I_NEW | I_CREATING); 1236 inode_wake_up_bit(inode, __I_NEW); 1237 spin_unlock(&inode->i_lock); 1238 } 1239 EXPORT_SYMBOL(unlock_new_inode); 1240 1241 void discard_new_inode(struct inode *inode) 1242 { 1243 lockdep_annotate_inode_mutex_key(inode); 1244 spin_lock(&inode->i_lock); 1245 WARN_ON(!(inode_state_read(inode) & I_NEW)); 1246 /* 1247 * Paired with igrab_from_hash() 1248 */ 1249 smp_wmb(); 1250 inode_state_clear(inode, I_NEW); 1251 inode_wake_up_bit(inode, __I_NEW); 1252 spin_unlock(&inode->i_lock); 1253 iput(inode); 1254 } 1255 EXPORT_SYMBOL(discard_new_inode); 1256 1257 /** 1258 * lock_two_nondirectories - take two i_mutexes on non-directory objects 1259 * 1260 * Lock any non-NULL argument. Passed objects must not be directories. 1261 * Zero, one or two objects may be locked by this function. 1262 * 1263 * @inode1: first inode to lock 1264 * @inode2: second inode to lock 1265 */ 1266 void lock_two_nondirectories(struct inode *inode1, struct inode *inode2) 1267 { 1268 if (inode1) 1269 WARN_ON_ONCE(S_ISDIR(inode1->i_mode)); 1270 if (inode2) 1271 WARN_ON_ONCE(S_ISDIR(inode2->i_mode)); 1272 if (inode1 > inode2) 1273 swap(inode1, inode2); 1274 if (inode1) 1275 inode_lock(inode1); 1276 if (inode2 && inode2 != inode1) 1277 inode_lock_nested(inode2, I_MUTEX_NONDIR2); 1278 } 1279 EXPORT_SYMBOL(lock_two_nondirectories); 1280 1281 /** 1282 * unlock_two_nondirectories - release locks from lock_two_nondirectories() 1283 * @inode1: first inode to unlock 1284 * @inode2: second inode to unlock 1285 */ 1286 void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2) 1287 { 1288 if (inode1) { 1289 WARN_ON_ONCE(S_ISDIR(inode1->i_mode)); 1290 inode_unlock(inode1); 1291 } 1292 if (inode2 && inode2 != inode1) { 1293 WARN_ON_ONCE(S_ISDIR(inode2->i_mode)); 1294 inode_unlock(inode2); 1295 } 1296 } 1297 EXPORT_SYMBOL(unlock_two_nondirectories); 1298 1299 /** 1300 * inode_insert5 - obtain an inode from a mounted file system 1301 * @inode: pre-allocated inode to use for insert to cache 1302 * @hashval: hash value (usually inode number) to get 1303 * @test: callback used for comparisons between inodes 1304 * @set: callback used to initialize a new struct inode 1305 * @data: opaque data pointer to pass to @test and @set 1306 * 1307 * Search for the inode specified by @hashval and @data in the inode cache, 1308 * and if present return it with an increased reference count. This is a 1309 * variant of iget5_locked() that doesn't allocate an inode. 1310 * 1311 * If the inode is not present in the cache, insert the pre-allocated inode and 1312 * return it locked, hashed, and with the I_NEW flag set. The file system gets 1313 * to fill it in before unlocking it via unlock_new_inode(). 1314 * 1315 * Note that both @test and @set are called with the inode_hash_lock held, so 1316 * they can't sleep. 1317 */ 1318 struct inode *inode_insert5(struct inode *inode, u64 hashval, 1319 int (*test)(struct inode *, void *), 1320 int (*set)(struct inode *, void *), void *data) 1321 { 1322 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval); 1323 struct inode *old; 1324 bool isnew; 1325 1326 might_sleep(); 1327 1328 again: 1329 spin_lock(&inode_hash_lock); 1330 old = find_inode(inode->i_sb, head, test, data, true, &isnew); 1331 if (unlikely(old)) { 1332 /* 1333 * Uhhuh, somebody else created the same inode under us. 1334 * Use the old inode instead of the preallocated one. 1335 */ 1336 spin_unlock(&inode_hash_lock); 1337 if (IS_ERR(old)) 1338 return NULL; 1339 if (unlikely(isnew)) 1340 wait_on_new_inode(old); 1341 if (unlikely(inode_unhashed(old))) { 1342 iput(old); 1343 goto again; 1344 } 1345 return old; 1346 } 1347 1348 if (set && unlikely(set(inode, data))) { 1349 spin_unlock(&inode_hash_lock); 1350 return NULL; 1351 } 1352 1353 /* 1354 * Return the locked inode with I_NEW set, the 1355 * caller is responsible for filling in the contents 1356 */ 1357 spin_lock(&inode->i_lock); 1358 inode_state_set(inode, I_NEW); 1359 hlist_add_head_rcu(&inode->i_hash, head); 1360 spin_unlock(&inode->i_lock); 1361 1362 spin_unlock(&inode_hash_lock); 1363 1364 /* 1365 * Add inode to the sb list if it's not already. It has I_NEW at this 1366 * point, so it should be safe to test i_sb_list locklessly. 1367 */ 1368 if (list_empty(&inode->i_sb_list)) 1369 inode_sb_list_add(inode); 1370 1371 return inode; 1372 } 1373 EXPORT_SYMBOL(inode_insert5); 1374 1375 /** 1376 * iget5_locked - obtain an inode from a mounted file system 1377 * @sb: super block of file system 1378 * @hashval: hash value (usually inode number) to get 1379 * @test: callback used for comparisons between inodes 1380 * @set: callback used to initialize a new struct inode 1381 * @data: opaque data pointer to pass to @test and @set 1382 * 1383 * Search for the inode specified by @hashval and @data in the inode cache, 1384 * and if present return it with an increased reference count. This is a 1385 * generalized version of iget_locked() for file systems where the inode 1386 * number is not sufficient for unique identification of an inode. 1387 * 1388 * If the inode is not present in the cache, allocate and insert a new inode 1389 * and return it locked, hashed, and with the I_NEW flag set. The file system 1390 * gets to fill it in before unlocking it via unlock_new_inode(). 1391 * 1392 * Note that both @test and @set are called with the inode_hash_lock held, so 1393 * they can't sleep. 1394 */ 1395 struct inode *iget5_locked(struct super_block *sb, u64 hashval, 1396 int (*test)(struct inode *, void *), 1397 int (*set)(struct inode *, void *), void *data) 1398 { 1399 struct inode *inode = ilookup5(sb, hashval, test, data); 1400 1401 if (!inode) { 1402 struct inode *new = alloc_inode(sb); 1403 1404 if (new) { 1405 inode = inode_insert5(new, hashval, test, set, data); 1406 if (unlikely(inode != new)) 1407 destroy_inode(new); 1408 } 1409 } 1410 return inode; 1411 } 1412 EXPORT_SYMBOL(iget5_locked); 1413 1414 /** 1415 * iget5_locked_rcu - obtain an inode from a mounted file system 1416 * @sb: super block of file system 1417 * @hashval: hash value (usually inode number) to get 1418 * @test: callback used for comparisons between inodes 1419 * @set: callback used to initialize a new struct inode 1420 * @data: opaque data pointer to pass to @test and @set 1421 * 1422 * This is equivalent to iget5_locked, except the @test callback must 1423 * tolerate the inode not being stable, including being mid-teardown. 1424 */ 1425 struct inode *iget5_locked_rcu(struct super_block *sb, u64 hashval, 1426 int (*test)(struct inode *, void *), 1427 int (*set)(struct inode *, void *), void *data) 1428 { 1429 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 1430 struct inode *inode, *new; 1431 bool isnew; 1432 1433 might_sleep(); 1434 1435 again: 1436 inode = find_inode(sb, head, test, data, false, &isnew); 1437 if (inode) { 1438 if (IS_ERR(inode)) 1439 return NULL; 1440 if (unlikely(isnew)) 1441 wait_on_new_inode(inode); 1442 if (unlikely(inode_unhashed(inode))) { 1443 iput(inode); 1444 goto again; 1445 } 1446 return inode; 1447 } 1448 1449 new = alloc_inode(sb); 1450 if (new) { 1451 inode = inode_insert5(new, hashval, test, set, data); 1452 if (unlikely(inode != new)) 1453 destroy_inode(new); 1454 } 1455 return inode; 1456 } 1457 EXPORT_SYMBOL_GPL(iget5_locked_rcu); 1458 1459 /** 1460 * iget_locked - obtain an inode from a mounted file system 1461 * @sb: super block of file system 1462 * @ino: inode number to get 1463 * 1464 * Search for the inode specified by @ino in the inode cache and if present 1465 * return it with an increased reference count. This is for file systems 1466 * where the inode number is sufficient for unique identification of an inode. 1467 * 1468 * If the inode is not in cache, allocate a new inode and return it locked, 1469 * hashed, and with the I_NEW flag set. The file system gets to fill it in 1470 * before unlocking it via unlock_new_inode(). 1471 */ 1472 struct inode *iget_locked(struct super_block *sb, u64 ino) 1473 { 1474 struct hlist_head *head = inode_hashtable + hash(sb, ino); 1475 struct inode *inode; 1476 bool isnew; 1477 1478 might_sleep(); 1479 1480 again: 1481 inode = find_inode_fast(sb, head, ino, false, &isnew); 1482 if (inode) { 1483 if (IS_ERR(inode)) 1484 return NULL; 1485 if (unlikely(isnew)) 1486 wait_on_new_inode(inode); 1487 if (unlikely(inode_unhashed(inode))) { 1488 iput(inode); 1489 goto again; 1490 } 1491 return inode; 1492 } 1493 1494 inode = alloc_inode(sb); 1495 if (inode) { 1496 struct inode *old; 1497 1498 spin_lock(&inode_hash_lock); 1499 /* We released the lock, so.. */ 1500 old = find_inode_fast(sb, head, ino, true, &isnew); 1501 if (!old) { 1502 inode->i_ino = ino; 1503 spin_lock(&inode->i_lock); 1504 inode_state_assign(inode, I_NEW); 1505 hlist_add_head_rcu(&inode->i_hash, head); 1506 spin_unlock(&inode->i_lock); 1507 spin_unlock(&inode_hash_lock); 1508 inode_sb_list_add(inode); 1509 1510 /* Return the locked inode with I_NEW set, the 1511 * caller is responsible for filling in the contents 1512 */ 1513 return inode; 1514 } 1515 1516 /* 1517 * Uhhuh, somebody else created the same inode under 1518 * us. Use the old inode instead of the one we just 1519 * allocated. 1520 */ 1521 spin_unlock(&inode_hash_lock); 1522 destroy_inode(inode); 1523 if (IS_ERR(old)) 1524 return NULL; 1525 inode = old; 1526 if (unlikely(isnew)) 1527 wait_on_new_inode(inode); 1528 if (unlikely(inode_unhashed(inode))) { 1529 iput(inode); 1530 goto again; 1531 } 1532 } 1533 return inode; 1534 } 1535 EXPORT_SYMBOL(iget_locked); 1536 1537 /* 1538 * search the inode cache for a matching inode number. 1539 * If we find one, then the inode number we are trying to 1540 * allocate is not unique and so we should not use it. 1541 * 1542 * Returns 1 if the inode number is unique, 0 if it is not. 1543 */ 1544 static int test_inode_iunique(struct super_block *sb, u64 ino) 1545 { 1546 struct hlist_head *b = inode_hashtable + hash(sb, ino); 1547 struct inode *inode; 1548 1549 hlist_for_each_entry_rcu(inode, b, i_hash) { 1550 if (inode->i_ino == ino && inode->i_sb == sb) 1551 return 0; 1552 } 1553 return 1; 1554 } 1555 1556 /** 1557 * iunique - get a unique inode number 1558 * @sb: superblock 1559 * @max_reserved: highest reserved inode number 1560 * 1561 * Obtain an inode number that is unique on the system for a given 1562 * superblock. This is used by file systems that have no natural 1563 * permanent inode numbering system. An inode number is returned that 1564 * is higher than the reserved limit but unique. 1565 * 1566 * BUGS: 1567 * With a large number of inodes live on the file system this function 1568 * currently becomes quite slow. 1569 */ 1570 ino_t iunique(struct super_block *sb, ino_t max_reserved) 1571 { 1572 /* 1573 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW 1574 * error if st_ino won't fit in target struct field. Use 32bit counter 1575 * here to attempt to avoid that. 1576 */ 1577 static DEFINE_SPINLOCK(iunique_lock); 1578 static unsigned int counter; 1579 ino_t res; 1580 1581 rcu_read_lock(); 1582 spin_lock(&iunique_lock); 1583 do { 1584 if (counter <= max_reserved) 1585 counter = max_reserved + 1; 1586 res = counter++; 1587 } while (!test_inode_iunique(sb, res)); 1588 spin_unlock(&iunique_lock); 1589 rcu_read_unlock(); 1590 1591 return res; 1592 } 1593 EXPORT_SYMBOL(iunique); 1594 1595 /** 1596 * ihold - get a reference on the inode, provided you already have one 1597 * @inode: inode to operate on 1598 */ 1599 void ihold(struct inode *inode) 1600 { 1601 VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); 1602 WARN_ON(atomic_inc_return(&inode->i_count) < 2); 1603 } 1604 EXPORT_SYMBOL(ihold); 1605 1606 struct inode *igrab(struct inode *inode) 1607 { 1608 /* 1609 * Read commentary above igrab_from_hash() for an explanation why this works. 1610 */ 1611 if (atomic_add_unless(&inode->i_count, 1, 0)) { 1612 VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); 1613 return inode; 1614 } 1615 1616 spin_lock(&inode->i_lock); 1617 if (!(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) { 1618 __iget(inode); 1619 spin_unlock(&inode->i_lock); 1620 } else { 1621 spin_unlock(&inode->i_lock); 1622 /* 1623 * Handle the case where s_op->clear_inode is not been 1624 * called yet, and somebody is calling igrab 1625 * while the inode is getting freed. 1626 */ 1627 inode = NULL; 1628 } 1629 return inode; 1630 } 1631 EXPORT_SYMBOL(igrab); 1632 1633 /* 1634 * igrab_from_hash - special inode refcount acquire primitive for the inode hash 1635 * 1636 * It provides lockless refcount acquire in the common case of no problematic 1637 * flags being set and the count being > 0. 1638 * 1639 * There are 4 state flags to worry about and the routine makes sure to not bump the 1640 * ref if any of them is present. 1641 * 1642 * I_NEW and I_CREATING can only legally get set *before* the inode becomes visible 1643 * during lookup. Thus if the flags are not spotted, they are guaranteed to not be 1644 * a factor. However, we need an acquire fence before returning the inode just 1645 * in case we raced against clearing the state to make sure our consumer picks up 1646 * any other changes made prior. atomic_add_unless provides a full fence, which 1647 * takes care of it. 1648 * 1649 * I_FREEING and I_WILL_FREE can only legally get set if ->i_count == 0 and it is 1650 * illegal to bump the ref if either is present. Consequently if atomic_add_unless 1651 * managed to replace a non-0 value with a bigger one, we have a guarantee neither 1652 * of these flags is set. Note this means explicitly checking of these flags below 1653 * is not necessary, it is only done because it does not cost anything on top of the 1654 * load which already needs to be done to handle the other flags. 1655 */ 1656 static bool igrab_from_hash(struct inode *inode) 1657 { 1658 if (inode_state_read_once(inode) & (I_NEW | I_CREATING | I_FREEING | I_WILL_FREE)) 1659 return false; 1660 /* 1661 * Paired with routines clearing I_NEW 1662 */ 1663 if (atomic_add_unless(&inode->i_count, 1, 0)) { 1664 VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); 1665 return true; 1666 } 1667 return false; 1668 } 1669 1670 /** 1671 * ilookup5_nowait - search for an inode in the inode cache 1672 * @sb: super block of file system to search 1673 * @hashval: hash value (usually inode number) to search for 1674 * @test: callback used for comparisons between inodes 1675 * @data: opaque data pointer to pass to @test 1676 * @isnew: return argument telling whether I_NEW was set when 1677 * the inode was found in hash (the caller needs to 1678 * wait for I_NEW to clear) 1679 * 1680 * Search for the inode specified by @hashval and @data in the inode cache. 1681 * If the inode is in the cache, the inode is returned with an incremented 1682 * reference count. 1683 * 1684 * Note: I_NEW is not waited upon so you have to be very careful what you do 1685 * with the returned inode. You probably should be using ilookup5() instead. 1686 * 1687 * Note2: @test is called with the inode_hash_lock held, so can't sleep. 1688 */ 1689 struct inode *ilookup5_nowait(struct super_block *sb, u64 hashval, 1690 int (*test)(struct inode *, void *), void *data, bool *isnew) 1691 { 1692 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 1693 struct inode *inode; 1694 1695 spin_lock(&inode_hash_lock); 1696 inode = find_inode(sb, head, test, data, true, isnew); 1697 spin_unlock(&inode_hash_lock); 1698 1699 return IS_ERR(inode) ? NULL : inode; 1700 } 1701 EXPORT_SYMBOL(ilookup5_nowait); 1702 1703 /** 1704 * ilookup5 - search for an inode in the inode cache 1705 * @sb: super block of file system to search 1706 * @hashval: hash value (usually inode number) to search for 1707 * @test: callback used for comparisons between inodes 1708 * @data: opaque data pointer to pass to @test 1709 * 1710 * Search for the inode specified by @hashval and @data in the inode cache, 1711 * and if the inode is in the cache, return the inode with an incremented 1712 * reference count. Waits on I_NEW before returning the inode. 1713 * returned with an incremented reference count. 1714 * 1715 * This is a generalized version of ilookup() for file systems where the 1716 * inode number is not sufficient for unique identification of an inode. 1717 * 1718 * Note: @test is called with the inode_hash_lock held, so can't sleep. 1719 */ 1720 struct inode *ilookup5(struct super_block *sb, u64 hashval, 1721 int (*test)(struct inode *, void *), void *data) 1722 { 1723 struct inode *inode; 1724 bool isnew; 1725 1726 might_sleep(); 1727 1728 again: 1729 inode = ilookup5_nowait(sb, hashval, test, data, &isnew); 1730 if (inode) { 1731 if (unlikely(isnew)) 1732 wait_on_new_inode(inode); 1733 if (unlikely(inode_unhashed(inode))) { 1734 iput(inode); 1735 goto again; 1736 } 1737 } 1738 return inode; 1739 } 1740 EXPORT_SYMBOL(ilookup5); 1741 1742 /** 1743 * ilookup - search for an inode in the inode cache 1744 * @sb: super block of file system to search 1745 * @ino: inode number to search for 1746 * 1747 * Search for the inode @ino in the inode cache, and if the inode is in the 1748 * cache, the inode is returned with an incremented reference count. 1749 */ 1750 struct inode *ilookup(struct super_block *sb, u64 ino) 1751 { 1752 struct hlist_head *head = inode_hashtable + hash(sb, ino); 1753 struct inode *inode; 1754 bool isnew; 1755 1756 might_sleep(); 1757 1758 again: 1759 inode = find_inode_fast(sb, head, ino, false, &isnew); 1760 1761 if (inode) { 1762 if (IS_ERR(inode)) 1763 return NULL; 1764 if (unlikely(isnew)) 1765 wait_on_new_inode(inode); 1766 if (unlikely(inode_unhashed(inode))) { 1767 iput(inode); 1768 goto again; 1769 } 1770 } 1771 return inode; 1772 } 1773 EXPORT_SYMBOL(ilookup); 1774 1775 /** 1776 * find_inode_nowait - find an inode in the inode cache 1777 * @sb: super block of file system to search 1778 * @hashval: hash value (usually inode number) to search for 1779 * @match: callback used for comparisons between inodes 1780 * @data: opaque data pointer to pass to @match 1781 * 1782 * Search for the inode specified by @hashval and @data in the inode 1783 * cache, where the helper function @match will return 0 if the inode 1784 * does not match, 1 if the inode does match, and -1 if the search 1785 * should be stopped. The @match function must be responsible for 1786 * taking the i_lock spin_lock and checking i_state for an inode being 1787 * freed or being initialized, and incrementing the reference count 1788 * before returning 1. It also must not sleep, since it is called with 1789 * the inode_hash_lock spinlock held. 1790 * 1791 * This is a even more generalized version of ilookup5() when the 1792 * function must never block --- find_inode() can block in 1793 * __wait_on_freeing_inode() --- or when the caller can not increment 1794 * the reference count because the resulting iput() might cause an 1795 * inode eviction. The tradeoff is that the @match function must be 1796 * very carefully implemented. 1797 */ 1798 struct inode *find_inode_nowait(struct super_block *sb, 1799 u64 hashval, 1800 int (*match)(struct inode *, u64, 1801 void *), 1802 void *data) 1803 { 1804 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 1805 struct inode *inode, *ret_inode = NULL; 1806 int mval; 1807 1808 spin_lock(&inode_hash_lock); 1809 hlist_for_each_entry(inode, head, i_hash) { 1810 if (inode->i_sb != sb) 1811 continue; 1812 mval = match(inode, hashval, data); 1813 if (mval == 0) 1814 continue; 1815 if (mval == 1) 1816 ret_inode = inode; 1817 goto out; 1818 } 1819 out: 1820 spin_unlock(&inode_hash_lock); 1821 return ret_inode; 1822 } 1823 EXPORT_SYMBOL(find_inode_nowait); 1824 1825 /** 1826 * find_inode_rcu - find an inode in the inode cache 1827 * @sb: Super block of file system to search 1828 * @hashval: Key to hash 1829 * @test: Function to test match on an inode 1830 * @data: Data for test function 1831 * 1832 * Search for the inode specified by @hashval and @data in the inode cache, 1833 * where the helper function @test will return 0 if the inode does not match 1834 * and 1 if it does. The @test function must be responsible for taking the 1835 * i_lock spin_lock and checking i_state for an inode being freed or being 1836 * initialized. 1837 * 1838 * If successful, this will return the inode for which the @test function 1839 * returned 1 and NULL otherwise. 1840 * 1841 * The @test function is not permitted to take a ref on any inode presented. 1842 * It is also not permitted to sleep. 1843 * 1844 * The caller must hold the RCU read lock. 1845 */ 1846 struct inode *find_inode_rcu(struct super_block *sb, u64 hashval, 1847 int (*test)(struct inode *, void *), void *data) 1848 { 1849 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 1850 struct inode *inode; 1851 1852 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), 1853 "suspicious find_inode_rcu() usage"); 1854 1855 hlist_for_each_entry_rcu(inode, head, i_hash) { 1856 if (inode->i_sb == sb && 1857 !(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE)) && 1858 test(inode, data)) 1859 return inode; 1860 } 1861 return NULL; 1862 } 1863 EXPORT_SYMBOL(find_inode_rcu); 1864 1865 /** 1866 * find_inode_by_ino_rcu - Find an inode in the inode cache 1867 * @sb: Super block of file system to search 1868 * @ino: The inode number to match 1869 * 1870 * Search for the inode specified by @hashval and @data in the inode cache, 1871 * where the helper function @test will return 0 if the inode does not match 1872 * and 1 if it does. The @test function must be responsible for taking the 1873 * i_lock spin_lock and checking i_state for an inode being freed or being 1874 * initialized. 1875 * 1876 * If successful, this will return the inode for which the @test function 1877 * returned 1 and NULL otherwise. 1878 * 1879 * The @test function is not permitted to take a ref on any inode presented. 1880 * It is also not permitted to sleep. 1881 * 1882 * The caller must hold the RCU read lock. 1883 */ 1884 struct inode *find_inode_by_ino_rcu(struct super_block *sb, 1885 u64 ino) 1886 { 1887 struct hlist_head *head = inode_hashtable + hash(sb, ino); 1888 struct inode *inode; 1889 1890 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), 1891 "suspicious find_inode_by_ino_rcu() usage"); 1892 1893 hlist_for_each_entry_rcu(inode, head, i_hash) { 1894 if (inode->i_ino == ino && 1895 inode->i_sb == sb && 1896 !(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE))) 1897 return inode; 1898 } 1899 return NULL; 1900 } 1901 EXPORT_SYMBOL(find_inode_by_ino_rcu); 1902 1903 int insert_inode_locked(struct inode *inode) 1904 { 1905 struct super_block *sb = inode->i_sb; 1906 u64 ino = inode->i_ino; 1907 struct hlist_head *head = inode_hashtable + hash(sb, ino); 1908 bool isnew; 1909 1910 might_sleep(); 1911 1912 while (1) { 1913 struct inode *old = NULL; 1914 spin_lock(&inode_hash_lock); 1915 repeat: 1916 hlist_for_each_entry(old, head, i_hash) { 1917 if (old->i_ino != ino) 1918 continue; 1919 if (old->i_sb != sb) 1920 continue; 1921 spin_lock(&old->i_lock); 1922 break; 1923 } 1924 if (likely(!old)) { 1925 spin_lock(&inode->i_lock); 1926 inode_state_set(inode, I_NEW | I_CREATING); 1927 hlist_add_head_rcu(&inode->i_hash, head); 1928 spin_unlock(&inode->i_lock); 1929 spin_unlock(&inode_hash_lock); 1930 return 0; 1931 } 1932 if (inode_state_read(old) & (I_FREEING | I_WILL_FREE)) { 1933 __wait_on_freeing_inode(old, true, false); 1934 old = NULL; 1935 goto repeat; 1936 } 1937 if (unlikely(inode_state_read(old) & I_CREATING)) { 1938 spin_unlock(&old->i_lock); 1939 spin_unlock(&inode_hash_lock); 1940 return -EBUSY; 1941 } 1942 __iget(old); 1943 isnew = !!(inode_state_read(old) & I_NEW); 1944 spin_unlock(&old->i_lock); 1945 spin_unlock(&inode_hash_lock); 1946 if (isnew) 1947 wait_on_new_inode(old); 1948 if (unlikely(!inode_unhashed(old))) { 1949 iput(old); 1950 return -EBUSY; 1951 } 1952 iput(old); 1953 } 1954 } 1955 EXPORT_SYMBOL(insert_inode_locked); 1956 1957 int insert_inode_locked4(struct inode *inode, u64 hashval, 1958 int (*test)(struct inode *, void *), void *data) 1959 { 1960 struct inode *old; 1961 1962 might_sleep(); 1963 1964 inode_state_set_raw(inode, I_CREATING); 1965 old = inode_insert5(inode, hashval, test, NULL, data); 1966 1967 if (old != inode) { 1968 iput(old); 1969 return -EBUSY; 1970 } 1971 return 0; 1972 } 1973 EXPORT_SYMBOL(insert_inode_locked4); 1974 1975 1976 int inode_just_drop(struct inode *inode) 1977 { 1978 return 1; 1979 } 1980 EXPORT_SYMBOL(inode_just_drop); 1981 1982 /* 1983 * Called when we're dropping the last reference 1984 * to an inode. 1985 * 1986 * Call the FS "drop_inode()" function, defaulting to 1987 * the legacy UNIX filesystem behaviour. If it tells 1988 * us to evict inode, do so. Otherwise, retain inode 1989 * in cache if fs is alive, sync and evict if fs is 1990 * shutting down. 1991 */ 1992 static void iput_final(struct inode *inode) 1993 { 1994 struct super_block *sb = inode->i_sb; 1995 const struct super_operations *op = inode->i_sb->s_op; 1996 int drop; 1997 1998 WARN_ON(inode_state_read(inode) & I_NEW); 1999 VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); 2000 2001 if (op->drop_inode) 2002 drop = op->drop_inode(inode); 2003 else 2004 drop = inode_generic_drop(inode); 2005 2006 if (!drop && 2007 !(inode_state_read(inode) & I_DONTCACHE) && 2008 (sb->s_flags & SB_ACTIVE)) { 2009 __inode_lru_list_add(inode, true); 2010 spin_unlock(&inode->i_lock); 2011 return; 2012 } 2013 2014 /* 2015 * Re-check ->i_count in case the ->drop_inode() hooks played games. 2016 * Note we only execute this if the verdict was to drop the inode. 2017 */ 2018 VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); 2019 2020 if (drop) { 2021 inode_state_set(inode, I_FREEING); 2022 } else { 2023 inode_state_set(inode, I_WILL_FREE); 2024 spin_unlock(&inode->i_lock); 2025 2026 write_inode_now(inode, 1); 2027 2028 spin_lock(&inode->i_lock); 2029 WARN_ON(inode_state_read(inode) & I_NEW); 2030 inode_state_replace(inode, I_WILL_FREE, I_FREEING); 2031 } 2032 2033 inode_lru_list_del(inode); 2034 spin_unlock(&inode->i_lock); 2035 2036 evict(inode); 2037 } 2038 2039 /** 2040 * iput - put an inode 2041 * @inode: inode to put 2042 * 2043 * Puts an inode, dropping its usage count. If the inode use count hits 2044 * zero, the inode is then freed and may also be destroyed. 2045 * 2046 * Consequently, iput() can sleep. 2047 */ 2048 void iput(struct inode *inode) 2049 { 2050 might_sleep(); 2051 if (unlikely(!inode)) 2052 return; 2053 2054 retry: 2055 lockdep_assert_not_held(&inode->i_lock); 2056 VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); 2057 /* 2058 * Note this assert is technically racy as if the count is bogusly 2059 * equal to one, then two CPUs racing to further drop it can both 2060 * conclude it's fine. 2061 */ 2062 VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); 2063 2064 if (atomic_add_unless(&inode->i_count, -1, 1)) 2065 return; 2066 2067 if (inode->i_nlink && sync_lazytime(inode)) 2068 goto retry; 2069 2070 spin_lock(&inode->i_lock); 2071 if (unlikely((inode_state_read(inode) & I_DIRTY_TIME) && inode->i_nlink)) { 2072 spin_unlock(&inode->i_lock); 2073 goto retry; 2074 } 2075 2076 if (!atomic_dec_and_test(&inode->i_count)) { 2077 spin_unlock(&inode->i_lock); 2078 return; 2079 } 2080 2081 /* 2082 * iput_final() drops ->i_lock, we can't assert on it as the inode may 2083 * be deallocated by the time the call returns. 2084 */ 2085 iput_final(inode); 2086 } 2087 EXPORT_SYMBOL(iput); 2088 2089 /** 2090 * iput_not_last - put an inode assuming this is not the last reference 2091 * @inode: inode to put 2092 */ 2093 void iput_not_last(struct inode *inode) 2094 { 2095 VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); 2096 VFS_BUG_ON_INODE(icount_read_once(inode) < 2, inode); 2097 2098 WARN_ON(atomic_sub_return(1, &inode->i_count) == 0); 2099 } 2100 EXPORT_SYMBOL(iput_not_last); 2101 2102 #ifdef CONFIG_BLOCK 2103 /** 2104 * bmap - find a block number in a file 2105 * @inode: inode owning the block number being requested 2106 * @block: pointer containing the block to find 2107 * 2108 * Replaces the value in ``*block`` with the block number on the device holding 2109 * corresponding to the requested block number in the file. 2110 * That is, asked for block 4 of inode 1 the function will replace the 2111 * 4 in ``*block``, with disk block relative to the disk start that holds that 2112 * block of the file. 2113 * 2114 * Returns -EINVAL in case of error, 0 otherwise. If mapping falls into a 2115 * hole, returns 0 and ``*block`` is also set to 0. 2116 */ 2117 int bmap(struct inode *inode, sector_t *block) 2118 { 2119 if (!inode->i_mapping->a_ops->bmap) 2120 return -EINVAL; 2121 2122 *block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block); 2123 return 0; 2124 } 2125 EXPORT_SYMBOL(bmap); 2126 #endif 2127 2128 /* 2129 * With relative atime, only update atime if the previous atime is 2130 * earlier than or equal to either the ctime or mtime, 2131 * or if at least a day has passed since the last atime update. 2132 */ 2133 static bool relatime_need_update(struct vfsmount *mnt, struct inode *inode, 2134 struct timespec64 now) 2135 { 2136 struct timespec64 atime, mtime, ctime; 2137 2138 if (!(mnt->mnt_flags & MNT_RELATIME)) 2139 return true; 2140 /* 2141 * Is mtime younger than or equal to atime? If yes, update atime: 2142 */ 2143 atime = inode_get_atime(inode); 2144 mtime = inode_get_mtime(inode); 2145 if (timespec64_compare(&mtime, &atime) >= 0) 2146 return true; 2147 /* 2148 * Is ctime younger than or equal to atime? If yes, update atime: 2149 */ 2150 ctime = inode_get_ctime(inode); 2151 if (timespec64_compare(&ctime, &atime) >= 0) 2152 return true; 2153 2154 /* 2155 * Is the previous atime value older than a day? If yes, 2156 * update atime: 2157 */ 2158 if ((long)(now.tv_sec - atime.tv_sec) >= 24*60*60) 2159 return true; 2160 /* 2161 * Good, we can skip the atime update: 2162 */ 2163 return false; 2164 } 2165 2166 static int inode_update_atime(struct inode *inode) 2167 { 2168 struct timespec64 atime = inode_get_atime(inode); 2169 struct timespec64 now = current_time(inode); 2170 2171 if (timespec64_equal(&now, &atime)) 2172 return 0; 2173 2174 inode_set_atime_to_ts(inode, now); 2175 return inode_time_dirty_flag(inode); 2176 } 2177 2178 static int inode_update_cmtime(struct inode *inode, unsigned int flags) 2179 { 2180 struct timespec64 ctime = inode_get_ctime(inode); 2181 struct timespec64 mtime = inode_get_mtime(inode); 2182 struct timespec64 now = inode_set_ctime_current(inode); 2183 unsigned int dirty = 0; 2184 bool mtime_changed; 2185 2186 mtime_changed = !timespec64_equal(&now, &mtime); 2187 if (mtime_changed || !timespec64_equal(&now, &ctime)) 2188 dirty = inode_time_dirty_flag(inode); 2189 2190 /* 2191 * Pure timestamp updates can be recorded in the inode without blocking 2192 * by not dirtying the inode. But when the file system requires 2193 * i_version updates, the update of i_version can still block. 2194 * Error out if we'd actually have to update i_version or don't support 2195 * lazytime. 2196 */ 2197 if (IS_I_VERSION(inode)) { 2198 if (flags & IOCB_NOWAIT) { 2199 if (!(inode->i_sb->s_flags & SB_LAZYTIME) || 2200 inode_iversion_need_inc(inode)) 2201 return -EAGAIN; 2202 } else { 2203 /* 2204 * Don't force iversion increment for pure lazytime 2205 * updates (I_DIRTY_TIME only), let I_VERSION_QUERIED 2206 * dictate whether the increment is needed. 2207 */ 2208 if (inode_maybe_inc_iversion(inode, 2209 dirty != I_DIRTY_TIME)) 2210 dirty |= I_DIRTY_SYNC; 2211 } 2212 } 2213 2214 if (mtime_changed) 2215 inode_set_mtime_to_ts(inode, now); 2216 return dirty; 2217 } 2218 2219 /** 2220 * inode_update_time - update either atime or c/mtime and i_version on the inode 2221 * @inode: inode to be updated 2222 * @type: timestamp to be updated 2223 * @flags: flags for the update 2224 * 2225 * Update either atime or c/mtime and version in a inode if needed for a file 2226 * access or modification. It is up to the caller to mark the inode dirty 2227 * appropriately. 2228 * 2229 * Returns the positive I_DIRTY_* flags for __mark_inode_dirty() if the inode 2230 * needs to be marked dirty, 0 if it did not, or a negative errno if an error 2231 * happened. 2232 */ 2233 int inode_update_time(struct inode *inode, enum fs_update_time type, 2234 unsigned int flags) 2235 { 2236 switch (type) { 2237 case FS_UPD_ATIME: 2238 return inode_update_atime(inode); 2239 case FS_UPD_CMTIME: 2240 return inode_update_cmtime(inode, flags); 2241 default: 2242 WARN_ON_ONCE(1); 2243 return -EIO; 2244 } 2245 } 2246 EXPORT_SYMBOL(inode_update_time); 2247 2248 /** 2249 * generic_update_time - update the timestamps on the inode 2250 * @inode: inode to be updated 2251 * @type: timestamp to be updated 2252 * @flags: flags for the update 2253 * 2254 * Returns a negative error value on error, else 0. 2255 */ 2256 int generic_update_time(struct inode *inode, enum fs_update_time type, 2257 unsigned int flags) 2258 { 2259 int dirty; 2260 2261 /* 2262 * ->dirty_inode is what could make generic timestamp updates block. 2263 * Don't support non-blocking timestamp updates here if it is set. 2264 * File systems that implement ->dirty_inode but want to support 2265 * non-blocking timestamp updates should call inode_update_time 2266 * directly. 2267 */ 2268 if ((flags & IOCB_NOWAIT) && inode->i_sb->s_op->dirty_inode) 2269 return -EAGAIN; 2270 2271 dirty = inode_update_time(inode, type, flags); 2272 if (dirty <= 0) 2273 return dirty; 2274 __mark_inode_dirty(inode, dirty); 2275 return 0; 2276 } 2277 EXPORT_SYMBOL(generic_update_time); 2278 2279 /** 2280 * atime_needs_update - update the access time 2281 * @path: the &struct path to update 2282 * @inode: inode to update 2283 * 2284 * Update the accessed time on an inode and mark it for writeback. 2285 * This function automatically handles read only file systems and media, 2286 * as well as the "noatime" flag and inode specific "noatime" markers. 2287 */ 2288 bool atime_needs_update(const struct path *path, struct inode *inode) 2289 { 2290 struct vfsmount *mnt = path->mnt; 2291 struct timespec64 now, atime; 2292 2293 if (inode->i_flags & S_NOATIME) 2294 return false; 2295 2296 /* Atime updates will likely cause i_uid and i_gid to be written 2297 * back improprely if their true value is unknown to the vfs. 2298 */ 2299 if (HAS_UNMAPPED_ID(mnt_idmap(mnt), inode)) 2300 return false; 2301 2302 if (IS_NOATIME(inode)) 2303 return false; 2304 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode)) 2305 return false; 2306 2307 if (mnt->mnt_flags & MNT_NOATIME) 2308 return false; 2309 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) 2310 return false; 2311 2312 now = current_time(inode); 2313 2314 if (!relatime_need_update(mnt, inode, now)) 2315 return false; 2316 2317 atime = inode_get_atime(inode); 2318 if (timespec64_equal(&atime, &now)) 2319 return false; 2320 2321 return true; 2322 } 2323 2324 void touch_atime(const struct path *path) 2325 { 2326 struct vfsmount *mnt = path->mnt; 2327 struct inode *inode = d_inode(path->dentry); 2328 2329 if (!atime_needs_update(path, inode)) 2330 return; 2331 2332 if (!sb_start_write_trylock(inode->i_sb)) 2333 return; 2334 2335 if (mnt_get_write_access(mnt) != 0) 2336 goto skip_update; 2337 /* 2338 * File systems can error out when updating inodes if they need to 2339 * allocate new space to modify an inode (such is the case for 2340 * Btrfs), but since we touch atime while walking down the path we 2341 * really don't care if we failed to update the atime of the file, 2342 * so just ignore the return value. 2343 * We may also fail on filesystems that have the ability to make parts 2344 * of the fs read only, e.g. subvolumes in Btrfs. 2345 */ 2346 if (inode->i_op->update_time) 2347 inode->i_op->update_time(inode, FS_UPD_ATIME, 0); 2348 else 2349 generic_update_time(inode, FS_UPD_ATIME, 0); 2350 mnt_put_write_access(mnt); 2351 skip_update: 2352 sb_end_write(inode->i_sb); 2353 } 2354 EXPORT_SYMBOL(touch_atime); 2355 2356 /* 2357 * Return mask of changes for notify_change() that need to be done as a 2358 * response to write or truncate. Return 0 if nothing has to be changed. 2359 * Negative value on error (change should be denied). 2360 */ 2361 int dentry_needs_remove_privs(const struct mnt_idmap *idmap, 2362 struct dentry *dentry) 2363 { 2364 struct inode *inode = d_inode(dentry); 2365 int mask = 0; 2366 int ret; 2367 2368 if (IS_NOSEC(inode)) 2369 return 0; 2370 2371 mask = setattr_should_drop_suidgid(idmap, inode); 2372 ret = security_inode_need_killpriv(dentry); 2373 if (ret < 0) 2374 return ret; 2375 if (ret) 2376 mask |= ATTR_KILL_PRIV; 2377 return mask; 2378 } 2379 2380 static int __remove_privs(const struct mnt_idmap *idmap, 2381 struct dentry *dentry, int kill) 2382 { 2383 struct iattr newattrs; 2384 2385 newattrs.ia_valid = ATTR_FORCE | kill; 2386 /* 2387 * Note we call this on write, so notify_change will not 2388 * encounter any conflicting delegations: 2389 */ 2390 return notify_change(idmap, dentry, &newattrs, NULL); 2391 } 2392 2393 static int file_remove_privs_flags(struct file *file, unsigned int flags) 2394 { 2395 struct dentry *dentry = file_dentry(file); 2396 struct inode *inode = file_inode(file); 2397 int error = 0; 2398 int kill; 2399 2400 if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode)) 2401 return 0; 2402 2403 kill = dentry_needs_remove_privs(file_mnt_idmap(file), dentry); 2404 if (kill < 0) 2405 return kill; 2406 2407 if (kill) { 2408 if (flags & IOCB_NOWAIT) 2409 return -EAGAIN; 2410 2411 error = __remove_privs(file_mnt_idmap(file), dentry, kill); 2412 } 2413 2414 if (!error) 2415 inode_has_no_xattr(inode); 2416 return error; 2417 } 2418 2419 /** 2420 * file_remove_privs - remove special file privileges (suid, capabilities) 2421 * @file: file to remove privileges from 2422 * 2423 * When file is modified by a write or truncation ensure that special 2424 * file privileges are removed. 2425 * 2426 * Return: 0 on success, negative errno on failure. 2427 */ 2428 int file_remove_privs(struct file *file) 2429 { 2430 return file_remove_privs_flags(file, 0); 2431 } 2432 EXPORT_SYMBOL(file_remove_privs); 2433 2434 /** 2435 * current_time - Return FS time (possibly fine-grained) 2436 * @inode: inode. 2437 * 2438 * Return the current time truncated to the time granularity supported by 2439 * the fs, as suitable for a ctime/mtime change. If the ctime is flagged 2440 * as having been QUERIED, get a fine-grained timestamp, but don't update 2441 * the floor. 2442 * 2443 * For a multigrain inode, this is effectively an estimate of the timestamp 2444 * that a file would receive. An actual update must go through 2445 * inode_set_ctime_current(). 2446 */ 2447 struct timespec64 current_time(struct inode *inode) 2448 { 2449 struct timespec64 now; 2450 u32 cns; 2451 2452 ktime_get_coarse_real_ts64_mg(&now); 2453 2454 if (!is_mgtime(inode)) 2455 goto out; 2456 2457 /* If nothing has queried it, then coarse time is fine */ 2458 cns = smp_load_acquire(&inode->i_ctime_nsec); 2459 if (cns & I_CTIME_QUERIED) { 2460 /* 2461 * If there is no apparent change, then get a fine-grained 2462 * timestamp. 2463 */ 2464 if (now.tv_nsec == (cns & ~I_CTIME_QUERIED)) 2465 ktime_get_real_ts64(&now); 2466 } 2467 out: 2468 return timestamp_truncate(now, inode); 2469 } 2470 EXPORT_SYMBOL(current_time); 2471 2472 static inline bool need_cmtime_update(struct inode *inode) 2473 { 2474 struct timespec64 now = current_time(inode), ts; 2475 2476 ts = inode_get_mtime(inode); 2477 if (!timespec64_equal(&ts, &now)) 2478 return true; 2479 ts = inode_get_ctime(inode); 2480 if (!timespec64_equal(&ts, &now)) 2481 return true; 2482 return IS_I_VERSION(inode) && inode_iversion_need_inc(inode); 2483 } 2484 2485 static int file_update_time_flags(struct file *file, unsigned int flags) 2486 { 2487 struct inode *inode = file_inode(file); 2488 int ret; 2489 2490 /* First try to exhaust all avenues to not sync */ 2491 if (IS_NOCMTIME(inode)) 2492 return 0; 2493 if (unlikely(file->f_mode & FMODE_NOCMTIME)) 2494 return 0; 2495 if (!need_cmtime_update(inode)) 2496 return 0; 2497 2498 flags &= IOCB_NOWAIT; 2499 if (mnt_get_write_access_file(file)) 2500 return 0; 2501 if (inode->i_op->update_time) 2502 ret = inode->i_op->update_time(inode, FS_UPD_CMTIME, flags); 2503 else 2504 ret = generic_update_time(inode, FS_UPD_CMTIME, flags); 2505 mnt_put_write_access_file(file); 2506 return ret; 2507 } 2508 2509 /** 2510 * file_update_time - update mtime and ctime time 2511 * @file: file accessed 2512 * 2513 * Update the mtime and ctime members of an inode and mark the inode for 2514 * writeback. Note that this function is meant exclusively for usage in 2515 * the file write path of filesystems, and filesystems may choose to 2516 * explicitly ignore updates via this function with the _NOCMTIME inode 2517 * flag, e.g. for network filesystem where these imestamps are handled 2518 * by the server. This can return an error for file systems who need to 2519 * allocate space in order to update an inode. 2520 * 2521 * Return: 0 on success, negative errno on failure. 2522 */ 2523 int file_update_time(struct file *file) 2524 { 2525 return file_update_time_flags(file, 0); 2526 } 2527 EXPORT_SYMBOL(file_update_time); 2528 2529 /** 2530 * file_modified_flags - handle mandated vfs changes when modifying a file 2531 * @file: file that was modified 2532 * @flags: kiocb flags 2533 * 2534 * When file has been modified ensure that special 2535 * file privileges are removed and time settings are updated. 2536 * 2537 * If IOCB_NOWAIT is set, special file privileges will not be removed and 2538 * time settings will not be updated. It will return -EAGAIN. 2539 * 2540 * Context: Caller must hold the file's inode lock. 2541 * 2542 * Return: 0 on success, negative errno on failure. 2543 */ 2544 static int file_modified_flags(struct file *file, int flags) 2545 { 2546 int ret; 2547 2548 /* 2549 * Clear the security bits if the process is not being run by root. 2550 * This keeps people from modifying setuid and setgid binaries. 2551 */ 2552 ret = file_remove_privs_flags(file, flags); 2553 if (ret) 2554 return ret; 2555 return file_update_time_flags(file, flags); 2556 } 2557 2558 /** 2559 * file_modified - handle mandated vfs changes when modifying a file 2560 * @file: file that was modified 2561 * 2562 * When file has been modified ensure that special 2563 * file privileges are removed and time settings are updated. 2564 * 2565 * Context: Caller must hold the file's inode lock. 2566 * 2567 * Return: 0 on success, negative errno on failure. 2568 */ 2569 int file_modified(struct file *file) 2570 { 2571 return file_modified_flags(file, 0); 2572 } 2573 EXPORT_SYMBOL(file_modified); 2574 2575 /** 2576 * kiocb_modified - handle mandated vfs changes when modifying a file 2577 * @iocb: iocb that was modified 2578 * 2579 * When file has been modified ensure that special 2580 * file privileges are removed and time settings are updated. 2581 * 2582 * Context: Caller must hold the file's inode lock. 2583 * 2584 * Return: 0 on success, negative errno on failure. 2585 */ 2586 int kiocb_modified(struct kiocb *iocb) 2587 { 2588 return file_modified_flags(iocb->ki_filp, iocb->ki_flags); 2589 } 2590 EXPORT_SYMBOL_GPL(kiocb_modified); 2591 2592 int inode_needs_sync(struct inode *inode) 2593 { 2594 if (IS_SYNC(inode)) 2595 return 1; 2596 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) 2597 return 1; 2598 return 0; 2599 } 2600 EXPORT_SYMBOL(inode_needs_sync); 2601 2602 /* 2603 * If we try to find an inode in the inode hash while it is being 2604 * deleted, we have to wait until the filesystem completes its 2605 * deletion before reporting that it isn't found. This function waits 2606 * until the deletion _might_ have completed. Callers are responsible 2607 * to recheck inode state. 2608 * 2609 * It doesn't matter if I_NEW is not set initially, a call to 2610 * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list 2611 * will DTRT. 2612 */ 2613 static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked) 2614 { 2615 struct wait_bit_queue_entry wqe; 2616 struct wait_queue_head *wq_head; 2617 2618 VFS_BUG_ON(!hash_locked && !rcu_locked); 2619 2620 /* 2621 * Handle racing against evict(), see that routine for more details. 2622 */ 2623 if (unlikely(inode_unhashed(inode))) { 2624 WARN_ON(hash_locked); 2625 spin_unlock(&inode->i_lock); 2626 return; 2627 } 2628 2629 wq_head = inode_bit_waitqueue(&wqe, inode, __I_NEW); 2630 prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); 2631 spin_unlock(&inode->i_lock); 2632 if (rcu_locked) 2633 rcu_read_unlock(); 2634 if (hash_locked) 2635 spin_unlock(&inode_hash_lock); 2636 schedule(); 2637 finish_wait(wq_head, &wqe.wq_entry); 2638 if (hash_locked) 2639 spin_lock(&inode_hash_lock); 2640 if (rcu_locked) 2641 rcu_read_lock(); 2642 } 2643 2644 static __initdata unsigned long ihash_entries; 2645 static int __init set_ihash_entries(char *str) 2646 { 2647 return kstrtoul(str, 0, &ihash_entries) == 0; 2648 } 2649 __setup("ihash_entries=", set_ihash_entries); 2650 2651 /* 2652 * Initialize the waitqueues and inode hash table. 2653 */ 2654 void __init inode_init_early(void) 2655 { 2656 /* If hashes are distributed across NUMA nodes, defer 2657 * hash allocation until vmalloc space is available. 2658 */ 2659 if (hashdist) 2660 return; 2661 2662 inode_hashtable = 2663 alloc_large_system_hash("Inode-cache", 2664 sizeof(struct hlist_head), 2665 ihash_entries, 2666 14, 2667 HASH_EARLY | HASH_ZERO, 2668 &i_hash_shift, 2669 &i_hash_mask, 2670 0, 2671 0); 2672 } 2673 2674 void __init inode_init(void) 2675 { 2676 /* inode slab cache */ 2677 inode_cachep = kmem_cache_create("inode_cache", 2678 sizeof(struct inode), 2679 0, 2680 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 2681 SLAB_ACCOUNT), 2682 init_once); 2683 2684 /* Hash may have been set up in inode_init_early */ 2685 if (!hashdist) 2686 return; 2687 2688 inode_hashtable = 2689 alloc_large_system_hash("Inode-cache", 2690 sizeof(struct hlist_head), 2691 ihash_entries, 2692 14, 2693 HASH_ZERO, 2694 &i_hash_shift, 2695 &i_hash_mask, 2696 0, 2697 0); 2698 } 2699 2700 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev) 2701 { 2702 inode->i_mode = mode; 2703 switch (inode->i_mode & S_IFMT) { 2704 case S_IFCHR: 2705 inode->i_fop = &def_chr_fops; 2706 inode->i_rdev = rdev; 2707 break; 2708 case S_IFBLK: 2709 if (IS_ENABLED(CONFIG_BLOCK)) 2710 inode->i_fop = &def_blk_fops; 2711 inode->i_rdev = rdev; 2712 break; 2713 case S_IFIFO: 2714 inode->i_fop = &pipefifo_fops; 2715 break; 2716 case S_IFSOCK: 2717 /* leave it no_open_fops */ 2718 break; 2719 default: 2720 pr_debug("init_special_inode: bogus i_mode (%o) for inode %s:%llu\n", 2721 mode, inode->i_sb->s_id, inode->i_ino); 2722 break; 2723 } 2724 } 2725 EXPORT_SYMBOL(init_special_inode); 2726 2727 /** 2728 * inode_init_owner - Init uid,gid,mode for new inode according to posix standards 2729 * @idmap: idmap of the mount the inode was created from 2730 * @inode: New inode 2731 * @dir: Directory inode 2732 * @mode: mode of the new inode 2733 * 2734 * If the inode has been created through an idmapped mount the idmap of 2735 * the vfsmount must be passed through @idmap. This function will then take 2736 * care to map the inode according to @idmap before checking permissions 2737 * and initializing i_uid and i_gid. On non-idmapped mounts or if permission 2738 * checking is to be performed on the raw inode simply pass @nop_mnt_idmap. 2739 */ 2740 void inode_init_owner(const struct mnt_idmap *idmap, struct inode *inode, 2741 const struct inode *dir, umode_t mode) 2742 { 2743 inode_fsuid_set(inode, idmap); 2744 if (dir && dir->i_mode & S_ISGID) { 2745 inode->i_gid = dir->i_gid; 2746 2747 /* Directories are special, and always inherit S_ISGID */ 2748 if (S_ISDIR(mode)) 2749 mode |= S_ISGID; 2750 } else 2751 inode_fsgid_set(inode, idmap); 2752 inode->i_mode = mode; 2753 } 2754 EXPORT_SYMBOL(inode_init_owner); 2755 2756 /** 2757 * inode_owner_or_capable - check current task permissions to inode 2758 * @idmap: idmap of the mount the inode was found from 2759 * @inode: inode being checked 2760 * 2761 * Return true if current either has CAP_FOWNER in a namespace with the 2762 * inode owner uid mapped, or owns the file. 2763 * 2764 * If the inode has been found through an idmapped mount the idmap of 2765 * the vfsmount must be passed through @idmap. This function will then take 2766 * care to map the inode according to @idmap before checking permissions. 2767 * On non-idmapped mounts or if permission checking is to be performed on the 2768 * raw inode simply pass @nop_mnt_idmap. 2769 */ 2770 bool inode_owner_or_capable(const struct mnt_idmap *idmap, 2771 const struct inode *inode) 2772 { 2773 vfsuid_t vfsuid; 2774 struct user_namespace *ns; 2775 2776 vfsuid = i_uid_into_vfsuid(idmap, inode); 2777 if (vfsuid_eq_kuid(vfsuid, current_fsuid())) 2778 return true; 2779 2780 ns = current_user_ns(); 2781 if (vfsuid_has_mapping(ns, vfsuid) && ns_capable(ns, CAP_FOWNER)) 2782 return true; 2783 return false; 2784 } 2785 EXPORT_SYMBOL(inode_owner_or_capable); 2786 2787 /* 2788 * Direct i/o helper functions 2789 */ 2790 bool inode_dio_finished(const struct inode *inode) 2791 { 2792 return atomic_read(&inode->i_dio_count) == 0; 2793 } 2794 EXPORT_SYMBOL(inode_dio_finished); 2795 2796 /** 2797 * inode_dio_wait - wait for outstanding DIO requests to finish 2798 * @inode: inode to wait for 2799 * 2800 * Waits for all pending direct I/O requests to finish so that we can 2801 * proceed with a truncate or equivalent operation. 2802 * 2803 * Must be called under a lock that serializes taking new references 2804 * to i_dio_count, usually by inode->i_rwsem. 2805 */ 2806 void inode_dio_wait(struct inode *inode) 2807 { 2808 wait_var_event(&inode->i_dio_count, inode_dio_finished(inode)); 2809 } 2810 EXPORT_SYMBOL(inode_dio_wait); 2811 2812 void inode_dio_wait_interruptible(struct inode *inode) 2813 { 2814 wait_var_event_interruptible(&inode->i_dio_count, 2815 inode_dio_finished(inode)); 2816 } 2817 EXPORT_SYMBOL(inode_dio_wait_interruptible); 2818 2819 /* 2820 * inode_set_flags - atomically set some inode flags 2821 * 2822 * Note: the caller should be holding i_rwsem exclusively, or else be sure that 2823 * they have exclusive access to the inode structure (i.e., while the 2824 * inode is being instantiated). The reason for the cmpxchg() loop 2825 * --- which wouldn't be necessary if all code paths which modify 2826 * i_flags actually followed this rule, is that there is at least one 2827 * code path which doesn't today so we use cmpxchg() out of an abundance 2828 * of caution. 2829 * 2830 * In the long run, i_rwsem is overkill, and we should probably look 2831 * at using the i_lock spinlock to protect i_flags, and then make sure 2832 * it is so documented in include/linux/fs.h and that all code follows 2833 * the locking convention!! 2834 */ 2835 void inode_set_flags(struct inode *inode, unsigned int flags, 2836 unsigned int mask) 2837 { 2838 WARN_ON_ONCE(flags & ~mask); 2839 set_mask_bits(&inode->i_flags, mask, flags); 2840 } 2841 EXPORT_SYMBOL(inode_set_flags); 2842 2843 void inode_nohighmem(struct inode *inode) 2844 { 2845 mapping_set_gfp_mask(inode->i_mapping, GFP_USER); 2846 } 2847 EXPORT_SYMBOL(inode_nohighmem); 2848 2849 struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts) 2850 { 2851 trace_inode_set_ctime_to_ts(inode, &ts); 2852 set_normalized_timespec64(&ts, ts.tv_sec, ts.tv_nsec); 2853 WRITE_ONCE(inode->i_ctime_sec, ts.tv_sec); 2854 WRITE_ONCE(inode->i_ctime_nsec, ts.tv_nsec); 2855 return ts; 2856 } 2857 EXPORT_SYMBOL(inode_set_ctime_to_ts); 2858 2859 /** 2860 * timestamp_truncate - Truncate timespec to a granularity 2861 * @t: Timespec 2862 * @inode: inode being updated 2863 * 2864 * Truncate a timespec to the granularity supported by the fs 2865 * containing the inode. Always rounds down. gran must 2866 * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns). 2867 */ 2868 struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode) 2869 { 2870 struct super_block *sb = inode->i_sb; 2871 unsigned int gran = sb->s_time_gran; 2872 time64_t seconds = t.tv_sec; 2873 2874 t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max); 2875 if (unlikely(t.tv_sec != seconds)) 2876 t.tv_nsec = 0; 2877 2878 /* Avoid division in the common cases 1 ns and 1 s. */ 2879 if (gran == 1) 2880 ; /* nothing */ 2881 else if (gran == NSEC_PER_SEC) 2882 t.tv_nsec = 0; 2883 else if (gran > 1 && gran < NSEC_PER_SEC) 2884 t.tv_nsec -= t.tv_nsec % gran; 2885 else 2886 WARN(1, "invalid file time granularity: %u", gran); 2887 return t; 2888 } 2889 EXPORT_SYMBOL(timestamp_truncate); 2890 2891 /** 2892 * inode_set_ctime_current - set the ctime to current_time 2893 * @inode: inode 2894 * 2895 * Set the inode's ctime to the current value for the inode. Returns the 2896 * current value that was assigned. If this is not a multigrain inode, then we 2897 * set it to the later of the coarse time and floor value. 2898 * 2899 * If it is multigrain, then we first see if the coarse-grained timestamp is 2900 * distinct from what is already there. If so, then use that. Otherwise, get a 2901 * fine-grained timestamp. 2902 * 2903 * After that, try to swap the new value into i_ctime_nsec. Accept the 2904 * resulting ctime, regardless of the outcome of the swap. If it has 2905 * already been replaced, then that timestamp is later than the earlier 2906 * unacceptable one, and is thus acceptable. 2907 */ 2908 struct timespec64 inode_set_ctime_current(struct inode *inode) 2909 { 2910 struct timespec64 now; 2911 u32 cns, cur; 2912 2913 ktime_get_coarse_real_ts64_mg(&now); 2914 now = timestamp_truncate(now, inode); 2915 2916 /* Just return that if this is not a multigrain fs */ 2917 if (!is_mgtime(inode)) { 2918 inode_set_ctime_to_ts(inode, now); 2919 goto out; 2920 } 2921 2922 /* 2923 * A fine-grained time is only needed if someone has queried 2924 * for timestamps, and the current coarse grained time isn't 2925 * later than what's already there. 2926 */ 2927 cns = smp_load_acquire(&inode->i_ctime_nsec); 2928 if (cns & I_CTIME_QUERIED) { 2929 struct timespec64 ctime = { .tv_sec = inode_get_ctime_sec(inode), 2930 .tv_nsec = cns & ~I_CTIME_QUERIED }; 2931 2932 if (timespec64_compare(&now, &ctime) <= 0) { 2933 ktime_get_real_ts64_mg(&now); 2934 now = timestamp_truncate(now, inode); 2935 mgtime_counter_inc(mg_fine_stamps); 2936 } 2937 } 2938 mgtime_counter_inc(mg_ctime_updates); 2939 2940 /* No need to cmpxchg if it's exactly the same */ 2941 if (cns == now.tv_nsec && inode_get_ctime_sec(inode) == now.tv_sec) { 2942 trace_ctime_xchg_skip(inode, &now); 2943 goto out; 2944 } 2945 cur = cns; 2946 retry: 2947 /* Try to swap the nsec value into place. */ 2948 if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { 2949 /* If swap occurred, then we're (mostly) done */ 2950 WRITE_ONCE(inode->i_ctime_sec, now.tv_sec); 2951 trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); 2952 mgtime_counter_inc(mg_ctime_swaps); 2953 } else { 2954 /* 2955 * Was the change due to someone marking the old ctime QUERIED? 2956 * If so then retry the swap. This can only happen once since 2957 * the only way to clear I_CTIME_QUERIED is to stamp the inode 2958 * with a new ctime. 2959 */ 2960 if (!(cns & I_CTIME_QUERIED) && (cns | I_CTIME_QUERIED) == cur) { 2961 cns = cur; 2962 goto retry; 2963 } 2964 /* Otherwise, keep the existing ctime */ 2965 now.tv_sec = inode_get_ctime_sec(inode); 2966 now.tv_nsec = cur & ~I_CTIME_QUERIED; 2967 } 2968 out: 2969 return now; 2970 } 2971 EXPORT_SYMBOL(inode_set_ctime_current); 2972 2973 /** 2974 * inode_set_ctime_deleg - try to update the ctime on a delegated inode 2975 * @inode: inode to update 2976 * @update: timespec64 to set the ctime 2977 * 2978 * Attempt to atomically update the ctime on behalf of a delegation holder. 2979 * 2980 * The nfs server can call back the holder of a delegation to get updated 2981 * inode attributes, including the mtime. When updating the mtime, update 2982 * the ctime to a value at least equal to that. 2983 * 2984 * This can race with concurrent updates to the inode, in which 2985 * case the update is skipped. 2986 * 2987 * Note that this works even when multigrain timestamps are not enabled, 2988 * so it is used in either case. 2989 */ 2990 struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 update) 2991 { 2992 struct timespec64 now, cur_ts; 2993 u32 cur, old; 2994 2995 /* pairs with try_cmpxchg below */ 2996 cur = smp_load_acquire(&inode->i_ctime_nsec); 2997 cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; 2998 cur_ts.tv_sec = inode_get_ctime_sec(inode); 2999 3000 /* If the update is older than the existing value, skip it. */ 3001 if (timespec64_compare(&update, &cur_ts) <= 0) 3002 return cur_ts; 3003 3004 ktime_get_coarse_real_ts64_mg(&now); 3005 3006 /* Clamp the update to "now" if it's in the future */ 3007 if (timespec64_compare(&update, &now) > 0) 3008 update = now; 3009 3010 update = timestamp_truncate(update, inode); 3011 3012 /* No need to update if the values are already the same */ 3013 if (timespec64_equal(&update, &cur_ts)) 3014 return cur_ts; 3015 3016 /* 3017 * Try to swap the nsec value into place. If it fails, that means 3018 * it raced with an update due to a write or similar activity. That 3019 * stamp takes precedence, so just skip the update. 3020 */ 3021 retry: 3022 old = cur; 3023 if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { 3024 WRITE_ONCE(inode->i_ctime_sec, update.tv_sec); 3025 mgtime_counter_inc(mg_ctime_swaps); 3026 return update; 3027 } 3028 3029 /* 3030 * Was the change due to another task marking the old ctime QUERIED? 3031 * 3032 * If so, then retry the swap. This can only happen once since 3033 * the only way to clear I_CTIME_QUERIED is to stamp the inode 3034 * with a new ctime. 3035 */ 3036 if (!(old & I_CTIME_QUERIED) && (cur == (old | I_CTIME_QUERIED))) 3037 goto retry; 3038 3039 /* Otherwise, it was a new timestamp. */ 3040 cur_ts.tv_sec = inode_get_ctime_sec(inode); 3041 cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; 3042 return cur_ts; 3043 } 3044 EXPORT_SYMBOL(inode_set_ctime_deleg); 3045 3046 /** 3047 * in_group_or_capable - check whether caller is CAP_FSETID privileged 3048 * @idmap: idmap of the mount @inode was found from 3049 * @inode: inode to check 3050 * @vfsgid: the new/current vfsgid of @inode 3051 * 3052 * Check whether @vfsgid is in the caller's group list or if the caller is 3053 * privileged with CAP_FSETID over @inode. This can be used to determine 3054 * whether the setgid bit can be kept or must be dropped. 3055 * 3056 * Return: true if the caller is sufficiently privileged, false if not. 3057 */ 3058 bool in_group_or_capable(const struct mnt_idmap *idmap, 3059 const struct inode *inode, vfsgid_t vfsgid) 3060 { 3061 if (vfsgid_in_group_p(vfsgid)) 3062 return true; 3063 if (capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID)) 3064 return true; 3065 return false; 3066 } 3067 EXPORT_SYMBOL(in_group_or_capable); 3068 3069 /** 3070 * mode_strip_sgid - handle the sgid bit for non-directories 3071 * @idmap: idmap of the mount the inode was created from 3072 * @dir: parent directory inode 3073 * @mode: mode of the file to be created in @dir 3074 * 3075 * If the @mode of the new file has both the S_ISGID and S_IXGRP bit 3076 * raised and @dir has the S_ISGID bit raised ensure that the caller is 3077 * either in the group of the parent directory or they have CAP_FSETID 3078 * in their user namespace and are privileged over the parent directory. 3079 * In all other cases, strip the S_ISGID bit from @mode. 3080 * 3081 * Return: the new mode to use for the file 3082 */ 3083 umode_t mode_strip_sgid(const struct mnt_idmap *idmap, 3084 const struct inode *dir, umode_t mode) 3085 { 3086 if ((mode & (S_ISGID | S_IXGRP)) != (S_ISGID | S_IXGRP)) 3087 return mode; 3088 if (S_ISDIR(mode) || !dir || !(dir->i_mode & S_ISGID)) 3089 return mode; 3090 if (in_group_or_capable(idmap, dir, i_gid_into_vfsgid(idmap, dir))) 3091 return mode; 3092 return mode & ~S_ISGID; 3093 } 3094 EXPORT_SYMBOL(mode_strip_sgid); 3095 3096 #ifdef CONFIG_DEBUG_VFS 3097 /** 3098 * dump_inode - dump an inode. 3099 * @inode: inode to dump 3100 * @reason: reason for dumping 3101 * 3102 * If inode is an invalid pointer, we don't want to crash accessing it, 3103 * so probe everything depending on it carefully with get_kernel_nofault(). 3104 */ 3105 void dump_inode(struct inode *inode, const char *reason) 3106 { 3107 struct super_block *sb; 3108 struct file_system_type *s_type; 3109 const char *fs_name_ptr; 3110 char fs_name[32] = {}; 3111 umode_t mode; 3112 unsigned short opflags; 3113 unsigned int flags; 3114 unsigned int state; 3115 int count; 3116 3117 if (get_kernel_nofault(sb, &inode->i_sb) || 3118 get_kernel_nofault(mode, &inode->i_mode) || 3119 get_kernel_nofault(opflags, &inode->i_opflags) || 3120 get_kernel_nofault(flags, &inode->i_flags)) { 3121 pr_warn("%s: unreadable inode:%px\n", reason, inode); 3122 return; 3123 } 3124 3125 state = inode_state_read_once(inode); 3126 count = icount_read_once(inode); 3127 3128 if (!sb || 3129 get_kernel_nofault(s_type, &sb->s_type) || !s_type || 3130 get_kernel_nofault(fs_name_ptr, &s_type->name) || !fs_name_ptr || 3131 strncpy_from_kernel_nofault(fs_name, fs_name_ptr, sizeof(fs_name) - 1) < 0) 3132 strscpy(fs_name, "<unknown, sb unreadable>"); 3133 3134 pr_warn("%s: inode:%px fs:%s mode:%ho opflags:%#x flags:%#x state:%#x count:%d\n", 3135 reason, inode, fs_name, mode, opflags, flags, state, count); 3136 } 3137 EXPORT_SYMBOL(dump_inode); 3138 #endif 3139