Lines Matching refs:dentry
126 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, in in_lookup_hash()
302 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) in dentry_cmp() argument
320 const unsigned char *cs = READ_ONCE(dentry->d_name.name); in dentry_cmp()
337 static inline struct external_name *external_name(struct dentry *dentry) in external_name() argument
339 return container_of(dentry->d_name.name, struct external_name, name[0]); in external_name()
344 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free() local
346 kmem_cache_free(dentry_cache, dentry); in __d_free()
351 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free_external() local
352 kfree(external_name(dentry)); in __d_free_external()
353 kmem_cache_free(dentry_cache, dentry); in __d_free_external()
356 static inline int dname_external(const struct dentry *dentry) in dname_external() argument
358 return dentry->d_name.name != dentry->d_shortname.string; in dname_external()
361 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry) in take_dentry_name_snapshot() argument
368 seq = read_seqcount_begin(&dentry->d_seq); in take_dentry_name_snapshot()
369 s = READ_ONCE(dentry->d_name.name); in take_dentry_name_snapshot()
370 name->name.hash_len = dentry->d_name.hash_len; in take_dentry_name_snapshot()
372 if (likely(s == dentry->d_shortname.string)) { in take_dentry_name_snapshot()
373 name->inline_name = dentry->d_shortname; in take_dentry_name_snapshot()
382 if (read_seqcount_retry(&dentry->d_seq, seq)) { in take_dentry_name_snapshot()
401 static inline void __d_set_inode_and_type(struct dentry *dentry, in __d_set_inode_and_type() argument
407 dentry->d_inode = inode; in __d_set_inode_and_type()
408 flags = READ_ONCE(dentry->d_flags); in __d_set_inode_and_type()
411 smp_store_release(&dentry->d_flags, flags); in __d_set_inode_and_type()
414 static inline void __d_clear_type_and_inode(struct dentry *dentry) in __d_clear_type_and_inode() argument
416 unsigned flags = READ_ONCE(dentry->d_flags); in __d_clear_type_and_inode()
419 WRITE_ONCE(dentry->d_flags, flags); in __d_clear_type_and_inode()
420 dentry->d_inode = NULL; in __d_clear_type_and_inode()
429 static void dentry_free(struct dentry *dentry) in dentry_free() argument
431 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); in dentry_free()
432 if (unlikely(dname_external(dentry))) { in dentry_free()
433 struct external_name *p = external_name(dentry); in dentry_free()
435 call_rcu(&dentry->d_u.d_rcu, __d_free_external); in dentry_free()
440 if (dentry->d_flags & DCACHE_NORCU) in dentry_free()
441 __d_free(&dentry->d_u.d_rcu); in dentry_free()
443 call_rcu(&dentry->d_u.d_rcu, __d_free); in dentry_free()
450 static void dentry_unlink_inode(struct dentry * dentry) in dentry_unlink_inode() argument
451 __releases(dentry->d_lock) in dentry_unlink_inode()
452 __releases(dentry->d_inode->i_lock) in dentry_unlink_inode()
454 struct inode *inode = dentry->d_inode; in dentry_unlink_inode()
456 raw_write_seqcount_begin(&dentry->d_seq); in dentry_unlink_inode()
457 __d_clear_type_and_inode(dentry); in dentry_unlink_inode()
458 hlist_del_init(&dentry->d_u.d_alias); in dentry_unlink_inode()
459 raw_write_seqcount_end(&dentry->d_seq); in dentry_unlink_inode()
460 spin_unlock(&dentry->d_lock); in dentry_unlink_inode()
464 if (dentry->d_op && dentry->d_op->d_iput) in dentry_unlink_inode()
465 dentry->d_op->d_iput(dentry, inode); in dentry_unlink_inode()
489 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_… argument
490 static void d_lru_add(struct dentry *dentry) in d_lru_add() argument
492 D_FLAG_VERIFY(dentry, 0); in d_lru_add()
493 dentry->d_flags |= DCACHE_LRU_LIST; in d_lru_add()
495 if (d_is_negative(dentry)) in d_lru_add()
498 &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_add()
501 static void d_lru_del(struct dentry *dentry) in d_lru_del() argument
503 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_del()
504 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_del()
506 if (d_is_negative(dentry)) in d_lru_del()
509 &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_del()
512 static void d_shrink_del(struct dentry *dentry) in d_shrink_del() argument
514 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
515 list_del_init(&dentry->d_lru); in d_shrink_del()
516 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
520 static void d_shrink_add(struct dentry *dentry, struct list_head *list) in d_shrink_add() argument
522 D_FLAG_VERIFY(dentry, 0); in d_shrink_add()
523 list_add(&dentry->d_lru, list); in d_shrink_add()
524 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; in d_shrink_add()
534 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry) in d_lru_isolate() argument
536 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_isolate()
537 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_isolate()
539 if (d_is_negative(dentry)) in d_lru_isolate()
541 list_lru_isolate(lru, &dentry->d_lru); in d_lru_isolate()
544 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, in d_lru_shrink_move() argument
547 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_shrink_move()
548 dentry->d_flags |= DCACHE_SHRINK_LIST; in d_lru_shrink_move()
549 if (d_is_negative(dentry)) in d_lru_shrink_move()
551 list_lru_isolate_move(lru, &dentry->d_lru, list); in d_lru_shrink_move()
554 static void ___d_drop(struct dentry *dentry) in ___d_drop() argument
562 if (unlikely(IS_ROOT(dentry))) in ___d_drop()
563 b = &dentry->d_sb->s_roots; in ___d_drop()
565 b = d_hash(dentry->d_name.hash); in ___d_drop()
568 __hlist_bl_del(&dentry->d_hash); in ___d_drop()
572 void __d_drop(struct dentry *dentry) in __d_drop() argument
574 if (!d_unhashed(dentry)) { in __d_drop()
575 ___d_drop(dentry); in __d_drop()
576 dentry->d_hash.pprev = NULL; in __d_drop()
577 write_seqcount_invalidate(&dentry->d_seq); in __d_drop()
600 void d_drop(struct dentry *dentry) in d_drop() argument
602 spin_lock(&dentry->d_lock); in d_drop()
603 __d_drop(dentry); in d_drop()
604 spin_unlock(&dentry->d_lock); in d_drop()
608 static inline void dentry_unlist(struct dentry *dentry) in dentry_unlist() argument
610 struct dentry *next; in dentry_unlist()
615 dentry->d_flags |= DCACHE_DENTRY_KILLED; in dentry_unlist()
616 if (unlikely(hlist_unhashed(&dentry->d_sib))) in dentry_unlist()
618 __hlist_del(&dentry->d_sib); in dentry_unlist()
638 while (dentry->d_sib.next) { in dentry_unlist()
639 next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib); in dentry_unlist()
642 dentry->d_sib.next = next->d_sib.next; in dentry_unlist()
646 static struct dentry *__dentry_kill(struct dentry *dentry) in __dentry_kill() argument
648 struct dentry *parent = NULL; in __dentry_kill()
654 lockref_mark_dead(&dentry->d_lockref); in __dentry_kill()
660 if (dentry->d_flags & DCACHE_OP_PRUNE) in __dentry_kill()
661 dentry->d_op->d_prune(dentry); in __dentry_kill()
663 if (dentry->d_flags & DCACHE_LRU_LIST) { in __dentry_kill()
664 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) in __dentry_kill()
665 d_lru_del(dentry); in __dentry_kill()
668 __d_drop(dentry); in __dentry_kill()
669 if (dentry->d_inode) in __dentry_kill()
670 dentry_unlink_inode(dentry); in __dentry_kill()
672 spin_unlock(&dentry->d_lock); in __dentry_kill()
674 if (dentry->d_op && dentry->d_op->d_release) in __dentry_kill()
675 dentry->d_op->d_release(dentry); in __dentry_kill()
679 if (!IS_ROOT(dentry)) { in __dentry_kill()
680 parent = dentry->d_parent; in __dentry_kill()
683 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in __dentry_kill()
684 dentry_unlist(dentry); in __dentry_kill()
685 if (dentry->d_flags & DCACHE_SHRINK_LIST) in __dentry_kill()
687 spin_unlock(&dentry->d_lock); in __dentry_kill()
689 dentry_free(dentry); in __dentry_kill()
708 static bool lock_for_kill(struct dentry *dentry) in lock_for_kill() argument
710 struct inode *inode = dentry->d_inode; in lock_for_kill()
712 if (unlikely(dentry->d_lockref.count)) in lock_for_kill()
719 spin_unlock(&dentry->d_lock); in lock_for_kill()
721 spin_lock(&dentry->d_lock); in lock_for_kill()
722 if (likely(inode == dentry->d_inode)) in lock_for_kill()
725 inode = dentry->d_inode; in lock_for_kill()
727 if (likely(!dentry->d_lockref.count)) in lock_for_kill()
745 static inline bool retain_dentry(struct dentry *dentry, bool locked) in retain_dentry() argument
750 d_flags = READ_ONCE(dentry->d_flags); in retain_dentry()
753 if (unlikely(d_unhashed(dentry))) in retain_dentry()
763 if (!locked || dentry->d_op->d_delete(dentry)) in retain_dentry()
779 d_lru_add(dentry); in retain_dentry()
783 dentry->d_flags |= DCACHE_REFERENCED; in retain_dentry()
790 struct dentry *de; in d_mark_dontcache()
813 static inline bool fast_dput(struct dentry *dentry) in fast_dput() argument
820 ret = lockref_put_return(&dentry->d_lockref); in fast_dput()
828 spin_lock(&dentry->d_lock); in fast_dput()
829 if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) { in fast_dput()
830 spin_unlock(&dentry->d_lock); in fast_dput()
833 dentry->d_lockref.count--; in fast_dput()
849 if (retain_dentry(dentry, false)) in fast_dput()
857 spin_lock(&dentry->d_lock); in fast_dput()
866 if (dentry->d_lockref.count || retain_dentry(dentry, true)) { in fast_dput()
867 spin_unlock(&dentry->d_lock); in fast_dput()
873 static void finish_dput(struct dentry *dentry) in finish_dput() argument
874 __releases(dentry->d_lock) in finish_dput()
877 while (lock_for_kill(dentry)) { in finish_dput()
879 dentry = __dentry_kill(dentry); in finish_dput()
880 if (!dentry) in finish_dput()
882 if (retain_dentry(dentry, true)) { in finish_dput()
883 spin_unlock(&dentry->d_lock); in finish_dput()
889 spin_unlock(&dentry->d_lock); in finish_dput()
918 void dput(struct dentry *dentry) in dput() argument
920 if (!dentry) in dput()
924 if (likely(fast_dput(dentry))) { in dput()
928 finish_dput(dentry); in dput()
932 void d_make_discardable(struct dentry *dentry) in d_make_discardable() argument
934 spin_lock(&dentry->d_lock); in d_make_discardable()
935 WARN_ON(!(dentry->d_flags & DCACHE_PERSISTENT)); in d_make_discardable()
936 dentry->d_flags &= ~DCACHE_PERSISTENT; in d_make_discardable()
937 dentry->d_lockref.count--; in d_make_discardable()
939 finish_dput(dentry); in d_make_discardable()
943 static void to_shrink_list(struct dentry *dentry, struct list_head *list) in to_shrink_list() argument
944 __must_hold(&dentry->d_lock) in to_shrink_list()
946 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) { in to_shrink_list()
947 if (dentry->d_flags & DCACHE_LRU_LIST) in to_shrink_list()
948 d_lru_del(dentry); in to_shrink_list()
949 d_shrink_add(dentry, list); in to_shrink_list()
953 void dput_to_list(struct dentry *dentry, struct list_head *list) in dput_to_list() argument
956 if (likely(fast_dput(dentry))) { in dput_to_list()
961 to_shrink_list(dentry, list); in dput_to_list()
962 spin_unlock(&dentry->d_lock); in dput_to_list()
965 struct dentry *dget_parent(struct dentry *dentry) in dget_parent() argument
968 struct dentry *ret; in dget_parent()
976 seq = raw_seqcount_begin(&dentry->d_seq); in dget_parent()
977 ret = READ_ONCE(dentry->d_parent); in dget_parent()
981 if (!read_seqcount_retry(&dentry->d_seq, seq)) in dget_parent()
992 ret = dentry->d_parent; in dget_parent()
994 if (unlikely(ret != dentry->d_parent)) { in dget_parent()
1007 static struct dentry * __d_find_any_alias(struct inode *inode) in __d_find_any_alias()
1009 struct dentry *alias; in __d_find_any_alias()
1013 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias); in __d_find_any_alias()
1025 struct dentry *d_find_any_alias(struct inode *inode) in d_find_any_alias()
1027 struct dentry *de; in d_find_any_alias()
1036 static struct dentry *__d_find_alias(struct inode *inode) in __d_find_alias()
1038 struct dentry *alias; in __d_find_alias()
1069 struct dentry *d_find_alias(struct inode *inode) in d_find_alias()
1071 struct dentry *de = NULL; in d_find_alias()
1086 struct dentry *d_find_alias_rcu(struct inode *inode) in d_find_alias_rcu()
1089 struct dentry *de = NULL; in d_find_alias_rcu()
1096 de = hlist_entry(l->first, struct dentry, d_u.d_alias); in d_find_alias_rcu()
1107 void d_dispose_if_unused(struct dentry *dentry, struct list_head *dispose) in d_dispose_if_unused() argument
1109 spin_lock(&dentry->d_lock); in d_dispose_if_unused()
1110 if (!dentry->d_lockref.count) in d_dispose_if_unused()
1111 to_shrink_list(dentry, dispose); in d_dispose_if_unused()
1112 spin_unlock(&dentry->d_lock); in d_dispose_if_unused()
1123 struct dentry *dentry; in d_prune_aliases() local
1126 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) in d_prune_aliases()
1127 d_dispose_if_unused(dentry, &dispose); in d_prune_aliases()
1133 static inline void shrink_kill(struct dentry *victim) in shrink_kill()
1148 struct dentry *dentry; in shrink_dentry_list() local
1150 dentry = list_entry(list->prev, struct dentry, d_lru); in shrink_dentry_list()
1151 spin_lock(&dentry->d_lock); in shrink_dentry_list()
1153 if (!lock_for_kill(dentry)) { in shrink_dentry_list()
1156 d_shrink_del(dentry); in shrink_dentry_list()
1157 can_free = dentry->d_flags & DCACHE_DENTRY_KILLED; in shrink_dentry_list()
1158 spin_unlock(&dentry->d_lock); in shrink_dentry_list()
1160 dentry_free(dentry); in shrink_dentry_list()
1163 d_shrink_del(dentry); in shrink_dentry_list()
1164 shrink_kill(dentry); in shrink_dentry_list()
1173 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate() local
1181 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate()
1189 if (dentry->d_lockref.count) { in dentry_lru_isolate()
1190 d_lru_isolate(lru, dentry); in dentry_lru_isolate()
1191 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1195 if (dentry->d_flags & DCACHE_REFERENCED) { in dentry_lru_isolate()
1196 dentry->d_flags &= ~DCACHE_REFERENCED; in dentry_lru_isolate()
1197 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1221 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate()
1222 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1254 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate_shrink() local
1261 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate_shrink()
1264 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate_shrink()
1265 spin_unlock(&dentry->d_lock); in dentry_lru_isolate_shrink()
1312 static void d_walk(struct dentry *parent, void *data, in d_walk() argument
1313 enum d_walk_ret (*enter)(void *, struct dentry *)) in d_walk()
1315 struct dentry *this_parent, *dentry; in d_walk() local
1337 dentry = d_first_child(this_parent); in d_walk()
1339 hlist_for_each_entry_from(dentry, d_sib) { in d_walk()
1340 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) in d_walk()
1343 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_walk()
1345 ret = enter(data, dentry); in d_walk()
1350 spin_unlock(&dentry->d_lock); in d_walk()
1356 spin_unlock(&dentry->d_lock); in d_walk()
1360 if (!hlist_empty(&dentry->d_children)) { in d_walk()
1362 spin_release(&dentry->d_lock.dep_map, _RET_IP_); in d_walk()
1363 this_parent = dentry; in d_walk()
1367 spin_unlock(&dentry->d_lock); in d_walk()
1375 dentry = this_parent; in d_walk()
1376 this_parent = dentry->d_parent; in d_walk()
1378 spin_unlock(&dentry->d_lock); in d_walk()
1385 hlist_for_each_entry_continue(dentry, d_sib) { in d_walk()
1386 if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) { in d_walk()
1418 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry) in path_check_mount() argument
1421 struct path path = { .mnt = info->mnt, .dentry = dentry }; in path_check_mount()
1423 if (likely(!d_mountpoint(dentry))) in path_check_mount()
1445 d_walk(parent->dentry, &data, path_check_mount); in path_has_submounts()
1459 int d_set_mounted(struct dentry *dentry) in d_set_mounted() argument
1461 struct dentry *p; in d_set_mounted()
1464 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { in d_set_mounted()
1473 spin_lock(&dentry->d_lock); in d_set_mounted()
1474 if (!d_unlinked(dentry)) { in d_set_mounted()
1476 if (!d_mountpoint(dentry)) { in d_set_mounted()
1477 dentry->d_flags |= DCACHE_MOUNTED; in d_set_mounted()
1481 spin_unlock(&dentry->d_lock); in d_set_mounted()
1503 struct dentry *start;
1506 struct dentry *victim;
1511 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) in select_collect() argument
1516 if (data->start == dentry) in select_collect()
1519 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in select_collect()
1521 } else if (!dentry->d_lockref.count) { in select_collect()
1522 to_shrink_list(dentry, &data->dispose); in select_collect()
1524 } else if (dentry->d_lockref.count < 0) { in select_collect()
1538 static enum d_walk_ret select_collect_umount(void *_data, struct dentry *dentry) in select_collect_umount() argument
1540 if (dentry->d_flags & DCACHE_PERSISTENT) { in select_collect_umount()
1541 dentry->d_flags &= ~DCACHE_PERSISTENT; in select_collect_umount()
1542 dentry->d_lockref.count--; in select_collect_umount()
1544 return select_collect(_data, dentry); in select_collect_umount()
1547 static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry) in select_collect2() argument
1552 if (data->start == dentry) in select_collect2()
1555 if (!dentry->d_lockref.count) { in select_collect2()
1556 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in select_collect2()
1558 data->victim = dentry; in select_collect2()
1561 to_shrink_list(dentry, &data->dispose); in select_collect2()
1581 static void shrink_dcache_tree(struct dentry *parent, bool for_umount) in shrink_dcache_tree()
1614 void shrink_dcache_parent(struct dentry *parent) in shrink_dcache_parent()
1620 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) in umount_check() argument
1623 if (!hlist_empty(&dentry->d_children)) in umount_check()
1627 if (dentry == _data && dentry->d_lockref.count == 1) in umount_check()
1632 dentry, in umount_check()
1633 dentry->d_inode ? in umount_check()
1634 dentry->d_inode->i_ino : 0UL, in umount_check()
1635 dentry, in umount_check()
1636 dentry->d_lockref.count, in umount_check()
1637 dentry->d_sb->s_type->name, in umount_check()
1638 dentry->d_sb->s_id); in umount_check()
1642 static void do_one_tree(struct dentry *dentry) in do_one_tree() argument
1644 shrink_dcache_tree(dentry, true); in do_one_tree()
1645 d_walk(dentry, dentry, umount_check); in do_one_tree()
1646 d_drop(dentry); in do_one_tree()
1647 dput(dentry); in do_one_tree()
1655 struct dentry *dentry; in shrink_dcache_for_umount() local
1659 dentry = sb->s_root; in shrink_dcache_for_umount()
1661 do_one_tree(dentry); in shrink_dcache_for_umount()
1664 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash)); in shrink_dcache_for_umount()
1665 do_one_tree(dentry); in shrink_dcache_for_umount()
1669 static enum d_walk_ret find_submount(void *_data, struct dentry *dentry) in find_submount() argument
1671 struct dentry **victim = _data; in find_submount()
1672 if (d_mountpoint(dentry)) { in find_submount()
1673 *victim = dget_dlock(dentry); in find_submount()
1683 void d_invalidate(struct dentry *dentry) in d_invalidate() argument
1686 spin_lock(&dentry->d_lock); in d_invalidate()
1687 if (d_unhashed(dentry)) { in d_invalidate()
1688 spin_unlock(&dentry->d_lock); in d_invalidate()
1691 __d_drop(dentry); in d_invalidate()
1692 spin_unlock(&dentry->d_lock); in d_invalidate()
1695 if (!dentry->d_inode) in d_invalidate()
1698 shrink_dcache_parent(dentry); in d_invalidate()
1700 struct dentry *victim = NULL; in d_invalidate()
1701 d_walk(dentry, &victim, find_submount); in d_invalidate()
1704 shrink_dcache_parent(dentry); in d_invalidate()
1724 static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) in __d_alloc()
1726 struct dentry *dentry; in __d_alloc() local
1730 dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru, in __d_alloc()
1732 if (!dentry) in __d_alloc()
1741 dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0; in __d_alloc()
1744 dname = dentry->d_shortname.string; in __d_alloc()
1751 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1757 dname = dentry->d_shortname.string; in __d_alloc()
1760 dentry->__d_name.len = name->len; in __d_alloc()
1761 dentry->__d_name.hash = name->hash; in __d_alloc()
1766 smp_store_release(&dentry->__d_name.name, dname); /* ^^^ */ in __d_alloc()
1768 dentry->d_flags = 0; in __d_alloc()
1769 lockref_init(&dentry->d_lockref); in __d_alloc()
1770 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock); in __d_alloc()
1771 dentry->d_inode = NULL; in __d_alloc()
1772 dentry->d_parent = dentry; in __d_alloc()
1773 dentry->d_sb = sb; in __d_alloc()
1774 dentry->d_op = sb->__s_d_op; in __d_alloc()
1775 dentry->d_flags = sb->s_d_flags; in __d_alloc()
1776 dentry->d_fsdata = NULL; in __d_alloc()
1777 INIT_HLIST_BL_NODE(&dentry->d_hash); in __d_alloc()
1778 INIT_LIST_HEAD(&dentry->d_lru); in __d_alloc()
1779 INIT_HLIST_HEAD(&dentry->d_children); in __d_alloc()
1780 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_alloc()
1781 INIT_HLIST_NODE(&dentry->d_sib); in __d_alloc()
1783 if (dentry->d_op && dentry->d_op->d_init) { in __d_alloc()
1784 err = dentry->d_op->d_init(dentry); in __d_alloc()
1786 if (dname_external(dentry)) in __d_alloc()
1787 kfree(external_name(dentry)); in __d_alloc()
1788 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1795 return dentry; in __d_alloc()
1807 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) in d_alloc()
1809 struct dentry *dentry = __d_alloc(parent->d_sb, name); in d_alloc() local
1810 if (!dentry) in d_alloc()
1817 dentry->d_parent = dget_dlock(parent); in d_alloc()
1818 hlist_add_head(&dentry->d_sib, &parent->d_children); in d_alloc()
1821 return dentry; in d_alloc()
1825 struct dentry *d_alloc_anon(struct super_block *sb) in d_alloc_anon()
1831 struct dentry *d_alloc_cursor(struct dentry * parent) in d_alloc_cursor()
1833 struct dentry *dentry = d_alloc_anon(parent->d_sb); in d_alloc_cursor() local
1834 if (dentry) { in d_alloc_cursor()
1835 dentry->d_flags |= DCACHE_DENTRY_CURSOR; in d_alloc_cursor()
1836 dentry->d_parent = dget(parent); in d_alloc_cursor()
1838 return dentry; in d_alloc_cursor()
1856 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) in d_alloc_pseudo()
1861 struct dentry *dentry = __d_alloc(sb, name); in d_alloc_pseudo() local
1862 if (likely(dentry)) { in d_alloc_pseudo()
1863 dentry->d_flags |= DCACHE_NORCU; in d_alloc_pseudo()
1865 if (!dentry->d_op) in d_alloc_pseudo()
1866 dentry->d_op = &anon_ops; in d_alloc_pseudo()
1868 return dentry; in d_alloc_pseudo()
1871 struct dentry *d_alloc_name(struct dentry *parent, const char *name) in d_alloc_name()
1908 static void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) in d_set_d_op() argument
1911 WARN_ON_ONCE(dentry->d_op); in d_set_d_op()
1912 WARN_ON_ONCE(dentry->d_flags & DCACHE_OP_FLAGS); in d_set_d_op()
1913 dentry->d_op = op; in d_set_d_op()
1915 dentry->d_flags |= flags; in d_set_d_op()
1961 static void __d_instantiate(struct dentry *dentry, struct inode *inode) in __d_instantiate() argument
1964 WARN_ON(d_in_lookup(dentry)); in __d_instantiate()
1970 if ((dentry->d_flags & in __d_instantiate()
1973 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_instantiate()
1974 raw_write_seqcount_begin(&dentry->d_seq); in __d_instantiate()
1975 __d_set_inode_and_type(dentry, inode, add_flags); in __d_instantiate()
1976 raw_write_seqcount_end(&dentry->d_seq); in __d_instantiate()
1977 fsnotify_update_flags(dentry); in __d_instantiate()
1995 void d_instantiate(struct dentry *entry, struct inode * inode) in d_instantiate()
2015 void d_instantiate_new(struct dentry *entry, struct inode *inode) in d_instantiate_new()
2032 struct dentry *d_make_root(struct inode *root_inode) in d_make_root()
2034 struct dentry *res = NULL; in d_make_root()
2047 static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected) in __d_obtain_alias()
2050 struct dentry *new, *res; in __d_obtain_alias()
2118 struct dentry *d_obtain_alias(struct inode *inode) in d_obtain_alias()
2139 struct dentry *d_obtain_root(struct inode *inode) in d_obtain_root()
2161 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, in d_add_ci() argument
2164 struct dentry *found, *res; in d_add_ci()
2170 found = d_hash_and_lookup(dentry->d_parent, name); in d_add_ci()
2175 if (d_in_lookup(dentry)) { in d_add_ci()
2176 found = d_alloc_parallel(dentry->d_parent, name, in d_add_ci()
2177 dentry->d_wait); in d_add_ci()
2183 found = d_alloc(dentry->d_parent, name); in d_add_ci()
2207 bool d_same_name(const struct dentry *dentry, const struct dentry *parent, in d_same_name() argument
2211 if (dentry->d_name.len != name->len) in d_same_name()
2213 return dentry_cmp(dentry, name->name, name->len) == 0; in d_same_name()
2215 return parent->d_op->d_compare(dentry, in d_same_name()
2216 dentry->d_name.len, dentry->d_name.name, in d_same_name()
2225 static noinline struct dentry *__d_lookup_rcu_op_compare( in __d_lookup_rcu_op_compare()
2226 const struct dentry *parent, in __d_lookup_rcu_op_compare()
2233 struct dentry *dentry; in __d_lookup_rcu_op_compare() local
2235 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup_rcu_op_compare()
2241 seq = raw_seqcount_begin(&dentry->d_seq); in __d_lookup_rcu_op_compare()
2242 if (dentry->d_parent != parent) in __d_lookup_rcu_op_compare()
2244 if (d_unhashed(dentry)) in __d_lookup_rcu_op_compare()
2246 if (dentry->d_name.hash != hashlen_hash(hashlen)) in __d_lookup_rcu_op_compare()
2248 tlen = dentry->d_name.len; in __d_lookup_rcu_op_compare()
2249 tname = dentry->d_name.name; in __d_lookup_rcu_op_compare()
2251 if (read_seqcount_retry(&dentry->d_seq, seq)) { in __d_lookup_rcu_op_compare()
2255 if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0) in __d_lookup_rcu_op_compare()
2258 return dentry; in __d_lookup_rcu_op_compare()
2289 struct dentry *__d_lookup_rcu(const struct dentry *parent, in __d_lookup_rcu()
2297 struct dentry *dentry; in __d_lookup_rcu() local
2322 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup_rcu()
2342 seq = raw_seqcount_begin(&dentry->d_seq); in __d_lookup_rcu()
2343 if (dentry->d_parent != parent) in __d_lookup_rcu()
2345 if (dentry->d_name.hash_len != hashlen) in __d_lookup_rcu()
2347 if (unlikely(dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)) in __d_lookup_rcu()
2358 if (unlikely(d_unhashed(dentry))) in __d_lookup_rcu()
2361 return dentry; in __d_lookup_rcu()
2377 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) in d_lookup()
2379 struct dentry *dentry; in d_lookup() local
2384 dentry = __d_lookup(parent, name); in d_lookup()
2385 if (dentry) in d_lookup()
2388 return dentry; in d_lookup()
2407 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) in __d_lookup()
2412 struct dentry *found = NULL; in __d_lookup()
2413 struct dentry *dentry; in __d_lookup() local
2437 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup()
2439 if (dentry->d_name.hash != hash) in __d_lookup()
2442 spin_lock(&dentry->d_lock); in __d_lookup()
2443 if (dentry->d_parent != parent) in __d_lookup()
2445 if (d_unhashed(dentry)) in __d_lookup()
2448 if (!d_same_name(dentry, parent, name)) in __d_lookup()
2451 dentry->d_lockref.count++; in __d_lookup()
2452 found = dentry; in __d_lookup()
2453 spin_unlock(&dentry->d_lock); in __d_lookup()
2456 spin_unlock(&dentry->d_lock); in __d_lookup()
2470 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) in d_hash_and_lookup()
2507 void d_delete(struct dentry * dentry) in d_delete() argument
2509 struct inode *inode = dentry->d_inode; in d_delete()
2512 spin_lock(&dentry->d_lock); in d_delete()
2516 if (dentry->d_lockref.count == 1) { in d_delete()
2518 __d_drop(dentry); in d_delete()
2519 dentry->d_flags &= ~DCACHE_CANT_MOUNT; in d_delete()
2520 dentry_unlink_inode(dentry); in d_delete()
2522 __d_drop(dentry); in d_delete()
2523 spin_unlock(&dentry->d_lock); in d_delete()
2529 static void __d_rehash(struct dentry *entry) in __d_rehash()
2545 void d_rehash(struct dentry * entry) in d_rehash()
2573 static void d_wait_lookup(struct dentry *dentry) in d_wait_lookup() argument
2575 if (d_in_lookup(dentry)) { in d_wait_lookup()
2577 add_wait_queue(dentry->d_wait, &wait); in d_wait_lookup()
2580 spin_unlock(&dentry->d_lock); in d_wait_lookup()
2582 spin_lock(&dentry->d_lock); in d_wait_lookup()
2583 } while (d_in_lookup(dentry)); in d_wait_lookup()
2587 struct dentry *d_alloc_parallel(struct dentry *parent, in d_alloc_parallel()
2594 struct dentry *new = __d_alloc(parent->d_sb, name); in d_alloc_parallel()
2595 struct dentry *dentry; in d_alloc_parallel() local
2613 dentry = __d_lookup_rcu(parent, name, &d_seq); in d_alloc_parallel()
2614 if (unlikely(dentry)) { in d_alloc_parallel()
2615 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2619 if (read_seqcount_retry(&dentry->d_seq, d_seq)) { in d_alloc_parallel()
2621 dput(dentry); in d_alloc_parallel()
2626 return dentry; in d_alloc_parallel()
2651 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) { in d_alloc_parallel()
2652 if (dentry->d_name.hash != hash) in d_alloc_parallel()
2654 if (dentry->d_parent != parent) in d_alloc_parallel()
2656 if (!d_same_name(dentry, parent, name)) in d_alloc_parallel()
2660 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2670 spin_lock(&dentry->d_lock); in d_alloc_parallel()
2671 d_wait_lookup(dentry); in d_alloc_parallel()
2678 if (unlikely(dentry->d_name.hash != hash)) in d_alloc_parallel()
2680 if (unlikely(dentry->d_parent != parent)) in d_alloc_parallel()
2682 if (unlikely(d_unhashed(dentry))) in d_alloc_parallel()
2684 if (unlikely(!d_same_name(dentry, parent, name))) in d_alloc_parallel()
2687 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2689 return dentry; in d_alloc_parallel()
2697 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2698 dput(dentry); in d_alloc_parallel()
2708 static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry) in __d_lookup_unhash() argument
2713 lockdep_assert_held(&dentry->d_lock); in __d_lookup_unhash()
2715 b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash); in __d_lookup_unhash()
2717 dentry->d_flags &= ~DCACHE_PAR_LOOKUP; in __d_lookup_unhash()
2718 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash); in __d_lookup_unhash()
2719 d_wait = dentry->d_wait; in __d_lookup_unhash()
2720 dentry->d_wait = NULL; in __d_lookup_unhash()
2722 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_lookup_unhash()
2723 INIT_LIST_HEAD(&dentry->d_lru); in __d_lookup_unhash()
2727 void __d_lookup_unhash_wake(struct dentry *dentry) in __d_lookup_unhash_wake() argument
2729 spin_lock(&dentry->d_lock); in __d_lookup_unhash_wake()
2730 wake_up_all(__d_lookup_unhash(dentry)); in __d_lookup_unhash_wake()
2731 spin_unlock(&dentry->d_lock); in __d_lookup_unhash_wake()
2737 static inline void __d_add(struct dentry *dentry, struct inode *inode, in __d_add() argument
2743 spin_lock(&dentry->d_lock); in __d_add()
2744 if (unlikely(d_in_lookup(dentry))) { in __d_add()
2745 dir = dentry->d_parent->d_inode; in __d_add()
2747 d_wait = __d_lookup_unhash(dentry); in __d_add()
2750 d_set_d_op(dentry, ops); in __d_add()
2753 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_add()
2754 raw_write_seqcount_begin(&dentry->d_seq); in __d_add()
2755 __d_set_inode_and_type(dentry, inode, add_flags); in __d_add()
2756 raw_write_seqcount_end(&dentry->d_seq); in __d_add()
2757 fsnotify_update_flags(dentry); in __d_add()
2759 __d_rehash(dentry); in __d_add()
2762 spin_unlock(&dentry->d_lock); in __d_add()
2776 void d_add(struct dentry *entry, struct inode *inode) in d_add()
2786 struct dentry *d_make_persistent(struct dentry *dentry, struct inode *inode) in d_make_persistent() argument
2788 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); in d_make_persistent()
2790 security_d_instantiate(dentry, inode); in d_make_persistent()
2792 spin_lock(&dentry->d_lock); in d_make_persistent()
2793 __d_instantiate(dentry, inode); in d_make_persistent()
2794 dentry->d_flags |= DCACHE_PERSISTENT; in d_make_persistent()
2795 dget_dlock(dentry); in d_make_persistent()
2796 if (d_unhashed(dentry)) in d_make_persistent()
2797 __d_rehash(dentry); in d_make_persistent()
2798 spin_unlock(&dentry->d_lock); in d_make_persistent()
2800 return dentry; in d_make_persistent()
2804 static void swap_names(struct dentry *dentry, struct dentry *target) in swap_names() argument
2807 if (unlikely(dname_external(dentry))) { in swap_names()
2811 swap(target->__d_name.name, dentry->__d_name.name); in swap_names()
2817 dentry->__d_name.name = target->__d_name.name; in swap_names()
2818 target->d_shortname = dentry->d_shortname; in swap_names()
2822 if (unlikely(dname_external(dentry))) { in swap_names()
2827 target->__d_name.name = dentry->__d_name.name; in swap_names()
2828 dentry->d_shortname = target->d_shortname; in swap_names()
2829 dentry->__d_name.name = dentry->d_shortname.string; in swap_names()
2835 swap(dentry->d_shortname.words[i], in swap_names()
2839 swap(dentry->__d_name.hash_len, target->__d_name.hash_len); in swap_names()
2842 static void copy_name(struct dentry *dentry, struct dentry *target) in copy_name() argument
2845 if (unlikely(dname_external(dentry))) in copy_name()
2846 old_name = external_name(dentry); in copy_name()
2849 dentry->__d_name = target->__d_name; in copy_name()
2851 dentry->d_shortname = target->d_shortname; in copy_name()
2852 dentry->__d_name.name = dentry->d_shortname.string; in copy_name()
2853 dentry->__d_name.hash_len = target->__d_name.hash_len; in copy_name()
2870 static void __d_move(struct dentry *dentry, struct dentry *target, in __d_move() argument
2873 struct dentry *old_parent, *p; in __d_move()
2878 WARN_ON(!dentry->d_inode); in __d_move()
2879 if (WARN_ON(dentry == target)) in __d_move()
2882 BUG_ON(d_ancestor(target, dentry)); in __d_move()
2883 old_parent = dentry->d_parent; in __d_move()
2885 if (IS_ROOT(dentry)) { in __d_move()
2893 BUG_ON(p == dentry); in __d_move()
2899 spin_lock_nested(&dentry->d_lock, 2); in __d_move()
2908 write_seqcount_begin(&dentry->d_seq); in __d_move()
2912 if (!d_unhashed(dentry)) in __d_move()
2913 ___d_drop(dentry); in __d_move()
2918 dentry->d_parent = target->d_parent; in __d_move()
2920 copy_name(dentry, target); in __d_move()
2922 dentry->d_parent->d_lockref.count++; in __d_move()
2923 if (dentry != old_parent) /* wasn't IS_ROOT */ in __d_move()
2927 swap_names(dentry, target); in __d_move()
2934 if (!hlist_unhashed(&dentry->d_sib)) in __d_move()
2935 __hlist_del(&dentry->d_sib); in __d_move()
2936 hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children); in __d_move()
2937 __d_rehash(dentry); in __d_move()
2938 fsnotify_update_flags(dentry); in __d_move()
2939 fscrypt_handle_d_move(dentry); in __d_move()
2942 write_seqcount_end(&dentry->d_seq); in __d_move()
2947 if (dentry->d_parent != old_parent) in __d_move()
2948 spin_unlock(&dentry->d_parent->d_lock); in __d_move()
2949 if (dentry != old_parent) in __d_move()
2952 spin_unlock(&dentry->d_lock); in __d_move()
2964 void d_move(struct dentry *dentry, struct dentry *target) in d_move() argument
2967 __d_move(dentry, target, false); in d_move()
2977 void d_exchange(struct dentry *dentry1, struct dentry *dentry2) in d_exchange()
3000 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) in d_ancestor()
3002 struct dentry *p; in d_ancestor()
3020 static int __d_unalias(struct dentry *dentry, struct dentry *alias) in __d_unalias() argument
3027 if (alias->d_parent == dentry->d_parent) in __d_unalias()
3031 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) in __d_unalias()
3033 m1 = &dentry->d_sb->s_vfs_rename_mutex; in __d_unalias()
3041 __d_move(alias, dentry, false); in __d_unalias()
3053 struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry, in d_splice_alias_ops() argument
3059 BUG_ON(!d_unhashed(dentry)); in d_splice_alias_ops()
3064 security_d_instantiate(dentry, inode); in d_splice_alias_ops()
3067 struct dentry *new = __d_find_any_alias(inode); in d_splice_alias_ops()
3072 if (unlikely(d_ancestor(new, dentry))) { in d_splice_alias_ops()
3079 dentry->d_name.name, in d_splice_alias_ops()
3083 struct dentry *old_parent = dget(new->d_parent); in d_splice_alias_ops()
3084 int err = __d_unalias(dentry, new); in d_splice_alias_ops()
3092 __d_move(new, dentry, false); in d_splice_alias_ops()
3100 __d_add(dentry, inode, ops); in d_splice_alias_ops()
3127 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) in d_splice_alias() argument
3129 return d_splice_alias_ops(inode, dentry, NULL); in d_splice_alias()
3149 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) in is_subdir()
3175 struct dentry *dentry = file->f_path.dentry; in d_mark_tmpfile() local
3177 BUG_ON(dname_external(dentry) || in d_mark_tmpfile()
3178 !hlist_unhashed(&dentry->d_u.d_alias) || in d_mark_tmpfile()
3179 !d_unlinked(dentry)); in d_mark_tmpfile()
3180 spin_lock(&dentry->d_parent->d_lock); in d_mark_tmpfile()
3181 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_mark_tmpfile()
3182 dentry->__d_name.len = sprintf(dentry->d_shortname.string, "#%llu", in d_mark_tmpfile()
3184 spin_unlock(&dentry->d_lock); in d_mark_tmpfile()
3185 spin_unlock(&dentry->d_parent->d_lock); in d_mark_tmpfile()
3191 struct dentry *dentry = file->f_path.dentry; in d_tmpfile() local
3195 d_instantiate(dentry, inode); in d_tmpfile()
3202 ino_t d_parent_ino(struct dentry *dentry) in d_parent_ino() argument
3204 struct dentry *parent; in d_parent_ino()
3210 seq = raw_seqcount_begin(&dentry->d_seq); in d_parent_ino()
3211 parent = READ_ONCE(dentry->d_parent); in d_parent_ino()
3215 if (!read_seqcount_retry(&dentry->d_seq, seq)) in d_parent_ino()
3220 spin_lock(&dentry->d_lock); in d_parent_ino()
3221 ret = dentry->d_parent->d_inode->i_ino; in d_parent_ino()
3222 spin_unlock(&dentry->d_lock); in d_parent_ino()
3268 __dentry_cache = KMEM_CACHE_USERCOPY(dentry, in dcache_init()