123 	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);  in nfsd_file_slab_free()  local
125 	put_cred(nf->nf_cred);  in nfsd_file_slab_free()
126 	kmem_cache_free(nfsd_file_slab, nf);  in nfsd_file_slab_free()
214 	struct nfsd_file *nf;  in nfsd_file_alloc()  local
216 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);  in nfsd_file_alloc()
217 	if (unlikely(!nf))  in nfsd_file_alloc()
221 	INIT_LIST_HEAD(&nf->nf_lru);  in nfsd_file_alloc()
222 	INIT_LIST_HEAD(&nf->nf_gc);  in nfsd_file_alloc()
223 	nf->nf_birthtime = ktime_get();  in nfsd_file_alloc()
224 	nf->nf_file = NULL;  in nfsd_file_alloc()
225 	nf->nf_cred = get_current_cred();  in nfsd_file_alloc()
226 	nf->nf_net = net;  in nfsd_file_alloc()
227 	nf->nf_flags = want_gc ?  in nfsd_file_alloc()
230 	nf->nf_inode = inode;  in nfsd_file_alloc()
231 	refcount_set(&nf->nf_ref, 1);  in nfsd_file_alloc()
232 	nf->nf_may = need;  in nfsd_file_alloc()
233 	nf->nf_mark = NULL;  in nfsd_file_alloc()
234 	nf->nf_dio_mem_align = 0;  in nfsd_file_alloc()
235 	nf->nf_dio_offset_align = 0;  in nfsd_file_alloc()
236 	nf->nf_dio_read_offset_align = 0;  in nfsd_file_alloc()
237 	return nf;  in nfsd_file_alloc()
248 nfsd_file_check_write_error(struct nfsd_file *nf)  in nfsd_file_check_write_error()  argument
250 	struct file *file = nf->nf_file;  in nfsd_file_check_write_error()
254 		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));  in nfsd_file_check_write_error()
258 nfsd_file_hash_remove(struct nfsd_file *nf)  in nfsd_file_hash_remove()  argument
260 	trace_nfsd_file_unhash(nf);  in nfsd_file_hash_remove()
261 	rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,  in nfsd_file_hash_remove()
266 nfsd_file_unhash(struct nfsd_file *nf)  in nfsd_file_unhash()  argument
268 	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {  in nfsd_file_unhash()
269 		nfsd_file_hash_remove(nf);  in nfsd_file_unhash()
276 nfsd_file_free(struct nfsd_file *nf)  in nfsd_file_free()  argument
278 	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));  in nfsd_file_free()
280 	trace_nfsd_file_free(nf);  in nfsd_file_free()
285 	nfsd_file_unhash(nf);  in nfsd_file_free()
286 	if (nf->nf_mark)  in nfsd_file_free()
287 		nfsd_file_mark_put(nf->nf_mark);  in nfsd_file_free()
288 	if (nf->nf_file) {  in nfsd_file_free()
289 		nfsd_file_check_write_error(nf);  in nfsd_file_free()
290 		nfsd_filp_close(nf->nf_file);  in nfsd_file_free()
297 	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))  in nfsd_file_free()
300 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);  in nfsd_file_free()
304 nfsd_file_check_writeback(struct nfsd_file *nf)  in nfsd_file_check_writeback()  argument
306 	struct file *file = nf->nf_file;  in nfsd_file_check_writeback()
325 static void nfsd_file_lru_add(struct nfsd_file *nf)  in nfsd_file_lru_add()  argument
327 	refcount_inc(&nf->nf_ref);  in nfsd_file_lru_add()
328 	if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru))  in nfsd_file_lru_add()
329 		trace_nfsd_file_lru_add(nf);  in nfsd_file_lru_add()
335 static bool nfsd_file_lru_remove(struct nfsd_file *nf)  in nfsd_file_lru_remove()  argument
337 	if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) {  in nfsd_file_lru_remove()
338 		trace_nfsd_file_lru_del(nf);  in nfsd_file_lru_remove()
345 nfsd_file_get(struct nfsd_file *nf)  in nfsd_file_get()  argument
347 	if (nf && refcount_inc_not_zero(&nf->nf_ref))  in nfsd_file_get()
348 		return nf;  in nfsd_file_get()
361 nfsd_file_put(struct nfsd_file *nf)  in nfsd_file_put()  argument
364 	trace_nfsd_file_put(nf);  in nfsd_file_put()
366 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&  in nfsd_file_put()
367 	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {  in nfsd_file_put()
368 		set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);  in nfsd_file_put()
369 		set_bit(NFSD_FILE_RECENT, &nf->nf_flags);  in nfsd_file_put()
372 	if (refcount_dec_and_test(&nf->nf_ref))  in nfsd_file_put()
373 		nfsd_file_free(nf);  in nfsd_file_put()
386 	struct nfsd_file *nf;  in nfsd_file_put_local()  local
389 	nf = unrcu_pointer(xchg(pnf, NULL));  in nfsd_file_put_local()
390 	if (nf) {  in nfsd_file_put_local()
391 		net = nf->nf_net;  in nfsd_file_put_local()
392 		nfsd_file_put(nf);  in nfsd_file_put_local()
404 nfsd_file_file(struct nfsd_file *nf)  in nfsd_file_file()  argument
406 	return nf->nf_file;  in nfsd_file_file()
412 	struct nfsd_file *nf;  in nfsd_file_dispose_list()  local
415 		nf = list_first_entry(dispose, struct nfsd_file, nf_gc);  in nfsd_file_dispose_list()
416 		list_del_init(&nf->nf_gc);  in nfsd_file_dispose_list()
417 		nfsd_file_free(nf);  in nfsd_file_dispose_list()
432 		struct nfsd_file *nf = list_first_entry(dispose,  in nfsd_file_dispose_list_delayed()  local
434 		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);  in nfsd_file_dispose_list_delayed()
439 		list_move_tail(&nf->nf_gc, &l->freeme);  in nfsd_file_dispose_list_delayed()
499 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);  in nfsd_file_lru_cb()  local
502 	WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));  in nfsd_file_lru_cb()
508 	if (nfsd_file_check_writeback(nf)) {  in nfsd_file_lru_cb()
509 		trace_nfsd_file_gc_writeback(nf);  in nfsd_file_lru_cb()
514 	if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {  in nfsd_file_lru_cb()
515 		trace_nfsd_file_gc_referenced(nf);  in nfsd_file_lru_cb()
523 	if (!refcount_dec_if_one(&nf->nf_ref)) {  in nfsd_file_lru_cb()
524 		trace_nfsd_file_gc_in_use(nf);  in nfsd_file_lru_cb()
529 	nfsd_file_unhash(nf);  in nfsd_file_lru_cb()
530 	list_lru_isolate(lru, &nf->nf_lru);  in nfsd_file_lru_cb()
531 	list_add(&nf->nf_gc, head);  in nfsd_file_lru_cb()
533 	trace_nfsd_file_gc_disposed(nf);  in nfsd_file_lru_cb()
541 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);  in nfsd_file_gc_cb()  local
543 	if (test_and_clear_bit(NFSD_FILE_RECENT, &nf->nf_flags)) {  in nfsd_file_gc_cb()
548 		clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);  in nfsd_file_gc_cb()
549 		trace_nfsd_file_gc_aged(nf);  in nfsd_file_gc_cb()
631 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)  in nfsd_file_cond_queue()  argument
637 	if (!nfsd_file_unhash(nf))  in nfsd_file_cond_queue()
641 	if (!nfsd_file_get(nf))  in nfsd_file_cond_queue()
645 	if (nfsd_file_lru_remove(nf))  in nfsd_file_cond_queue()
649 	if (refcount_sub_and_test(decrement, &nf->nf_ref)) {  in nfsd_file_cond_queue()
650 		list_add(&nf->nf_gc, dispose);  in nfsd_file_cond_queue()
651 		trace_nfsd_file_closing(nf);  in nfsd_file_cond_queue()
675 	struct nfsd_file *nf;  in nfsd_file_queue_for_close()  local
680 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {  in nfsd_file_queue_for_close()
681 		if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))  in nfsd_file_queue_for_close()
683 		nfsd_file_cond_queue(nf, dispose);  in nfsd_file_queue_for_close()
868 	struct nfsd_file *nf;  in __nfsd_file_cache_purge()  local
883 		nf = rhashtable_walk_next(&iter);  in __nfsd_file_cache_purge()
884 		while (!IS_ERR_OR_NULL(nf)) {  in __nfsd_file_cache_purge()
885 			if (!net || nf->nf_net == net)  in __nfsd_file_cache_purge()
886 				nfsd_file_cond_queue(nf, &dispose);  in __nfsd_file_cache_purge()
887 			nf = rhashtable_walk_next(&iter);  in __nfsd_file_cache_purge()
891 	} while (nf == ERR_PTR(-EAGAIN));  in __nfsd_file_cache_purge()
999 	struct nfsd_file *nf;  in nfsd_file_lookup_locked()  local
1003 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {  in nfsd_file_lookup_locked()
1004 		if (nf->nf_may != need)  in nfsd_file_lookup_locked()
1006 		if (nf->nf_net != net)  in nfsd_file_lookup_locked()
1008 		if (!nfsd_match_cred(nf->nf_cred, cred))  in nfsd_file_lookup_locked()
1010 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)  in nfsd_file_lookup_locked()
1012 		if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)  in nfsd_file_lookup_locked()
1015 		if (!nfsd_file_get(nf))  in nfsd_file_lookup_locked()
1017 		return nf;  in nfsd_file_lookup_locked()
1037 	struct nfsd_file *nf;  in nfsd_file_is_cached()  local
1043 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)  in nfsd_file_is_cached()
1044 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {  in nfsd_file_is_cached()
1055 nfsd_file_get_dio_attrs(const struct svc_fh *fhp, struct nfsd_file *nf)  in nfsd_file_get_dio_attrs()  argument
1057 	struct inode *inode = file_inode(nf->nf_file);  in nfsd_file_get_dio_attrs()
1072 		nf->nf_dio_mem_align = stat.dio_mem_align;  in nfsd_file_get_dio_attrs()
1073 		nf->nf_dio_offset_align = stat.dio_offset_align;  in nfsd_file_get_dio_attrs()
1076 		nf->nf_dio_read_offset_align = stat.dio_read_offset_align;  in nfsd_file_get_dio_attrs()
1078 		nf->nf_dio_read_offset_align = nf->nf_dio_offset_align;  in nfsd_file_get_dio_attrs()
1092 	struct nfsd_file *new, *nf;  in nfsd_file_do_acquire()  local
1112 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);  in nfsd_file_do_acquire()
1115 	if (nf)  in nfsd_file_do_acquire()
1126 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);  in nfsd_file_do_acquire()
1127 	if (unlikely(nf)) {  in nfsd_file_do_acquire()
1133 	nf = new;  in nfsd_file_do_acquire()
1134 	ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,  in nfsd_file_do_acquire()
1146 	wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);  in nfsd_file_do_acquire()
1149 	if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {  in nfsd_file_do_acquire()
1150 		trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);  in nfsd_file_do_acquire()
1155 		nfsd_file_put(nf);  in nfsd_file_do_acquire()
1162 	status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));  in nfsd_file_do_acquire()
1164 		nfsd_file_put(nf);  in nfsd_file_do_acquire()
1165 		nf = NULL;  in nfsd_file_do_acquire()
1171 		nfsd_file_check_write_error(nf);  in nfsd_file_do_acquire()
1172 		*pnf = nf;  in nfsd_file_do_acquire()
1174 	trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);  in nfsd_file_do_acquire()
1178 	trace_nfsd_file_alloc(nf);  in nfsd_file_do_acquire()
1181 		nf->nf_mark = nfsd_file_mark_find_or_create(inode);  in nfsd_file_do_acquire()
1183 	if (type != S_IFREG || nf->nf_mark) {  in nfsd_file_do_acquire()
1186 			nf->nf_file = file;  in nfsd_file_do_acquire()
1188 			trace_nfsd_file_opened(nf, status);  in nfsd_file_do_acquire()
1190 			ret = nfsd_open_verified(fhp, type, may_flags, &nf->nf_file);  in nfsd_file_do_acquire()
1193 				nfsd_file_unhash(nf);  in nfsd_file_do_acquire()
1195 						      &nf->nf_flags);  in nfsd_file_do_acquire()
1196 				if (refcount_dec_and_test(&nf->nf_ref))  in nfsd_file_do_acquire()
1197 					nfsd_file_free(nf);  in nfsd_file_do_acquire()
1198 				nf = NULL;  in nfsd_file_do_acquire()
1203 			trace_nfsd_file_open(nf, status);  in nfsd_file_do_acquire()
1205 				status = nfsd_file_get_dio_attrs(fhp, nf);  in nfsd_file_do_acquire()
1214 		nfsd_file_unhash(nf);  in nfsd_file_do_acquire()
1216 		nfsd_file_lru_add(nf);  in nfsd_file_do_acquire()
1218 	clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);  in nfsd_file_do_acquire()
1223 	if (refcount_dec_and_test(&nf->nf_ref))  in nfsd_file_do_acquire()
1224 		nfsd_file_free(nf);  in nfsd_file_do_acquire()
1225 	nf = NULL;  in nfsd_file_do_acquire()