xref: /linux/fs/nfsd/filecache.c (revision 7f71507851fc7764b36a3221839607d3a45c2025)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * The NFSD open file cache.
4  *
5  * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
6  *
7  * An nfsd_file object is a per-file collection of open state that binds
8  * together:
9  *   - a struct file *
10  *   - a user credential
11  *   - a network namespace
12  *   - a read-ahead context
13  *   - monitoring for writeback errors
14  *
15  * nfsd_file objects are reference-counted. Consumers acquire a new
16  * object via the nfsd_file_acquire API. They manage their interest in
17  * the acquired object, and hence the object's reference count, via
18  * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
19  * object:
20  *
21  *  * non-garbage-collected: When a consumer wants to precisely control
22  *    the lifetime of a file's open state, it acquires a non-garbage-
23  *    collected nfsd_file. The final nfsd_file_put releases the open
24  *    state immediately.
25  *
26  *  * garbage-collected: When a consumer does not control the lifetime
27  *    of open state, it acquires a garbage-collected nfsd_file. The
28  *    final nfsd_file_put allows the open state to linger for a period
29  *    during which it may be re-used.
30  */
31 
32 #include <linux/hash.h>
33 #include <linux/slab.h>
34 #include <linux/file.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched.h>
37 #include <linux/list_lru.h>
38 #include <linux/fsnotify_backend.h>
39 #include <linux/fsnotify.h>
40 #include <linux/seq_file.h>
41 #include <linux/rhashtable.h>
42 
43 #include "vfs.h"
44 #include "nfsd.h"
45 #include "nfsfh.h"
46 #include "netns.h"
47 #include "filecache.h"
48 #include "trace.h"
49 
50 #define NFSD_LAUNDRETTE_DELAY		     (2 * HZ)
51 
52 #define NFSD_FILE_CACHE_UP		     (0)
53 
54 /* We only care about NFSD_MAY_READ/WRITE for this cache */
55 #define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO)
56 
57 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
58 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
59 static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);
60 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
61 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
62 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
63 
64 struct nfsd_fcache_disposal {
65 	spinlock_t lock;
66 	struct list_head freeme;
67 };
68 
69 static struct kmem_cache		*nfsd_file_slab;
70 static struct kmem_cache		*nfsd_file_mark_slab;
71 static struct list_lru			nfsd_file_lru;
72 static unsigned long			nfsd_file_flags;
73 static struct fsnotify_group		*nfsd_file_fsnotify_group;
74 static struct delayed_work		nfsd_filecache_laundrette;
75 static struct rhltable			nfsd_file_rhltable
76 						____cacheline_aligned_in_smp;
77 
78 static bool
79 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
80 {
81 	int i;
82 
83 	if (!uid_eq(c1->fsuid, c2->fsuid))
84 		return false;
85 	if (!gid_eq(c1->fsgid, c2->fsgid))
86 		return false;
87 	if (c1->group_info == NULL || c2->group_info == NULL)
88 		return c1->group_info == c2->group_info;
89 	if (c1->group_info->ngroups != c2->group_info->ngroups)
90 		return false;
91 	for (i = 0; i < c1->group_info->ngroups; i++) {
92 		if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
93 			return false;
94 	}
95 	return true;
96 }
97 
98 static const struct rhashtable_params nfsd_file_rhash_params = {
99 	.key_len		= sizeof_field(struct nfsd_file, nf_inode),
100 	.key_offset		= offsetof(struct nfsd_file, nf_inode),
101 	.head_offset		= offsetof(struct nfsd_file, nf_rlist),
102 
103 	/*
104 	 * Start with a single page hash table to reduce resizing churn
105 	 * on light workloads.
106 	 */
107 	.min_size		= 256,
108 	.automatic_shrinking	= true,
109 };
110 
111 static void
112 nfsd_file_schedule_laundrette(void)
113 {
114 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
115 		queue_delayed_work(system_unbound_wq, &nfsd_filecache_laundrette,
116 				   NFSD_LAUNDRETTE_DELAY);
117 }
118 
119 static void
120 nfsd_file_slab_free(struct rcu_head *rcu)
121 {
122 	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
123 
124 	put_cred(nf->nf_cred);
125 	kmem_cache_free(nfsd_file_slab, nf);
126 }
127 
128 static void
129 nfsd_file_mark_free(struct fsnotify_mark *mark)
130 {
131 	struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
132 						  nfm_mark);
133 
134 	kmem_cache_free(nfsd_file_mark_slab, nfm);
135 }
136 
137 static struct nfsd_file_mark *
138 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
139 {
140 	if (!refcount_inc_not_zero(&nfm->nfm_ref))
141 		return NULL;
142 	return nfm;
143 }
144 
145 static void
146 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
147 {
148 	if (refcount_dec_and_test(&nfm->nfm_ref)) {
149 		fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
150 		fsnotify_put_mark(&nfm->nfm_mark);
151 	}
152 }
153 
154 static struct nfsd_file_mark *
155 nfsd_file_mark_find_or_create(struct inode *inode)
156 {
157 	int			err;
158 	struct fsnotify_mark	*mark;
159 	struct nfsd_file_mark	*nfm = NULL, *new;
160 
161 	do {
162 		fsnotify_group_lock(nfsd_file_fsnotify_group);
163 		mark = fsnotify_find_inode_mark(inode,
164 						nfsd_file_fsnotify_group);
165 		if (mark) {
166 			nfm = nfsd_file_mark_get(container_of(mark,
167 						 struct nfsd_file_mark,
168 						 nfm_mark));
169 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
170 			if (nfm) {
171 				fsnotify_put_mark(mark);
172 				break;
173 			}
174 			/* Avoid soft lockup race with nfsd_file_mark_put() */
175 			fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
176 			fsnotify_put_mark(mark);
177 		} else {
178 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
179 		}
180 
181 		/* allocate a new nfm */
182 		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
183 		if (!new)
184 			return NULL;
185 		fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
186 		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
187 		refcount_set(&new->nfm_ref, 1);
188 
189 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
190 
191 		/*
192 		 * If the add was successful, then return the object.
193 		 * Otherwise, we need to put the reference we hold on the
194 		 * nfm_mark. The fsnotify code will take a reference and put
195 		 * it on failure, so we can't just free it directly. It's also
196 		 * not safe to call fsnotify_destroy_mark on it as the
197 		 * mark->group will be NULL. Thus, we can't let the nfm_ref
198 		 * counter drive the destruction at this point.
199 		 */
200 		if (likely(!err))
201 			nfm = new;
202 		else
203 			fsnotify_put_mark(&new->nfm_mark);
204 	} while (unlikely(err == -EEXIST));
205 
206 	return nfm;
207 }
208 
209 static struct nfsd_file *
210 nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,
211 		bool want_gc)
212 {
213 	struct nfsd_file *nf;
214 
215 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
216 	if (unlikely(!nf))
217 		return NULL;
218 
219 	this_cpu_inc(nfsd_file_allocations);
220 	INIT_LIST_HEAD(&nf->nf_lru);
221 	INIT_LIST_HEAD(&nf->nf_gc);
222 	nf->nf_birthtime = ktime_get();
223 	nf->nf_file = NULL;
224 	nf->nf_cred = get_current_cred();
225 	nf->nf_net = net;
226 	nf->nf_flags = want_gc ?
227 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :
228 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);
229 	nf->nf_inode = inode;
230 	refcount_set(&nf->nf_ref, 1);
231 	nf->nf_may = need;
232 	nf->nf_mark = NULL;
233 	return nf;
234 }
235 
236 /**
237  * nfsd_file_check_write_error - check for writeback errors on a file
238  * @nf: nfsd_file to check for writeback errors
239  *
240  * Check whether a nfsd_file has an unseen error. Reset the write
241  * verifier if so.
242  */
243 static void
244 nfsd_file_check_write_error(struct nfsd_file *nf)
245 {
246 	struct file *file = nf->nf_file;
247 
248 	if ((file->f_mode & FMODE_WRITE) &&
249 	    filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
250 		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
251 }
252 
253 static void
254 nfsd_file_hash_remove(struct nfsd_file *nf)
255 {
256 	trace_nfsd_file_unhash(nf);
257 	rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,
258 			nfsd_file_rhash_params);
259 }
260 
261 static bool
262 nfsd_file_unhash(struct nfsd_file *nf)
263 {
264 	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
265 		nfsd_file_hash_remove(nf);
266 		return true;
267 	}
268 	return false;
269 }
270 
271 static void
272 nfsd_file_free(struct nfsd_file *nf)
273 {
274 	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
275 
276 	trace_nfsd_file_free(nf);
277 
278 	this_cpu_inc(nfsd_file_releases);
279 	this_cpu_add(nfsd_file_total_age, age);
280 
281 	nfsd_file_unhash(nf);
282 	if (nf->nf_mark)
283 		nfsd_file_mark_put(nf->nf_mark);
284 	if (nf->nf_file) {
285 		nfsd_file_check_write_error(nf);
286 		nfsd_filp_close(nf->nf_file);
287 	}
288 
289 	/*
290 	 * If this item is still linked via nf_lru, that's a bug.
291 	 * WARN and leak it to preserve system stability.
292 	 */
293 	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
294 		return;
295 
296 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
297 }
298 
299 static bool
300 nfsd_file_check_writeback(struct nfsd_file *nf)
301 {
302 	struct file *file = nf->nf_file;
303 	struct address_space *mapping;
304 
305 	/* File not open for write? */
306 	if (!(file->f_mode & FMODE_WRITE))
307 		return false;
308 
309 	/*
310 	 * Some filesystems (e.g. NFS) flush all dirty data on close.
311 	 * On others, there is no need to wait for writeback.
312 	 */
313 	if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))
314 		return false;
315 
316 	mapping = file->f_mapping;
317 	return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
318 		mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
319 }
320 
321 
322 static bool nfsd_file_lru_add(struct nfsd_file *nf)
323 {
324 	set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
325 	if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) {
326 		trace_nfsd_file_lru_add(nf);
327 		return true;
328 	}
329 	return false;
330 }
331 
332 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
333 {
334 	if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) {
335 		trace_nfsd_file_lru_del(nf);
336 		return true;
337 	}
338 	return false;
339 }
340 
341 struct nfsd_file *
342 nfsd_file_get(struct nfsd_file *nf)
343 {
344 	if (nf && refcount_inc_not_zero(&nf->nf_ref))
345 		return nf;
346 	return NULL;
347 }
348 
349 /**
350  * nfsd_file_put - put the reference to a nfsd_file
351  * @nf: nfsd_file of which to put the reference
352  *
353  * Put a reference to a nfsd_file. In the non-GC case, we just put the
354  * reference immediately. In the GC case, if the reference would be
355  * the last one, the put it on the LRU instead to be cleaned up later.
356  */
357 void
358 nfsd_file_put(struct nfsd_file *nf)
359 {
360 	might_sleep();
361 	trace_nfsd_file_put(nf);
362 
363 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
364 	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
365 		/*
366 		 * If this is the last reference (nf_ref == 1), then try to
367 		 * transfer it to the LRU.
368 		 */
369 		if (refcount_dec_not_one(&nf->nf_ref))
370 			return;
371 
372 		/* Try to add it to the LRU.  If that fails, decrement. */
373 		if (nfsd_file_lru_add(nf)) {
374 			/* If it's still hashed, we're done */
375 			if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
376 				nfsd_file_schedule_laundrette();
377 				return;
378 			}
379 
380 			/*
381 			 * We're racing with unhashing, so try to remove it from
382 			 * the LRU. If removal fails, then someone else already
383 			 * has our reference.
384 			 */
385 			if (!nfsd_file_lru_remove(nf))
386 				return;
387 		}
388 	}
389 	if (refcount_dec_and_test(&nf->nf_ref))
390 		nfsd_file_free(nf);
391 }
392 
393 /**
394  * nfsd_file_put_local - put nfsd_file reference and arm nfsd_serv_put in caller
395  * @nf: nfsd_file of which to put the reference
396  *
397  * First save the associated net to return to caller, then put
398  * the reference of the nfsd_file.
399  */
400 struct net *
401 nfsd_file_put_local(struct nfsd_file *nf)
402 {
403 	struct net *net = nf->nf_net;
404 
405 	nfsd_file_put(nf);
406 	return net;
407 }
408 
409 /**
410  * nfsd_file_file - get the backing file of an nfsd_file
411  * @nf: nfsd_file of which to access the backing file.
412  *
413  * Return backing file for @nf.
414  */
415 struct file *
416 nfsd_file_file(struct nfsd_file *nf)
417 {
418 	return nf->nf_file;
419 }
420 
421 static void
422 nfsd_file_dispose_list(struct list_head *dispose)
423 {
424 	struct nfsd_file *nf;
425 
426 	while (!list_empty(dispose)) {
427 		nf = list_first_entry(dispose, struct nfsd_file, nf_gc);
428 		list_del_init(&nf->nf_gc);
429 		nfsd_file_free(nf);
430 	}
431 }
432 
433 /**
434  * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list
435  * @dispose: list of nfsd_files to be disposed
436  *
437  * Transfers each file to the "freeme" list for its nfsd_net, to eventually
438  * be disposed of by the per-net garbage collector.
439  */
440 static void
441 nfsd_file_dispose_list_delayed(struct list_head *dispose)
442 {
443 	while(!list_empty(dispose)) {
444 		struct nfsd_file *nf = list_first_entry(dispose,
445 						struct nfsd_file, nf_gc);
446 		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
447 		struct nfsd_fcache_disposal *l = nn->fcache_disposal;
448 
449 		spin_lock(&l->lock);
450 		list_move_tail(&nf->nf_gc, &l->freeme);
451 		spin_unlock(&l->lock);
452 		svc_wake_up(nn->nfsd_serv);
453 	}
454 }
455 
456 /**
457  * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed.
458  * @nn: nfsd_net in which to find files to be disposed.
459  *
460  * When files held open for nfsv3 are removed from the filecache, whether
461  * due to memory pressure or garbage collection, they are queued to
462  * a per-net-ns queue.  This function completes the disposal, either
463  * directly or by waking another nfsd thread to help with the work.
464  */
465 void nfsd_file_net_dispose(struct nfsd_net *nn)
466 {
467 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
468 
469 	if (!list_empty(&l->freeme)) {
470 		LIST_HEAD(dispose);
471 		int i;
472 
473 		spin_lock(&l->lock);
474 		for (i = 0; i < 8 && !list_empty(&l->freeme); i++)
475 			list_move(l->freeme.next, &dispose);
476 		spin_unlock(&l->lock);
477 		if (!list_empty(&l->freeme))
478 			/* Wake up another thread to share the work
479 			 * *before* doing any actual disposing.
480 			 */
481 			svc_wake_up(nn->nfsd_serv);
482 		nfsd_file_dispose_list(&dispose);
483 	}
484 }
485 
486 /**
487  * nfsd_file_lru_cb - Examine an entry on the LRU list
488  * @item: LRU entry to examine
489  * @lru: controlling LRU
490  * @arg: dispose list
491  *
492  * Return values:
493  *   %LRU_REMOVED: @item was removed from the LRU
494  *   %LRU_ROTATE: @item is to be moved to the LRU tail
495  *   %LRU_SKIP: @item cannot be evicted
496  */
497 static enum lru_status
498 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
499 		 void *arg)
500 {
501 	struct list_head *head = arg;
502 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
503 
504 	/* We should only be dealing with GC entries here */
505 	WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
506 
507 	/*
508 	 * Don't throw out files that are still undergoing I/O or
509 	 * that have uncleared errors pending.
510 	 */
511 	if (nfsd_file_check_writeback(nf)) {
512 		trace_nfsd_file_gc_writeback(nf);
513 		return LRU_SKIP;
514 	}
515 
516 	/* If it was recently added to the list, skip it */
517 	if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
518 		trace_nfsd_file_gc_referenced(nf);
519 		return LRU_ROTATE;
520 	}
521 
522 	/*
523 	 * Put the reference held on behalf of the LRU. If it wasn't the last
524 	 * one, then just remove it from the LRU and ignore it.
525 	 */
526 	if (!refcount_dec_and_test(&nf->nf_ref)) {
527 		trace_nfsd_file_gc_in_use(nf);
528 		list_lru_isolate(lru, &nf->nf_lru);
529 		return LRU_REMOVED;
530 	}
531 
532 	/* Refcount went to zero. Unhash it and queue it to the dispose list */
533 	nfsd_file_unhash(nf);
534 	list_lru_isolate(lru, &nf->nf_lru);
535 	list_add(&nf->nf_gc, head);
536 	this_cpu_inc(nfsd_file_evictions);
537 	trace_nfsd_file_gc_disposed(nf);
538 	return LRU_REMOVED;
539 }
540 
541 static void
542 nfsd_file_gc(void)
543 {
544 	LIST_HEAD(dispose);
545 	unsigned long ret;
546 
547 	ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
548 			    &dispose, list_lru_count(&nfsd_file_lru));
549 	trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
550 	nfsd_file_dispose_list_delayed(&dispose);
551 }
552 
553 static void
554 nfsd_file_gc_worker(struct work_struct *work)
555 {
556 	nfsd_file_gc();
557 	if (list_lru_count(&nfsd_file_lru))
558 		nfsd_file_schedule_laundrette();
559 }
560 
561 static unsigned long
562 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
563 {
564 	return list_lru_count(&nfsd_file_lru);
565 }
566 
567 static unsigned long
568 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
569 {
570 	LIST_HEAD(dispose);
571 	unsigned long ret;
572 
573 	ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
574 				   nfsd_file_lru_cb, &dispose);
575 	trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
576 	nfsd_file_dispose_list_delayed(&dispose);
577 	return ret;
578 }
579 
580 static struct shrinker *nfsd_file_shrinker;
581 
582 /**
583  * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
584  * @nf: nfsd_file to attempt to queue
585  * @dispose: private list to queue successfully-put objects
586  *
587  * Unhash an nfsd_file, try to get a reference to it, and then put that
588  * reference. If it's the last reference, queue it to the dispose list.
589  */
590 static void
591 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
592 	__must_hold(RCU)
593 {
594 	int decrement = 1;
595 
596 	/* If we raced with someone else unhashing, ignore it */
597 	if (!nfsd_file_unhash(nf))
598 		return;
599 
600 	/* If we can't get a reference, ignore it */
601 	if (!nfsd_file_get(nf))
602 		return;
603 
604 	/* Extra decrement if we remove from the LRU */
605 	if (nfsd_file_lru_remove(nf))
606 		++decrement;
607 
608 	/* If refcount goes to 0, then put on the dispose list */
609 	if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
610 		list_add(&nf->nf_gc, dispose);
611 		trace_nfsd_file_closing(nf);
612 	}
613 }
614 
615 /**
616  * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
617  * @inode:   inode on which to close out nfsd_files
618  * @dispose: list on which to gather nfsd_files to close out
619  *
620  * An nfsd_file represents a struct file being held open on behalf of nfsd.
621  * An open file however can block other activity (such as leases), or cause
622  * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
623  *
624  * This function is intended to find open nfsd_files when this sort of
625  * conflicting access occurs and then attempt to close those files out.
626  *
627  * Populates the dispose list with entries that have already had their
628  * refcounts go to zero. The actual free of an nfsd_file can be expensive,
629  * so we leave it up to the caller whether it wants to wait or not.
630  */
631 static void
632 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
633 {
634 	struct rhlist_head *tmp, *list;
635 	struct nfsd_file *nf;
636 
637 	rcu_read_lock();
638 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
639 			       nfsd_file_rhash_params);
640 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
641 		if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))
642 			continue;
643 		nfsd_file_cond_queue(nf, dispose);
644 	}
645 	rcu_read_unlock();
646 }
647 
648 /**
649  * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
650  * @inode: inode of the file to attempt to remove
651  *
652  * Close out any open nfsd_files that can be reaped for @inode. The
653  * actual freeing is deferred to the dispose_list_delayed infrastructure.
654  *
655  * This is used by the fsnotify callbacks and setlease notifier.
656  */
657 static void
658 nfsd_file_close_inode(struct inode *inode)
659 {
660 	LIST_HEAD(dispose);
661 
662 	nfsd_file_queue_for_close(inode, &dispose);
663 	nfsd_file_dispose_list_delayed(&dispose);
664 }
665 
666 /**
667  * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
668  * @inode: inode of the file to attempt to remove
669  *
670  * Close out any open nfsd_files that can be reaped for @inode. The
671  * nfsd_files are closed out synchronously.
672  *
673  * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
674  * when reexporting NFS.
675  */
676 void
677 nfsd_file_close_inode_sync(struct inode *inode)
678 {
679 	struct nfsd_file *nf;
680 	LIST_HEAD(dispose);
681 
682 	trace_nfsd_file_close(inode);
683 
684 	nfsd_file_queue_for_close(inode, &dispose);
685 	while (!list_empty(&dispose)) {
686 		nf = list_first_entry(&dispose, struct nfsd_file, nf_gc);
687 		list_del_init(&nf->nf_gc);
688 		nfsd_file_free(nf);
689 	}
690 }
691 
692 static int
693 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
694 			    void *data)
695 {
696 	struct file_lease *fl = data;
697 
698 	/* Only close files for F_SETLEASE leases */
699 	if (fl->c.flc_flags & FL_LEASE)
700 		nfsd_file_close_inode(file_inode(fl->c.flc_file));
701 	return 0;
702 }
703 
704 static struct notifier_block nfsd_file_lease_notifier = {
705 	.notifier_call = nfsd_file_lease_notifier_call,
706 };
707 
708 static int
709 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
710 				struct inode *inode, struct inode *dir,
711 				const struct qstr *name, u32 cookie)
712 {
713 	if (WARN_ON_ONCE(!inode))
714 		return 0;
715 
716 	trace_nfsd_file_fsnotify_handle_event(inode, mask);
717 
718 	/* Should be no marks on non-regular files */
719 	if (!S_ISREG(inode->i_mode)) {
720 		WARN_ON_ONCE(1);
721 		return 0;
722 	}
723 
724 	/* don't close files if this was not the last link */
725 	if (mask & FS_ATTRIB) {
726 		if (inode->i_nlink)
727 			return 0;
728 	}
729 
730 	nfsd_file_close_inode(inode);
731 	return 0;
732 }
733 
734 
735 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
736 	.handle_inode_event = nfsd_file_fsnotify_handle_event,
737 	.free_mark = nfsd_file_mark_free,
738 };
739 
740 int
741 nfsd_file_cache_init(void)
742 {
743 	int ret;
744 
745 	lockdep_assert_held(&nfsd_mutex);
746 	if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
747 		return 0;
748 
749 	ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params);
750 	if (ret)
751 		goto out;
752 
753 	ret = -ENOMEM;
754 	nfsd_file_slab = KMEM_CACHE(nfsd_file, 0);
755 	if (!nfsd_file_slab) {
756 		pr_err("nfsd: unable to create nfsd_file_slab\n");
757 		goto out_err;
758 	}
759 
760 	nfsd_file_mark_slab = KMEM_CACHE(nfsd_file_mark, 0);
761 	if (!nfsd_file_mark_slab) {
762 		pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
763 		goto out_err;
764 	}
765 
766 	ret = list_lru_init(&nfsd_file_lru);
767 	if (ret) {
768 		pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
769 		goto out_err;
770 	}
771 
772 	nfsd_file_shrinker = shrinker_alloc(0, "nfsd-filecache");
773 	if (!nfsd_file_shrinker) {
774 		ret = -ENOMEM;
775 		pr_err("nfsd: failed to allocate nfsd_file_shrinker\n");
776 		goto out_lru;
777 	}
778 
779 	nfsd_file_shrinker->count_objects = nfsd_file_lru_count;
780 	nfsd_file_shrinker->scan_objects = nfsd_file_lru_scan;
781 	nfsd_file_shrinker->seeks = 1;
782 
783 	shrinker_register(nfsd_file_shrinker);
784 
785 	ret = lease_register_notifier(&nfsd_file_lease_notifier);
786 	if (ret) {
787 		pr_err("nfsd: unable to register lease notifier: %d\n", ret);
788 		goto out_shrinker;
789 	}
790 
791 	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
792 							0);
793 	if (IS_ERR(nfsd_file_fsnotify_group)) {
794 		pr_err("nfsd: unable to create fsnotify group: %ld\n",
795 			PTR_ERR(nfsd_file_fsnotify_group));
796 		ret = PTR_ERR(nfsd_file_fsnotify_group);
797 		nfsd_file_fsnotify_group = NULL;
798 		goto out_notifier;
799 	}
800 
801 	INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
802 out:
803 	if (ret)
804 		clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags);
805 	return ret;
806 out_notifier:
807 	lease_unregister_notifier(&nfsd_file_lease_notifier);
808 out_shrinker:
809 	shrinker_free(nfsd_file_shrinker);
810 out_lru:
811 	list_lru_destroy(&nfsd_file_lru);
812 out_err:
813 	kmem_cache_destroy(nfsd_file_slab);
814 	nfsd_file_slab = NULL;
815 	kmem_cache_destroy(nfsd_file_mark_slab);
816 	nfsd_file_mark_slab = NULL;
817 	rhltable_destroy(&nfsd_file_rhltable);
818 	goto out;
819 }
820 
821 /**
822  * __nfsd_file_cache_purge: clean out the cache for shutdown
823  * @net: net-namespace to shut down the cache (may be NULL)
824  *
825  * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
826  * then close out everything. Called when an nfsd instance is being shut down,
827  * and when the exports table is flushed.
828  */
829 static void
830 __nfsd_file_cache_purge(struct net *net)
831 {
832 	struct rhashtable_iter iter;
833 	struct nfsd_file *nf;
834 	LIST_HEAD(dispose);
835 
836 	rhltable_walk_enter(&nfsd_file_rhltable, &iter);
837 	do {
838 		rhashtable_walk_start(&iter);
839 
840 		nf = rhashtable_walk_next(&iter);
841 		while (!IS_ERR_OR_NULL(nf)) {
842 			if (!net || nf->nf_net == net)
843 				nfsd_file_cond_queue(nf, &dispose);
844 			nf = rhashtable_walk_next(&iter);
845 		}
846 
847 		rhashtable_walk_stop(&iter);
848 	} while (nf == ERR_PTR(-EAGAIN));
849 	rhashtable_walk_exit(&iter);
850 
851 	nfsd_file_dispose_list(&dispose);
852 }
853 
854 static struct nfsd_fcache_disposal *
855 nfsd_alloc_fcache_disposal(void)
856 {
857 	struct nfsd_fcache_disposal *l;
858 
859 	l = kmalloc(sizeof(*l), GFP_KERNEL);
860 	if (!l)
861 		return NULL;
862 	spin_lock_init(&l->lock);
863 	INIT_LIST_HEAD(&l->freeme);
864 	return l;
865 }
866 
867 static void
868 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
869 {
870 	nfsd_file_dispose_list(&l->freeme);
871 	kfree(l);
872 }
873 
874 static void
875 nfsd_free_fcache_disposal_net(struct net *net)
876 {
877 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
878 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
879 
880 	nfsd_free_fcache_disposal(l);
881 }
882 
883 int
884 nfsd_file_cache_start_net(struct net *net)
885 {
886 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
887 
888 	nn->fcache_disposal = nfsd_alloc_fcache_disposal();
889 	return nn->fcache_disposal ? 0 : -ENOMEM;
890 }
891 
892 /**
893  * nfsd_file_cache_purge - Remove all cache items associated with @net
894  * @net: target net namespace
895  *
896  */
897 void
898 nfsd_file_cache_purge(struct net *net)
899 {
900 	lockdep_assert_held(&nfsd_mutex);
901 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
902 		__nfsd_file_cache_purge(net);
903 }
904 
905 void
906 nfsd_file_cache_shutdown_net(struct net *net)
907 {
908 	nfsd_file_cache_purge(net);
909 	nfsd_free_fcache_disposal_net(net);
910 }
911 
912 void
913 nfsd_file_cache_shutdown(void)
914 {
915 	int i;
916 
917 	lockdep_assert_held(&nfsd_mutex);
918 	if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
919 		return;
920 
921 	lease_unregister_notifier(&nfsd_file_lease_notifier);
922 	shrinker_free(nfsd_file_shrinker);
923 	/*
924 	 * make sure all callers of nfsd_file_lru_cb are done before
925 	 * calling nfsd_file_cache_purge
926 	 */
927 	cancel_delayed_work_sync(&nfsd_filecache_laundrette);
928 	__nfsd_file_cache_purge(NULL);
929 	list_lru_destroy(&nfsd_file_lru);
930 	rcu_barrier();
931 	fsnotify_put_group(nfsd_file_fsnotify_group);
932 	nfsd_file_fsnotify_group = NULL;
933 	kmem_cache_destroy(nfsd_file_slab);
934 	nfsd_file_slab = NULL;
935 	fsnotify_wait_marks_destroyed();
936 	kmem_cache_destroy(nfsd_file_mark_slab);
937 	nfsd_file_mark_slab = NULL;
938 	rhltable_destroy(&nfsd_file_rhltable);
939 
940 	for_each_possible_cpu(i) {
941 		per_cpu(nfsd_file_cache_hits, i) = 0;
942 		per_cpu(nfsd_file_acquisitions, i) = 0;
943 		per_cpu(nfsd_file_allocations, i) = 0;
944 		per_cpu(nfsd_file_releases, i) = 0;
945 		per_cpu(nfsd_file_total_age, i) = 0;
946 		per_cpu(nfsd_file_evictions, i) = 0;
947 	}
948 }
949 
950 static struct nfsd_file *
951 nfsd_file_lookup_locked(const struct net *net, const struct cred *cred,
952 			struct inode *inode, unsigned char need,
953 			bool want_gc)
954 {
955 	struct rhlist_head *tmp, *list;
956 	struct nfsd_file *nf;
957 
958 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
959 			       nfsd_file_rhash_params);
960 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
961 		if (nf->nf_may != need)
962 			continue;
963 		if (nf->nf_net != net)
964 			continue;
965 		if (!nfsd_match_cred(nf->nf_cred, cred))
966 			continue;
967 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)
968 			continue;
969 		if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
970 			continue;
971 
972 		if (!nfsd_file_get(nf))
973 			continue;
974 		return nf;
975 	}
976 	return NULL;
977 }
978 
979 /**
980  * nfsd_file_is_cached - are there any cached open files for this inode?
981  * @inode: inode to check
982  *
983  * The lookup matches inodes in all net namespaces and is atomic wrt
984  * nfsd_file_acquire().
985  *
986  * Return values:
987  *   %true: filecache contains at least one file matching this inode
988  *   %false: filecache contains no files matching this inode
989  */
990 bool
991 nfsd_file_is_cached(struct inode *inode)
992 {
993 	struct rhlist_head *tmp, *list;
994 	struct nfsd_file *nf;
995 	bool ret = false;
996 
997 	rcu_read_lock();
998 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
999 			       nfsd_file_rhash_params);
1000 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)
1001 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
1002 			ret = true;
1003 			break;
1004 		}
1005 	rcu_read_unlock();
1006 
1007 	trace_nfsd_file_is_cached(inode, (int)ret);
1008 	return ret;
1009 }
1010 
1011 static __be32
1012 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net,
1013 		     struct svc_cred *cred,
1014 		     struct auth_domain *client,
1015 		     struct svc_fh *fhp,
1016 		     unsigned int may_flags, struct file *file,
1017 		     struct nfsd_file **pnf, bool want_gc)
1018 {
1019 	unsigned char need = may_flags & NFSD_FILE_MAY_MASK;
1020 	struct nfsd_file *new, *nf;
1021 	bool stale_retry = true;
1022 	bool open_retry = true;
1023 	struct inode *inode;
1024 	__be32 status;
1025 	int ret;
1026 
1027 retry:
1028 	if (rqstp) {
1029 		status = fh_verify(rqstp, fhp, S_IFREG,
1030 				   may_flags|NFSD_MAY_OWNER_OVERRIDE);
1031 	} else {
1032 		status = fh_verify_local(net, cred, client, fhp, S_IFREG,
1033 					 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1034 	}
1035 	if (status != nfs_ok)
1036 		return status;
1037 	inode = d_inode(fhp->fh_dentry);
1038 
1039 	rcu_read_lock();
1040 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1041 	rcu_read_unlock();
1042 
1043 	if (nf) {
1044 		/*
1045 		 * If the nf is on the LRU then it holds an extra reference
1046 		 * that must be put if it's removed. It had better not be
1047 		 * the last one however, since we should hold another.
1048 		 */
1049 		if (nfsd_file_lru_remove(nf))
1050 			refcount_dec(&nf->nf_ref);
1051 		goto wait_for_construction;
1052 	}
1053 
1054 	new = nfsd_file_alloc(net, inode, need, want_gc);
1055 	if (!new) {
1056 		status = nfserr_jukebox;
1057 		goto out;
1058 	}
1059 
1060 	rcu_read_lock();
1061 	spin_lock(&inode->i_lock);
1062 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1063 	if (unlikely(nf)) {
1064 		spin_unlock(&inode->i_lock);
1065 		rcu_read_unlock();
1066 		nfsd_file_free(new);
1067 		goto wait_for_construction;
1068 	}
1069 	nf = new;
1070 	ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,
1071 			      nfsd_file_rhash_params);
1072 	spin_unlock(&inode->i_lock);
1073 	rcu_read_unlock();
1074 	if (likely(ret == 0))
1075 		goto open_file;
1076 
1077 	trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret);
1078 	status = nfserr_jukebox;
1079 	goto construction_err;
1080 
1081 wait_for_construction:
1082 	wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1083 
1084 	/* Did construction of this file fail? */
1085 	if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1086 		trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);
1087 		if (!open_retry) {
1088 			status = nfserr_jukebox;
1089 			goto construction_err;
1090 		}
1091 		nfsd_file_put(nf);
1092 		open_retry = false;
1093 		fh_put(fhp);
1094 		goto retry;
1095 	}
1096 	this_cpu_inc(nfsd_file_cache_hits);
1097 
1098 	status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1099 	if (status != nfs_ok) {
1100 		nfsd_file_put(nf);
1101 		nf = NULL;
1102 	}
1103 
1104 out:
1105 	if (status == nfs_ok) {
1106 		this_cpu_inc(nfsd_file_acquisitions);
1107 		nfsd_file_check_write_error(nf);
1108 		*pnf = nf;
1109 	}
1110 	trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);
1111 	return status;
1112 
1113 open_file:
1114 	trace_nfsd_file_alloc(nf);
1115 	nf->nf_mark = nfsd_file_mark_find_or_create(inode);
1116 	if (nf->nf_mark) {
1117 		if (file) {
1118 			get_file(file);
1119 			nf->nf_file = file;
1120 			status = nfs_ok;
1121 			trace_nfsd_file_opened(nf, status);
1122 		} else {
1123 			ret = nfsd_open_verified(fhp, may_flags, &nf->nf_file);
1124 			if (ret == -EOPENSTALE && stale_retry) {
1125 				stale_retry = false;
1126 				nfsd_file_unhash(nf);
1127 				clear_and_wake_up_bit(NFSD_FILE_PENDING,
1128 						      &nf->nf_flags);
1129 				if (refcount_dec_and_test(&nf->nf_ref))
1130 					nfsd_file_free(nf);
1131 				nf = NULL;
1132 				fh_put(fhp);
1133 				goto retry;
1134 			}
1135 			status = nfserrno(ret);
1136 			trace_nfsd_file_open(nf, status);
1137 		}
1138 	} else
1139 		status = nfserr_jukebox;
1140 	/*
1141 	 * If construction failed, or we raced with a call to unlink()
1142 	 * then unhash.
1143 	 */
1144 	if (status != nfs_ok || inode->i_nlink == 0)
1145 		nfsd_file_unhash(nf);
1146 	clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);
1147 	if (status == nfs_ok)
1148 		goto out;
1149 
1150 construction_err:
1151 	if (refcount_dec_and_test(&nf->nf_ref))
1152 		nfsd_file_free(nf);
1153 	nf = NULL;
1154 	goto out;
1155 }
1156 
1157 /**
1158  * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1159  * @rqstp: the RPC transaction being executed
1160  * @fhp: the NFS filehandle of the file to be opened
1161  * @may_flags: NFSD_MAY_ settings for the file
1162  * @pnf: OUT: new or found "struct nfsd_file" object
1163  *
1164  * The nfsd_file object returned by this API is reference-counted
1165  * and garbage-collected. The object is retained for a few
1166  * seconds after the final nfsd_file_put() in case the caller
1167  * wants to re-use it.
1168  *
1169  * Return values:
1170  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1171  *   count boosted.
1172  *
1173  * On error, an nfsstat value in network byte order is returned.
1174  */
1175 __be32
1176 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1177 		     unsigned int may_flags, struct nfsd_file **pnf)
1178 {
1179 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1180 				    fhp, may_flags, NULL, pnf, true);
1181 }
1182 
1183 /**
1184  * nfsd_file_acquire - Get a struct nfsd_file with an open file
1185  * @rqstp: the RPC transaction being executed
1186  * @fhp: the NFS filehandle of the file to be opened
1187  * @may_flags: NFSD_MAY_ settings for the file
1188  * @pnf: OUT: new or found "struct nfsd_file" object
1189  *
1190  * The nfsd_file_object returned by this API is reference-counted
1191  * but not garbage-collected. The object is unhashed after the
1192  * final nfsd_file_put().
1193  *
1194  * Return values:
1195  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1196  *   count boosted.
1197  *
1198  * On error, an nfsstat value in network byte order is returned.
1199  */
1200 __be32
1201 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1202 		  unsigned int may_flags, struct nfsd_file **pnf)
1203 {
1204 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1205 				    fhp, may_flags, NULL, pnf, false);
1206 }
1207 
1208 /**
1209  * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio
1210  * @net: The network namespace in which to perform a lookup
1211  * @cred: the user credential with which to validate access
1212  * @client: the auth_domain for LOCALIO lookup
1213  * @fhp: the NFS filehandle of the file to be opened
1214  * @may_flags: NFSD_MAY_ settings for the file
1215  * @pnf: OUT: new or found "struct nfsd_file" object
1216  *
1217  * This file lookup interface provide access to a file given the
1218  * filehandle and credential.  No connection-based authorisation
1219  * is performed and in that way it is quite different to other
1220  * file access mediated by nfsd.  It allows a kernel module such as the NFS
1221  * client to reach across network and filesystem namespaces to access
1222  * a file.  The security implications of this should be carefully
1223  * considered before use.
1224  *
1225  * The nfsd_file object returned by this API is reference-counted
1226  * and garbage-collected. The object is retained for a few
1227  * seconds after the final nfsd_file_put() in case the caller
1228  * wants to re-use it.
1229  *
1230  * Return values:
1231  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1232  *   count boosted.
1233  *
1234  * On error, an nfsstat value in network byte order is returned.
1235  */
1236 __be32
1237 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred,
1238 			struct auth_domain *client, struct svc_fh *fhp,
1239 			unsigned int may_flags, struct nfsd_file **pnf)
1240 {
1241 	/*
1242 	 * Save creds before calling nfsd_file_do_acquire() (which calls
1243 	 * nfsd_setuser). Important because caller (LOCALIO) is from
1244 	 * client context.
1245 	 */
1246 	const struct cred *save_cred = get_current_cred();
1247 	__be32 beres;
1248 
1249 	beres = nfsd_file_do_acquire(NULL, net, cred, client,
1250 				     fhp, may_flags, NULL, pnf, true);
1251 	revert_creds(save_cred);
1252 	return beres;
1253 }
1254 
1255 /**
1256  * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1257  * @rqstp: the RPC transaction being executed
1258  * @fhp: the NFS filehandle of the file just created
1259  * @may_flags: NFSD_MAY_ settings for the file
1260  * @file: cached, already-open file (may be NULL)
1261  * @pnf: OUT: new or found "struct nfsd_file" object
1262  *
1263  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1264  * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1265  * opening a new one.
1266  *
1267  * Return values:
1268  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1269  *   count boosted.
1270  *
1271  * On error, an nfsstat value in network byte order is returned.
1272  */
1273 __be32
1274 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1275 			 unsigned int may_flags, struct file *file,
1276 			 struct nfsd_file **pnf)
1277 {
1278 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1279 				    fhp, may_flags, file, pnf, false);
1280 }
1281 
1282 /*
1283  * Note that fields may be added, removed or reordered in the future. Programs
1284  * scraping this file for info should test the labels to ensure they're
1285  * getting the correct field.
1286  */
1287 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1288 {
1289 	unsigned long allocations = 0, releases = 0, evictions = 0;
1290 	unsigned long hits = 0, acquisitions = 0;
1291 	unsigned int i, count = 0, buckets = 0;
1292 	unsigned long lru = 0, total_age = 0;
1293 
1294 	/* Serialize with server shutdown */
1295 	mutex_lock(&nfsd_mutex);
1296 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1297 		struct bucket_table *tbl;
1298 		struct rhashtable *ht;
1299 
1300 		lru = list_lru_count(&nfsd_file_lru);
1301 
1302 		rcu_read_lock();
1303 		ht = &nfsd_file_rhltable.ht;
1304 		count = atomic_read(&ht->nelems);
1305 		tbl = rht_dereference_rcu(ht->tbl, ht);
1306 		buckets = tbl->size;
1307 		rcu_read_unlock();
1308 	}
1309 	mutex_unlock(&nfsd_mutex);
1310 
1311 	for_each_possible_cpu(i) {
1312 		hits += per_cpu(nfsd_file_cache_hits, i);
1313 		acquisitions += per_cpu(nfsd_file_acquisitions, i);
1314 		allocations += per_cpu(nfsd_file_allocations, i);
1315 		releases += per_cpu(nfsd_file_releases, i);
1316 		total_age += per_cpu(nfsd_file_total_age, i);
1317 		evictions += per_cpu(nfsd_file_evictions, i);
1318 	}
1319 
1320 	seq_printf(m, "total inodes:  %u\n", count);
1321 	seq_printf(m, "hash buckets:  %u\n", buckets);
1322 	seq_printf(m, "lru entries:   %lu\n", lru);
1323 	seq_printf(m, "cache hits:    %lu\n", hits);
1324 	seq_printf(m, "acquisitions:  %lu\n", acquisitions);
1325 	seq_printf(m, "allocations:   %lu\n", allocations);
1326 	seq_printf(m, "releases:      %lu\n", releases);
1327 	seq_printf(m, "evictions:     %lu\n", evictions);
1328 	if (releases)
1329 		seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1330 	else
1331 		seq_printf(m, "mean age (ms): -\n");
1332 	return 0;
1333 }
1334