xref: /linux/fs/nfsd/filecache.c (revision 070a542f08acb7e8cf197287f5c44658c715d2d1)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * The NFSD open file cache.
4  *
5  * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
6  *
7  * An nfsd_file object is a per-file collection of open state that binds
8  * together:
9  *   - a struct file *
10  *   - a user credential
11  *   - a network namespace
12  *   - a read-ahead context
13  *   - monitoring for writeback errors
14  *
15  * nfsd_file objects are reference-counted. Consumers acquire a new
16  * object via the nfsd_file_acquire API. They manage their interest in
17  * the acquired object, and hence the object's reference count, via
18  * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
19  * object:
20  *
21  *  * non-garbage-collected: When a consumer wants to precisely control
22  *    the lifetime of a file's open state, it acquires a non-garbage-
23  *    collected nfsd_file. The final nfsd_file_put releases the open
24  *    state immediately.
25  *
26  *  * garbage-collected: When a consumer does not control the lifetime
27  *    of open state, it acquires a garbage-collected nfsd_file. The
28  *    final nfsd_file_put allows the open state to linger for a period
29  *    during which it may be re-used.
30  */
31 
32 #include <linux/hash.h>
33 #include <linux/slab.h>
34 #include <linux/file.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched.h>
37 #include <linux/list_lru.h>
38 #include <linux/fsnotify_backend.h>
39 #include <linux/fsnotify.h>
40 #include <linux/seq_file.h>
41 #include <linux/rhashtable.h>
42 #include <linux/nfslocalio.h>
43 
44 #include "vfs.h"
45 #include "nfsd.h"
46 #include "nfsfh.h"
47 #include "netns.h"
48 #include "filecache.h"
49 #include "trace.h"
50 
51 #define NFSD_LAUNDRETTE_DELAY		     (2 * HZ)
52 
53 #define NFSD_FILE_CACHE_UP		     (0)
54 
55 /* We only care about NFSD_MAY_READ/WRITE for this cache */
56 #define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO)
57 
58 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
59 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
60 static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);
61 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
62 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
63 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
64 
65 struct nfsd_fcache_disposal {
66 	spinlock_t lock;
67 	struct list_head freeme;
68 };
69 
70 static struct kmem_cache		*nfsd_file_slab;
71 static struct kmem_cache		*nfsd_file_mark_slab;
72 static struct list_lru			nfsd_file_lru;
73 static unsigned long			nfsd_file_flags;
74 static struct fsnotify_group		*nfsd_file_fsnotify_group;
75 static struct delayed_work		nfsd_filecache_laundrette;
76 static struct rhltable			nfsd_file_rhltable
77 						____cacheline_aligned_in_smp;
78 
79 static bool
nfsd_match_cred(const struct cred * c1,const struct cred * c2)80 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
81 {
82 	int i;
83 
84 	if (!uid_eq(c1->fsuid, c2->fsuid))
85 		return false;
86 	if (!gid_eq(c1->fsgid, c2->fsgid))
87 		return false;
88 	if (c1->group_info == NULL || c2->group_info == NULL)
89 		return c1->group_info == c2->group_info;
90 	if (c1->group_info->ngroups != c2->group_info->ngroups)
91 		return false;
92 	for (i = 0; i < c1->group_info->ngroups; i++) {
93 		if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
94 			return false;
95 	}
96 	return true;
97 }
98 
99 static const struct rhashtable_params nfsd_file_rhash_params = {
100 	.key_len		= sizeof_field(struct nfsd_file, nf_inode),
101 	.key_offset		= offsetof(struct nfsd_file, nf_inode),
102 	.head_offset		= offsetof(struct nfsd_file, nf_rlist),
103 
104 	/*
105 	 * Start with a single page hash table to reduce resizing churn
106 	 * on light workloads.
107 	 */
108 	.min_size		= 256,
109 	.automatic_shrinking	= true,
110 };
111 
112 static void
nfsd_file_schedule_laundrette(void)113 nfsd_file_schedule_laundrette(void)
114 {
115 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
116 		queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette,
117 				   NFSD_LAUNDRETTE_DELAY);
118 }
119 
120 static void
nfsd_file_slab_free(struct rcu_head * rcu)121 nfsd_file_slab_free(struct rcu_head *rcu)
122 {
123 	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
124 
125 	put_cred(nf->nf_cred);
126 	kmem_cache_free(nfsd_file_slab, nf);
127 }
128 
129 static void
nfsd_file_mark_free(struct fsnotify_mark * mark)130 nfsd_file_mark_free(struct fsnotify_mark *mark)
131 {
132 	struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
133 						  nfm_mark);
134 
135 	kmem_cache_free(nfsd_file_mark_slab, nfm);
136 }
137 
138 static struct nfsd_file_mark *
nfsd_file_mark_get(struct nfsd_file_mark * nfm)139 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
140 {
141 	if (!refcount_inc_not_zero(&nfm->nfm_ref))
142 		return NULL;
143 	return nfm;
144 }
145 
146 static void
nfsd_file_mark_put(struct nfsd_file_mark * nfm)147 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
148 {
149 	if (refcount_dec_and_test(&nfm->nfm_ref)) {
150 		fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
151 		fsnotify_put_mark(&nfm->nfm_mark);
152 	}
153 }
154 
155 static struct nfsd_file_mark *
nfsd_file_mark_find_or_create(struct inode * inode)156 nfsd_file_mark_find_or_create(struct inode *inode)
157 {
158 	int			err;
159 	struct fsnotify_mark	*mark;
160 	struct nfsd_file_mark	*nfm = NULL, *new;
161 
162 	do {
163 		fsnotify_group_lock(nfsd_file_fsnotify_group);
164 		mark = fsnotify_find_inode_mark(inode,
165 						nfsd_file_fsnotify_group);
166 		if (mark) {
167 			nfm = nfsd_file_mark_get(container_of(mark,
168 						 struct nfsd_file_mark,
169 						 nfm_mark));
170 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
171 			if (nfm) {
172 				fsnotify_put_mark(mark);
173 				break;
174 			}
175 			/* Avoid soft lockup race with nfsd_file_mark_put() */
176 			fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
177 			fsnotify_put_mark(mark);
178 		} else {
179 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
180 		}
181 
182 		/* allocate a new nfm */
183 		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
184 		if (!new)
185 			return NULL;
186 		fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
187 		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
188 		refcount_set(&new->nfm_ref, 1);
189 
190 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
191 
192 		/*
193 		 * If the add was successful, then return the object.
194 		 * Otherwise, we need to put the reference we hold on the
195 		 * nfm_mark. The fsnotify code will take a reference and put
196 		 * it on failure, so we can't just free it directly. It's also
197 		 * not safe to call fsnotify_destroy_mark on it as the
198 		 * mark->group will be NULL. Thus, we can't let the nfm_ref
199 		 * counter drive the destruction at this point.
200 		 */
201 		if (likely(!err))
202 			nfm = new;
203 		else
204 			fsnotify_put_mark(&new->nfm_mark);
205 	} while (unlikely(err == -EEXIST));
206 
207 	return nfm;
208 }
209 
210 static struct nfsd_file *
nfsd_file_alloc(struct net * net,struct inode * inode,unsigned char need,bool want_gc)211 nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,
212 		bool want_gc)
213 {
214 	struct nfsd_file *nf;
215 
216 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
217 	if (unlikely(!nf))
218 		return NULL;
219 
220 	this_cpu_inc(nfsd_file_allocations);
221 	INIT_LIST_HEAD(&nf->nf_lru);
222 	INIT_LIST_HEAD(&nf->nf_gc);
223 	nf->nf_birthtime = ktime_get();
224 	nf->nf_file = NULL;
225 	nf->nf_cred = get_current_cred();
226 	nf->nf_net = net;
227 	nf->nf_flags = want_gc ?
228 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :
229 		BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);
230 	nf->nf_inode = inode;
231 	refcount_set(&nf->nf_ref, 1);
232 	nf->nf_may = need;
233 	nf->nf_mark = NULL;
234 	nf->nf_dio_mem_align = 0;
235 	nf->nf_dio_offset_align = 0;
236 	nf->nf_dio_read_offset_align = 0;
237 	return nf;
238 }
239 
240 /**
241  * nfsd_file_check_write_error - check for writeback errors on a file
242  * @nf: nfsd_file to check for writeback errors
243  *
244  * Check whether a nfsd_file has an unseen error. Reset the write
245  * verifier if so.
246  */
247 static void
nfsd_file_check_write_error(struct nfsd_file * nf)248 nfsd_file_check_write_error(struct nfsd_file *nf)
249 {
250 	struct file *file = nf->nf_file;
251 
252 	if ((file->f_mode & FMODE_WRITE) &&
253 	    filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
254 		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
255 }
256 
257 static void
nfsd_file_hash_remove(struct nfsd_file * nf)258 nfsd_file_hash_remove(struct nfsd_file *nf)
259 {
260 	trace_nfsd_file_unhash(nf);
261 	rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,
262 			nfsd_file_rhash_params);
263 }
264 
265 static bool
nfsd_file_unhash(struct nfsd_file * nf)266 nfsd_file_unhash(struct nfsd_file *nf)
267 {
268 	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
269 		nfsd_file_hash_remove(nf);
270 		return true;
271 	}
272 	return false;
273 }
274 
275 static void
nfsd_file_free(struct nfsd_file * nf)276 nfsd_file_free(struct nfsd_file *nf)
277 {
278 	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
279 
280 	trace_nfsd_file_free(nf);
281 
282 	this_cpu_inc(nfsd_file_releases);
283 	this_cpu_add(nfsd_file_total_age, age);
284 
285 	nfsd_file_unhash(nf);
286 	if (nf->nf_mark)
287 		nfsd_file_mark_put(nf->nf_mark);
288 	if (nf->nf_file) {
289 		nfsd_file_check_write_error(nf);
290 		nfsd_filp_close(nf->nf_file);
291 	}
292 
293 	/*
294 	 * If this item is still linked via nf_lru, that's a bug.
295 	 * WARN and leak it to preserve system stability.
296 	 */
297 	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
298 		return;
299 
300 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
301 }
302 
303 static bool
nfsd_file_check_writeback(struct nfsd_file * nf)304 nfsd_file_check_writeback(struct nfsd_file *nf)
305 {
306 	struct file *file = nf->nf_file;
307 	struct address_space *mapping;
308 
309 	/* File not open for write? */
310 	if (!(file->f_mode & FMODE_WRITE))
311 		return false;
312 
313 	/*
314 	 * Some filesystems (e.g. NFS) flush all dirty data on close.
315 	 * On others, there is no need to wait for writeback.
316 	 */
317 	if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))
318 		return false;
319 
320 	mapping = file->f_mapping;
321 	return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
322 		mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
323 }
324 
nfsd_file_lru_add(struct nfsd_file * nf)325 static void nfsd_file_lru_add(struct nfsd_file *nf)
326 {
327 	refcount_inc(&nf->nf_ref);
328 	if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru))
329 		trace_nfsd_file_lru_add(nf);
330 	else
331 		WARN_ON(1);
332 	nfsd_file_schedule_laundrette();
333 }
334 
nfsd_file_lru_remove(struct nfsd_file * nf)335 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
336 {
337 	if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) {
338 		trace_nfsd_file_lru_del(nf);
339 		return true;
340 	}
341 	return false;
342 }
343 
344 struct nfsd_file *
nfsd_file_get(struct nfsd_file * nf)345 nfsd_file_get(struct nfsd_file *nf)
346 {
347 	if (nf && refcount_inc_not_zero(&nf->nf_ref))
348 		return nf;
349 	return NULL;
350 }
351 
352 /**
353  * nfsd_file_put - put the reference to a nfsd_file
354  * @nf: nfsd_file of which to put the reference
355  *
356  * Put a reference to a nfsd_file. In the non-GC case, we just put the
357  * reference immediately. In the GC case, if the reference would be
358  * the last one, the put it on the LRU instead to be cleaned up later.
359  */
360 void
nfsd_file_put(struct nfsd_file * nf)361 nfsd_file_put(struct nfsd_file *nf)
362 {
363 	might_sleep();
364 	trace_nfsd_file_put(nf);
365 
366 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
367 	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
368 		set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
369 		set_bit(NFSD_FILE_RECENT, &nf->nf_flags);
370 	}
371 
372 	if (refcount_dec_and_test(&nf->nf_ref))
373 		nfsd_file_free(nf);
374 }
375 
376 /**
377  * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller
378  * @pnf: nfsd_file of which to put the reference
379  *
380  * First save the associated net to return to caller, then put
381  * the reference of the nfsd_file.
382  */
383 struct net *
nfsd_file_put_local(struct nfsd_file __rcu ** pnf)384 nfsd_file_put_local(struct nfsd_file __rcu **pnf)
385 {
386 	struct nfsd_file *nf;
387 	struct net *net = NULL;
388 
389 	nf = unrcu_pointer(xchg(pnf, NULL));
390 	if (nf) {
391 		net = nf->nf_net;
392 		nfsd_file_put(nf);
393 	}
394 	return net;
395 }
396 
397 /**
398  * nfsd_file_get_local - get nfsd_file reference and reference to net
399  * @nf: nfsd_file of which to put the reference
400  *
401  * Get reference to both the nfsd_file and nf->nf_net.
402  */
403 struct nfsd_file *
nfsd_file_get_local(struct nfsd_file * nf)404 nfsd_file_get_local(struct nfsd_file *nf)
405 {
406 	struct net *net = nf->nf_net;
407 
408 	if (nfsd_net_try_get(net)) {
409 		nf = nfsd_file_get(nf);
410 		if (!nf)
411 			nfsd_net_put(net);
412 	} else {
413 		nf = NULL;
414 	}
415 	return nf;
416 }
417 
418 /**
419  * nfsd_file_file - get the backing file of an nfsd_file
420  * @nf: nfsd_file of which to access the backing file.
421  *
422  * Return backing file for @nf.
423  */
424 struct file *
nfsd_file_file(struct nfsd_file * nf)425 nfsd_file_file(struct nfsd_file *nf)
426 {
427 	return nf->nf_file;
428 }
429 
430 static void
nfsd_file_dispose_list(struct list_head * dispose)431 nfsd_file_dispose_list(struct list_head *dispose)
432 {
433 	struct nfsd_file *nf;
434 
435 	while (!list_empty(dispose)) {
436 		nf = list_first_entry(dispose, struct nfsd_file, nf_gc);
437 		list_del_init(&nf->nf_gc);
438 		nfsd_file_free(nf);
439 	}
440 }
441 
442 /**
443  * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list
444  * @dispose: list of nfsd_files to be disposed
445  *
446  * Transfers each file to the "freeme" list for its nfsd_net, to eventually
447  * be disposed of by the per-net garbage collector.
448  */
449 static void
nfsd_file_dispose_list_delayed(struct list_head * dispose)450 nfsd_file_dispose_list_delayed(struct list_head *dispose)
451 {
452 	while(!list_empty(dispose)) {
453 		struct nfsd_file *nf = list_first_entry(dispose,
454 						struct nfsd_file, nf_gc);
455 		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
456 		struct nfsd_fcache_disposal *l = nn->fcache_disposal;
457 		struct svc_serv *serv;
458 
459 		spin_lock(&l->lock);
460 		list_move_tail(&nf->nf_gc, &l->freeme);
461 		spin_unlock(&l->lock);
462 
463 		/*
464 		 * The filecache laundrette is shut down after the
465 		 * nn->nfsd_serv pointer is cleared, but before the
466 		 * svc_serv is freed.
467 		 */
468 		serv = nn->nfsd_serv;
469 		if (serv)
470 			svc_wake_up(serv);
471 	}
472 }
473 
474 /**
475  * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed.
476  * @nn: nfsd_net in which to find files to be disposed.
477  *
478  * When files held open for nfsv3 are removed from the filecache, whether
479  * due to memory pressure or garbage collection, they are queued to
480  * a per-net-ns queue.  This function completes the disposal, either
481  * directly or by waking another nfsd thread to help with the work.
482  */
nfsd_file_net_dispose(struct nfsd_net * nn)483 void nfsd_file_net_dispose(struct nfsd_net *nn)
484 {
485 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
486 
487 	if (!list_empty(&l->freeme)) {
488 		LIST_HEAD(dispose);
489 		int i;
490 
491 		spin_lock(&l->lock);
492 		for (i = 0; i < 8 && !list_empty(&l->freeme); i++)
493 			list_move(l->freeme.next, &dispose);
494 		spin_unlock(&l->lock);
495 		if (!list_empty(&l->freeme))
496 			/* Wake up another thread to share the work
497 			 * *before* doing any actual disposing.
498 			 */
499 			svc_wake_up(nn->nfsd_serv);
500 		nfsd_file_dispose_list(&dispose);
501 	}
502 }
503 
504 /**
505  * nfsd_file_lru_cb - Examine an entry on the LRU list
506  * @item: LRU entry to examine
507  * @lru: controlling LRU
508  * @arg: dispose list
509  *
510  * Return values:
511  *   %LRU_REMOVED: @item was removed from the LRU
512  *   %LRU_ROTATE: @item is to be moved to the LRU tail
513  *   %LRU_SKIP: @item cannot be evicted
514  */
515 static enum lru_status
nfsd_file_lru_cb(struct list_head * item,struct list_lru_one * lru,void * arg)516 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
517 		 void *arg)
518 {
519 	struct list_head *head = arg;
520 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
521 
522 	/* We should only be dealing with GC entries here */
523 	WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
524 
525 	/*
526 	 * Don't throw out files that are still undergoing I/O or
527 	 * that have uncleared errors pending.
528 	 */
529 	if (nfsd_file_check_writeback(nf)) {
530 		trace_nfsd_file_gc_writeback(nf);
531 		return LRU_SKIP;
532 	}
533 
534 	/* If it was recently added to the list, skip it */
535 	if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
536 		trace_nfsd_file_gc_referenced(nf);
537 		return LRU_ROTATE;
538 	}
539 
540 	/*
541 	 * Put the reference held on behalf of the LRU if it is the last
542 	 * reference, else rotate.
543 	 */
544 	if (!refcount_dec_if_one(&nf->nf_ref)) {
545 		trace_nfsd_file_gc_in_use(nf);
546 		return LRU_ROTATE;
547 	}
548 
549 	/* Refcount went to zero. Unhash it and queue it to the dispose list */
550 	nfsd_file_unhash(nf);
551 	list_lru_isolate(lru, &nf->nf_lru);
552 	list_add(&nf->nf_gc, head);
553 	this_cpu_inc(nfsd_file_evictions);
554 	trace_nfsd_file_gc_disposed(nf);
555 	return LRU_REMOVED;
556 }
557 
558 static enum lru_status
nfsd_file_gc_cb(struct list_head * item,struct list_lru_one * lru,void * arg)559 nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru,
560 		 void *arg)
561 {
562 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
563 
564 	if (test_and_clear_bit(NFSD_FILE_RECENT, &nf->nf_flags)) {
565 		/*
566 		 * "REFERENCED" really means "should be at the end of the
567 		 * LRU. As we are putting it there we can clear the flag.
568 		 */
569 		clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
570 		trace_nfsd_file_gc_aged(nf);
571 		return LRU_ROTATE;
572 	}
573 	return nfsd_file_lru_cb(item, lru, arg);
574 }
575 
576 /* If the shrinker runs between calls to list_lru_walk_node() in
577  * nfsd_file_gc(), the "remaining" count will be wrong.  This could
578  * result in premature freeing of some files.  This may not matter much
579  * but is easy to fix with this spinlock which temporarily disables
580  * the shrinker.
581  */
582 static DEFINE_SPINLOCK(nfsd_gc_lock);
583 static void
nfsd_file_gc(void)584 nfsd_file_gc(void)
585 {
586 	unsigned long ret = 0;
587 	LIST_HEAD(dispose);
588 	int nid;
589 
590 	spin_lock(&nfsd_gc_lock);
591 	for_each_node_state(nid, N_NORMAL_MEMORY) {
592 		unsigned long remaining = list_lru_count_node(&nfsd_file_lru, nid);
593 
594 		while (remaining > 0) {
595 			unsigned long nr = min(remaining, NFSD_FILE_GC_BATCH);
596 
597 			remaining -= nr;
598 			ret += list_lru_walk_node(&nfsd_file_lru, nid, nfsd_file_gc_cb,
599 						  &dispose, &nr);
600 			if (nr)
601 				/* walk aborted early */
602 				remaining = 0;
603 		}
604 	}
605 	spin_unlock(&nfsd_gc_lock);
606 	trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
607 	nfsd_file_dispose_list_delayed(&dispose);
608 }
609 
610 static void
nfsd_file_gc_worker(struct work_struct * work)611 nfsd_file_gc_worker(struct work_struct *work)
612 {
613 	if (list_lru_count(&nfsd_file_lru))
614 		nfsd_file_gc();
615 	nfsd_file_schedule_laundrette();
616 }
617 
618 static unsigned long
nfsd_file_lru_count(struct shrinker * s,struct shrink_control * sc)619 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
620 {
621 	return list_lru_count(&nfsd_file_lru);
622 }
623 
624 static unsigned long
nfsd_file_lru_scan(struct shrinker * s,struct shrink_control * sc)625 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
626 {
627 	LIST_HEAD(dispose);
628 	unsigned long ret;
629 
630 	if (!spin_trylock(&nfsd_gc_lock))
631 		return SHRINK_STOP;
632 
633 	ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
634 				   nfsd_file_lru_cb, &dispose);
635 	spin_unlock(&nfsd_gc_lock);
636 	trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
637 	nfsd_file_dispose_list_delayed(&dispose);
638 	return ret;
639 }
640 
641 static struct shrinker *nfsd_file_shrinker;
642 
643 /**
644  * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
645  * @nf: nfsd_file to attempt to queue
646  * @dispose: private list to queue successfully-put objects
647  *
648  * Unhash an nfsd_file, try to get a reference to it, and then put that
649  * reference. If it's the last reference, queue it to the dispose list.
650  */
651 static void
nfsd_file_cond_queue(struct nfsd_file * nf,struct list_head * dispose)652 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
653 	__must_hold(RCU)
654 {
655 	int decrement = 1;
656 
657 	/* If we raced with someone else unhashing, ignore it */
658 	if (!nfsd_file_unhash(nf))
659 		return;
660 
661 	/* If we can't get a reference, ignore it */
662 	if (!nfsd_file_get(nf))
663 		return;
664 
665 	/* Extra decrement if we remove from the LRU */
666 	if (nfsd_file_lru_remove(nf))
667 		++decrement;
668 
669 	/* If refcount goes to 0, then put on the dispose list */
670 	if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
671 		list_add(&nf->nf_gc, dispose);
672 		trace_nfsd_file_closing(nf);
673 	}
674 }
675 
676 /**
677  * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
678  * @inode:   inode on which to close out nfsd_files
679  * @dispose: list on which to gather nfsd_files to close out
680  *
681  * An nfsd_file represents a struct file being held open on behalf of nfsd.
682  * An open file however can block other activity (such as leases), or cause
683  * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
684  *
685  * This function is intended to find open nfsd_files when this sort of
686  * conflicting access occurs and then attempt to close those files out.
687  *
688  * Populates the dispose list with entries that have already had their
689  * refcounts go to zero. The actual free of an nfsd_file can be expensive,
690  * so we leave it up to the caller whether it wants to wait or not.
691  */
692 static void
nfsd_file_queue_for_close(struct inode * inode,struct list_head * dispose)693 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
694 {
695 	struct rhlist_head *tmp, *list;
696 	struct nfsd_file *nf;
697 
698 	rcu_read_lock();
699 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
700 			       nfsd_file_rhash_params);
701 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
702 		if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))
703 			continue;
704 		nfsd_file_cond_queue(nf, dispose);
705 	}
706 	rcu_read_unlock();
707 }
708 
709 /**
710  * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
711  * @inode: inode of the file to attempt to remove
712  *
713  * Close out any open nfsd_files that can be reaped for @inode. The
714  * actual freeing is deferred to the dispose_list_delayed infrastructure.
715  *
716  * This is used by the fsnotify callbacks and setlease notifier.
717  */
718 static void
nfsd_file_close_inode(struct inode * inode)719 nfsd_file_close_inode(struct inode *inode)
720 {
721 	LIST_HEAD(dispose);
722 
723 	nfsd_file_queue_for_close(inode, &dispose);
724 	nfsd_file_dispose_list_delayed(&dispose);
725 }
726 
727 /**
728  * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
729  * @inode: inode of the file to attempt to remove
730  *
731  * Close out any open nfsd_files that can be reaped for @inode. The
732  * nfsd_files are closed out synchronously.
733  *
734  * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
735  * when reexporting NFS.
736  */
737 void
nfsd_file_close_inode_sync(struct inode * inode)738 nfsd_file_close_inode_sync(struct inode *inode)
739 {
740 	LIST_HEAD(dispose);
741 
742 	trace_nfsd_file_close(inode);
743 
744 	nfsd_file_queue_for_close(inode, &dispose);
745 	nfsd_file_dispose_list(&dispose);
746 }
747 
748 static int
nfsd_file_lease_notifier_call(struct notifier_block * nb,unsigned long arg,void * data)749 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
750 			    void *data)
751 {
752 	struct file_lease *fl = data;
753 
754 	/* Only close files for F_SETLEASE leases */
755 	if (fl->c.flc_flags & FL_LEASE)
756 		nfsd_file_close_inode(file_inode(fl->c.flc_file));
757 	return 0;
758 }
759 
760 static struct notifier_block nfsd_file_lease_notifier = {
761 	.notifier_call = nfsd_file_lease_notifier_call,
762 };
763 
764 static int
nfsd_file_fsnotify_handle_event(struct fsnotify_mark * mark,u32 mask,struct inode * inode,struct inode * dir,const struct qstr * name,u32 cookie)765 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
766 				struct inode *inode, struct inode *dir,
767 				const struct qstr *name, u32 cookie)
768 {
769 	if (WARN_ON_ONCE(!inode))
770 		return 0;
771 
772 	trace_nfsd_file_fsnotify_handle_event(inode, mask);
773 
774 	/* Should be no marks on non-regular files */
775 	if (!S_ISREG(inode->i_mode)) {
776 		WARN_ON_ONCE(1);
777 		return 0;
778 	}
779 
780 	/* don't close files if this was not the last link */
781 	if (mask & FS_ATTRIB) {
782 		if (inode->i_nlink)
783 			return 0;
784 	}
785 
786 	nfsd_file_close_inode(inode);
787 	return 0;
788 }
789 
790 
791 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
792 	.handle_inode_event = nfsd_file_fsnotify_handle_event,
793 	.free_mark = nfsd_file_mark_free,
794 };
795 
796 int
nfsd_file_cache_init(void)797 nfsd_file_cache_init(void)
798 {
799 	int ret;
800 
801 	lockdep_assert_held(&nfsd_mutex);
802 	if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
803 		return 0;
804 
805 	ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params);
806 	if (ret)
807 		goto out;
808 
809 	ret = -ENOMEM;
810 	nfsd_file_slab = KMEM_CACHE(nfsd_file, 0);
811 	if (!nfsd_file_slab) {
812 		pr_err("nfsd: unable to create nfsd_file_slab\n");
813 		goto out_err;
814 	}
815 
816 	nfsd_file_mark_slab = KMEM_CACHE(nfsd_file_mark, 0);
817 	if (!nfsd_file_mark_slab) {
818 		pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
819 		goto out_err;
820 	}
821 
822 	ret = list_lru_init(&nfsd_file_lru);
823 	if (ret) {
824 		pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
825 		goto out_err;
826 	}
827 
828 	nfsd_file_shrinker = shrinker_alloc(0, "nfsd-filecache");
829 	if (!nfsd_file_shrinker) {
830 		ret = -ENOMEM;
831 		pr_err("nfsd: failed to allocate nfsd_file_shrinker\n");
832 		goto out_lru;
833 	}
834 
835 	nfsd_file_shrinker->count_objects = nfsd_file_lru_count;
836 	nfsd_file_shrinker->scan_objects = nfsd_file_lru_scan;
837 	nfsd_file_shrinker->seeks = 1;
838 
839 	shrinker_register(nfsd_file_shrinker);
840 
841 	ret = lease_register_notifier(&nfsd_file_lease_notifier);
842 	if (ret) {
843 		pr_err("nfsd: unable to register lease notifier: %d\n", ret);
844 		goto out_shrinker;
845 	}
846 
847 	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
848 							0);
849 	if (IS_ERR(nfsd_file_fsnotify_group)) {
850 		pr_err("nfsd: unable to create fsnotify group: %ld\n",
851 			PTR_ERR(nfsd_file_fsnotify_group));
852 		ret = PTR_ERR(nfsd_file_fsnotify_group);
853 		nfsd_file_fsnotify_group = NULL;
854 		goto out_notifier;
855 	}
856 
857 	INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
858 out:
859 	if (ret)
860 		clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags);
861 	return ret;
862 out_notifier:
863 	lease_unregister_notifier(&nfsd_file_lease_notifier);
864 out_shrinker:
865 	shrinker_free(nfsd_file_shrinker);
866 out_lru:
867 	list_lru_destroy(&nfsd_file_lru);
868 out_err:
869 	kmem_cache_destroy(nfsd_file_slab);
870 	nfsd_file_slab = NULL;
871 	kmem_cache_destroy(nfsd_file_mark_slab);
872 	nfsd_file_mark_slab = NULL;
873 	rhltable_destroy(&nfsd_file_rhltable);
874 	goto out;
875 }
876 
877 /**
878  * __nfsd_file_cache_purge: clean out the cache for shutdown
879  * @net: net-namespace to shut down the cache (may be NULL)
880  *
881  * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
882  * then close out everything. Called when an nfsd instance is being shut down,
883  * and when the exports table is flushed.
884  */
885 static void
__nfsd_file_cache_purge(struct net * net)886 __nfsd_file_cache_purge(struct net *net)
887 {
888 	struct rhashtable_iter iter;
889 	struct nfsd_file *nf;
890 	LIST_HEAD(dispose);
891 
892 #if IS_ENABLED(CONFIG_NFS_LOCALIO)
893 	if (net) {
894 		struct nfsd_net *nn = net_generic(net, nfsd_net_id);
895 		nfs_localio_invalidate_clients(&nn->local_clients,
896 					       &nn->local_clients_lock);
897 	}
898 #endif
899 
900 	rhltable_walk_enter(&nfsd_file_rhltable, &iter);
901 	do {
902 		rhashtable_walk_start(&iter);
903 
904 		nf = rhashtable_walk_next(&iter);
905 		while (!IS_ERR_OR_NULL(nf)) {
906 			if (!net || nf->nf_net == net)
907 				nfsd_file_cond_queue(nf, &dispose);
908 			nf = rhashtable_walk_next(&iter);
909 		}
910 
911 		rhashtable_walk_stop(&iter);
912 	} while (nf == ERR_PTR(-EAGAIN));
913 	rhashtable_walk_exit(&iter);
914 
915 	nfsd_file_dispose_list(&dispose);
916 }
917 
918 static struct nfsd_fcache_disposal *
nfsd_alloc_fcache_disposal(void)919 nfsd_alloc_fcache_disposal(void)
920 {
921 	struct nfsd_fcache_disposal *l;
922 
923 	l = kmalloc(sizeof(*l), GFP_KERNEL);
924 	if (!l)
925 		return NULL;
926 	spin_lock_init(&l->lock);
927 	INIT_LIST_HEAD(&l->freeme);
928 	return l;
929 }
930 
931 static void
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal * l)932 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
933 {
934 	nfsd_file_dispose_list(&l->freeme);
935 	kfree(l);
936 }
937 
938 static void
nfsd_free_fcache_disposal_net(struct net * net)939 nfsd_free_fcache_disposal_net(struct net *net)
940 {
941 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
942 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
943 
944 	nfsd_free_fcache_disposal(l);
945 }
946 
947 int
nfsd_file_cache_start_net(struct net * net)948 nfsd_file_cache_start_net(struct net *net)
949 {
950 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
951 
952 	nn->fcache_disposal = nfsd_alloc_fcache_disposal();
953 	return nn->fcache_disposal ? 0 : -ENOMEM;
954 }
955 
956 /**
957  * nfsd_file_cache_purge - Remove all cache items associated with @net
958  * @net: target net namespace
959  *
960  */
961 void
nfsd_file_cache_purge(struct net * net)962 nfsd_file_cache_purge(struct net *net)
963 {
964 	lockdep_assert_held(&nfsd_mutex);
965 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
966 		__nfsd_file_cache_purge(net);
967 }
968 
969 void
nfsd_file_cache_shutdown_net(struct net * net)970 nfsd_file_cache_shutdown_net(struct net *net)
971 {
972 	nfsd_file_cache_purge(net);
973 	nfsd_free_fcache_disposal_net(net);
974 }
975 
976 void
nfsd_file_cache_shutdown(void)977 nfsd_file_cache_shutdown(void)
978 {
979 	int i;
980 
981 	lockdep_assert_held(&nfsd_mutex);
982 	if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
983 		return;
984 
985 	lease_unregister_notifier(&nfsd_file_lease_notifier);
986 	shrinker_free(nfsd_file_shrinker);
987 	/*
988 	 * make sure all callers of nfsd_file_lru_cb are done before
989 	 * calling nfsd_file_cache_purge
990 	 */
991 	cancel_delayed_work_sync(&nfsd_filecache_laundrette);
992 	__nfsd_file_cache_purge(NULL);
993 	list_lru_destroy(&nfsd_file_lru);
994 	rcu_barrier();
995 	fsnotify_put_group(nfsd_file_fsnotify_group);
996 	nfsd_file_fsnotify_group = NULL;
997 	kmem_cache_destroy(nfsd_file_slab);
998 	nfsd_file_slab = NULL;
999 	fsnotify_wait_marks_destroyed();
1000 	kmem_cache_destroy(nfsd_file_mark_slab);
1001 	nfsd_file_mark_slab = NULL;
1002 	rhltable_destroy(&nfsd_file_rhltable);
1003 
1004 	for_each_possible_cpu(i) {
1005 		per_cpu(nfsd_file_cache_hits, i) = 0;
1006 		per_cpu(nfsd_file_acquisitions, i) = 0;
1007 		per_cpu(nfsd_file_allocations, i) = 0;
1008 		per_cpu(nfsd_file_releases, i) = 0;
1009 		per_cpu(nfsd_file_total_age, i) = 0;
1010 		per_cpu(nfsd_file_evictions, i) = 0;
1011 	}
1012 }
1013 
1014 static struct nfsd_file *
nfsd_file_lookup_locked(const struct net * net,const struct cred * cred,struct inode * inode,unsigned char need,bool want_gc)1015 nfsd_file_lookup_locked(const struct net *net, const struct cred *cred,
1016 			struct inode *inode, unsigned char need,
1017 			bool want_gc)
1018 {
1019 	struct rhlist_head *tmp, *list;
1020 	struct nfsd_file *nf;
1021 
1022 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
1023 			       nfsd_file_rhash_params);
1024 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
1025 		if (nf->nf_may != need)
1026 			continue;
1027 		if (nf->nf_net != net)
1028 			continue;
1029 		if (!nfsd_match_cred(nf->nf_cred, cred))
1030 			continue;
1031 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)
1032 			continue;
1033 		if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
1034 			continue;
1035 
1036 		if (!nfsd_file_get(nf))
1037 			continue;
1038 		return nf;
1039 	}
1040 	return NULL;
1041 }
1042 
1043 /**
1044  * nfsd_file_is_cached - are there any cached open files for this inode?
1045  * @inode: inode to check
1046  *
1047  * The lookup matches inodes in all net namespaces and is atomic wrt
1048  * nfsd_file_acquire().
1049  *
1050  * Return values:
1051  *   %true: filecache contains at least one file matching this inode
1052  *   %false: filecache contains no files matching this inode
1053  */
1054 bool
nfsd_file_is_cached(struct inode * inode)1055 nfsd_file_is_cached(struct inode *inode)
1056 {
1057 	struct rhlist_head *tmp, *list;
1058 	struct nfsd_file *nf;
1059 	bool ret = false;
1060 
1061 	rcu_read_lock();
1062 	list = rhltable_lookup(&nfsd_file_rhltable, &inode,
1063 			       nfsd_file_rhash_params);
1064 	rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)
1065 		if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
1066 			ret = true;
1067 			break;
1068 		}
1069 	rcu_read_unlock();
1070 
1071 	trace_nfsd_file_is_cached(inode, (int)ret);
1072 	return ret;
1073 }
1074 
1075 static __be32
nfsd_file_get_dio_attrs(const struct svc_fh * fhp,struct nfsd_file * nf)1076 nfsd_file_get_dio_attrs(const struct svc_fh *fhp, struct nfsd_file *nf)
1077 {
1078 	struct inode *inode = file_inode(nf->nf_file);
1079 	struct kstat stat;
1080 	__be32 status;
1081 
1082 	/* Currently only need to get DIO alignment info for regular files */
1083 	if (!S_ISREG(inode->i_mode))
1084 		return nfs_ok;
1085 
1086 	status = fh_getattr(fhp, &stat);
1087 	if (status != nfs_ok)
1088 		return status;
1089 
1090 	trace_nfsd_file_get_dio_attrs(inode, &stat);
1091 
1092 	if (stat.result_mask & STATX_DIOALIGN) {
1093 		nf->nf_dio_mem_align = stat.dio_mem_align;
1094 		nf->nf_dio_offset_align = stat.dio_offset_align;
1095 	}
1096 	if (stat.result_mask & STATX_DIO_READ_ALIGN)
1097 		nf->nf_dio_read_offset_align = stat.dio_read_offset_align;
1098 	else
1099 		nf->nf_dio_read_offset_align = nf->nf_dio_offset_align;
1100 
1101 	return nfs_ok;
1102 }
1103 
1104 static __be32
nfsd_file_do_acquire(struct svc_rqst * rqstp,struct net * net,struct svc_cred * cred,struct auth_domain * client,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf,bool want_gc)1105 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net,
1106 		     struct svc_cred *cred,
1107 		     struct auth_domain *client,
1108 		     struct svc_fh *fhp,
1109 		     unsigned int may_flags, struct file *file,
1110 		     struct nfsd_file **pnf, bool want_gc)
1111 {
1112 	unsigned char need = may_flags & NFSD_FILE_MAY_MASK;
1113 	struct nfsd_file *new, *nf;
1114 	bool stale_retry = true;
1115 	bool open_retry = true;
1116 	struct inode *inode;
1117 	__be32 status;
1118 	int ret;
1119 
1120 retry:
1121 	if (rqstp) {
1122 		status = fh_verify(rqstp, fhp, S_IFREG,
1123 				   may_flags|NFSD_MAY_OWNER_OVERRIDE);
1124 	} else {
1125 		status = fh_verify_local(net, cred, client, fhp, S_IFREG,
1126 					 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1127 	}
1128 	if (status != nfs_ok)
1129 		return status;
1130 	inode = d_inode(fhp->fh_dentry);
1131 
1132 	rcu_read_lock();
1133 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1134 	rcu_read_unlock();
1135 
1136 	if (nf)
1137 		goto wait_for_construction;
1138 
1139 	new = nfsd_file_alloc(net, inode, need, want_gc);
1140 	if (!new) {
1141 		status = nfserr_jukebox;
1142 		goto out;
1143 	}
1144 
1145 	rcu_read_lock();
1146 	spin_lock(&inode->i_lock);
1147 	nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1148 	if (unlikely(nf)) {
1149 		spin_unlock(&inode->i_lock);
1150 		rcu_read_unlock();
1151 		nfsd_file_free(new);
1152 		goto wait_for_construction;
1153 	}
1154 	nf = new;
1155 	ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,
1156 			      nfsd_file_rhash_params);
1157 	spin_unlock(&inode->i_lock);
1158 	rcu_read_unlock();
1159 	if (likely(ret == 0))
1160 		goto open_file;
1161 
1162 	trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret);
1163 	status = nfserr_jukebox;
1164 	goto construction_err;
1165 
1166 wait_for_construction:
1167 	wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1168 
1169 	/* Did construction of this file fail? */
1170 	if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1171 		trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);
1172 		if (!open_retry) {
1173 			status = nfserr_jukebox;
1174 			goto construction_err;
1175 		}
1176 		nfsd_file_put(nf);
1177 		open_retry = false;
1178 		fh_put(fhp);
1179 		goto retry;
1180 	}
1181 	this_cpu_inc(nfsd_file_cache_hits);
1182 
1183 	status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1184 	if (status != nfs_ok) {
1185 		nfsd_file_put(nf);
1186 		nf = NULL;
1187 	}
1188 
1189 out:
1190 	if (status == nfs_ok) {
1191 		this_cpu_inc(nfsd_file_acquisitions);
1192 		nfsd_file_check_write_error(nf);
1193 		*pnf = nf;
1194 	}
1195 	trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);
1196 	return status;
1197 
1198 open_file:
1199 	trace_nfsd_file_alloc(nf);
1200 	nf->nf_mark = nfsd_file_mark_find_or_create(inode);
1201 	if (nf->nf_mark) {
1202 		if (file) {
1203 			get_file(file);
1204 			nf->nf_file = file;
1205 			status = nfs_ok;
1206 			trace_nfsd_file_opened(nf, status);
1207 		} else {
1208 			ret = nfsd_open_verified(fhp, may_flags, &nf->nf_file);
1209 			if (ret == -EOPENSTALE && stale_retry) {
1210 				stale_retry = false;
1211 				nfsd_file_unhash(nf);
1212 				clear_and_wake_up_bit(NFSD_FILE_PENDING,
1213 						      &nf->nf_flags);
1214 				if (refcount_dec_and_test(&nf->nf_ref))
1215 					nfsd_file_free(nf);
1216 				nf = NULL;
1217 				fh_put(fhp);
1218 				goto retry;
1219 			}
1220 			status = nfserrno(ret);
1221 			trace_nfsd_file_open(nf, status);
1222 			if (status == nfs_ok)
1223 				status = nfsd_file_get_dio_attrs(fhp, nf);
1224 		}
1225 	} else
1226 		status = nfserr_jukebox;
1227 	/*
1228 	 * If construction failed, or we raced with a call to unlink()
1229 	 * then unhash.
1230 	 */
1231 	if (status != nfs_ok || inode->i_nlink == 0)
1232 		nfsd_file_unhash(nf);
1233 	else if (want_gc)
1234 		nfsd_file_lru_add(nf);
1235 
1236 	clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);
1237 	if (status == nfs_ok)
1238 		goto out;
1239 
1240 construction_err:
1241 	if (refcount_dec_and_test(&nf->nf_ref))
1242 		nfsd_file_free(nf);
1243 	nf = NULL;
1244 	goto out;
1245 }
1246 
1247 /**
1248  * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1249  * @rqstp: the RPC transaction being executed
1250  * @fhp: the NFS filehandle of the file to be opened
1251  * @may_flags: NFSD_MAY_ settings for the file
1252  * @pnf: OUT: new or found "struct nfsd_file" object
1253  *
1254  * The nfsd_file object returned by this API is reference-counted
1255  * and garbage-collected. The object is retained for a few
1256  * seconds after the final nfsd_file_put() in case the caller
1257  * wants to re-use it.
1258  *
1259  * Return values:
1260  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1261  *   count boosted.
1262  *
1263  * On error, an nfsstat value in network byte order is returned.
1264  */
1265 __be32
nfsd_file_acquire_gc(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1266 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1267 		     unsigned int may_flags, struct nfsd_file **pnf)
1268 {
1269 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1270 				    fhp, may_flags, NULL, pnf, true);
1271 }
1272 
1273 /**
1274  * nfsd_file_acquire - Get a struct nfsd_file with an open file
1275  * @rqstp: the RPC transaction being executed
1276  * @fhp: the NFS filehandle of the file to be opened
1277  * @may_flags: NFSD_MAY_ settings for the file
1278  * @pnf: OUT: new or found "struct nfsd_file" object
1279  *
1280  * The nfsd_file_object returned by this API is reference-counted
1281  * but not garbage-collected. The object is unhashed after the
1282  * final nfsd_file_put().
1283  *
1284  * Return values:
1285  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1286  *   count boosted.
1287  *
1288  * On error, an nfsstat value in network byte order is returned.
1289  */
1290 __be32
nfsd_file_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1291 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1292 		  unsigned int may_flags, struct nfsd_file **pnf)
1293 {
1294 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1295 				    fhp, may_flags, NULL, pnf, false);
1296 }
1297 
1298 /**
1299  * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio
1300  * @net: The network namespace in which to perform a lookup
1301  * @cred: the user credential with which to validate access
1302  * @client: the auth_domain for LOCALIO lookup
1303  * @fhp: the NFS filehandle of the file to be opened
1304  * @may_flags: NFSD_MAY_ settings for the file
1305  * @pnf: OUT: new or found "struct nfsd_file" object
1306  *
1307  * This file lookup interface provide access to a file given the
1308  * filehandle and credential.  No connection-based authorisation
1309  * is performed and in that way it is quite different to other
1310  * file access mediated by nfsd.  It allows a kernel module such as the NFS
1311  * client to reach across network and filesystem namespaces to access
1312  * a file.  The security implications of this should be carefully
1313  * considered before use.
1314  *
1315  * The nfsd_file_object returned by this API is reference-counted
1316  * but not garbage-collected. The object is unhashed after the
1317  * final nfsd_file_put().
1318  *
1319  * Return values:
1320  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1321  *   count boosted.
1322  *
1323  * On error, an nfsstat value in network byte order is returned.
1324  */
1325 __be32
nfsd_file_acquire_local(struct net * net,struct svc_cred * cred,struct auth_domain * client,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1326 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred,
1327 			struct auth_domain *client, struct svc_fh *fhp,
1328 			unsigned int may_flags, struct nfsd_file **pnf)
1329 {
1330 	/*
1331 	 * Save creds before calling nfsd_file_do_acquire() (which calls
1332 	 * nfsd_setuser). Important because caller (LOCALIO) is from
1333 	 * client context.
1334 	 */
1335 	const struct cred *save_cred = get_current_cred();
1336 	__be32 beres;
1337 
1338 	beres = nfsd_file_do_acquire(NULL, net, cred, client,
1339 				     fhp, may_flags, NULL, pnf, false);
1340 	put_cred(revert_creds(save_cred));
1341 	return beres;
1342 }
1343 
1344 /**
1345  * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1346  * @rqstp: the RPC transaction being executed
1347  * @fhp: the NFS filehandle of the file just created
1348  * @may_flags: NFSD_MAY_ settings for the file
1349  * @file: cached, already-open file (may be NULL)
1350  * @pnf: OUT: new or found "struct nfsd_file" object
1351  *
1352  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1353  * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1354  * opening a new one.
1355  *
1356  * Return values:
1357  *   %nfs_ok - @pnf points to an nfsd_file with its reference
1358  *   count boosted.
1359  *
1360  * On error, an nfsstat value in network byte order is returned.
1361  */
1362 __be32
nfsd_file_acquire_opened(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf)1363 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1364 			 unsigned int may_flags, struct file *file,
1365 			 struct nfsd_file **pnf)
1366 {
1367 	return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1368 				    fhp, may_flags, file, pnf, false);
1369 }
1370 
1371 /*
1372  * Note that fields may be added, removed or reordered in the future. Programs
1373  * scraping this file for info should test the labels to ensure they're
1374  * getting the correct field.
1375  */
nfsd_file_cache_stats_show(struct seq_file * m,void * v)1376 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1377 {
1378 	unsigned long allocations = 0, releases = 0, evictions = 0;
1379 	unsigned long hits = 0, acquisitions = 0;
1380 	unsigned int i, count = 0, buckets = 0;
1381 	unsigned long lru = 0, total_age = 0;
1382 
1383 	/* Serialize with server shutdown */
1384 	mutex_lock(&nfsd_mutex);
1385 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1386 		struct bucket_table *tbl;
1387 		struct rhashtable *ht;
1388 
1389 		lru = list_lru_count(&nfsd_file_lru);
1390 
1391 		rcu_read_lock();
1392 		ht = &nfsd_file_rhltable.ht;
1393 		count = atomic_read(&ht->nelems);
1394 		tbl = rht_dereference_rcu(ht->tbl, ht);
1395 		buckets = tbl->size;
1396 		rcu_read_unlock();
1397 	}
1398 	mutex_unlock(&nfsd_mutex);
1399 
1400 	for_each_possible_cpu(i) {
1401 		hits += per_cpu(nfsd_file_cache_hits, i);
1402 		acquisitions += per_cpu(nfsd_file_acquisitions, i);
1403 		allocations += per_cpu(nfsd_file_allocations, i);
1404 		releases += per_cpu(nfsd_file_releases, i);
1405 		total_age += per_cpu(nfsd_file_total_age, i);
1406 		evictions += per_cpu(nfsd_file_evictions, i);
1407 	}
1408 
1409 	seq_printf(m, "total inodes:  %u\n", count);
1410 	seq_printf(m, "hash buckets:  %u\n", buckets);
1411 	seq_printf(m, "lru entries:   %lu\n", lru);
1412 	seq_printf(m, "cache hits:    %lu\n", hits);
1413 	seq_printf(m, "acquisitions:  %lu\n", acquisitions);
1414 	seq_printf(m, "allocations:   %lu\n", allocations);
1415 	seq_printf(m, "releases:      %lu\n", releases);
1416 	seq_printf(m, "evictions:     %lu\n", evictions);
1417 	if (releases)
1418 		seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1419 	else
1420 		seq_printf(m, "mean age (ms): -\n");
1421 	return 0;
1422 }
1423