1 // SPDX-License-Identifier: GPL-2.0
4  * change in the future and be a per-client cache.
64  * XXX: these limits are per-container, so memory used will increase
71 	unsigned long low_pages = totalram_pages() - totalhigh_pages();  in nfsd_cache_size_limit()
73 	limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10);  in nfsd_cache_size_limit()
95 		rp->c_state = RC_UNUSED;  in nfsd_cacherep_alloc()
96 		rp->c_type = RC_NOCACHE;  in nfsd_cacherep_alloc()
97 		RB_CLEAR_NODE(&rp->c_node);  in nfsd_cacherep_alloc()
98 		INIT_LIST_HEAD(&rp->c_lru);  in nfsd_cacherep_alloc()
100 		memset(&rp->c_key, 0, sizeof(rp->c_key));  in nfsd_cacherep_alloc()
101 		rp->c_key.k_xid = rqstp->rq_xid;  in nfsd_cacherep_alloc()
102 		rp->c_key.k_proc = rqstp->rq_proc;  in nfsd_cacherep_alloc()
103 		rpc_copy_addr((struct sockaddr *)&rp->c_key.k_addr, svc_addr(rqstp));  in nfsd_cacherep_alloc()
104 		rpc_set_port((struct sockaddr *)&rp->c_key.k_addr, rpc_get_port(svc_addr(rqstp)));  in nfsd_cacherep_alloc()
105 		rp->c_key.k_prot = rqstp->rq_prot;  in nfsd_cacherep_alloc()
106 		rp->c_key.k_vers = rqstp->rq_vers;  in nfsd_cacherep_alloc()
107 		rp->c_key.k_len = rqstp->rq_arg.len;  in nfsd_cacherep_alloc()
108 		rp->c_key.k_csum = csum;  in nfsd_cacherep_alloc()
115 	if (rp->c_type == RC_REPLBUFF)  in nfsd_cacherep_free()
116 		kfree(rp->c_replvec.iov_base);  in nfsd_cacherep_free()
128 		list_del(&rp->c_lru);  in nfsd_cacherep_dispose()
139 	if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base)  in nfsd_cacherep_unlink_locked()
140 		nfsd_stats_drc_mem_usage_sub(nn, rp->c_replvec.iov_len);  in nfsd_cacherep_unlink_locked()
141 	if (rp->c_state != RC_UNUSED) {  in nfsd_cacherep_unlink_locked()
142 		rb_erase(&rp->c_node, &b->rb_head);  in nfsd_cacherep_unlink_locked()
143 		list_del(&rp->c_lru);  in nfsd_cacherep_unlink_locked()
144 		atomic_dec(&nn->num_drc_entries);  in nfsd_cacherep_unlink_locked()
161 	spin_lock(&b->cache_lock);  in nfsd_reply_cache_free()
163 	spin_unlock(&b->cache_lock);  in nfsd_reply_cache_free()
170 	return drc_slab ? 0: -ENOMEM;  in nfsd_drc_slab_create()
183 	nn->max_drc_entries = nfsd_cache_size_limit();  in nfsd_reply_cache_init()
184 	atomic_set(&nn->num_drc_entries, 0);  in nfsd_reply_cache_init()
185 	hashsize = nfsd_hashsize(nn->max_drc_entries);  in nfsd_reply_cache_init()
186 	nn->maskbits = ilog2(hashsize);  in nfsd_reply_cache_init()
188 	nn->drc_hashtbl = kvzalloc(array_size(hashsize,  in nfsd_reply_cache_init()
189 				sizeof(*nn->drc_hashtbl)), GFP_KERNEL);  in nfsd_reply_cache_init()
190 	if (!nn->drc_hashtbl)  in nfsd_reply_cache_init()
191 		return -ENOMEM;  in nfsd_reply_cache_init()
193 	nn->nfsd_reply_cache_shrinker = shrinker_alloc(0, "nfsd-reply:%s",  in nfsd_reply_cache_init()
194 						       nn->nfsd_name);  in nfsd_reply_cache_init()
195 	if (!nn->nfsd_reply_cache_shrinker)  in nfsd_reply_cache_init()
198 	nn->nfsd_reply_cache_shrinker->scan_objects = nfsd_reply_cache_scan;  in nfsd_reply_cache_init()
199 	nn->nfsd_reply_cache_shrinker->count_objects = nfsd_reply_cache_count;  in nfsd_reply_cache_init()
200 	nn->nfsd_reply_cache_shrinker->seeks = 1;  in nfsd_reply_cache_init()
201 	nn->nfsd_reply_cache_shrinker->private_data = nn;  in nfsd_reply_cache_init()
203 	shrinker_register(nn->nfsd_reply_cache_shrinker);  in nfsd_reply_cache_init()
206 		INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head);  in nfsd_reply_cache_init()
207 		spin_lock_init(&nn->drc_hashtbl[i].cache_lock);  in nfsd_reply_cache_init()
209 	nn->drc_hashsize = hashsize;  in nfsd_reply_cache_init()
213 	kvfree(nn->drc_hashtbl);  in nfsd_reply_cache_init()
215 	return -ENOMEM;  in nfsd_reply_cache_init()
223 	shrinker_free(nn->nfsd_reply_cache_shrinker);  in nfsd_reply_cache_shutdown()
225 	for (i = 0; i < nn->drc_hashsize; i++) {  in nfsd_reply_cache_shutdown()
226 		struct list_head *head = &nn->drc_hashtbl[i].lru_head;  in nfsd_reply_cache_shutdown()
229 			nfsd_reply_cache_free_locked(&nn->drc_hashtbl[i],  in nfsd_reply_cache_shutdown()
234 	kvfree(nn->drc_hashtbl);  in nfsd_reply_cache_shutdown()
235 	nn->drc_hashtbl = NULL;  in nfsd_reply_cache_shutdown()
236 	nn->drc_hashsize = 0;  in nfsd_reply_cache_shutdown()
243 	rp->c_timestamp = jiffies;  in lru_put_end()
244 	list_move_tail(&rp->c_lru, &b->lru_head);  in lru_put_end()
250 	unsigned int hash = hash_32((__force u32)xid, nn->maskbits);  in nfsd_cache_bucket_find()
252 	return &nn->drc_hashtbl[hash];  in nfsd_cache_bucket_find()
263 	unsigned long expiry = jiffies - RC_EXPIRE;  in nfsd_prune_bucket_locked()
267 	lockdep_assert_held(&b->cache_lock);  in nfsd_prune_bucket_locked()
269 	/* The bucket LRU is ordered oldest-first. */  in nfsd_prune_bucket_locked()
270 	list_for_each_entry_safe(rp, tmp, &b->lru_head, c_lru) {  in nfsd_prune_bucket_locked()
271 		if (atomic_read(&nn->num_drc_entries) <= nn->max_drc_entries &&  in nfsd_prune_bucket_locked()
272 		    time_before(expiry, rp->c_timestamp))  in nfsd_prune_bucket_locked()
276 		list_add(&rp->c_lru, dispose);  in nfsd_prune_bucket_locked()
284  * nfsd_reply_cache_count - count_objects method for the DRC shrinker
296 	struct nfsd_net *nn = shrink->private_data;  in nfsd_reply_cache_count()
298 	return atomic_read(&nn->num_drc_entries);  in nfsd_reply_cache_count()
302  * nfsd_reply_cache_scan - scan_objects method for the DRC shrinker
315 	struct nfsd_net *nn = shrink->private_data;  in nfsd_reply_cache_scan()
320 	for (i = 0; i < nn->drc_hashsize; i++) {  in nfsd_reply_cache_scan()
321 		struct nfsd_drc_bucket *b = &nn->drc_hashtbl[i];  in nfsd_reply_cache_scan()
323 		if (list_empty(&b->lru_head))  in nfsd_reply_cache_scan()
326 		spin_lock(&b->cache_lock);  in nfsd_reply_cache_scan()
328 		spin_unlock(&b->cache_lock);  in nfsd_reply_cache_scan()
331 		if (freed > sc->nr_to_scan)  in nfsd_reply_cache_scan()
338  * nfsd_cache_csum - Checksum incoming NFS Call arguments
352  * Returns a 32-bit checksum value, as defined in RFC 793.
372 		remaining -= len;  in nfsd_cache_csum()
380 		len = min_t(unsigned int, PAGE_SIZE - base, remaining);  in nfsd_cache_csum()
382 		remaining -= len;  in nfsd_cache_csum()
393 	if (key->c_key.k_xid == rp->c_key.k_xid &&  in nfsd_cache_key_cmp()
394 	    key->c_key.k_csum != rp->c_key.k_csum) {  in nfsd_cache_key_cmp()
399 	return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key));  in nfsd_cache_key_cmp()
412 	struct rb_node		**p = &b->rb_head.rb_node,  in nfsd_cache_insert()
424 			p = &parent->rb_left;  in nfsd_cache_insert()
426 			p = &parent->rb_right;  in nfsd_cache_insert()
432 	rb_link_node(&key->c_node, parent, p);  in nfsd_cache_insert()
433 	rb_insert_color(&key->c_node, &b->rb_head);  in nfsd_cache_insert()
436 	if (entries > nn->longest_chain) {  in nfsd_cache_insert()
437 		nn->longest_chain = entries;  in nfsd_cache_insert()
438 		nn->longest_chain_cachesize = atomic_read(&nn->num_drc_entries);  in nfsd_cache_insert()
439 	} else if (entries == nn->longest_chain) {  in nfsd_cache_insert()
441 		nn->longest_chain_cachesize = min_t(unsigned int,  in nfsd_cache_insert()
442 				nn->longest_chain_cachesize,  in nfsd_cache_insert()
443 				atomic_read(&nn->num_drc_entries));  in nfsd_cache_insert()
449  * nfsd_cache_lookup - Find an entry in the duplicate reply cache
451  * @start: starting byte in @rqstp->rq_arg of the NFS Call header
473 	int type = rqstp->rq_cachetype;  in nfsd_cache_lookup()
482 	csum = nfsd_cache_csum(&rqstp->rq_arg, start, len);  in nfsd_cache_lookup()
492 	b = nfsd_cache_bucket_find(rqstp->rq_xid, nn);  in nfsd_cache_lookup()
493 	spin_lock(&b->cache_lock);  in nfsd_cache_lookup()
498 	rp->c_state = RC_INPROG;  in nfsd_cache_lookup()
500 	spin_unlock(&b->cache_lock);  in nfsd_cache_lookup()
505 	atomic_inc(&nn->num_drc_entries);  in nfsd_cache_lookup()
517 	if (rp->c_state == RC_INPROG)  in nfsd_cache_lookup()
520 	/* From the hall of fame of impractical attacks:  in nfsd_cache_lookup()
523 	if (!test_bit(RQ_SECURE, &rqstp->rq_flags) && rp->c_secure)  in nfsd_cache_lookup()
527 	switch (rp->c_type) {  in nfsd_cache_lookup()
531 		xdr_stream_encode_be32(&rqstp->rq_res_stream, rp->c_replstat);  in nfsd_cache_lookup()
535 		if (!nfsd_cache_append(rqstp, &rp->c_replvec))  in nfsd_cache_lookup()
540 		WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp->c_type);  in nfsd_cache_lookup()
546 	spin_unlock(&b->cache_lock);  in nfsd_cache_lookup()
552  * nfsd_cache_update - Update an entry in the duplicate reply cache.
559  * executed and the complete reply is in rqstp->rq_res.
562  * the toplevel loop requires max-sized buffers, which would be a waste
576 	struct kvec	*resv = &rqstp->rq_res.head[0], *cachv;  in nfsd_cache_update()
584 	b = nfsd_cache_bucket_find(rp->c_key.k_xid, nn);  in nfsd_cache_update()
586 	len = resv->iov_len - ((char*)statp - (char*)resv->iov_base);  in nfsd_cache_update()
595 	switch (cachetype) {  in nfsd_cache_update()
599 		rp->c_replstat = *statp;  in nfsd_cache_update()
602 		cachv = &rp->c_replvec;  in nfsd_cache_update()
604 		cachv->iov_base = kmalloc(bufsize, GFP_KERNEL);  in nfsd_cache_update()
605 		if (!cachv->iov_base) {  in nfsd_cache_update()
609 		cachv->iov_len = bufsize;  in nfsd_cache_update()
610 		memcpy(cachv->iov_base, statp, bufsize);  in nfsd_cache_update()
616 	spin_lock(&b->cache_lock);  in nfsd_cache_update()
619 	rp->c_secure = test_bit(RQ_SECURE, &rqstp->rq_flags);  in nfsd_cache_update()
620 	rp->c_type = cachetype;  in nfsd_cache_update()
621 	rp->c_state = RC_DONE;  in nfsd_cache_update()
622 	spin_unlock(&b->cache_lock);  in nfsd_cache_update()
631 	p = xdr_reserve_space(&rqstp->rq_res_stream, data->iov_len);  in nfsd_cache_append()
634 	memcpy(p, data->iov_base, data->iov_len);  in nfsd_cache_append()
635 	xdr_commit_encode(&rqstp->rq_res_stream);  in nfsd_cache_append()
646 	struct nfsd_net *nn = net_generic(file_inode(m->file)->i_sb->s_fs_info,  in nfsd_reply_cache_stats_show()
649 	seq_printf(m, "max entries:           %u\n", nn->max_drc_entries);  in nfsd_reply_cache_stats_show()
651 		   atomic_read(&nn->num_drc_entries));  in nfsd_reply_cache_stats_show()
652 	seq_printf(m, "hash buckets:          %u\n", 1 << nn->maskbits);  in nfsd_reply_cache_stats_show()
654 		   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_DRC_MEM_USAGE]));  in nfsd_reply_cache_stats_show()
656 		   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]));  in nfsd_reply_cache_stats_show()
658 		   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]));  in nfsd_reply_cache_stats_show()
660 		   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]));  in nfsd_reply_cache_stats_show()
662 		   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_PAYLOAD_MISSES]));  in nfsd_reply_cache_stats_show()
663 	seq_printf(m, "longest chain len:     %u\n", nn->longest_chain);  in nfsd_reply_cache_stats_show()
664 	seq_printf(m, "cachesize at longest:  %u\n", nn->longest_chain_cachesize);  in nfsd_reply_cache_stats_show()