xref: /linux/net/sunrpc/svc.c (revision a1c3be890440a1769ed6f822376a3e3ab0d42994)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * linux/net/sunrpc/svc.c
4  *
5  * High-level RPC service routines
6  *
7  * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
8  *
9  * Multiple threads pools and NUMAisation
10  * Copyright (c) 2006 Silicon Graphics, Inc.
11  * by Greg Banks <gnb@melbourne.sgi.com>
12  */
13 
14 #include <linux/linkage.h>
15 #include <linux/sched/signal.h>
16 #include <linux/errno.h>
17 #include <linux/net.h>
18 #include <linux/in.h>
19 #include <linux/mm.h>
20 #include <linux/interrupt.h>
21 #include <linux/module.h>
22 #include <linux/kthread.h>
23 #include <linux/slab.h>
24 
25 #include <linux/sunrpc/types.h>
26 #include <linux/sunrpc/xdr.h>
27 #include <linux/sunrpc/stats.h>
28 #include <linux/sunrpc/svcsock.h>
29 #include <linux/sunrpc/clnt.h>
30 #include <linux/sunrpc/bc_xprt.h>
31 
32 #include <trace/events/sunrpc.h>
33 
34 #define RPCDBG_FACILITY	RPCDBG_SVCDSP
35 
36 static void svc_unregister(const struct svc_serv *serv, struct net *net);
37 
38 #define svc_serv_is_pooled(serv)    ((serv)->sv_ops->svo_function)
39 
40 #define SVC_POOL_DEFAULT	SVC_POOL_GLOBAL
41 
42 /*
43  * Structure for mapping cpus to pools and vice versa.
44  * Setup once during sunrpc initialisation.
45  */
46 struct svc_pool_map svc_pool_map = {
47 	.mode = SVC_POOL_DEFAULT
48 };
49 EXPORT_SYMBOL_GPL(svc_pool_map);
50 
51 static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
52 
53 static int
54 param_set_pool_mode(const char *val, const struct kernel_param *kp)
55 {
56 	int *ip = (int *)kp->arg;
57 	struct svc_pool_map *m = &svc_pool_map;
58 	int err;
59 
60 	mutex_lock(&svc_pool_map_mutex);
61 
62 	err = -EBUSY;
63 	if (m->count)
64 		goto out;
65 
66 	err = 0;
67 	if (!strncmp(val, "auto", 4))
68 		*ip = SVC_POOL_AUTO;
69 	else if (!strncmp(val, "global", 6))
70 		*ip = SVC_POOL_GLOBAL;
71 	else if (!strncmp(val, "percpu", 6))
72 		*ip = SVC_POOL_PERCPU;
73 	else if (!strncmp(val, "pernode", 7))
74 		*ip = SVC_POOL_PERNODE;
75 	else
76 		err = -EINVAL;
77 
78 out:
79 	mutex_unlock(&svc_pool_map_mutex);
80 	return err;
81 }
82 
83 static int
84 param_get_pool_mode(char *buf, const struct kernel_param *kp)
85 {
86 	int *ip = (int *)kp->arg;
87 
88 	switch (*ip)
89 	{
90 	case SVC_POOL_AUTO:
91 		return strlcpy(buf, "auto\n", 20);
92 	case SVC_POOL_GLOBAL:
93 		return strlcpy(buf, "global\n", 20);
94 	case SVC_POOL_PERCPU:
95 		return strlcpy(buf, "percpu\n", 20);
96 	case SVC_POOL_PERNODE:
97 		return strlcpy(buf, "pernode\n", 20);
98 	default:
99 		return sprintf(buf, "%d\n", *ip);
100 	}
101 }
102 
103 module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
104 		 &svc_pool_map.mode, 0644);
105 
106 /*
107  * Detect best pool mapping mode heuristically,
108  * according to the machine's topology.
109  */
110 static int
111 svc_pool_map_choose_mode(void)
112 {
113 	unsigned int node;
114 
115 	if (nr_online_nodes > 1) {
116 		/*
117 		 * Actually have multiple NUMA nodes,
118 		 * so split pools on NUMA node boundaries
119 		 */
120 		return SVC_POOL_PERNODE;
121 	}
122 
123 	node = first_online_node;
124 	if (nr_cpus_node(node) > 2) {
125 		/*
126 		 * Non-trivial SMP, or CONFIG_NUMA on
127 		 * non-NUMA hardware, e.g. with a generic
128 		 * x86_64 kernel on Xeons.  In this case we
129 		 * want to divide the pools on cpu boundaries.
130 		 */
131 		return SVC_POOL_PERCPU;
132 	}
133 
134 	/* default: one global pool */
135 	return SVC_POOL_GLOBAL;
136 }
137 
138 /*
139  * Allocate the to_pool[] and pool_to[] arrays.
140  * Returns 0 on success or an errno.
141  */
142 static int
143 svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
144 {
145 	m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
146 	if (!m->to_pool)
147 		goto fail;
148 	m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
149 	if (!m->pool_to)
150 		goto fail_free;
151 
152 	return 0;
153 
154 fail_free:
155 	kfree(m->to_pool);
156 	m->to_pool = NULL;
157 fail:
158 	return -ENOMEM;
159 }
160 
161 /*
162  * Initialise the pool map for SVC_POOL_PERCPU mode.
163  * Returns number of pools or <0 on error.
164  */
165 static int
166 svc_pool_map_init_percpu(struct svc_pool_map *m)
167 {
168 	unsigned int maxpools = nr_cpu_ids;
169 	unsigned int pidx = 0;
170 	unsigned int cpu;
171 	int err;
172 
173 	err = svc_pool_map_alloc_arrays(m, maxpools);
174 	if (err)
175 		return err;
176 
177 	for_each_online_cpu(cpu) {
178 		BUG_ON(pidx >= maxpools);
179 		m->to_pool[cpu] = pidx;
180 		m->pool_to[pidx] = cpu;
181 		pidx++;
182 	}
183 	/* cpus brought online later all get mapped to pool0, sorry */
184 
185 	return pidx;
186 };
187 
188 
189 /*
190  * Initialise the pool map for SVC_POOL_PERNODE mode.
191  * Returns number of pools or <0 on error.
192  */
193 static int
194 svc_pool_map_init_pernode(struct svc_pool_map *m)
195 {
196 	unsigned int maxpools = nr_node_ids;
197 	unsigned int pidx = 0;
198 	unsigned int node;
199 	int err;
200 
201 	err = svc_pool_map_alloc_arrays(m, maxpools);
202 	if (err)
203 		return err;
204 
205 	for_each_node_with_cpus(node) {
206 		/* some architectures (e.g. SN2) have cpuless nodes */
207 		BUG_ON(pidx > maxpools);
208 		m->to_pool[node] = pidx;
209 		m->pool_to[pidx] = node;
210 		pidx++;
211 	}
212 	/* nodes brought online later all get mapped to pool0, sorry */
213 
214 	return pidx;
215 }
216 
217 
218 /*
219  * Add a reference to the global map of cpus to pools (and
220  * vice versa).  Initialise the map if we're the first user.
221  * Returns the number of pools.
222  */
223 unsigned int
224 svc_pool_map_get(void)
225 {
226 	struct svc_pool_map *m = &svc_pool_map;
227 	int npools = -1;
228 
229 	mutex_lock(&svc_pool_map_mutex);
230 
231 	if (m->count++) {
232 		mutex_unlock(&svc_pool_map_mutex);
233 		return m->npools;
234 	}
235 
236 	if (m->mode == SVC_POOL_AUTO)
237 		m->mode = svc_pool_map_choose_mode();
238 
239 	switch (m->mode) {
240 	case SVC_POOL_PERCPU:
241 		npools = svc_pool_map_init_percpu(m);
242 		break;
243 	case SVC_POOL_PERNODE:
244 		npools = svc_pool_map_init_pernode(m);
245 		break;
246 	}
247 
248 	if (npools < 0) {
249 		/* default, or memory allocation failure */
250 		npools = 1;
251 		m->mode = SVC_POOL_GLOBAL;
252 	}
253 	m->npools = npools;
254 
255 	mutex_unlock(&svc_pool_map_mutex);
256 	return m->npools;
257 }
258 EXPORT_SYMBOL_GPL(svc_pool_map_get);
259 
260 /*
261  * Drop a reference to the global map of cpus to pools.
262  * When the last reference is dropped, the map data is
263  * freed; this allows the sysadmin to change the pool
264  * mode using the pool_mode module option without
265  * rebooting or re-loading sunrpc.ko.
266  */
267 void
268 svc_pool_map_put(void)
269 {
270 	struct svc_pool_map *m = &svc_pool_map;
271 
272 	mutex_lock(&svc_pool_map_mutex);
273 
274 	if (!--m->count) {
275 		kfree(m->to_pool);
276 		m->to_pool = NULL;
277 		kfree(m->pool_to);
278 		m->pool_to = NULL;
279 		m->npools = 0;
280 	}
281 
282 	mutex_unlock(&svc_pool_map_mutex);
283 }
284 EXPORT_SYMBOL_GPL(svc_pool_map_put);
285 
286 static int svc_pool_map_get_node(unsigned int pidx)
287 {
288 	const struct svc_pool_map *m = &svc_pool_map;
289 
290 	if (m->count) {
291 		if (m->mode == SVC_POOL_PERCPU)
292 			return cpu_to_node(m->pool_to[pidx]);
293 		if (m->mode == SVC_POOL_PERNODE)
294 			return m->pool_to[pidx];
295 	}
296 	return NUMA_NO_NODE;
297 }
298 /*
299  * Set the given thread's cpus_allowed mask so that it
300  * will only run on cpus in the given pool.
301  */
302 static inline void
303 svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
304 {
305 	struct svc_pool_map *m = &svc_pool_map;
306 	unsigned int node = m->pool_to[pidx];
307 
308 	/*
309 	 * The caller checks for sv_nrpools > 1, which
310 	 * implies that we've been initialized.
311 	 */
312 	WARN_ON_ONCE(m->count == 0);
313 	if (m->count == 0)
314 		return;
315 
316 	switch (m->mode) {
317 	case SVC_POOL_PERCPU:
318 	{
319 		set_cpus_allowed_ptr(task, cpumask_of(node));
320 		break;
321 	}
322 	case SVC_POOL_PERNODE:
323 	{
324 		set_cpus_allowed_ptr(task, cpumask_of_node(node));
325 		break;
326 	}
327 	}
328 }
329 
330 /*
331  * Use the mapping mode to choose a pool for a given CPU.
332  * Used when enqueueing an incoming RPC.  Always returns
333  * a non-NULL pool pointer.
334  */
335 struct svc_pool *
336 svc_pool_for_cpu(struct svc_serv *serv, int cpu)
337 {
338 	struct svc_pool_map *m = &svc_pool_map;
339 	unsigned int pidx = 0;
340 
341 	/*
342 	 * An uninitialised map happens in a pure client when
343 	 * lockd is brought up, so silently treat it the
344 	 * same as SVC_POOL_GLOBAL.
345 	 */
346 	if (svc_serv_is_pooled(serv)) {
347 		switch (m->mode) {
348 		case SVC_POOL_PERCPU:
349 			pidx = m->to_pool[cpu];
350 			break;
351 		case SVC_POOL_PERNODE:
352 			pidx = m->to_pool[cpu_to_node(cpu)];
353 			break;
354 		}
355 	}
356 	return &serv->sv_pools[pidx % serv->sv_nrpools];
357 }
358 
359 int svc_rpcb_setup(struct svc_serv *serv, struct net *net)
360 {
361 	int err;
362 
363 	err = rpcb_create_local(net);
364 	if (err)
365 		return err;
366 
367 	/* Remove any stale portmap registrations */
368 	svc_unregister(serv, net);
369 	return 0;
370 }
371 EXPORT_SYMBOL_GPL(svc_rpcb_setup);
372 
373 void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net)
374 {
375 	svc_unregister(serv, net);
376 	rpcb_put_local(net);
377 }
378 EXPORT_SYMBOL_GPL(svc_rpcb_cleanup);
379 
380 static int svc_uses_rpcbind(struct svc_serv *serv)
381 {
382 	struct svc_program	*progp;
383 	unsigned int		i;
384 
385 	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
386 		for (i = 0; i < progp->pg_nvers; i++) {
387 			if (progp->pg_vers[i] == NULL)
388 				continue;
389 			if (!progp->pg_vers[i]->vs_hidden)
390 				return 1;
391 		}
392 	}
393 
394 	return 0;
395 }
396 
397 int svc_bind(struct svc_serv *serv, struct net *net)
398 {
399 	if (!svc_uses_rpcbind(serv))
400 		return 0;
401 	return svc_rpcb_setup(serv, net);
402 }
403 EXPORT_SYMBOL_GPL(svc_bind);
404 
405 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
406 static void
407 __svc_init_bc(struct svc_serv *serv)
408 {
409 	INIT_LIST_HEAD(&serv->sv_cb_list);
410 	spin_lock_init(&serv->sv_cb_lock);
411 	init_waitqueue_head(&serv->sv_cb_waitq);
412 }
413 #else
414 static void
415 __svc_init_bc(struct svc_serv *serv)
416 {
417 }
418 #endif
419 
420 /*
421  * Create an RPC service
422  */
423 static struct svc_serv *
424 __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
425 	     const struct svc_serv_ops *ops)
426 {
427 	struct svc_serv	*serv;
428 	unsigned int vers;
429 	unsigned int xdrsize;
430 	unsigned int i;
431 
432 	if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
433 		return NULL;
434 	serv->sv_name      = prog->pg_name;
435 	serv->sv_program   = prog;
436 	serv->sv_nrthreads = 1;
437 	serv->sv_stats     = prog->pg_stats;
438 	if (bufsize > RPCSVC_MAXPAYLOAD)
439 		bufsize = RPCSVC_MAXPAYLOAD;
440 	serv->sv_max_payload = bufsize? bufsize : 4096;
441 	serv->sv_max_mesg  = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
442 	serv->sv_ops = ops;
443 	xdrsize = 0;
444 	while (prog) {
445 		prog->pg_lovers = prog->pg_nvers-1;
446 		for (vers=0; vers<prog->pg_nvers ; vers++)
447 			if (prog->pg_vers[vers]) {
448 				prog->pg_hivers = vers;
449 				if (prog->pg_lovers > vers)
450 					prog->pg_lovers = vers;
451 				if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
452 					xdrsize = prog->pg_vers[vers]->vs_xdrsize;
453 			}
454 		prog = prog->pg_next;
455 	}
456 	serv->sv_xdrsize   = xdrsize;
457 	INIT_LIST_HEAD(&serv->sv_tempsocks);
458 	INIT_LIST_HEAD(&serv->sv_permsocks);
459 	timer_setup(&serv->sv_temptimer, NULL, 0);
460 	spin_lock_init(&serv->sv_lock);
461 
462 	__svc_init_bc(serv);
463 
464 	serv->sv_nrpools = npools;
465 	serv->sv_pools =
466 		kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
467 			GFP_KERNEL);
468 	if (!serv->sv_pools) {
469 		kfree(serv);
470 		return NULL;
471 	}
472 
473 	for (i = 0; i < serv->sv_nrpools; i++) {
474 		struct svc_pool *pool = &serv->sv_pools[i];
475 
476 		dprintk("svc: initialising pool %u for %s\n",
477 				i, serv->sv_name);
478 
479 		pool->sp_id = i;
480 		INIT_LIST_HEAD(&pool->sp_sockets);
481 		INIT_LIST_HEAD(&pool->sp_all_threads);
482 		spin_lock_init(&pool->sp_lock);
483 	}
484 
485 	return serv;
486 }
487 
488 struct svc_serv *
489 svc_create(struct svc_program *prog, unsigned int bufsize,
490 	   const struct svc_serv_ops *ops)
491 {
492 	return __svc_create(prog, bufsize, /*npools*/1, ops);
493 }
494 EXPORT_SYMBOL_GPL(svc_create);
495 
496 struct svc_serv *
497 svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
498 		  const struct svc_serv_ops *ops)
499 {
500 	struct svc_serv *serv;
501 	unsigned int npools = svc_pool_map_get();
502 
503 	serv = __svc_create(prog, bufsize, npools, ops);
504 	if (!serv)
505 		goto out_err;
506 	return serv;
507 out_err:
508 	svc_pool_map_put();
509 	return NULL;
510 }
511 EXPORT_SYMBOL_GPL(svc_create_pooled);
512 
513 void svc_shutdown_net(struct svc_serv *serv, struct net *net)
514 {
515 	svc_close_net(serv, net);
516 
517 	if (serv->sv_ops->svo_shutdown)
518 		serv->sv_ops->svo_shutdown(serv, net);
519 }
520 EXPORT_SYMBOL_GPL(svc_shutdown_net);
521 
522 /*
523  * Destroy an RPC service. Should be called with appropriate locking to
524  * protect the sv_nrthreads, sv_permsocks and sv_tempsocks.
525  */
526 void
527 svc_destroy(struct svc_serv *serv)
528 {
529 	dprintk("svc: svc_destroy(%s, %d)\n",
530 				serv->sv_program->pg_name,
531 				serv->sv_nrthreads);
532 
533 	if (serv->sv_nrthreads) {
534 		if (--(serv->sv_nrthreads) != 0) {
535 			svc_sock_update_bufs(serv);
536 			return;
537 		}
538 	} else
539 		printk("svc_destroy: no threads for serv=%p!\n", serv);
540 
541 	del_timer_sync(&serv->sv_temptimer);
542 
543 	/*
544 	 * The last user is gone and thus all sockets have to be destroyed to
545 	 * the point. Check this.
546 	 */
547 	BUG_ON(!list_empty(&serv->sv_permsocks));
548 	BUG_ON(!list_empty(&serv->sv_tempsocks));
549 
550 	cache_clean_deferred(serv);
551 
552 	if (svc_serv_is_pooled(serv))
553 		svc_pool_map_put();
554 
555 	kfree(serv->sv_pools);
556 	kfree(serv);
557 }
558 EXPORT_SYMBOL_GPL(svc_destroy);
559 
560 /*
561  * Allocate an RPC server's buffer space.
562  * We allocate pages and place them in rq_pages.
563  */
564 static int
565 svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
566 {
567 	unsigned int pages, arghi;
568 
569 	/* bc_xprt uses fore channel allocated buffers */
570 	if (svc_is_backchannel(rqstp))
571 		return 1;
572 
573 	pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
574 				       * We assume one is at most one page
575 				       */
576 	arghi = 0;
577 	WARN_ON_ONCE(pages > RPCSVC_MAXPAGES);
578 	if (pages > RPCSVC_MAXPAGES)
579 		pages = RPCSVC_MAXPAGES;
580 	while (pages) {
581 		struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
582 		if (!p)
583 			break;
584 		rqstp->rq_pages[arghi++] = p;
585 		pages--;
586 	}
587 	return pages == 0;
588 }
589 
590 /*
591  * Release an RPC server buffer
592  */
593 static void
594 svc_release_buffer(struct svc_rqst *rqstp)
595 {
596 	unsigned int i;
597 
598 	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
599 		if (rqstp->rq_pages[i])
600 			put_page(rqstp->rq_pages[i]);
601 }
602 
603 struct svc_rqst *
604 svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)
605 {
606 	struct svc_rqst	*rqstp;
607 
608 	rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
609 	if (!rqstp)
610 		return rqstp;
611 
612 	__set_bit(RQ_BUSY, &rqstp->rq_flags);
613 	spin_lock_init(&rqstp->rq_lock);
614 	rqstp->rq_server = serv;
615 	rqstp->rq_pool = pool;
616 
617 	rqstp->rq_scratch_page = alloc_pages_node(node, GFP_KERNEL, 0);
618 	if (!rqstp->rq_scratch_page)
619 		goto out_enomem;
620 
621 	rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
622 	if (!rqstp->rq_argp)
623 		goto out_enomem;
624 
625 	rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
626 	if (!rqstp->rq_resp)
627 		goto out_enomem;
628 
629 	if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
630 		goto out_enomem;
631 
632 	return rqstp;
633 out_enomem:
634 	svc_rqst_free(rqstp);
635 	return NULL;
636 }
637 EXPORT_SYMBOL_GPL(svc_rqst_alloc);
638 
639 struct svc_rqst *
640 svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
641 {
642 	struct svc_rqst	*rqstp;
643 
644 	rqstp = svc_rqst_alloc(serv, pool, node);
645 	if (!rqstp)
646 		return ERR_PTR(-ENOMEM);
647 
648 	serv->sv_nrthreads++;
649 	spin_lock_bh(&pool->sp_lock);
650 	pool->sp_nrthreads++;
651 	list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
652 	spin_unlock_bh(&pool->sp_lock);
653 	return rqstp;
654 }
655 EXPORT_SYMBOL_GPL(svc_prepare_thread);
656 
657 /*
658  * Choose a pool in which to create a new thread, for svc_set_num_threads
659  */
660 static inline struct svc_pool *
661 choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
662 {
663 	if (pool != NULL)
664 		return pool;
665 
666 	return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
667 }
668 
669 /*
670  * Choose a thread to kill, for svc_set_num_threads
671  */
672 static inline struct task_struct *
673 choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
674 {
675 	unsigned int i;
676 	struct task_struct *task = NULL;
677 
678 	if (pool != NULL) {
679 		spin_lock_bh(&pool->sp_lock);
680 	} else {
681 		/* choose a pool in round-robin fashion */
682 		for (i = 0; i < serv->sv_nrpools; i++) {
683 			pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
684 			spin_lock_bh(&pool->sp_lock);
685 			if (!list_empty(&pool->sp_all_threads))
686 				goto found_pool;
687 			spin_unlock_bh(&pool->sp_lock);
688 		}
689 		return NULL;
690 	}
691 
692 found_pool:
693 	if (!list_empty(&pool->sp_all_threads)) {
694 		struct svc_rqst *rqstp;
695 
696 		/*
697 		 * Remove from the pool->sp_all_threads list
698 		 * so we don't try to kill it again.
699 		 */
700 		rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
701 		set_bit(RQ_VICTIM, &rqstp->rq_flags);
702 		list_del_rcu(&rqstp->rq_all);
703 		task = rqstp->rq_task;
704 	}
705 	spin_unlock_bh(&pool->sp_lock);
706 
707 	return task;
708 }
709 
710 /* create new threads */
711 static int
712 svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
713 {
714 	struct svc_rqst	*rqstp;
715 	struct task_struct *task;
716 	struct svc_pool *chosen_pool;
717 	unsigned int state = serv->sv_nrthreads-1;
718 	int node;
719 
720 	do {
721 		nrservs--;
722 		chosen_pool = choose_pool(serv, pool, &state);
723 
724 		node = svc_pool_map_get_node(chosen_pool->sp_id);
725 		rqstp = svc_prepare_thread(serv, chosen_pool, node);
726 		if (IS_ERR(rqstp))
727 			return PTR_ERR(rqstp);
728 
729 		__module_get(serv->sv_ops->svo_module);
730 		task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
731 					      node, "%s", serv->sv_name);
732 		if (IS_ERR(task)) {
733 			module_put(serv->sv_ops->svo_module);
734 			svc_exit_thread(rqstp);
735 			return PTR_ERR(task);
736 		}
737 
738 		rqstp->rq_task = task;
739 		if (serv->sv_nrpools > 1)
740 			svc_pool_map_set_cpumask(task, chosen_pool->sp_id);
741 
742 		svc_sock_update_bufs(serv);
743 		wake_up_process(task);
744 	} while (nrservs > 0);
745 
746 	return 0;
747 }
748 
749 
750 /* destroy old threads */
751 static int
752 svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
753 {
754 	struct task_struct *task;
755 	unsigned int state = serv->sv_nrthreads-1;
756 
757 	/* destroy old threads */
758 	do {
759 		task = choose_victim(serv, pool, &state);
760 		if (task == NULL)
761 			break;
762 		send_sig(SIGINT, task, 1);
763 		nrservs++;
764 	} while (nrservs < 0);
765 
766 	return 0;
767 }
768 
769 /*
770  * Create or destroy enough new threads to make the number
771  * of threads the given number.  If `pool' is non-NULL, applies
772  * only to threads in that pool, otherwise round-robins between
773  * all pools.  Caller must ensure that mutual exclusion between this and
774  * server startup or shutdown.
775  *
776  * Destroying threads relies on the service threads filling in
777  * rqstp->rq_task, which only the nfs ones do.  Assumes the serv
778  * has been created using svc_create_pooled().
779  *
780  * Based on code that used to be in nfsd_svc() but tweaked
781  * to be pool-aware.
782  */
783 int
784 svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
785 {
786 	if (pool == NULL) {
787 		/* The -1 assumes caller has done a svc_get() */
788 		nrservs -= (serv->sv_nrthreads-1);
789 	} else {
790 		spin_lock_bh(&pool->sp_lock);
791 		nrservs -= pool->sp_nrthreads;
792 		spin_unlock_bh(&pool->sp_lock);
793 	}
794 
795 	if (nrservs > 0)
796 		return svc_start_kthreads(serv, pool, nrservs);
797 	if (nrservs < 0)
798 		return svc_signal_kthreads(serv, pool, nrservs);
799 	return 0;
800 }
801 EXPORT_SYMBOL_GPL(svc_set_num_threads);
802 
803 /* destroy old threads */
804 static int
805 svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
806 {
807 	struct task_struct *task;
808 	unsigned int state = serv->sv_nrthreads-1;
809 
810 	/* destroy old threads */
811 	do {
812 		task = choose_victim(serv, pool, &state);
813 		if (task == NULL)
814 			break;
815 		kthread_stop(task);
816 		nrservs++;
817 	} while (nrservs < 0);
818 	return 0;
819 }
820 
821 int
822 svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
823 {
824 	if (pool == NULL) {
825 		/* The -1 assumes caller has done a svc_get() */
826 		nrservs -= (serv->sv_nrthreads-1);
827 	} else {
828 		spin_lock_bh(&pool->sp_lock);
829 		nrservs -= pool->sp_nrthreads;
830 		spin_unlock_bh(&pool->sp_lock);
831 	}
832 
833 	if (nrservs > 0)
834 		return svc_start_kthreads(serv, pool, nrservs);
835 	if (nrservs < 0)
836 		return svc_stop_kthreads(serv, pool, nrservs);
837 	return 0;
838 }
839 EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);
840 
841 /*
842  * Called from a server thread as it's exiting. Caller must hold the "service
843  * mutex" for the service.
844  */
845 void
846 svc_rqst_free(struct svc_rqst *rqstp)
847 {
848 	svc_release_buffer(rqstp);
849 	put_page(rqstp->rq_scratch_page);
850 	kfree(rqstp->rq_resp);
851 	kfree(rqstp->rq_argp);
852 	kfree(rqstp->rq_auth_data);
853 	kfree_rcu(rqstp, rq_rcu_head);
854 }
855 EXPORT_SYMBOL_GPL(svc_rqst_free);
856 
857 void
858 svc_exit_thread(struct svc_rqst *rqstp)
859 {
860 	struct svc_serv	*serv = rqstp->rq_server;
861 	struct svc_pool	*pool = rqstp->rq_pool;
862 
863 	spin_lock_bh(&pool->sp_lock);
864 	pool->sp_nrthreads--;
865 	if (!test_and_set_bit(RQ_VICTIM, &rqstp->rq_flags))
866 		list_del_rcu(&rqstp->rq_all);
867 	spin_unlock_bh(&pool->sp_lock);
868 
869 	svc_rqst_free(rqstp);
870 
871 	/* Release the server */
872 	if (serv)
873 		svc_destroy(serv);
874 }
875 EXPORT_SYMBOL_GPL(svc_exit_thread);
876 
877 /*
878  * Register an "inet" protocol family netid with the local
879  * rpcbind daemon via an rpcbind v4 SET request.
880  *
881  * No netconfig infrastructure is available in the kernel, so
882  * we map IP_ protocol numbers to netids by hand.
883  *
884  * Returns zero on success; a negative errno value is returned
885  * if any error occurs.
886  */
887 static int __svc_rpcb_register4(struct net *net, const u32 program,
888 				const u32 version,
889 				const unsigned short protocol,
890 				const unsigned short port)
891 {
892 	const struct sockaddr_in sin = {
893 		.sin_family		= AF_INET,
894 		.sin_addr.s_addr	= htonl(INADDR_ANY),
895 		.sin_port		= htons(port),
896 	};
897 	const char *netid;
898 	int error;
899 
900 	switch (protocol) {
901 	case IPPROTO_UDP:
902 		netid = RPCBIND_NETID_UDP;
903 		break;
904 	case IPPROTO_TCP:
905 		netid = RPCBIND_NETID_TCP;
906 		break;
907 	default:
908 		return -ENOPROTOOPT;
909 	}
910 
911 	error = rpcb_v4_register(net, program, version,
912 					(const struct sockaddr *)&sin, netid);
913 
914 	/*
915 	 * User space didn't support rpcbind v4, so retry this
916 	 * registration request with the legacy rpcbind v2 protocol.
917 	 */
918 	if (error == -EPROTONOSUPPORT)
919 		error = rpcb_register(net, program, version, protocol, port);
920 
921 	return error;
922 }
923 
924 #if IS_ENABLED(CONFIG_IPV6)
925 /*
926  * Register an "inet6" protocol family netid with the local
927  * rpcbind daemon via an rpcbind v4 SET request.
928  *
929  * No netconfig infrastructure is available in the kernel, so
930  * we map IP_ protocol numbers to netids by hand.
931  *
932  * Returns zero on success; a negative errno value is returned
933  * if any error occurs.
934  */
935 static int __svc_rpcb_register6(struct net *net, const u32 program,
936 				const u32 version,
937 				const unsigned short protocol,
938 				const unsigned short port)
939 {
940 	const struct sockaddr_in6 sin6 = {
941 		.sin6_family		= AF_INET6,
942 		.sin6_addr		= IN6ADDR_ANY_INIT,
943 		.sin6_port		= htons(port),
944 	};
945 	const char *netid;
946 	int error;
947 
948 	switch (protocol) {
949 	case IPPROTO_UDP:
950 		netid = RPCBIND_NETID_UDP6;
951 		break;
952 	case IPPROTO_TCP:
953 		netid = RPCBIND_NETID_TCP6;
954 		break;
955 	default:
956 		return -ENOPROTOOPT;
957 	}
958 
959 	error = rpcb_v4_register(net, program, version,
960 					(const struct sockaddr *)&sin6, netid);
961 
962 	/*
963 	 * User space didn't support rpcbind version 4, so we won't
964 	 * use a PF_INET6 listener.
965 	 */
966 	if (error == -EPROTONOSUPPORT)
967 		error = -EAFNOSUPPORT;
968 
969 	return error;
970 }
971 #endif	/* IS_ENABLED(CONFIG_IPV6) */
972 
973 /*
974  * Register a kernel RPC service via rpcbind version 4.
975  *
976  * Returns zero on success; a negative errno value is returned
977  * if any error occurs.
978  */
979 static int __svc_register(struct net *net, const char *progname,
980 			  const u32 program, const u32 version,
981 			  const int family,
982 			  const unsigned short protocol,
983 			  const unsigned short port)
984 {
985 	int error = -EAFNOSUPPORT;
986 
987 	switch (family) {
988 	case PF_INET:
989 		error = __svc_rpcb_register4(net, program, version,
990 						protocol, port);
991 		break;
992 #if IS_ENABLED(CONFIG_IPV6)
993 	case PF_INET6:
994 		error = __svc_rpcb_register6(net, program, version,
995 						protocol, port);
996 #endif
997 	}
998 
999 	trace_svc_register(progname, version, protocol, port, family, error);
1000 	return error;
1001 }
1002 
1003 int svc_rpcbind_set_version(struct net *net,
1004 			    const struct svc_program *progp,
1005 			    u32 version, int family,
1006 			    unsigned short proto,
1007 			    unsigned short port)
1008 {
1009 	return __svc_register(net, progp->pg_name, progp->pg_prog,
1010 				version, family, proto, port);
1011 
1012 }
1013 EXPORT_SYMBOL_GPL(svc_rpcbind_set_version);
1014 
1015 int svc_generic_rpcbind_set(struct net *net,
1016 			    const struct svc_program *progp,
1017 			    u32 version, int family,
1018 			    unsigned short proto,
1019 			    unsigned short port)
1020 {
1021 	const struct svc_version *vers = progp->pg_vers[version];
1022 	int error;
1023 
1024 	if (vers == NULL)
1025 		return 0;
1026 
1027 	if (vers->vs_hidden) {
1028 		trace_svc_noregister(progp->pg_name, version, proto,
1029 				     port, family, 0);
1030 		return 0;
1031 	}
1032 
1033 	/*
1034 	 * Don't register a UDP port if we need congestion
1035 	 * control.
1036 	 */
1037 	if (vers->vs_need_cong_ctrl && proto == IPPROTO_UDP)
1038 		return 0;
1039 
1040 	error = svc_rpcbind_set_version(net, progp, version,
1041 					family, proto, port);
1042 
1043 	return (vers->vs_rpcb_optnl) ? 0 : error;
1044 }
1045 EXPORT_SYMBOL_GPL(svc_generic_rpcbind_set);
1046 
1047 /**
1048  * svc_register - register an RPC service with the local portmapper
1049  * @serv: svc_serv struct for the service to register
1050  * @net: net namespace for the service to register
1051  * @family: protocol family of service's listener socket
1052  * @proto: transport protocol number to advertise
1053  * @port: port to advertise
1054  *
1055  * Service is registered for any address in the passed-in protocol family
1056  */
1057 int svc_register(const struct svc_serv *serv, struct net *net,
1058 		 const int family, const unsigned short proto,
1059 		 const unsigned short port)
1060 {
1061 	struct svc_program	*progp;
1062 	unsigned int		i;
1063 	int			error = 0;
1064 
1065 	WARN_ON_ONCE(proto == 0 && port == 0);
1066 	if (proto == 0 && port == 0)
1067 		return -EINVAL;
1068 
1069 	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
1070 		for (i = 0; i < progp->pg_nvers; i++) {
1071 
1072 			error = progp->pg_rpcbind_set(net, progp, i,
1073 					family, proto, port);
1074 			if (error < 0) {
1075 				printk(KERN_WARNING "svc: failed to register "
1076 					"%sv%u RPC service (errno %d).\n",
1077 					progp->pg_name, i, -error);
1078 				break;
1079 			}
1080 		}
1081 	}
1082 
1083 	return error;
1084 }
1085 
1086 /*
1087  * If user space is running rpcbind, it should take the v4 UNSET
1088  * and clear everything for this [program, version].  If user space
1089  * is running portmap, it will reject the v4 UNSET, but won't have
1090  * any "inet6" entries anyway.  So a PMAP_UNSET should be sufficient
1091  * in this case to clear all existing entries for [program, version].
1092  */
1093 static void __svc_unregister(struct net *net, const u32 program, const u32 version,
1094 			     const char *progname)
1095 {
1096 	int error;
1097 
1098 	error = rpcb_v4_register(net, program, version, NULL, "");
1099 
1100 	/*
1101 	 * User space didn't support rpcbind v4, so retry this
1102 	 * request with the legacy rpcbind v2 protocol.
1103 	 */
1104 	if (error == -EPROTONOSUPPORT)
1105 		error = rpcb_register(net, program, version, 0, 0);
1106 
1107 	trace_svc_unregister(progname, version, error);
1108 }
1109 
1110 /*
1111  * All netids, bind addresses and ports registered for [program, version]
1112  * are removed from the local rpcbind database (if the service is not
1113  * hidden) to make way for a new instance of the service.
1114  *
1115  * The result of unregistration is reported via dprintk for those who want
1116  * verification of the result, but is otherwise not important.
1117  */
1118 static void svc_unregister(const struct svc_serv *serv, struct net *net)
1119 {
1120 	struct svc_program *progp;
1121 	unsigned long flags;
1122 	unsigned int i;
1123 
1124 	clear_thread_flag(TIF_SIGPENDING);
1125 
1126 	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
1127 		for (i = 0; i < progp->pg_nvers; i++) {
1128 			if (progp->pg_vers[i] == NULL)
1129 				continue;
1130 			if (progp->pg_vers[i]->vs_hidden)
1131 				continue;
1132 			__svc_unregister(net, progp->pg_prog, i, progp->pg_name);
1133 		}
1134 	}
1135 
1136 	spin_lock_irqsave(&current->sighand->siglock, flags);
1137 	recalc_sigpending();
1138 	spin_unlock_irqrestore(&current->sighand->siglock, flags);
1139 }
1140 
1141 /*
1142  * dprintk the given error with the address of the client that caused it.
1143  */
1144 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
1145 static __printf(2, 3)
1146 void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
1147 {
1148 	struct va_format vaf;
1149 	va_list args;
1150 	char 	buf[RPC_MAX_ADDRBUFLEN];
1151 
1152 	va_start(args, fmt);
1153 
1154 	vaf.fmt = fmt;
1155 	vaf.va = &args;
1156 
1157 	dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
1158 
1159 	va_end(args);
1160 }
1161 #else
1162 static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
1163 #endif
1164 
1165 __be32
1166 svc_return_autherr(struct svc_rqst *rqstp, __be32 auth_err)
1167 {
1168 	set_bit(RQ_AUTHERR, &rqstp->rq_flags);
1169 	return auth_err;
1170 }
1171 EXPORT_SYMBOL_GPL(svc_return_autherr);
1172 
1173 static __be32
1174 svc_get_autherr(struct svc_rqst *rqstp, __be32 *statp)
1175 {
1176 	if (test_and_clear_bit(RQ_AUTHERR, &rqstp->rq_flags))
1177 		return *statp;
1178 	return rpc_auth_ok;
1179 }
1180 
1181 static int
1182 svc_generic_dispatch(struct svc_rqst *rqstp, __be32 *statp)
1183 {
1184 	struct kvec *argv = &rqstp->rq_arg.head[0];
1185 	struct kvec *resv = &rqstp->rq_res.head[0];
1186 	const struct svc_procedure *procp = rqstp->rq_procinfo;
1187 
1188 	/*
1189 	 * Decode arguments
1190 	 * XXX: why do we ignore the return value?
1191 	 */
1192 	if (procp->pc_decode &&
1193 	    !procp->pc_decode(rqstp, argv->iov_base)) {
1194 		*statp = rpc_garbage_args;
1195 		return 1;
1196 	}
1197 
1198 	*statp = procp->pc_func(rqstp);
1199 
1200 	if (*statp == rpc_drop_reply ||
1201 	    test_bit(RQ_DROPME, &rqstp->rq_flags))
1202 		return 0;
1203 
1204 	if (test_bit(RQ_AUTHERR, &rqstp->rq_flags))
1205 		return 1;
1206 
1207 	if (*statp != rpc_success)
1208 		return 1;
1209 
1210 	/* Encode reply */
1211 	if (procp->pc_encode &&
1212 	    !procp->pc_encode(rqstp, resv->iov_base + resv->iov_len)) {
1213 		dprintk("svc: failed to encode reply\n");
1214 		/* serv->sv_stats->rpcsystemerr++; */
1215 		*statp = rpc_system_err;
1216 	}
1217 	return 1;
1218 }
1219 
1220 __be32
1221 svc_generic_init_request(struct svc_rqst *rqstp,
1222 		const struct svc_program *progp,
1223 		struct svc_process_info *ret)
1224 {
1225 	const struct svc_version *versp = NULL;	/* compiler food */
1226 	const struct svc_procedure *procp = NULL;
1227 
1228 	if (rqstp->rq_vers >= progp->pg_nvers )
1229 		goto err_bad_vers;
1230 	versp = progp->pg_vers[rqstp->rq_vers];
1231 	if (!versp)
1232 		goto err_bad_vers;
1233 
1234 	/*
1235 	 * Some protocol versions (namely NFSv4) require some form of
1236 	 * congestion control.  (See RFC 7530 section 3.1 paragraph 2)
1237 	 * In other words, UDP is not allowed. We mark those when setting
1238 	 * up the svc_xprt, and verify that here.
1239 	 *
1240 	 * The spec is not very clear about what error should be returned
1241 	 * when someone tries to access a server that is listening on UDP
1242 	 * for lower versions. RPC_PROG_MISMATCH seems to be the closest
1243 	 * fit.
1244 	 */
1245 	if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&
1246 	    !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))
1247 		goto err_bad_vers;
1248 
1249 	if (rqstp->rq_proc >= versp->vs_nproc)
1250 		goto err_bad_proc;
1251 	rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];
1252 	if (!procp)
1253 		goto err_bad_proc;
1254 
1255 	/* Initialize storage for argp and resp */
1256 	memset(rqstp->rq_argp, 0, procp->pc_argsize);
1257 	memset(rqstp->rq_resp, 0, procp->pc_ressize);
1258 
1259 	/* Bump per-procedure stats counter */
1260 	versp->vs_count[rqstp->rq_proc]++;
1261 
1262 	ret->dispatch = versp->vs_dispatch;
1263 	return rpc_success;
1264 err_bad_vers:
1265 	ret->mismatch.lovers = progp->pg_lovers;
1266 	ret->mismatch.hivers = progp->pg_hivers;
1267 	return rpc_prog_mismatch;
1268 err_bad_proc:
1269 	return rpc_proc_unavail;
1270 }
1271 EXPORT_SYMBOL_GPL(svc_generic_init_request);
1272 
1273 /*
1274  * Common routine for processing the RPC request.
1275  */
1276 static int
1277 svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
1278 {
1279 	struct svc_program	*progp;
1280 	const struct svc_procedure *procp = NULL;
1281 	struct svc_serv		*serv = rqstp->rq_server;
1282 	struct svc_process_info process;
1283 	__be32			*statp;
1284 	u32			prog, vers;
1285 	__be32			auth_stat, rpc_stat;
1286 	int			auth_res;
1287 	__be32			*reply_statp;
1288 
1289 	rpc_stat = rpc_success;
1290 
1291 	if (argv->iov_len < 6*4)
1292 		goto err_short_len;
1293 
1294 	/* Will be turned off by GSS integrity and privacy services */
1295 	set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
1296 	/* Will be turned off only when NFSv4 Sessions are used */
1297 	set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
1298 	clear_bit(RQ_DROPME, &rqstp->rq_flags);
1299 
1300 	svc_putu32(resv, rqstp->rq_xid);
1301 
1302 	vers = svc_getnl(argv);
1303 
1304 	/* First words of reply: */
1305 	svc_putnl(resv, 1);		/* REPLY */
1306 
1307 	if (vers != 2)		/* RPC version number */
1308 		goto err_bad_rpc;
1309 
1310 	/* Save position in case we later decide to reject: */
1311 	reply_statp = resv->iov_base + resv->iov_len;
1312 
1313 	svc_putnl(resv, 0);		/* ACCEPT */
1314 
1315 	rqstp->rq_prog = prog = svc_getnl(argv);	/* program number */
1316 	rqstp->rq_vers = svc_getnl(argv);	/* version number */
1317 	rqstp->rq_proc = svc_getnl(argv);	/* procedure number */
1318 
1319 	for (progp = serv->sv_program; progp; progp = progp->pg_next)
1320 		if (prog == progp->pg_prog)
1321 			break;
1322 
1323 	/*
1324 	 * Decode auth data, and add verifier to reply buffer.
1325 	 * We do this before anything else in order to get a decent
1326 	 * auth verifier.
1327 	 */
1328 	auth_res = svc_authenticate(rqstp, &auth_stat);
1329 	/* Also give the program a chance to reject this call: */
1330 	if (auth_res == SVC_OK && progp) {
1331 		auth_stat = rpc_autherr_badcred;
1332 		auth_res = progp->pg_authenticate(rqstp);
1333 	}
1334 	if (auth_res != SVC_OK)
1335 		trace_svc_authenticate(rqstp, auth_res, auth_stat);
1336 	switch (auth_res) {
1337 	case SVC_OK:
1338 		break;
1339 	case SVC_GARBAGE:
1340 		goto err_garbage;
1341 	case SVC_SYSERR:
1342 		rpc_stat = rpc_system_err;
1343 		goto err_bad;
1344 	case SVC_DENIED:
1345 		goto err_bad_auth;
1346 	case SVC_CLOSE:
1347 		goto close;
1348 	case SVC_DROP:
1349 		goto dropit;
1350 	case SVC_COMPLETE:
1351 		goto sendit;
1352 	}
1353 
1354 	if (progp == NULL)
1355 		goto err_bad_prog;
1356 
1357 	rpc_stat = progp->pg_init_request(rqstp, progp, &process);
1358 	switch (rpc_stat) {
1359 	case rpc_success:
1360 		break;
1361 	case rpc_prog_unavail:
1362 		goto err_bad_prog;
1363 	case rpc_prog_mismatch:
1364 		goto err_bad_vers;
1365 	case rpc_proc_unavail:
1366 		goto err_bad_proc;
1367 	}
1368 
1369 	procp = rqstp->rq_procinfo;
1370 	/* Should this check go into the dispatcher? */
1371 	if (!procp || !procp->pc_func)
1372 		goto err_bad_proc;
1373 
1374 	/* Syntactic check complete */
1375 	serv->sv_stats->rpccnt++;
1376 	trace_svc_process(rqstp, progp->pg_name);
1377 
1378 	/* Build the reply header. */
1379 	statp = resv->iov_base +resv->iov_len;
1380 	svc_putnl(resv, RPC_SUCCESS);
1381 
1382 	/* un-reserve some of the out-queue now that we have a
1383 	 * better idea of reply size
1384 	 */
1385 	if (procp->pc_xdrressize)
1386 		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
1387 
1388 	/* Call the function that processes the request. */
1389 	if (!process.dispatch) {
1390 		if (!svc_generic_dispatch(rqstp, statp))
1391 			goto release_dropit;
1392 		if (*statp == rpc_garbage_args)
1393 			goto err_garbage;
1394 		auth_stat = svc_get_autherr(rqstp, statp);
1395 		if (auth_stat != rpc_auth_ok)
1396 			goto err_release_bad_auth;
1397 	} else {
1398 		dprintk("svc: calling dispatcher\n");
1399 		if (!process.dispatch(rqstp, statp))
1400 			goto release_dropit; /* Release reply info */
1401 	}
1402 
1403 	/* Check RPC status result */
1404 	if (*statp != rpc_success)
1405 		resv->iov_len = ((void*)statp)  - resv->iov_base + 4;
1406 
1407 	/* Release reply info */
1408 	if (procp->pc_release)
1409 		procp->pc_release(rqstp);
1410 
1411 	if (procp->pc_encode == NULL)
1412 		goto dropit;
1413 
1414  sendit:
1415 	if (svc_authorise(rqstp))
1416 		goto close;
1417 	return 1;		/* Caller can now send it */
1418 
1419 release_dropit:
1420 	if (procp->pc_release)
1421 		procp->pc_release(rqstp);
1422  dropit:
1423 	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
1424 	dprintk("svc: svc_process dropit\n");
1425 	return 0;
1426 
1427  close:
1428 	if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
1429 		svc_close_xprt(rqstp->rq_xprt);
1430 	dprintk("svc: svc_process close\n");
1431 	return 0;
1432 
1433 err_short_len:
1434 	svc_printk(rqstp, "short len %zd, dropping request\n",
1435 			argv->iov_len);
1436 	goto close;
1437 
1438 err_bad_rpc:
1439 	serv->sv_stats->rpcbadfmt++;
1440 	svc_putnl(resv, 1);	/* REJECT */
1441 	svc_putnl(resv, 0);	/* RPC_MISMATCH */
1442 	svc_putnl(resv, 2);	/* Only RPCv2 supported */
1443 	svc_putnl(resv, 2);
1444 	goto sendit;
1445 
1446 err_release_bad_auth:
1447 	if (procp->pc_release)
1448 		procp->pc_release(rqstp);
1449 err_bad_auth:
1450 	dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
1451 	serv->sv_stats->rpcbadauth++;
1452 	/* Restore write pointer to location of accept status: */
1453 	xdr_ressize_check(rqstp, reply_statp);
1454 	svc_putnl(resv, 1);	/* REJECT */
1455 	svc_putnl(resv, 1);	/* AUTH_ERROR */
1456 	svc_putnl(resv, ntohl(auth_stat));	/* status */
1457 	goto sendit;
1458 
1459 err_bad_prog:
1460 	dprintk("svc: unknown program %d\n", prog);
1461 	serv->sv_stats->rpcbadfmt++;
1462 	svc_putnl(resv, RPC_PROG_UNAVAIL);
1463 	goto sendit;
1464 
1465 err_bad_vers:
1466 	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1467 		       rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);
1468 
1469 	serv->sv_stats->rpcbadfmt++;
1470 	svc_putnl(resv, RPC_PROG_MISMATCH);
1471 	svc_putnl(resv, process.mismatch.lovers);
1472 	svc_putnl(resv, process.mismatch.hivers);
1473 	goto sendit;
1474 
1475 err_bad_proc:
1476 	svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);
1477 
1478 	serv->sv_stats->rpcbadfmt++;
1479 	svc_putnl(resv, RPC_PROC_UNAVAIL);
1480 	goto sendit;
1481 
1482 err_garbage:
1483 	svc_printk(rqstp, "failed to decode args\n");
1484 
1485 	rpc_stat = rpc_garbage_args;
1486 err_bad:
1487 	serv->sv_stats->rpcbadfmt++;
1488 	svc_putnl(resv, ntohl(rpc_stat));
1489 	goto sendit;
1490 }
1491 
1492 /*
1493  * Process the RPC request.
1494  */
1495 int
1496 svc_process(struct svc_rqst *rqstp)
1497 {
1498 	struct kvec		*argv = &rqstp->rq_arg.head[0];
1499 	struct kvec		*resv = &rqstp->rq_res.head[0];
1500 	struct svc_serv		*serv = rqstp->rq_server;
1501 	u32			dir;
1502 
1503 	/*
1504 	 * Setup response xdr_buf.
1505 	 * Initially it has just one page
1506 	 */
1507 	rqstp->rq_next_page = &rqstp->rq_respages[1];
1508 	resv->iov_base = page_address(rqstp->rq_respages[0]);
1509 	resv->iov_len = 0;
1510 	rqstp->rq_res.pages = rqstp->rq_respages + 1;
1511 	rqstp->rq_res.len = 0;
1512 	rqstp->rq_res.page_base = 0;
1513 	rqstp->rq_res.page_len = 0;
1514 	rqstp->rq_res.buflen = PAGE_SIZE;
1515 	rqstp->rq_res.tail[0].iov_base = NULL;
1516 	rqstp->rq_res.tail[0].iov_len = 0;
1517 
1518 	dir  = svc_getnl(argv);
1519 	if (dir != 0) {
1520 		/* direction != CALL */
1521 		svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
1522 		serv->sv_stats->rpcbadfmt++;
1523 		goto out_drop;
1524 	}
1525 
1526 	/* Returns 1 for send, 0 for drop */
1527 	if (likely(svc_process_common(rqstp, argv, resv)))
1528 		return svc_send(rqstp);
1529 
1530 out_drop:
1531 	svc_drop(rqstp);
1532 	return 0;
1533 }
1534 EXPORT_SYMBOL_GPL(svc_process);
1535 
1536 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
1537 /*
1538  * Process a backchannel RPC request that arrived over an existing
1539  * outbound connection
1540  */
1541 int
1542 bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
1543 	       struct svc_rqst *rqstp)
1544 {
1545 	struct kvec	*argv = &rqstp->rq_arg.head[0];
1546 	struct kvec	*resv = &rqstp->rq_res.head[0];
1547 	struct rpc_task *task;
1548 	int proc_error;
1549 	int error;
1550 
1551 	dprintk("svc: %s(%p)\n", __func__, req);
1552 
1553 	/* Build the svc_rqst used by the common processing routine */
1554 	rqstp->rq_xid = req->rq_xid;
1555 	rqstp->rq_prot = req->rq_xprt->prot;
1556 	rqstp->rq_server = serv;
1557 	rqstp->rq_bc_net = req->rq_xprt->xprt_net;
1558 
1559 	rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
1560 	memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
1561 	memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
1562 	memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
1563 
1564 	/* Adjust the argument buffer length */
1565 	rqstp->rq_arg.len = req->rq_private_buf.len;
1566 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
1567 		rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
1568 		rqstp->rq_arg.page_len = 0;
1569 	} else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
1570 			rqstp->rq_arg.page_len)
1571 		rqstp->rq_arg.page_len = rqstp->rq_arg.len -
1572 			rqstp->rq_arg.head[0].iov_len;
1573 	else
1574 		rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
1575 			rqstp->rq_arg.page_len;
1576 
1577 	/* reset result send buffer "put" position */
1578 	resv->iov_len = 0;
1579 
1580 	/*
1581 	 * Skip the next two words because they've already been
1582 	 * processed in the transport
1583 	 */
1584 	svc_getu32(argv);	/* XID */
1585 	svc_getnl(argv);	/* CALLDIR */
1586 
1587 	/* Parse and execute the bc call */
1588 	proc_error = svc_process_common(rqstp, argv, resv);
1589 
1590 	atomic_dec(&req->rq_xprt->bc_slot_count);
1591 	if (!proc_error) {
1592 		/* Processing error: drop the request */
1593 		xprt_free_bc_request(req);
1594 		error = -EINVAL;
1595 		goto out;
1596 	}
1597 	/* Finally, send the reply synchronously */
1598 	memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
1599 	task = rpc_run_bc_task(req);
1600 	if (IS_ERR(task)) {
1601 		error = PTR_ERR(task);
1602 		goto out;
1603 	}
1604 
1605 	WARN_ON_ONCE(atomic_read(&task->tk_count) != 1);
1606 	error = task->tk_status;
1607 	rpc_put_task(task);
1608 
1609 out:
1610 	dprintk("svc: %s(), error=%d\n", __func__, error);
1611 	return error;
1612 }
1613 EXPORT_SYMBOL_GPL(bc_svc_process);
1614 #endif /* CONFIG_SUNRPC_BACKCHANNEL */
1615 
1616 /*
1617  * Return (transport-specific) limit on the rpc payload.
1618  */
1619 u32 svc_max_payload(const struct svc_rqst *rqstp)
1620 {
1621 	u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
1622 
1623 	if (rqstp->rq_server->sv_max_payload < max)
1624 		max = rqstp->rq_server->sv_max_payload;
1625 	return max;
1626 }
1627 EXPORT_SYMBOL_GPL(svc_max_payload);
1628 
1629 /**
1630  * svc_encode_result_payload - mark a range of bytes as a result payload
1631  * @rqstp: svc_rqst to operate on
1632  * @offset: payload's byte offset in rqstp->rq_res
1633  * @length: size of payload, in bytes
1634  *
1635  * Returns zero on success, or a negative errno if a permanent
1636  * error occurred.
1637  */
1638 int svc_encode_result_payload(struct svc_rqst *rqstp, unsigned int offset,
1639 			      unsigned int length)
1640 {
1641 	return rqstp->rq_xprt->xpt_ops->xpo_result_payload(rqstp, offset,
1642 							   length);
1643 }
1644 EXPORT_SYMBOL_GPL(svc_encode_result_payload);
1645 
1646 /**
1647  * svc_fill_write_vector - Construct data argument for VFS write call
1648  * @rqstp: svc_rqst to operate on
1649  * @pages: list of pages containing data payload
1650  * @first: buffer containing first section of write payload
1651  * @total: total number of bytes of write payload
1652  *
1653  * Fills in rqstp::rq_vec, and returns the number of elements.
1654  */
1655 unsigned int svc_fill_write_vector(struct svc_rqst *rqstp, struct page **pages,
1656 				   struct kvec *first, size_t total)
1657 {
1658 	struct kvec *vec = rqstp->rq_vec;
1659 	unsigned int i;
1660 
1661 	/* Some types of transport can present the write payload
1662 	 * entirely in rq_arg.pages. In this case, @first is empty.
1663 	 */
1664 	i = 0;
1665 	if (first->iov_len) {
1666 		vec[i].iov_base = first->iov_base;
1667 		vec[i].iov_len = min_t(size_t, total, first->iov_len);
1668 		total -= vec[i].iov_len;
1669 		++i;
1670 	}
1671 
1672 	while (total) {
1673 		vec[i].iov_base = page_address(*pages);
1674 		vec[i].iov_len = min_t(size_t, total, PAGE_SIZE);
1675 		total -= vec[i].iov_len;
1676 		++i;
1677 		++pages;
1678 	}
1679 
1680 	WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
1681 	return i;
1682 }
1683 EXPORT_SYMBOL_GPL(svc_fill_write_vector);
1684 
1685 /**
1686  * svc_fill_symlink_pathname - Construct pathname argument for VFS symlink call
1687  * @rqstp: svc_rqst to operate on
1688  * @first: buffer containing first section of pathname
1689  * @p: buffer containing remaining section of pathname
1690  * @total: total length of the pathname argument
1691  *
1692  * The VFS symlink API demands a NUL-terminated pathname in mapped memory.
1693  * Returns pointer to a NUL-terminated string, or an ERR_PTR. Caller must free
1694  * the returned string.
1695  */
1696 char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,
1697 				void *p, size_t total)
1698 {
1699 	size_t len, remaining;
1700 	char *result, *dst;
1701 
1702 	result = kmalloc(total + 1, GFP_KERNEL);
1703 	if (!result)
1704 		return ERR_PTR(-ESERVERFAULT);
1705 
1706 	dst = result;
1707 	remaining = total;
1708 
1709 	len = min_t(size_t, total, first->iov_len);
1710 	if (len) {
1711 		memcpy(dst, first->iov_base, len);
1712 		dst += len;
1713 		remaining -= len;
1714 	}
1715 
1716 	if (remaining) {
1717 		len = min_t(size_t, remaining, PAGE_SIZE);
1718 		memcpy(dst, p, len);
1719 		dst += len;
1720 	}
1721 
1722 	*dst = '\0';
1723 
1724 	/* Sanity check: Linux doesn't allow the pathname argument to
1725 	 * contain a NUL byte.
1726 	 */
1727 	if (strlen(result) != total) {
1728 		kfree(result);
1729 		return ERR_PTR(-EINVAL);
1730 	}
1731 	return result;
1732 }
1733 EXPORT_SYMBOL_GPL(svc_fill_symlink_pathname);
1734