xref: /linux/net/sunrpc/xprtmultipath.c (revision 96ac6d435100450f0565708d9b885ea2a7400e0a) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Multipath support for RPC
4  *
5  * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
6  *
7  * Trond Myklebust <trond.myklebust@primarydata.com>
8  *
9  */
10 #include <linux/types.h>
11 #include <linux/kref.h>
12 #include <linux/list.h>
13 #include <linux/rcupdate.h>
14 #include <linux/rculist.h>
15 #include <linux/slab.h>
16 #include <asm/cmpxchg.h>
17 #include <linux/spinlock.h>
18 #include <linux/sunrpc/xprt.h>
19 #include <linux/sunrpc/addr.h>
20 #include <linux/sunrpc/xprtmultipath.h>
21 
22 typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
23 		const struct rpc_xprt *cur);
24 
25 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
26 static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
27 static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
28 
29 static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
30 		struct rpc_xprt *xprt)
31 {
32 	if (unlikely(xprt_get(xprt) == NULL))
33 		return;
34 	list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
35 	smp_wmb();
36 	if (xps->xps_nxprts == 0)
37 		xps->xps_net = xprt->xprt_net;
38 	xps->xps_nxprts++;
39 }
40 
41 /**
42  * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
43  * @xps: pointer to struct rpc_xprt_switch
44  * @xprt: pointer to struct rpc_xprt
45  *
46  * Adds xprt to the end of the list of struct rpc_xprt in xps.
47  */
48 void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
49 		struct rpc_xprt *xprt)
50 {
51 	if (xprt == NULL)
52 		return;
53 	spin_lock(&xps->xps_lock);
54 	if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) &&
55 	    !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr))
56 		xprt_switch_add_xprt_locked(xps, xprt);
57 	spin_unlock(&xps->xps_lock);
58 }
59 
60 static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
61 		struct rpc_xprt *xprt)
62 {
63 	if (unlikely(xprt == NULL))
64 		return;
65 	xps->xps_nxprts--;
66 	if (xps->xps_nxprts == 0)
67 		xps->xps_net = NULL;
68 	smp_wmb();
69 	list_del_rcu(&xprt->xprt_switch);
70 }
71 
72 /**
73  * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
74  * @xps: pointer to struct rpc_xprt_switch
75  * @xprt: pointer to struct rpc_xprt
76  *
77  * Removes xprt from the list of struct rpc_xprt in xps.
78  */
79 void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
80 		struct rpc_xprt *xprt)
81 {
82 	spin_lock(&xps->xps_lock);
83 	xprt_switch_remove_xprt_locked(xps, xprt);
84 	spin_unlock(&xps->xps_lock);
85 	xprt_put(xprt);
86 }
87 
88 /**
89  * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
90  * @xprt: pointer to struct rpc_xprt
91  * @gfp_flags: allocation flags
92  *
93  * On success, returns an initialised struct rpc_xprt_switch, containing
94  * the entry xprt. Returns NULL on failure.
95  */
96 struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
97 		gfp_t gfp_flags)
98 {
99 	struct rpc_xprt_switch *xps;
100 
101 	xps = kmalloc(sizeof(*xps), gfp_flags);
102 	if (xps != NULL) {
103 		spin_lock_init(&xps->xps_lock);
104 		kref_init(&xps->xps_kref);
105 		xps->xps_nxprts = 0;
106 		INIT_LIST_HEAD(&xps->xps_xprt_list);
107 		xps->xps_iter_ops = &rpc_xprt_iter_singular;
108 		xprt_switch_add_xprt_locked(xps, xprt);
109 	}
110 
111 	return xps;
112 }
113 
114 static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
115 {
116 	spin_lock(&xps->xps_lock);
117 	while (!list_empty(&xps->xps_xprt_list)) {
118 		struct rpc_xprt *xprt;
119 
120 		xprt = list_first_entry(&xps->xps_xprt_list,
121 				struct rpc_xprt, xprt_switch);
122 		xprt_switch_remove_xprt_locked(xps, xprt);
123 		spin_unlock(&xps->xps_lock);
124 		xprt_put(xprt);
125 		spin_lock(&xps->xps_lock);
126 	}
127 	spin_unlock(&xps->xps_lock);
128 }
129 
130 static void xprt_switch_free(struct kref *kref)
131 {
132 	struct rpc_xprt_switch *xps = container_of(kref,
133 			struct rpc_xprt_switch, xps_kref);
134 
135 	xprt_switch_free_entries(xps);
136 	kfree_rcu(xps, xps_rcu);
137 }
138 
139 /**
140  * xprt_switch_get - Return a reference to a rpc_xprt_switch
141  * @xps: pointer to struct rpc_xprt_switch
142  *
143  * Returns a reference to xps unless the refcount is already zero.
144  */
145 struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
146 {
147 	if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
148 		return xps;
149 	return NULL;
150 }
151 
152 /**
153  * xprt_switch_put - Release a reference to a rpc_xprt_switch
154  * @xps: pointer to struct rpc_xprt_switch
155  *
156  * Release the reference to xps, and free it once the refcount is zero.
157  */
158 void xprt_switch_put(struct rpc_xprt_switch *xps)
159 {
160 	if (xps != NULL)
161 		kref_put(&xps->xps_kref, xprt_switch_free);
162 }
163 
164 /**
165  * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
166  * @xps: pointer to struct rpc_xprt_switch
167  *
168  * Sets a round-robin default policy for iterators acting on xps.
169  */
170 void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
171 {
172 	if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
173 		WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
174 }
175 
176 static
177 const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
178 {
179 	if (xpi->xpi_ops != NULL)
180 		return xpi->xpi_ops;
181 	return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
182 }
183 
184 static
185 void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
186 {
187 }
188 
189 static
190 void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
191 {
192 	WRITE_ONCE(xpi->xpi_cursor, NULL);
193 }
194 
195 static
196 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
197 {
198 	return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
199 }
200 
201 static
202 struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
203 {
204 	struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
205 
206 	if (xps == NULL)
207 		return NULL;
208 	return xprt_switch_find_first_entry(&xps->xps_xprt_list);
209 }
210 
211 static
212 struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
213 		const struct rpc_xprt *cur)
214 {
215 	struct rpc_xprt *pos;
216 
217 	list_for_each_entry_rcu(pos, head, xprt_switch) {
218 		if (cur == pos)
219 			return pos;
220 	}
221 	return NULL;
222 }
223 
224 static
225 struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
226 {
227 	struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
228 	struct list_head *head;
229 
230 	if (xps == NULL)
231 		return NULL;
232 	head = &xps->xps_xprt_list;
233 	if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
234 		return xprt_switch_find_first_entry(head);
235 	return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
236 }
237 
238 bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
239 			      const struct sockaddr *sap)
240 {
241 	struct list_head *head;
242 	struct rpc_xprt *pos;
243 
244 	if (xps == NULL || sap == NULL)
245 		return false;
246 
247 	head = &xps->xps_xprt_list;
248 	list_for_each_entry_rcu(pos, head, xprt_switch) {
249 		if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) {
250 			pr_info("RPC:   addr %s already in xprt switch\n",
251 				pos->address_strings[RPC_DISPLAY_ADDR]);
252 			return true;
253 		}
254 	}
255 	return false;
256 }
257 
258 static
259 struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
260 		const struct rpc_xprt *cur)
261 {
262 	struct rpc_xprt *pos, *prev = NULL;
263 
264 	list_for_each_entry_rcu(pos, head, xprt_switch) {
265 		if (cur == prev)
266 			return pos;
267 		prev = pos;
268 	}
269 	return NULL;
270 }
271 
272 static
273 struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
274 		struct rpc_xprt **cursor,
275 		xprt_switch_find_xprt_t find_next)
276 {
277 	struct rpc_xprt *cur, *pos, *old;
278 
279 	cur = READ_ONCE(*cursor);
280 	for (;;) {
281 		old = cur;
282 		pos = find_next(head, old);
283 		if (pos == NULL)
284 			break;
285 		cur = cmpxchg_relaxed(cursor, old, pos);
286 		if (cur == old)
287 			break;
288 	}
289 	return pos;
290 }
291 
292 static
293 struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
294 		xprt_switch_find_xprt_t find_next)
295 {
296 	struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
297 
298 	if (xps == NULL)
299 		return NULL;
300 	return xprt_switch_set_next_cursor(&xps->xps_xprt_list,
301 			&xpi->xpi_cursor,
302 			find_next);
303 }
304 
305 static
306 struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
307 		const struct rpc_xprt *cur)
308 {
309 	struct rpc_xprt *ret;
310 
311 	ret = xprt_switch_find_next_entry(head, cur);
312 	if (ret != NULL)
313 		return ret;
314 	return xprt_switch_find_first_entry(head);
315 }
316 
317 static
318 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
319 {
320 	return xprt_iter_next_entry_multiple(xpi,
321 			xprt_switch_find_next_entry_roundrobin);
322 }
323 
324 static
325 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
326 {
327 	return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
328 }
329 
330 /*
331  * xprt_iter_rewind - Resets the xprt iterator
332  * @xpi: pointer to rpc_xprt_iter
333  *
334  * Resets xpi to ensure that it points to the first entry in the list
335  * of transports.
336  */
337 static
338 void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
339 {
340 	rcu_read_lock();
341 	xprt_iter_ops(xpi)->xpi_rewind(xpi);
342 	rcu_read_unlock();
343 }
344 
345 static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
346 		struct rpc_xprt_switch *xps,
347 		const struct rpc_xprt_iter_ops *ops)
348 {
349 	rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
350 	xpi->xpi_cursor = NULL;
351 	xpi->xpi_ops = ops;
352 }
353 
354 /**
355  * xprt_iter_init - Initialise an xprt iterator
356  * @xpi: pointer to rpc_xprt_iter
357  * @xps: pointer to rpc_xprt_switch
358  *
359  * Initialises the iterator to use the default iterator ops
360  * as set in xps. This function is mainly intended for internal
361  * use in the rpc_client.
362  */
363 void xprt_iter_init(struct rpc_xprt_iter *xpi,
364 		struct rpc_xprt_switch *xps)
365 {
366 	__xprt_iter_init(xpi, xps, NULL);
367 }
368 
369 /**
370  * xprt_iter_init_listall - Initialise an xprt iterator
371  * @xpi: pointer to rpc_xprt_iter
372  * @xps: pointer to rpc_xprt_switch
373  *
374  * Initialises the iterator to iterate once through the entire list
375  * of entries in xps.
376  */
377 void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
378 		struct rpc_xprt_switch *xps)
379 {
380 	__xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
381 }
382 
383 /**
384  * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
385  * @xpi: pointer to rpc_xprt_iter
386  * @newswitch: pointer to a new rpc_xprt_switch or NULL
387  *
388  * Swaps out the existing xpi->xpi_xpswitch with a new value.
389  */
390 struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
391 		struct rpc_xprt_switch *newswitch)
392 {
393 	struct rpc_xprt_switch __rcu *oldswitch;
394 
395 	/* Atomically swap out the old xpswitch */
396 	oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
397 	if (newswitch != NULL)
398 		xprt_iter_rewind(xpi);
399 	return rcu_dereference_protected(oldswitch, true);
400 }
401 
402 /**
403  * xprt_iter_destroy - Destroys the xprt iterator
404  * @xpi: pointer to rpc_xprt_iter
405  */
406 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
407 {
408 	xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
409 }
410 
411 /**
412  * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
413  * @xpi: pointer to rpc_xprt_iter
414  *
415  * Returns a pointer to the struct rpc_xprt that is currently
416  * pointed to by the cursor.
417  * Caller must be holding rcu_read_lock().
418  */
419 struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
420 {
421 	WARN_ON_ONCE(!rcu_read_lock_held());
422 	return xprt_iter_ops(xpi)->xpi_xprt(xpi);
423 }
424 
425 static
426 struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
427 		struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
428 {
429 	struct rpc_xprt *ret;
430 
431 	do {
432 		ret = fn(xpi);
433 		if (ret == NULL)
434 			break;
435 		ret = xprt_get(ret);
436 	} while (ret == NULL);
437 	return ret;
438 }
439 
440 /**
441  * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
442  * @xpi: pointer to rpc_xprt_iter
443  *
444  * Returns a reference to the struct rpc_xprt that is currently
445  * pointed to by the cursor.
446  */
447 struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
448 {
449 	struct rpc_xprt *xprt;
450 
451 	rcu_read_lock();
452 	xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
453 	rcu_read_unlock();
454 	return xprt;
455 }
456 
457 /**
458  * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
459  * @xpi: pointer to rpc_xprt_iter
460  *
461  * Returns a reference to the struct rpc_xprt that immediately follows the
462  * entry pointed to by the cursor.
463  */
464 struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
465 {
466 	struct rpc_xprt *xprt;
467 
468 	rcu_read_lock();
469 	xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
470 	rcu_read_unlock();
471 	return xprt;
472 }
473 
474 /* Policy for always returning the first entry in the rpc_xprt_switch */
475 static
476 const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
477 	.xpi_rewind = xprt_iter_no_rewind,
478 	.xpi_xprt = xprt_iter_first_entry,
479 	.xpi_next = xprt_iter_first_entry,
480 };
481 
482 /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
483 static
484 const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
485 	.xpi_rewind = xprt_iter_default_rewind,
486 	.xpi_xprt = xprt_iter_current_entry,
487 	.xpi_next = xprt_iter_next_entry_roundrobin,
488 };
489 
490 /* Policy for once-through iteration of entries in the rpc_xprt_switch */
491 static
492 const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
493 	.xpi_rewind = xprt_iter_default_rewind,
494 	.xpi_xprt = xprt_iter_current_entry,
495 	.xpi_next = xprt_iter_next_entry_all,
496 };
497