xref: /linux/net/mptcp/pm_kernel.c (revision 49b09ae0bf57b4230187d4cdd0a370558e44a6e8)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3  *
4  * Copyright (c) 2025, Matthieu Baerts.
5  */
6 
7 #define pr_fmt(fmt) "MPTCP: " fmt
8 
9 #include <net/netns/generic.h>
10 
11 #include "protocol.h"
12 #include "mib.h"
13 #include "mptcp_pm_gen.h"
14 
15 static int pm_nl_pernet_id;
16 
17 struct pm_nl_pernet {
18 	/* protects pernet updates */
19 	spinlock_t		lock;
20 	struct list_head	endp_list;
21 	u8			endpoints;
22 	u8			endp_signal_max;
23 	u8			endp_subflow_max;
24 	u8			endp_laminar_max;
25 	u8			endp_fullmesh_max;
26 	u8			limit_add_addr_accepted;
27 	u8			limit_extra_subflows;
28 	u8			next_id;
29 	DECLARE_BITMAP(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
30 };
31 
32 #define MPTCP_PM_ADDR_MAX	8
33 #define MPTCP_PM_SUBFLOWS_MAX	64
34 
35 static struct pm_nl_pernet *pm_nl_get_pernet(const struct net *net)
36 {
37 	return net_generic(net, pm_nl_pernet_id);
38 }
39 
40 static struct pm_nl_pernet *
41 pm_nl_get_pernet_from_msk(const struct mptcp_sock *msk)
42 {
43 	return pm_nl_get_pernet(sock_net((struct sock *)msk));
44 }
45 
46 static struct pm_nl_pernet *genl_info_pm_nl(struct genl_info *info)
47 {
48 	return pm_nl_get_pernet(genl_info_net(info));
49 }
50 
51 u8 mptcp_pm_get_endp_signal_max(const struct mptcp_sock *msk)
52 {
53 	const struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
54 
55 	return READ_ONCE(pernet->endp_signal_max);
56 }
57 EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_signal_max);
58 
59 u8 mptcp_pm_get_endp_subflow_max(const struct mptcp_sock *msk)
60 {
61 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
62 
63 	return READ_ONCE(pernet->endp_subflow_max);
64 }
65 EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_subflow_max);
66 
67 u8 mptcp_pm_get_endp_laminar_max(const struct mptcp_sock *msk)
68 {
69 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
70 
71 	return READ_ONCE(pernet->endp_laminar_max);
72 }
73 EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_laminar_max);
74 
75 u8 mptcp_pm_get_endp_fullmesh_max(const struct mptcp_sock *msk)
76 {
77 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
78 
79 	return READ_ONCE(pernet->endp_fullmesh_max);
80 }
81 EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_fullmesh_max);
82 
83 u8 mptcp_pm_get_limit_add_addr_accepted(const struct mptcp_sock *msk)
84 {
85 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
86 
87 	return READ_ONCE(pernet->limit_add_addr_accepted);
88 }
89 EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_add_addr_accepted);
90 
91 u8 mptcp_pm_get_limit_extra_subflows(const struct mptcp_sock *msk)
92 {
93 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
94 
95 	return READ_ONCE(pernet->limit_extra_subflows);
96 }
97 EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_extra_subflows);
98 
99 static bool has_subflow_daddr(const struct mptcp_sock *msk,
100 			      const struct mptcp_addr_info *daddr)
101 {
102 	struct mptcp_subflow_context *subflow;
103 	struct mptcp_addr_info cur;
104 
105 	mptcp_for_each_subflow(msk, subflow) {
106 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
107 
108 		if (!((1 << inet_sk_state_load(ssk)) &
109 		      (TCPF_ESTABLISHED | TCPF_SYN_SENT | TCPF_SYN_RECV)))
110 			continue;
111 
112 		mptcp_remote_address((struct sock_common *)ssk, &cur);
113 		if (mptcp_addresses_equal(&cur, daddr, daddr->port))
114 			return true;
115 	}
116 
117 	return false;
118 }
119 
120 static bool
121 select_local_address(const struct pm_nl_pernet *pernet,
122 		     const struct mptcp_sock *msk,
123 		     struct mptcp_pm_local *new_local)
124 {
125 	struct mptcp_pm_addr_entry *entry;
126 	bool found = false;
127 
128 	msk_owned_by_me(msk);
129 
130 	rcu_read_lock();
131 	list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
132 		if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW))
133 			continue;
134 
135 		if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap))
136 			continue;
137 
138 		new_local->addr = entry->addr;
139 		new_local->flags = entry->flags;
140 		new_local->ifindex = entry->ifindex;
141 		found = true;
142 		break;
143 	}
144 	rcu_read_unlock();
145 
146 	return found;
147 }
148 
149 static bool
150 select_signal_address(struct pm_nl_pernet *pernet, const struct mptcp_sock *msk,
151 		      struct mptcp_pm_local *new_local)
152 {
153 	struct mptcp_pm_addr_entry *entry;
154 	bool found = false;
155 
156 	rcu_read_lock();
157 	/* do not keep any additional per socket state, just signal
158 	 * the address list in order.
159 	 * Note: removal from the local address list during the msk life-cycle
160 	 * can lead to additional addresses not being announced.
161 	 */
162 	list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
163 		if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap))
164 			continue;
165 
166 		if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL))
167 			continue;
168 
169 		new_local->addr = entry->addr;
170 		new_local->flags = entry->flags;
171 		new_local->ifindex = entry->ifindex;
172 		found = true;
173 		break;
174 	}
175 	rcu_read_unlock();
176 
177 	return found;
178 }
179 
180 static unsigned int
181 fill_remote_addr(struct mptcp_sock *msk, struct mptcp_addr_info *local,
182 		 struct mptcp_addr_info *addrs)
183 {
184 	bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0);
185 	struct mptcp_addr_info remote = { 0 };
186 	struct sock *sk = (struct sock *)msk;
187 
188 	if (deny_id0)
189 		return 0;
190 
191 	mptcp_remote_address((struct sock_common *)sk, &remote);
192 
193 	if (!mptcp_pm_addr_families_match(sk, local, &remote))
194 		return 0;
195 
196 	msk->pm.extra_subflows++;
197 	*addrs = remote;
198 
199 	return 1;
200 }
201 
202 static unsigned int
203 fill_remote_addresses_fullmesh(struct mptcp_sock *msk,
204 			       struct mptcp_addr_info *local,
205 			       struct mptcp_addr_info *addrs,
206 			       int addrs_size)
207 {
208 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
209 	bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0);
210 	DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
211 	struct sock *sk = (struct sock *)msk, *ssk;
212 	struct mptcp_subflow_context *subflow;
213 	int i = 0;
214 
215 	/* Forbid creation of new subflows matching existing ones, possibly
216 	 * already created by incoming ADD_ADDR
217 	 */
218 	bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
219 	mptcp_for_each_subflow(msk, subflow)
220 		if (READ_ONCE(subflow->local_id) == local->id)
221 			__set_bit(subflow->remote_id, unavail_id);
222 
223 	mptcp_for_each_subflow(msk, subflow) {
224 		ssk = mptcp_subflow_tcp_sock(subflow);
225 		mptcp_remote_address((struct sock_common *)ssk, &addrs[i]);
226 		addrs[i].id = READ_ONCE(subflow->remote_id);
227 		if (deny_id0 && !addrs[i].id)
228 			continue;
229 
230 		if (test_bit(addrs[i].id, unavail_id))
231 			continue;
232 
233 		if (!mptcp_pm_addr_families_match(sk, local, &addrs[i]))
234 			continue;
235 
236 		/* forbid creating multiple address towards this id */
237 		__set_bit(addrs[i].id, unavail_id);
238 		msk->pm.extra_subflows++;
239 		i++;
240 
241 		if (msk->pm.extra_subflows >= limit_extra_subflows ||
242 		    i == addrs_size)
243 			break;
244 	}
245 
246 	return i;
247 }
248 
249 /* Fill all the remote addresses into the array addrs[],
250  * and return the array size.
251  */
252 static unsigned int
253 fill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local,
254 			  bool fullmesh, struct mptcp_addr_info *addrs,
255 			  int addrs_size)
256 {
257 	/* Non-fullmesh: fill in the single entry corresponding to the primary
258 	 * MPC subflow remote address, and return 1, corresponding to 1 entry.
259 	 */
260 	if (!fullmesh)
261 		return fill_remote_addr(msk, local, addrs);
262 
263 	/* Fullmesh endpoint: fill all possible remote addresses */
264 	return fill_remote_addresses_fullmesh(msk, local, addrs, addrs_size);
265 }
266 
267 static struct mptcp_pm_addr_entry *
268 __lookup_addr_by_id(struct pm_nl_pernet *pernet, unsigned int id)
269 {
270 	struct mptcp_pm_addr_entry *entry;
271 
272 	list_for_each_entry_rcu(entry, &pernet->endp_list, list,
273 				lockdep_is_held(&pernet->lock)) {
274 		if (entry->addr.id == id)
275 			return entry;
276 	}
277 	return NULL;
278 }
279 
280 static struct mptcp_pm_addr_entry *
281 __lookup_addr(struct pm_nl_pernet *pernet, const struct mptcp_addr_info *info)
282 {
283 	struct mptcp_pm_addr_entry *entry;
284 
285 	list_for_each_entry_rcu(entry, &pernet->endp_list, list,
286 				lockdep_is_held(&pernet->lock)) {
287 		if (mptcp_addresses_equal(&entry->addr, info, entry->addr.port))
288 			return entry;
289 	}
290 	return NULL;
291 }
292 
293 static u8 mptcp_endp_get_local_id(struct mptcp_sock *msk,
294 				  const struct mptcp_addr_info *addr)
295 {
296 	return msk->mpc_endpoint_id == addr->id ? 0 : addr->id;
297 }
298 
299 /* Set mpc_endpoint_id, and send MP_PRIO for ID0 if needed */
300 static void mptcp_mpc_endpoint_setup(struct mptcp_sock *msk)
301 {
302 	struct mptcp_subflow_context *subflow;
303 	struct mptcp_pm_addr_entry *entry;
304 	struct mptcp_addr_info mpc_addr;
305 	struct pm_nl_pernet *pernet;
306 	bool backup = false;
307 
308 	/* do lazy endpoint usage accounting for the MPC subflows */
309 	if (likely(msk->pm.status & BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED)) ||
310 	    !msk->first)
311 		return;
312 
313 	subflow = mptcp_subflow_ctx(msk->first);
314 	pernet = pm_nl_get_pernet_from_msk(msk);
315 
316 	mptcp_local_address((struct sock_common *)msk->first, &mpc_addr);
317 	rcu_read_lock();
318 	entry = __lookup_addr(pernet, &mpc_addr);
319 	if (entry) {
320 		__clear_bit(entry->addr.id, msk->pm.id_avail_bitmap);
321 		msk->mpc_endpoint_id = entry->addr.id;
322 		backup = !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
323 	}
324 	rcu_read_unlock();
325 
326 	/* Send MP_PRIO */
327 	if (backup)
328 		mptcp_pm_send_ack(msk, subflow, true, backup);
329 
330 	msk->pm.status |= BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED);
331 }
332 
333 static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
334 {
335 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
336 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
337 	u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk);
338 	u8 endp_signal_max = mptcp_pm_get_endp_signal_max(msk);
339 	struct sock *sk = (struct sock *)msk;
340 	bool signal_and_subflow = false;
341 	struct mptcp_pm_local local;
342 
343 	mptcp_mpc_endpoint_setup(msk);
344 	if (!mptcp_is_fully_established(sk))
345 		return;
346 
347 	pr_debug("local %d:%d signal %d:%d subflows %d:%d\n",
348 		 msk->pm.local_addr_used, endp_subflow_max,
349 		 msk->pm.add_addr_signaled, endp_signal_max,
350 		 msk->pm.extra_subflows, limit_extra_subflows);
351 
352 	/* check first for announce */
353 	if (msk->pm.add_addr_signaled < endp_signal_max) {
354 		u8 endp_id;
355 
356 		/* due to racing events on both ends we can reach here while
357 		 * previous add address is still running: if we invoke now
358 		 * mptcp_pm_announce_addr(), that will fail and the
359 		 * corresponding id will be marked as used.
360 		 * Instead let the PM machinery reschedule us when the
361 		 * current address announce will be completed.
362 		 */
363 		if (msk->pm.addr_signal & BIT(MPTCP_ADD_ADDR_SIGNAL))
364 			return;
365 
366 		if (!select_signal_address(pernet, msk, &local))
367 			goto subflow;
368 
369 		/* Special case for ID0: set the correct ID */
370 		endp_id = local.addr.id;
371 		if (endp_id == msk->mpc_endpoint_id)
372 			local.addr.id = 0;
373 
374 		/* If the alloc fails, we are on memory pressure, not worth
375 		 * continuing, and trying to create subflows.
376 		 */
377 		if (!mptcp_pm_announced_alloc(msk, &local.addr))
378 			return;
379 
380 		__clear_bit(endp_id, msk->pm.id_avail_bitmap);
381 		msk->pm.add_addr_signaled++;
382 
383 		mptcp_pm_announce_addr(msk, &local.addr, false);
384 		mptcp_pm_addr_send_ack(msk);
385 
386 		if (local.flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)
387 			signal_and_subflow = true;
388 	}
389 
390 subflow:
391 	/* No need to try establishing subflows to remote id0 if not allowed */
392 	if (mptcp_pm_add_addr_c_flag_case(msk))
393 		goto exit;
394 
395 	/* check if should create a new subflow */
396 	while (msk->pm.local_addr_used < endp_subflow_max &&
397 	       msk->pm.extra_subflows < limit_extra_subflows) {
398 		struct mptcp_addr_info addrs[MPTCP_PM_ADDR_MAX];
399 		bool fullmesh;
400 		int i, nr;
401 
402 		if (signal_and_subflow)
403 			signal_and_subflow = false;
404 		else if (!select_local_address(pernet, msk, &local))
405 			break;
406 
407 		fullmesh = !!(local.flags & MPTCP_PM_ADDR_FLAG_FULLMESH);
408 
409 		__clear_bit(local.addr.id, msk->pm.id_avail_bitmap);
410 
411 		/* Special case for ID0: set the correct ID */
412 		if (local.addr.id == msk->mpc_endpoint_id)
413 			local.addr.id = 0;
414 		else /* local_addr_used is not decr for ID 0 */
415 			msk->pm.local_addr_used++;
416 
417 		nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh,
418 					       addrs, ARRAY_SIZE(addrs));
419 		if (nr == 0)
420 			continue;
421 
422 		spin_unlock_bh(&msk->pm.lock);
423 		for (i = 0; i < nr; i++)
424 			__mptcp_subflow_connect(sk, &local, &addrs[i]);
425 		spin_lock_bh(&msk->pm.lock);
426 	}
427 
428 exit:
429 	/* If an endpoint has both the signal and subflow flags, but it is not
430 	 * possible to create subflows -- the 'while' loop body above never
431 	 * executed --  then still mark the endp as used, which is somehow the
432 	 * case. This avoids issues later when removing the endpoint and calling
433 	 * __mark_subflow_endp_available(), which expects the increment here.
434 	 */
435 	if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id)
436 		msk->pm.local_addr_used++;
437 
438 	mptcp_pm_nl_check_work_pending(msk);
439 }
440 
441 static void mptcp_pm_nl_fully_established(struct mptcp_sock *msk)
442 {
443 	mptcp_pm_create_subflow_or_signal_addr(msk);
444 }
445 
446 static void mptcp_pm_nl_subflow_established(struct mptcp_sock *msk)
447 {
448 	mptcp_pm_create_subflow_or_signal_addr(msk);
449 }
450 
451 static unsigned int
452 fill_local_addresses_vec_fullmesh(struct mptcp_sock *msk,
453 				  struct mptcp_addr_info *remote,
454 				  struct mptcp_pm_local *locals,
455 				  int locals_size,
456 				  bool c_flag_case)
457 {
458 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
459 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
460 	struct sock *sk = (struct sock *)msk;
461 	struct mptcp_pm_addr_entry *entry;
462 	struct mptcp_pm_local *local;
463 	int i = 0;
464 
465 	rcu_read_lock();
466 	list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
467 		bool is_id0;
468 
469 		if (!(entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH))
470 			continue;
471 
472 		if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote))
473 			continue;
474 
475 		local = &locals[i];
476 		local->addr = entry->addr;
477 		local->flags = entry->flags;
478 		local->ifindex = entry->ifindex;
479 
480 		is_id0 = local->addr.id == msk->mpc_endpoint_id;
481 
482 		if (c_flag_case &&
483 		    (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) {
484 			__clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
485 
486 			if (!is_id0)
487 				msk->pm.local_addr_used++;
488 		}
489 
490 		/* Special case for ID0: set the correct ID */
491 		if (is_id0)
492 			local->addr.id = 0;
493 
494 		msk->pm.extra_subflows++;
495 		i++;
496 
497 		if (msk->pm.extra_subflows >= limit_extra_subflows ||
498 		    i == locals_size)
499 			break;
500 	}
501 	rcu_read_unlock();
502 
503 	return i;
504 }
505 
506 static unsigned int
507 fill_local_laminar_endp(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
508 			struct mptcp_pm_local *locals)
509 {
510 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
511 	DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
512 	struct mptcp_subflow_context *subflow;
513 	struct sock *sk = (struct sock *)msk;
514 	struct mptcp_pm_addr_entry *entry;
515 	struct mptcp_pm_local *local;
516 	int found = 0;
517 
518 	/* Forbid creation of new subflows matching existing ones, possibly
519 	 * already created by 'subflow' endpoints
520 	 */
521 	bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1);
522 	mptcp_for_each_subflow(msk, subflow) {
523 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
524 
525 		if ((1 << inet_sk_state_load(ssk)) &
526 		    (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING |
527 		     TCPF_CLOSE))
528 			continue;
529 
530 		__set_bit(subflow_get_local_id(subflow), unavail_id);
531 	}
532 
533 	rcu_read_lock();
534 	list_for_each_entry_rcu(entry, &pernet->endp_list, list) {
535 		if (!(entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR))
536 			continue;
537 
538 		if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote))
539 			continue;
540 
541 		if (test_bit(mptcp_endp_get_local_id(msk, &entry->addr),
542 			     unavail_id))
543 			continue;
544 
545 		local = &locals[0];
546 		local->addr = entry->addr;
547 		local->flags = entry->flags;
548 		local->ifindex = entry->ifindex;
549 
550 		if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
551 			__clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
552 
553 			if (local->addr.id != msk->mpc_endpoint_id)
554 				msk->pm.local_addr_used++;
555 		}
556 
557 		msk->pm.extra_subflows++;
558 		found = 1;
559 		break;
560 	}
561 	rcu_read_unlock();
562 
563 	return found;
564 }
565 
566 static unsigned int
567 fill_local_addresses_vec_c_flag(struct mptcp_sock *msk,
568 				struct mptcp_addr_info *remote,
569 				struct mptcp_pm_local *locals,
570 				int locals_size)
571 {
572 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
573 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
574 	u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk);
575 	struct sock *sk = (struct sock *)msk;
576 	struct mptcp_pm_local *local;
577 	int i = 0;
578 
579 	while (msk->pm.local_addr_used < endp_subflow_max) {
580 		local = &locals[i];
581 
582 		if (!select_local_address(pernet, msk, local))
583 			break;
584 
585 		__clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
586 
587 		if (!mptcp_pm_addr_families_match(sk, &local->addr, remote))
588 			continue;
589 
590 		if (local->addr.id == msk->mpc_endpoint_id)
591 			continue;
592 
593 		msk->pm.local_addr_used++;
594 		msk->pm.extra_subflows++;
595 		i++;
596 
597 		if (msk->pm.extra_subflows >= limit_extra_subflows ||
598 		    i == locals_size)
599 			break;
600 	}
601 
602 	return i;
603 }
604 
605 static unsigned int
606 fill_local_address_any(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
607 		       struct mptcp_pm_local *local)
608 {
609 	struct sock *sk = (struct sock *)msk;
610 
611 	memset(local, 0, sizeof(*local));
612 	local->addr.family =
613 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
614 			remote->family == AF_INET6 &&
615 			ipv6_addr_v4mapped(&remote->addr6) ? AF_INET :
616 #endif
617 			remote->family;
618 
619 	if (!mptcp_pm_addr_families_match(sk, &local->addr, remote))
620 		return 0;
621 
622 	msk->pm.extra_subflows++;
623 
624 	return 1;
625 }
626 
627 /* Fill all the local addresses into the array addrs[],
628  * and return the array size.
629  */
630 static unsigned int
631 fill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
632 			 struct mptcp_pm_local *locals, int locals_size)
633 {
634 	bool c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk);
635 
636 	/* If there is at least one MPTCP endpoint with a fullmesh flag */
637 	if (mptcp_pm_get_endp_fullmesh_max(msk))
638 		return fill_local_addresses_vec_fullmesh(msk, remote, locals,
639 							 locals_size,
640 							 c_flag_case);
641 
642 	/* If there is at least one MPTCP endpoint with a laminar flag */
643 	if (mptcp_pm_get_endp_laminar_max(msk))
644 		return fill_local_laminar_endp(msk, remote, locals);
645 
646 	/* Special case: peer sets the C flag, accept one ADD_ADDR if default
647 	 * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints
648 	 */
649 	if (c_flag_case)
650 		return fill_local_addresses_vec_c_flag(msk, remote, locals,
651 						       locals_size);
652 
653 	/* No special case: fill in the single 'IPADDRANY' local address */
654 	return fill_local_address_any(msk, remote, &locals[0]);
655 }
656 
657 static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
658 {
659 	u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk);
660 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
661 	struct mptcp_pm_local locals[MPTCP_PM_ADDR_MAX];
662 	struct sock *sk = (struct sock *)msk;
663 	struct mptcp_addr_info remote;
664 	bool sf_created = false;
665 	int i, nr;
666 
667 	pr_debug("accepted %d:%d remote family %d\n",
668 		 msk->pm.add_addr_accepted, limit_add_addr_accepted,
669 		 msk->pm.remote.family);
670 
671 	remote = msk->pm.remote;
672 	mptcp_pm_announce_addr(msk, &remote, true);
673 	mptcp_pm_addr_send_ack(msk);
674 	mptcp_mpc_endpoint_setup(msk);
675 
676 	if (has_subflow_daddr(msk, &remote))
677 		return;
678 
679 	/* pick id 0 port, if none is provided the remote address */
680 	if (!remote.port)
681 		remote.port = sk->sk_dport;
682 
683 	/* connect to the specified remote address, using whatever
684 	 * local address the routing configuration will pick.
685 	 */
686 	nr = fill_local_addresses_vec(msk, &remote, locals, ARRAY_SIZE(locals));
687 	if (nr == 0)
688 		return;
689 
690 	spin_unlock_bh(&msk->pm.lock);
691 	for (i = 0; i < nr; i++)
692 		if (__mptcp_subflow_connect(sk, &locals[i], &remote) == 0)
693 			sf_created = true;
694 	spin_lock_bh(&msk->pm.lock);
695 
696 	if (sf_created) {
697 		/* add_addr_accepted is not decr for ID 0 */
698 		if (remote.id)
699 			msk->pm.add_addr_accepted++;
700 		if (msk->pm.add_addr_accepted >= limit_add_addr_accepted ||
701 		    msk->pm.extra_subflows >= limit_extra_subflows)
702 			WRITE_ONCE(msk->pm.accept_addr, false);
703 	}
704 }
705 
706 void mptcp_pm_nl_rm_addr(struct mptcp_sock *msk, u8 rm_id)
707 {
708 	if (rm_id && !WARN_ON_ONCE(msk->pm.add_addr_accepted == 0)) {
709 		u8 limit_add_addr_accepted =
710 			mptcp_pm_get_limit_add_addr_accepted(msk);
711 
712 		/* Note: if the subflow has been closed before, this
713 		 * add_addr_accepted counter will not be decremented.
714 		 */
715 		if (--msk->pm.add_addr_accepted < limit_add_addr_accepted)
716 			WRITE_ONCE(msk->pm.accept_addr, true);
717 	}
718 }
719 
720 static bool address_use_port(struct mptcp_pm_addr_entry *entry)
721 {
722 	return (entry->flags &
723 		(MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_SUBFLOW)) ==
724 		MPTCP_PM_ADDR_FLAG_SIGNAL;
725 }
726 
727 /* caller must ensure the RCU grace period is already elapsed */
728 static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry)
729 {
730 	if (entry->lsk)
731 		sock_release(entry->lsk);
732 	kfree(entry);
733 }
734 
735 static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet,
736 					     struct mptcp_pm_addr_entry *entry,
737 					     bool replace)
738 {
739 	struct mptcp_pm_addr_entry *cur, *del_entry = NULL;
740 	int ret = -EINVAL;
741 	u8 addr_max;
742 
743 	spin_lock_bh(&pernet->lock);
744 	/* to keep the code simple, don't do IDR-like allocation for address ID,
745 	 * just bail when we exceed limits
746 	 */
747 	if (pernet->next_id == MPTCP_PM_MAX_ADDR_ID)
748 		pernet->next_id = 1;
749 	if (pernet->endpoints == MPTCP_PM_MAX_ADDR_ID) {
750 		ret = -ERANGE;
751 		goto out;
752 	}
753 	if (test_bit(entry->addr.id, pernet->id_bitmap)) {
754 		ret = -EBUSY;
755 		goto out;
756 	}
757 
758 	/* do not insert duplicate address, differentiate on port only
759 	 * singled addresses
760 	 */
761 	if (!address_use_port(entry))
762 		entry->addr.port = 0;
763 	list_for_each_entry(cur, &pernet->endp_list, list) {
764 		if (mptcp_addresses_equal(&cur->addr, &entry->addr,
765 					  cur->addr.port || entry->addr.port)) {
766 			/* allow replacing the exiting endpoint only if such
767 			 * endpoint is an implicit one and the user-space
768 			 * did not provide an endpoint id
769 			 */
770 			if (!(cur->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)) {
771 				ret = -EEXIST;
772 				goto out;
773 			}
774 			if (entry->addr.id)
775 				goto out;
776 
777 			/* allow callers that only need to look up the local
778 			 * addr's id to skip replacement. This allows them to
779 			 * avoid calling synchronize_rcu in the packet recv
780 			 * path.
781 			 */
782 			if (!replace) {
783 				kfree(entry);
784 				ret = cur->addr.id;
785 				goto out;
786 			}
787 
788 			pernet->endpoints--;
789 			entry->addr.id = cur->addr.id;
790 			list_del_rcu(&cur->list);
791 			del_entry = cur;
792 			break;
793 		}
794 	}
795 
796 	if (!entry->addr.id) {
797 find_next:
798 		entry->addr.id = find_next_zero_bit(pernet->id_bitmap,
799 						    MPTCP_PM_MAX_ADDR_ID + 1,
800 						    pernet->next_id);
801 		if (!entry->addr.id && pernet->next_id != 1) {
802 			pernet->next_id = 1;
803 			goto find_next;
804 		}
805 	}
806 
807 	if (!entry->addr.id)
808 		goto out;
809 
810 	__set_bit(entry->addr.id, pernet->id_bitmap);
811 	if (entry->addr.id > pernet->next_id)
812 		pernet->next_id = entry->addr.id;
813 
814 	if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) {
815 		addr_max = pernet->endp_signal_max;
816 		WRITE_ONCE(pernet->endp_signal_max, addr_max + 1);
817 	}
818 	if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
819 		addr_max = pernet->endp_subflow_max;
820 		WRITE_ONCE(pernet->endp_subflow_max, addr_max + 1);
821 	}
822 	if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) {
823 		addr_max = pernet->endp_laminar_max;
824 		WRITE_ONCE(pernet->endp_laminar_max, addr_max + 1);
825 	}
826 	if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
827 		addr_max = pernet->endp_fullmesh_max;
828 		WRITE_ONCE(pernet->endp_fullmesh_max, addr_max + 1);
829 	}
830 
831 	pernet->endpoints++;
832 	if (!entry->addr.port)
833 		list_add_tail_rcu(&entry->list, &pernet->endp_list);
834 	else
835 		list_add_rcu(&entry->list, &pernet->endp_list);
836 	ret = entry->addr.id;
837 
838 out:
839 	spin_unlock_bh(&pernet->lock);
840 
841 	/* just replaced an existing entry, free it */
842 	if (del_entry) {
843 		synchronize_rcu();
844 		__mptcp_pm_release_addr_entry(del_entry);
845 	}
846 	return ret;
847 }
848 
849 static struct lock_class_key mptcp_slock_keys[2];
850 static struct lock_class_key mptcp_keys[2];
851 
852 static int mptcp_pm_nl_create_listen_socket(struct sock *sk,
853 					    struct mptcp_pm_addr_entry *entry)
854 {
855 	bool is_ipv6 = entry->addr.family == AF_INET6;
856 	int addrlen = sizeof(struct sockaddr_in);
857 	struct sockaddr_storage addr;
858 	struct sock *newsk, *ssk;
859 	int backlog = 1024;
860 	int err;
861 
862 	err = sock_create_kern(sock_net(sk), entry->addr.family,
863 			       SOCK_STREAM, IPPROTO_MPTCP, &entry->lsk);
864 	if (err)
865 		return err;
866 
867 	newsk = entry->lsk->sk;
868 	if (!newsk)
869 		return -EINVAL;
870 
871 	/* The subflow socket lock is acquired in a nested to the msk one
872 	 * in several places, even by the TCP stack, and this msk is a kernel
873 	 * socket: lockdep complains. Instead of propagating the _nested
874 	 * modifiers in several places, re-init the lock class for the msk
875 	 * socket to an mptcp specific one.
876 	 */
877 	sock_lock_init_class_and_name(newsk,
878 				      is_ipv6 ? "mlock-AF_INET6" : "mlock-AF_INET",
879 				      &mptcp_slock_keys[is_ipv6],
880 				      is_ipv6 ? "msk_lock-AF_INET6" : "msk_lock-AF_INET",
881 				      &mptcp_keys[is_ipv6]);
882 
883 	lock_sock(newsk);
884 	ssk = __mptcp_nmpc_sk(mptcp_sk(newsk));
885 	release_sock(newsk);
886 	if (IS_ERR(ssk))
887 		return PTR_ERR(ssk);
888 
889 	mptcp_info2sockaddr(&entry->addr, &addr, entry->addr.family);
890 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
891 	if (entry->addr.family == AF_INET6)
892 		addrlen = sizeof(struct sockaddr_in6);
893 #endif
894 	if (ssk->sk_family == AF_INET)
895 		err = inet_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);
896 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
897 	else if (ssk->sk_family == AF_INET6)
898 		err = inet6_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);
899 #endif
900 	if (err)
901 		return err;
902 
903 	/* We don't use mptcp_set_state() here because it needs to be called
904 	 * under the msk socket lock. For the moment, that will not bring
905 	 * anything more than only calling inet_sk_state_store(), because the
906 	 * old status is known (TCP_CLOSE).
907 	 */
908 	inet_sk_state_store(newsk, TCP_LISTEN);
909 	sock_set_flag(newsk, SOCK_RCU_FREE);
910 	lock_sock(ssk);
911 	WRITE_ONCE(mptcp_subflow_ctx(ssk)->pm_listener, true);
912 	err = __inet_listen_sk(ssk, backlog);
913 	if (!err)
914 		mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED);
915 	release_sock(ssk);
916 	return err;
917 }
918 
919 int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk,
920 			     struct mptcp_pm_addr_entry *skc)
921 {
922 	struct mptcp_pm_addr_entry *entry;
923 	struct pm_nl_pernet *pernet;
924 	int ret;
925 
926 	pernet = pm_nl_get_pernet_from_msk(msk);
927 
928 	rcu_read_lock();
929 	entry = __lookup_addr(pernet, &skc->addr);
930 	ret = entry ? entry->addr.id : -1;
931 	rcu_read_unlock();
932 	if (ret >= 0)
933 		return ret;
934 
935 	/* address not found, add to local list */
936 	entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC);
937 	if (!entry)
938 		return -ENOMEM;
939 
940 	entry->addr.port = 0;
941 	ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false);
942 	if (ret < 0)
943 		kfree(entry);
944 
945 	return ret;
946 }
947 
948 bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc)
949 {
950 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
951 	struct mptcp_pm_addr_entry *entry;
952 	bool backup;
953 
954 	rcu_read_lock();
955 	entry = __lookup_addr(pernet, skc);
956 	backup = entry && !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
957 	rcu_read_unlock();
958 
959 	return backup;
960 }
961 
962 static int mptcp_nl_add_subflow_or_signal_addr(struct net *net,
963 					       struct mptcp_addr_info *addr)
964 {
965 	struct mptcp_sock *msk;
966 	long s_slot = 0, s_num = 0;
967 
968 	while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
969 		struct sock *sk = (struct sock *)msk;
970 		struct mptcp_addr_info mpc_addr;
971 
972 		if (!READ_ONCE(msk->fully_established) ||
973 		    mptcp_pm_is_userspace(msk))
974 			goto next;
975 
976 		/* if the endp linked to the init sf is re-added with a != ID */
977 		mptcp_local_address((struct sock_common *)msk, &mpc_addr);
978 
979 		lock_sock(sk);
980 		spin_lock_bh(&msk->pm.lock);
981 		if (mptcp_addresses_equal(addr, &mpc_addr, addr->port))
982 			msk->mpc_endpoint_id = addr->id;
983 		mptcp_pm_create_subflow_or_signal_addr(msk);
984 		spin_unlock_bh(&msk->pm.lock);
985 		release_sock(sk);
986 
987 next:
988 		sock_put(sk);
989 		cond_resched();
990 	}
991 
992 	return 0;
993 }
994 
995 /* Add an MPTCP endpoint */
996 int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info)
997 {
998 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
999 	struct mptcp_pm_addr_entry addr, *entry;
1000 	struct nlattr *attr;
1001 	int ret;
1002 
1003 	if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR))
1004 		return -EINVAL;
1005 
1006 	attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR];
1007 	ret = mptcp_pm_parse_entry(attr, info, true, &addr);
1008 	if (ret < 0)
1009 		return ret;
1010 
1011 	if (addr.addr.port && !address_use_port(&addr)) {
1012 		NL_SET_ERR_MSG_ATTR(info->extack, attr,
1013 				    "flags must have signal and not subflow when using port");
1014 		return -EINVAL;
1015 	}
1016 
1017 	if (addr.flags & MPTCP_PM_ADDR_FLAG_SIGNAL &&
1018 	    addr.flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1019 		NL_SET_ERR_MSG_ATTR(info->extack, attr,
1020 				    "flags mustn't have both signal and fullmesh");
1021 		return -EINVAL;
1022 	}
1023 
1024 	if (addr.flags & MPTCP_PM_ADDR_FLAG_IMPLICIT) {
1025 		NL_SET_ERR_MSG_ATTR(info->extack, attr,
1026 				    "can't create IMPLICIT endpoint");
1027 		return -EINVAL;
1028 	}
1029 
1030 	entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT);
1031 	if (!entry) {
1032 		GENL_SET_ERR_MSG(info, "can't allocate addr");
1033 		return -ENOMEM;
1034 	}
1035 
1036 	if (entry->addr.port) {
1037 		ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry);
1038 		if (ret) {
1039 			GENL_SET_ERR_MSG_FMT(info, "create listen socket error: %d", ret);
1040 			goto out_free;
1041 		}
1042 	}
1043 	ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true);
1044 	if (ret < 0) {
1045 		GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret);
1046 		goto out_free;
1047 	}
1048 
1049 	mptcp_nl_add_subflow_or_signal_addr(sock_net(skb->sk), &entry->addr);
1050 	return 0;
1051 
1052 out_free:
1053 	__mptcp_pm_release_addr_entry(entry);
1054 	return ret;
1055 }
1056 
1057 static void mptcp_pm_remove_announced(struct mptcp_sock *msk,
1058 				      const struct mptcp_addr_info *addr,
1059 				      bool force)
1060 {
1061 	struct mptcp_rm_list list = { .nr = 0 };
1062 	bool announced;
1063 
1064 	list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
1065 
1066 	announced = mptcp_pm_announced_remove(msk, addr);
1067 	if (announced || force) {
1068 		spin_lock_bh(&msk->pm.lock);
1069 		if (announced)
1070 			msk->pm.add_addr_signaled--;
1071 		mptcp_pm_remove_addr(msk, &list);
1072 		spin_unlock_bh(&msk->pm.lock);
1073 	}
1074 }
1075 
1076 static void __mark_subflow_endp_available(struct mptcp_sock *msk, u8 id)
1077 {
1078 	/* If it was marked as used, and not ID 0, decrement local_addr_used */
1079 	if (!__test_and_set_bit(id ? : msk->mpc_endpoint_id, msk->pm.id_avail_bitmap) &&
1080 	    id && !WARN_ON_ONCE(msk->pm.local_addr_used == 0))
1081 		msk->pm.local_addr_used--;
1082 }
1083 
1084 static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net,
1085 						   const struct mptcp_pm_addr_entry *entry)
1086 {
1087 	const struct mptcp_addr_info *addr = &entry->addr;
1088 	struct mptcp_rm_list list = { .nr = 1 };
1089 	long s_slot = 0, s_num = 0;
1090 	struct mptcp_sock *msk;
1091 
1092 	pr_debug("remove_id=%d\n", addr->id);
1093 
1094 	while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1095 		struct sock *sk = (struct sock *)msk;
1096 		bool remove_subflow;
1097 
1098 		if (mptcp_pm_is_userspace(msk))
1099 			goto next;
1100 
1101 		lock_sock(sk);
1102 		remove_subflow = mptcp_pm_has_subflow_saddr(msk, addr);
1103 		mptcp_pm_remove_announced(msk, addr, remove_subflow &&
1104 					  !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT));
1105 
1106 		list.ids[0] = mptcp_endp_get_local_id(msk, addr);
1107 
1108 		spin_lock_bh(&msk->pm.lock);
1109 		if (remove_subflow)
1110 			mptcp_pm_rm_subflow(msk, &list);
1111 		if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)
1112 			__mark_subflow_endp_available(msk, list.ids[0]);
1113 		else /* mark endp ID as available, e.g. Signal or MPC endp */
1114 			__set_bit(addr->id, msk->pm.id_avail_bitmap);
1115 		spin_unlock_bh(&msk->pm.lock);
1116 
1117 		if (msk->mpc_endpoint_id == entry->addr.id)
1118 			msk->mpc_endpoint_id = 0;
1119 		release_sock(sk);
1120 
1121 next:
1122 		sock_put(sk);
1123 		cond_resched();
1124 	}
1125 
1126 	return 0;
1127 }
1128 
1129 static int mptcp_nl_remove_id_zero_address(struct net *net,
1130 					   struct mptcp_addr_info *addr)
1131 {
1132 	struct mptcp_rm_list list = { .nr = 0 };
1133 	long s_slot = 0, s_num = 0;
1134 	struct mptcp_sock *msk;
1135 
1136 	list.ids[list.nr++] = 0;
1137 
1138 	while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1139 		struct sock *sk = (struct sock *)msk;
1140 		struct mptcp_addr_info msk_local;
1141 		struct mptcp_addr_info anno_addr;
1142 		bool announced;
1143 
1144 		if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk))
1145 			goto next;
1146 
1147 		mptcp_local_address((struct sock_common *)msk, &msk_local);
1148 		if (!mptcp_addresses_equal(&msk_local, addr, addr->port))
1149 			goto next;
1150 
1151 		lock_sock(sk);
1152 		/* Drop a possibly pending ADD_ADDR for this address. */
1153 		anno_addr = msk_local;
1154 		anno_addr.port = 0;
1155 		announced = mptcp_pm_announced_remove(msk, &anno_addr);
1156 		spin_lock_bh(&msk->pm.lock);
1157 		if (announced)
1158 			msk->pm.add_addr_signaled--;
1159 		mptcp_pm_remove_addr(msk, &list);
1160 		mptcp_pm_rm_subflow(msk, &list);
1161 		__mark_subflow_endp_available(msk, 0);
1162 		spin_unlock_bh(&msk->pm.lock);
1163 		release_sock(sk);
1164 
1165 next:
1166 		sock_put(sk);
1167 		cond_resched();
1168 	}
1169 
1170 	return 0;
1171 }
1172 
1173 /* Remove an MPTCP endpoint */
1174 int mptcp_pm_nl_del_addr_doit(struct sk_buff *skb, struct genl_info *info)
1175 {
1176 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1177 	struct mptcp_pm_addr_entry addr, *entry;
1178 	struct nlattr *attr;
1179 	u8 addr_max;
1180 	int ret;
1181 
1182 	if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR))
1183 		return -EINVAL;
1184 
1185 	attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR];
1186 	ret = mptcp_pm_parse_entry(attr, info, false, &addr);
1187 	if (ret < 0)
1188 		return ret;
1189 
1190 	/* the zero id address is special: the first address used by the msk
1191 	 * always gets such an id, so different subflows can have different zero
1192 	 * id addresses. Additionally zero id is not accounted for in id_bitmap.
1193 	 * Let's use an 'mptcp_rm_list' instead of the common remove code.
1194 	 */
1195 	if (addr.addr.id == 0)
1196 		return mptcp_nl_remove_id_zero_address(sock_net(skb->sk), &addr.addr);
1197 
1198 	spin_lock_bh(&pernet->lock);
1199 	entry = __lookup_addr_by_id(pernet, addr.addr.id);
1200 	if (!entry) {
1201 		NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found");
1202 		spin_unlock_bh(&pernet->lock);
1203 		return -EINVAL;
1204 	}
1205 	if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) {
1206 		addr_max = pernet->endp_signal_max;
1207 		WRITE_ONCE(pernet->endp_signal_max, addr_max - 1);
1208 	}
1209 	if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) {
1210 		addr_max = pernet->endp_subflow_max;
1211 		WRITE_ONCE(pernet->endp_subflow_max, addr_max - 1);
1212 	}
1213 	if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) {
1214 		addr_max = pernet->endp_laminar_max;
1215 		WRITE_ONCE(pernet->endp_laminar_max, addr_max - 1);
1216 	}
1217 	if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1218 		addr_max = pernet->endp_fullmesh_max;
1219 		WRITE_ONCE(pernet->endp_fullmesh_max, addr_max - 1);
1220 	}
1221 
1222 	pernet->endpoints--;
1223 	list_del_rcu(&entry->list);
1224 	__clear_bit(entry->addr.id, pernet->id_bitmap);
1225 	spin_unlock_bh(&pernet->lock);
1226 
1227 	mptcp_nl_remove_subflow_and_signal_addr(sock_net(skb->sk), entry);
1228 	synchronize_rcu();
1229 	__mptcp_pm_release_addr_entry(entry);
1230 
1231 	return ret;
1232 }
1233 
1234 static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk,
1235 					      struct list_head *rm_list,
1236 					      struct mptcp_pm_addr_entry *entry)
1237 {
1238 	struct mptcp_rm_list alist, slist;
1239 	bool more;
1240 
1241 again:
1242 	alist.nr = 0;
1243 	slist.nr = 0;
1244 	more = false;
1245 
1246 	entry = list_prepare_entry(entry, rm_list, list);
1247 	list_for_each_entry_continue(entry, rm_list, list) {
1248 		if (mptcp_pm_has_subflow_saddr(msk, &entry->addr))
1249 			slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
1250 
1251 		if (mptcp_pm_announced_remove(msk, &entry->addr))
1252 			alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
1253 
1254 		if (slist.nr == MPTCP_RM_IDS_MAX ||
1255 		    alist.nr == MPTCP_RM_IDS_MAX) {
1256 			more = !list_is_last(&entry->list, rm_list);
1257 			break;
1258 		}
1259 	}
1260 
1261 	spin_lock_bh(&msk->pm.lock);
1262 	if (alist.nr) {
1263 		msk->pm.add_addr_signaled -= alist.nr;
1264 		mptcp_pm_remove_addr(msk, &alist);
1265 	}
1266 	if (slist.nr)
1267 		mptcp_pm_rm_subflow(msk, &slist);
1268 	/* Reset counters: maybe some subflows have been removed before */
1269 	if (!more) {
1270 		bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
1271 		msk->pm.local_addr_used = 0;
1272 	}
1273 	spin_unlock_bh(&msk->pm.lock);
1274 
1275 	if (more)
1276 		goto again;
1277 }
1278 
1279 static void mptcp_nl_flush_addrs_list(struct net *net,
1280 				      struct list_head *rm_list)
1281 {
1282 	long s_slot = 0, s_num = 0;
1283 	struct mptcp_sock *msk;
1284 
1285 	if (list_empty(rm_list))
1286 		return;
1287 
1288 	while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1289 		struct sock *sk = (struct sock *)msk;
1290 
1291 		if (!mptcp_pm_is_userspace(msk)) {
1292 			lock_sock(sk);
1293 			mptcp_pm_flush_addrs_and_subflows(msk, rm_list, NULL);
1294 			release_sock(sk);
1295 		}
1296 
1297 		sock_put(sk);
1298 		cond_resched();
1299 	}
1300 }
1301 
1302 /* caller must ensure the RCU grace period is already elapsed */
1303 static void __flush_addrs(struct list_head *list)
1304 {
1305 	while (!list_empty(list)) {
1306 		struct mptcp_pm_addr_entry *cur;
1307 
1308 		cur = list_entry(list->next,
1309 				 struct mptcp_pm_addr_entry, list);
1310 		list_del_rcu(&cur->list);
1311 		__mptcp_pm_release_addr_entry(cur);
1312 	}
1313 }
1314 
1315 static void __reset_counters(struct pm_nl_pernet *pernet)
1316 {
1317 	WRITE_ONCE(pernet->endp_signal_max, 0);
1318 	WRITE_ONCE(pernet->endp_subflow_max, 0);
1319 	WRITE_ONCE(pernet->endp_laminar_max, 0);
1320 	WRITE_ONCE(pernet->endp_fullmesh_max, 0);
1321 	pernet->endpoints = 0;
1322 }
1323 
1324 int mptcp_pm_nl_flush_addrs_doit(struct sk_buff *skb, struct genl_info *info)
1325 {
1326 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1327 	struct list_head free_list;
1328 
1329 	spin_lock_bh(&pernet->lock);
1330 	free_list = pernet->endp_list;
1331 	INIT_LIST_HEAD_RCU(&pernet->endp_list);
1332 	__reset_counters(pernet);
1333 	pernet->next_id = 1;
1334 	bitmap_zero(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
1335 	spin_unlock_bh(&pernet->lock);
1336 
1337 	if (free_list.next == &pernet->endp_list)
1338 		return 0;
1339 
1340 	synchronize_rcu();
1341 
1342 	/* Adjust the pointers to free_list instead of pernet->endp_list */
1343 	free_list.prev->next = &free_list;
1344 	free_list.next->prev = &free_list;
1345 
1346 	mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list);
1347 	__flush_addrs(&free_list);
1348 	return 0;
1349 }
1350 
1351 int mptcp_pm_nl_get_addr(u8 id, struct mptcp_pm_addr_entry *addr,
1352 			 struct genl_info *info)
1353 {
1354 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1355 	struct mptcp_pm_addr_entry *entry;
1356 	int ret = -EINVAL;
1357 
1358 	rcu_read_lock();
1359 	entry = __lookup_addr_by_id(pernet, id);
1360 	if (entry) {
1361 		*addr = *entry;
1362 		ret = 0;
1363 	}
1364 	rcu_read_unlock();
1365 
1366 	return ret;
1367 }
1368 
1369 int mptcp_pm_nl_dump_addr(struct sk_buff *msg,
1370 			  struct netlink_callback *cb)
1371 {
1372 	struct net *net = sock_net(msg->sk);
1373 	struct mptcp_pm_addr_entry *entry;
1374 	struct pm_nl_pernet *pernet;
1375 	int id = cb->args[0];
1376 	int i;
1377 
1378 	pernet = pm_nl_get_pernet(net);
1379 
1380 	rcu_read_lock();
1381 	for (i = id; i < MPTCP_PM_MAX_ADDR_ID + 1; i++) {
1382 		if (test_bit(i, pernet->id_bitmap)) {
1383 			entry = __lookup_addr_by_id(pernet, i);
1384 			if (!entry)
1385 				break;
1386 
1387 			if (entry->addr.id <= id)
1388 				continue;
1389 
1390 			if (mptcp_pm_genl_fill_addr(msg, cb, entry) < 0)
1391 				break;
1392 
1393 			id = entry->addr.id;
1394 		}
1395 	}
1396 	rcu_read_unlock();
1397 
1398 	cb->args[0] = id;
1399 	return msg->len;
1400 }
1401 
1402 static int parse_limit(struct genl_info *info, int id, unsigned int *limit)
1403 {
1404 	struct nlattr *attr = info->attrs[id];
1405 
1406 	if (!attr)
1407 		return 0;
1408 
1409 	*limit = nla_get_u32(attr);
1410 	if (*limit > MPTCP_PM_SUBFLOWS_MAX) {
1411 		NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr,
1412 					"limit greater than maximum (%u)",
1413 					MPTCP_PM_SUBFLOWS_MAX);
1414 		return -EINVAL;
1415 	}
1416 	return 0;
1417 }
1418 
1419 int mptcp_pm_nl_set_limits_doit(struct sk_buff *skb, struct genl_info *info)
1420 {
1421 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1422 	unsigned int rcv_addrs, subflows;
1423 	int ret;
1424 
1425 	spin_lock_bh(&pernet->lock);
1426 	rcv_addrs = pernet->limit_add_addr_accepted;
1427 	ret = parse_limit(info, MPTCP_PM_ATTR_RCV_ADD_ADDRS, &rcv_addrs);
1428 	if (ret)
1429 		goto unlock;
1430 
1431 	subflows = pernet->limit_extra_subflows;
1432 	ret = parse_limit(info, MPTCP_PM_ATTR_SUBFLOWS, &subflows);
1433 	if (ret)
1434 		goto unlock;
1435 
1436 	WRITE_ONCE(pernet->limit_add_addr_accepted, rcv_addrs);
1437 	WRITE_ONCE(pernet->limit_extra_subflows, subflows);
1438 
1439 unlock:
1440 	spin_unlock_bh(&pernet->lock);
1441 	return ret;
1442 }
1443 
1444 int mptcp_pm_nl_get_limits_doit(struct sk_buff *skb, struct genl_info *info)
1445 {
1446 	struct pm_nl_pernet *pernet = genl_info_pm_nl(info);
1447 	struct sk_buff *msg;
1448 	void *reply;
1449 
1450 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1451 	if (!msg)
1452 		return -ENOMEM;
1453 
1454 	reply = genlmsg_put_reply(msg, info, &mptcp_genl_family, 0,
1455 				  MPTCP_PM_CMD_GET_LIMITS);
1456 	if (!reply)
1457 		goto fail;
1458 
1459 	if (nla_put_u32(msg, MPTCP_PM_ATTR_RCV_ADD_ADDRS,
1460 			READ_ONCE(pernet->limit_add_addr_accepted)))
1461 		goto fail;
1462 
1463 	if (nla_put_u32(msg, MPTCP_PM_ATTR_SUBFLOWS,
1464 			READ_ONCE(pernet->limit_extra_subflows)))
1465 		goto fail;
1466 
1467 	genlmsg_end(msg, reply);
1468 	return genlmsg_reply(msg, info);
1469 
1470 fail:
1471 	GENL_SET_ERR_MSG(info, "not enough space in Netlink message");
1472 	nlmsg_free(msg);
1473 	return -EMSGSIZE;
1474 }
1475 
1476 static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk,
1477 				 struct mptcp_addr_info *addr)
1478 {
1479 	struct mptcp_rm_list list = { .nr = 0 };
1480 
1481 	list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
1482 
1483 	spin_lock_bh(&msk->pm.lock);
1484 	mptcp_pm_rm_subflow(msk, &list);
1485 	__mark_subflow_endp_available(msk, list.ids[0]);
1486 	mptcp_pm_create_subflow_or_signal_addr(msk);
1487 	spin_unlock_bh(&msk->pm.lock);
1488 }
1489 
1490 static void mptcp_pm_nl_set_flags_all(struct net *net,
1491 				      struct mptcp_pm_addr_entry *local,
1492 				      u8 changed)
1493 {
1494 	u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW);
1495 	u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1496 	long s_slot = 0, s_num = 0;
1497 	struct mptcp_sock *msk;
1498 
1499 	if (changed == MPTCP_PM_ADDR_FLAG_FULLMESH && !is_subflow)
1500 		return;
1501 
1502 	while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) {
1503 		struct sock *sk = (struct sock *)msk;
1504 
1505 		if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk))
1506 			goto next;
1507 
1508 		lock_sock(sk);
1509 		if (changed & MPTCP_PM_ADDR_FLAG_BACKUP)
1510 			mptcp_pm_mp_prio_send_ack(msk, &local->addr, NULL, bkup);
1511 		/* Subflows will only be recreated if the SUBFLOW flag is set */
1512 		if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH))
1513 			mptcp_pm_nl_fullmesh(msk, &local->addr);
1514 		release_sock(sk);
1515 
1516 next:
1517 		sock_put(sk);
1518 		cond_resched();
1519 	}
1520 }
1521 
1522 int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local,
1523 			  struct genl_info *info)
1524 {
1525 	struct nlattr *attr = info->attrs[MPTCP_PM_ATTR_ADDR];
1526 	u8 changed, mask = MPTCP_PM_ADDR_FLAG_BACKUP |
1527 			   MPTCP_PM_ADDR_FLAG_FULLMESH;
1528 	struct net *net = genl_info_net(info);
1529 	struct mptcp_pm_addr_entry *entry;
1530 	struct pm_nl_pernet *pernet;
1531 	u8 lookup_by_id = 0;
1532 
1533 	pernet = pm_nl_get_pernet(net);
1534 
1535 	if (local->addr.family == AF_UNSPEC) {
1536 		lookup_by_id = 1;
1537 		if (!local->addr.id) {
1538 			NL_SET_ERR_MSG_ATTR(info->extack, attr,
1539 					    "missing address ID");
1540 			return -EOPNOTSUPP;
1541 		}
1542 	}
1543 
1544 	spin_lock_bh(&pernet->lock);
1545 	entry = lookup_by_id ? __lookup_addr_by_id(pernet, local->addr.id) :
1546 			       __lookup_addr(pernet, &local->addr);
1547 	if (!entry) {
1548 		spin_unlock_bh(&pernet->lock);
1549 		NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found");
1550 		return -EINVAL;
1551 	}
1552 	if ((local->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) &&
1553 	    (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL |
1554 			     MPTCP_PM_ADDR_FLAG_IMPLICIT))) {
1555 		spin_unlock_bh(&pernet->lock);
1556 		NL_SET_ERR_MSG_ATTR(info->extack, attr, "invalid addr flags");
1557 		return -EINVAL;
1558 	}
1559 
1560 	changed = (local->flags ^ entry->flags) & mask;
1561 	entry->flags = (entry->flags & ~mask) | (local->flags & mask);
1562 	*local = *entry;
1563 
1564 	if (changed & MPTCP_PM_ADDR_FLAG_FULLMESH) {
1565 		u8 addr_max = pernet->endp_fullmesh_max;
1566 
1567 		if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH)
1568 			addr_max++;
1569 		else
1570 			addr_max--;
1571 
1572 		WRITE_ONCE(pernet->endp_fullmesh_max, addr_max);
1573 	}
1574 
1575 	spin_unlock_bh(&pernet->lock);
1576 
1577 	mptcp_pm_nl_set_flags_all(net, local, changed);
1578 	return 0;
1579 }
1580 
1581 bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk)
1582 {
1583 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
1584 
1585 	if (msk->pm.extra_subflows == mptcp_pm_get_limit_extra_subflows(msk) ||
1586 	    (find_next_and_bit(pernet->id_bitmap, msk->pm.id_avail_bitmap,
1587 			       MPTCP_PM_MAX_ADDR_ID + 1, 0) == MPTCP_PM_MAX_ADDR_ID + 1)) {
1588 		WRITE_ONCE(msk->pm.work_pending, false);
1589 		return false;
1590 	}
1591 	return true;
1592 }
1593 
1594 /* Called under PM lock */
1595 void __mptcp_pm_kernel_worker(struct mptcp_sock *msk)
1596 {
1597 	struct mptcp_pm_data *pm = &msk->pm;
1598 
1599 	if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) {
1600 		pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED);
1601 		mptcp_pm_nl_add_addr_received(msk);
1602 	}
1603 	if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) {
1604 		pm->status &= ~BIT(MPTCP_PM_ESTABLISHED);
1605 		mptcp_pm_nl_fully_established(msk);
1606 	}
1607 	if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) {
1608 		pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED);
1609 		mptcp_pm_nl_subflow_established(msk);
1610 	}
1611 }
1612 
1613 static int __net_init pm_nl_init_net(struct net *net)
1614 {
1615 	struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
1616 
1617 	INIT_LIST_HEAD_RCU(&pernet->endp_list);
1618 
1619 	/* Cit. 2 subflows ought to be enough for anybody. */
1620 	pernet->limit_extra_subflows = 2;
1621 	pernet->next_id = 1;
1622 	spin_lock_init(&pernet->lock);
1623 
1624 	/* No need to initialize other pernet fields, the struct is zeroed at
1625 	 * allocation time.
1626 	 */
1627 
1628 	return 0;
1629 }
1630 
1631 static void __net_exit pm_nl_exit_net(struct list_head *net_list)
1632 {
1633 	struct net *net;
1634 
1635 	list_for_each_entry(net, net_list, exit_list) {
1636 		struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
1637 
1638 		/* net is removed from namespace list, can't race with
1639 		 * other modifiers, also netns core already waited for a
1640 		 * RCU grace period.
1641 		 */
1642 		__flush_addrs(&pernet->endp_list);
1643 	}
1644 }
1645 
1646 static struct pernet_operations mptcp_pm_pernet_ops = {
1647 	.init = pm_nl_init_net,
1648 	.exit_batch = pm_nl_exit_net,
1649 	.id = &pm_nl_pernet_id,
1650 	.size = sizeof(struct pm_nl_pernet),
1651 };
1652 
1653 struct mptcp_pm_ops mptcp_pm_kernel = {
1654 	.name			= "kernel",
1655 	.owner			= THIS_MODULE,
1656 };
1657 
1658 void __init mptcp_pm_kernel_register(void)
1659 {
1660 	if (register_pernet_subsys(&mptcp_pm_pernet_ops) < 0)
1661 		panic("Failed to register MPTCP PM pernet subsystem.\n");
1662 
1663 	mptcp_pm_register(&mptcp_pm_kernel);
1664 }
1665