1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1991, 1993, 1995
5 * The Regents of the University of California.
6 * Copyright (c) 2007-2009 Robert N. M. Watson
7 * Copyright (c) 2010-2011 Juniper Networks, Inc.
8 * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
9 * All rights reserved.
10 *
11 * Portions of this software were developed by Robert N. M. Watson under
12 * contract to Juniper Networks, Inc.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
16 * are met:
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39 #include "opt_ddb.h"
40 #include "opt_ipsec.h"
41 #include "opt_inet.h"
42 #include "opt_inet6.h"
43 #include "opt_ratelimit.h"
44 #include "opt_rss.h"
45
46 #include <sys/param.h>
47 #include <sys/hash.h>
48 #include <sys/systm.h>
49 #include <sys/libkern.h>
50 #include <sys/lock.h>
51 #include <sys/malloc.h>
52 #include <sys/mbuf.h>
53 #include <sys/eventhandler.h>
54 #include <sys/domain.h>
55 #include <sys/proc.h>
56 #include <sys/protosw.h>
57 #include <sys/smp.h>
58 #include <sys/smr.h>
59 #include <sys/socket.h>
60 #include <sys/socketvar.h>
61 #include <sys/sockio.h>
62 #include <sys/priv.h>
63 #include <sys/proc.h>
64 #include <sys/refcount.h>
65 #include <sys/jail.h>
66 #include <sys/kernel.h>
67 #include <sys/sysctl.h>
68
69 #ifdef DDB
70 #include <ddb/ddb.h>
71 #endif
72
73 #include <vm/uma.h>
74 #include <vm/vm.h>
75
76 #include <net/if.h>
77 #include <net/if_var.h>
78 #include <net/if_private.h>
79 #include <net/if_types.h>
80 #include <net/if_llatbl.h>
81 #include <net/route.h>
82 #include <net/rss_config.h>
83 #include <net/vnet.h>
84
85 #include <netinet/in.h>
86 #include <netinet/in_pcb.h>
87 #include <netinet/in_pcb_var.h>
88 #include <netinet/tcp.h>
89 #ifdef INET
90 #include <netinet/in_var.h>
91 #include <netinet/in_fib.h>
92 #endif
93 #include <netinet/ip_var.h>
94 #ifdef INET6
95 #include <netinet/ip6.h>
96 #include <netinet6/in6_pcb.h>
97 #include <netinet6/in6_var.h>
98 #include <netinet6/ip6_var.h>
99 #endif
100 #include <net/route/nhop.h>
101
102 #include <netipsec/ipsec_support.h>
103
104 #include <security/mac/mac_framework.h>
105
106 #define INPCBLBGROUP_SIZMIN 8
107 #define INPCBLBGROUP_SIZMAX 256
108
109 #define INP_FREED 0x00000200 /* Went through in_pcbfree(). */
110
111 /*
112 * These configure the range of local port addresses assigned to
113 * "unspecified" outgoing connections/packets/whatever.
114 */
115 VNET_DEFINE(u_int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */
116 VNET_DEFINE(u_int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */
117 VNET_DEFINE(u_int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */
118 VNET_DEFINE(u_int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */
119 VNET_DEFINE(u_int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */
120 VNET_DEFINE(u_int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */
121
122 /*
123 * Reserved ports accessible only to root. There are significant
124 * security considerations that must be accounted for when changing these,
125 * but the security benefits can be great. Please be careful.
126 */
127 VNET_DEFINE(u_int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
128 VNET_DEFINE(u_int, ipport_reservedlow);
129
130 /* Enable random ephemeral port allocation by default. */
131 VNET_DEFINE(bool, ipport_randomized) = true;
132
133 #ifdef INET
134 static struct inpcb *in_pcblookup_internal(struct inpcbinfo_ctx *ipictx,
135 struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
136 u_int lport_arg, int lookupflags, uint8_t numa_domain, int fib);
137 static struct inpcb *in_pcblookup_exact(struct inpcbinfo *pcbinfo,
138 struct in_addr faddr, u_short fport, struct in_addr laddr, u_short lport,
139 struct inpbucket **bucketp);
140 static struct inpcb *in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo,
141 struct in_addr laddr, u_short lport, int fib, struct inpbucket **bucketp);
142
143 #define RANGECHK(var, min, max) \
144 if ((var) < (min)) { (var) = (min); } \
145 else if ((var) > (max)) { (var) = (max); }
146
147 static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)148 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
149 {
150 int error;
151
152 error = sysctl_handle_int(oidp, arg1, arg2, req);
153 if (error == 0) {
154 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
155 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
156 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
157 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
158 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
159 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
160 }
161 return (error);
162 }
163
164 #undef RANGECHK
165
166 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
168 "IP Ports");
169
170 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
171 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
172 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "IU",
173 "");
174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
175 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
176 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "IU",
177 "");
178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
179 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
180 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "IU",
181 "");
182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
183 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
184 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "IU",
185 "");
186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
187 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
188 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "IU",
189 "");
190 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
191 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
192 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "IU",
193 "");
194 SYSCTL_UINT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
195 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
196 &VNET_NAME(ipport_reservedhigh), 0, "");
197 SYSCTL_UINT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
198 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
199 SYSCTL_BOOL(_net_inet_ip_portrange, OID_AUTO, randomized,
200 CTLFLAG_VNET | CTLFLAG_RW,
201 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
202
203 #ifdef RATELIMIT
204 counter_u64_t rate_limit_new;
205 counter_u64_t rate_limit_chg;
206 counter_u64_t rate_limit_active;
207 counter_u64_t rate_limit_alloc_fail;
208 counter_u64_t rate_limit_set_ok;
209
210 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
211 "IP Rate Limiting");
212 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
213 &rate_limit_active, "Active rate limited connections");
214 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
215 &rate_limit_alloc_fail, "Rate limited connection failures");
216 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
217 &rate_limit_set_ok, "Rate limited setting succeeded");
218 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
219 &rate_limit_new, "Total Rate limit new attempts");
220 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
221 &rate_limit_chg, "Total Rate limited change attempts");
222 #endif /* RATELIMIT */
223
224 #endif /* INET */
225
226 VNET_DEFINE(uint32_t, in_pcbhashseed);
227 static void
in_pcbhashseed_init(void)228 in_pcbhashseed_init(void)
229 {
230
231 V_in_pcbhashseed = arc4random();
232 }
233 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
234 in_pcbhashseed_init, NULL);
235
236 #ifdef INET
237 VNET_DEFINE_STATIC(int, connect_inaddr_wild) = 0;
238 #define V_connect_inaddr_wild VNET(connect_inaddr_wild)
239 SYSCTL_INT(_net_inet_ip, OID_AUTO, connect_inaddr_wild,
240 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_inaddr_wild), 0,
241 "Allow connecting to INADDR_ANY or INADDR_BROADCAST for connect(2)");
242 #endif
243
244 static struct inpcblbgroup *
in_pcblbgroup_alloc(struct ucred * cred,u_char vflag,uint16_t port,const union in_dependaddr * addr,int size,uint8_t numa_domain,int fib)245 in_pcblbgroup_alloc(struct ucred *cred, u_char vflag, uint16_t port,
246 const union in_dependaddr *addr, int size, uint8_t numa_domain, int fib)
247 {
248 struct inpcblbgroup *grp;
249 size_t bytes;
250
251 bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
252 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
253 if (grp == NULL)
254 return (NULL);
255 LIST_INIT(&grp->il_pending);
256 grp->il_cred = crhold(cred);
257 grp->il_vflag = vflag;
258 grp->il_lport = port;
259 grp->il_numa_domain = numa_domain;
260 grp->il_fibnum = fib;
261 grp->il_dependladdr = *addr;
262 grp->il_inpsiz = size;
263 return (grp);
264 }
265
266 static void
in_pcblbgroup_free_deferred(epoch_context_t ctx)267 in_pcblbgroup_free_deferred(epoch_context_t ctx)
268 {
269 struct inpcblbgroup *grp;
270
271 grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
272 crfree(grp->il_cred);
273 free(grp, M_PCB);
274 }
275
276 static void
in_pcblbgroup_free(struct inpcblbgroup * grp)277 in_pcblbgroup_free(struct inpcblbgroup *grp)
278 {
279 KASSERT(LIST_EMPTY(&grp->il_pending),
280 ("local group %p still has pending inps", grp));
281
282 CK_LIST_REMOVE(grp, il_list);
283 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
284 }
285
286 /*
287 * Returns with locked bucket on success and not locked in case of failure.
288 */
289 struct inpcblbgroup *
in_pcblbgroup_find(struct inpcb * inp,struct lbgroupbucket ** bucket)290 in_pcblbgroup_find(struct inpcb *inp, struct lbgroupbucket **bucket)
291 {
292 const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
293 struct inpcblbgroup *grp;
294
295 INP_WLOCK_ASSERT(inp);
296
297 *bucket = &pcbinfo->ipi_lbgrouphashbase[
298 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
299 INPBUCKET_LOCK(*bucket);
300 CK_LIST_FOREACH(grp, &(*bucket)->head, il_list) {
301 struct inpcb *inp1;
302
303 for (unsigned int i = 0; i < grp->il_inpcnt; i++) {
304 if (inp == grp->il_inp[i])
305 return (grp);
306 }
307 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {
308 if (inp == inp1)
309 return (grp);
310 }
311 }
312 INPBUCKET_UNLOCK(*bucket);
313 return (NULL);
314 }
315
316 static void
in_pcblbgroup_insert(struct inpcblbgroup * grp,struct inpcb * inp)317 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp)
318 {
319 /* NB: bucket is locked */
320 KASSERT(grp->il_inpcnt < grp->il_inpsiz,
321 ("invalid local group size %d and count %d", grp->il_inpsiz,
322 grp->il_inpcnt));
323 INP_WLOCK_ASSERT(inp);
324
325 if (inp->inp_socket->so_proto->pr_listen != pr_listen_notsupp &&
326 !SOLISTENING(inp->inp_socket)) {
327 /*
328 * If this is a TCP socket, it should not be visible to lbgroup
329 * lookups until listen() has been called.
330 */
331 LIST_INSERT_HEAD(&grp->il_pending, inp, inp_lbgroup_list);
332 grp->il_pendcnt++;
333 } else {
334 grp->il_inp[grp->il_inpcnt] = inp;
335
336 /*
337 * Synchronize with in_pcblookup_lbgroup(): make sure that we
338 * don't expose a null slot to the lookup path.
339 */
340 atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1);
341 }
342
343 inp->inp_flags |= INP_INLBGROUP;
344 }
345
346 static struct inpcblbgroup *
in_pcblbgroup_resize(struct lbgroupbucket * bucket,struct inpcblbgroup * old_grp,int size)347 in_pcblbgroup_resize(struct lbgroupbucket *bucket,
348 struct inpcblbgroup *old_grp, int size)
349 {
350 struct inpcblbgroup *grp;
351 int i;
352
353 INPBUCKET_LOCK_ASSERT(bucket);
354
355 grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag,
356 old_grp->il_lport, &old_grp->il_dependladdr, size,
357 old_grp->il_numa_domain, old_grp->il_fibnum);
358 if (grp == NULL)
359 return (NULL);
360
361 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
362 ("invalid new local group size %d and old local group count %d",
363 grp->il_inpsiz, old_grp->il_inpcnt));
364
365 for (i = 0; i < old_grp->il_inpcnt; ++i)
366 grp->il_inp[i] = old_grp->il_inp[i];
367 grp->il_inpcnt = old_grp->il_inpcnt;
368 CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list);
369 LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb,
370 inp_lbgroup_list);
371 grp->il_pendcnt = old_grp->il_pendcnt;
372 old_grp->il_pendcnt = 0;
373 in_pcblbgroup_free(old_grp);
374 return (grp);
375 }
376
377 /*
378 * Add PCB to load balance group for SO_REUSEPORT_LB option.
379 */
380 static int
in_pcbinslbgrouphash(struct inpcb * inp,uint8_t numa_domain)381 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
382 {
383 const static struct timeval interval = { 60, 0 };
384 static struct timeval lastprint;
385 const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
386 struct lbgroupbucket *bucket;
387 struct inpcblbgroup *grp;
388 uint32_t idx;
389 int fib;
390
391 INP_WLOCK_ASSERT(inp);
392
393 fib = (inp->inp_flags & INP_BOUNDFIB) != 0 ?
394 inp->inp_inc.inc_fibnum : RT_ALL_FIBS;
395
396 #ifdef INET6
397 /*
398 * Don't allow IPv4 mapped INET6 wild socket.
399 */
400 if ((inp->inp_vflag & INP_IPV4) &&
401 inp->inp_laddr.s_addr == INADDR_ANY &&
402 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
403 return (0);
404 }
405 #endif
406
407 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
408 bucket = &pcbinfo->ipi_lbgrouphashbase[idx];
409 INPBUCKET_LOCK(bucket);
410 CK_LIST_FOREACH(grp, &bucket->head, il_list) {
411 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
412 grp->il_vflag == inp->inp_vflag &&
413 grp->il_lport == inp->inp_lport &&
414 grp->il_numa_domain == numa_domain &&
415 grp->il_fibnum == fib &&
416 memcmp(&grp->il_dependladdr,
417 &inp->inp_inc.inc_ie.ie_dependladdr,
418 sizeof(grp->il_dependladdr)) == 0) {
419 break;
420 }
421 }
422 if (grp == NULL) {
423 /* Create new load balance group. */
424 grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,
425 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
426 INPCBLBGROUP_SIZMIN, numa_domain, fib);
427 if (grp == NULL) {
428 INPBUCKET_UNLOCK(bucket);
429 return (ENOMEM);
430 }
431 in_pcblbgroup_insert(grp, inp);
432 CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list);
433 } else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) {
434 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
435 INPBUCKET_UNLOCK(bucket);
436 if (ratecheck(&lastprint, &interval))
437 printf("lb group port %d, limit reached\n",
438 ntohs(grp->il_lport));
439 return (0);
440 }
441
442 /* Expand this local group. */
443 grp = in_pcblbgroup_resize(bucket, grp, grp->il_inpsiz * 2);
444 if (grp == NULL) {
445 INPBUCKET_UNLOCK(bucket);
446 return (ENOMEM);
447 }
448 in_pcblbgroup_insert(grp, inp);
449 } else {
450 in_pcblbgroup_insert(grp, inp);
451 }
452 INPBUCKET_UNLOCK(bucket);
453 return (0);
454 }
455
456 /*
457 * Remove PCB from load balance group. Expects locked bucket.
458 */
459 static void
in_pcbremlbgrouphash(struct lbgroupbucket * bucket,struct inpcb * inp)460 in_pcbremlbgrouphash(struct lbgroupbucket *bucket, struct inpcb *inp)
461 {
462 struct inpcblbgroup *grp;
463 struct inpcb *inp1;
464 int i;
465
466 INP_WLOCK_ASSERT(inp);
467 MPASS(inp->inp_flags & INP_INLBGROUP);
468 MPASS(bucket == &inp->inp_pcbinfo->ipi_lbgrouphashbase[
469 INP_PCBPORTHASH(inp->inp_lport,
470 inp->inp_pcbinfo->ipi_lbgrouphashmask)]);
471 INPBUCKET_LOCK_ASSERT(bucket);
472
473 CK_LIST_FOREACH(grp, &bucket->head, il_list) {
474 for (i = 0; i < grp->il_inpcnt; ++i) {
475 if (grp->il_inp[i] != inp)
476 continue;
477
478 if (grp->il_inpcnt == 1 &&
479 LIST_EMPTY(&grp->il_pending)) {
480 /* We are the last, free this local group. */
481 in_pcblbgroup_free(grp);
482 } else {
483 grp->il_inp[i] =
484 grp->il_inp[grp->il_inpcnt - 1];
485
486 /*
487 * Synchronize with in_pcblookup_lbgroup().
488 */
489 atomic_store_rel_int(&grp->il_inpcnt,
490 grp->il_inpcnt - 1);
491 }
492 inp->inp_flags &= ~INP_INLBGROUP;
493 return;
494 }
495 LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {
496 if (inp == inp1) {
497 LIST_REMOVE(inp, inp_lbgroup_list);
498 grp->il_pendcnt--;
499 inp->inp_flags &= ~INP_INLBGROUP;
500 return;
501 }
502 }
503 }
504 __assert_unreachable();
505 }
506
507 int
in_pcblbgroup_numa(struct inpcb * inp,int arg)508 in_pcblbgroup_numa(struct inpcb *inp, int arg)
509 {
510 struct lbgroupbucket *bucket;
511 uint8_t numa_domain;
512
513 INP_WLOCK_ASSERT(inp);
514
515 switch (arg) {
516 case TCP_REUSPORT_LB_NUMA_NODOM:
517 numa_domain = M_NODOM;
518 break;
519 case TCP_REUSPORT_LB_NUMA_CURDOM:
520 numa_domain = PCPU_GET(domain);
521 break;
522 default:
523 if (arg < 0 || arg >= vm_ndomains)
524 return (EINVAL);
525 numa_domain = arg;
526 }
527
528 if (in_pcblbgroup_find(inp, &bucket) == NULL)
529 return (ENOENT);
530 /* Remove it from the old group. */
531 in_pcbremlbgrouphash(bucket, inp);
532 INPBUCKET_UNLOCK(bucket);
533 /* Add it to the new group based on numa domain. */
534 in_pcbinslbgrouphash(inp, numa_domain);
535
536 return (0);
537 }
538
539 /*
540 * Initialize an inpcbinfo - a per-VNET instance of connections db.
541 */
542 void
in_pcbinfo_init(struct inpcbinfo * pcbinfo,struct inpcbstorage * pcbstor,u_int hash_nelements,u_int porthash_nelements,u_int lbgrouphash_nelements)543 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
544 u_int hash_nelements, u_int porthash_nelements, u_int lbgrouphash_nelements)
545 {
546 struct hashalloc_args ha = {
547 .mtype = M_PCB,
548 .mflags = M_WAITOK,
549 .head = HASH_HEAD_CK_LIST,
550 .lock = HASH_LOCK_MTX,
551 .lopts = MTX_DEF,
552 };
553
554 mtx_init(&pcbinfo->ipi_list_unconn.lock, pcbstor->ips_hashlock_name,
555 NULL, MTX_DEF);
556 CK_LIST_INIT(&pcbinfo->ipi_list_unconn.head);
557 pcbinfo->ipi_count = 0;
558
559 ha.size = hash_nelements;
560 ha.lname = "inpcb_exact_bucket";
561 pcbinfo->ipi_hash_exact = hashalloc(&ha);
562 ha.lname = "inpcb_wild_bucket";
563 pcbinfo->ipi_hash_wild = hashalloc(&ha);
564 pcbinfo->ipi_hashmask = ha.size - 1;
565
566 if (porthash_nelements > 0) {
567 ha.size = imin(porthash_nelements, IPPORT_MAX + 1);
568 ha.lname = "inpcb_port_bucket";
569 pcbinfo->ipi_porthash = hashalloc(&ha);
570 pcbinfo->ipi_porthashmask = ha.size - 1;
571 } else
572 pcbinfo->ipi_porthash = NULL;
573 if (lbgrouphash_nelements > 0) {
574 ha.size = imin(lbgrouphash_nelements, IPPORT_MAX + 1);
575 ha.lname = "inpcb_lbgroup_bucket";
576 pcbinfo->ipi_lbgrouphashbase = hashalloc(&ha);
577 pcbinfo->ipi_lbgrouphashmask = ha.size - 1;
578 } else
579 pcbinfo->ipi_lbgrouphashbase = NULL;
580
581 pcbinfo->ipi_zone = pcbstor->ips_zone;
582 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
583 }
584
585 /*
586 * Destroy an inpcbinfo.
587 */
588 void
in_pcbinfo_destroy(struct inpcbinfo * pcbinfo)589 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
590 {
591 struct hashalloc_args ha = {
592 .mtype = M_PCB,
593 .head = HASH_HEAD_CK_LIST,
594 .lock = HASH_LOCK_MTX,
595 };
596
597 KASSERT(pcbinfo->ipi_count == 0,
598 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
599
600 ha.size = pcbinfo->ipi_hashmask + 1;
601 hashfree(pcbinfo->ipi_hash_exact, &ha);
602 hashfree(pcbinfo->ipi_hash_wild, &ha);
603 if (pcbinfo->ipi_porthash != NULL) {
604 ha.size = pcbinfo->ipi_porthashmask + 1;
605 hashfree(pcbinfo->ipi_porthash, &ha);
606 }
607 if (pcbinfo->ipi_lbgrouphashbase != NULL) {
608 ha.size = pcbinfo->ipi_lbgrouphashmask + 1;
609 hashfree(pcbinfo->ipi_lbgrouphashbase, &ha);
610 }
611 mtx_destroy(&pcbinfo->ipi_list_unconn.lock);
612 }
613
614 /*
615 * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
616 */
617 static void inpcb_fini(void *, int);
618 void
in_pcbstorage_init(void * arg)619 in_pcbstorage_init(void *arg)
620 {
621 struct inpcbstorage *pcbstor = arg;
622
623 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
624 pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit,
625 inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR);
626 }
627
628 /*
629 * Destroy a pcbstorage - used by unloadable protocols.
630 */
631 void
in_pcbstorage_destroy(void * arg)632 in_pcbstorage_destroy(void *arg)
633 {
634 struct inpcbstorage *pcbstor = arg;
635
636 uma_zdestroy(pcbstor->ips_zone);
637 }
638
639 /*
640 * Allocate a PCB and associate it with the socket.
641 * On success: return with the PCB locked.
642 */
643 int
in_pcballoc(struct socket * so,struct inpcbinfo * pcbinfo)644 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
645 {
646 struct inpcb *inp;
647 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
648 int error;
649 #endif
650
651 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
652 if (inp == NULL)
653 return (ENOBUFS);
654 bzero(&inp->inp_start_zero, inp_zero_size);
655 #ifdef NUMA
656 inp->inp_numa_domain = M_NODOM;
657 #endif
658 inp->inp_pcbinfo = pcbinfo;
659 inp->inp_socket = so;
660 inp->inp_cred = crhold(so->so_cred);
661 inp->inp_inc.inc_fibnum = so->so_fibnum;
662 #ifdef MAC
663 error = mac_inpcb_init(inp, M_NOWAIT);
664 if (error != 0)
665 goto out;
666 mac_inpcb_create(so, inp);
667 #endif
668 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
669 error = ipsec_init_pcbpolicy(inp);
670 if (error != 0) {
671 #ifdef MAC
672 mac_inpcb_destroy(inp);
673 #endif
674 goto out;
675 }
676 #endif /*IPSEC*/
677 #ifdef INET6
678 if (INP_SOCKAF(so) == AF_INET6) {
679 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
680 if (V_ip6_v6only)
681 inp->inp_flags |= IN6P_IPV6_V6ONLY;
682 #ifdef INET
683 else
684 inp->inp_vflag |= INP_IPV4;
685 #endif
686 if (V_ip6_auto_flowlabel)
687 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
688 inp->in6p_hops = -1; /* use kernel default */
689 }
690 #endif
691 #if defined(INET) && defined(INET6)
692 else
693 #endif
694 #ifdef INET
695 inp->inp_vflag |= INP_IPV4;
696 #endif
697 inp->inp_smr = SMR_SEQ_INVALID;
698
699 /*
700 * Routes in inpcb's can cache L2 as well; they are guaranteed
701 * to be cleaned up.
702 */
703 inp->inp_route.ro_flags = RT_LLE_CACHE;
704 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
705 inp->inp_flags |= INP_UNCONNECTED;
706 INP_WLOCK(inp);
707 IPI_LOCK(pcbinfo);
708 pcbinfo->ipi_count++;
709 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
710 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp,
711 inp_unconn_list);
712 IPI_UNLOCK(pcbinfo);
713 so->so_pcb = inp;
714
715 return (0);
716
717 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
718 out:
719 crfree(inp->inp_cred);
720 #ifdef INVARIANTS
721 inp->inp_cred = NULL;
722 #endif
723 uma_zfree_smr(pcbinfo->ipi_zone, inp);
724 return (error);
725 #endif
726 }
727
728 /*
729 * Assign a local port like in_pcb_lport(), but also used with connect()
730 * and a foreign address and port. If fsa is non-NULL, choose a local port
731 * that is unused with those, otherwise one that is completely unused.
732 * lsa can be NULL for IPv6.
733 * On success: returns with locked context and fills lportp.
734 */
735 int
in_pcb_lport_dest(struct inpcbinfo_ctx * ipictx,struct inpcb * inp,const struct sockaddr * lsa,const struct sockaddr * fsa,u_short fport,struct ucred * cred,int lookupflags,u_short * lportp)736 in_pcb_lport_dest(struct inpcbinfo_ctx *ipictx, struct inpcb *inp,
737 const struct sockaddr *lsa, const struct sockaddr *fsa, u_short fport,
738 struct ucred *cred, int lookupflags, u_short *lportp)
739 {
740 u_int count, first, last, port, *lastport;
741 int error;
742 u_short lport;
743 #ifdef INET
744 struct in_addr laddr, faddr;
745 #endif
746 #ifdef INET6
747 const struct in6_addr *laddr6, *faddr6;
748 #endif
749
750 INP_LOCK_ASSERT(inp);
751
752 if (inp->inp_flags & INP_HIGHPORT) {
753 first = V_ipport_hifirstauto; /* sysctl */
754 last = V_ipport_hilastauto;
755 lastport = &ipictx->pcbinfo->ipi_lasthi;
756 } else if (inp->inp_flags & INP_LOWPORT) {
757 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
758 if (error)
759 return (error);
760 first = V_ipport_lowfirstauto; /* 1023 */
761 last = V_ipport_lowlastauto; /* 600 */
762 lastport = &ipictx->pcbinfo->ipi_lastlow;
763 } else {
764 first = V_ipport_firstauto; /* sysctl */
765 last = V_ipport_lastauto;
766 lastport = &ipictx->pcbinfo->ipi_lastport;
767 }
768
769 /*
770 * Instead of having two loops further down counting up or down
771 * make sure that first is always <= last and go with only one
772 * code path implementing all logic.
773 */
774 if (first > last) {
775 __typeof(first) aux;
776
777 aux = first;
778 first = last;
779 last = aux;
780 }
781
782 #ifdef INET
783 laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */
784 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
785 if (lsa != NULL)
786 laddr = ((const struct sockaddr_in *)lsa)->sin_addr;
787 if (fsa != NULL)
788 faddr = ((const struct sockaddr_in *)fsa)->sin_addr;
789 }
790 #endif
791 #ifdef INET6
792 laddr6 = NULL;
793 if ((inp->inp_vflag & INP_IPV6) != 0) {
794 if (lsa != NULL)
795 laddr6 = &((const struct sockaddr_in6 *)lsa)->sin6_addr;
796 if (fsa != NULL)
797 faddr6 = &((const struct sockaddr_in6 *)fsa)->sin6_addr;
798 }
799 #endif
800
801 if (V_ipport_randomized) {
802 port = first + (arc4random() % (last - first));
803 } else {
804 port = atomic_fetchadd_int(lastport, 1);
805 if (__predict_false(port < first || port > last)) {
806 atomic_store_int(lastport, first);
807 port = first;
808 }
809 }
810
811 for (count = last - first; count > 0; count--) {
812 struct inpcbinfo_ctx tmpctx = {
813 .pcbinfo = ipictx->pcbinfo
814 };
815 struct inpcb *tmpinp = NULL;
816
817 lport = htons(port);
818
819 if (fsa != NULL) {
820 #ifdef INET
821 if (lsa->sa_family == AF_INET) {
822 tmpinp = in_pcblookup_internal(&tmpctx,
823 faddr, fport, laddr, lport, lookupflags,
824 M_NODOM, RT_ALL_FIBS);
825 }
826 #endif
827 #ifdef INET6
828 if (lsa->sa_family == AF_INET6) {
829 tmpinp = in6_pcblookup_internal(&tmpctx,
830 faddr6, fport, laddr6, lport, lookupflags,
831 M_NODOM, RT_ALL_FIBS);
832 }
833 #endif
834 } else {
835 #ifdef INET6
836 if ((inp->inp_vflag & INP_IPV6) != 0) {
837 tmpinp = in6_pcblookup_local(&tmpctx,
838 &inp->in6p_laddr, lport, RT_ALL_FIBS,
839 lookupflags, cred);
840 #ifdef INET
841 if (tmpinp == NULL &&
842 (inp->inp_vflag & INP_IPV4))
843 tmpinp = in_pcblookup_local(&tmpctx,
844 laddr, lport, RT_ALL_FIBS,
845 lookupflags, cred);
846 #endif
847 }
848 #endif
849 #if defined(INET) && defined(INET6)
850 else
851 #endif
852 #ifdef INET
853 tmpinp = in_pcblookup_local(&tmpctx, laddr,
854 lport, RT_ALL_FIBS, lookupflags, cred);
855 #endif
856 }
857 if (tmpinp == NULL) {
858 MPASS(ipictx->ebucket == NULL);
859 MPASS(ipictx->wbucket == NULL);
860 MPASS(ipictx->pbucket == NULL);
861 MPASS(ipictx->lbbucket == NULL);
862 *ipictx = tmpctx;
863 break;
864 }
865 inpcbinfo_ctx_release(&tmpctx);
866 ++port;
867 if (port < first || port > last)
868 port = first;
869 }
870
871 if (count == 0) /* completely used? */
872 return (EADDRNOTAVAIL);
873
874 *lportp = lport;
875
876 return (0);
877 }
878
879 /*
880 * Select a local port (number) to use.
881 * On success: returns with locked context and fills lportp.
882 */
883 int
in_pcb_lport(struct inpcbinfo_ctx * ipictx,struct inpcb * inp,struct in_addr * laddrp,u_short * lportp,struct ucred * cred,int lookupflags)884 in_pcb_lport(struct inpcbinfo_ctx *ipictx, struct inpcb *inp,
885 struct in_addr *laddrp, u_short *lportp, struct ucred *cred,
886 int lookupflags)
887 {
888 struct sockaddr_in laddr;
889
890 if (laddrp) {
891 bzero(&laddr, sizeof(laddr));
892 laddr.sin_family = AF_INET;
893 laddr.sin_addr = *laddrp;
894 }
895 return (in_pcb_lport_dest(ipictx, inp,
896 laddrp ? (struct sockaddr *) &laddr : NULL,
897 NULL, 0, cred, lookupflags, lportp));
898 }
899
900 #ifdef INET
901 /*
902 * Determine whether the inpcb can be bound to the specified address/port tuple.
903 * May return with partially locked context.
904 */
905 static int
in_pcbbind_avail(struct inpcbinfo_ctx * ipictx,struct inpcb * inp,const struct in_addr laddr,const u_short lport,const int fib,int sooptions,int lookupflags,struct ucred * cred)906 in_pcbbind_avail(struct inpcbinfo_ctx *ipictx, struct inpcb *inp,
907 const struct in_addr laddr, const u_short lport, const int fib,
908 int sooptions, int lookupflags, struct ucred *cred)
909 {
910 int reuseport, reuseport_lb;
911
912 INP_LOCK_ASSERT(inp);
913
914 reuseport = (sooptions & SO_REUSEPORT);
915 reuseport_lb = (sooptions & SO_REUSEPORT_LB);
916
917 if (IN_MULTICAST(ntohl(laddr.s_addr))) {
918 /*
919 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
920 * allow complete duplication of binding if
921 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
922 * and a multicast address is bound on both
923 * new and duplicated sockets.
924 */
925 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0)
926 reuseport = SO_REUSEADDR | SO_REUSEPORT;
927 /*
928 * XXX: How to deal with SO_REUSEPORT_LB here?
929 * Treat same as SO_REUSEPORT for now.
930 */
931 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0)
932 reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB;
933 } else if (!in_nullhost(laddr)) {
934 struct sockaddr_in sin;
935
936 memset(&sin, 0, sizeof(sin));
937 sin.sin_family = AF_INET;
938 sin.sin_len = sizeof(sin);
939 sin.sin_addr = laddr;
940
941 /*
942 * Is the address a local IP address?
943 * If INP_BINDANY is set, then the socket may be bound
944 * to any endpoint address, local or not.
945 */
946 if ((inp->inp_flags & INP_BINDANY) == 0 &&
947 ifa_ifwithaddr_fib_check((const struct sockaddr *)&sin, fib) == 0)
948 return (EADDRNOTAVAIL);
949 }
950
951 if (lport != 0) {
952 struct inpcb *t;
953
954 if (ntohs(lport) <= V_ipport_reservedhigh &&
955 ntohs(lport) >= V_ipport_reservedlow &&
956 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
957 return (EACCES);
958
959 if (!IN_MULTICAST(ntohl(laddr.s_addr)) &&
960 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
961 /*
962 * If a socket owned by a different user is already
963 * bound to this port, fail. In particular, SO_REUSE*
964 * can only be used to share a port among sockets owned
965 * by the same user.
966 *
967 * However, we can share a port with a connected socket
968 * which has a unique 4-tuple.
969 */
970 t = in_pcblookup_local(ipictx, laddr, lport,
971 RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
972 if (t != NULL &&
973 (inp->inp_socket->so_type != SOCK_STREAM ||
974 in_nullhost(t->inp_faddr)) &&
975 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
976 return (EADDRINUSE);
977 }
978 t = in_pcblookup_local(ipictx, laddr, lport, fib,
979 lookupflags, cred);
980 if (t != NULL && ((reuseport | reuseport_lb) &
981 t->inp_socket->so_options) == 0) {
982 #ifdef INET6
983 if (!in_nullhost(laddr) ||
984 !in_nullhost(t->inp_laddr) ||
985 (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
986 (t->inp_vflag & INP_IPV6PROTO) == 0)
987 #endif
988 return (EADDRINUSE);
989 }
990 }
991 return (0);
992 }
993
994 /*
995 * Set up a bind operation on a PCB, performing port allocation
996 * as required, but do not actually modify the PCB. Callers can
997 * either complete the bind by setting inp_laddr/inp_lport and
998 * calling in_pcbinshash(), or they can just use the resulting
999 * port and address to authorise the sending of a once-off packet.
1000 *
1001 * On error, the values of *laddrp and *lportp are not changed.
1002 */
1003 static int
in_pcbbind_setup_locked(struct inpcbinfo_ctx * ipictx,struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,int flags,struct ucred * cred)1004 in_pcbbind_setup_locked(struct inpcbinfo_ctx *ipictx, struct inpcb *inp,
1005 struct sockaddr_in *sin, in_addr_t *laddrp, u_short *lportp, int flags,
1006 struct ucred *cred)
1007 {
1008 struct socket *so = inp->inp_socket;
1009 struct in_addr laddr;
1010 u_short lport = 0;
1011 int error, fib, lookupflags, sooptions;
1012
1013 /* No state changes, so read lock is sufficient here. */
1014 INP_LOCK_ASSERT(inp);
1015
1016 laddr.s_addr = *laddrp;
1017 if (sin != NULL && laddr.s_addr != INADDR_ANY)
1018 return (EINVAL);
1019
1020 lookupflags = 0;
1021 sooptions = atomic_load_int(&so->so_options);
1022 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0)
1023 lookupflags = INPLOOKUP_WILDCARD;
1024 if (sin == NULL) {
1025 if ((error = prison_local_ip4(cred, &laddr)) != 0)
1026 return (error);
1027 } else {
1028 KASSERT(sin->sin_family == AF_INET,
1029 ("%s: invalid family for address %p", __func__, sin));
1030 KASSERT(sin->sin_len == sizeof(*sin),
1031 ("%s: invalid length for address %p", __func__, sin));
1032
1033 error = prison_local_ip4(cred, &sin->sin_addr);
1034 if (error)
1035 return (error);
1036 if (sin->sin_port != *lportp) {
1037 /* Don't allow the port to change. */
1038 if (*lportp != 0)
1039 return (EINVAL);
1040 lport = sin->sin_port;
1041 }
1042 laddr = sin->sin_addr;
1043
1044 fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum :
1045 RT_ALL_FIBS;
1046
1047 /* See if this address/port combo is available. */
1048 error = in_pcbbind_avail(ipictx, inp, laddr, lport, fib,
1049 sooptions, lookupflags, cred);
1050 if (error != 0)
1051 return (error);
1052 }
1053 if (*lportp != 0)
1054 lport = *lportp;
1055 if (lport == 0) {
1056 error = in_pcb_lport(ipictx, inp, &laddr, &lport, cred,
1057 lookupflags);
1058 if (error != 0)
1059 return (error);
1060 }
1061 *laddrp = laddr.s_addr;
1062 *lportp = lport;
1063 if ((flags & INPBIND_FIB) != 0)
1064 inp->inp_flags |= INP_BOUNDFIB;
1065 return (0);
1066 }
1067
1068 int
in_pcbbind_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,int flags,struct ucred * cred)1069 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
1070 u_short *lportp, int flags, struct ucred *cred)
1071 {
1072 struct inpcbinfo_ctx ipictx = {
1073 .pcbinfo = inp->inp_pcbinfo
1074 };
1075 int error;
1076
1077 error = in_pcbbind_setup_locked(&ipictx, inp, sin, laddrp, lportp,
1078 flags, cred);
1079 inpcbinfo_ctx_release(&ipictx);
1080
1081 return (error);
1082 }
1083
1084 #ifdef INET
1085 int
in_pcbbind(struct inpcb * inp,struct sockaddr_in * sin,int flags,struct ucred * cred)1086 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, int flags,
1087 struct ucred *cred)
1088 {
1089 struct inpcbinfo_ctx ipictx = {
1090 .pcbinfo = inp->inp_pcbinfo
1091 };
1092 int error;
1093 bool anonport;
1094
1095 KASSERT(sin == NULL || sin->sin_family == AF_INET,
1096 ("%s: invalid address family for %p", __func__, sin));
1097 KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in),
1098 ("%s: invalid address length for %p", __func__, sin));
1099 INP_WLOCK_ASSERT(inp);
1100
1101 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
1102 return (EINVAL);
1103 anonport = sin == NULL || sin->sin_port == 0;
1104
1105 error = in_pcbbind_setup_locked(&ipictx, inp, sin,
1106 &inp->inp_laddr.s_addr, &inp->inp_lport, flags, cred);
1107 if (error) {
1108 inpcbinfo_ctx_release(&ipictx);
1109 return (error);
1110 }
1111 if (__predict_false((error = in_pcbinshash(inp, &ipictx)) != 0)) {
1112 inpcbinfo_ctx_release(&ipictx);
1113 MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB);
1114 inp->inp_laddr.s_addr = INADDR_ANY;
1115 inp->inp_lport = 0;
1116 inp->inp_flags &= ~INP_BOUNDFIB;
1117 return (error);
1118 }
1119 inpcbinfo_ctx_release(&ipictx);
1120 if (anonport)
1121 inp->inp_flags |= INP_ANONPORT;
1122 return (0);
1123 }
1124 #endif
1125
1126 /*
1127 * Connect from a socket to a specified address.
1128 * Both address and port must be specified in argument sin.
1129 * If don't have a local address for this socket yet,
1130 * then pick one.
1131 */
1132 int
in_pcbconnect(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred)1133 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
1134 {
1135 struct inpcbinfo_ctx ipictx = {
1136 .pcbinfo = inp->inp_pcbinfo
1137 };
1138 struct in_addr laddr, faddr;
1139 u_short lport;
1140 int error;
1141 bool anonport;
1142
1143 NET_EPOCH_ASSERT();
1144 INP_WLOCK_ASSERT(inp);
1145 KASSERT(in_nullhost(inp->inp_faddr),
1146 ("%s: inp is already connected", __func__));
1147 KASSERT(sin->sin_family == AF_INET,
1148 ("%s: invalid address family for %p", __func__, sin));
1149 KASSERT(sin->sin_len == sizeof(*sin),
1150 ("%s: invalid address length for %p", __func__, sin));
1151
1152 if (sin->sin_port == 0)
1153 return (EADDRNOTAVAIL);
1154
1155 anonport = (inp->inp_lport == 0);
1156
1157 if (__predict_false(in_broadcast(sin->sin_addr))) {
1158 if (!V_connect_inaddr_wild || CK_STAILQ_EMPTY(&V_in_ifaddrhead))
1159 return (ENETUNREACH);
1160 /*
1161 * If the destination address is INADDR_ANY, use the primary
1162 * local address. If the supplied address is INADDR_BROADCAST,
1163 * and the primary interface supports broadcast, choose the
1164 * broadcast address for that interface.
1165 */
1166 if (in_nullhost(sin->sin_addr)) {
1167 faddr =
1168 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1169 if ((error = prison_get_ip4(cred, &faddr)) != 0)
1170 return (error);
1171 } else if (sin->sin_addr.s_addr == INADDR_BROADCAST &&
1172 CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags
1173 & IFF_BROADCAST) {
1174 faddr = satosin(&CK_STAILQ_FIRST(
1175 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1176 } else
1177 faddr = sin->sin_addr;
1178 } else
1179 faddr = sin->sin_addr;
1180
1181 if (in_nullhost(inp->inp_laddr)) {
1182 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1183 if (__predict_false(error)) {
1184 return (error);
1185 }
1186 } else
1187 laddr = inp->inp_laddr;
1188
1189 /*
1190 * Depending on if the inpcb was bind(2)-ed to a local port before
1191 * connect(2), different actions are taken.
1192 * If the inpcb is newborn and lives just on the ipi_list_unconn, then
1193 * in_pcb_lport_dest() will find the exact hash slot and also port
1194 * slot (if applicable). Locking these two buckets is required for the
1195 * later call to in_pcbinshash().
1196 * If the inpcb is already bind(2)-ed, then it is already inserted in
1197 * the port hash (if applicable) and doesn't need to move within it.
1198 * But it needs to move from the wild hash to the exact hash. Both
1199 * buckets need to be locked for in_pcbrehash().
1200 */
1201 if (anonport) {
1202 struct sockaddr_in lsin = {
1203 .sin_family = AF_INET,
1204 .sin_addr = laddr,
1205 };
1206 struct sockaddr_in fsin = {
1207 .sin_family = AF_INET,
1208 .sin_addr = faddr,
1209 };
1210
1211 MPASS(inp->inp_flags & INP_UNCONNECTED);
1212
1213 error = in_pcb_lport_dest(&ipictx, inp,
1214 (struct sockaddr *)&lsin, (struct sockaddr *)&fsin,
1215 sin->sin_port, cred, INPLOOKUP_WILDCARD, &lport);
1216 if (__predict_false(error))
1217 /* in_pcb_lport_dest() shouldn't obtained a lock */
1218 return (error);
1219 } else {
1220 if (in_pcblookup_exact(ipictx.pcbinfo, faddr, sin->sin_port,
1221 laddr, inp->inp_lport, &ipictx.ebucket) != NULL) {
1222 inpcbinfo_ctx_release(&ipictx);
1223 return (EADDRINUSE);
1224 }
1225 if ((inp->inp_flags & INP_UNCONNECTED) == 0) {
1226 struct inpcb *i __diagused;
1227 struct inpcblbgroup *g __diagused;
1228
1229 i = in_pcblookup_wild_locked(ipictx.pcbinfo,
1230 laddr, inp->inp_lport, RT_ALL_FIBS,
1231 &ipictx.wbucket);
1232 MPASS(i);
1233 if (inp->inp_flags & INP_INLBGROUP) {
1234 g = in_pcblbgroup_find(inp, &ipictx.lbbucket);
1235 MPASS(g);
1236 }
1237 }
1238 lport = inp->inp_lport;
1239 }
1240
1241 MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 ||
1242 (inp->inp_flags & INP_UNCONNECTED));
1243
1244 inp->inp_faddr = faddr;
1245 inp->inp_fport = sin->sin_port;
1246 inp->inp_laddr = laddr;
1247 inp->inp_lport = lport;
1248
1249 if (inp->inp_flags & INP_UNCONNECTED) {
1250 error = in_pcbinshash(inp, &ipictx);
1251 MPASS(error == 0);
1252 } else
1253 in_pcbrehash(inp, &ipictx);
1254 inpcbinfo_ctx_release(&ipictx);
1255
1256 if (V_fib_hash_outbound) {
1257 uint32_t hash_val, hash_type;
1258
1259 hash_val = fib4_calc_software_hash(inp->inp_laddr,
1260 inp->inp_faddr, 0, sin->sin_port,
1261 inp->inp_socket->so_proto->pr_protocol, &hash_type);
1262
1263 inp->inp_flowid = hash_val;
1264 inp->inp_flowtype = hash_type;
1265 }
1266 if (anonport)
1267 inp->inp_flags |= INP_ANONPORT;
1268 return (0);
1269 }
1270
1271 /*
1272 * Do proper source address selection on an unbound socket in case
1273 * of connect. Take jails into account as well.
1274 */
1275 int
in_pcbladdr(const struct inpcb * inp,struct in_addr * faddr,struct in_addr * laddr,struct ucred * cred)1276 in_pcbladdr(const struct inpcb *inp, struct in_addr *faddr,
1277 struct in_addr *laddr, struct ucred *cred)
1278 {
1279 struct ifaddr *ifa;
1280 struct sockaddr *sa;
1281 struct sockaddr_in *sin, dst;
1282 struct nhop_object *nh;
1283 int error;
1284
1285 NET_EPOCH_ASSERT();
1286 KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1287
1288 /*
1289 * Bypass source address selection and use the primary jail IP
1290 * if requested.
1291 */
1292 if (!prison_saddrsel_ip4(cred, laddr))
1293 return (0);
1294
1295 /*
1296 * If the destination address is multicast and an outgoing
1297 * interface has been set as a multicast option, prefer the
1298 * address of that interface as our source address.
1299 */
1300 if (IN_MULTICAST(ntohl(faddr->s_addr)) && inp->inp_moptions != NULL &&
1301 inp->inp_moptions->imo_multicast_ifp != NULL) {
1302 struct ifnet *ifp = inp->inp_moptions->imo_multicast_ifp;
1303 struct in_ifaddr *ia;
1304
1305 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1306 if (ia->ia_ifp == ifp &&
1307 prison_check_ip4(cred, &ia->ia_addr.sin_addr) == 0)
1308 break;
1309 }
1310 if (ia == NULL)
1311 return (EADDRNOTAVAIL);
1312 *laddr = ia->ia_addr.sin_addr;
1313 return (0);
1314 }
1315
1316 error = 0;
1317
1318 nh = NULL;
1319 bzero(&dst, sizeof(dst));
1320 sin = &dst;
1321 sin->sin_family = AF_INET;
1322 sin->sin_len = sizeof(struct sockaddr_in);
1323 sin->sin_addr.s_addr = faddr->s_addr;
1324
1325 /*
1326 * If route is known our src addr is taken from the i/f,
1327 * else punt.
1328 *
1329 * Find out route to destination.
1330 */
1331 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1332 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1333 0, NHR_NONE, 0);
1334
1335 /*
1336 * If we found a route, use the address corresponding to
1337 * the outgoing interface.
1338 *
1339 * Otherwise assume faddr is reachable on a directly connected
1340 * network and try to find a corresponding interface to take
1341 * the source address from.
1342 */
1343 if (nh == NULL || nh->nh_ifp == NULL) {
1344 struct in_ifaddr *ia;
1345 struct ifnet *ifp;
1346
1347 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1348 inp->inp_socket->so_fibnum));
1349 if (ia == NULL) {
1350 ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1351 inp->inp_socket->so_fibnum));
1352 }
1353 if (ia == NULL) {
1354 error = ENETUNREACH;
1355 goto done;
1356 }
1357
1358 if (!prison_flag(cred, PR_IP4)) {
1359 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1360 goto done;
1361 }
1362
1363 ifp = ia->ia_ifp;
1364 ia = NULL;
1365 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1366 sa = ifa->ifa_addr;
1367 if (sa->sa_family != AF_INET)
1368 continue;
1369 sin = (struct sockaddr_in *)sa;
1370 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1371 ia = (struct in_ifaddr *)ifa;
1372 break;
1373 }
1374 }
1375 if (ia != NULL) {
1376 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1377 goto done;
1378 }
1379
1380 /* 3. As a last resort return the 'default' jail address. */
1381 error = prison_get_ip4(cred, laddr);
1382 goto done;
1383 }
1384
1385 /*
1386 * If the outgoing interface on the route found is not
1387 * a loopback interface, use the address from that interface.
1388 * In case of jails do those three steps:
1389 * 1. check if the interface address belongs to the jail. If so use it.
1390 * 2. check if we have any address on the outgoing interface
1391 * belonging to this jail. If so use it.
1392 * 3. as a last resort return the 'default' jail address.
1393 */
1394 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1395 struct in_ifaddr *ia;
1396 struct ifnet *ifp;
1397
1398 /* If not jailed, use the default returned. */
1399 if (!prison_flag(cred, PR_IP4)) {
1400 ia = (struct in_ifaddr *)nh->nh_ifa;
1401 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1402 goto done;
1403 }
1404
1405 /* Jailed. */
1406 /* 1. Check if the iface address belongs to the jail. */
1407 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1408 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1409 ia = (struct in_ifaddr *)nh->nh_ifa;
1410 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1411 goto done;
1412 }
1413
1414 /*
1415 * 2. Check if we have any address on the outgoing interface
1416 * belonging to this jail.
1417 */
1418 ia = NULL;
1419 ifp = nh->nh_ifp;
1420 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1421 sa = ifa->ifa_addr;
1422 if (sa->sa_family != AF_INET)
1423 continue;
1424 sin = (struct sockaddr_in *)sa;
1425 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1426 ia = (struct in_ifaddr *)ifa;
1427 break;
1428 }
1429 }
1430 if (ia != NULL) {
1431 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1432 goto done;
1433 }
1434
1435 /* 3. As a last resort return the 'default' jail address. */
1436 error = prison_get_ip4(cred, laddr);
1437 goto done;
1438 }
1439
1440 /*
1441 * The outgoing interface is marked with 'loopback net', so a route
1442 * to ourselves is here.
1443 * Try to find the interface of the destination address and then
1444 * take the address from there. That interface is not necessarily
1445 * a loopback interface.
1446 * In case of jails, check that it is an address of the jail
1447 * and if we cannot find, fall back to the 'default' jail address.
1448 */
1449 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1450 struct in_ifaddr *ia;
1451
1452 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1453 inp->inp_socket->so_fibnum));
1454 if (ia == NULL)
1455 ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1456 inp->inp_socket->so_fibnum));
1457 if (ia == NULL)
1458 ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1459
1460 if (!prison_flag(cred, PR_IP4)) {
1461 if (ia == NULL) {
1462 error = ENETUNREACH;
1463 goto done;
1464 }
1465 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1466 goto done;
1467 }
1468
1469 /* Jailed. */
1470 if (ia != NULL) {
1471 struct ifnet *ifp;
1472
1473 ifp = ia->ia_ifp;
1474 ia = NULL;
1475 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1476 sa = ifa->ifa_addr;
1477 if (sa->sa_family != AF_INET)
1478 continue;
1479 sin = (struct sockaddr_in *)sa;
1480 if (prison_check_ip4(cred,
1481 &sin->sin_addr) == 0) {
1482 ia = (struct in_ifaddr *)ifa;
1483 break;
1484 }
1485 }
1486 if (ia != NULL) {
1487 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1488 goto done;
1489 }
1490 }
1491
1492 /* 3. As a last resort return the 'default' jail address. */
1493 error = prison_get_ip4(cred, laddr);
1494 goto done;
1495 }
1496
1497 done:
1498 if (error == 0 && laddr->s_addr == INADDR_ANY)
1499 return (EHOSTUNREACH);
1500 return (error);
1501 }
1502
1503 void
in_pcbdisconnect(struct inpcb * inp)1504 in_pcbdisconnect(struct inpcb *inp)
1505 {
1506
1507 INP_WLOCK_ASSERT(inp);
1508 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1509 ("%s: inp %p was already disconnected", __func__, inp));
1510
1511 if (inp->inp_flags & INP_UNCONNECTED)
1512 return;
1513
1514 in_pcbremhash(inp);
1515 IPI_LOCK(inp->inp_pcbinfo);
1516 CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn.head, inp,
1517 inp_unconn_list);
1518 IPI_UNLOCK(inp->inp_pcbinfo);
1519 inp->inp_flags |= INP_UNCONNECTED;
1520
1521 if ((inp->inp_socket->so_proto->pr_flags & PR_CONNREQUIRED) == 0) {
1522 /* See the comment in in_pcbinshash(). */
1523 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1524 inp->inp_faddr.s_addr = INADDR_ANY;
1525 inp->inp_fport = 0;
1526 }
1527 }
1528 #endif /* INET */
1529
1530 void
in_pcblisten(struct inpcb * inp)1531 in_pcblisten(struct inpcb *inp)
1532 {
1533 struct lbgroupbucket *bucket;
1534 struct inpcblbgroup *grp;
1535
1536 INP_WLOCK_ASSERT(inp);
1537
1538 if ((inp->inp_flags & INP_INLBGROUP) == 0)
1539 return;
1540
1541 grp = in_pcblbgroup_find(inp, &bucket);
1542 LIST_REMOVE(inp, inp_lbgroup_list);
1543 grp->il_pendcnt--;
1544 in_pcblbgroup_insert(grp, inp);
1545 INPBUCKET_UNLOCK(bucket);
1546 }
1547
1548 /*
1549 * inpcb hash lookups are protected by SMR section.
1550 *
1551 * Once desired pcb has been found, switching from SMR section to a pcb
1552 * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1553 * here because SMR is a critical section.
1554 * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1555 */
1556 void
inp_lock(struct inpcb * inp,const inp_lookup_t lock)1557 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1558 {
1559
1560 lock == INPLOOKUP_RLOCKPCB ?
1561 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1562 }
1563
1564 void
inp_unlock(struct inpcb * inp,const inp_lookup_t lock)1565 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1566 {
1567
1568 lock == INPLOOKUP_RLOCKPCB ?
1569 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1570 }
1571
1572 int
inp_trylock(struct inpcb * inp,const inp_lookup_t lock)1573 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1574 {
1575
1576 return (lock == INPLOOKUP_RLOCKPCB ?
1577 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1578 }
1579
1580 static inline bool
_inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock,const int ignflags)1581 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1582 {
1583
1584 MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1585 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1586
1587 if (__predict_true(inp_trylock(inp, lock))) {
1588 if (__predict_false(inp->inp_flags & ignflags)) {
1589 smr_exit(inp->inp_pcbinfo->ipi_smr);
1590 inp_unlock(inp, lock);
1591 return (false);
1592 }
1593 smr_exit(inp->inp_pcbinfo->ipi_smr);
1594 return (true);
1595 }
1596
1597 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1598 smr_exit(inp->inp_pcbinfo->ipi_smr);
1599 inp_lock(inp, lock);
1600 if (__predict_false(in_pcbrele(inp, lock)))
1601 return (false);
1602 /*
1603 * inp acquired through refcount & lock for sure didn't went
1604 * through uma_zfree(). However, it may have already went
1605 * through in_pcbfree() and has another reference, that
1606 * prevented its release by our in_pcbrele().
1607 */
1608 if (__predict_false(inp->inp_flags & ignflags)) {
1609 inp_unlock(inp, lock);
1610 return (false);
1611 }
1612 return (true);
1613 } else {
1614 smr_exit(inp->inp_pcbinfo->ipi_smr);
1615 return (false);
1616 }
1617 }
1618
1619 bool
inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock)1620 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1621 {
1622
1623 /*
1624 * in_pcblookup() family of functions shall ignore not onlu pcbs that
1625 * had been freed that may be found due to lockless access to the hash,
1626 * but also pcbs that were removed from the hash, but are still around.
1627 */
1628 return (_inp_smr_lock(inp, lock, INP_FREED | INP_UNCONNECTED));
1629 }
1630
1631 /*
1632 * inp_next() - inpcb hash/list traversal iterator
1633 *
1634 * Requires initialized struct inpcb_iterator for context.
1635 * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1636 *
1637 * - Iterator can have either write-lock or read-lock semantics, that can not
1638 * be changed later.
1639 * - Iterator has three modes of operation, defined by value of .hash member
1640 * on the first call:
1641 * - .hash = INP_ALL_LIST: the iterator will go through the unconnected
1642 * list, then all wildcard hash slots and then all exact hash slots.
1643 * - .hash = INP_UNCONN_LIST: the iterator will go through the list of
1644 * unconnected pcbs only.
1645 * - .hash initialized with an arbitrary positive value: iterator will go
1646 * through this exact hash slot only.
1647 * Note: only rip_input() and sysctl_setsockopt() use the latter.
1648 * The interface may be extended for iteration over single wildcard hash
1649 * slot, but there is no use case for that today.
1650 * - Iterator may have optional bool matching function. The matching function
1651 * will be executed for each inpcb in the SMR context, so it can not acquire
1652 * locks and can safely access only immutable fields of inpcb.
1653 *
1654 * A fresh initialized iterator has NULL inpcb in its context and that
1655 * means that inp_next() call would return the very first inpcb on the list
1656 * locked with desired semantic. In all following calls the context pointer
1657 * shall hold the current inpcb pointer. The KPI user is not supposed to
1658 * unlock the current inpcb! Upon end of traversal inp_next() will return NULL
1659 * and write NULL to its context. After end of traversal an iterator can be
1660 * reused.
1661 *
1662 * List traversals have the following features/constraints:
1663 * - New entries won't be seen, as they are always added to the head of a list.
1664 * - Removed entries won't stop traversal as long as they are not added to
1665 * a different list. This is violated by in_pcbrehash().
1666 */
1667 static inline struct inpcb *
ii_list_first(const struct inpcb_iterator * ii)1668 ii_list_first(const struct inpcb_iterator *ii)
1669 {
1670 const struct inpcbinfo *ipi = ii->ipi;
1671 const int hash = ii->hash;
1672
1673 if (hash < 0)
1674 return (CK_LIST_FIRST(&ipi->ipi_list_unconn.head));
1675 else if (hash <= ipi->ipi_hashmask)
1676 return (CK_LIST_FIRST(&ipi->ipi_hash_wild[hash].head));
1677 else
1678 return (CK_LIST_FIRST(
1679 &ipi->ipi_hash_exact[hash - ipi->ipi_hashmask - 1].head));
1680 }
1681
1682 static inline struct inpcb *
ii_list_next(const struct inpcb_iterator * ii,struct inpcb * inp)1683 ii_list_next(const struct inpcb_iterator *ii, struct inpcb *inp)
1684 {
1685 if (ii->hash < 0)
1686 return (CK_LIST_NEXT(inp, inp_unconn_list));
1687 else if (ii->hash <= ii->ipi->ipi_hashmask)
1688 return (CK_LIST_NEXT(inp, inp_hash_wild));
1689 else
1690 return (CK_LIST_NEXT(inp, inp_hash_exact));
1691 }
1692
1693 struct inpcb *
inp_next(struct inpcb_iterator * ii)1694 inp_next(struct inpcb_iterator *ii)
1695 {
1696 const struct inpcbinfo *ipi = ii->ipi;
1697 const int hashmax = (ipi->ipi_hashmask + 1) * 2;
1698 inp_match_t *match = ii->match;
1699 void *ctx = ii->ctx;
1700 inp_lookup_t lock = ii->lock;
1701 struct inpcb *inp;
1702
1703 if (ii->inp == NULL) { /* First call. */
1704 if ((ii->hash = ii->mode) >= 0) {
1705 /* Targeted iterators support only the exact hash. */
1706 MPASS(ii->hash <= ipi->ipi_hashmask);
1707 ii->hash += ipi->ipi_hashmask + 1;
1708 }
1709 smr_enter(ipi->ipi_smr);
1710 next_first:
1711 /* This is unrolled CK_LIST_FOREACH() over different headers. */
1712 for (inp = ii_list_first(ii);
1713 inp != NULL;
1714 inp = ii_list_next(ii, inp)) {
1715 if (match != NULL && (match)(inp, ctx) == false)
1716 continue;
1717 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1718 break;
1719 else {
1720 smr_enter(ipi->ipi_smr);
1721 MPASS(inp != ii_list_first(ii));
1722 inp = ii_list_first(ii);
1723 if (inp == NULL)
1724 break;
1725 }
1726 }
1727
1728 if (inp == NULL) {
1729 if (ii->mode == INP_ALL_LIST && ++ii->hash < hashmax)
1730 goto next_first;
1731 smr_exit(ipi->ipi_smr);
1732 } else
1733 ii->inp = inp;
1734
1735 return (inp);
1736 }
1737
1738 /* Not a first call. */
1739 smr_enter(ipi->ipi_smr);
1740 restart:
1741 inp = ii->inp;
1742 rw_assert(&inp->inp_lock,
1743 lock == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED);
1744 next:
1745 inp = ii_list_next(ii, inp);
1746 if (inp == NULL) {
1747 if (ii->mode == INP_ALL_LIST && ++ii->hash < hashmax) {
1748 inp_unlock(ii->inp, lock);
1749 ii->inp = NULL;
1750 goto next_first;
1751 }
1752 smr_exit(ipi->ipi_smr);
1753 goto found;
1754 }
1755
1756 if (match != NULL && (match)(inp, ctx) == false)
1757 goto next;
1758
1759 if (__predict_true(inp_trylock(inp, lock))) {
1760 if (__predict_false(inp->inp_flags & INP_FREED)) {
1761 /*
1762 * Entries are never inserted in middle of a list, thus
1763 * as long as we are in SMR, we can continue traversal.
1764 * Jump to 'next' should yield in the same result, but
1765 * could produce unnecessary looping. Could this
1766 * looping be unbound?
1767 */
1768 inp_unlock(inp, lock);
1769 goto next;
1770 } else {
1771 smr_exit(ipi->ipi_smr);
1772 goto found;
1773 }
1774 }
1775
1776 /*
1777 * Can't obtain lock immediately, thus going hard. Once we exit the
1778 * SMR section we can no longer jump to 'next', and our only stable
1779 * anchoring point is ii->inp, which we keep locked for this case, so
1780 * we jump to 'restart'.
1781 */
1782 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1783 smr_exit(ipi->ipi_smr);
1784 inp_lock(inp, lock);
1785 if (__predict_false(in_pcbrele(inp, lock))) {
1786 smr_enter(ipi->ipi_smr);
1787 goto restart;
1788 }
1789 /*
1790 * See comment in inp_smr_lock().
1791 */
1792 if (__predict_false(inp->inp_flags & INP_FREED)) {
1793 inp_unlock(inp, lock);
1794 smr_enter(ipi->ipi_smr);
1795 goto restart;
1796 }
1797 } else
1798 goto next;
1799
1800 found:
1801 inp_unlock(ii->inp, lock);
1802 ii->inp = inp;
1803
1804 return (ii->inp);
1805 }
1806
1807 /*
1808 * in_pcbref() bumps the reference count on an inpcb in order to maintain
1809 * stability of an inpcb pointer despite the inpcb lock being released or
1810 * SMR section exited.
1811 *
1812 * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1813 */
1814 void
in_pcbref(struct inpcb * inp)1815 in_pcbref(struct inpcb *inp)
1816 {
1817 u_int old __diagused;
1818
1819 old = refcount_acquire(&inp->inp_refcount);
1820 KASSERT(old > 0, ("%s: refcount 0", __func__));
1821 }
1822
1823 /*
1824 * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1825 * freeing the pcb, if the reference was very last.
1826 */
1827 bool
in_pcbrele_rlocked(struct inpcb * inp)1828 in_pcbrele_rlocked(struct inpcb *inp)
1829 {
1830
1831 INP_RLOCK_ASSERT(inp);
1832
1833 if (!refcount_release(&inp->inp_refcount))
1834 return (false);
1835
1836 MPASS(inp->inp_flags & INP_FREED);
1837 MPASS(inp->inp_socket == NULL);
1838 crfree(inp->inp_cred);
1839 #ifdef INVARIANTS
1840 inp->inp_cred = NULL;
1841 #endif
1842 INP_RUNLOCK(inp);
1843 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1844 return (true);
1845 }
1846
1847 bool
in_pcbrele_wlocked(struct inpcb * inp)1848 in_pcbrele_wlocked(struct inpcb *inp)
1849 {
1850
1851 INP_WLOCK_ASSERT(inp);
1852
1853 if (!refcount_release(&inp->inp_refcount))
1854 return (false);
1855
1856 MPASS(inp->inp_flags & INP_FREED);
1857 MPASS(inp->inp_socket == NULL);
1858 crfree(inp->inp_cred);
1859 #ifdef INVARIANTS
1860 inp->inp_cred = NULL;
1861 #endif
1862 INP_WUNLOCK(inp);
1863 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1864 return (true);
1865 }
1866
1867 bool
in_pcbrele(struct inpcb * inp,const inp_lookup_t lock)1868 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1869 {
1870
1871 return (lock == INPLOOKUP_RLOCKPCB ?
1872 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1873 }
1874
1875 /*
1876 * Dereference and rlock inp, for which the caller must own the
1877 * reference. Returns true if inp no longer usable, false otherwise.
1878 */
1879 bool
in_pcbrele_rlock(struct inpcb * inp)1880 in_pcbrele_rlock(struct inpcb *inp)
1881 {
1882 INP_RLOCK(inp);
1883 if (in_pcbrele_rlocked(inp))
1884 return (true);
1885 if ((inp->inp_flags & INP_FREED) != 0) {
1886 INP_RUNLOCK(inp);
1887 return (true);
1888 }
1889 return (false);
1890 }
1891
1892 /*
1893 * Unconditionally schedule an inpcb to be freed by decrementing its
1894 * reference count, which should occur only after the inpcb has been detached
1895 * from its socket. If another thread holds a temporary reference (acquired
1896 * using in_pcbref()) then the free is deferred until that reference is
1897 * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1898 * Almost all work, including removal from global lists, is done in this
1899 * context, where the pcbinfo lock is held.
1900 */
1901 void
in_pcbfree(struct inpcb * inp)1902 in_pcbfree(struct inpcb *inp)
1903 {
1904 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1905 #ifdef INET
1906 struct ip_moptions *imo;
1907 #endif
1908 #ifdef INET6
1909 struct ip6_moptions *im6o;
1910 #endif
1911
1912 INP_WLOCK_ASSERT(inp);
1913 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1914 KASSERT((inp->inp_flags & INP_FREED) == 0,
1915 ("%s: called twice for pcb %p", __func__, inp));
1916
1917 /*
1918 * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb
1919 * from the hash without acquiring inpcb lock, they rely on the hash
1920 * lock, thus in_pcbremhash() should be the first action.
1921 */
1922 if (inp->inp_flags & INP_UNCONNECTED) {
1923 IPI_LOCK(pcbinfo);
1924 CK_LIST_REMOVE(inp, inp_unconn_list);
1925 } else {
1926 in_pcbremhash(inp);
1927 IPI_LOCK(pcbinfo);
1928 }
1929 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1930 pcbinfo->ipi_count--;
1931 IPI_UNLOCK(pcbinfo);
1932
1933 #ifdef RATELIMIT
1934 if (inp->inp_snd_tag != NULL)
1935 in_pcbdetach_txrtlmt(inp);
1936 #endif
1937 inp->inp_flags |= INP_FREED;
1938 inp->inp_socket->so_pcb = NULL;
1939 inp->inp_socket = NULL;
1940
1941 RO_INVALIDATE_CACHE(&inp->inp_route);
1942 #ifdef MAC
1943 mac_inpcb_destroy(inp);
1944 #endif
1945 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1946 if (inp->inp_sp != NULL)
1947 ipsec_delete_pcbpolicy(inp);
1948 #endif
1949 #ifdef INET
1950 if (inp->inp_options)
1951 (void)m_free(inp->inp_options);
1952 DEBUG_POISON_POINTER(inp->inp_options);
1953 imo = inp->inp_moptions;
1954 DEBUG_POISON_POINTER(inp->inp_moptions);
1955 #endif
1956 #ifdef INET6
1957 if (inp->inp_vflag & INP_IPV6PROTO) {
1958 ip6_freepcbopts(inp->in6p_outputopts);
1959 DEBUG_POISON_POINTER(inp->in6p_outputopts);
1960 im6o = inp->in6p_moptions;
1961 DEBUG_POISON_POINTER(inp->in6p_moptions);
1962 } else
1963 im6o = NULL;
1964 #endif
1965
1966 if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1967 INP_WUNLOCK(inp);
1968 }
1969 #ifdef INET6
1970 ip6_freemoptions(im6o);
1971 #endif
1972 #ifdef INET
1973 inp_freemoptions(imo);
1974 #endif
1975 }
1976
1977 /*
1978 * Different protocols initialize their inpcbs differently - giving
1979 * different name to the lock. But they all are disposed the same.
1980 */
1981 static void
inpcb_fini(void * mem,int size)1982 inpcb_fini(void *mem, int size)
1983 {
1984 struct inpcb *inp = mem;
1985
1986 INP_LOCK_DESTROY(inp);
1987 }
1988
1989 #ifdef INET
1990 /*
1991 * Common routines to return the socket addresses associated with inpcbs.
1992 */
1993 int
in_getsockaddr(struct socket * so,struct sockaddr * sa)1994 in_getsockaddr(struct socket *so, struct sockaddr *sa)
1995 {
1996 struct inpcb *inp;
1997
1998 inp = sotoinpcb(so);
1999 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
2000
2001 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
2002 .sin_len = sizeof(struct sockaddr_in),
2003 .sin_family = AF_INET,
2004 .sin_port = inp->inp_lport,
2005 .sin_addr = inp->inp_laddr,
2006 };
2007
2008 return (0);
2009 }
2010
2011 int
in_getpeeraddr(struct socket * so,struct sockaddr * sa)2012 in_getpeeraddr(struct socket *so, struct sockaddr *sa)
2013 {
2014 struct inpcb *inp;
2015
2016 inp = sotoinpcb(so);
2017 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
2018
2019 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
2020 .sin_len = sizeof(struct sockaddr_in),
2021 .sin_family = AF_INET,
2022 .sin_port = inp->inp_fport,
2023 .sin_addr = inp->inp_faddr,
2024 };
2025
2026 return (0);
2027 }
2028
2029 static bool
inp_v4_multi_match(const struct inpcb * inp,void * v __unused)2030 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
2031 {
2032
2033 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
2034 return (true);
2035 else
2036 return (false);
2037 }
2038
2039 void
in_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)2040 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
2041 {
2042 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
2043 inp_v4_multi_match, NULL);
2044 struct inpcb *inp;
2045 struct in_multi *inm;
2046 struct in_mfilter *imf;
2047 struct ip_moptions *imo;
2048
2049 IN_MULTI_LOCK_ASSERT();
2050
2051 while ((inp = inp_next(&inpi)) != NULL) {
2052 INP_WLOCK_ASSERT(inp);
2053
2054 imo = inp->inp_moptions;
2055 /*
2056 * Unselect the outgoing interface if it is being
2057 * detached.
2058 */
2059 if (imo->imo_multicast_ifp == ifp)
2060 imo->imo_multicast_ifp = NULL;
2061
2062 /*
2063 * Drop multicast group membership if we joined
2064 * through the interface being detached.
2065 *
2066 * XXX This can all be deferred to an epoch_call
2067 */
2068 restart:
2069 IP_MFILTER_FOREACH(imf, &imo->imo_head) {
2070 if ((inm = imf->imf_inm) == NULL)
2071 continue;
2072 if (inm->inm_ifp != ifp)
2073 continue;
2074 ip_mfilter_remove(&imo->imo_head, imf);
2075 in_leavegroup_locked(inm, NULL);
2076 ip_mfilter_free(imf);
2077 goto restart;
2078 }
2079 }
2080 }
2081
2082 /*
2083 * Lookup a PCB based on the local address and port.
2084 * No inpcb locks or references are acquired.
2085 *
2086 * Always returns with a locked context.
2087 */
2088 #define INP_LOOKUP_MAPPED_PCB_COST 3
2089 struct inpcb *
in_pcblookup_local(struct inpcbinfo_ctx * ipictx,struct in_addr laddr,u_short lport,int fib,int lookupflags,struct ucred * cred)2090 in_pcblookup_local(struct inpcbinfo_ctx *ipictx, struct in_addr laddr,
2091 u_short lport, int fib, int lookupflags, struct ucred *cred)
2092 {
2093 struct inpcb *inp;
2094 #ifdef INET6
2095 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
2096 #else
2097 int matchwild = 3;
2098 #endif
2099 int wildcard;
2100
2101 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
2102 ("%s: invalid lookup flags %d", __func__, lookupflags));
2103 KASSERT(fib == RT_ALL_FIBS || (fib >= 0 && fib < V_rt_numfibs),
2104 ("%s: invalid fib %d", __func__, fib));
2105
2106 /*
2107 * Lock the wildcard bucket regardless of lookupflags. It needs to be
2108 * locked before the ports bucket. Those callers that call
2109 * in_pcblookup_local() as a check before in_pcbinshash() require
2110 * wildcard bucket locked.
2111 * This function may be called repeatedly and inpcbinfo_ctx_wildlock()
2112 * takes care against lock recursion.
2113 */
2114 inpcbinfo_ctx_wildlock(ipictx, lport);
2115
2116 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
2117 /*
2118 * Look for an unconnected (wildcard foreign addr) PCB that
2119 * matches the local address and port we're looking for.
2120 */
2121 CK_LIST_FOREACH(inp, &ipictx->wbucket->head, inp_hash_wild) {
2122 #ifdef INET6
2123 /* XXX inp locking */
2124 if ((inp->inp_vflag & INP_IPV4) == 0)
2125 continue;
2126 #endif
2127 if (inp->inp_laddr.s_addr == laddr.s_addr &&
2128 inp->inp_lport == lport && (fib == RT_ALL_FIBS ||
2129 inp->inp_inc.inc_fibnum == fib)) {
2130 /*
2131 * Found?
2132 */
2133 if (prison_equal_ip4(cred->cr_prison,
2134 inp->inp_cred->cr_prison))
2135 return (inp);
2136 }
2137 }
2138 /*
2139 * Not found.
2140 */
2141 return (NULL);
2142 } else {
2143 struct inpcb *match = NULL;
2144
2145 /*
2146 * Port may be in use by one or more PCBs. Look for best fit.
2147 */
2148 inpcbinfo_ctx_portlock(ipictx, lport);
2149 CK_LIST_FOREACH(inp, &ipictx->pbucket->head, inp_portlist) {
2150 if (inp->inp_lport != lport)
2151 continue;
2152 if (!prison_equal_ip4(inp->inp_cred->cr_prison,
2153 cred->cr_prison))
2154 continue;
2155 if (fib != RT_ALL_FIBS &&
2156 inp->inp_inc.inc_fibnum != fib)
2157 continue;
2158 wildcard = 0;
2159 #ifdef INET6
2160 /* XXX inp locking */
2161 if ((inp->inp_vflag & INP_IPV4) == 0)
2162 continue;
2163 /*
2164 * We never select the PCB that has INP_IPV6 flag and
2165 * is bound to :: if we have another PCB which is bound
2166 * to 0.0.0.0. If a PCB has the INP_IPV6 flag, then we
2167 * set its cost higher than IPv4 only PCBs.
2168 *
2169 * Note that the case only happens when a socket is
2170 * bound to ::, under the condition that the use of the
2171 * mapped address is allowed.
2172 */
2173 if ((inp->inp_vflag & INP_IPV6) != 0)
2174 wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2175 #endif
2176 if (inp->inp_faddr.s_addr != INADDR_ANY)
2177 wildcard++;
2178 if (inp->inp_laddr.s_addr != INADDR_ANY) {
2179 if (laddr.s_addr == INADDR_ANY)
2180 wildcard++;
2181 else if (inp->inp_laddr.s_addr != laddr.s_addr)
2182 continue;
2183 } else {
2184 if (laddr.s_addr != INADDR_ANY)
2185 wildcard++;
2186 }
2187 if (wildcard < matchwild) {
2188 match = inp;
2189 matchwild = wildcard;
2190 if (matchwild == 0)
2191 break;
2192 }
2193 }
2194 return (match);
2195 }
2196 }
2197 #undef INP_LOOKUP_MAPPED_PCB_COST
2198
2199 static bool
in_pcblookup_lb_match(const struct inpcblbgroup * grp,int domain,int fib)2200 in_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib)
2201 {
2202 return ((domain == M_NODOM || domain == grp->il_numa_domain) &&
2203 (fib == RT_ALL_FIBS || fib == grp->il_fibnum));
2204 }
2205
2206 static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in_addr * faddr,uint16_t fport,const struct in_addr * laddr,uint16_t lport,int domain,int fib,struct lbgroupbucket ** bucketp)2207 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2208 const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr,
2209 uint16_t lport, int domain, int fib, struct lbgroupbucket **bucketp)
2210 {
2211 struct lbgroupbucket *bucket;
2212 struct inpcblbgroup *grp;
2213 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
2214 struct inpcb *inp;
2215 u_int count;
2216
2217 NET_EPOCH_ASSERT();
2218 MPASS(bucketp != NULL || SMR_ENTERED(pcbinfo->ipi_smr));
2219
2220 /*
2221 * Search for an LB group match based on the following criteria:
2222 * - prefer jailed groups to non-jailed groups
2223 * - prefer exact source address matches to wildcard matches
2224 * - prefer groups bound to the specified NUMA domain
2225 */
2226 jail_exact = jail_wild = local_exact = local_wild = NULL;
2227 bucket = &pcbinfo->ipi_lbgrouphashbase[
2228 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2229 if (bucketp != NULL) {
2230 MPASS(*bucketp == NULL);
2231 *bucketp = bucket;
2232 INPBUCKET_LOCK(bucket);
2233 }
2234 CK_LIST_FOREACH(grp, &bucket->head, il_list) {
2235 bool injail;
2236
2237 #ifdef INET6
2238 if (!(grp->il_vflag & INP_IPV4))
2239 continue;
2240 #endif
2241 if (grp->il_lport != lport)
2242 continue;
2243
2244 injail = prison_flag(grp->il_cred, PR_IP4) != 0;
2245 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,
2246 laddr) != 0)
2247 continue;
2248
2249 if (grp->il_laddr.s_addr == laddr->s_addr) {
2250 if (injail) {
2251 jail_exact = grp;
2252 if (in_pcblookup_lb_match(grp, domain, fib))
2253 /* This is a perfect match. */
2254 goto out;
2255 } else if (local_exact == NULL ||
2256 in_pcblookup_lb_match(grp, domain, fib)) {
2257 local_exact = grp;
2258 }
2259 } else if (grp->il_laddr.s_addr == INADDR_ANY) {
2260 if (injail) {
2261 if (jail_wild == NULL ||
2262 in_pcblookup_lb_match(grp, domain, fib))
2263 jail_wild = grp;
2264 } else if (local_wild == NULL ||
2265 in_pcblookup_lb_match(grp, domain, fib)) {
2266 local_wild = grp;
2267 }
2268 }
2269 }
2270
2271 if (jail_exact != NULL)
2272 grp = jail_exact;
2273 else if (jail_wild != NULL)
2274 grp = jail_wild;
2275 else if (local_exact != NULL)
2276 grp = local_exact;
2277 else
2278 grp = local_wild;
2279 if (grp == NULL)
2280 return (NULL);
2281
2282 out:
2283 /*
2284 * Synchronize with in_pcblbgroup_insert().
2285 */
2286 count = atomic_load_acq_int(&grp->il_inpcnt);
2287 if (count == 0)
2288 return (NULL);
2289 inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
2290 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
2291 return (inp);
2292 }
2293
2294 static bool
in_pcblookup_exact_match(const struct inpcb * inp,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2295 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2296 u_short fport, struct in_addr laddr, u_short lport)
2297 {
2298 #ifdef INET6
2299 /* XXX inp locking */
2300 if ((inp->inp_vflag & INP_IPV4) == 0)
2301 return (false);
2302 #endif
2303 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2304 inp->inp_laddr.s_addr == laddr.s_addr &&
2305 inp->inp_fport == fport &&
2306 inp->inp_lport == lport)
2307 return (true);
2308 return (false);
2309 }
2310
2311 /*
2312 * Look up inpcb in the exact hash.
2313 *
2314 * For packet processing called in the SMR section. For conflict checks the
2315 * bucket pointer shall be provided that will be returned locked.
2316 */
2317 static struct inpcb *
in_pcblookup_exact(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport,struct inpbucket ** bucketp)2318 in_pcblookup_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2319 u_short fport, struct in_addr laddr, u_short lport,
2320 struct inpbucket **bucketp)
2321 {
2322 struct inpbucket *bucket;
2323 struct inpcb *inp;
2324
2325 MPASS(bucketp != NULL || SMR_ENTERED(pcbinfo->ipi_smr));
2326
2327 bucket = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
2328 pcbinfo->ipi_hashmask)];
2329 if (bucketp != NULL) {
2330 MPASS(*bucketp == NULL);
2331 *bucketp = bucket;
2332 INPBUCKET_LOCK(bucket);
2333 }
2334 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_exact) {
2335 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2336 return (inp);
2337 }
2338 return (NULL);
2339 }
2340
2341 typedef enum {
2342 INPLOOKUP_MATCH_NONE = 0,
2343 INPLOOKUP_MATCH_WILD = 1,
2344 INPLOOKUP_MATCH_LADDR = 2,
2345 } inp_lookup_match_t;
2346
2347 static inp_lookup_match_t
in_pcblookup_wild_match(const struct inpcb * inp,struct in_addr laddr,u_short lport,int fib)2348 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2349 u_short lport, int fib)
2350 {
2351 #ifdef INET6
2352 /* XXX inp locking */
2353 if ((inp->inp_vflag & INP_IPV4) == 0)
2354 return (INPLOOKUP_MATCH_NONE);
2355 #endif
2356 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2357 return (INPLOOKUP_MATCH_NONE);
2358 if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib)
2359 return (INPLOOKUP_MATCH_NONE);
2360 if (inp->inp_laddr.s_addr == INADDR_ANY)
2361 return (INPLOOKUP_MATCH_WILD);
2362 if (inp->inp_laddr.s_addr == laddr.s_addr)
2363 return (INPLOOKUP_MATCH_LADDR);
2364 return (INPLOOKUP_MATCH_NONE);
2365 }
2366
2367 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
2368
2369 static struct inpcb *
in_pcblookup_wild_smr(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int fib,const inp_lookup_t lockflags)2370 in_pcblookup_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2371 u_short lport, int fib, const inp_lookup_t lockflags)
2372 {
2373 struct inpbucket *bucket;
2374 struct inpcb *inp;
2375
2376 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
2377 ("%s: not in SMR read section", __func__));
2378
2379 bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2380 pcbinfo->ipi_hashmask)];
2381 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
2382 inp_lookup_match_t match;
2383
2384 match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2385 if (match == INPLOOKUP_MATCH_NONE)
2386 continue;
2387
2388 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2389 match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2390 if (match != INPLOOKUP_MATCH_NONE &&
2391 prison_check_ip4_locked(inp->inp_cred->cr_prison,
2392 &laddr) == 0)
2393 return (inp);
2394 inp_unlock(inp, lockflags);
2395 }
2396
2397 /*
2398 * The matching socket disappeared out from under us. Fall back
2399 * to a serialized lookup.
2400 */
2401 return (INP_LOOKUP_AGAIN);
2402 }
2403 return (NULL);
2404 }
2405
2406 /*
2407 * Always returns with locked bucket.
2408 */
2409 static struct inpcb *
in_pcblookup_wild_locked(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int fib,struct inpbucket ** bucketp)2410 in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2411 u_short lport, int fib, struct inpbucket **bucketp)
2412 {
2413 struct inpbucket *bucket;
2414 struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2415 #ifdef INET6
2416 struct inpcb *local_wild_mapped;
2417 #endif
2418
2419 MPASS(bucketp != NULL);
2420
2421 /*
2422 * Order of socket selection - we always prefer jails.
2423 * 1. jailed, non-wild.
2424 * 2. jailed, wild.
2425 * 3. non-jailed, non-wild.
2426 * 4. non-jailed, wild.
2427 */
2428 local_wild = local_exact = jail_wild = NULL;
2429 #ifdef INET6
2430 local_wild_mapped = NULL;
2431 #endif
2432 bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2433 pcbinfo->ipi_hashmask)];
2434 MPASS(*bucketp == NULL);
2435 *bucketp = bucket;
2436 INPBUCKET_LOCK(bucket);
2437 CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
2438 inp_lookup_match_t match;
2439 bool injail;
2440
2441 match = in_pcblookup_wild_match(inp, laddr, lport, fib);
2442 if (match == INPLOOKUP_MATCH_NONE)
2443 continue;
2444
2445 injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2446 if (injail) {
2447 if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2448 &laddr) != 0)
2449 continue;
2450 } else {
2451 if (local_exact != NULL)
2452 continue;
2453 }
2454
2455 if (match == INPLOOKUP_MATCH_LADDR) {
2456 if (injail)
2457 return (inp);
2458 local_exact = inp;
2459 } else {
2460 #ifdef INET6
2461 /* XXX inp locking, NULL check */
2462 if (inp->inp_vflag & INP_IPV6PROTO)
2463 local_wild_mapped = inp;
2464 else
2465 #endif
2466 if (injail)
2467 jail_wild = inp;
2468 else
2469 local_wild = inp;
2470 }
2471 }
2472 if (jail_wild != NULL)
2473 return (jail_wild);
2474 if (local_exact != NULL)
2475 return (local_exact);
2476 if (local_wild != NULL)
2477 return (local_wild);
2478 #ifdef INET6
2479 if (local_wild_mapped != NULL)
2480 return (local_wild_mapped);
2481 #endif
2482 return (NULL);
2483 }
2484
2485 /*
2486 * Returns with fully locked context in case of failure and partially
2487 * locked in case of successful lookup.
2488 */
2489 static struct inpcb *
in_pcblookup_internal(struct inpcbinfo_ctx * ipictx,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)2490 in_pcblookup_internal(struct inpcbinfo_ctx *ipictx, struct in_addr faddr,
2491 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2492 uint8_t numa_domain, int fib)
2493 {
2494 const u_short fport = fport_arg, lport = lport_arg;
2495 struct inpcb *inp;
2496
2497 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0,
2498 ("%s: invalid lookup flags %d", __func__, lookupflags));
2499 KASSERT(faddr.s_addr != INADDR_ANY,
2500 ("%s: invalid foreign address", __func__));
2501 KASSERT(laddr.s_addr != INADDR_ANY,
2502 ("%s: invalid local address", __func__));
2503
2504 inp = in_pcblookup_exact(ipictx->pcbinfo, faddr, fport, laddr, lport,
2505 &ipictx->ebucket);
2506 if (inp != NULL)
2507 return (inp);
2508
2509 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2510 inp = in_pcblookup_lbgroup(ipictx->pcbinfo, &faddr, fport,
2511 &laddr, lport, numa_domain, fib, &ipictx->lbbucket);
2512 if (inp == NULL) {
2513 /*
2514 * Unlock the lbbucket to avoid LOR. If the lookup
2515 * is made for insertion, it will be acquired later.
2516 * It is safe to do so, since load balance groups
2517 * by design have "conflicting" inpcbs.
2518 */
2519 INPBUCKET_UNLOCK(ipictx->lbbucket);
2520 ipictx->lbbucket = NULL;
2521 inp = in_pcblookup_wild_locked(ipictx->pcbinfo, laddr,
2522 lport, fib, &ipictx->wbucket);
2523 }
2524 }
2525
2526 return (inp);
2527 }
2528
2529 /*
2530 * Lookup inpcb using locks. Used by in_pcblookup_smr() in case inp_smr_lock()
2531 * failed.
2532 * Returns with locked inpcb.
2533 */
2534 static struct inpcb *
in_pcblookup_with_lock(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,uint8_t numa_domain,int fib)2535 in_pcblookup_with_lock(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2536 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2537 uint8_t numa_domain, int fib)
2538 {
2539 struct inpcbinfo_ctx ipictx = {
2540 .pcbinfo = pcbinfo
2541 };
2542 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2543 struct inpcb *inp;
2544
2545 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2546 ("%s: LOCKPCB not set", __func__));
2547
2548 inp = in_pcblookup_internal(&ipictx, faddr, fport, laddr, lport,
2549 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
2550 if (inp != NULL && !inp_trylock(inp, lockflags)) {
2551 in_pcbref(inp);
2552 inpcbinfo_ctx_release(&ipictx);
2553 inp_lock(inp, lockflags);
2554 if (in_pcbrele(inp, lockflags))
2555 /* XXX-MJ or retry until we get a negative match? */
2556 inp = NULL;
2557 } else
2558 inpcbinfo_ctx_release(&ipictx);
2559 return (inp);
2560 }
2561
2562 static struct inpcb *
in_pcblookup_smr(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)2563 in_pcblookup_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2564 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2565 uint8_t numa_domain, int fib)
2566 {
2567 struct inpcb *inp;
2568 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2569 const u_short fport = fport_arg, lport = lport_arg;
2570
2571 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2572 ("%s: invalid lookup flags %d", __func__, lookupflags));
2573 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2574 ("%s: LOCKPCB not set", __func__));
2575
2576 smr_enter(pcbinfo->ipi_smr);
2577 inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport, NULL);
2578 if (inp != NULL) {
2579 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2580 /*
2581 * Revalidate the 4-tuple, the socket could have been
2582 * disconnected.
2583 */
2584 if (__predict_true(in_pcblookup_exact_match(inp,
2585 faddr, fport, laddr, lport)))
2586 return (inp);
2587 inp_unlock(inp, lockflags);
2588 }
2589
2590 /*
2591 * We failed to lock the inpcb, or its connection state changed
2592 * out from under us. Fall back to a precise search.
2593 */
2594 return (in_pcblookup_with_lock(pcbinfo, faddr, fport, laddr,
2595 lport, lookupflags, numa_domain, fib));
2596 }
2597
2598 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2599 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2600 &laddr, lport, numa_domain, fib, NULL);
2601 if (inp != NULL) {
2602 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2603 if (__predict_true(in_pcblookup_wild_match(inp,
2604 laddr, lport, fib) != INPLOOKUP_MATCH_NONE))
2605 return (inp);
2606 inp_unlock(inp, lockflags);
2607 }
2608 inp = INP_LOOKUP_AGAIN;
2609 } else {
2610 inp = in_pcblookup_wild_smr(pcbinfo, laddr, lport,
2611 fib, lockflags);
2612 }
2613 if (inp == INP_LOOKUP_AGAIN) {
2614 return (in_pcblookup_with_lock(pcbinfo, faddr, fport,
2615 laddr, lport, lookupflags, numa_domain, fib));
2616 }
2617 }
2618
2619 if (inp == NULL)
2620 smr_exit(pcbinfo->ipi_smr);
2621
2622 return (inp);
2623 }
2624
2625 /*
2626 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2627 * from which a pre-calculated hash value may be extracted.
2628 */
2629 struct inpcb *
in_pcblookup(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp)2630 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2631 struct in_addr laddr, u_int lport, int lookupflags,
2632 struct ifnet *ifp)
2633 {
2634 int fib;
2635
2636 fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS;
2637 return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport,
2638 lookupflags, M_NODOM, fib));
2639 }
2640
2641 struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)2642 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2643 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2644 struct ifnet *ifp __unused, struct mbuf *m)
2645 {
2646 int fib;
2647
2648 M_ASSERTPKTHDR(m);
2649 fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS;
2650 return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport,
2651 lookupflags, m->m_pkthdr.numa_domain, fib));
2652 }
2653 #endif /* INET */
2654
2655 static bool
in_pcbjailed(const struct inpcb * inp,unsigned int flag)2656 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2657 {
2658 return (prison_flag(inp->inp_cred, flag) != 0);
2659 }
2660
2661 /*
2662 * Insert the PCB into a hash chain using ordering rules which ensure that
2663 * in_pcblookup_wild_*() always encounter the highest-ranking PCB first.
2664 *
2665 * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2666 * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs
2667 * always appear last no matter whether they are jailed.
2668 */
2669 static void
_in_pcbinshash_wild(struct inpbucket * bucket,struct inpcb * inp)2670 _in_pcbinshash_wild(struct inpbucket *bucket, struct inpcb *inp)
2671 {
2672 struct inpcb *last;
2673 bool bound, injail;
2674
2675 INP_LOCK_ASSERT(inp);
2676 INPBUCKET_LOCK_ASSERT(bucket);
2677
2678 last = NULL;
2679 bound = inp->inp_laddr.s_addr != INADDR_ANY;
2680 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2681 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
2682 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2683 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2684 return;
2685 }
2686 }
2687 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
2688 return;
2689 }
2690
2691 injail = in_pcbjailed(inp, PR_IP4);
2692 if (!injail) {
2693 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
2694 if (!in_pcbjailed(last, PR_IP4))
2695 break;
2696 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2697 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2698 return;
2699 }
2700 }
2701 } else if (!CK_LIST_EMPTY(&bucket->head) &&
2702 !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP4)) {
2703 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
2704 return;
2705 }
2706 if (!bound) {
2707 CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) {
2708 if (last->inp_laddr.s_addr == INADDR_ANY)
2709 break;
2710 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2711 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2712 return;
2713 }
2714 }
2715 }
2716 if (last == NULL)
2717 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
2718 else
2719 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2720 }
2721
2722 #ifdef INET6
2723 /*
2724 * See the comment above _in_pcbinshash_wild().
2725 */
2726 static void
_in6_pcbinshash_wild(struct inpbucket * bucket,struct inpcb * inp)2727 _in6_pcbinshash_wild(struct inpbucket *bucket, struct inpcb *inp)
2728 {
2729 struct inpcb *last;
2730 bool bound, injail;
2731
2732 INP_LOCK_ASSERT(inp);
2733 INPBUCKET_LOCK_ASSERT(bucket);
2734
2735 last = NULL;
2736 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2737 injail = in_pcbjailed(inp, PR_IP6);
2738 if (!injail) {
2739 CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
2740 if (!in_pcbjailed(last, PR_IP6))
2741 break;
2742 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2743 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2744 return;
2745 }
2746 }
2747 } else if (!CK_LIST_EMPTY(&bucket->head) &&
2748 !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP6)) {
2749 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
2750 return;
2751 }
2752 if (!bound) {
2753 CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) {
2754 if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
2755 break;
2756 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2757 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2758 return;
2759 }
2760 }
2761 }
2762 if (last == NULL)
2763 CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
2764 else
2765 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2766 }
2767 #endif
2768
2769 /*
2770 * Insert PCB onto various hash lists.
2771 *
2772 * With normal sockets this function shall not fail, so it could return void.
2773 * But for SO_REUSEPORT_LB it may need to allocate memory with locks held,
2774 * that's the only condition when it can fail.
2775 */
2776 int
in_pcbinshash(struct inpcb * inp,struct inpcbinfo_ctx * ipictx)2777 in_pcbinshash(struct inpcb *inp, struct inpcbinfo_ctx *ipictx)
2778 {
2779 uint32_t hash __diagused;
2780 bool connected;
2781
2782 INP_WLOCK_ASSERT(inp);
2783 MPASS(inp->inp_flags & INP_UNCONNECTED);
2784
2785 #ifdef INET6
2786 if (inp->inp_vflag & INP_IPV6) {
2787 #ifdef INVARIANTS
2788 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2789 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
2790 #endif
2791 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2792 } else
2793 #endif
2794 {
2795 #ifdef INVARIANTS
2796 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2797 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
2798 #endif
2799 connected = !in_nullhost(inp->inp_faddr);
2800 }
2801
2802 #ifdef INVARIANTS
2803 if (connected) {
2804 MPASS(ipictx->ebucket ==
2805 &ipictx->pcbinfo->ipi_hash_exact[hash]);
2806 INPBUCKET_LOCK_ASSERT(ipictx->ebucket);
2807 } else {
2808 MPASS(ipictx->wbucket == &ipictx->pcbinfo->ipi_hash_wild[hash]);
2809 INPBUCKET_LOCK_ASSERT(ipictx->wbucket);
2810 }
2811 #endif
2812
2813 /*
2814 * Ignore SO_REUSEPORT_LB if the socket is connected. Really this case
2815 * should be an error, but for UDP sockets it is not, and some
2816 * applications erroneously set it on connected UDP sockets, so we can't
2817 * change this without breaking compatibility.
2818 */
2819 if (!connected &&
2820 (inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2821 int error = in_pcbinslbgrouphash(inp, M_NODOM);
2822 if (error != 0)
2823 return (error);
2824 }
2825
2826 /*
2827 * The PCB may have been disconnected in the past. Before we can safely
2828 * make it visible in the hash table, we must wait for all readers which
2829 * may be traversing this PCB to finish.
2830 */
2831 if (inp->inp_smr != SMR_SEQ_INVALID) {
2832 smr_wait(ipictx->pcbinfo->ipi_smr, inp->inp_smr);
2833 inp->inp_smr = SMR_SEQ_INVALID;
2834 }
2835
2836 IPI_LOCK(ipictx->pcbinfo);
2837 CK_LIST_REMOVE(inp, inp_unconn_list);
2838 IPI_UNLOCK(ipictx->pcbinfo);
2839
2840 if (connected) {
2841 CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp,
2842 inp_hash_exact);
2843 } else {
2844 #ifdef INET6
2845 if ((inp->inp_vflag & INP_IPV6) != 0)
2846 _in6_pcbinshash_wild(ipictx->wbucket, inp);
2847 else
2848 #endif
2849 _in_pcbinshash_wild(ipictx->wbucket, inp);
2850 }
2851 /*
2852 * In most code paths we arrive here without locked pbucket. This is
2853 * always true for connect(2) and also true for bind(2) when the socket
2854 * has SO_REUSEADDR set then port hash checks were not performed.
2855 */
2856 inpcbinfo_ctx_portlock(ipictx, inp->inp_lport);
2857 CK_LIST_INSERT_HEAD(&ipictx->pbucket->head, inp, inp_portlist);
2858 inp->inp_flags &= ~INP_UNCONNECTED;
2859
2860 return (0);
2861 }
2862
2863 void
in_pcbremhash(struct inpcb * inp)2864 in_pcbremhash(struct inpcb *inp)
2865 {
2866 struct inpbucket *bucket;
2867 uint32_t hash;
2868 bool connected;
2869
2870 INP_WLOCK_ASSERT(inp);
2871 MPASS(!(inp->inp_flags & INP_UNCONNECTED));
2872
2873 if ((inp->inp_flags & INP_INLBGROUP) != 0) {
2874 struct lbgroupbucket *lbbucket;
2875 struct inpcblbgroup *g __diagused;
2876
2877 g = in_pcblbgroup_find(inp, &lbbucket);
2878 MPASS(g);
2879 in_pcbremlbgrouphash(lbbucket, inp);
2880 INPBUCKET_UNLOCK(lbbucket);
2881 }
2882
2883 #ifdef INET6
2884 if (inp->inp_vflag & INP_IPV6) {
2885 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2886 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
2887 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2888 } else
2889 #endif
2890 {
2891 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2892 inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
2893 connected = !in_nullhost(inp->inp_faddr);
2894 }
2895
2896 if (connected) {
2897 bucket = &inp->inp_pcbinfo->ipi_hash_exact[hash];
2898 INPBUCKET_LOCK(bucket);
2899 CK_LIST_REMOVE(inp, inp_hash_exact);
2900 } else {
2901 bucket = &inp->inp_pcbinfo->ipi_hash_wild[hash];
2902 INPBUCKET_LOCK(bucket);
2903 CK_LIST_REMOVE(inp, inp_hash_wild);
2904 }
2905 INPBUCKET_UNLOCK(bucket);
2906
2907 bucket = &inp->inp_pcbinfo->ipi_porthash[
2908 INP_PCBPORTHASH(inp->inp_lport,
2909 inp->inp_pcbinfo->ipi_porthashmask)];
2910 INPBUCKET_LOCK(bucket);
2911 CK_LIST_REMOVE(inp, inp_portlist);
2912 INPBUCKET_UNLOCK(bucket);
2913 }
2914
2915 /*
2916 * Move PCB to the proper hash bucket when { faddr, fport } have been
2917 * changed. NOTE: This does not handle the case of the lport changing (the
2918 * hashed port list would have to be updated as well), so the lport must
2919 * not change after in_pcbinshash() has been called.
2920 */
2921 void
in_pcbrehash(struct inpcb * inp,struct inpcbinfo_ctx * ipictx)2922 in_pcbrehash(struct inpcb *inp, struct inpcbinfo_ctx *ipictx)
2923 {
2924 #ifdef INVARIANTS
2925 uint32_t hash;
2926
2927 INP_WLOCK_ASSERT(inp);
2928 MPASS(!(inp->inp_flags & INP_UNCONNECTED));
2929 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2930 ("%s: inp was disconnected", __func__));
2931 #ifdef INET6
2932 if (inp->inp_vflag & INP_IPV6) {
2933 MPASS(!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr));
2934 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2935 inp->inp_fport, ipictx->pcbinfo->ipi_hashmask);
2936 } else
2937 #endif
2938 {
2939 MPASS(!in_nullhost(inp->inp_faddr));
2940 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2941 inp->inp_fport, ipictx->pcbinfo->ipi_hashmask);
2942 }
2943 MPASS(ipictx->ebucket == &ipictx->pcbinfo->ipi_hash_exact[hash]);
2944 INPBUCKET_LOCK_ASSERT(ipictx->ebucket);
2945 INPBUCKET_LOCK_ASSERT(ipictx->wbucket);
2946 #endif
2947
2948 /* See the comment in in_pcbinshash(). */
2949 if ((inp->inp_flags & INP_INLBGROUP) != 0) {
2950 INPBUCKET_LOCK_ASSERT(ipictx->lbbucket);
2951 in_pcbremlbgrouphash(ipictx->lbbucket, inp);
2952 }
2953
2954 CK_LIST_REMOVE(inp, inp_hash_wild);
2955 CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp, inp_hash_exact);
2956 }
2957
2958 void
ripcb_connect(struct inpcb * inp)2959 ripcb_connect(struct inpcb *inp)
2960 {
2961 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2962 uint32_t hash;
2963
2964 INP_WLOCK_ASSERT(inp);
2965 MPASS(inp->inp_flags & INP_UNCONNECTED);
2966
2967 hash = RIPCB_HASH(inp) & pcbinfo->ipi_hashmask;
2968
2969 IPI_LOCK(pcbinfo);
2970 CK_LIST_REMOVE(inp, inp_unconn_list);
2971 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_hash_exact[hash].head, inp,
2972 inp_hash_exact);
2973 IPI_UNLOCK(pcbinfo);
2974 inp->inp_flags &= ~INP_UNCONNECTED;
2975 }
2976
2977 void
ripcb_disconnect(struct inpcb * inp)2978 ripcb_disconnect(struct inpcb *inp)
2979 {
2980 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2981
2982 INP_WLOCK_ASSERT(inp);
2983
2984 if (inp->inp_flags & INP_UNCONNECTED)
2985 return;
2986
2987 IPI_LOCK(pcbinfo);
2988 CK_LIST_REMOVE(inp, inp_hash_exact);
2989 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp,
2990 inp_unconn_list);
2991 IPI_UNLOCK(pcbinfo);
2992 inp->inp_flags |= INP_UNCONNECTED;
2993 }
2994
2995 /*
2996 * Check for alternatives when higher level complains
2997 * about service problems. For now, invalidate cached
2998 * routing information. If the route was created dynamically
2999 * (by a redirect), time to try a default gateway again.
3000 */
3001 void
in_losing(struct inpcb * inp)3002 in_losing(struct inpcb *inp)
3003 {
3004
3005 RO_INVALIDATE_CACHE(&inp->inp_route);
3006 return;
3007 }
3008
3009 /*
3010 * A set label operation has occurred at the socket layer, propagate the
3011 * label change into the in_pcb for the socket.
3012 */
3013 void
in_pcbsosetlabel(struct socket * so)3014 in_pcbsosetlabel(struct socket *so)
3015 {
3016 #ifdef MAC
3017 struct inpcb *inp;
3018
3019 inp = sotoinpcb(so);
3020 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
3021
3022 INP_WLOCK(inp);
3023 SOCK_LOCK(so);
3024 mac_inpcb_sosetlabel(so, inp);
3025 SOCK_UNLOCK(so);
3026 INP_WUNLOCK(inp);
3027 #endif
3028 }
3029
3030 void
inp_wlock(struct inpcb * inp)3031 inp_wlock(struct inpcb *inp)
3032 {
3033
3034 INP_WLOCK(inp);
3035 }
3036
3037 void
inp_wunlock(struct inpcb * inp)3038 inp_wunlock(struct inpcb *inp)
3039 {
3040
3041 INP_WUNLOCK(inp);
3042 }
3043
3044 void
inp_rlock(struct inpcb * inp)3045 inp_rlock(struct inpcb *inp)
3046 {
3047
3048 INP_RLOCK(inp);
3049 }
3050
3051 void
inp_runlock(struct inpcb * inp)3052 inp_runlock(struct inpcb *inp)
3053 {
3054
3055 INP_RUNLOCK(inp);
3056 }
3057
3058 #ifdef INVARIANT_SUPPORT
3059 void
inp_lock_assert(struct inpcb * inp)3060 inp_lock_assert(struct inpcb *inp)
3061 {
3062
3063 INP_WLOCK_ASSERT(inp);
3064 }
3065
3066 void
inp_unlock_assert(struct inpcb * inp)3067 inp_unlock_assert(struct inpcb *inp)
3068 {
3069
3070 INP_UNLOCK_ASSERT(inp);
3071 }
3072 #endif
3073
3074 void
inp_apply_all(struct inpcbinfo * pcbinfo,void (* func)(struct inpcb *,void *),void * arg)3075 inp_apply_all(struct inpcbinfo *pcbinfo,
3076 void (*func)(struct inpcb *, void *), void *arg)
3077 {
3078 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
3079 INPLOOKUP_WLOCKPCB);
3080 struct inpcb *inp;
3081
3082 while ((inp = inp_next(&inpi)) != NULL)
3083 func(inp, arg);
3084 }
3085
3086 struct socket *
inp_inpcbtosocket(struct inpcb * inp)3087 inp_inpcbtosocket(struct inpcb *inp)
3088 {
3089
3090 INP_WLOCK_ASSERT(inp);
3091 return (inp->inp_socket);
3092 }
3093
3094 void
inp_4tuple_get(struct inpcb * inp,uint32_t * laddr,uint16_t * lp,uint32_t * faddr,uint16_t * fp)3095 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
3096 uint32_t *faddr, uint16_t *fp)
3097 {
3098
3099 INP_LOCK_ASSERT(inp);
3100 *laddr = inp->inp_laddr.s_addr;
3101 *faddr = inp->inp_faddr.s_addr;
3102 *lp = inp->inp_lport;
3103 *fp = inp->inp_fport;
3104 }
3105
3106 /*
3107 * Create an external-format (``xinpcb'') structure using the information in
3108 * the kernel-format in_pcb structure pointed to by inp. This is done to
3109 * reduce the spew of irrelevant information over this interface, to isolate
3110 * user code from changes in the kernel structure, and potentially to provide
3111 * information-hiding if we decide that some of this information should be
3112 * hidden from users.
3113 */
3114 void
in_pcbtoxinpcb(const struct inpcb * inp,struct xinpcb * xi)3115 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
3116 {
3117
3118 bzero(xi, sizeof(*xi));
3119 xi->xi_len = sizeof(struct xinpcb);
3120 if (inp->inp_socket)
3121 sotoxsocket(inp->inp_socket, &xi->xi_socket);
3122 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
3123 xi->inp_gencnt = inp->inp_gencnt;
3124 xi->inp_flow = inp->inp_flow;
3125 xi->inp_flowid = inp->inp_flowid;
3126 xi->inp_flowtype = inp->inp_flowtype;
3127 xi->inp_flags = inp->inp_flags;
3128 xi->inp_flags2 = inp->inp_flags2;
3129 xi->in6p_cksum = inp->in6p_cksum;
3130 xi->in6p_hops = inp->in6p_hops;
3131 xi->inp_ip_tos = inp->inp_ip_tos;
3132 xi->inp_vflag = inp->inp_vflag;
3133 xi->inp_ip_ttl = inp->inp_ip_ttl;
3134 xi->inp_ip_p = inp->inp_ip_p;
3135 xi->inp_ip_minttl = inp->inp_ip_minttl;
3136 }
3137
3138 int
sysctl_setsockopt(SYSCTL_HANDLER_ARGS,struct inpcbinfo * pcbinfo,int (* ctloutput_set)(struct inpcb *,struct sockopt *))3139 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
3140 int (*ctloutput_set)(struct inpcb *, struct sockopt *))
3141 {
3142 struct sockopt sopt;
3143 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
3144 INPLOOKUP_WLOCKPCB);
3145 struct inpcb *inp;
3146 struct sockopt_parameters *params;
3147 struct socket *so;
3148 int error;
3149 char buf[1024];
3150
3151 if (req->oldptr != NULL || req->oldlen != 0)
3152 return (EINVAL);
3153 if (req->newptr == NULL)
3154 return (EPERM);
3155 if (req->newlen > sizeof(buf))
3156 return (ENOMEM);
3157 error = SYSCTL_IN(req, buf, req->newlen);
3158 if (error != 0)
3159 return (error);
3160 if (req->newlen < sizeof(struct sockopt_parameters))
3161 return (EINVAL);
3162 params = (struct sockopt_parameters *)buf;
3163 sopt.sopt_level = params->sop_level;
3164 sopt.sopt_name = params->sop_optname;
3165 sopt.sopt_dir = SOPT_SET;
3166 sopt.sopt_val = params->sop_optval;
3167 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
3168 sopt.sopt_td = NULL;
3169 #ifdef INET6
3170 if (params->sop_inc.inc_flags & INC_ISIPV6) {
3171 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr))
3172 params->sop_inc.inc6_laddr.s6_addr16[1] =
3173 htons(params->sop_inc.inc6_zoneid & 0xffff);
3174 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr))
3175 params->sop_inc.inc6_faddr.s6_addr16[1] =
3176 htons(params->sop_inc.inc6_zoneid & 0xffff);
3177 }
3178 #endif
3179 if (params->sop_inc.inc_lport != htons(0) &&
3180 params->sop_inc.inc_fport != htons(0)) {
3181 #ifdef INET6
3182 if (params->sop_inc.inc_flags & INC_ISIPV6)
3183 inpi.hash = INP6_PCBHASH(
3184 ¶ms->sop_inc.inc6_faddr,
3185 params->sop_inc.inc_lport,
3186 params->sop_inc.inc_fport,
3187 pcbinfo->ipi_hashmask);
3188 else
3189 #endif
3190 inpi.hash = INP_PCBHASH(
3191 ¶ms->sop_inc.inc_faddr,
3192 params->sop_inc.inc_lport,
3193 params->sop_inc.inc_fport,
3194 pcbinfo->ipi_hashmask);
3195 }
3196 while ((inp = inp_next(&inpi)) != NULL)
3197 if (inp->inp_gencnt == params->sop_id) {
3198 /*
3199 * XXXGL
3200 * 1) the inp_next() that ignores INP_UNCONNECTED needs
3201 * to be generally supported.
3202 * 2) Why do we ECONNRESET instead of continueing?
3203 */
3204 if (inp->inp_flags & INP_UNCONNECTED) {
3205 INP_WUNLOCK(inp);
3206 return (ECONNRESET);
3207 }
3208 so = inp->inp_socket;
3209 KASSERT(so != NULL, ("inp_socket == NULL"));
3210 soref(so);
3211 if (params->sop_level == SOL_SOCKET) {
3212 INP_WUNLOCK(inp);
3213 error = sosetopt(so, &sopt);
3214 } else
3215 error = (*ctloutput_set)(inp, &sopt);
3216 sorele(so);
3217 break;
3218 }
3219 if (inp == NULL)
3220 error = ESRCH;
3221 return (error);
3222 }
3223
3224 #ifdef DDB
3225 static void
db_print_indent(int indent)3226 db_print_indent(int indent)
3227 {
3228 int i;
3229
3230 for (i = 0; i < indent; i++)
3231 db_printf(" ");
3232 }
3233
3234 static void
db_print_inconninfo(struct in_conninfo * inc,const char * name,int indent)3235 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
3236 {
3237 char faddr_str[48], laddr_str[48];
3238
3239 db_print_indent(indent);
3240 db_printf("%s at %p\n", name, inc);
3241
3242 indent += 2;
3243
3244 #ifdef INET6
3245 if (inc->inc_flags & INC_ISIPV6) {
3246 /* IPv6. */
3247 ip6_sprintf(laddr_str, &inc->inc6_laddr);
3248 ip6_sprintf(faddr_str, &inc->inc6_faddr);
3249 } else
3250 #endif
3251 {
3252 /* IPv4. */
3253 inet_ntoa_r(inc->inc_laddr, laddr_str);
3254 inet_ntoa_r(inc->inc_faddr, faddr_str);
3255 }
3256 db_print_indent(indent);
3257 db_printf("inc_laddr %s inc_lport %u\n", laddr_str,
3258 ntohs(inc->inc_lport));
3259 db_print_indent(indent);
3260 db_printf("inc_faddr %s inc_fport %u\n", faddr_str,
3261 ntohs(inc->inc_fport));
3262 }
3263
3264 void
db_print_inpcb(struct inpcb * inp,const char * name,int indent)3265 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3266 {
3267
3268 db_print_indent(indent);
3269 db_printf("%s at %p\n", name, inp);
3270
3271 indent += 2;
3272
3273 db_print_indent(indent);
3274 db_printf("inp_flow: 0x%x inp_label: %p\n", inp->inp_flow,
3275 inp->inp_label);
3276
3277 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3278
3279 db_print_indent(indent);
3280 db_printf("inp_flags: 0x%b\n", inp->inp_flags, INP_FLAGS_BITS);
3281
3282 db_print_indent(indent);
3283 db_printf("inp_flags2: 0x%b\n", inp->inp_flags2, INP_FLAGS2_BITS);
3284
3285 db_print_indent(indent);
3286 db_printf("inp_sp: %p inp_vflag: 0x%b\n", inp->inp_sp,
3287 inp->inp_vflag, INP_VFLAGS_BITS);
3288
3289 db_print_indent(indent);
3290 db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n",
3291 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3292
3293 #ifdef INET6
3294 if (inp->inp_vflag & INP_IPV6) {
3295 db_print_indent(indent);
3296 db_printf("in6p_options: %p in6p_outputopts: %p "
3297 "in6p_moptions: %p\n", inp->in6p_options,
3298 inp->in6p_outputopts, inp->in6p_moptions);
3299 db_print_indent(indent);
3300 db_printf("in6p_icmp6filt: %p in6p_cksum %d "
3301 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3302 inp->in6p_hops);
3303 } else
3304 #endif
3305 {
3306 db_print_indent(indent);
3307 db_printf("inp_ip_tos: %d inp_ip_options: %p "
3308 "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3309 inp->inp_options, inp->inp_moptions);
3310 }
3311
3312 db_print_indent(indent);
3313 db_printf("inp_gencnt: %ju\n", (uintmax_t)inp->inp_gencnt);
3314 }
3315
DB_SHOW_COMMAND(inpcb,db_show_inpcb)3316 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3317 {
3318 struct inpcb *inp;
3319
3320 if (!have_addr) {
3321 db_printf("usage: show inpcb <addr>\n");
3322 return;
3323 }
3324 inp = (struct inpcb *)addr;
3325
3326 db_print_inpcb(inp, "inpcb", 0);
3327 }
3328 #endif /* DDB */
3329
3330 #ifdef RATELIMIT
3331 /*
3332 * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3333 * if any.
3334 */
3335 int
in_pcbmodify_txrtlmt(struct inpcb * inp,uint32_t max_pacing_rate)3336 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3337 {
3338 union if_snd_tag_modify_params params = {
3339 .rate_limit.max_rate = max_pacing_rate,
3340 .rate_limit.flags = M_NOWAIT,
3341 };
3342 struct m_snd_tag *mst;
3343 int error;
3344
3345 mst = inp->inp_snd_tag;
3346 if (mst == NULL)
3347 return (EINVAL);
3348
3349 if (mst->sw->snd_tag_modify == NULL) {
3350 error = EOPNOTSUPP;
3351 } else {
3352 error = mst->sw->snd_tag_modify(mst, ¶ms);
3353 }
3354 return (error);
3355 }
3356
3357 /*
3358 * Query existing TX rate limit based on the existing
3359 * "inp->inp_snd_tag", if any.
3360 */
3361 int
in_pcbquery_txrtlmt(struct inpcb * inp,uint32_t * p_max_pacing_rate)3362 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3363 {
3364 union if_snd_tag_query_params params = { };
3365 struct m_snd_tag *mst;
3366 int error;
3367
3368 mst = inp->inp_snd_tag;
3369 if (mst == NULL)
3370 return (EINVAL);
3371
3372 if (mst->sw->snd_tag_query == NULL) {
3373 error = EOPNOTSUPP;
3374 } else {
3375 error = mst->sw->snd_tag_query(mst, ¶ms);
3376 if (error == 0 && p_max_pacing_rate != NULL)
3377 *p_max_pacing_rate = params.rate_limit.max_rate;
3378 }
3379 return (error);
3380 }
3381
3382 /*
3383 * Query existing TX queue level based on the existing
3384 * "inp->inp_snd_tag", if any.
3385 */
3386 int
in_pcbquery_txrlevel(struct inpcb * inp,uint32_t * p_txqueue_level)3387 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3388 {
3389 union if_snd_tag_query_params params = { };
3390 struct m_snd_tag *mst;
3391 int error;
3392
3393 mst = inp->inp_snd_tag;
3394 if (mst == NULL)
3395 return (EINVAL);
3396
3397 if (mst->sw->snd_tag_query == NULL)
3398 return (EOPNOTSUPP);
3399
3400 error = mst->sw->snd_tag_query(mst, ¶ms);
3401 if (error == 0 && p_txqueue_level != NULL)
3402 *p_txqueue_level = params.rate_limit.queue_level;
3403 return (error);
3404 }
3405
3406 /*
3407 * Allocate a new TX rate limit send tag from the network interface
3408 * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3409 */
3410 int
in_pcbattach_txrtlmt(struct inpcb * inp,struct ifnet * ifp,uint32_t flowtype,uint32_t flowid,uint32_t max_pacing_rate,struct m_snd_tag ** st)3411 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3412 uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3413
3414 {
3415 union if_snd_tag_alloc_params params = {
3416 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3417 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3418 .rate_limit.hdr.flowid = flowid,
3419 .rate_limit.hdr.flowtype = flowtype,
3420 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3421 .rate_limit.max_rate = max_pacing_rate,
3422 .rate_limit.flags = M_NOWAIT,
3423 };
3424 int error;
3425
3426 INP_WLOCK_ASSERT(inp);
3427
3428 /*
3429 * If there is already a send tag, or the INP is being torn
3430 * down, allocating a new send tag is not allowed. Else send
3431 * tags may leak.
3432 */
3433 if (*st != NULL || (inp->inp_flags & INP_UNCONNECTED))
3434 return (EINVAL);
3435
3436 error = m_snd_tag_alloc(ifp, ¶ms, st);
3437 #ifdef INET
3438 if (error == 0) {
3439 counter_u64_add(rate_limit_set_ok, 1);
3440 counter_u64_add(rate_limit_active, 1);
3441 } else if (error != EOPNOTSUPP)
3442 counter_u64_add(rate_limit_alloc_fail, 1);
3443 #endif
3444 return (error);
3445 }
3446
3447 void
in_pcbdetach_tag(struct m_snd_tag * mst)3448 in_pcbdetach_tag(struct m_snd_tag *mst)
3449 {
3450
3451 m_snd_tag_rele(mst);
3452 #ifdef INET
3453 counter_u64_add(rate_limit_active, -1);
3454 #endif
3455 }
3456
3457 /*
3458 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3459 * if any:
3460 */
3461 void
in_pcbdetach_txrtlmt(struct inpcb * inp)3462 in_pcbdetach_txrtlmt(struct inpcb *inp)
3463 {
3464 struct m_snd_tag *mst;
3465
3466 INP_WLOCK_ASSERT(inp);
3467
3468 mst = inp->inp_snd_tag;
3469 inp->inp_snd_tag = NULL;
3470
3471 if (mst == NULL)
3472 return;
3473
3474 m_snd_tag_rele(mst);
3475 #ifdef INET
3476 counter_u64_add(rate_limit_active, -1);
3477 #endif
3478 }
3479
3480 int
in_pcboutput_txrtlmt_locked(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb,uint32_t max_pacing_rate)3481 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3482 {
3483 int error;
3484
3485 /*
3486 * If the existing send tag is for the wrong interface due to
3487 * a route change, first drop the existing tag. Set the
3488 * CHANGED flag so that we will keep trying to allocate a new
3489 * tag if we fail to allocate one this time.
3490 */
3491 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3492 in_pcbdetach_txrtlmt(inp);
3493 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3494 }
3495
3496 /*
3497 * NOTE: When attaching to a network interface a reference is
3498 * made to ensure the network interface doesn't go away until
3499 * all ratelimit connections are gone. The network interface
3500 * pointers compared below represent valid network interfaces,
3501 * except when comparing towards NULL.
3502 */
3503 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3504 error = 0;
3505 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3506 if (inp->inp_snd_tag != NULL)
3507 in_pcbdetach_txrtlmt(inp);
3508 error = 0;
3509 } else if (inp->inp_snd_tag == NULL) {
3510 /*
3511 * In order to utilize packet pacing with RSS, we need
3512 * to wait until there is a valid RSS hash before we
3513 * can proceed:
3514 */
3515 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3516 error = EAGAIN;
3517 } else {
3518 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3519 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3520 }
3521 } else {
3522 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3523 }
3524 if (error == 0 || error == EOPNOTSUPP)
3525 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3526
3527 return (error);
3528 }
3529
3530 /*
3531 * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3532 * is set in the fast path and will attach/detach/modify the TX rate
3533 * limit send tag based on the socket's so_max_pacing_rate value.
3534 */
3535 void
in_pcboutput_txrtlmt(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb)3536 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3537 {
3538 struct socket *socket;
3539 uint32_t max_pacing_rate;
3540 bool did_upgrade;
3541
3542 if (inp == NULL)
3543 return;
3544
3545 socket = inp->inp_socket;
3546 if (socket == NULL)
3547 return;
3548
3549 if (!INP_WLOCKED(inp)) {
3550 /*
3551 * NOTE: If the write locking fails, we need to bail
3552 * out and use the non-ratelimited ring for the
3553 * transmit until there is a new chance to get the
3554 * write lock.
3555 */
3556 if (!INP_TRY_UPGRADE(inp))
3557 return;
3558 did_upgrade = 1;
3559 } else {
3560 did_upgrade = 0;
3561 }
3562
3563 /*
3564 * NOTE: The so_max_pacing_rate value is read unlocked,
3565 * because atomic updates are not required since the variable
3566 * is checked at every mbuf we send. It is assumed that the
3567 * variable read itself will be atomic.
3568 */
3569 max_pacing_rate = socket->so_max_pacing_rate;
3570
3571 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3572
3573 if (did_upgrade)
3574 INP_DOWNGRADE(inp);
3575 }
3576
3577 /*
3578 * Track route changes for TX rate limiting.
3579 */
3580 void
in_pcboutput_eagain(struct inpcb * inp)3581 in_pcboutput_eagain(struct inpcb *inp)
3582 {
3583 bool did_upgrade;
3584
3585 if (inp == NULL)
3586 return;
3587
3588 if (inp->inp_snd_tag == NULL)
3589 return;
3590
3591 if (!INP_WLOCKED(inp)) {
3592 /*
3593 * NOTE: If the write locking fails, we need to bail
3594 * out and use the non-ratelimited ring for the
3595 * transmit until there is a new chance to get the
3596 * write lock.
3597 */
3598 if (!INP_TRY_UPGRADE(inp))
3599 return;
3600 did_upgrade = 1;
3601 } else {
3602 did_upgrade = 0;
3603 }
3604
3605 /* detach rate limiting */
3606 in_pcbdetach_txrtlmt(inp);
3607
3608 /* make sure new mbuf send tag allocation is made */
3609 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3610
3611 if (did_upgrade)
3612 INP_DOWNGRADE(inp);
3613 }
3614
3615 #ifdef INET
3616 static void
rl_init(void * st)3617 rl_init(void *st)
3618 {
3619 rate_limit_new = counter_u64_alloc(M_WAITOK);
3620 rate_limit_chg = counter_u64_alloc(M_WAITOK);
3621 rate_limit_active = counter_u64_alloc(M_WAITOK);
3622 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3623 rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3624 }
3625
3626 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3627 #endif
3628 #endif /* RATELIMIT */
3629