1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1991, 1993, 1995
5 * The Regents of the University of California.
6 * Copyright (c) 2007-2009 Robert N. M. Watson
7 * Copyright (c) 2010-2011 Juniper Networks, Inc.
8 * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
9 * All rights reserved.
10 *
11 * Portions of this software were developed by Robert N. M. Watson under
12 * contract to Juniper Networks, Inc.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
16 * are met:
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39 #include <sys/cdefs.h>
40 #include "opt_ddb.h"
41 #include "opt_ipsec.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_ratelimit.h"
45 #include "opt_route.h"
46 #include "opt_rss.h"
47
48 #include <sys/param.h>
49 #include <sys/hash.h>
50 #include <sys/systm.h>
51 #include <sys/libkern.h>
52 #include <sys/lock.h>
53 #include <sys/malloc.h>
54 #include <sys/mbuf.h>
55 #include <sys/eventhandler.h>
56 #include <sys/domain.h>
57 #include <sys/proc.h>
58 #include <sys/protosw.h>
59 #include <sys/smp.h>
60 #include <sys/smr.h>
61 #include <sys/socket.h>
62 #include <sys/socketvar.h>
63 #include <sys/sockio.h>
64 #include <sys/priv.h>
65 #include <sys/proc.h>
66 #include <sys/refcount.h>
67 #include <sys/jail.h>
68 #include <sys/kernel.h>
69 #include <sys/sysctl.h>
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #endif
74
75 #include <vm/uma.h>
76 #include <vm/vm.h>
77
78 #include <net/if.h>
79 #include <net/if_var.h>
80 #include <net/if_private.h>
81 #include <net/if_types.h>
82 #include <net/if_llatbl.h>
83 #include <net/route.h>
84 #include <net/rss_config.h>
85 #include <net/vnet.h>
86
87 #if defined(INET) || defined(INET6)
88 #include <netinet/in.h>
89 #include <netinet/in_pcb.h>
90 #include <netinet/in_pcb_var.h>
91 #include <netinet/tcp.h>
92 #ifdef INET
93 #include <netinet/in_var.h>
94 #include <netinet/in_fib.h>
95 #endif
96 #include <netinet/ip_var.h>
97 #ifdef INET6
98 #include <netinet/ip6.h>
99 #include <netinet6/in6_pcb.h>
100 #include <netinet6/in6_var.h>
101 #include <netinet6/ip6_var.h>
102 #endif /* INET6 */
103 #include <net/route/nhop.h>
104 #endif
105
106 #include <netipsec/ipsec_support.h>
107
108 #include <security/mac/mac_framework.h>
109
110 #define INPCBLBGROUP_SIZMIN 8
111 #define INPCBLBGROUP_SIZMAX 256
112
113 #define INP_FREED 0x00000200 /* Went through in_pcbfree(). */
114 #define INP_INLBGROUP 0x01000000 /* Inserted into inpcblbgroup. */
115
116 /*
117 * These configure the range of local port addresses assigned to
118 * "unspecified" outgoing connections/packets/whatever.
119 */
120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1; /* 1023 */
121 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART; /* 600 */
122 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST; /* 10000 */
123 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST; /* 65535 */
124 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO; /* 49152 */
125 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO; /* 65535 */
126
127 /*
128 * Reserved ports accessible only to root. There are significant
129 * security considerations that must be accounted for when changing these,
130 * but the security benefits can be great. Please be careful.
131 */
132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
133 VNET_DEFINE(int, ipport_reservedlow);
134
135 /* Enable random ephemeral port allocation by default. */
136 VNET_DEFINE(int, ipport_randomized) = 1;
137
138 #ifdef INET
139 static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
140 struct in_addr faddr, u_int fport_arg,
141 struct in_addr laddr, u_int lport_arg,
142 int lookupflags, uint8_t numa_domain);
143
144 #define RANGECHK(var, min, max) \
145 if ((var) < (min)) { (var) = (min); } \
146 else if ((var) > (max)) { (var) = (max); }
147
148 static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)149 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
150 {
151 int error;
152
153 error = sysctl_handle_int(oidp, arg1, arg2, req);
154 if (error == 0) {
155 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
156 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
157 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
158 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
159 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
160 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
161 }
162 return (error);
163 }
164
165 #undef RANGECHK
166
167 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
168 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
169 "IP Ports");
170
171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
172 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
173 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
174 "");
175 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
176 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
177 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
178 "");
179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
180 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
181 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
182 "");
183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
184 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
185 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
186 "");
187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
188 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
189 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
190 "");
191 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
192 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
193 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
194 "");
195 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
196 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
197 &VNET_NAME(ipport_reservedhigh), 0, "");
198 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
199 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
200 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
201 CTLFLAG_VNET | CTLFLAG_RW,
202 &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
203
204 #ifdef RATELIMIT
205 counter_u64_t rate_limit_new;
206 counter_u64_t rate_limit_chg;
207 counter_u64_t rate_limit_active;
208 counter_u64_t rate_limit_alloc_fail;
209 counter_u64_t rate_limit_set_ok;
210
211 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
212 "IP Rate Limiting");
213 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
214 &rate_limit_active, "Active rate limited connections");
215 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
216 &rate_limit_alloc_fail, "Rate limited connection failures");
217 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
218 &rate_limit_set_ok, "Rate limited setting succeeded");
219 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
220 &rate_limit_new, "Total Rate limit new attempts");
221 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
222 &rate_limit_chg, "Total Rate limited change attempts");
223 #endif /* RATELIMIT */
224
225 #endif /* INET */
226
227 VNET_DEFINE(uint32_t, in_pcbhashseed);
228 static void
in_pcbhashseed_init(void)229 in_pcbhashseed_init(void)
230 {
231
232 V_in_pcbhashseed = arc4random();
233 }
234 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
235 in_pcbhashseed_init, NULL);
236
237 #ifdef INET
238 VNET_DEFINE_STATIC(int, connect_inaddr_wild) = 1;
239 #define V_connect_inaddr_wild VNET(connect_inaddr_wild)
240 SYSCTL_INT(_net_inet_ip, OID_AUTO, connect_inaddr_wild,
241 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_inaddr_wild), 0,
242 "Allow connecting to INADDR_ANY or INADDR_BROADCAST for connect(2)");
243 #endif
244
245 static void in_pcbremhash(struct inpcb *);
246
247 /*
248 * in_pcb.c: manage the Protocol Control Blocks.
249 *
250 * NOTE: It is assumed that most of these functions will be called with
251 * the pcbinfo lock held, and often, the inpcb lock held, as these utility
252 * functions often modify hash chains or addresses in pcbs.
253 */
254
255 static struct inpcblbgroup *
in_pcblbgroup_alloc(struct ucred * cred,u_char vflag,uint16_t port,const union in_dependaddr * addr,int size,uint8_t numa_domain)256 in_pcblbgroup_alloc(struct ucred *cred, u_char vflag, uint16_t port,
257 const union in_dependaddr *addr, int size, uint8_t numa_domain)
258 {
259 struct inpcblbgroup *grp;
260 size_t bytes;
261
262 bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
263 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
264 if (grp == NULL)
265 return (NULL);
266 grp->il_cred = crhold(cred);
267 grp->il_vflag = vflag;
268 grp->il_lport = port;
269 grp->il_numa_domain = numa_domain;
270 grp->il_dependladdr = *addr;
271 grp->il_inpsiz = size;
272 return (grp);
273 }
274
275 static void
in_pcblbgroup_free_deferred(epoch_context_t ctx)276 in_pcblbgroup_free_deferred(epoch_context_t ctx)
277 {
278 struct inpcblbgroup *grp;
279
280 grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
281 crfree(grp->il_cred);
282 free(grp, M_PCB);
283 }
284
285 static void
in_pcblbgroup_free(struct inpcblbgroup * grp)286 in_pcblbgroup_free(struct inpcblbgroup *grp)
287 {
288
289 CK_LIST_REMOVE(grp, il_list);
290 NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
291 }
292
293 static void
in_pcblbgroup_insert(struct inpcblbgroup * grp,struct inpcb * inp)294 in_pcblbgroup_insert(struct inpcblbgroup *grp, struct inpcb *inp)
295 {
296 KASSERT(grp->il_inpcnt < grp->il_inpsiz,
297 ("invalid local group size %d and count %d", grp->il_inpsiz,
298 grp->il_inpcnt));
299 INP_WLOCK_ASSERT(inp);
300
301 inp->inp_flags |= INP_INLBGROUP;
302 grp->il_inp[grp->il_inpcnt] = inp;
303
304 /*
305 * Synchronize with in_pcblookup_lbgroup(): make sure that we don't
306 * expose a null slot to the lookup path.
307 */
308 atomic_store_rel_int(&grp->il_inpcnt, grp->il_inpcnt + 1);
309 }
310
311 static struct inpcblbgroup *
in_pcblbgroup_resize(struct inpcblbgrouphead * hdr,struct inpcblbgroup * old_grp,int size)312 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
313 struct inpcblbgroup *old_grp, int size)
314 {
315 struct inpcblbgroup *grp;
316 int i;
317
318 grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag,
319 old_grp->il_lport, &old_grp->il_dependladdr, size,
320 old_grp->il_numa_domain);
321 if (grp == NULL)
322 return (NULL);
323
324 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
325 ("invalid new local group size %d and old local group count %d",
326 grp->il_inpsiz, old_grp->il_inpcnt));
327
328 for (i = 0; i < old_grp->il_inpcnt; ++i)
329 grp->il_inp[i] = old_grp->il_inp[i];
330 grp->il_inpcnt = old_grp->il_inpcnt;
331 CK_LIST_INSERT_HEAD(hdr, grp, il_list);
332 in_pcblbgroup_free(old_grp);
333 return (grp);
334 }
335
336 /*
337 * Add PCB to load balance group for SO_REUSEPORT_LB option.
338 */
339 static int
in_pcbinslbgrouphash(struct inpcb * inp,uint8_t numa_domain)340 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
341 {
342 const static struct timeval interval = { 60, 0 };
343 static struct timeval lastprint;
344 struct inpcbinfo *pcbinfo;
345 struct inpcblbgrouphead *hdr;
346 struct inpcblbgroup *grp;
347 uint32_t idx;
348
349 pcbinfo = inp->inp_pcbinfo;
350
351 INP_WLOCK_ASSERT(inp);
352 INP_HASH_WLOCK_ASSERT(pcbinfo);
353
354 #ifdef INET6
355 /*
356 * Don't allow IPv4 mapped INET6 wild socket.
357 */
358 if ((inp->inp_vflag & INP_IPV4) &&
359 inp->inp_laddr.s_addr == INADDR_ANY &&
360 INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
361 return (0);
362 }
363 #endif
364
365 idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
366 hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
367 CK_LIST_FOREACH(grp, hdr, il_list) {
368 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
369 grp->il_vflag == inp->inp_vflag &&
370 grp->il_lport == inp->inp_lport &&
371 grp->il_numa_domain == numa_domain &&
372 memcmp(&grp->il_dependladdr,
373 &inp->inp_inc.inc_ie.ie_dependladdr,
374 sizeof(grp->il_dependladdr)) == 0) {
375 break;
376 }
377 }
378 if (grp == NULL) {
379 /* Create new load balance group. */
380 grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,
381 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
382 INPCBLBGROUP_SIZMIN, numa_domain);
383 if (grp == NULL)
384 return (ENOBUFS);
385 in_pcblbgroup_insert(grp, inp);
386 CK_LIST_INSERT_HEAD(hdr, grp, il_list);
387 } else if (grp->il_inpcnt == grp->il_inpsiz) {
388 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
389 if (ratecheck(&lastprint, &interval))
390 printf("lb group port %d, limit reached\n",
391 ntohs(grp->il_lport));
392 return (0);
393 }
394
395 /* Expand this local group. */
396 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
397 if (grp == NULL)
398 return (ENOBUFS);
399 in_pcblbgroup_insert(grp, inp);
400 } else {
401 in_pcblbgroup_insert(grp, inp);
402 }
403 return (0);
404 }
405
406 /*
407 * Remove PCB from load balance group.
408 */
409 static void
in_pcbremlbgrouphash(struct inpcb * inp)410 in_pcbremlbgrouphash(struct inpcb *inp)
411 {
412 struct inpcbinfo *pcbinfo;
413 struct inpcblbgrouphead *hdr;
414 struct inpcblbgroup *grp;
415 int i;
416
417 pcbinfo = inp->inp_pcbinfo;
418
419 INP_WLOCK_ASSERT(inp);
420 MPASS(inp->inp_flags & INP_INLBGROUP);
421 INP_HASH_WLOCK_ASSERT(pcbinfo);
422
423 hdr = &pcbinfo->ipi_lbgrouphashbase[
424 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
425 CK_LIST_FOREACH(grp, hdr, il_list) {
426 for (i = 0; i < grp->il_inpcnt; ++i) {
427 if (grp->il_inp[i] != inp)
428 continue;
429
430 if (grp->il_inpcnt == 1) {
431 /* We are the last, free this local group. */
432 in_pcblbgroup_free(grp);
433 } else {
434 KASSERT(grp->il_inpcnt >= 2,
435 ("invalid local group count %d",
436 grp->il_inpcnt));
437 grp->il_inp[i] =
438 grp->il_inp[grp->il_inpcnt - 1];
439
440 /*
441 * Synchronize with in_pcblookup_lbgroup().
442 */
443 atomic_store_rel_int(&grp->il_inpcnt,
444 grp->il_inpcnt - 1);
445 }
446 inp->inp_flags &= ~INP_INLBGROUP;
447 return;
448 }
449 }
450 KASSERT(0, ("%s: did not find %p", __func__, inp));
451 }
452
453 int
in_pcblbgroup_numa(struct inpcb * inp,int arg)454 in_pcblbgroup_numa(struct inpcb *inp, int arg)
455 {
456 struct inpcbinfo *pcbinfo;
457 struct inpcblbgrouphead *hdr;
458 struct inpcblbgroup *grp;
459 int err, i;
460 uint8_t numa_domain;
461
462 switch (arg) {
463 case TCP_REUSPORT_LB_NUMA_NODOM:
464 numa_domain = M_NODOM;
465 break;
466 case TCP_REUSPORT_LB_NUMA_CURDOM:
467 numa_domain = PCPU_GET(domain);
468 break;
469 default:
470 if (arg < 0 || arg >= vm_ndomains)
471 return (EINVAL);
472 numa_domain = arg;
473 }
474
475 err = 0;
476 pcbinfo = inp->inp_pcbinfo;
477 INP_WLOCK_ASSERT(inp);
478 INP_HASH_WLOCK(pcbinfo);
479 hdr = &pcbinfo->ipi_lbgrouphashbase[
480 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
481 CK_LIST_FOREACH(grp, hdr, il_list) {
482 for (i = 0; i < grp->il_inpcnt; ++i) {
483 if (grp->il_inp[i] != inp)
484 continue;
485
486 if (grp->il_numa_domain == numa_domain) {
487 goto abort_with_hash_wlock;
488 }
489
490 /* Remove it from the old group. */
491 in_pcbremlbgrouphash(inp);
492
493 /* Add it to the new group based on numa domain. */
494 in_pcbinslbgrouphash(inp, numa_domain);
495 goto abort_with_hash_wlock;
496 }
497 }
498 err = ENOENT;
499 abort_with_hash_wlock:
500 INP_HASH_WUNLOCK(pcbinfo);
501 return (err);
502 }
503
504 /* Make sure it is safe to use hashinit(9) on CK_LIST. */
505 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb)));
506
507 /*
508 * Initialize an inpcbinfo - a per-VNET instance of connections db.
509 */
510 void
in_pcbinfo_init(struct inpcbinfo * pcbinfo,struct inpcbstorage * pcbstor,u_int hash_nelements,u_int porthash_nelements)511 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
512 u_int hash_nelements, u_int porthash_nelements)
513 {
514
515 mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);
516 mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
517 NULL, MTX_DEF);
518 #ifdef VIMAGE
519 pcbinfo->ipi_vnet = curvnet;
520 #endif
521 CK_LIST_INIT(&pcbinfo->ipi_listhead);
522 pcbinfo->ipi_count = 0;
523 pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,
524 &pcbinfo->ipi_hashmask);
525 pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,
526 &pcbinfo->ipi_hashmask);
527 porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
528 pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
529 &pcbinfo->ipi_porthashmask);
530 pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
531 &pcbinfo->ipi_lbgrouphashmask);
532 pcbinfo->ipi_zone = pcbstor->ips_zone;
533 pcbinfo->ipi_portzone = pcbstor->ips_portzone;
534 pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
535 }
536
537 /*
538 * Destroy an inpcbinfo.
539 */
540 void
in_pcbinfo_destroy(struct inpcbinfo * pcbinfo)541 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
542 {
543
544 KASSERT(pcbinfo->ipi_count == 0,
545 ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
546
547 hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);
548 hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);
549 hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
550 pcbinfo->ipi_porthashmask);
551 hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
552 pcbinfo->ipi_lbgrouphashmask);
553 mtx_destroy(&pcbinfo->ipi_hash_lock);
554 mtx_destroy(&pcbinfo->ipi_lock);
555 }
556
557 /*
558 * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
559 */
560 static void inpcb_fini(void *, int);
561 void
in_pcbstorage_init(void * arg)562 in_pcbstorage_init(void *arg)
563 {
564 struct inpcbstorage *pcbstor = arg;
565
566 pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
567 pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit,
568 inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR);
569 pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name,
570 sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
571 uma_zone_set_smr(pcbstor->ips_portzone,
572 uma_zone_get_smr(pcbstor->ips_zone));
573 }
574
575 /*
576 * Destroy a pcbstorage - used by unloadable protocols.
577 */
578 void
in_pcbstorage_destroy(void * arg)579 in_pcbstorage_destroy(void *arg)
580 {
581 struct inpcbstorage *pcbstor = arg;
582
583 uma_zdestroy(pcbstor->ips_zone);
584 uma_zdestroy(pcbstor->ips_portzone);
585 }
586
587 /*
588 * Allocate a PCB and associate it with the socket.
589 * On success return with the PCB locked.
590 */
591 int
in_pcballoc(struct socket * so,struct inpcbinfo * pcbinfo)592 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
593 {
594 struct inpcb *inp;
595 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
596 int error;
597 #endif
598
599 inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
600 if (inp == NULL)
601 return (ENOBUFS);
602 bzero(&inp->inp_start_zero, inp_zero_size);
603 #ifdef NUMA
604 inp->inp_numa_domain = M_NODOM;
605 #endif
606 inp->inp_pcbinfo = pcbinfo;
607 inp->inp_socket = so;
608 inp->inp_cred = crhold(so->so_cred);
609 inp->inp_inc.inc_fibnum = so->so_fibnum;
610 #ifdef MAC
611 error = mac_inpcb_init(inp, M_NOWAIT);
612 if (error != 0)
613 goto out;
614 mac_inpcb_create(so, inp);
615 #endif
616 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
617 error = ipsec_init_pcbpolicy(inp);
618 if (error != 0) {
619 #ifdef MAC
620 mac_inpcb_destroy(inp);
621 #endif
622 goto out;
623 }
624 #endif /*IPSEC*/
625 #ifdef INET6
626 if (INP_SOCKAF(so) == AF_INET6) {
627 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
628 if (V_ip6_v6only)
629 inp->inp_flags |= IN6P_IPV6_V6ONLY;
630 #ifdef INET
631 else
632 inp->inp_vflag |= INP_IPV4;
633 #endif
634 if (V_ip6_auto_flowlabel)
635 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
636 inp->in6p_hops = -1; /* use kernel default */
637 }
638 #endif
639 #if defined(INET) && defined(INET6)
640 else
641 #endif
642 #ifdef INET
643 inp->inp_vflag |= INP_IPV4;
644 #endif
645 inp->inp_smr = SMR_SEQ_INVALID;
646
647 /*
648 * Routes in inpcb's can cache L2 as well; they are guaranteed
649 * to be cleaned up.
650 */
651 inp->inp_route.ro_flags = RT_LLE_CACHE;
652 refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
653 INP_WLOCK(inp);
654 INP_INFO_WLOCK(pcbinfo);
655 pcbinfo->ipi_count++;
656 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
657 CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
658 INP_INFO_WUNLOCK(pcbinfo);
659 so->so_pcb = inp;
660
661 return (0);
662
663 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
664 out:
665 crfree(inp->inp_cred);
666 #ifdef INVARIANTS
667 inp->inp_cred = NULL;
668 #endif
669 uma_zfree_smr(pcbinfo->ipi_zone, inp);
670 return (error);
671 #endif
672 }
673
674 #ifdef INET
675 int
in_pcbbind(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred)676 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
677 {
678 int anonport, error;
679
680 KASSERT(sin == NULL || sin->sin_family == AF_INET,
681 ("%s: invalid address family for %p", __func__, sin));
682 KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in),
683 ("%s: invalid address length for %p", __func__, sin));
684 INP_WLOCK_ASSERT(inp);
685 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
686
687 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
688 return (EINVAL);
689 anonport = sin == NULL || sin->sin_port == 0;
690 error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr,
691 &inp->inp_lport, cred);
692 if (error)
693 return (error);
694 if (in_pcbinshash(inp) != 0) {
695 inp->inp_laddr.s_addr = INADDR_ANY;
696 inp->inp_lport = 0;
697 return (EAGAIN);
698 }
699 if (anonport)
700 inp->inp_flags |= INP_ANONPORT;
701 return (0);
702 }
703 #endif
704
705 #if defined(INET) || defined(INET6)
706 /*
707 * Assign a local port like in_pcb_lport(), but also used with connect()
708 * and a foreign address and port. If fsa is non-NULL, choose a local port
709 * that is unused with those, otherwise one that is completely unused.
710 * lsa can be NULL for IPv6.
711 */
712 int
in_pcb_lport_dest(struct inpcb * inp,struct sockaddr * lsa,u_short * lportp,struct sockaddr * fsa,u_short fport,struct ucred * cred,int lookupflags)713 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
714 struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
715 {
716 struct inpcbinfo *pcbinfo;
717 struct inpcb *tmpinp;
718 unsigned short *lastport;
719 int count, error;
720 u_short aux, first, last, lport;
721 #ifdef INET
722 struct in_addr laddr, faddr;
723 #endif
724 #ifdef INET6
725 struct in6_addr *laddr6, *faddr6;
726 #endif
727
728 pcbinfo = inp->inp_pcbinfo;
729
730 /*
731 * Because no actual state changes occur here, a global write lock on
732 * the pcbinfo isn't required.
733 */
734 INP_LOCK_ASSERT(inp);
735 INP_HASH_LOCK_ASSERT(pcbinfo);
736
737 if (inp->inp_flags & INP_HIGHPORT) {
738 first = V_ipport_hifirstauto; /* sysctl */
739 last = V_ipport_hilastauto;
740 lastport = &pcbinfo->ipi_lasthi;
741 } else if (inp->inp_flags & INP_LOWPORT) {
742 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
743 if (error)
744 return (error);
745 first = V_ipport_lowfirstauto; /* 1023 */
746 last = V_ipport_lowlastauto; /* 600 */
747 lastport = &pcbinfo->ipi_lastlow;
748 } else {
749 first = V_ipport_firstauto; /* sysctl */
750 last = V_ipport_lastauto;
751 lastport = &pcbinfo->ipi_lastport;
752 }
753
754 /*
755 * Instead of having two loops further down counting up or down
756 * make sure that first is always <= last and go with only one
757 * code path implementing all logic.
758 */
759 if (first > last) {
760 aux = first;
761 first = last;
762 last = aux;
763 }
764
765 #ifdef INET
766 laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */
767 if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
768 if (lsa != NULL)
769 laddr = ((struct sockaddr_in *)lsa)->sin_addr;
770 if (fsa != NULL)
771 faddr = ((struct sockaddr_in *)fsa)->sin_addr;
772 }
773 #endif
774 #ifdef INET6
775 laddr6 = NULL;
776 if ((inp->inp_vflag & INP_IPV6) != 0) {
777 if (lsa != NULL)
778 laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
779 if (fsa != NULL)
780 faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
781 }
782 #endif
783
784 tmpinp = NULL;
785 lport = *lportp;
786
787 if (V_ipport_randomized)
788 *lastport = first + (arc4random() % (last - first));
789
790 count = last - first;
791
792 do {
793 if (count-- < 0) /* completely used? */
794 return (EADDRNOTAVAIL);
795 ++*lastport;
796 if (*lastport < first || *lastport > last)
797 *lastport = first;
798 lport = htons(*lastport);
799
800 if (fsa != NULL) {
801 #ifdef INET
802 if (lsa->sa_family == AF_INET) {
803 tmpinp = in_pcblookup_hash_locked(pcbinfo,
804 faddr, fport, laddr, lport, lookupflags,
805 M_NODOM);
806 }
807 #endif
808 #ifdef INET6
809 if (lsa->sa_family == AF_INET6) {
810 tmpinp = in6_pcblookup_hash_locked(pcbinfo,
811 faddr6, fport, laddr6, lport, lookupflags,
812 M_NODOM);
813 }
814 #endif
815 } else {
816 #ifdef INET6
817 if ((inp->inp_vflag & INP_IPV6) != 0) {
818 tmpinp = in6_pcblookup_local(pcbinfo,
819 &inp->in6p_laddr, lport, lookupflags, cred);
820 #ifdef INET
821 if (tmpinp == NULL &&
822 (inp->inp_vflag & INP_IPV4))
823 tmpinp = in_pcblookup_local(pcbinfo,
824 laddr, lport, lookupflags, cred);
825 #endif
826 }
827 #endif
828 #if defined(INET) && defined(INET6)
829 else
830 #endif
831 #ifdef INET
832 tmpinp = in_pcblookup_local(pcbinfo, laddr,
833 lport, lookupflags, cred);
834 #endif
835 }
836 } while (tmpinp != NULL);
837
838 *lportp = lport;
839
840 return (0);
841 }
842
843 /*
844 * Select a local port (number) to use.
845 */
846 int
in_pcb_lport(struct inpcb * inp,struct in_addr * laddrp,u_short * lportp,struct ucred * cred,int lookupflags)847 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
848 struct ucred *cred, int lookupflags)
849 {
850 struct sockaddr_in laddr;
851
852 if (laddrp) {
853 bzero(&laddr, sizeof(laddr));
854 laddr.sin_family = AF_INET;
855 laddr.sin_addr = *laddrp;
856 }
857 return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
858 NULL, lportp, NULL, 0, cred, lookupflags));
859 }
860 #endif /* INET || INET6 */
861
862 #ifdef INET
863 /*
864 * Determine whether the inpcb can be bound to the specified address/port tuple.
865 */
866 static int
in_pcbbind_avail(struct inpcb * inp,const struct in_addr laddr,const u_short lport,int sooptions,int lookupflags,struct ucred * cred)867 in_pcbbind_avail(struct inpcb *inp, const struct in_addr laddr,
868 const u_short lport, int sooptions, int lookupflags, struct ucred *cred)
869 {
870 int reuseport, reuseport_lb;
871
872 INP_LOCK_ASSERT(inp);
873 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
874
875 reuseport = (sooptions & SO_REUSEPORT);
876 reuseport_lb = (sooptions & SO_REUSEPORT_LB);
877
878 if (IN_MULTICAST(ntohl(laddr.s_addr))) {
879 /*
880 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
881 * allow complete duplication of binding if
882 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
883 * and a multicast address is bound on both
884 * new and duplicated sockets.
885 */
886 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0)
887 reuseport = SO_REUSEADDR | SO_REUSEPORT;
888 /*
889 * XXX: How to deal with SO_REUSEPORT_LB here?
890 * Treat same as SO_REUSEPORT for now.
891 */
892 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0)
893 reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB;
894 } else if (!in_nullhost(laddr)) {
895 struct sockaddr_in sin;
896
897 memset(&sin, 0, sizeof(sin));
898 sin.sin_family = AF_INET;
899 sin.sin_len = sizeof(sin);
900 sin.sin_addr = laddr;
901
902 /*
903 * Is the address a local IP address?
904 * If INP_BINDANY is set, then the socket may be bound
905 * to any endpoint address, local or not.
906 */
907 if ((inp->inp_flags & INP_BINDANY) == 0 &&
908 ifa_ifwithaddr_check((const struct sockaddr *)&sin) == 0)
909 return (EADDRNOTAVAIL);
910 }
911
912 if (lport != 0) {
913 struct inpcb *t;
914
915 if (ntohs(lport) <= V_ipport_reservedhigh &&
916 ntohs(lport) >= V_ipport_reservedlow &&
917 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
918 return (EACCES);
919
920 if (!IN_MULTICAST(ntohl(laddr.s_addr)) &&
921 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
922 /*
923 * If a socket owned by a different user is already
924 * bound to this port, fail. In particular, SO_REUSE*
925 * can only be used to share a port among sockets owned
926 * by the same user.
927 *
928 * However, we can share a port with a connected socket
929 * which has a unique 4-tuple.
930 */
931 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
932 INPLOOKUP_WILDCARD, cred);
933 if (t != NULL &&
934 (inp->inp_socket->so_type != SOCK_STREAM ||
935 in_nullhost(t->inp_faddr)) &&
936 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
937 return (EADDRINUSE);
938 }
939 t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
940 lookupflags, cred);
941 if (t != NULL && ((reuseport | reuseport_lb) &
942 t->inp_socket->so_options) == 0) {
943 #ifdef INET6
944 if (!in_nullhost(laddr) ||
945 !in_nullhost(t->inp_laddr) ||
946 (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
947 (t->inp_vflag & INP_IPV6PROTO) == 0)
948 #endif
949 return (EADDRINUSE);
950 }
951 }
952 return (0);
953 }
954
955 /*
956 * Set up a bind operation on a PCB, performing port allocation
957 * as required, but do not actually modify the PCB. Callers can
958 * either complete the bind by setting inp_laddr/inp_lport and
959 * calling in_pcbinshash(), or they can just use the resulting
960 * port and address to authorise the sending of a once-off packet.
961 *
962 * On error, the values of *laddrp and *lportp are not changed.
963 */
964 int
in_pcbbind_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,struct ucred * cred)965 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
966 u_short *lportp, struct ucred *cred)
967 {
968 struct socket *so = inp->inp_socket;
969 struct in_addr laddr;
970 u_short lport = 0;
971 int lookupflags, sooptions;
972 int error;
973
974 /*
975 * No state changes, so read locks are sufficient here.
976 */
977 INP_LOCK_ASSERT(inp);
978 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
979
980 laddr.s_addr = *laddrp;
981 if (sin != NULL && laddr.s_addr != INADDR_ANY)
982 return (EINVAL);
983
984 lookupflags = 0;
985 sooptions = atomic_load_int(&so->so_options);
986 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0)
987 lookupflags = INPLOOKUP_WILDCARD;
988 if (sin == NULL) {
989 if ((error = prison_local_ip4(cred, &laddr)) != 0)
990 return (error);
991 } else {
992 KASSERT(sin->sin_family == AF_INET,
993 ("%s: invalid family for address %p", __func__, sin));
994 KASSERT(sin->sin_len == sizeof(*sin),
995 ("%s: invalid length for address %p", __func__, sin));
996
997 error = prison_local_ip4(cred, &sin->sin_addr);
998 if (error)
999 return (error);
1000 if (sin->sin_port != *lportp) {
1001 /* Don't allow the port to change. */
1002 if (*lportp != 0)
1003 return (EINVAL);
1004 lport = sin->sin_port;
1005 }
1006 laddr = sin->sin_addr;
1007
1008 /* See if this address/port combo is available. */
1009 error = in_pcbbind_avail(inp, laddr, lport, sooptions,
1010 lookupflags, cred);
1011 if (error != 0)
1012 return (error);
1013 }
1014 if (*lportp != 0)
1015 lport = *lportp;
1016 if (lport == 0) {
1017 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1018 if (error != 0)
1019 return (error);
1020 }
1021 *laddrp = laddr.s_addr;
1022 *lportp = lport;
1023 return (0);
1024 }
1025
1026 /*
1027 * Connect from a socket to a specified address.
1028 * Both address and port must be specified in argument sin.
1029 * If don't have a local address for this socket yet,
1030 * then pick one.
1031 */
1032 int
in_pcbconnect(struct inpcb * inp,struct sockaddr_in * sin,struct ucred * cred)1033 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
1034 {
1035 u_short lport, fport;
1036 in_addr_t laddr, faddr;
1037 int anonport, error;
1038
1039 INP_WLOCK_ASSERT(inp);
1040 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1041 KASSERT(in_nullhost(inp->inp_faddr),
1042 ("%s: inp is already connected", __func__));
1043
1044 lport = inp->inp_lport;
1045 laddr = inp->inp_laddr.s_addr;
1046 anonport = (lport == 0);
1047 error = in_pcbconnect_setup(inp, sin, &laddr, &lport, &faddr, &fport,
1048 cred);
1049 if (error)
1050 return (error);
1051
1052 inp->inp_faddr.s_addr = faddr;
1053 inp->inp_fport = fport;
1054
1055 /* Do the initial binding of the local address if required. */
1056 if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1057 inp->inp_lport = lport;
1058 inp->inp_laddr.s_addr = laddr;
1059 if (in_pcbinshash(inp) != 0) {
1060 inp->inp_laddr.s_addr = inp->inp_faddr.s_addr =
1061 INADDR_ANY;
1062 inp->inp_lport = inp->inp_fport = 0;
1063 return (EAGAIN);
1064 }
1065 } else {
1066 inp->inp_lport = lport;
1067 inp->inp_laddr.s_addr = laddr;
1068 if ((inp->inp_flags & INP_INHASHLIST) != 0)
1069 in_pcbrehash(inp);
1070 else
1071 in_pcbinshash(inp);
1072 }
1073
1074 if (anonport)
1075 inp->inp_flags |= INP_ANONPORT;
1076 return (0);
1077 }
1078
1079 /*
1080 * Do proper source address selection on an unbound socket in case
1081 * of connect. Take jails into account as well.
1082 */
1083 int
in_pcbladdr(struct inpcb * inp,struct in_addr * faddr,struct in_addr * laddr,struct ucred * cred)1084 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1085 struct ucred *cred)
1086 {
1087 struct ifaddr *ifa;
1088 struct sockaddr *sa;
1089 struct sockaddr_in *sin, dst;
1090 struct nhop_object *nh;
1091 int error;
1092
1093 NET_EPOCH_ASSERT();
1094 KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1095
1096 /*
1097 * Bypass source address selection and use the primary jail IP
1098 * if requested.
1099 */
1100 if (!prison_saddrsel_ip4(cred, laddr))
1101 return (0);
1102
1103 error = 0;
1104
1105 nh = NULL;
1106 bzero(&dst, sizeof(dst));
1107 sin = &dst;
1108 sin->sin_family = AF_INET;
1109 sin->sin_len = sizeof(struct sockaddr_in);
1110 sin->sin_addr.s_addr = faddr->s_addr;
1111
1112 /*
1113 * If route is known our src addr is taken from the i/f,
1114 * else punt.
1115 *
1116 * Find out route to destination.
1117 */
1118 if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1119 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1120 0, NHR_NONE, 0);
1121
1122 /*
1123 * If we found a route, use the address corresponding to
1124 * the outgoing interface.
1125 *
1126 * Otherwise assume faddr is reachable on a directly connected
1127 * network and try to find a corresponding interface to take
1128 * the source address from.
1129 */
1130 if (nh == NULL || nh->nh_ifp == NULL) {
1131 struct in_ifaddr *ia;
1132 struct ifnet *ifp;
1133
1134 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1135 inp->inp_socket->so_fibnum));
1136 if (ia == NULL) {
1137 ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1138 inp->inp_socket->so_fibnum));
1139 }
1140 if (ia == NULL) {
1141 error = ENETUNREACH;
1142 goto done;
1143 }
1144
1145 if (!prison_flag(cred, PR_IP4)) {
1146 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1147 goto done;
1148 }
1149
1150 ifp = ia->ia_ifp;
1151 ia = NULL;
1152 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1153 sa = ifa->ifa_addr;
1154 if (sa->sa_family != AF_INET)
1155 continue;
1156 sin = (struct sockaddr_in *)sa;
1157 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1158 ia = (struct in_ifaddr *)ifa;
1159 break;
1160 }
1161 }
1162 if (ia != NULL) {
1163 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1164 goto done;
1165 }
1166
1167 /* 3. As a last resort return the 'default' jail address. */
1168 error = prison_get_ip4(cred, laddr);
1169 goto done;
1170 }
1171
1172 /*
1173 * If the outgoing interface on the route found is not
1174 * a loopback interface, use the address from that interface.
1175 * In case of jails do those three steps:
1176 * 1. check if the interface address belongs to the jail. If so use it.
1177 * 2. check if we have any address on the outgoing interface
1178 * belonging to this jail. If so use it.
1179 * 3. as a last resort return the 'default' jail address.
1180 */
1181 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1182 struct in_ifaddr *ia;
1183 struct ifnet *ifp;
1184
1185 /* If not jailed, use the default returned. */
1186 if (!prison_flag(cred, PR_IP4)) {
1187 ia = (struct in_ifaddr *)nh->nh_ifa;
1188 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1189 goto done;
1190 }
1191
1192 /* Jailed. */
1193 /* 1. Check if the iface address belongs to the jail. */
1194 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1195 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1196 ia = (struct in_ifaddr *)nh->nh_ifa;
1197 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1198 goto done;
1199 }
1200
1201 /*
1202 * 2. Check if we have any address on the outgoing interface
1203 * belonging to this jail.
1204 */
1205 ia = NULL;
1206 ifp = nh->nh_ifp;
1207 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1208 sa = ifa->ifa_addr;
1209 if (sa->sa_family != AF_INET)
1210 continue;
1211 sin = (struct sockaddr_in *)sa;
1212 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1213 ia = (struct in_ifaddr *)ifa;
1214 break;
1215 }
1216 }
1217 if (ia != NULL) {
1218 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1219 goto done;
1220 }
1221
1222 /* 3. As a last resort return the 'default' jail address. */
1223 error = prison_get_ip4(cred, laddr);
1224 goto done;
1225 }
1226
1227 /*
1228 * The outgoing interface is marked with 'loopback net', so a route
1229 * to ourselves is here.
1230 * Try to find the interface of the destination address and then
1231 * take the address from there. That interface is not necessarily
1232 * a loopback interface.
1233 * In case of jails, check that it is an address of the jail
1234 * and if we cannot find, fall back to the 'default' jail address.
1235 */
1236 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1237 struct in_ifaddr *ia;
1238
1239 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1240 inp->inp_socket->so_fibnum));
1241 if (ia == NULL)
1242 ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1243 inp->inp_socket->so_fibnum));
1244 if (ia == NULL)
1245 ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1246
1247 if (!prison_flag(cred, PR_IP4)) {
1248 if (ia == NULL) {
1249 error = ENETUNREACH;
1250 goto done;
1251 }
1252 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1253 goto done;
1254 }
1255
1256 /* Jailed. */
1257 if (ia != NULL) {
1258 struct ifnet *ifp;
1259
1260 ifp = ia->ia_ifp;
1261 ia = NULL;
1262 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1263 sa = ifa->ifa_addr;
1264 if (sa->sa_family != AF_INET)
1265 continue;
1266 sin = (struct sockaddr_in *)sa;
1267 if (prison_check_ip4(cred,
1268 &sin->sin_addr) == 0) {
1269 ia = (struct in_ifaddr *)ifa;
1270 break;
1271 }
1272 }
1273 if (ia != NULL) {
1274 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1275 goto done;
1276 }
1277 }
1278
1279 /* 3. As a last resort return the 'default' jail address. */
1280 error = prison_get_ip4(cred, laddr);
1281 goto done;
1282 }
1283
1284 done:
1285 if (error == 0 && laddr->s_addr == INADDR_ANY)
1286 return (EHOSTUNREACH);
1287 return (error);
1288 }
1289
1290 /*
1291 * Set up for a connect from a socket to the specified address.
1292 * On entry, *laddrp and *lportp should contain the current local
1293 * address and port for the PCB; these are updated to the values
1294 * that should be placed in inp_laddr and inp_lport to complete
1295 * the connect.
1296 *
1297 * On success, *faddrp and *fportp will be set to the remote address
1298 * and port. These are not updated in the error case.
1299 */
1300 int
in_pcbconnect_setup(struct inpcb * inp,struct sockaddr_in * sin,in_addr_t * laddrp,u_short * lportp,in_addr_t * faddrp,u_short * fportp,struct ucred * cred)1301 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr_in *sin,
1302 in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1303 struct ucred *cred)
1304 {
1305 struct in_ifaddr *ia;
1306 struct in_addr laddr, faddr;
1307 u_short lport, fport;
1308 int error;
1309
1310 KASSERT(sin->sin_family == AF_INET,
1311 ("%s: invalid address family for %p", __func__, sin));
1312 KASSERT(sin->sin_len == sizeof(*sin),
1313 ("%s: invalid address length for %p", __func__, sin));
1314
1315 /*
1316 * Because a global state change doesn't actually occur here, a read
1317 * lock is sufficient.
1318 */
1319 NET_EPOCH_ASSERT();
1320 INP_LOCK_ASSERT(inp);
1321 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1322
1323 if (sin->sin_port == 0)
1324 return (EADDRNOTAVAIL);
1325 laddr.s_addr = *laddrp;
1326 lport = *lportp;
1327 faddr = sin->sin_addr;
1328 fport = sin->sin_port;
1329 #ifdef ROUTE_MPATH
1330 if (CALC_FLOWID_OUTBOUND) {
1331 uint32_t hash_val, hash_type;
1332
1333 hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
1334 inp->inp_socket->so_proto->pr_protocol, &hash_type);
1335
1336 inp->inp_flowid = hash_val;
1337 inp->inp_flowtype = hash_type;
1338 }
1339 #endif
1340 if (V_connect_inaddr_wild && !CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1341 /*
1342 * If the destination address is INADDR_ANY,
1343 * use the primary local address.
1344 * If the supplied address is INADDR_BROADCAST,
1345 * and the primary interface supports broadcast,
1346 * choose the broadcast address for that interface.
1347 */
1348 if (faddr.s_addr == INADDR_ANY) {
1349 faddr =
1350 IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1351 if ((error = prison_get_ip4(cred, &faddr)) != 0)
1352 return (error);
1353 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1354 if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1355 IFF_BROADCAST)
1356 faddr = satosin(&CK_STAILQ_FIRST(
1357 &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1358 }
1359 } else if (faddr.s_addr == INADDR_ANY) {
1360 return (ENETUNREACH);
1361 }
1362 if (laddr.s_addr == INADDR_ANY) {
1363 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1364 /*
1365 * If the destination address is multicast and an outgoing
1366 * interface has been set as a multicast option, prefer the
1367 * address of that interface as our source address.
1368 */
1369 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1370 inp->inp_moptions != NULL) {
1371 struct ip_moptions *imo;
1372 struct ifnet *ifp;
1373
1374 imo = inp->inp_moptions;
1375 if (imo->imo_multicast_ifp != NULL) {
1376 ifp = imo->imo_multicast_ifp;
1377 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1378 if (ia->ia_ifp == ifp &&
1379 prison_check_ip4(cred,
1380 &ia->ia_addr.sin_addr) == 0)
1381 break;
1382 }
1383 if (ia == NULL)
1384 error = EADDRNOTAVAIL;
1385 else {
1386 laddr = ia->ia_addr.sin_addr;
1387 error = 0;
1388 }
1389 }
1390 }
1391 if (error)
1392 return (error);
1393 }
1394
1395 if (lport != 0) {
1396 if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1397 fport, laddr, lport, 0, M_NODOM) != NULL)
1398 return (EADDRINUSE);
1399 } else {
1400 struct sockaddr_in lsin, fsin;
1401
1402 bzero(&lsin, sizeof(lsin));
1403 bzero(&fsin, sizeof(fsin));
1404 lsin.sin_family = AF_INET;
1405 lsin.sin_addr = laddr;
1406 fsin.sin_family = AF_INET;
1407 fsin.sin_addr = faddr;
1408 error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1409 &lport, (struct sockaddr *)& fsin, fport, cred,
1410 INPLOOKUP_WILDCARD);
1411 if (error)
1412 return (error);
1413 }
1414 *laddrp = laddr.s_addr;
1415 *lportp = lport;
1416 *faddrp = faddr.s_addr;
1417 *fportp = fport;
1418 return (0);
1419 }
1420
1421 void
in_pcbdisconnect(struct inpcb * inp)1422 in_pcbdisconnect(struct inpcb *inp)
1423 {
1424
1425 INP_WLOCK_ASSERT(inp);
1426 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1427 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1428 ("%s: inp %p was already disconnected", __func__, inp));
1429
1430 in_pcbremhash_locked(inp);
1431
1432 /* See the comment in in_pcbinshash(). */
1433 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1434 inp->inp_laddr.s_addr = INADDR_ANY;
1435 inp->inp_faddr.s_addr = INADDR_ANY;
1436 inp->inp_fport = 0;
1437 }
1438 #endif /* INET */
1439
1440 /*
1441 * inpcb hash lookups are protected by SMR section.
1442 *
1443 * Once desired pcb has been found, switching from SMR section to a pcb
1444 * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1445 * here because SMR is a critical section.
1446 * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1447 */
1448 void
inp_lock(struct inpcb * inp,const inp_lookup_t lock)1449 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1450 {
1451
1452 lock == INPLOOKUP_RLOCKPCB ?
1453 rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1454 }
1455
1456 void
inp_unlock(struct inpcb * inp,const inp_lookup_t lock)1457 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1458 {
1459
1460 lock == INPLOOKUP_RLOCKPCB ?
1461 rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1462 }
1463
1464 int
inp_trylock(struct inpcb * inp,const inp_lookup_t lock)1465 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1466 {
1467
1468 return (lock == INPLOOKUP_RLOCKPCB ?
1469 rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1470 }
1471
1472 static inline bool
_inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock,const int ignflags)1473 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1474 {
1475
1476 MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1477 SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1478
1479 if (__predict_true(inp_trylock(inp, lock))) {
1480 if (__predict_false(inp->inp_flags & ignflags)) {
1481 smr_exit(inp->inp_pcbinfo->ipi_smr);
1482 inp_unlock(inp, lock);
1483 return (false);
1484 }
1485 smr_exit(inp->inp_pcbinfo->ipi_smr);
1486 return (true);
1487 }
1488
1489 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1490 smr_exit(inp->inp_pcbinfo->ipi_smr);
1491 inp_lock(inp, lock);
1492 if (__predict_false(in_pcbrele(inp, lock)))
1493 return (false);
1494 /*
1495 * inp acquired through refcount & lock for sure didn't went
1496 * through uma_zfree(). However, it may have already went
1497 * through in_pcbfree() and has another reference, that
1498 * prevented its release by our in_pcbrele().
1499 */
1500 if (__predict_false(inp->inp_flags & ignflags)) {
1501 inp_unlock(inp, lock);
1502 return (false);
1503 }
1504 return (true);
1505 } else {
1506 smr_exit(inp->inp_pcbinfo->ipi_smr);
1507 return (false);
1508 }
1509 }
1510
1511 bool
inp_smr_lock(struct inpcb * inp,const inp_lookup_t lock)1512 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1513 {
1514
1515 /*
1516 * in_pcblookup() family of functions ignore not only freed entries,
1517 * that may be found due to lockless access to the hash, but dropped
1518 * entries, too.
1519 */
1520 return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED));
1521 }
1522
1523 /*
1524 * inp_next() - inpcb hash/list traversal iterator
1525 *
1526 * Requires initialized struct inpcb_iterator for context.
1527 * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1528 *
1529 * - Iterator can have either write-lock or read-lock semantics, that can not
1530 * be changed later.
1531 * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1532 * a single hash slot. Note: only rip_input() does the latter.
1533 * - Iterator may have optional bool matching function. The matching function
1534 * will be executed for each inpcb in the SMR context, so it can not acquire
1535 * locks and can safely access only immutable fields of inpcb.
1536 *
1537 * A fresh initialized iterator has NULL inpcb in its context and that
1538 * means that inp_next() call would return the very first inpcb on the list
1539 * locked with desired semantic. In all following calls the context pointer
1540 * shall hold the current inpcb pointer. The KPI user is not supposed to
1541 * unlock the current inpcb! Upon end of traversal inp_next() will return NULL
1542 * and write NULL to its context. After end of traversal an iterator can be
1543 * reused.
1544 *
1545 * List traversals have the following features/constraints:
1546 * - New entries won't be seen, as they are always added to the head of a list.
1547 * - Removed entries won't stop traversal as long as they are not added to
1548 * a different list. This is violated by in_pcbrehash().
1549 */
1550 #define II_LIST_FIRST(ipi, hash) \
1551 (((hash) == INP_ALL_LIST) ? \
1552 CK_LIST_FIRST(&(ipi)->ipi_listhead) : \
1553 CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)]))
1554 #define II_LIST_NEXT(inp, hash) \
1555 (((hash) == INP_ALL_LIST) ? \
1556 CK_LIST_NEXT((inp), inp_list) : \
1557 CK_LIST_NEXT((inp), inp_hash_exact))
1558 #define II_LOCK_ASSERT(inp, lock) \
1559 rw_assert(&(inp)->inp_lock, \
1560 (lock) == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED )
1561 struct inpcb *
inp_next(struct inpcb_iterator * ii)1562 inp_next(struct inpcb_iterator *ii)
1563 {
1564 const struct inpcbinfo *ipi = ii->ipi;
1565 inp_match_t *match = ii->match;
1566 void *ctx = ii->ctx;
1567 inp_lookup_t lock = ii->lock;
1568 int hash = ii->hash;
1569 struct inpcb *inp;
1570
1571 if (ii->inp == NULL) { /* First call. */
1572 smr_enter(ipi->ipi_smr);
1573 /* This is unrolled CK_LIST_FOREACH(). */
1574 for (inp = II_LIST_FIRST(ipi, hash);
1575 inp != NULL;
1576 inp = II_LIST_NEXT(inp, hash)) {
1577 if (match != NULL && (match)(inp, ctx) == false)
1578 continue;
1579 if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1580 break;
1581 else {
1582 smr_enter(ipi->ipi_smr);
1583 MPASS(inp != II_LIST_FIRST(ipi, hash));
1584 inp = II_LIST_FIRST(ipi, hash);
1585 if (inp == NULL)
1586 break;
1587 }
1588 }
1589
1590 if (inp == NULL)
1591 smr_exit(ipi->ipi_smr);
1592 else
1593 ii->inp = inp;
1594
1595 return (inp);
1596 }
1597
1598 /* Not a first call. */
1599 smr_enter(ipi->ipi_smr);
1600 restart:
1601 inp = ii->inp;
1602 II_LOCK_ASSERT(inp, lock);
1603 next:
1604 inp = II_LIST_NEXT(inp, hash);
1605 if (inp == NULL) {
1606 smr_exit(ipi->ipi_smr);
1607 goto found;
1608 }
1609
1610 if (match != NULL && (match)(inp, ctx) == false)
1611 goto next;
1612
1613 if (__predict_true(inp_trylock(inp, lock))) {
1614 if (__predict_false(inp->inp_flags & INP_FREED)) {
1615 /*
1616 * Entries are never inserted in middle of a list, thus
1617 * as long as we are in SMR, we can continue traversal.
1618 * Jump to 'restart' should yield in the same result,
1619 * but could produce unnecessary looping. Could this
1620 * looping be unbound?
1621 */
1622 inp_unlock(inp, lock);
1623 goto next;
1624 } else {
1625 smr_exit(ipi->ipi_smr);
1626 goto found;
1627 }
1628 }
1629
1630 /*
1631 * Can't obtain lock immediately, thus going hard. Once we exit the
1632 * SMR section we can no longer jump to 'next', and our only stable
1633 * anchoring point is ii->inp, which we keep locked for this case, so
1634 * we jump to 'restart'.
1635 */
1636 if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1637 smr_exit(ipi->ipi_smr);
1638 inp_lock(inp, lock);
1639 if (__predict_false(in_pcbrele(inp, lock))) {
1640 smr_enter(ipi->ipi_smr);
1641 goto restart;
1642 }
1643 /*
1644 * See comment in inp_smr_lock().
1645 */
1646 if (__predict_false(inp->inp_flags & INP_FREED)) {
1647 inp_unlock(inp, lock);
1648 smr_enter(ipi->ipi_smr);
1649 goto restart;
1650 }
1651 } else
1652 goto next;
1653
1654 found:
1655 inp_unlock(ii->inp, lock);
1656 ii->inp = inp;
1657
1658 return (ii->inp);
1659 }
1660
1661 /*
1662 * in_pcbref() bumps the reference count on an inpcb in order to maintain
1663 * stability of an inpcb pointer despite the inpcb lock being released or
1664 * SMR section exited.
1665 *
1666 * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1667 */
1668 void
in_pcbref(struct inpcb * inp)1669 in_pcbref(struct inpcb *inp)
1670 {
1671 u_int old __diagused;
1672
1673 old = refcount_acquire(&inp->inp_refcount);
1674 KASSERT(old > 0, ("%s: refcount 0", __func__));
1675 }
1676
1677 /*
1678 * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1679 * freeing the pcb, if the reference was very last.
1680 */
1681 bool
in_pcbrele_rlocked(struct inpcb * inp)1682 in_pcbrele_rlocked(struct inpcb *inp)
1683 {
1684
1685 INP_RLOCK_ASSERT(inp);
1686
1687 if (!refcount_release(&inp->inp_refcount))
1688 return (false);
1689
1690 MPASS(inp->inp_flags & INP_FREED);
1691 MPASS(inp->inp_socket == NULL);
1692 crfree(inp->inp_cred);
1693 #ifdef INVARIANTS
1694 inp->inp_cred = NULL;
1695 #endif
1696 INP_RUNLOCK(inp);
1697 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1698 return (true);
1699 }
1700
1701 bool
in_pcbrele_wlocked(struct inpcb * inp)1702 in_pcbrele_wlocked(struct inpcb *inp)
1703 {
1704
1705 INP_WLOCK_ASSERT(inp);
1706
1707 if (!refcount_release(&inp->inp_refcount))
1708 return (false);
1709
1710 MPASS(inp->inp_flags & INP_FREED);
1711 MPASS(inp->inp_socket == NULL);
1712 crfree(inp->inp_cred);
1713 #ifdef INVARIANTS
1714 inp->inp_cred = NULL;
1715 #endif
1716 INP_WUNLOCK(inp);
1717 uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1718 return (true);
1719 }
1720
1721 bool
in_pcbrele(struct inpcb * inp,const inp_lookup_t lock)1722 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1723 {
1724
1725 return (lock == INPLOOKUP_RLOCKPCB ?
1726 in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1727 }
1728
1729 /*
1730 * Unconditionally schedule an inpcb to be freed by decrementing its
1731 * reference count, which should occur only after the inpcb has been detached
1732 * from its socket. If another thread holds a temporary reference (acquired
1733 * using in_pcbref()) then the free is deferred until that reference is
1734 * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1735 * Almost all work, including removal from global lists, is done in this
1736 * context, where the pcbinfo lock is held.
1737 */
1738 void
in_pcbfree(struct inpcb * inp)1739 in_pcbfree(struct inpcb *inp)
1740 {
1741 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1742 #ifdef INET
1743 struct ip_moptions *imo;
1744 #endif
1745 #ifdef INET6
1746 struct ip6_moptions *im6o;
1747 #endif
1748
1749 INP_WLOCK_ASSERT(inp);
1750 KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1751 KASSERT((inp->inp_flags & INP_FREED) == 0,
1752 ("%s: called twice for pcb %p", __func__, inp));
1753
1754 /*
1755 * in_pcblookup_local() and in6_pcblookup_local() may return an inpcb
1756 * from the hash without acquiring inpcb lock, they rely on the hash
1757 * lock, thus in_pcbremhash() should be the first action.
1758 */
1759 if (inp->inp_flags & INP_INHASHLIST)
1760 in_pcbremhash(inp);
1761 INP_INFO_WLOCK(pcbinfo);
1762 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1763 pcbinfo->ipi_count--;
1764 CK_LIST_REMOVE(inp, inp_list);
1765 INP_INFO_WUNLOCK(pcbinfo);
1766
1767 #ifdef RATELIMIT
1768 if (inp->inp_snd_tag != NULL)
1769 in_pcbdetach_txrtlmt(inp);
1770 #endif
1771 inp->inp_flags |= INP_FREED;
1772 inp->inp_socket->so_pcb = NULL;
1773 inp->inp_socket = NULL;
1774
1775 RO_INVALIDATE_CACHE(&inp->inp_route);
1776 #ifdef MAC
1777 mac_inpcb_destroy(inp);
1778 #endif
1779 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1780 if (inp->inp_sp != NULL)
1781 ipsec_delete_pcbpolicy(inp);
1782 #endif
1783 #ifdef INET
1784 if (inp->inp_options)
1785 (void)m_free(inp->inp_options);
1786 DEBUG_POISON_POINTER(inp->inp_options);
1787 imo = inp->inp_moptions;
1788 DEBUG_POISON_POINTER(inp->inp_moptions);
1789 #endif
1790 #ifdef INET6
1791 if (inp->inp_vflag & INP_IPV6PROTO) {
1792 ip6_freepcbopts(inp->in6p_outputopts);
1793 DEBUG_POISON_POINTER(inp->in6p_outputopts);
1794 im6o = inp->in6p_moptions;
1795 DEBUG_POISON_POINTER(inp->in6p_moptions);
1796 } else
1797 im6o = NULL;
1798 #endif
1799
1800 if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1801 INP_WUNLOCK(inp);
1802 }
1803 #ifdef INET6
1804 ip6_freemoptions(im6o);
1805 #endif
1806 #ifdef INET
1807 inp_freemoptions(imo);
1808 #endif
1809 }
1810
1811 /*
1812 * Different protocols initialize their inpcbs differently - giving
1813 * different name to the lock. But they all are disposed the same.
1814 */
1815 static void
inpcb_fini(void * mem,int size)1816 inpcb_fini(void *mem, int size)
1817 {
1818 struct inpcb *inp = mem;
1819
1820 INP_LOCK_DESTROY(inp);
1821 }
1822
1823 /*
1824 * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1825 * port reservation, and preventing it from being returned by inpcb lookups.
1826 *
1827 * It is used by TCP to mark an inpcb as unused and avoid future packet
1828 * delivery or event notification when a socket remains open but TCP has
1829 * closed. This might occur as a result of a shutdown()-initiated TCP close
1830 * or a RST on the wire, and allows the port binding to be reused while still
1831 * maintaining the invariant that so_pcb always points to a valid inpcb until
1832 * in_pcbdetach().
1833 *
1834 * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1835 * in_pcbpurgeif0()?
1836 */
1837 void
in_pcbdrop(struct inpcb * inp)1838 in_pcbdrop(struct inpcb *inp)
1839 {
1840
1841 INP_WLOCK_ASSERT(inp);
1842
1843 inp->inp_flags |= INP_DROPPED;
1844 if (inp->inp_flags & INP_INHASHLIST)
1845 in_pcbremhash(inp);
1846 }
1847
1848 #ifdef INET
1849 /*
1850 * Common routines to return the socket addresses associated with inpcbs.
1851 */
1852 int
in_getsockaddr(struct socket * so,struct sockaddr * sa)1853 in_getsockaddr(struct socket *so, struct sockaddr *sa)
1854 {
1855 struct inpcb *inp;
1856
1857 inp = sotoinpcb(so);
1858 KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1859
1860 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
1861 .sin_len = sizeof(struct sockaddr_in),
1862 .sin_family = AF_INET,
1863 .sin_port = inp->inp_lport,
1864 .sin_addr = inp->inp_laddr,
1865 };
1866
1867 return (0);
1868 }
1869
1870 int
in_getpeeraddr(struct socket * so,struct sockaddr * sa)1871 in_getpeeraddr(struct socket *so, struct sockaddr *sa)
1872 {
1873 struct inpcb *inp;
1874
1875 inp = sotoinpcb(so);
1876 KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1877
1878 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
1879 .sin_len = sizeof(struct sockaddr_in),
1880 .sin_family = AF_INET,
1881 .sin_port = inp->inp_fport,
1882 .sin_addr = inp->inp_faddr,
1883 };
1884
1885 return (0);
1886 }
1887
1888 static bool
inp_v4_multi_match(const struct inpcb * inp,void * v __unused)1889 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
1890 {
1891
1892 if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
1893 return (true);
1894 else
1895 return (false);
1896 }
1897
1898 void
in_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)1899 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1900 {
1901 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
1902 inp_v4_multi_match, NULL);
1903 struct inpcb *inp;
1904 struct in_multi *inm;
1905 struct in_mfilter *imf;
1906 struct ip_moptions *imo;
1907
1908 IN_MULTI_LOCK_ASSERT();
1909
1910 while ((inp = inp_next(&inpi)) != NULL) {
1911 INP_WLOCK_ASSERT(inp);
1912
1913 imo = inp->inp_moptions;
1914 /*
1915 * Unselect the outgoing interface if it is being
1916 * detached.
1917 */
1918 if (imo->imo_multicast_ifp == ifp)
1919 imo->imo_multicast_ifp = NULL;
1920
1921 /*
1922 * Drop multicast group membership if we joined
1923 * through the interface being detached.
1924 *
1925 * XXX This can all be deferred to an epoch_call
1926 */
1927 restart:
1928 IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1929 if ((inm = imf->imf_inm) == NULL)
1930 continue;
1931 if (inm->inm_ifp != ifp)
1932 continue;
1933 ip_mfilter_remove(&imo->imo_head, imf);
1934 in_leavegroup_locked(inm, NULL);
1935 ip_mfilter_free(imf);
1936 goto restart;
1937 }
1938 }
1939 }
1940
1941 /*
1942 * Lookup a PCB based on the local address and port. Caller must hold the
1943 * hash lock. No inpcb locks or references are acquired.
1944 */
1945 #define INP_LOOKUP_MAPPED_PCB_COST 3
1946 struct inpcb *
in_pcblookup_local(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,int lookupflags,struct ucred * cred)1947 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1948 u_short lport, int lookupflags, struct ucred *cred)
1949 {
1950 struct inpcb *inp;
1951 #ifdef INET6
1952 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1953 #else
1954 int matchwild = 3;
1955 #endif
1956 int wildcard;
1957
1958 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1959 ("%s: invalid lookup flags %d", __func__, lookupflags));
1960 INP_HASH_LOCK_ASSERT(pcbinfo);
1961
1962 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1963 struct inpcbhead *head;
1964 /*
1965 * Look for an unconnected (wildcard foreign addr) PCB that
1966 * matches the local address and port we're looking for.
1967 */
1968 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1969 pcbinfo->ipi_hashmask)];
1970 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1971 #ifdef INET6
1972 /* XXX inp locking */
1973 if ((inp->inp_vflag & INP_IPV4) == 0)
1974 continue;
1975 #endif
1976 if (inp->inp_faddr.s_addr == INADDR_ANY &&
1977 inp->inp_laddr.s_addr == laddr.s_addr &&
1978 inp->inp_lport == lport) {
1979 /*
1980 * Found?
1981 */
1982 if (prison_equal_ip4(cred->cr_prison,
1983 inp->inp_cred->cr_prison))
1984 return (inp);
1985 }
1986 }
1987 /*
1988 * Not found.
1989 */
1990 return (NULL);
1991 } else {
1992 struct inpcbporthead *porthash;
1993 struct inpcbport *phd;
1994 struct inpcb *match = NULL;
1995 /*
1996 * Best fit PCB lookup.
1997 *
1998 * First see if this local port is in use by looking on the
1999 * port hash list.
2000 */
2001 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
2002 pcbinfo->ipi_porthashmask)];
2003 CK_LIST_FOREACH(phd, porthash, phd_hash) {
2004 if (phd->phd_port == lport)
2005 break;
2006 }
2007 if (phd != NULL) {
2008 /*
2009 * Port is in use by one or more PCBs. Look for best
2010 * fit.
2011 */
2012 CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
2013 wildcard = 0;
2014 if (!prison_equal_ip4(inp->inp_cred->cr_prison,
2015 cred->cr_prison))
2016 continue;
2017 #ifdef INET6
2018 /* XXX inp locking */
2019 if ((inp->inp_vflag & INP_IPV4) == 0)
2020 continue;
2021 /*
2022 * We never select the PCB that has
2023 * INP_IPV6 flag and is bound to :: if
2024 * we have another PCB which is bound
2025 * to 0.0.0.0. If a PCB has the
2026 * INP_IPV6 flag, then we set its cost
2027 * higher than IPv4 only PCBs.
2028 *
2029 * Note that the case only happens
2030 * when a socket is bound to ::, under
2031 * the condition that the use of the
2032 * mapped address is allowed.
2033 */
2034 if ((inp->inp_vflag & INP_IPV6) != 0)
2035 wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2036 #endif
2037 if (inp->inp_faddr.s_addr != INADDR_ANY)
2038 wildcard++;
2039 if (inp->inp_laddr.s_addr != INADDR_ANY) {
2040 if (laddr.s_addr == INADDR_ANY)
2041 wildcard++;
2042 else if (inp->inp_laddr.s_addr != laddr.s_addr)
2043 continue;
2044 } else {
2045 if (laddr.s_addr != INADDR_ANY)
2046 wildcard++;
2047 }
2048 if (wildcard < matchwild) {
2049 match = inp;
2050 matchwild = wildcard;
2051 if (matchwild == 0)
2052 break;
2053 }
2054 }
2055 }
2056 return (match);
2057 }
2058 }
2059 #undef INP_LOOKUP_MAPPED_PCB_COST
2060
2061 static bool
in_pcblookup_lb_numa_match(const struct inpcblbgroup * grp,int domain)2062 in_pcblookup_lb_numa_match(const struct inpcblbgroup *grp, int domain)
2063 {
2064 return (domain == M_NODOM || domain == grp->il_numa_domain);
2065 }
2066
2067 static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in_addr * faddr,uint16_t fport,const struct in_addr * laddr,uint16_t lport,int domain)2068 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2069 const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr,
2070 uint16_t lport, int domain)
2071 {
2072 const struct inpcblbgrouphead *hdr;
2073 struct inpcblbgroup *grp;
2074 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
2075 struct inpcb *inp;
2076 u_int count;
2077
2078 INP_HASH_LOCK_ASSERT(pcbinfo);
2079 NET_EPOCH_ASSERT();
2080
2081 hdr = &pcbinfo->ipi_lbgrouphashbase[
2082 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2083
2084 /*
2085 * Search for an LB group match based on the following criteria:
2086 * - prefer jailed groups to non-jailed groups
2087 * - prefer exact source address matches to wildcard matches
2088 * - prefer groups bound to the specified NUMA domain
2089 */
2090 jail_exact = jail_wild = local_exact = local_wild = NULL;
2091 CK_LIST_FOREACH(grp, hdr, il_list) {
2092 bool injail;
2093
2094 #ifdef INET6
2095 if (!(grp->il_vflag & INP_IPV4))
2096 continue;
2097 #endif
2098 if (grp->il_lport != lport)
2099 continue;
2100
2101 injail = prison_flag(grp->il_cred, PR_IP4) != 0;
2102 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,
2103 laddr) != 0)
2104 continue;
2105
2106 if (grp->il_laddr.s_addr == laddr->s_addr) {
2107 if (injail) {
2108 jail_exact = grp;
2109 if (in_pcblookup_lb_numa_match(grp, domain))
2110 /* This is a perfect match. */
2111 goto out;
2112 } else if (local_exact == NULL ||
2113 in_pcblookup_lb_numa_match(grp, domain)) {
2114 local_exact = grp;
2115 }
2116 } else if (grp->il_laddr.s_addr == INADDR_ANY) {
2117 if (injail) {
2118 if (jail_wild == NULL ||
2119 in_pcblookup_lb_numa_match(grp, domain))
2120 jail_wild = grp;
2121 } else if (local_wild == NULL ||
2122 in_pcblookup_lb_numa_match(grp, domain)) {
2123 local_wild = grp;
2124 }
2125 }
2126 }
2127
2128 if (jail_exact != NULL)
2129 grp = jail_exact;
2130 else if (jail_wild != NULL)
2131 grp = jail_wild;
2132 else if (local_exact != NULL)
2133 grp = local_exact;
2134 else
2135 grp = local_wild;
2136 if (grp == NULL)
2137 return (NULL);
2138
2139 out:
2140 /*
2141 * Synchronize with in_pcblbgroup_insert().
2142 */
2143 count = atomic_load_acq_int(&grp->il_inpcnt);
2144 if (count == 0)
2145 return (NULL);
2146 inp = grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
2147 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
2148 return (inp);
2149 }
2150
2151 static bool
in_pcblookup_exact_match(const struct inpcb * inp,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2152 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2153 u_short fport, struct in_addr laddr, u_short lport)
2154 {
2155 #ifdef INET6
2156 /* XXX inp locking */
2157 if ((inp->inp_vflag & INP_IPV4) == 0)
2158 return (false);
2159 #endif
2160 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2161 inp->inp_laddr.s_addr == laddr.s_addr &&
2162 inp->inp_fport == fport &&
2163 inp->inp_lport == lport)
2164 return (true);
2165 return (false);
2166 }
2167
2168 static struct inpcb *
in_pcblookup_hash_exact(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_short fport,struct in_addr laddr,u_short lport)2169 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2170 u_short fport, struct in_addr laddr, u_short lport)
2171 {
2172 struct inpcbhead *head;
2173 struct inpcb *inp;
2174
2175 INP_HASH_LOCK_ASSERT(pcbinfo);
2176
2177 head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
2178 pcbinfo->ipi_hashmask)];
2179 CK_LIST_FOREACH(inp, head, inp_hash_exact) {
2180 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2181 return (inp);
2182 }
2183 return (NULL);
2184 }
2185
2186 typedef enum {
2187 INPLOOKUP_MATCH_NONE = 0,
2188 INPLOOKUP_MATCH_WILD = 1,
2189 INPLOOKUP_MATCH_LADDR = 2,
2190 } inp_lookup_match_t;
2191
2192 static inp_lookup_match_t
in_pcblookup_wild_match(const struct inpcb * inp,struct in_addr laddr,u_short lport)2193 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2194 u_short lport)
2195 {
2196 #ifdef INET6
2197 /* XXX inp locking */
2198 if ((inp->inp_vflag & INP_IPV4) == 0)
2199 return (INPLOOKUP_MATCH_NONE);
2200 #endif
2201 if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2202 return (INPLOOKUP_MATCH_NONE);
2203 if (inp->inp_laddr.s_addr == INADDR_ANY)
2204 return (INPLOOKUP_MATCH_WILD);
2205 if (inp->inp_laddr.s_addr == laddr.s_addr)
2206 return (INPLOOKUP_MATCH_LADDR);
2207 return (INPLOOKUP_MATCH_NONE);
2208 }
2209
2210 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
2211
2212 static struct inpcb *
in_pcblookup_hash_wild_smr(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport,const inp_lookup_t lockflags)2213 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2214 u_short lport, const inp_lookup_t lockflags)
2215 {
2216 struct inpcbhead *head;
2217 struct inpcb *inp;
2218
2219 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
2220 ("%s: not in SMR read section", __func__));
2221
2222 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2223 pcbinfo->ipi_hashmask)];
2224 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2225 inp_lookup_match_t match;
2226
2227 match = in_pcblookup_wild_match(inp, laddr, lport);
2228 if (match == INPLOOKUP_MATCH_NONE)
2229 continue;
2230
2231 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2232 match = in_pcblookup_wild_match(inp, laddr, lport);
2233 if (match != INPLOOKUP_MATCH_NONE &&
2234 prison_check_ip4_locked(inp->inp_cred->cr_prison,
2235 &laddr) == 0)
2236 return (inp);
2237 inp_unlock(inp, lockflags);
2238 }
2239
2240 /*
2241 * The matching socket disappeared out from under us. Fall back
2242 * to a serialized lookup.
2243 */
2244 return (INP_LOOKUP_AGAIN);
2245 }
2246 return (NULL);
2247 }
2248
2249 static struct inpcb *
in_pcblookup_hash_wild_locked(struct inpcbinfo * pcbinfo,struct in_addr laddr,u_short lport)2250 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,
2251 u_short lport)
2252 {
2253 struct inpcbhead *head;
2254 struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2255 #ifdef INET6
2256 struct inpcb *local_wild_mapped;
2257 #endif
2258
2259 INP_HASH_LOCK_ASSERT(pcbinfo);
2260
2261 /*
2262 * Order of socket selection - we always prefer jails.
2263 * 1. jailed, non-wild.
2264 * 2. jailed, wild.
2265 * 3. non-jailed, non-wild.
2266 * 4. non-jailed, wild.
2267 */
2268 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2269 pcbinfo->ipi_hashmask)];
2270 local_wild = local_exact = jail_wild = NULL;
2271 #ifdef INET6
2272 local_wild_mapped = NULL;
2273 #endif
2274 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2275 inp_lookup_match_t match;
2276 bool injail;
2277
2278 match = in_pcblookup_wild_match(inp, laddr, lport);
2279 if (match == INPLOOKUP_MATCH_NONE)
2280 continue;
2281
2282 injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2283 if (injail) {
2284 if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2285 &laddr) != 0)
2286 continue;
2287 } else {
2288 if (local_exact != NULL)
2289 continue;
2290 }
2291
2292 if (match == INPLOOKUP_MATCH_LADDR) {
2293 if (injail)
2294 return (inp);
2295 local_exact = inp;
2296 } else {
2297 #ifdef INET6
2298 /* XXX inp locking, NULL check */
2299 if (inp->inp_vflag & INP_IPV6PROTO)
2300 local_wild_mapped = inp;
2301 else
2302 #endif
2303 if (injail)
2304 jail_wild = inp;
2305 else
2306 local_wild = inp;
2307 }
2308 }
2309 if (jail_wild != NULL)
2310 return (jail_wild);
2311 if (local_exact != NULL)
2312 return (local_exact);
2313 if (local_wild != NULL)
2314 return (local_wild);
2315 #ifdef INET6
2316 if (local_wild_mapped != NULL)
2317 return (local_wild_mapped);
2318 #endif
2319 return (NULL);
2320 }
2321
2322 /*
2323 * Lookup PCB in hash list, using pcbinfo tables. This variation assumes
2324 * that the caller has either locked the hash list, which usually happens
2325 * for bind(2) operations, or is in SMR section, which happens when sorting
2326 * out incoming packets.
2327 */
2328 static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain)2329 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2330 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2331 uint8_t numa_domain)
2332 {
2333 struct inpcb *inp;
2334 const u_short fport = fport_arg, lport = lport_arg;
2335
2336 KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0,
2337 ("%s: invalid lookup flags %d", __func__, lookupflags));
2338 KASSERT(faddr.s_addr != INADDR_ANY,
2339 ("%s: invalid foreign address", __func__));
2340 KASSERT(laddr.s_addr != INADDR_ANY,
2341 ("%s: invalid local address", __func__));
2342 INP_HASH_WLOCK_ASSERT(pcbinfo);
2343
2344 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2345 if (inp != NULL)
2346 return (inp);
2347
2348 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2349 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2350 &laddr, lport, numa_domain);
2351 if (inp == NULL) {
2352 inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr,
2353 lport);
2354 }
2355 }
2356
2357 return (inp);
2358 }
2359
2360 static struct inpcb *
in_pcblookup_hash(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,uint8_t numa_domain)2361 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2362 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2363 uint8_t numa_domain)
2364 {
2365 struct inpcb *inp;
2366 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2367
2368 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2369 ("%s: LOCKPCB not set", __func__));
2370
2371 INP_HASH_WLOCK(pcbinfo);
2372 inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2373 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain);
2374 if (inp != NULL && !inp_trylock(inp, lockflags)) {
2375 in_pcbref(inp);
2376 INP_HASH_WUNLOCK(pcbinfo);
2377 inp_lock(inp, lockflags);
2378 if (in_pcbrele(inp, lockflags))
2379 /* XXX-MJ or retry until we get a negative match? */
2380 inp = NULL;
2381 } else {
2382 INP_HASH_WUNLOCK(pcbinfo);
2383 }
2384 return (inp);
2385 }
2386
2387 static struct inpcb *
in_pcblookup_hash_smr(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport_arg,struct in_addr laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain)2388 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2389 u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2390 uint8_t numa_domain)
2391 {
2392 struct inpcb *inp;
2393 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2394 const u_short fport = fport_arg, lport = lport_arg;
2395
2396 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2397 ("%s: invalid lookup flags %d", __func__, lookupflags));
2398 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2399 ("%s: LOCKPCB not set", __func__));
2400
2401 smr_enter(pcbinfo->ipi_smr);
2402 inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2403 if (inp != NULL) {
2404 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2405 /*
2406 * Revalidate the 4-tuple, the socket could have been
2407 * disconnected.
2408 */
2409 if (__predict_true(in_pcblookup_exact_match(inp,
2410 faddr, fport, laddr, lport)))
2411 return (inp);
2412 inp_unlock(inp, lockflags);
2413 }
2414
2415 /*
2416 * We failed to lock the inpcb, or its connection state changed
2417 * out from under us. Fall back to a precise search.
2418 */
2419 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2420 lookupflags, numa_domain));
2421 }
2422
2423 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2424 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2425 &laddr, lport, numa_domain);
2426 if (inp != NULL) {
2427 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2428 if (__predict_true(in_pcblookup_wild_match(inp,
2429 laddr, lport) != INPLOOKUP_MATCH_NONE))
2430 return (inp);
2431 inp_unlock(inp, lockflags);
2432 }
2433 inp = INP_LOOKUP_AGAIN;
2434 } else {
2435 inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,
2436 lockflags);
2437 }
2438 if (inp == INP_LOOKUP_AGAIN) {
2439 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,
2440 lport, lookupflags, numa_domain));
2441 }
2442 }
2443
2444 if (inp == NULL)
2445 smr_exit(pcbinfo->ipi_smr);
2446
2447 return (inp);
2448 }
2449
2450 /*
2451 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2452 * from which a pre-calculated hash value may be extracted.
2453 */
2454 struct inpcb *
in_pcblookup(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused)2455 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2456 struct in_addr laddr, u_int lport, int lookupflags,
2457 struct ifnet *ifp __unused)
2458 {
2459 return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2460 lookupflags, M_NODOM));
2461 }
2462
2463 struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo * pcbinfo,struct in_addr faddr,u_int fport,struct in_addr laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)2464 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2465 u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2466 struct ifnet *ifp __unused, struct mbuf *m)
2467 {
2468 return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2469 lookupflags, m->m_pkthdr.numa_domain));
2470 }
2471 #endif /* INET */
2472
2473 static bool
in_pcbjailed(const struct inpcb * inp,unsigned int flag)2474 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2475 {
2476 return (prison_flag(inp->inp_cred, flag) != 0);
2477 }
2478
2479 /*
2480 * Insert the PCB into a hash chain using ordering rules which ensure that
2481 * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first.
2482 *
2483 * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2484 * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs
2485 * always appear last no matter whether they are jailed.
2486 */
2487 static void
_in_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2488 _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2489 {
2490 struct inpcb *last;
2491 bool bound, injail;
2492
2493 INP_LOCK_ASSERT(inp);
2494 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2495
2496 last = NULL;
2497 bound = inp->inp_laddr.s_addr != INADDR_ANY;
2498 if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2499 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2500 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2501 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2502 return;
2503 }
2504 }
2505 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2506 return;
2507 }
2508
2509 injail = in_pcbjailed(inp, PR_IP4);
2510 if (!injail) {
2511 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2512 if (!in_pcbjailed(last, PR_IP4))
2513 break;
2514 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2515 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2516 return;
2517 }
2518 }
2519 } else if (!CK_LIST_EMPTY(pcbhash) &&
2520 !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) {
2521 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2522 return;
2523 }
2524 if (!bound) {
2525 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2526 if (last->inp_laddr.s_addr == INADDR_ANY)
2527 break;
2528 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2529 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2530 return;
2531 }
2532 }
2533 }
2534 if (last == NULL)
2535 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2536 else
2537 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2538 }
2539
2540 #ifdef INET6
2541 /*
2542 * See the comment above _in_pcbinshash_wild().
2543 */
2544 static void
_in6_pcbinshash_wild(struct inpcbhead * pcbhash,struct inpcb * inp)2545 _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2546 {
2547 struct inpcb *last;
2548 bool bound, injail;
2549
2550 INP_LOCK_ASSERT(inp);
2551 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2552
2553 last = NULL;
2554 bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2555 injail = in_pcbjailed(inp, PR_IP6);
2556 if (!injail) {
2557 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2558 if (!in_pcbjailed(last, PR_IP6))
2559 break;
2560 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2561 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2562 return;
2563 }
2564 }
2565 } else if (!CK_LIST_EMPTY(pcbhash) &&
2566 !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) {
2567 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2568 return;
2569 }
2570 if (!bound) {
2571 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2572 if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
2573 break;
2574 if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2575 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2576 return;
2577 }
2578 }
2579 }
2580 if (last == NULL)
2581 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2582 else
2583 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2584 }
2585 #endif
2586
2587 /*
2588 * Insert PCB onto various hash lists.
2589 */
2590 int
in_pcbinshash(struct inpcb * inp)2591 in_pcbinshash(struct inpcb *inp)
2592 {
2593 struct inpcbhead *pcbhash;
2594 struct inpcbporthead *pcbporthash;
2595 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2596 struct inpcbport *phd;
2597 uint32_t hash;
2598 bool connected;
2599
2600 INP_WLOCK_ASSERT(inp);
2601 INP_HASH_WLOCK_ASSERT(pcbinfo);
2602 KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2603 ("in_pcbinshash: INP_INHASHLIST"));
2604
2605 #ifdef INET6
2606 if (inp->inp_vflag & INP_IPV6) {
2607 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2608 inp->inp_fport, pcbinfo->ipi_hashmask);
2609 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2610 } else
2611 #endif
2612 {
2613 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2614 inp->inp_fport, pcbinfo->ipi_hashmask);
2615 connected = !in_nullhost(inp->inp_faddr);
2616 }
2617
2618 if (connected)
2619 pcbhash = &pcbinfo->ipi_hash_exact[hash];
2620 else
2621 pcbhash = &pcbinfo->ipi_hash_wild[hash];
2622
2623 pcbporthash = &pcbinfo->ipi_porthashbase[
2624 INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2625
2626 /*
2627 * Add entry to load balance group.
2628 * Only do this if SO_REUSEPORT_LB is set.
2629 */
2630 if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2631 int error = in_pcbinslbgrouphash(inp, M_NODOM);
2632 if (error != 0)
2633 return (error);
2634 }
2635
2636 /*
2637 * Go through port list and look for a head for this lport.
2638 */
2639 CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2640 if (phd->phd_port == inp->inp_lport)
2641 break;
2642 }
2643
2644 /*
2645 * If none exists, malloc one and tack it on.
2646 */
2647 if (phd == NULL) {
2648 phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT);
2649 if (phd == NULL) {
2650 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2651 in_pcbremlbgrouphash(inp);
2652 return (ENOMEM);
2653 }
2654 phd->phd_port = inp->inp_lport;
2655 CK_LIST_INIT(&phd->phd_pcblist);
2656 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2657 }
2658 inp->inp_phd = phd;
2659 CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2660
2661 /*
2662 * The PCB may have been disconnected in the past. Before we can safely
2663 * make it visible in the hash table, we must wait for all readers which
2664 * may be traversing this PCB to finish.
2665 */
2666 if (inp->inp_smr != SMR_SEQ_INVALID) {
2667 smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
2668 inp->inp_smr = SMR_SEQ_INVALID;
2669 }
2670
2671 if (connected)
2672 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
2673 else {
2674 #ifdef INET6
2675 if ((inp->inp_vflag & INP_IPV6) != 0)
2676 _in6_pcbinshash_wild(pcbhash, inp);
2677 else
2678 #endif
2679 _in_pcbinshash_wild(pcbhash, inp);
2680 }
2681 inp->inp_flags |= INP_INHASHLIST;
2682
2683 return (0);
2684 }
2685
2686 void
in_pcbremhash_locked(struct inpcb * inp)2687 in_pcbremhash_locked(struct inpcb *inp)
2688 {
2689 struct inpcbport *phd = inp->inp_phd;
2690
2691 INP_WLOCK_ASSERT(inp);
2692 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2693 MPASS(inp->inp_flags & INP_INHASHLIST);
2694
2695 if ((inp->inp_flags & INP_INLBGROUP) != 0)
2696 in_pcbremlbgrouphash(inp);
2697 #ifdef INET6
2698 if (inp->inp_vflag & INP_IPV6) {
2699 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
2700 CK_LIST_REMOVE(inp, inp_hash_wild);
2701 else
2702 CK_LIST_REMOVE(inp, inp_hash_exact);
2703 } else
2704 #endif
2705 {
2706 if (in_nullhost(inp->inp_faddr))
2707 CK_LIST_REMOVE(inp, inp_hash_wild);
2708 else
2709 CK_LIST_REMOVE(inp, inp_hash_exact);
2710 }
2711 CK_LIST_REMOVE(inp, inp_portlist);
2712 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
2713 CK_LIST_REMOVE(phd, phd_hash);
2714 uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd);
2715 }
2716 inp->inp_flags &= ~INP_INHASHLIST;
2717 }
2718
2719 static void
in_pcbremhash(struct inpcb * inp)2720 in_pcbremhash(struct inpcb *inp)
2721 {
2722 INP_HASH_WLOCK(inp->inp_pcbinfo);
2723 in_pcbremhash_locked(inp);
2724 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
2725 }
2726
2727 /*
2728 * Move PCB to the proper hash bucket when { faddr, fport } have been
2729 * changed. NOTE: This does not handle the case of the lport changing (the
2730 * hashed port list would have to be updated as well), so the lport must
2731 * not change after in_pcbinshash() has been called.
2732 */
2733 void
in_pcbrehash(struct inpcb * inp)2734 in_pcbrehash(struct inpcb *inp)
2735 {
2736 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2737 struct inpcbhead *head;
2738 uint32_t hash;
2739 bool connected;
2740
2741 INP_WLOCK_ASSERT(inp);
2742 INP_HASH_WLOCK_ASSERT(pcbinfo);
2743 KASSERT(inp->inp_flags & INP_INHASHLIST,
2744 ("%s: !INP_INHASHLIST", __func__));
2745 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2746 ("%s: inp was disconnected", __func__));
2747
2748 #ifdef INET6
2749 if (inp->inp_vflag & INP_IPV6) {
2750 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2751 inp->inp_fport, pcbinfo->ipi_hashmask);
2752 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2753 } else
2754 #endif
2755 {
2756 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2757 inp->inp_fport, pcbinfo->ipi_hashmask);
2758 connected = !in_nullhost(inp->inp_faddr);
2759 }
2760
2761 /*
2762 * When rehashing, the caller must ensure that either the new or the old
2763 * foreign address was unspecified.
2764 */
2765 if (connected)
2766 CK_LIST_REMOVE(inp, inp_hash_wild);
2767 else
2768 CK_LIST_REMOVE(inp, inp_hash_exact);
2769
2770 if (connected) {
2771 head = &pcbinfo->ipi_hash_exact[hash];
2772 CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
2773 } else {
2774 head = &pcbinfo->ipi_hash_wild[hash];
2775 CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild);
2776 }
2777 }
2778
2779 /*
2780 * Check for alternatives when higher level complains
2781 * about service problems. For now, invalidate cached
2782 * routing information. If the route was created dynamically
2783 * (by a redirect), time to try a default gateway again.
2784 */
2785 void
in_losing(struct inpcb * inp)2786 in_losing(struct inpcb *inp)
2787 {
2788
2789 RO_INVALIDATE_CACHE(&inp->inp_route);
2790 return;
2791 }
2792
2793 /*
2794 * A set label operation has occurred at the socket layer, propagate the
2795 * label change into the in_pcb for the socket.
2796 */
2797 void
in_pcbsosetlabel(struct socket * so)2798 in_pcbsosetlabel(struct socket *so)
2799 {
2800 #ifdef MAC
2801 struct inpcb *inp;
2802
2803 inp = sotoinpcb(so);
2804 KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2805
2806 INP_WLOCK(inp);
2807 SOCK_LOCK(so);
2808 mac_inpcb_sosetlabel(so, inp);
2809 SOCK_UNLOCK(so);
2810 INP_WUNLOCK(inp);
2811 #endif
2812 }
2813
2814 void
inp_wlock(struct inpcb * inp)2815 inp_wlock(struct inpcb *inp)
2816 {
2817
2818 INP_WLOCK(inp);
2819 }
2820
2821 void
inp_wunlock(struct inpcb * inp)2822 inp_wunlock(struct inpcb *inp)
2823 {
2824
2825 INP_WUNLOCK(inp);
2826 }
2827
2828 void
inp_rlock(struct inpcb * inp)2829 inp_rlock(struct inpcb *inp)
2830 {
2831
2832 INP_RLOCK(inp);
2833 }
2834
2835 void
inp_runlock(struct inpcb * inp)2836 inp_runlock(struct inpcb *inp)
2837 {
2838
2839 INP_RUNLOCK(inp);
2840 }
2841
2842 #ifdef INVARIANT_SUPPORT
2843 void
inp_lock_assert(struct inpcb * inp)2844 inp_lock_assert(struct inpcb *inp)
2845 {
2846
2847 INP_WLOCK_ASSERT(inp);
2848 }
2849
2850 void
inp_unlock_assert(struct inpcb * inp)2851 inp_unlock_assert(struct inpcb *inp)
2852 {
2853
2854 INP_UNLOCK_ASSERT(inp);
2855 }
2856 #endif
2857
2858 void
inp_apply_all(struct inpcbinfo * pcbinfo,void (* func)(struct inpcb *,void *),void * arg)2859 inp_apply_all(struct inpcbinfo *pcbinfo,
2860 void (*func)(struct inpcb *, void *), void *arg)
2861 {
2862 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2863 INPLOOKUP_WLOCKPCB);
2864 struct inpcb *inp;
2865
2866 while ((inp = inp_next(&inpi)) != NULL)
2867 func(inp, arg);
2868 }
2869
2870 struct socket *
inp_inpcbtosocket(struct inpcb * inp)2871 inp_inpcbtosocket(struct inpcb *inp)
2872 {
2873
2874 INP_WLOCK_ASSERT(inp);
2875 return (inp->inp_socket);
2876 }
2877
2878 void
inp_4tuple_get(struct inpcb * inp,uint32_t * laddr,uint16_t * lp,uint32_t * faddr,uint16_t * fp)2879 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2880 uint32_t *faddr, uint16_t *fp)
2881 {
2882
2883 INP_LOCK_ASSERT(inp);
2884 *laddr = inp->inp_laddr.s_addr;
2885 *faddr = inp->inp_faddr.s_addr;
2886 *lp = inp->inp_lport;
2887 *fp = inp->inp_fport;
2888 }
2889
2890 /*
2891 * Create an external-format (``xinpcb'') structure using the information in
2892 * the kernel-format in_pcb structure pointed to by inp. This is done to
2893 * reduce the spew of irrelevant information over this interface, to isolate
2894 * user code from changes in the kernel structure, and potentially to provide
2895 * information-hiding if we decide that some of this information should be
2896 * hidden from users.
2897 */
2898 void
in_pcbtoxinpcb(const struct inpcb * inp,struct xinpcb * xi)2899 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2900 {
2901
2902 bzero(xi, sizeof(*xi));
2903 xi->xi_len = sizeof(struct xinpcb);
2904 if (inp->inp_socket)
2905 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2906 bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2907 xi->inp_gencnt = inp->inp_gencnt;
2908 xi->inp_flow = inp->inp_flow;
2909 xi->inp_flowid = inp->inp_flowid;
2910 xi->inp_flowtype = inp->inp_flowtype;
2911 xi->inp_flags = inp->inp_flags;
2912 xi->inp_flags2 = inp->inp_flags2;
2913 xi->in6p_cksum = inp->in6p_cksum;
2914 xi->in6p_hops = inp->in6p_hops;
2915 xi->inp_ip_tos = inp->inp_ip_tos;
2916 xi->inp_vflag = inp->inp_vflag;
2917 xi->inp_ip_ttl = inp->inp_ip_ttl;
2918 xi->inp_ip_p = inp->inp_ip_p;
2919 xi->inp_ip_minttl = inp->inp_ip_minttl;
2920 }
2921
2922 int
sysctl_setsockopt(SYSCTL_HANDLER_ARGS,struct inpcbinfo * pcbinfo,int (* ctloutput_set)(struct inpcb *,struct sockopt *))2923 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
2924 int (*ctloutput_set)(struct inpcb *, struct sockopt *))
2925 {
2926 struct sockopt sopt;
2927 struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2928 INPLOOKUP_WLOCKPCB);
2929 struct inpcb *inp;
2930 struct sockopt_parameters *params;
2931 struct socket *so;
2932 int error;
2933 char buf[1024];
2934
2935 if (req->oldptr != NULL || req->oldlen != 0)
2936 return (EINVAL);
2937 if (req->newptr == NULL)
2938 return (EPERM);
2939 if (req->newlen > sizeof(buf))
2940 return (ENOMEM);
2941 error = SYSCTL_IN(req, buf, req->newlen);
2942 if (error != 0)
2943 return (error);
2944 if (req->newlen < sizeof(struct sockopt_parameters))
2945 return (EINVAL);
2946 params = (struct sockopt_parameters *)buf;
2947 sopt.sopt_level = params->sop_level;
2948 sopt.sopt_name = params->sop_optname;
2949 sopt.sopt_dir = SOPT_SET;
2950 sopt.sopt_val = params->sop_optval;
2951 sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
2952 sopt.sopt_td = NULL;
2953 #ifdef INET6
2954 if (params->sop_inc.inc_flags & INC_ISIPV6) {
2955 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr))
2956 params->sop_inc.inc6_laddr.s6_addr16[1] =
2957 htons(params->sop_inc.inc6_zoneid & 0xffff);
2958 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr))
2959 params->sop_inc.inc6_faddr.s6_addr16[1] =
2960 htons(params->sop_inc.inc6_zoneid & 0xffff);
2961 }
2962 #endif
2963 if (params->sop_inc.inc_lport != htons(0) &&
2964 params->sop_inc.inc_fport != htons(0)) {
2965 #ifdef INET6
2966 if (params->sop_inc.inc_flags & INC_ISIPV6)
2967 inpi.hash = INP6_PCBHASH(
2968 ¶ms->sop_inc.inc6_faddr,
2969 params->sop_inc.inc_lport,
2970 params->sop_inc.inc_fport,
2971 pcbinfo->ipi_hashmask);
2972 else
2973 #endif
2974 inpi.hash = INP_PCBHASH(
2975 ¶ms->sop_inc.inc_faddr,
2976 params->sop_inc.inc_lport,
2977 params->sop_inc.inc_fport,
2978 pcbinfo->ipi_hashmask);
2979 }
2980 while ((inp = inp_next(&inpi)) != NULL)
2981 if (inp->inp_gencnt == params->sop_id) {
2982 if (inp->inp_flags & INP_DROPPED) {
2983 INP_WUNLOCK(inp);
2984 return (ECONNRESET);
2985 }
2986 so = inp->inp_socket;
2987 KASSERT(so != NULL, ("inp_socket == NULL"));
2988 soref(so);
2989 if (params->sop_level == SOL_SOCKET) {
2990 INP_WUNLOCK(inp);
2991 error = sosetopt(so, &sopt);
2992 } else
2993 error = (*ctloutput_set)(inp, &sopt);
2994 sorele(so);
2995 break;
2996 }
2997 if (inp == NULL)
2998 error = ESRCH;
2999 return (error);
3000 }
3001
3002 #ifdef DDB
3003 static void
db_print_indent(int indent)3004 db_print_indent(int indent)
3005 {
3006 int i;
3007
3008 for (i = 0; i < indent; i++)
3009 db_printf(" ");
3010 }
3011
3012 static void
db_print_inconninfo(struct in_conninfo * inc,const char * name,int indent)3013 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
3014 {
3015 char faddr_str[48], laddr_str[48];
3016
3017 db_print_indent(indent);
3018 db_printf("%s at %p\n", name, inc);
3019
3020 indent += 2;
3021
3022 #ifdef INET6
3023 if (inc->inc_flags & INC_ISIPV6) {
3024 /* IPv6. */
3025 ip6_sprintf(laddr_str, &inc->inc6_laddr);
3026 ip6_sprintf(faddr_str, &inc->inc6_faddr);
3027 } else
3028 #endif
3029 {
3030 /* IPv4. */
3031 inet_ntoa_r(inc->inc_laddr, laddr_str);
3032 inet_ntoa_r(inc->inc_faddr, faddr_str);
3033 }
3034 db_print_indent(indent);
3035 db_printf("inc_laddr %s inc_lport %u\n", laddr_str,
3036 ntohs(inc->inc_lport));
3037 db_print_indent(indent);
3038 db_printf("inc_faddr %s inc_fport %u\n", faddr_str,
3039 ntohs(inc->inc_fport));
3040 }
3041
3042 static void
db_print_inpflags(int inp_flags)3043 db_print_inpflags(int inp_flags)
3044 {
3045 int comma;
3046
3047 comma = 0;
3048 if (inp_flags & INP_RECVOPTS) {
3049 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
3050 comma = 1;
3051 }
3052 if (inp_flags & INP_RECVRETOPTS) {
3053 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
3054 comma = 1;
3055 }
3056 if (inp_flags & INP_RECVDSTADDR) {
3057 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
3058 comma = 1;
3059 }
3060 if (inp_flags & INP_ORIGDSTADDR) {
3061 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
3062 comma = 1;
3063 }
3064 if (inp_flags & INP_HDRINCL) {
3065 db_printf("%sINP_HDRINCL", comma ? ", " : "");
3066 comma = 1;
3067 }
3068 if (inp_flags & INP_HIGHPORT) {
3069 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
3070 comma = 1;
3071 }
3072 if (inp_flags & INP_LOWPORT) {
3073 db_printf("%sINP_LOWPORT", comma ? ", " : "");
3074 comma = 1;
3075 }
3076 if (inp_flags & INP_ANONPORT) {
3077 db_printf("%sINP_ANONPORT", comma ? ", " : "");
3078 comma = 1;
3079 }
3080 if (inp_flags & INP_RECVIF) {
3081 db_printf("%sINP_RECVIF", comma ? ", " : "");
3082 comma = 1;
3083 }
3084 if (inp_flags & INP_MTUDISC) {
3085 db_printf("%sINP_MTUDISC", comma ? ", " : "");
3086 comma = 1;
3087 }
3088 if (inp_flags & INP_RECVTTL) {
3089 db_printf("%sINP_RECVTTL", comma ? ", " : "");
3090 comma = 1;
3091 }
3092 if (inp_flags & INP_DONTFRAG) {
3093 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
3094 comma = 1;
3095 }
3096 if (inp_flags & INP_RECVTOS) {
3097 db_printf("%sINP_RECVTOS", comma ? ", " : "");
3098 comma = 1;
3099 }
3100 if (inp_flags & IN6P_IPV6_V6ONLY) {
3101 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3102 comma = 1;
3103 }
3104 if (inp_flags & IN6P_PKTINFO) {
3105 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3106 comma = 1;
3107 }
3108 if (inp_flags & IN6P_HOPLIMIT) {
3109 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3110 comma = 1;
3111 }
3112 if (inp_flags & IN6P_HOPOPTS) {
3113 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3114 comma = 1;
3115 }
3116 if (inp_flags & IN6P_DSTOPTS) {
3117 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3118 comma = 1;
3119 }
3120 if (inp_flags & IN6P_RTHDR) {
3121 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3122 comma = 1;
3123 }
3124 if (inp_flags & IN6P_RTHDRDSTOPTS) {
3125 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3126 comma = 1;
3127 }
3128 if (inp_flags & IN6P_TCLASS) {
3129 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3130 comma = 1;
3131 }
3132 if (inp_flags & IN6P_AUTOFLOWLABEL) {
3133 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3134 comma = 1;
3135 }
3136 if (inp_flags & INP_ONESBCAST) {
3137 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3138 comma = 1;
3139 }
3140 if (inp_flags & INP_DROPPED) {
3141 db_printf("%sINP_DROPPED", comma ? ", " : "");
3142 comma = 1;
3143 }
3144 if (inp_flags & INP_SOCKREF) {
3145 db_printf("%sINP_SOCKREF", comma ? ", " : "");
3146 comma = 1;
3147 }
3148 if (inp_flags & IN6P_RFC2292) {
3149 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3150 comma = 1;
3151 }
3152 if (inp_flags & IN6P_MTU) {
3153 db_printf("IN6P_MTU%s", comma ? ", " : "");
3154 comma = 1;
3155 }
3156 }
3157
3158 static void
db_print_inpvflag(u_char inp_vflag)3159 db_print_inpvflag(u_char inp_vflag)
3160 {
3161 int comma;
3162
3163 comma = 0;
3164 if (inp_vflag & INP_IPV4) {
3165 db_printf("%sINP_IPV4", comma ? ", " : "");
3166 comma = 1;
3167 }
3168 if (inp_vflag & INP_IPV6) {
3169 db_printf("%sINP_IPV6", comma ? ", " : "");
3170 comma = 1;
3171 }
3172 if (inp_vflag & INP_IPV6PROTO) {
3173 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3174 comma = 1;
3175 }
3176 }
3177
3178 static void
db_print_inpcb(struct inpcb * inp,const char * name,int indent)3179 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3180 {
3181
3182 db_print_indent(indent);
3183 db_printf("%s at %p\n", name, inp);
3184
3185 indent += 2;
3186
3187 db_print_indent(indent);
3188 db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3189
3190 db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3191
3192 db_print_indent(indent);
3193 db_printf("inp_label: %p inp_flags: 0x%x (",
3194 inp->inp_label, inp->inp_flags);
3195 db_print_inpflags(inp->inp_flags);
3196 db_printf(")\n");
3197
3198 db_print_indent(indent);
3199 db_printf("inp_sp: %p inp_vflag: 0x%x (", inp->inp_sp,
3200 inp->inp_vflag);
3201 db_print_inpvflag(inp->inp_vflag);
3202 db_printf(")\n");
3203
3204 db_print_indent(indent);
3205 db_printf("inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n",
3206 inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3207
3208 db_print_indent(indent);
3209 #ifdef INET6
3210 if (inp->inp_vflag & INP_IPV6) {
3211 db_printf("in6p_options: %p in6p_outputopts: %p "
3212 "in6p_moptions: %p\n", inp->in6p_options,
3213 inp->in6p_outputopts, inp->in6p_moptions);
3214 db_printf("in6p_icmp6filt: %p in6p_cksum %d "
3215 "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3216 inp->in6p_hops);
3217 } else
3218 #endif
3219 {
3220 db_printf("inp_ip_tos: %d inp_ip_options: %p "
3221 "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3222 inp->inp_options, inp->inp_moptions);
3223 }
3224
3225 db_print_indent(indent);
3226 db_printf("inp_phd: %p inp_gencnt: %ju\n", inp->inp_phd,
3227 (uintmax_t)inp->inp_gencnt);
3228 }
3229
DB_SHOW_COMMAND(inpcb,db_show_inpcb)3230 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3231 {
3232 struct inpcb *inp;
3233
3234 if (!have_addr) {
3235 db_printf("usage: show inpcb <addr>\n");
3236 return;
3237 }
3238 inp = (struct inpcb *)addr;
3239
3240 db_print_inpcb(inp, "inpcb", 0);
3241 }
3242 #endif /* DDB */
3243
3244 #ifdef RATELIMIT
3245 /*
3246 * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3247 * if any.
3248 */
3249 int
in_pcbmodify_txrtlmt(struct inpcb * inp,uint32_t max_pacing_rate)3250 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3251 {
3252 union if_snd_tag_modify_params params = {
3253 .rate_limit.max_rate = max_pacing_rate,
3254 .rate_limit.flags = M_NOWAIT,
3255 };
3256 struct m_snd_tag *mst;
3257 int error;
3258
3259 mst = inp->inp_snd_tag;
3260 if (mst == NULL)
3261 return (EINVAL);
3262
3263 if (mst->sw->snd_tag_modify == NULL) {
3264 error = EOPNOTSUPP;
3265 } else {
3266 error = mst->sw->snd_tag_modify(mst, ¶ms);
3267 }
3268 return (error);
3269 }
3270
3271 /*
3272 * Query existing TX rate limit based on the existing
3273 * "inp->inp_snd_tag", if any.
3274 */
3275 int
in_pcbquery_txrtlmt(struct inpcb * inp,uint32_t * p_max_pacing_rate)3276 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3277 {
3278 union if_snd_tag_query_params params = { };
3279 struct m_snd_tag *mst;
3280 int error;
3281
3282 mst = inp->inp_snd_tag;
3283 if (mst == NULL)
3284 return (EINVAL);
3285
3286 if (mst->sw->snd_tag_query == NULL) {
3287 error = EOPNOTSUPP;
3288 } else {
3289 error = mst->sw->snd_tag_query(mst, ¶ms);
3290 if (error == 0 && p_max_pacing_rate != NULL)
3291 *p_max_pacing_rate = params.rate_limit.max_rate;
3292 }
3293 return (error);
3294 }
3295
3296 /*
3297 * Query existing TX queue level based on the existing
3298 * "inp->inp_snd_tag", if any.
3299 */
3300 int
in_pcbquery_txrlevel(struct inpcb * inp,uint32_t * p_txqueue_level)3301 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3302 {
3303 union if_snd_tag_query_params params = { };
3304 struct m_snd_tag *mst;
3305 int error;
3306
3307 mst = inp->inp_snd_tag;
3308 if (mst == NULL)
3309 return (EINVAL);
3310
3311 if (mst->sw->snd_tag_query == NULL)
3312 return (EOPNOTSUPP);
3313
3314 error = mst->sw->snd_tag_query(mst, ¶ms);
3315 if (error == 0 && p_txqueue_level != NULL)
3316 *p_txqueue_level = params.rate_limit.queue_level;
3317 return (error);
3318 }
3319
3320 /*
3321 * Allocate a new TX rate limit send tag from the network interface
3322 * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3323 */
3324 int
in_pcbattach_txrtlmt(struct inpcb * inp,struct ifnet * ifp,uint32_t flowtype,uint32_t flowid,uint32_t max_pacing_rate,struct m_snd_tag ** st)3325 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3326 uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3327
3328 {
3329 union if_snd_tag_alloc_params params = {
3330 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3331 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3332 .rate_limit.hdr.flowid = flowid,
3333 .rate_limit.hdr.flowtype = flowtype,
3334 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3335 .rate_limit.max_rate = max_pacing_rate,
3336 .rate_limit.flags = M_NOWAIT,
3337 };
3338 int error;
3339
3340 INP_WLOCK_ASSERT(inp);
3341
3342 /*
3343 * If there is already a send tag, or the INP is being torn
3344 * down, allocating a new send tag is not allowed. Else send
3345 * tags may leak.
3346 */
3347 if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0)
3348 return (EINVAL);
3349
3350 error = m_snd_tag_alloc(ifp, ¶ms, st);
3351 #ifdef INET
3352 if (error == 0) {
3353 counter_u64_add(rate_limit_set_ok, 1);
3354 counter_u64_add(rate_limit_active, 1);
3355 } else if (error != EOPNOTSUPP)
3356 counter_u64_add(rate_limit_alloc_fail, 1);
3357 #endif
3358 return (error);
3359 }
3360
3361 void
in_pcbdetach_tag(struct m_snd_tag * mst)3362 in_pcbdetach_tag(struct m_snd_tag *mst)
3363 {
3364
3365 m_snd_tag_rele(mst);
3366 #ifdef INET
3367 counter_u64_add(rate_limit_active, -1);
3368 #endif
3369 }
3370
3371 /*
3372 * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3373 * if any:
3374 */
3375 void
in_pcbdetach_txrtlmt(struct inpcb * inp)3376 in_pcbdetach_txrtlmt(struct inpcb *inp)
3377 {
3378 struct m_snd_tag *mst;
3379
3380 INP_WLOCK_ASSERT(inp);
3381
3382 mst = inp->inp_snd_tag;
3383 inp->inp_snd_tag = NULL;
3384
3385 if (mst == NULL)
3386 return;
3387
3388 m_snd_tag_rele(mst);
3389 #ifdef INET
3390 counter_u64_add(rate_limit_active, -1);
3391 #endif
3392 }
3393
3394 int
in_pcboutput_txrtlmt_locked(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb,uint32_t max_pacing_rate)3395 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3396 {
3397 int error;
3398
3399 /*
3400 * If the existing send tag is for the wrong interface due to
3401 * a route change, first drop the existing tag. Set the
3402 * CHANGED flag so that we will keep trying to allocate a new
3403 * tag if we fail to allocate one this time.
3404 */
3405 if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3406 in_pcbdetach_txrtlmt(inp);
3407 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3408 }
3409
3410 /*
3411 * NOTE: When attaching to a network interface a reference is
3412 * made to ensure the network interface doesn't go away until
3413 * all ratelimit connections are gone. The network interface
3414 * pointers compared below represent valid network interfaces,
3415 * except when comparing towards NULL.
3416 */
3417 if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3418 error = 0;
3419 } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3420 if (inp->inp_snd_tag != NULL)
3421 in_pcbdetach_txrtlmt(inp);
3422 error = 0;
3423 } else if (inp->inp_snd_tag == NULL) {
3424 /*
3425 * In order to utilize packet pacing with RSS, we need
3426 * to wait until there is a valid RSS hash before we
3427 * can proceed:
3428 */
3429 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3430 error = EAGAIN;
3431 } else {
3432 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3433 mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3434 }
3435 } else {
3436 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3437 }
3438 if (error == 0 || error == EOPNOTSUPP)
3439 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3440
3441 return (error);
3442 }
3443
3444 /*
3445 * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3446 * is set in the fast path and will attach/detach/modify the TX rate
3447 * limit send tag based on the socket's so_max_pacing_rate value.
3448 */
3449 void
in_pcboutput_txrtlmt(struct inpcb * inp,struct ifnet * ifp,struct mbuf * mb)3450 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3451 {
3452 struct socket *socket;
3453 uint32_t max_pacing_rate;
3454 bool did_upgrade;
3455
3456 if (inp == NULL)
3457 return;
3458
3459 socket = inp->inp_socket;
3460 if (socket == NULL)
3461 return;
3462
3463 if (!INP_WLOCKED(inp)) {
3464 /*
3465 * NOTE: If the write locking fails, we need to bail
3466 * out and use the non-ratelimited ring for the
3467 * transmit until there is a new chance to get the
3468 * write lock.
3469 */
3470 if (!INP_TRY_UPGRADE(inp))
3471 return;
3472 did_upgrade = 1;
3473 } else {
3474 did_upgrade = 0;
3475 }
3476
3477 /*
3478 * NOTE: The so_max_pacing_rate value is read unlocked,
3479 * because atomic updates are not required since the variable
3480 * is checked at every mbuf we send. It is assumed that the
3481 * variable read itself will be atomic.
3482 */
3483 max_pacing_rate = socket->so_max_pacing_rate;
3484
3485 in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3486
3487 if (did_upgrade)
3488 INP_DOWNGRADE(inp);
3489 }
3490
3491 /*
3492 * Track route changes for TX rate limiting.
3493 */
3494 void
in_pcboutput_eagain(struct inpcb * inp)3495 in_pcboutput_eagain(struct inpcb *inp)
3496 {
3497 bool did_upgrade;
3498
3499 if (inp == NULL)
3500 return;
3501
3502 if (inp->inp_snd_tag == NULL)
3503 return;
3504
3505 if (!INP_WLOCKED(inp)) {
3506 /*
3507 * NOTE: If the write locking fails, we need to bail
3508 * out and use the non-ratelimited ring for the
3509 * transmit until there is a new chance to get the
3510 * write lock.
3511 */
3512 if (!INP_TRY_UPGRADE(inp))
3513 return;
3514 did_upgrade = 1;
3515 } else {
3516 did_upgrade = 0;
3517 }
3518
3519 /* detach rate limiting */
3520 in_pcbdetach_txrtlmt(inp);
3521
3522 /* make sure new mbuf send tag allocation is made */
3523 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3524
3525 if (did_upgrade)
3526 INP_DOWNGRADE(inp);
3527 }
3528
3529 #ifdef INET
3530 static void
rl_init(void * st)3531 rl_init(void *st)
3532 {
3533 rate_limit_new = counter_u64_alloc(M_WAITOK);
3534 rate_limit_chg = counter_u64_alloc(M_WAITOK);
3535 rate_limit_active = counter_u64_alloc(M_WAITOK);
3536 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3537 rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3538 }
3539
3540 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3541 #endif
3542 #endif /* RATELIMIT */
3543