1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
5 * Copyright (c) 2010-2011 Juniper Networks, Inc.
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Robert N. M. Watson under
9 * contract to Juniper Networks, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. Neither the name of the project nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 *
35 * $KAME: in6_pcb.c,v 1.31 2001/05/21 05:45:10 jinmei Exp $
36 */
37
38 /*-
39 * Copyright (c) 1982, 1986, 1991, 1993
40 * The Regents of the University of California. All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 */
66
67 #include <sys/cdefs.h>
68 #include "opt_inet.h"
69 #include "opt_inet6.h"
70 #include "opt_ipsec.h"
71 #include "opt_route.h"
72 #include "opt_rss.h"
73
74 #include <sys/hash.h>
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/malloc.h>
78 #include <sys/mbuf.h>
79 #include <sys/domain.h>
80 #include <sys/proc.h>
81 #include <sys/protosw.h>
82 #include <sys/smr.h>
83 #include <sys/socket.h>
84 #include <sys/socketvar.h>
85 #include <sys/sockio.h>
86 #include <sys/sysctl.h>
87 #include <sys/errno.h>
88 #include <sys/time.h>
89 #include <sys/priv.h>
90 #include <sys/proc.h>
91 #include <sys/jail.h>
92
93 #include <vm/uma.h>
94
95 #include <net/if.h>
96 #include <net/if_var.h>
97 #include <net/if_llatbl.h>
98 #include <net/if_types.h>
99 #include <net/route.h>
100 #include <net/route/nhop.h>
101 #include <net/vnet.h>
102
103 #include <netinet/in.h>
104 #include <netinet/in_var.h>
105 #include <netinet/in_systm.h>
106 #include <netinet/ip6.h>
107 #include <netinet/ip_var.h>
108
109 #include <netinet6/ip6_var.h>
110 #include <netinet6/nd6.h>
111 #include <netinet/in_pcb.h>
112 #include <netinet/in_pcb_var.h>
113 #include <netinet6/in6_pcb.h>
114 #include <netinet6/in6_fib.h>
115 #include <netinet6/scope6_var.h>
116
117 SYSCTL_DECL(_net_inet6);
118 SYSCTL_DECL(_net_inet6_ip6);
119 VNET_DEFINE_STATIC(int, connect_in6addr_wild) = 1;
120 #define V_connect_in6addr_wild VNET(connect_in6addr_wild)
121 SYSCTL_INT(_net_inet6_ip6, OID_AUTO, connect_in6addr_wild,
122 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(connect_in6addr_wild), 0,
123 "Allow connecting to the unspecified address for connect(2)");
124
125 int
in6_pcbsetport(struct in6_addr * laddr,struct inpcb * inp,struct ucred * cred)126 in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct ucred *cred)
127 {
128 struct socket *so = inp->inp_socket;
129 u_int16_t lport = 0;
130 int error, lookupflags = 0;
131 #ifdef INVARIANTS
132 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
133 #endif
134
135 INP_WLOCK_ASSERT(inp);
136 INP_HASH_WLOCK_ASSERT(pcbinfo);
137
138 error = prison_local_ip6(cred, laddr,
139 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0));
140 if (error)
141 return(error);
142
143 /* XXX: this is redundant when called from in6_pcbbind */
144 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
145 lookupflags = INPLOOKUP_WILDCARD;
146
147 inp->inp_flags |= INP_ANONPORT;
148
149 error = in_pcb_lport(inp, NULL, &lport, cred, lookupflags);
150 if (error != 0)
151 return (error);
152
153 inp->inp_lport = lport;
154 if (in_pcbinshash(inp) != 0) {
155 inp->in6p_laddr = in6addr_any;
156 inp->inp_lport = 0;
157 return (EAGAIN);
158 }
159
160 return (0);
161 }
162
163 /*
164 * Determine whether the inpcb can be bound to the specified address/port tuple.
165 */
166 static int
in6_pcbbind_avail(struct inpcb * inp,const struct sockaddr_in6 * sin6,int fib,int sooptions,int lookupflags,struct ucred * cred)167 in6_pcbbind_avail(struct inpcb *inp, const struct sockaddr_in6 *sin6, int fib,
168 int sooptions, int lookupflags, struct ucred *cred)
169 {
170 const struct in6_addr *laddr;
171 int reuseport, reuseport_lb;
172 u_short lport;
173
174 INP_LOCK_ASSERT(inp);
175 INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
176
177 laddr = &sin6->sin6_addr;
178 lport = sin6->sin6_port;
179
180 reuseport = (sooptions & SO_REUSEPORT);
181 reuseport_lb = (sooptions & SO_REUSEPORT_LB);
182
183 if (IN6_IS_ADDR_MULTICAST(laddr)) {
184 /*
185 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
186 * allow compepte duplication of binding if
187 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
188 * and a multicast address is bound on both
189 * new and duplicated sockets.
190 */
191 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT)) != 0)
192 reuseport = SO_REUSEADDR | SO_REUSEPORT;
193 /*
194 * XXX: How to deal with SO_REUSEPORT_LB here?
195 * Treat same as SO_REUSEPORT for now.
196 */
197 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT_LB)) != 0)
198 reuseport_lb = SO_REUSEADDR | SO_REUSEPORT_LB;
199 } else if (!IN6_IS_ADDR_UNSPECIFIED(laddr)) {
200 struct sockaddr_in6 sin6;
201 struct epoch_tracker et;
202 struct ifaddr *ifa;
203
204 memset(&sin6, 0, sizeof(sin6));
205 sin6.sin6_family = AF_INET6;
206 sin6.sin6_len = sizeof(sin6);
207 sin6.sin6_addr = *laddr;
208
209 NET_EPOCH_ENTER(et);
210 if ((ifa = ifa_ifwithaddr((const struct sockaddr *)&sin6)) ==
211 NULL && (inp->inp_flags & INP_BINDANY) == 0) {
212 NET_EPOCH_EXIT(et);
213 return (EADDRNOTAVAIL);
214 }
215
216 /*
217 * We used to prohibit binding to an anycast address here,
218 * based on RFC3513, but that restriction was removed in
219 * RFC4291.
220 */
221 if (ifa != NULL &&
222 ((struct in6_ifaddr *)ifa)->ia6_flags &
223 (IN6_IFF_NOTREADY | IN6_IFF_DETACHED)) {
224 NET_EPOCH_EXIT(et);
225 return (EADDRNOTAVAIL);
226 }
227 NET_EPOCH_EXIT(et);
228 }
229
230 if (lport != 0) {
231 struct inpcb *t;
232
233 if (ntohs(lport) <= V_ipport_reservedhigh &&
234 ntohs(lport) >= V_ipport_reservedlow &&
235 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
236 return (EACCES);
237
238 if (!IN6_IS_ADDR_MULTICAST(laddr) &&
239 priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) !=
240 0) {
241 /*
242 * If a socket owned by a different user is already
243 * bound to this port, fail. In particular, SO_REUSE*
244 * can only be used to share a port among sockets owned
245 * by the same user.
246 *
247 * However, we can share a port with a connected socket
248 * which has a unique 4-tuple.
249 */
250 t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
251 RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
252 if (t != NULL &&
253 (inp->inp_socket->so_type != SOCK_STREAM ||
254 IN6_IS_ADDR_UNSPECIFIED(&t->in6p_faddr)) &&
255 (inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
256 return (EADDRINUSE);
257
258 #ifdef INET
259 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
260 IN6_IS_ADDR_UNSPECIFIED(laddr)) {
261 struct sockaddr_in sin;
262
263 in6_sin6_2_sin(&sin, sin6);
264 t = in_pcblookup_local(inp->inp_pcbinfo,
265 sin.sin_addr, lport, RT_ALL_FIBS,
266 INPLOOKUP_WILDCARD, cred);
267 if (t != NULL &&
268 (inp->inp_socket->so_type != SOCK_STREAM ||
269 in_nullhost(t->inp_faddr)) &&
270 (inp->inp_cred->cr_uid !=
271 t->inp_cred->cr_uid))
272 return (EADDRINUSE);
273 }
274 #endif
275 }
276 t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
277 fib, lookupflags, cred);
278 if (t != NULL && ((reuseport | reuseport_lb) &
279 t->inp_socket->so_options) == 0)
280 return (EADDRINUSE);
281 #ifdef INET
282 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
283 IN6_IS_ADDR_UNSPECIFIED(laddr)) {
284 struct sockaddr_in sin;
285
286 in6_sin6_2_sin(&sin, sin6);
287 t = in_pcblookup_local(inp->inp_pcbinfo, sin.sin_addr,
288 lport, RT_ALL_FIBS, lookupflags, cred);
289 if (t != NULL && ((reuseport | reuseport_lb) &
290 t->inp_socket->so_options) == 0 &&
291 (!in_nullhost(t->inp_laddr) ||
292 (t->inp_vflag & INP_IPV6PROTO) != 0)) {
293 return (EADDRINUSE);
294 }
295 }
296 #endif
297 }
298 return (0);
299 }
300
301 int
in6_pcbbind(struct inpcb * inp,struct sockaddr_in6 * sin6,int flags,struct ucred * cred)302 in6_pcbbind(struct inpcb *inp, struct sockaddr_in6 *sin6, int flags,
303 struct ucred *cred)
304 {
305 struct socket *so = inp->inp_socket;
306 u_short lport = 0;
307 int error, fib, lookupflags, sooptions;
308
309 INP_WLOCK_ASSERT(inp);
310 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
311
312 if (inp->inp_lport || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
313 return (EINVAL);
314
315 lookupflags = 0;
316 sooptions = atomic_load_int(&so->so_options);
317 if ((sooptions & (SO_REUSEADDR | SO_REUSEPORT | SO_REUSEPORT_LB)) == 0)
318 lookupflags = INPLOOKUP_WILDCARD;
319 if (sin6 == NULL) {
320 if ((error = prison_local_ip6(cred, &inp->in6p_laddr,
321 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
322 return (error);
323 } else {
324 KASSERT(sin6->sin6_family == AF_INET6,
325 ("%s: invalid address family for %p", __func__, sin6));
326 KASSERT(sin6->sin6_len == sizeof(*sin6),
327 ("%s: invalid address length for %p", __func__, sin6));
328
329 if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
330 return(error);
331
332 if ((error = prison_local_ip6(cred, &sin6->sin6_addr,
333 ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
334 return (error);
335
336 fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum :
337 RT_ALL_FIBS;
338
339 /* See if this address/port combo is available. */
340 error = in6_pcbbind_avail(inp, sin6, fib, sooptions, lookupflags,
341 cred);
342 if (error != 0)
343 return (error);
344
345 lport = sin6->sin6_port;
346 inp->in6p_laddr = sin6->sin6_addr;
347 }
348 if ((flags & INPBIND_FIB) != 0)
349 inp->inp_flags |= INP_BOUNDFIB;
350 if (lport == 0) {
351 if ((error = in6_pcbsetport(&inp->in6p_laddr, inp, cred)) != 0) {
352 /* Undo an address bind that may have occurred. */
353 inp->inp_flags &= ~INP_BOUNDFIB;
354 inp->in6p_laddr = in6addr_any;
355 return (error);
356 }
357 } else {
358 inp->inp_lport = lport;
359 if (in_pcbinshash(inp) != 0) {
360 inp->inp_flags &= ~INP_BOUNDFIB;
361 inp->in6p_laddr = in6addr_any;
362 inp->inp_lport = 0;
363 return (EAGAIN);
364 }
365 }
366 return (0);
367 }
368
369 /*
370 * Transform old in6_pcbconnect() into an inner subroutine for new
371 * in6_pcbconnect(): Do some validity-checking on the remote
372 * address (in mbuf 'nam') and then determine local host address
373 * (i.e., which interface) to use to access that remote host.
374 *
375 * This preserves definition of in6_pcbconnect(), while supporting a
376 * slightly different version for T/TCP. (This is more than
377 * a bit of a kludge, but cleaning up the internal interfaces would
378 * have forced minor changes in every protocol).
379 */
380 static int
in6_pcbladdr(struct inpcb * inp,struct sockaddr_in6 * sin6,struct in6_addr * plocal_addr6,bool sas_required)381 in6_pcbladdr(struct inpcb *inp, struct sockaddr_in6 *sin6,
382 struct in6_addr *plocal_addr6, bool sas_required)
383 {
384 int error = 0;
385 int scope_ambiguous = 0;
386 struct in6_addr in6a;
387
388 NET_EPOCH_ASSERT();
389 INP_WLOCK_ASSERT(inp);
390 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); /* XXXRW: why? */
391
392 if (sin6->sin6_port == 0)
393 return (EADDRNOTAVAIL);
394
395 if (sin6->sin6_scope_id == 0 && !V_ip6_use_defzone)
396 scope_ambiguous = 1;
397 if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
398 return(error);
399
400 if (V_connect_in6addr_wild && !CK_STAILQ_EMPTY(&V_in6_ifaddrhead)) {
401 /*
402 * If the destination address is UNSPECIFIED addr,
403 * use the loopback addr, e.g ::1.
404 */
405 if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
406 sin6->sin6_addr = in6addr_loopback;
407 } else if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
408 return (ENETUNREACH);
409 }
410
411 if ((error = prison_remote_ip6(inp->inp_cred, &sin6->sin6_addr)) != 0)
412 return (error);
413
414 if (sas_required) {
415 error = in6_selectsrc_socket(sin6, inp->in6p_outputopts,
416 inp, inp->inp_cred, scope_ambiguous, &in6a, NULL);
417 if (error)
418 return (error);
419 } else {
420 /*
421 * Source address selection isn't required when syncache
422 * has already established connection and both source and
423 * destination addresses was chosen.
424 *
425 * This also includes the case when fwd_tag was used to
426 * select source address in tcp_input().
427 */
428 in6a = inp->in6p_laddr;
429 }
430
431 if (IN6_IS_ADDR_UNSPECIFIED(&in6a))
432 return (EHOSTUNREACH);
433 /*
434 * Do not update this earlier, in case we return with an error.
435 *
436 * XXX: this in6_selectsrc_socket result might replace the bound local
437 * address with the address specified by setsockopt(IPV6_PKTINFO).
438 * Is it the intended behavior?
439 */
440 *plocal_addr6 = in6a;
441
442 /*
443 * Don't do pcblookup call here; return interface in
444 * plocal_addr6
445 * and exit to caller, that will do the lookup.
446 */
447
448 return (0);
449 }
450
451 /*
452 * Outer subroutine:
453 * Connect from a socket to a specified address.
454 * Both address and port must be specified in argument sin.
455 * If don't have a local address for this socket yet,
456 * then pick one.
457 */
458 int
in6_pcbconnect(struct inpcb * inp,struct sockaddr_in6 * sin6,struct ucred * cred,bool sas_required)459 in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred,
460 bool sas_required)
461 {
462 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
463 struct sockaddr_in6 laddr6;
464 int error;
465
466 NET_EPOCH_ASSERT();
467 INP_WLOCK_ASSERT(inp);
468 INP_HASH_WLOCK_ASSERT(pcbinfo);
469 KASSERT(sin6->sin6_family == AF_INET6,
470 ("%s: invalid address family for %p", __func__, sin6));
471 KASSERT(sin6->sin6_len == sizeof(*sin6),
472 ("%s: invalid address length for %p", __func__, sin6));
473 KASSERT(IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr),
474 ("%s: inp is already connected", __func__));
475
476 bzero(&laddr6, sizeof(laddr6));
477 laddr6.sin6_family = AF_INET6;
478
479 #ifdef ROUTE_MPATH
480 if (CALC_FLOWID_OUTBOUND) {
481 uint32_t hash_type, hash_val;
482
483 hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
484 &sin6->sin6_addr, 0, sin6->sin6_port,
485 inp->inp_socket->so_proto->pr_protocol, &hash_type);
486 inp->inp_flowid = hash_val;
487 inp->inp_flowtype = hash_type;
488 }
489 #endif
490 /*
491 * Call inner routine, to assign local interface address.
492 * in6_pcbladdr() may automatically fill in sin6_scope_id.
493 */
494 if ((error = in6_pcbladdr(inp, sin6, &laddr6.sin6_addr,
495 sas_required)) != 0)
496 return (error);
497
498 if (in6_pcblookup_hash_locked(pcbinfo, &sin6->sin6_addr,
499 sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ?
500 &laddr6.sin6_addr : &inp->in6p_laddr, inp->inp_lport, 0,
501 M_NODOM, RT_ALL_FIBS) != NULL)
502 return (EADDRINUSE);
503 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
504 if (inp->inp_lport == 0) {
505 error = in_pcb_lport_dest(inp,
506 (struct sockaddr *) &laddr6, &inp->inp_lport,
507 (struct sockaddr *) sin6, sin6->sin6_port, cred,
508 INPLOOKUP_WILDCARD);
509 if (error)
510 return (error);
511 }
512 inp->in6p_laddr = laddr6.sin6_addr;
513 }
514 inp->in6p_faddr = sin6->sin6_addr;
515 inp->inp_fport = sin6->sin6_port;
516 /* update flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
517 inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
518 if (inp->inp_flags & IN6P_AUTOFLOWLABEL)
519 inp->inp_flow |=
520 (htonl(ip6_randomflowlabel()) & IPV6_FLOWLABEL_MASK);
521
522 if ((inp->inp_flags & INP_INHASHLIST) != 0) {
523 in_pcbrehash(inp);
524 } else {
525 error = in_pcbinshash(inp);
526 MPASS(error == 0);
527 }
528
529 return (0);
530 }
531
532 void
in6_pcbdisconnect(struct inpcb * inp)533 in6_pcbdisconnect(struct inpcb *inp)
534 {
535
536 INP_WLOCK_ASSERT(inp);
537 INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
538 KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
539 ("%s: inp %p was already disconnected", __func__, inp));
540
541 in_pcbremhash_locked(inp);
542
543 /* See the comment in in_pcbinshash(). */
544 inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
545
546 /* XXX-MJ torn writes are visible to SMR lookup */
547 memset(&inp->in6p_laddr, 0, sizeof(inp->in6p_laddr));
548 memset(&inp->in6p_faddr, 0, sizeof(inp->in6p_faddr));
549 inp->inp_fport = 0;
550 /* clear flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
551 inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
552 }
553
554 int
in6_getsockaddr(struct socket * so,struct sockaddr * sa)555 in6_getsockaddr(struct socket *so, struct sockaddr *sa)
556 {
557 struct inpcb *inp;
558
559 inp = sotoinpcb(so);
560 KASSERT(inp != NULL, ("in6_getsockaddr: inp == NULL"));
561
562 *(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
563 .sin6_len = sizeof(struct sockaddr_in6),
564 .sin6_family = AF_INET6,
565 .sin6_port = inp->inp_lport,
566 .sin6_addr = inp->in6p_laddr,
567 };
568 /* XXX: should catch errors */
569 (void)sa6_recoverscope((struct sockaddr_in6 *)sa);
570
571 return (0);
572 }
573
574 int
in6_getpeeraddr(struct socket * so,struct sockaddr * sa)575 in6_getpeeraddr(struct socket *so, struct sockaddr *sa)
576 {
577 struct inpcb *inp;
578
579 inp = sotoinpcb(so);
580 KASSERT(inp != NULL, ("in6_getpeeraddr: inp == NULL"));
581
582 *(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
583 .sin6_len = sizeof(struct sockaddr_in6),
584 .sin6_family = AF_INET6,
585 .sin6_port = inp->inp_fport,
586 .sin6_addr = inp->in6p_faddr,
587 };
588 /* XXX: should catch errors */
589 (void)sa6_recoverscope((struct sockaddr_in6 *)sa);
590
591 return (0);
592 }
593
594 int
in6_mapped_sockaddr(struct socket * so,struct sockaddr * sa)595 in6_mapped_sockaddr(struct socket *so, struct sockaddr *sa)
596 {
597 int error;
598 #ifdef INET
599 struct inpcb *inp;
600
601 inp = sotoinpcb(so);
602 KASSERT(inp != NULL, ("in6_mapped_sockaddr: inp == NULL"));
603
604 if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
605 struct sockaddr_in sin;
606
607 error = in_getsockaddr(so, (struct sockaddr *)&sin);
608 if (error == 0)
609 in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
610 } else
611 #endif
612 {
613 /* scope issues will be handled in in6_getsockaddr(). */
614 error = in6_getsockaddr(so, sa);
615 }
616
617 return error;
618 }
619
620 int
in6_mapped_peeraddr(struct socket * so,struct sockaddr * sa)621 in6_mapped_peeraddr(struct socket *so, struct sockaddr *sa)
622 {
623 int error;
624 #ifdef INET
625 struct inpcb *inp;
626
627 inp = sotoinpcb(so);
628 KASSERT(inp != NULL, ("in6_mapped_peeraddr: inp == NULL"));
629
630 if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
631 struct sockaddr_in sin;
632
633 error = in_getpeeraddr(so, (struct sockaddr *)&sin);
634 if (error == 0)
635 in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
636 } else
637 #endif
638 {
639 /* scope issues will be handled in in6_getpeeraddr(). */
640 error = in6_getpeeraddr(so, sa);
641 }
642
643 return error;
644 }
645
646 /*
647 * Pass some notification to all connections of a protocol
648 * associated with address dst. The local address and/or port numbers
649 * may be specified to limit the search. The "usual action" will be
650 * taken, depending on the ctlinput cmd. The caller must filter any
651 * cmds that are uninteresting (e.g., no error in the map).
652 * Call the protocol specific routine (if any) to report
653 * any errors for each matching socket.
654 */
655 static bool
inp_match6(const struct inpcb * inp,void * v __unused)656 inp_match6(const struct inpcb *inp, void *v __unused)
657 {
658
659 return ((inp->inp_vflag & INP_IPV6) != 0);
660 }
661
662 void
in6_pcbnotify(struct inpcbinfo * pcbinfo,struct sockaddr_in6 * sa6_dst,u_int fport_arg,const struct sockaddr_in6 * src,u_int lport_arg,int errno,void * cmdarg,struct inpcb * (* notify)(struct inpcb *,int))663 in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr_in6 *sa6_dst,
664 u_int fport_arg, const struct sockaddr_in6 *src, u_int lport_arg,
665 int errno, void *cmdarg,
666 struct inpcb *(*notify)(struct inpcb *, int))
667 {
668 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
669 inp_match6, NULL);
670 struct inpcb *inp;
671 struct sockaddr_in6 sa6_src;
672 u_short fport = fport_arg, lport = lport_arg;
673 u_int32_t flowinfo;
674
675 if (IN6_IS_ADDR_UNSPECIFIED(&sa6_dst->sin6_addr))
676 return;
677
678 /*
679 * note that src can be NULL when we get notify by local fragmentation.
680 */
681 sa6_src = (src == NULL) ? sa6_any : *src;
682 flowinfo = sa6_src.sin6_flowinfo;
683
684 while ((inp = inp_next(&inpi)) != NULL) {
685 INP_WLOCK_ASSERT(inp);
686 /*
687 * If the error designates a new path MTU for a destination
688 * and the application (associated with this socket) wanted to
689 * know the value, notify.
690 * XXX: should we avoid to notify the value to TCP sockets?
691 */
692 if (errno == EMSGSIZE && cmdarg != NULL)
693 ip6_notify_pmtu(inp, sa6_dst, *(uint32_t *)cmdarg);
694
695 /*
696 * Detect if we should notify the error. If no source and
697 * destination ports are specified, but non-zero flowinfo and
698 * local address match, notify the error. This is the case
699 * when the error is delivered with an encrypted buffer
700 * by ESP. Otherwise, just compare addresses and ports
701 * as usual.
702 */
703 if (lport == 0 && fport == 0 && flowinfo &&
704 inp->inp_socket != NULL &&
705 flowinfo == (inp->inp_flow & IPV6_FLOWLABEL_MASK) &&
706 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, &sa6_src.sin6_addr))
707 goto do_notify;
708 else if (!IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr,
709 &sa6_dst->sin6_addr) ||
710 inp->inp_socket == 0 ||
711 (lport && inp->inp_lport != lport) ||
712 (!IN6_IS_ADDR_UNSPECIFIED(&sa6_src.sin6_addr) &&
713 !IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr,
714 &sa6_src.sin6_addr)) ||
715 (fport && inp->inp_fport != fport)) {
716 continue;
717 }
718
719 do_notify:
720 if (notify)
721 (*notify)(inp, errno);
722 }
723 }
724
725 /*
726 * Lookup a PCB based on the local address and port. Caller must hold the
727 * hash lock. No inpcb locks or references are acquired.
728 */
729 struct inpcb *
in6_pcblookup_local(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib,int lookupflags,struct ucred * cred)730 in6_pcblookup_local(struct inpcbinfo *pcbinfo, const struct in6_addr *laddr,
731 u_short lport, int fib, int lookupflags, struct ucred *cred)
732 {
733 struct inpcb *inp;
734 int matchwild = 3, wildcard;
735
736 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
737 ("%s: invalid lookup flags %d", __func__, lookupflags));
738 KASSERT(fib == RT_ALL_FIBS || (fib >= 0 && fib < V_rt_numfibs),
739 ("%s: invalid fib %d", __func__, fib));
740
741 INP_HASH_LOCK_ASSERT(pcbinfo);
742
743 if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
744 struct inpcbhead *head;
745 /*
746 * Look for an unconnected (wildcard foreign addr) PCB that
747 * matches the local address and port we're looking for.
748 */
749 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
750 pcbinfo->ipi_hashmask)];
751 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
752 /* XXX inp locking */
753 if ((inp->inp_vflag & INP_IPV6) == 0)
754 continue;
755 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) &&
756 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
757 inp->inp_lport == lport && (fib == RT_ALL_FIBS ||
758 inp->inp_inc.inc_fibnum == fib)) {
759 /* Found. */
760 if (prison_equal_ip6(cred->cr_prison,
761 inp->inp_cred->cr_prison))
762 return (inp);
763 }
764 }
765 /*
766 * Not found.
767 */
768 return (NULL);
769 } else {
770 struct inpcbhead *porthash;
771 struct inpcb *match = NULL;
772
773 /*
774 * Port is in use by one or more PCBs. Look for best
775 * fit.
776 */
777 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
778 pcbinfo->ipi_porthashmask)];
779 CK_LIST_FOREACH(inp, porthash, inp_portlist) {
780 if (inp->inp_lport != lport)
781 continue;
782 if (!prison_equal_ip6(cred->cr_prison,
783 inp->inp_cred->cr_prison))
784 continue;
785 /* XXX inp locking */
786 if ((inp->inp_vflag & INP_IPV6) == 0)
787 continue;
788 if (fib != RT_ALL_FIBS &&
789 inp->inp_inc.inc_fibnum != fib)
790 continue;
791 wildcard = 0;
792 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
793 wildcard++;
794 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
795 if (IN6_IS_ADDR_UNSPECIFIED(laddr))
796 wildcard++;
797 else if (!IN6_ARE_ADDR_EQUAL(
798 &inp->in6p_laddr, laddr))
799 continue;
800 } else {
801 if (!IN6_IS_ADDR_UNSPECIFIED(laddr))
802 wildcard++;
803 }
804 if (wildcard < matchwild) {
805 match = inp;
806 matchwild = wildcard;
807 if (matchwild == 0)
808 break;
809 }
810 }
811 return (match);
812 }
813 }
814
815 static bool
in6_multi_match(const struct inpcb * inp,void * v __unused)816 in6_multi_match(const struct inpcb *inp, void *v __unused)
817 {
818
819 if ((inp->inp_vflag & INP_IPV6) && inp->in6p_moptions != NULL)
820 return (true);
821 else
822 return (false);
823 }
824
825 void
in6_pcbpurgeif0(struct inpcbinfo * pcbinfo,struct ifnet * ifp)826 in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
827 {
828 struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_RLOCKPCB,
829 in6_multi_match, NULL);
830 struct inpcb *inp;
831 struct in6_multi *inm;
832 struct in6_mfilter *imf;
833 struct ip6_moptions *im6o;
834
835 IN6_MULTI_LOCK_ASSERT();
836
837 while ((inp = inp_next(&inpi)) != NULL) {
838 INP_RLOCK_ASSERT(inp);
839
840 im6o = inp->in6p_moptions;
841 /*
842 * Unselect the outgoing ifp for multicast if it
843 * is being detached.
844 */
845 if (im6o->im6o_multicast_ifp == ifp)
846 im6o->im6o_multicast_ifp = NULL;
847 /*
848 * Drop multicast group membership if we joined
849 * through the interface being detached.
850 */
851 restart:
852 IP6_MFILTER_FOREACH(imf, &im6o->im6o_head) {
853 if ((inm = imf->im6f_in6m) == NULL)
854 continue;
855 if (inm->in6m_ifp != ifp)
856 continue;
857 ip6_mfilter_remove(&im6o->im6o_head, imf);
858 in6_leavegroup_locked(inm, NULL);
859 ip6_mfilter_free(imf);
860 goto restart;
861 }
862 }
863 }
864
865 /*
866 * Check for alternatives when higher level complains
867 * about service problems. For now, invalidate cached
868 * routing information. If the route was created dynamically
869 * (by a redirect), time to try a default gateway again.
870 */
871 void
in6_losing(struct inpcb * inp)872 in6_losing(struct inpcb *inp)
873 {
874
875 RO_INVALIDATE_CACHE(&inp->inp_route6);
876 }
877
878 /*
879 * After a routing change, flush old routing
880 * and allocate a (hopefully) better one.
881 */
882 struct inpcb *
in6_rtchange(struct inpcb * inp,int errno __unused)883 in6_rtchange(struct inpcb *inp, int errno __unused)
884 {
885
886 RO_INVALIDATE_CACHE(&inp->inp_route6);
887 return inp;
888 }
889
890 static bool
in6_pcblookup_lb_match(const struct inpcblbgroup * grp,int domain,int fib)891 in6_pcblookup_lb_match(const struct inpcblbgroup *grp, int domain, int fib)
892 {
893 return ((domain == M_NODOM || domain == grp->il_numa_domain) &&
894 (fib == RT_ALL_FIBS || fib == grp->il_fibnum));
895 }
896
897 static struct inpcb *
in6_pcblookup_lbgroup(const struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,uint16_t fport,const struct in6_addr * laddr,uint16_t lport,uint8_t domain,int fib)898 in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
899 const struct in6_addr *faddr, uint16_t fport, const struct in6_addr *laddr,
900 uint16_t lport, uint8_t domain, int fib)
901 {
902 const struct inpcblbgrouphead *hdr;
903 struct inpcblbgroup *grp;
904 struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
905 struct inpcb *inp;
906 u_int count;
907
908 INP_HASH_LOCK_ASSERT(pcbinfo);
909 NET_EPOCH_ASSERT();
910
911 hdr = &pcbinfo->ipi_lbgrouphashbase[
912 INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
913
914 /*
915 * Search for an LB group match based on the following criteria:
916 * - prefer jailed groups to non-jailed groups
917 * - prefer exact source address matches to wildcard matches
918 * - prefer groups bound to the specified NUMA domain
919 */
920 jail_exact = jail_wild = local_exact = local_wild = NULL;
921 CK_LIST_FOREACH(grp, hdr, il_list) {
922 bool injail;
923
924 #ifdef INET
925 if (!(grp->il_vflag & INP_IPV6))
926 continue;
927 #endif
928 if (grp->il_lport != lport)
929 continue;
930
931 injail = prison_flag(grp->il_cred, PR_IP6) != 0;
932 if (injail && prison_check_ip6_locked(grp->il_cred->cr_prison,
933 laddr) != 0)
934 continue;
935
936 if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) {
937 if (injail) {
938 jail_exact = grp;
939 if (in6_pcblookup_lb_match(grp, domain, fib))
940 /* This is a perfect match. */
941 goto out;
942 } else if (local_exact == NULL ||
943 in6_pcblookup_lb_match(grp, domain, fib)) {
944 local_exact = grp;
945 }
946 } else if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr)) {
947 if (injail) {
948 if (jail_wild == NULL ||
949 in6_pcblookup_lb_match(grp, domain, fib))
950 jail_wild = grp;
951 } else if (local_wild == NULL ||
952 in6_pcblookup_lb_match(grp, domain, fib)) {
953 local_wild = grp;
954 }
955 }
956 }
957
958 if (jail_exact != NULL)
959 grp = jail_exact;
960 else if (jail_wild != NULL)
961 grp = jail_wild;
962 else if (local_exact != NULL)
963 grp = local_exact;
964 else
965 grp = local_wild;
966 if (grp == NULL)
967 return (NULL);
968 out:
969 /*
970 * Synchronize with in_pcblbgroup_insert().
971 */
972 count = atomic_load_acq_int(&grp->il_inpcnt);
973 if (count == 0)
974 return (NULL);
975 inp = grp->il_inp[INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) % count];
976 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
977 return (inp);
978 }
979
980 static bool
in6_pcblookup_exact_match(const struct inpcb * inp,const struct in6_addr * faddr,u_short fport,const struct in6_addr * laddr,u_short lport)981 in6_pcblookup_exact_match(const struct inpcb *inp, const struct in6_addr *faddr,
982 u_short fport, const struct in6_addr *laddr, u_short lport)
983 {
984 /* XXX inp locking */
985 if ((inp->inp_vflag & INP_IPV6) == 0)
986 return (false);
987 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, faddr) &&
988 IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
989 inp->inp_fport == fport && inp->inp_lport == lport)
990 return (true);
991 return (false);
992 }
993
994 static struct inpcb *
in6_pcblookup_hash_exact(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_short fport,const struct in6_addr * laddr,u_short lport)995 in6_pcblookup_hash_exact(struct inpcbinfo *pcbinfo,
996 const struct in6_addr *faddr, u_short fport,
997 const struct in6_addr *laddr, u_short lport)
998 {
999 struct inpcbhead *head;
1000 struct inpcb *inp;
1001
1002 INP_HASH_LOCK_ASSERT(pcbinfo);
1003
1004 /*
1005 * First look for an exact match.
1006 */
1007 head = &pcbinfo->ipi_hash_exact[INP6_PCBHASH(faddr, lport, fport,
1008 pcbinfo->ipi_hashmask)];
1009 CK_LIST_FOREACH(inp, head, inp_hash_exact) {
1010 if (in6_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
1011 return (inp);
1012 }
1013 return (NULL);
1014 }
1015
1016 typedef enum {
1017 INPLOOKUP_MATCH_NONE = 0,
1018 INPLOOKUP_MATCH_WILD = 1,
1019 INPLOOKUP_MATCH_LADDR = 2,
1020 } inp_lookup_match_t;
1021
1022 static inp_lookup_match_t
in6_pcblookup_wild_match(const struct inpcb * inp,const struct in6_addr * laddr,u_short lport,int fib)1023 in6_pcblookup_wild_match(const struct inpcb *inp, const struct in6_addr *laddr,
1024 u_short lport, int fib)
1025 {
1026 /* XXX inp locking */
1027 if ((inp->inp_vflag & INP_IPV6) == 0)
1028 return (INPLOOKUP_MATCH_NONE);
1029 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) ||
1030 inp->inp_lport != lport)
1031 return (INPLOOKUP_MATCH_NONE);
1032 if (fib != RT_ALL_FIBS && inp->inp_inc.inc_fibnum != fib)
1033 return (INPLOOKUP_MATCH_NONE);
1034 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
1035 return (INPLOOKUP_MATCH_WILD);
1036 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr))
1037 return (INPLOOKUP_MATCH_LADDR);
1038 return (INPLOOKUP_MATCH_NONE);
1039 }
1040
1041 #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
1042
1043 static struct inpcb *
in6_pcblookup_hash_wild_smr(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib,const inp_lookup_t lockflags)1044 in6_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo,
1045 const struct in6_addr *laddr, u_short lport, int fib,
1046 const inp_lookup_t lockflags)
1047 {
1048 struct inpcbhead *head;
1049 struct inpcb *inp;
1050
1051 KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
1052 ("%s: not in SMR read section", __func__));
1053
1054 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1055 pcbinfo->ipi_hashmask)];
1056 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1057 inp_lookup_match_t match;
1058
1059 match = in6_pcblookup_wild_match(inp, laddr, lport, fib);
1060 if (match == INPLOOKUP_MATCH_NONE)
1061 continue;
1062
1063 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1064 match = in6_pcblookup_wild_match(inp, laddr, lport,
1065 fib);
1066 if (match != INPLOOKUP_MATCH_NONE &&
1067 prison_check_ip6_locked(inp->inp_cred->cr_prison,
1068 laddr) == 0)
1069 return (inp);
1070 inp_unlock(inp, lockflags);
1071 }
1072
1073 /*
1074 * The matching socket disappeared out from under us. Fall back
1075 * to a serialized lookup.
1076 */
1077 return (INP_LOOKUP_AGAIN);
1078 }
1079 return (NULL);
1080 }
1081
1082 static struct inpcb *
in6_pcblookup_hash_wild_locked(struct inpcbinfo * pcbinfo,const struct in6_addr * laddr,u_short lport,int fib)1083 in6_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo,
1084 const struct in6_addr *laddr, u_short lport, int fib)
1085 {
1086 struct inpcbhead *head;
1087 struct inpcb *inp, *jail_wild, *local_exact, *local_wild;
1088
1089 INP_HASH_LOCK_ASSERT(pcbinfo);
1090
1091 /*
1092 * Order of socket selection - we always prefer jails.
1093 * 1. jailed, non-wild.
1094 * 2. jailed, wild.
1095 * 3. non-jailed, non-wild.
1096 * 4. non-jailed, wild.
1097 */
1098 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1099 pcbinfo->ipi_hashmask)];
1100 local_wild = local_exact = jail_wild = NULL;
1101 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1102 inp_lookup_match_t match;
1103 bool injail;
1104
1105 match = in6_pcblookup_wild_match(inp, laddr, lport, fib);
1106 if (match == INPLOOKUP_MATCH_NONE)
1107 continue;
1108
1109 injail = prison_flag(inp->inp_cred, PR_IP6) != 0;
1110 if (injail) {
1111 if (prison_check_ip6_locked(
1112 inp->inp_cred->cr_prison, laddr) != 0)
1113 continue;
1114 } else {
1115 if (local_exact != NULL)
1116 continue;
1117 }
1118
1119 if (match == INPLOOKUP_MATCH_LADDR) {
1120 if (injail)
1121 return (inp);
1122 else
1123 local_exact = inp;
1124 } else {
1125 if (injail)
1126 jail_wild = inp;
1127 else
1128 local_wild = inp;
1129 }
1130 }
1131
1132 if (jail_wild != NULL)
1133 return (jail_wild);
1134 if (local_exact != NULL)
1135 return (local_exact);
1136 if (local_wild != NULL)
1137 return (local_wild);
1138 return (NULL);
1139 }
1140
1141 struct inpcb *
in6_pcblookup_hash_locked(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport_arg,const struct in6_addr * laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)1142 in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
1143 const struct in6_addr *faddr, u_int fport_arg,
1144 const struct in6_addr *laddr, u_int lport_arg,
1145 int lookupflags, uint8_t numa_domain, int fib)
1146 {
1147 struct inpcb *inp;
1148 u_short fport = fport_arg, lport = lport_arg;
1149
1150 KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0,
1151 ("%s: invalid lookup flags %d", __func__, lookupflags));
1152 KASSERT(!IN6_IS_ADDR_UNSPECIFIED(faddr),
1153 ("%s: invalid foreign address", __func__));
1154 KASSERT(!IN6_IS_ADDR_UNSPECIFIED(laddr),
1155 ("%s: invalid local address", __func__));
1156 INP_HASH_LOCK_ASSERT(pcbinfo);
1157
1158 inp = in6_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
1159 if (inp != NULL)
1160 return (inp);
1161
1162 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1163 inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport, laddr,
1164 lport, numa_domain, fib);
1165 if (inp == NULL) {
1166 inp = in6_pcblookup_hash_wild_locked(pcbinfo,
1167 laddr, lport, fib);
1168 }
1169 }
1170 return (inp);
1171 }
1172
1173 static struct inpcb *
in6_pcblookup_hash(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,uint8_t numa_domain,int fib)1174 in6_pcblookup_hash(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1175 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1176 uint8_t numa_domain, int fib)
1177 {
1178 struct inpcb *inp;
1179 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
1180
1181 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1182 ("%s: LOCKPCB not set", __func__));
1183
1184 INP_HASH_WLOCK(pcbinfo);
1185 inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1186 lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
1187 if (inp != NULL && !inp_trylock(inp, lockflags)) {
1188 in_pcbref(inp);
1189 INP_HASH_WUNLOCK(pcbinfo);
1190 inp_lock(inp, lockflags);
1191 if (in_pcbrele(inp, lockflags))
1192 /* XXX-MJ or retry until we get a negative match? */
1193 inp = NULL;
1194 } else {
1195 INP_HASH_WUNLOCK(pcbinfo);
1196 }
1197 return (inp);
1198 }
1199
1200 static struct inpcb *
in6_pcblookup_hash_smr(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport_arg,const struct in6_addr * laddr,u_int lport_arg,int lookupflags,uint8_t numa_domain,int fib)1201 in6_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1202 u_int fport_arg, const struct in6_addr *laddr, u_int lport_arg,
1203 int lookupflags, uint8_t numa_domain, int fib)
1204 {
1205 struct inpcb *inp;
1206 const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
1207 const u_short fport = fport_arg, lport = lport_arg;
1208
1209 KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1210 ("%s: invalid lookup flags %d", __func__, lookupflags));
1211 KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1212 ("%s: LOCKPCB not set", __func__));
1213
1214 smr_enter(pcbinfo->ipi_smr);
1215 inp = in6_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
1216 if (inp != NULL) {
1217 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1218 if (__predict_true(in6_pcblookup_exact_match(inp,
1219 faddr, fport, laddr, lport)))
1220 return (inp);
1221 inp_unlock(inp, lockflags);
1222 }
1223 /*
1224 * We failed to lock the inpcb, or its connection state changed
1225 * out from under us. Fall back to a precise search.
1226 */
1227 return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
1228 lookupflags, numa_domain, fib));
1229 }
1230
1231 if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1232 inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport,
1233 laddr, lport, numa_domain, fib);
1234 if (inp != NULL) {
1235 if (__predict_true(inp_smr_lock(inp, lockflags))) {
1236 if (__predict_true(in6_pcblookup_wild_match(inp,
1237 laddr, lport, fib) != INPLOOKUP_MATCH_NONE))
1238 return (inp);
1239 inp_unlock(inp, lockflags);
1240 }
1241 inp = INP_LOOKUP_AGAIN;
1242 } else {
1243 inp = in6_pcblookup_hash_wild_smr(pcbinfo, laddr, lport,
1244 fib, lockflags);
1245 }
1246 if (inp == INP_LOOKUP_AGAIN) {
1247 return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr,
1248 lport, lookupflags, numa_domain, fib));
1249 }
1250 }
1251
1252 if (inp == NULL)
1253 smr_exit(pcbinfo->ipi_smr);
1254
1255 return (inp);
1256 }
1257
1258 /*
1259 * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1260 * from which a pre-calculated hash value may be extracted.
1261 */
1262 struct inpcb *
in6_pcblookup(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,struct ifnet * ifp)1263 in6_pcblookup(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1264 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1265 struct ifnet *ifp)
1266 {
1267 int fib;
1268
1269 fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS;
1270 return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
1271 lookupflags, M_NODOM, fib));
1272 }
1273
1274 struct inpcb *
in6_pcblookup_mbuf(struct inpcbinfo * pcbinfo,const struct in6_addr * faddr,u_int fport,const struct in6_addr * laddr,u_int lport,int lookupflags,struct ifnet * ifp __unused,struct mbuf * m)1275 in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, const struct in6_addr *faddr,
1276 u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
1277 struct ifnet *ifp __unused, struct mbuf *m)
1278 {
1279 int fib;
1280
1281 M_ASSERTPKTHDR(m);
1282 fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS;
1283 return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
1284 lookupflags, m->m_pkthdr.numa_domain, fib));
1285 }
1286
1287 void
init_sin6(struct sockaddr_in6 * sin6,struct mbuf * m,int srcordst)1288 init_sin6(struct sockaddr_in6 *sin6, struct mbuf *m, int srcordst)
1289 {
1290 struct ip6_hdr *ip;
1291
1292 ip = mtod(m, struct ip6_hdr *);
1293 bzero(sin6, sizeof(*sin6));
1294 sin6->sin6_len = sizeof(*sin6);
1295 sin6->sin6_family = AF_INET6;
1296 sin6->sin6_addr = srcordst ? ip->ip6_dst : ip->ip6_src;
1297
1298 (void)sa6_recoverscope(sin6); /* XXX: should catch errors... */
1299
1300 return;
1301 }
1302