Lines Matching refs:sk
46 static u32 sk_ehashfn(const struct sock *sk) in sk_ehashfn() argument
49 if (sk->sk_family == AF_INET6 && in sk_ehashfn()
50 !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) in sk_ehashfn()
51 return inet6_ehashfn(sock_net(sk), in sk_ehashfn()
52 &sk->sk_v6_rcv_saddr, sk->sk_num, in sk_ehashfn()
53 &sk->sk_v6_daddr, sk->sk_dport); in sk_ehashfn()
55 return inet_ehashfn(sock_net(sk), in sk_ehashfn()
56 sk->sk_rcv_saddr, sk->sk_num, in sk_ehashfn()
57 sk->sk_daddr, sk->sk_dport); in sk_ehashfn()
106 const struct sock *sk) in inet_bind2_bucket_init() argument
113 if (sk->sk_family == AF_INET6) { in inet_bind2_bucket_init()
114 tb2->addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); in inet_bind2_bucket_init()
115 tb2->v6_rcv_saddr = sk->sk_v6_rcv_saddr; in inet_bind2_bucket_init()
118 ipv6_addr_set_v4mapped(sk->sk_rcv_saddr, &tb2->v6_rcv_saddr); in inet_bind2_bucket_init()
121 tb2->rcv_saddr = sk->sk_rcv_saddr; in inet_bind2_bucket_init()
132 const struct sock *sk) in inet_bind2_bucket_create() argument
137 inet_bind2_bucket_init(tb2, net, head, tb, sk); in inet_bind2_bucket_create()
153 const struct sock *sk) in inet_bind2_bucket_addr_match() argument
156 if (sk->sk_family == AF_INET6) in inet_bind2_bucket_addr_match()
157 return ipv6_addr_equal(&tb2->v6_rcv_saddr, &sk->sk_v6_rcv_saddr); in inet_bind2_bucket_addr_match()
162 return tb2->rcv_saddr == sk->sk_rcv_saddr; in inet_bind2_bucket_addr_match()
165 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, in inet_bind_hash() argument
168 inet_sk(sk)->inet_num = port; in inet_bind_hash()
169 inet_csk(sk)->icsk_bind_hash = tb; in inet_bind_hash()
170 inet_csk(sk)->icsk_bind2_hash = tb2; in inet_bind_hash()
171 sk_add_bind_node(sk, &tb2->owners); in inet_bind_hash()
177 static void __inet_put_port(struct sock *sk) in __inet_put_port() argument
179 struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); in __inet_put_port()
181 struct net *net = sock_net(sk); in __inet_put_port()
185 bhash = inet_bhashfn(net, inet_sk(sk)->inet_num, hashinfo->bhash_size); in __inet_put_port()
187 head2 = inet_bhashfn_portaddr(hashinfo, sk, net, inet_sk(sk)->inet_num); in __inet_put_port()
190 tb = inet_csk(sk)->icsk_bind_hash; in __inet_put_port()
191 inet_csk(sk)->icsk_bind_hash = NULL; in __inet_put_port()
192 inet_sk(sk)->inet_num = 0; in __inet_put_port()
195 if (inet_csk(sk)->icsk_bind2_hash) { in __inet_put_port()
196 struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash; in __inet_put_port()
198 __sk_del_bind_node(sk); in __inet_put_port()
199 inet_csk(sk)->icsk_bind2_hash = NULL; in __inet_put_port()
208 void inet_put_port(struct sock *sk) in inet_put_port() argument
211 __inet_put_port(sk); in inet_put_port()
216 int __inet_inherit_port(const struct sock *sk, struct sock *child) in __inet_inherit_port() argument
218 struct inet_hashinfo *table = tcp_or_dccp_get_hashinfo(sk); in __inet_inherit_port()
222 struct net *net = sock_net(sk); in __inet_inherit_port()
234 tb = inet_csk(sk)->icsk_bind_hash; in __inet_inherit_port()
235 tb2 = inet_csk(sk)->icsk_bind2_hash; in __inet_inherit_port()
242 l3mdev = inet_sk_bound_l3mdev(sk); in __inet_inherit_port()
267 l3mdev = inet_sk_bound_l3mdev(sk); in __inet_inherit_port()
296 inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk) in inet_lhash2_bucket_sk() argument
301 if (sk->sk_family == AF_INET6) in inet_lhash2_bucket_sk()
302 hash = ipv6_portaddr_hash(sock_net(sk), in inet_lhash2_bucket_sk()
303 &sk->sk_v6_rcv_saddr, in inet_lhash2_bucket_sk()
304 inet_sk(sk)->inet_num); in inet_lhash2_bucket_sk()
307 hash = ipv4_portaddr_hash(sock_net(sk), in inet_lhash2_bucket_sk()
308 inet_sk(sk)->inet_rcv_saddr, in inet_lhash2_bucket_sk()
309 inet_sk(sk)->inet_num); in inet_lhash2_bucket_sk()
313 static inline int compute_score(struct sock *sk, const struct net *net, in compute_score() argument
319 if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && in compute_score()
320 !ipv6_only_sock(sk)) { in compute_score()
321 if (sk->sk_rcv_saddr != daddr) in compute_score()
324 if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) in compute_score()
326 score = sk->sk_bound_dev_if ? 2 : 1; in compute_score()
328 if (sk->sk_family == PF_INET) in compute_score()
330 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) in compute_score()
351 struct sock *inet_lookup_reuseport(const struct net *net, struct sock *sk, in inet_lookup_reuseport() argument
360 if (sk->sk_reuseport) { in inet_lookup_reuseport()
363 reuse_sk = reuseport_select_sock(sk, phash, skb, doff); in inet_lookup_reuseport()
384 struct sock *sk, *result = NULL; in inet_lhash2_lookup() local
388 sk_nulls_for_each_rcu(sk, node, &ilb2->nulls_head) { in inet_lhash2_lookup()
389 score = compute_score(sk, net, hnum, daddr, dif, sdif); in inet_lhash2_lookup()
391 result = inet_lookup_reuseport(net, sk, skb, doff, in inet_lhash2_lookup()
396 result = sk; in inet_lhash2_lookup()
411 struct sock *sk, *reuse_sk; in inet_lookup_run_sk_lookup() local
415 daddr, hnum, dif, &sk); in inet_lookup_run_sk_lookup()
416 if (no_reuseport || IS_ERR_OR_NULL(sk)) in inet_lookup_run_sk_lookup()
417 return sk; in inet_lookup_run_sk_lookup()
419 reuse_sk = inet_lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum, in inet_lookup_run_sk_lookup()
422 sk = reuse_sk; in inet_lookup_run_sk_lookup()
423 return sk; in inet_lookup_run_sk_lookup()
471 void sock_gen_put(struct sock *sk) in sock_gen_put() argument
473 if (!refcount_dec_and_test(&sk->sk_refcnt)) in sock_gen_put()
476 if (sk->sk_state == TCP_TIME_WAIT) in sock_gen_put()
477 inet_twsk_free(inet_twsk(sk)); in sock_gen_put()
478 else if (sk->sk_state == TCP_NEW_SYN_RECV) in sock_gen_put()
479 reqsk_free(inet_reqsk(sk)); in sock_gen_put()
481 sk_free(sk); in sock_gen_put()
487 sock_gen_put(skb->sk); in sock_edemux()
499 struct sock *sk; in __inet_lookup_established() local
509 sk_nulls_for_each_rcu(sk, node, &head->chain) { in __inet_lookup_established()
510 if (sk->sk_hash != hash) in __inet_lookup_established()
512 if (likely(inet_match(net, sk, acookie, ports, dif, sdif))) { in __inet_lookup_established()
513 if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) in __inet_lookup_established()
515 if (unlikely(!inet_match(net, sk, acookie, in __inet_lookup_established()
517 sock_gen_put(sk); in __inet_lookup_established()
531 sk = NULL; in __inet_lookup_established()
533 return sk; in __inet_lookup_established()
539 struct sock *sk, __u16 lport, in __inet_check_established() argument
543 struct inet_sock *inet = inet_sk(sk); in __inet_check_established()
546 int dif = sk->sk_bound_dev_if; in __inet_check_established()
547 struct net *net = sock_net(sk); in __inet_check_established()
568 if (sk->sk_protocol == IPPROTO_TCP && in __inet_check_established()
569 tcp_twsk_unique(sk, sk2, twp)) in __inet_check_established()
581 sk->sk_hash = hash; in __inet_check_established()
582 WARN_ON(!sk_unhashed(sk)); in __inet_check_established()
583 __sk_nulls_add_node_rcu(sk, &head->chain); in __inet_check_established()
589 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in __inet_check_established()
604 static u64 inet_sk_port_offset(const struct sock *sk) in inet_sk_port_offset() argument
606 const struct inet_sock *inet = inet_sk(sk); in inet_sk_port_offset()
616 static bool inet_ehash_lookup_by_sk(struct sock *sk, in inet_ehash_lookup_by_sk() argument
619 const __portpair ports = INET_COMBINED_PORTS(sk->sk_dport, sk->sk_num); in inet_ehash_lookup_by_sk()
620 const int sdif = sk->sk_bound_dev_if; in inet_ehash_lookup_by_sk()
621 const int dif = sk->sk_bound_dev_if; in inet_ehash_lookup_by_sk()
623 struct net *net = sock_net(sk); in inet_ehash_lookup_by_sk()
626 INET_ADDR_COOKIE(acookie, sk->sk_daddr, sk->sk_rcv_saddr); in inet_ehash_lookup_by_sk()
629 if (esk->sk_hash != sk->sk_hash) in inet_ehash_lookup_by_sk()
631 if (sk->sk_family == AF_INET) { in inet_ehash_lookup_by_sk()
638 else if (sk->sk_family == AF_INET6) { in inet_ehash_lookup_by_sk()
640 &sk->sk_v6_daddr, in inet_ehash_lookup_by_sk()
641 &sk->sk_v6_rcv_saddr, in inet_ehash_lookup_by_sk()
656 bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk) in inet_ehash_insert() argument
658 struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); in inet_ehash_insert()
664 WARN_ON_ONCE(!sk_unhashed(sk)); in inet_ehash_insert()
666 sk->sk_hash = sk_ehashfn(sk); in inet_ehash_insert()
667 head = inet_ehash_bucket(hashinfo, sk->sk_hash); in inet_ehash_insert()
669 lock = inet_ehash_lockp(hashinfo, sk->sk_hash); in inet_ehash_insert()
673 WARN_ON_ONCE(sk->sk_hash != osk->sk_hash); in inet_ehash_insert()
676 *found_dup_sk = inet_ehash_lookup_by_sk(sk, list); in inet_ehash_insert()
682 __sk_nulls_add_node_rcu(sk, list); in inet_ehash_insert()
689 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk) in inet_ehash_nolisten() argument
691 bool ok = inet_ehash_insert(sk, osk, found_dup_sk); in inet_ehash_nolisten()
694 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in inet_ehash_nolisten()
696 this_cpu_inc(*sk->sk_prot->orphan_count); in inet_ehash_nolisten()
697 inet_sk_set_state(sk, TCP_CLOSE); in inet_ehash_nolisten()
698 sock_set_flag(sk, SOCK_DEAD); in inet_ehash_nolisten()
699 inet_csk_destroy_sock(sk); in inet_ehash_nolisten()
705 static int inet_reuseport_add_sock(struct sock *sk, in inet_reuseport_add_sock() argument
708 struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash; in inet_reuseport_add_sock()
711 kuid_t uid = sock_i_uid(sk); in inet_reuseport_add_sock()
714 if (sk2 != sk && in inet_reuseport_add_sock()
715 sk2->sk_family == sk->sk_family && in inet_reuseport_add_sock()
716 ipv6_only_sock(sk2) == ipv6_only_sock(sk) && in inet_reuseport_add_sock()
717 sk2->sk_bound_dev_if == sk->sk_bound_dev_if && in inet_reuseport_add_sock()
720 inet_rcv_saddr_equal(sk, sk2, false)) in inet_reuseport_add_sock()
721 return reuseport_add_sock(sk, sk2, in inet_reuseport_add_sock()
722 inet_rcv_saddr_any(sk)); in inet_reuseport_add_sock()
725 return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); in inet_reuseport_add_sock()
728 int __inet_hash(struct sock *sk, struct sock *osk) in __inet_hash() argument
730 struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); in __inet_hash()
734 if (sk->sk_state != TCP_LISTEN) { in __inet_hash()
736 inet_ehash_nolisten(sk, osk, NULL); in __inet_hash()
740 WARN_ON(!sk_unhashed(sk)); in __inet_hash()
741 ilb2 = inet_lhash2_bucket_sk(hashinfo, sk); in __inet_hash()
744 if (sk->sk_reuseport) { in __inet_hash()
745 err = inet_reuseport_add_sock(sk, ilb2); in __inet_hash()
749 sock_set_flag(sk, SOCK_RCU_FREE); in __inet_hash()
750 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && in __inet_hash()
751 sk->sk_family == AF_INET6) in __inet_hash()
752 __sk_nulls_add_node_tail_rcu(sk, &ilb2->nulls_head); in __inet_hash()
754 __sk_nulls_add_node_rcu(sk, &ilb2->nulls_head); in __inet_hash()
755 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in __inet_hash()
763 int inet_hash(struct sock *sk) in inet_hash() argument
767 if (sk->sk_state != TCP_CLOSE) in inet_hash()
768 err = __inet_hash(sk, NULL); in inet_hash()
774 void inet_unhash(struct sock *sk) in inet_unhash() argument
776 struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); in inet_unhash()
778 if (sk_unhashed(sk)) in inet_unhash()
781 if (sk->sk_state == TCP_LISTEN) { in inet_unhash()
784 ilb2 = inet_lhash2_bucket_sk(hashinfo, sk); in inet_unhash()
789 if (sk_unhashed(sk)) { in inet_unhash()
794 if (rcu_access_pointer(sk->sk_reuseport_cb)) in inet_unhash()
795 reuseport_stop_listen_sock(sk); in inet_unhash()
797 __sk_nulls_del_node_init_rcu(sk); in inet_unhash()
798 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in inet_unhash()
801 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash); in inet_unhash()
804 if (sk_unhashed(sk)) { in inet_unhash()
808 __sk_nulls_del_node_init_rcu(sk); in inet_unhash()
809 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in inet_unhash()
817 int l3mdev, const struct sock *sk) in inet_bind2_bucket_match() argument
823 return inet_bind2_bucket_addr_match(tb, sk); in inet_bind2_bucket_match()
827 unsigned short port, int l3mdev, const struct sock *sk) in inet_bind2_bucket_match_addr_any() argument
840 if (sk->sk_family == AF_INET6 && in inet_bind2_bucket_match_addr_any()
841 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) in inet_bind2_bucket_match_addr_any()
850 unsigned short port, int l3mdev, const struct sock *sk) in inet_bind2_bucket_find() argument
855 if (inet_bind2_bucket_match(bhash2, net, port, l3mdev, sk)) in inet_bind2_bucket_find()
862 inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port) in inet_bhash2_addr_any_hashbucket() argument
864 struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); in inet_bhash2_addr_any_hashbucket()
868 if (sk->sk_family == AF_INET6) in inet_bhash2_addr_any_hashbucket()
877 static void inet_update_saddr(struct sock *sk, void *saddr, int family) in inet_update_saddr() argument
880 inet_sk(sk)->inet_saddr = *(__be32 *)saddr; in inet_update_saddr()
881 sk_rcv_saddr_set(sk, inet_sk(sk)->inet_saddr); in inet_update_saddr()
885 sk->sk_v6_rcv_saddr = *(struct in6_addr *)saddr; in inet_update_saddr()
890 static int __inet_bhash2_update_saddr(struct sock *sk, void *saddr, int family, bool reset) in __inet_bhash2_update_saddr() argument
892 struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); in __inet_bhash2_update_saddr()
895 int l3mdev = inet_sk_bound_l3mdev(sk); in __inet_bhash2_update_saddr()
896 int port = inet_sk(sk)->inet_num; in __inet_bhash2_update_saddr()
897 struct net *net = sock_net(sk); in __inet_bhash2_update_saddr()
900 if (!inet_csk(sk)->icsk_bind2_hash) { in __inet_bhash2_update_saddr()
903 inet_reset_saddr(sk); in __inet_bhash2_update_saddr()
905 inet_update_saddr(sk, saddr, family); in __inet_bhash2_update_saddr()
922 inet_put_port(sk); in __inet_bhash2_update_saddr()
923 inet_reset_saddr(sk); in __inet_bhash2_update_saddr()
931 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); in __inet_bhash2_update_saddr()
939 __sk_del_bind_node(sk); in __inet_bhash2_update_saddr()
940 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, inet_csk(sk)->icsk_bind2_hash); in __inet_bhash2_update_saddr()
944 inet_reset_saddr(sk); in __inet_bhash2_update_saddr()
946 inet_update_saddr(sk, saddr, family); in __inet_bhash2_update_saddr()
948 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); in __inet_bhash2_update_saddr()
951 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); in __inet_bhash2_update_saddr()
954 inet_bind2_bucket_init(tb2, net, head2, inet_csk(sk)->icsk_bind_hash, sk); in __inet_bhash2_update_saddr()
956 inet_csk(sk)->icsk_bind2_hash = tb2; in __inet_bhash2_update_saddr()
957 sk_add_bind_node(sk, &tb2->owners); in __inet_bhash2_update_saddr()
968 int inet_bhash2_update_saddr(struct sock *sk, void *saddr, int family) in inet_bhash2_update_saddr() argument
970 return __inet_bhash2_update_saddr(sk, saddr, family, false); in inet_bhash2_update_saddr()
974 void inet_bhash2_reset_saddr(struct sock *sk) in inet_bhash2_reset_saddr() argument
976 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) in inet_bhash2_reset_saddr()
977 __inet_bhash2_update_saddr(sk, NULL, 0, true); in inet_bhash2_reset_saddr()
995 struct sock *sk, u64 port_offset, in __inet_hash_connect() argument
1002 int port = inet_sk(sk)->inet_num; in __inet_hash_connect()
1003 struct net *net = sock_net(sk); in __inet_hash_connect()
1015 ret = check_established(death_row, sk, port, NULL); in __inet_hash_connect()
1020 l3mdev = inet_sk_bound_l3mdev(sk); in __inet_hash_connect()
1022 local_ports = inet_sk_get_local_port_range(sk, &low, &high); in __inet_hash_connect()
1062 if (!check_established(death_row, sk, in __inet_hash_connect()
1095 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); in __inet_hash_connect()
1098 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); in __inet_hash_connect()
1101 head2, tb, sk); in __inet_hash_connect()
1115 inet_bind_hash(sk, tb, tb2, port); in __inet_hash_connect()
1117 if (sk_unhashed(sk)) { in __inet_hash_connect()
1118 inet_sk(sk)->inet_sport = htons(port); in __inet_hash_connect()
1119 inet_ehash_nolisten(sk, (struct sock *)tw, NULL); in __inet_hash_connect()
1133 if (sk_hashed(sk)) { in __inet_hash_connect()
1134 spinlock_t *lock = inet_ehash_lockp(hinfo, sk->sk_hash); in __inet_hash_connect()
1136 sock_prot_inuse_add(net, sk->sk_prot, -1); in __inet_hash_connect()
1139 __sk_nulls_del_node_init_rcu(sk); in __inet_hash_connect()
1142 sk->sk_hash = 0; in __inet_hash_connect()
1143 inet_sk(sk)->inet_sport = 0; in __inet_hash_connect()
1144 inet_sk(sk)->inet_num = 0; in __inet_hash_connect()
1167 struct sock *sk) in inet_hash_connect() argument
1171 if (!inet_sk(sk)->inet_num) in inet_hash_connect()
1172 port_offset = inet_sk_port_offset(sk); in inet_hash_connect()
1173 return __inet_hash_connect(death_row, sk, port_offset, in inet_hash_connect()