Lines Matching +full:hw +full:- +full:gro
1 // SPDX-License-Identifier: GPL-2.0-or-later
20 #include <linux/bpf-cgroup.h>
55 #include <net/gro.h>
72 sk->sk_destruct = udpv6_destruct_sock;
73 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
91 lhash = (__force u32)laddr->s6_addr32[3];
103 ipv6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0);
106 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
113 &sk->sk_v6_rcv_saddr,
114 inet_sk(sk)->inet_num);
117 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
119 sk->sk_rcv_saddr, sk->sk_num,
120 sk->sk_daddr, sk->sk_dport);
123 &sk->sk_v6_rcv_saddr, sk->sk_num,
124 &sk->sk_v6_daddr, sk->sk_dport);
140 udp_sk(sk)->udp_port_hash != hnum ||
141 sk->sk_family != PF_INET6)
142 return -1;
144 if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
145 return -1;
150 if (inet->inet_dport) {
151 if (inet->inet_dport != sport)
152 return -1;
156 if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
157 if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))
158 return -1;
162 bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
165 return -1;
169 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
176 * udp6_lib_lookup1() - Simplified lookup using primary hash (destination port)
201 unsigned int slot = udp_hashfn(net, hnum, udptable->mask);
202 struct udp_hslot *hslot = &udptable->hash[slot];
206 sk_for_each_rcu(sk, &hslot->head) {
230 badness = -1;
231 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
242 if (sk->sk_state == TCP_ESTABLISHED) {
303 slot = hash4 & udptable->mask;
304 hslot4 = &udptable->hash4[slot];
307 udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
328 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
333 if (sk_unhashed(sk) || ipv6_addr_any(&sk->sk_v6_rcv_saddr))
336 hash = udp6_ehashfn(net, &sk->sk_v6_rcv_saddr, sk->sk_num,
337 &sk->sk_v6_daddr, sk->sk_dport);
365 /* Lookup connected or non-wildcard sockets */
369 if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
374 udptable == net->ipv4.udp_table) {
384 /* Got non-wildcard socket or error on first lookup */
415 return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
416 &iph->daddr, dport, inet6_iif(skb),
423 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
424 const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
425 struct net *net = dev_net(skb->dev);
430 return __udp6_lib_lookup(net, &iph->saddr, sport,
431 &iph->daddr, dport, iif,
432 sdif, net->ipv4.udp_table, NULL);
445 dif, 0, net->ipv4.udp_table, NULL);
446 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
459 return unlikely(inet6_is_jumbogram(skb)) ? skb->len : udp_skb_len(skb);
483 if (np->rxopt.bits.rxpmtu && READ_ONCE(np->rxpmtu))
494 if (copied > ulen - off)
495 copied = ulen - off;
497 msg->msg_flags |= MSG_TRUNC;
499 is_udp4 = (skb->protocol == htons(ETH_P_IP));
505 * coverage checksum (UDP-Lite), do it before the copy.
509 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
518 err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
523 if (err == -EINVAL)
540 if (msg->msg_name) {
541 DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name);
542 sin6->sin6_family = AF_INET6;
543 sin6->sin6_port = udp_hdr(skb)->source;
544 sin6->sin6_flowinfo = 0;
547 ipv6_addr_set_v4mapped(ip_hdr(skb)->saddr,
548 &sin6->sin6_addr);
549 sin6->sin6_scope_id = 0;
551 sin6->sin6_addr = ipv6_hdr(skb)->saddr;
552 sin6->sin6_scope_id =
553 ipv6_iface_scope_id(&sin6->sin6_addr,
566 if (np->rxopt.all)
574 if (np->rxopt.all)
582 skb_consume_udp(sk, skb, peeking ? -err : err);
586 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
595 msg->msg_flags &= ~MSG_TRUNC;
623 handler = encap->err_handler;
628 return -ENOENT;
673 lookup = READ_ONCE(up->encap_err_lookup);
680 sk = __udp6_lib_lookup(net, &hdr->daddr, uh->source,
681 &hdr->saddr, uh->dest,
686 lookup = READ_ONCE(up->encap_err_lookup);
708 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
709 const struct in6_addr *saddr = &hdr->saddr;
710 const struct in6_addr *daddr = seg6_get_daddr(skb, opt) ? : &hdr->daddr;
711 struct udphdr *uh = (struct udphdr *)(skb->data+offset);
716 struct net *net = dev_net(skb->dev);
718 sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source,
721 if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
730 sk = ERR_PTR(-ENOENT);
733 __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
748 if (READ_ONCE(np->pmtudisc) != IPV6_PMTUDISC_DONT)
754 READ_ONCE(sk->sk_mark),
764 if (udp_sk(sk)->encap_err_rcv)
765 udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest,
771 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
774 ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1));
777 sk->sk_err = err;
787 if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
801 if (rc == -ENOMEM) {
813 return -1;
824 dev_net(skb->dev)->ipv4.udp_table);
840 READ_ONCE(up->encap_type)) {
847 * up->encap_rcv() returns the following value:
851 * <0 if skb should be resubmitted as proto -N
855 encap_rcv = READ_ONCE(up->encap_rcv);
868 return -ret;
872 /* FALLTHROUGH -- it's a UDP Packet */
876 * UDP-Lite specific tests, ignored on UDP sockets (see net/ipv4/udp.c).
878 if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
879 u16 pcrlen = READ_ONCE(up->pcrlen);
883 UDP_SKB_CB(skb)->cscov, skb->len);
886 if (UDP_SKB_CB(skb)->cscov < pcrlen) {
888 UDP_SKB_CB(skb)->cscov, pcrlen);
893 prefetch(&sk->sk_rmem_alloc);
894 if (rcu_access_pointer(sk->sk_filter) &&
914 return -1;
925 __skb_push(skb, -skb_mac_offset(skb));
933 ip6_protocol_deliver_rcu(dev_net(skb->dev), skb, ret,
949 if (udp_sk(sk)->udp_port_hash != hnum ||
950 sk->sk_family != PF_INET6 ||
951 (inet->inet_dport && inet->inet_dport != rmt_port) ||
952 (!ipv6_addr_any(&sk->sk_v6_daddr) &&
953 !ipv6_addr_equal(&sk->sk_v6_daddr, rmt_addr)) ||
954 !udp_sk_bound_dev_eq(net, READ_ONCE(sk->sk_bound_dev_if), dif, sdif) ||
955 (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&
956 !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, loc_addr)))
968 net_dbg_ratelimited("IPv6: udp checksum is 0 for [%pI6c]:%u->[%pI6c]:%u\n",
969 &ipv6_hdr(skb)->saddr, ntohs(udp_hdr(skb)->source),
970 &ipv6_hdr(skb)->daddr, ntohs(udp_hdr(skb)->dest));
983 unsigned short hnum = ntohs(uh->dest);
986 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
994 udptable->mask;
995 hash2 = ipv6_portaddr_hash(net, daddr, hnum) & udptable->mask;
997 hslot = &udptable->hash2[hash2].hslot;
1001 sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
1002 if (!__udp_v6_is_mcast_sock(net, sk, uh->dest, daddr,
1003 uh->source, saddr, dif, sdif,
1009 if (!uh->check && !udp_get_no_check6_rx(sk))
1049 sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
1060 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
1076 struct net *net = dev_net(skb->dev);
1085 saddr = &ipv6_hdr(skb)->saddr;
1086 daddr = &ipv6_hdr(skb)->daddr;
1089 ulen = ntohs(uh->len);
1090 if (ulen > skb->len)
1098 ulen = skb->len;
1103 if (ulen < skb->len) {
1106 saddr = &ipv6_hdr(skb)->saddr;
1107 daddr = &ipv6_hdr(skb)->daddr;
1116 sk = inet6_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
1125 if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
1128 if (!uh->check && !udp_get_no_check6_rx(sk)) {
1148 sk = __udp6_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
1150 if (!uh->check && !udp_get_no_check6_rx(sk))
1157 if (!uh->check)
1177 proto == IPPROTO_UDPLITE ? "-Lite" : "",
1178 saddr, ntohs(uh->source),
1179 ulen, skb->len,
1180 daddr, ntohs(uh->dest));
1201 struct udp_table *udptable = net->ipv4.udp_table;
1212 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
1213 if (sk->sk_state == TCP_ESTABLISHED &&
1224 struct net *net = dev_net(skb->dev);
1228 int dif = skb->dev->ifindex;
1237 if (skb->pkt_type == PACKET_HOST)
1238 sk = __udp6_lib_demux_lookup(net, uh->dest,
1239 &ipv6_hdr(skb)->daddr,
1240 uh->source, &ipv6_hdr(skb)->saddr,
1248 skb->sk = sk;
1250 skb->destructor = sock_pfree;
1251 dst = rcu_dereference(sk->sk_rx_dst);
1254 dst = dst_check(dst, sk->sk_rx_dst_cookie);
1266 return __udp6_lib_rcv(skb, dev_net(skb->dev)->ipv4.udp_table, IPPROTO_UDP);
1276 if (up->pending == AF_INET)
1278 else if (up->pending) {
1279 up->len = 0;
1280 WRITE_ONCE(up->pending, 0);
1289 return -EINVAL;
1294 if (uaddr->sa_family == AF_INET) {
1296 return -EAFNOSUPPORT;
1301 return -EINVAL;
1319 * udp6_hwcsum_outgoing - handle outgoing HW checksumming
1321 * @skb: sk_buff containing the filled-in UDP header
1333 struct sk_buff *frags = skb_shinfo(skb)->frag_list;
1338 skb->csum_start = skb_transport_header(skb) - skb->head;
1339 skb->csum_offset = offsetof(struct udphdr, check);
1340 uh->check = ~csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, 0);
1343 * HW-checksum won't work as there are two or more
1348 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
1349 csum = skb->csum;
1351 skb->ip_summed = CHECKSUM_NONE;
1354 csum = csum_add(csum, frags->csum);
1355 } while ((frags = frags->next));
1357 uh->check = csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP,
1359 if (uh->check == 0)
1360 uh->check = CSUM_MANGLED_0;
1371 struct sock *sk = skb->sk;
1377 int len = skb->len - offset;
1378 int datalen = len - sizeof(*uh);
1384 uh->source = fl6->fl6_sport;
1385 uh->dest = fl6->fl6_dport;
1386 uh->len = htons(len);
1387 uh->check = 0;
1389 if (cork->gso_size) {
1393 if (hlen + min(datalen, cork->gso_size) > cork->fragsize) {
1395 return -EMSGSIZE;
1397 if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
1399 return -EINVAL;
1403 return -EINVAL;
1407 return -EIO;
1410 if (datalen > cork->gso_size) {
1411 skb_shinfo(skb)->gso_size = cork->gso_size;
1412 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
1413 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
1414 cork->gso_size);
1424 skb->ip_summed = CHECKSUM_NONE;
1426 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
1428 udp6_hwcsum_outgoing(sk, skb, &fl6->saddr, &fl6->daddr, len);
1433 /* add protocol-dependent pseudo-header */
1434 uh->check = csum_ipv6_magic(&fl6->saddr, &fl6->daddr,
1435 len, fl6->flowi6_proto, csum);
1436 if (uh->check == 0)
1437 uh->check = CSUM_MANGLED_0;
1442 if (err == -ENOBUFS && !inet6_test_bit(RECVERR6, sk)) {
1460 if (up->pending == AF_INET)
1467 err = udp_v6_send_skb(skb, &inet_sk(sk)->cork.fl.u.ip6,
1468 &inet_sk(sk)->cork.base);
1470 up->len = 0;
1471 WRITE_ONCE(up->pending, 0);
1481 DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name);
1490 int addr_len = msg->msg_namelen;
1493 int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
1499 ipc6.gso_size = READ_ONCE(up->gso_size);
1504 return -EINVAL;
1506 switch (sin6->sin6_family) {
1509 return -EINVAL;
1510 daddr = &sin6->sin6_addr;
1512 ipv6_addr_v4mapped(&np->saddr))
1519 msg->msg_name = sin6 = NULL;
1520 msg->msg_namelen = addr_len = 0;
1524 return -EINVAL;
1526 } else if (!READ_ONCE(up->pending)) {
1527 if (sk->sk_state != TCP_ESTABLISHED)
1528 return -EDESTADDRREQ;
1529 daddr = &sk->sk_v6_daddr;
1537 sin.sin_port = sin6 ? sin6->sin6_port : inet->inet_dport;
1538 sin.sin_addr.s_addr = daddr->s6_addr32[3];
1539 msg->msg_name = &sin;
1540 msg->msg_namelen = sizeof(sin);
1543 -ENETUNREACH : udp_sendmsg(sk, msg, len);
1544 msg->msg_name = sin6;
1545 msg->msg_namelen = addr_len;
1553 if (len > INT_MAX - sizeof(struct udphdr))
1554 return -EMSGSIZE;
1557 if (READ_ONCE(up->pending)) {
1558 if (READ_ONCE(up->pending) == AF_INET)
1565 if (likely(up->pending)) {
1566 if (unlikely(up->pending != AF_INET6)) {
1568 return -EAFNOSUPPORT;
1580 if (sin6->sin6_port == 0)
1581 return -EINVAL;
1583 fl6->fl6_dport = sin6->sin6_port;
1584 daddr = &sin6->sin6_addr;
1587 fl6->flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
1588 if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
1589 flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
1591 return -EINVAL;
1597 * sk->sk_dst_cache.
1599 if (sk->sk_state == TCP_ESTABLISHED &&
1600 ipv6_addr_equal(daddr, &sk->sk_v6_daddr))
1601 daddr = &sk->sk_v6_daddr;
1604 sin6->sin6_scope_id &&
1606 fl6->flowi6_oif = sin6->sin6_scope_id;
1608 if (sk->sk_state != TCP_ESTABLISHED)
1609 return -EDESTADDRREQ;
1611 fl6->fl6_dport = inet->inet_dport;
1612 daddr = &sk->sk_v6_daddr;
1613 fl6->flowlabel = np->flow_label;
1617 if (!fl6->flowi6_oif)
1618 fl6->flowi6_oif = READ_ONCE(sk->sk_bound_dev_if);
1620 if (!fl6->flowi6_oif)
1621 fl6->flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
1623 fl6->flowi6_uid = sk_uid(sk);
1625 if (msg->msg_controllen) {
1628 opt->tot_len = sizeof(*opt);
1641 if ((fl6->flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
1642 flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
1644 return -EINVAL;
1646 if (!(opt->opt_nflen|opt->opt_flen))
1658 fl6->flowi6_proto = sk->sk_protocol;
1659 fl6->flowi6_mark = ipc6.sockc.mark;
1660 fl6->daddr = *daddr;
1661 if (ipv6_addr_any(&fl6->saddr) && !ipv6_addr_any(&np->saddr))
1662 fl6->saddr = np->saddr;
1663 fl6->fl6_sport = inet->inet_sport;
1669 &fl6->saddr);
1673 if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
1674 /* BPF program rewrote IPv6-only by IPv4-mapped
1677 err = -ENOTSUPP;
1680 if (sin6->sin6_port == 0) {
1682 err = -EINVAL;
1685 fl6->fl6_dport = sin6->sin6_port;
1686 fl6->daddr = sin6->sin6_addr;
1690 if (ipv6_addr_any(&fl6->daddr))
1691 fl6->daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */
1697 if (!fl6->flowi6_oif && ipv6_addr_is_multicast(&fl6->daddr)) {
1698 fl6->flowi6_oif = READ_ONCE(np->mcast_oif);
1700 } else if (!fl6->flowi6_oif)
1701 fl6->flowi6_oif = READ_ONCE(np->ucast_oif);
1705 fl6->flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6->flowlabel);
1717 if (msg->msg_flags&MSG_CONFIRM)
1721 /* Lockless fast path for the non-corking case */
1728 msg->msg_flags, &cork);
1737 if (unlikely(up->pending)) {
1739 /* ... which is an evident application bug. --ANK */
1743 err = -EINVAL;
1747 WRITE_ONCE(up->pending, AF_INET6);
1750 up->len += ulen;
1753 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1758 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1759 WRITE_ONCE(up->pending, 0);
1779 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1786 if (msg->msg_flags & MSG_PROBE)
1787 dst_confirm_neigh(dst, &fl6->daddr);
1788 if (!(msg->msg_flags&MSG_PROBE) || len)
1797 struct sock *sk = sock->sk;
1800 if (!READ_ONCE(up->pending) || udp_test_bit(CORK, sk))
1804 if (up->pending && !udp_test_bit(CORK, sk))
1820 if (up->encap_type) {
1822 encap_destroy = READ_ONCE(up->encap_destroy);
1856 /* ------------------------------------------------------------------------ */
1863 int bucket = ((struct udp_iter_state *)seq->private)->bucket;
1865 __u16 srcp = ntohs(inet->inet_sport);
1866 __u16 destp = ntohs(inet->inet_dport);
1888 if (!proc_create_net_data("udp6", 0444, net->proc_net, &udp6_seq_ops,
1890 return -ENOMEM;
1896 remove_proc_entry("udp6", net->proc_net);
1900 /* ------------------------------------------------------------------------ */