udp.c (b7e2672d1a23a53bd2657704bf94a8dc8880cc49) udp.c (64f0f5d18a47c703c85576375cc010e83dac6a48)
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
8 * Authors: Ross Biro

--- 788 unchanged lines hidden (view full) ---

797 uh->source = inet->inet_sport;
798 uh->dest = fl4->fl4_dport;
799 uh->len = htons(len);
800 uh->check = 0;
801
802 if (is_udplite) /* UDP-Lite */
803 csum = udplite_csum(skb);
804
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
8 * Authors: Ross Biro

--- 788 unchanged lines hidden (view full) ---

797 uh->source = inet->inet_sport;
798 uh->dest = fl4->fl4_dport;
799 uh->len = htons(len);
800 uh->check = 0;
801
802 if (is_udplite) /* UDP-Lite */
803 csum = udplite_csum(skb);
804
805 else if (sk->sk_no_check_tx) { /* UDP csum disabled */
805 else if (sk->sk_no_check_tx && !skb_is_gso(skb)) { /* UDP csum off */
806
807 skb->ip_summed = CHECKSUM_NONE;
808 goto send;
809
810 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
811
812 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
813 goto send;

--- 344 unchanged lines hidden (view full) ---

1158 ret = udp_push_pending_frames(sk);
1159 if (!ret)
1160 ret = size;
1161out:
1162 release_sock(sk);
1163 return ret;
1164}
1165
806
807 skb->ip_summed = CHECKSUM_NONE;
808 goto send;
809
810 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
811
812 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
813 goto send;

--- 344 unchanged lines hidden (view full) ---

1158 ret = udp_push_pending_frames(sk);
1159 if (!ret)
1160 ret = size;
1161out:
1162 release_sock(sk);
1163 return ret;
1164}
1165
1166#if BITS_PER_LONG == 64
1166#define UDP_SKB_IS_STATELESS 0x80000000
1167
1167static void udp_set_dev_scratch(struct sk_buff *skb)
1168{
1168static void udp_set_dev_scratch(struct sk_buff *skb)
1169{
1169 struct udp_dev_scratch *scratch;
1170 struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
1170
1171 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
1171
1172 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
1172 scratch = (struct udp_dev_scratch *)&skb->dev_scratch;
1173 scratch->truesize = skb->truesize;
1173 scratch->_tsize_state = skb->truesize;
1174#if BITS_PER_LONG == 64
1174 scratch->len = skb->len;
1175 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1176 scratch->is_linear = !skb_is_nonlinear(skb);
1175 scratch->len = skb->len;
1176 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1177 scratch->is_linear = !skb_is_nonlinear(skb);
1178#endif
1179 if (likely(!skb->_skb_refdst))
1180 scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
1177}
1178
1179static int udp_skb_truesize(struct sk_buff *skb)
1180{
1181}
1182
1183static int udp_skb_truesize(struct sk_buff *skb)
1184{
1181 return ((struct udp_dev_scratch *)&skb->dev_scratch)->truesize;
1185 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
1182}
1186}
1183#else
1184static void udp_set_dev_scratch(struct sk_buff *skb)
1185{
1186 skb->dev_scratch = skb->truesize;
1187}
1188
1187
1189static int udp_skb_truesize(struct sk_buff *skb)
1188static bool udp_skb_has_head_state(struct sk_buff *skb)
1190{
1189{
1191 return skb->dev_scratch;
1190 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
1192}
1191}
1193#endif
1194
1195/* fully reclaim rmem/fwd memory allocated for skb */
1196static void udp_rmem_release(struct sock *sk, int size, int partial,
1197 bool rx_queue_lock_held)
1198{
1199 struct udp_sock *up = udp_sk(sk);
1200 struct sk_buff_head *sk_queue;
1201 int amt;

--- 181 unchanged lines hidden (view full) ---

1383{
1384 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1385 bool slow = lock_sock_fast(sk);
1386
1387 sk_peek_offset_bwd(sk, len);
1388 unlock_sock_fast(sk, slow);
1389 }
1390
1192
1193/* fully reclaim rmem/fwd memory allocated for skb */
1194static void udp_rmem_release(struct sock *sk, int size, int partial,
1195 bool rx_queue_lock_held)
1196{
1197 struct udp_sock *up = udp_sk(sk);
1198 struct sk_buff_head *sk_queue;
1199 int amt;

--- 181 unchanged lines hidden (view full) ---

1381{
1382 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1383 bool slow = lock_sock_fast(sk);
1384
1385 sk_peek_offset_bwd(sk, len);
1386 unlock_sock_fast(sk, slow);
1387 }
1388
1391 /* we cleared the head states previously only if the skb lacks any IP
1392 * options, see __udp_queue_rcv_skb().
1389 /* In the more common cases we cleared the head states previously,
1390 * see __udp_queue_rcv_skb().
1393 */
1391 */
1394 if (unlikely(IPCB(skb)->opt.optlen > 0))
1392 if (unlikely(udp_skb_has_head_state(skb)))
1395 skb_release_head_state(skb);
1396 consume_stateless_skb(skb);
1397}
1398EXPORT_SYMBOL_GPL(skb_consume_udp);
1399
1400static struct sk_buff *__first_packet_length(struct sock *sk,
1401 struct sk_buff_head *rcvq,
1402 int *total)

--- 168 unchanged lines hidden (view full) ---

1571 int err;
1572 int is_udplite = IS_UDPLITE(sk);
1573 bool checksum_valid = false;
1574
1575 if (flags & MSG_ERRQUEUE)
1576 return ip_recv_error(sk, msg, len, addr_len);
1577
1578try_again:
1393 skb_release_head_state(skb);
1394 consume_stateless_skb(skb);
1395}
1396EXPORT_SYMBOL_GPL(skb_consume_udp);
1397
1398static struct sk_buff *__first_packet_length(struct sock *sk,
1399 struct sk_buff_head *rcvq,
1400 int *total)

--- 168 unchanged lines hidden (view full) ---

1569 int err;
1570 int is_udplite = IS_UDPLITE(sk);
1571 bool checksum_valid = false;
1572
1573 if (flags & MSG_ERRQUEUE)
1574 return ip_recv_error(sk, msg, len, addr_len);
1575
1576try_again:
1579 peeking = off = sk_peek_offset(sk, flags);
1577 peeking = flags & MSG_PEEK;
1578 off = sk_peek_offset(sk, flags);
1580 skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
1581 if (!skb)
1582 return err;
1583
1584 ulen = udp_skb_len(skb);
1585 copied = len;
1586 if (copied > ulen - off)
1587 copied = ulen - off;

--- 191 unchanged lines hidden (view full) ---

1779 if (inet_sk(sk)->inet_daddr) {
1780 sock_rps_save_rxhash(sk, skb);
1781 sk_mark_napi_id(sk, skb);
1782 sk_incoming_cpu_update(sk);
1783 } else {
1784 sk_mark_napi_id_once(sk, skb);
1785 }
1786
1579 skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
1580 if (!skb)
1581 return err;
1582
1583 ulen = udp_skb_len(skb);
1584 copied = len;
1585 if (copied > ulen - off)
1586 copied = ulen - off;

--- 191 unchanged lines hidden (view full) ---

1778 if (inet_sk(sk)->inet_daddr) {
1779 sock_rps_save_rxhash(sk, skb);
1780 sk_mark_napi_id(sk, skb);
1781 sk_incoming_cpu_update(sk);
1782 } else {
1783 sk_mark_napi_id_once(sk, skb);
1784 }
1785
1787 /* At recvmsg() time we need skb->dst to process IP options-related
1788 * cmsg, elsewhere can we clear all pending head states while they are
1789 * hot in the cache
1786 /* At recvmsg() time we may access skb->dst or skb->sp depending on
1787 * the IP options and the cmsg flags, elsewhere can we clear all
1788 * pending head states while they are hot in the cache
1790 */
1789 */
1791 if (likely(IPCB(skb)->opt.optlen == 0))
1790 if (likely(IPCB(skb)->opt.optlen == 0 && !skb_sec_path(skb)))
1792 skb_release_head_state(skb);
1793
1794 rc = __udp_enqueue_schedule_skb(sk, skb);
1795 if (rc < 0) {
1796 int is_udplite = IS_UDPLITE(sk);
1797
1798 /* Note that an ENOMEM error is charged twice */
1799 if (rc == -ENOMEM)

--- 125 unchanged lines hidden (view full) ---

1925 atomic_inc(&sk->sk_drops);
1926 kfree_skb(skb);
1927 return -1;
1928}
1929
1930/* For TCP sockets, sk_rx_dst is protected by socket lock
1931 * For UDP, we use xchg() to guard against concurrent changes.
1932 */
1791 skb_release_head_state(skb);
1792
1793 rc = __udp_enqueue_schedule_skb(sk, skb);
1794 if (rc < 0) {
1795 int is_udplite = IS_UDPLITE(sk);
1796
1797 /* Note that an ENOMEM error is charged twice */
1798 if (rc == -ENOMEM)

--- 125 unchanged lines hidden (view full) ---

1924 atomic_inc(&sk->sk_drops);
1925 kfree_skb(skb);
1926 return -1;
1927}
1928
1929/* For TCP sockets, sk_rx_dst is protected by socket lock
1930 * For UDP, we use xchg() to guard against concurrent changes.
1931 */
1933static void udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
1932bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
1934{
1935 struct dst_entry *old;
1936
1937 if (dst_hold_safe(dst)) {
1938 old = xchg(&sk->sk_rx_dst, dst);
1939 dst_release(old);
1933{
1934 struct dst_entry *old;
1935
1936 if (dst_hold_safe(dst)) {
1937 old = xchg(&sk->sk_rx_dst, dst);
1938 dst_release(old);
1939 return old != dst;
1940 }
1940 }
1941 return false;
1941}
1942}
1943EXPORT_SYMBOL(udp_sk_rx_dst_set);
1942
1943/*
1944 * Multicasts and broadcasts go to each listener.
1945 *
1946 * Note: called only from the BH handler context.
1947 */
1948static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
1949 struct udphdr *uh,

--- 897 unchanged lines hidden --- 		1944
1945/*
1946 * Multicasts and broadcasts go to each listener.
1947 *
1948 * Note: called only from the BH handler context.
1949 */
1950static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
1951 struct udphdr *uh,

--- 897 unchanged lines hidden ---