12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * The User Datagram Protocol (UDP). 81da177e4SLinus Torvalds * 902c30a84SJesper Juhl * Authors: Ross Biro 101da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 111da177e4SLinus Torvalds * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 12113aa838SAlan Cox * Alan Cox, <alan@lxorguk.ukuu.org.uk> 131da177e4SLinus Torvalds * Hirokazu Takahashi, <taka@valinux.co.jp> 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * Fixes: 161da177e4SLinus Torvalds * Alan Cox : verify_area() calls 171da177e4SLinus Torvalds * Alan Cox : stopped close while in use off icmp 181da177e4SLinus Torvalds * messages. Not a fix but a botch that 191da177e4SLinus Torvalds * for udp at least is 'valid'. 201da177e4SLinus Torvalds * Alan Cox : Fixed icmp handling properly 211da177e4SLinus Torvalds * Alan Cox : Correct error for oversized datagrams 221da177e4SLinus Torvalds * Alan Cox : Tidied select() semantics. 231da177e4SLinus Torvalds * Alan Cox : udp_err() fixed properly, also now 241da177e4SLinus Torvalds * select and read wake correctly on errors 251da177e4SLinus Torvalds * Alan Cox : udp_send verify_area moved to avoid mem leak 261da177e4SLinus Torvalds * Alan Cox : UDP can count its memory 271da177e4SLinus Torvalds * Alan Cox : send to an unknown connection causes 281da177e4SLinus Torvalds * an ECONNREFUSED off the icmp, but 291da177e4SLinus Torvalds * does NOT close. 301da177e4SLinus Torvalds * Alan Cox : Switched to new sk_buff handlers. No more backlog! 311da177e4SLinus Torvalds * Alan Cox : Using generic datagram code. Even smaller and the PEEK 321da177e4SLinus Torvalds * bug no longer crashes it. 331da177e4SLinus Torvalds * Fred Van Kempen : Net2e support for sk->broadcast. 341da177e4SLinus Torvalds * Alan Cox : Uses skb_free_datagram 351da177e4SLinus Torvalds * Alan Cox : Added get/set sockopt support. 361da177e4SLinus Torvalds * Alan Cox : Broadcasting without option set returns EACCES. 371da177e4SLinus Torvalds * Alan Cox : No wakeup calls. Instead we now use the callbacks. 381da177e4SLinus Torvalds * Alan Cox : Use ip_tos and ip_ttl 391da177e4SLinus Torvalds * Alan Cox : SNMP Mibs 401da177e4SLinus Torvalds * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support. 411da177e4SLinus Torvalds * Matt Dillon : UDP length checks. 421da177e4SLinus Torvalds * Alan Cox : Smarter af_inet used properly. 431da177e4SLinus Torvalds * Alan Cox : Use new kernel side addressing. 441da177e4SLinus Torvalds * Alan Cox : Incorrect return on truncated datagram receive. 451da177e4SLinus Torvalds * Arnt Gulbrandsen : New udp_send and stuff 461da177e4SLinus Torvalds * Alan Cox : Cache last socket 471da177e4SLinus Torvalds * Alan Cox : Route cache 481da177e4SLinus Torvalds * Jon Peatfield : Minor efficiency fix to sendto(). 491da177e4SLinus Torvalds * Mike Shaver : RFC1122 checks. 501da177e4SLinus Torvalds * Alan Cox : Nonblocking error fix. 511da177e4SLinus Torvalds * Willy Konynenberg : Transparent proxying support. 521da177e4SLinus Torvalds * Mike McLagan : Routing by source 531da177e4SLinus Torvalds * David S. Miller : New socket lookup architecture. 541da177e4SLinus Torvalds * Last socket cache retained as it 551da177e4SLinus Torvalds * does have a high hit rate. 561da177e4SLinus Torvalds * Olaf Kirch : Don't linearise iovec on sendmsg. 571da177e4SLinus Torvalds * Andi Kleen : Some cleanups, cache destination entry 581da177e4SLinus Torvalds * for connect. 591da177e4SLinus Torvalds * Vitaly E. Lavrov : Transparent proxy revived after year coma. 601da177e4SLinus Torvalds * Melvin Smith : Check msg_name not msg_namelen in sendto(), 611da177e4SLinus Torvalds * return ENOTCONN for unconnected sockets (POSIX) 621da177e4SLinus Torvalds * Janos Farkas : don't deliver multi/broadcasts to a different 631da177e4SLinus Torvalds * bound-to-device socket 641da177e4SLinus Torvalds * Hirokazu Takahashi : HW checksumming for outgoing UDP 651da177e4SLinus Torvalds * datagrams. 661da177e4SLinus Torvalds * Hirokazu Takahashi : sendfile() on UDP works now. 671da177e4SLinus Torvalds * Arnaldo C. Melo : convert /proc/net/udp to seq_file 681da177e4SLinus Torvalds * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 691da177e4SLinus Torvalds * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind 701da177e4SLinus Torvalds * a single port at the same time. 711da177e4SLinus Torvalds * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support 72342f0234SJames Chapman * James Chapman : Add L2TP encapsulation type. 731da177e4SLinus Torvalds */ 741da177e4SLinus Torvalds 75afd46503SJoe Perches #define pr_fmt(fmt) "UDP: " fmt 76afd46503SJoe Perches 777c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 781da177e4SLinus Torvalds #include <asm/ioctls.h> 7957c8a661SMike Rapoport #include <linux/memblock.h> 808203efb3SEric Dumazet #include <linux/highmem.h> 818203efb3SEric Dumazet #include <linux/swap.h> 821da177e4SLinus Torvalds #include <linux/types.h> 831da177e4SLinus Torvalds #include <linux/fcntl.h> 841da177e4SLinus Torvalds #include <linux/module.h> 851da177e4SLinus Torvalds #include <linux/socket.h> 861da177e4SLinus Torvalds #include <linux/sockios.h> 8714c85021SArnaldo Carvalho de Melo #include <linux/igmp.h> 886e540309SShawn Bohrer #include <linux/inetdevice.h> 891da177e4SLinus Torvalds #include <linux/in.h> 901da177e4SLinus Torvalds #include <linux/errno.h> 911da177e4SLinus Torvalds #include <linux/timer.h> 921da177e4SLinus Torvalds #include <linux/mm.h> 931da177e4SLinus Torvalds #include <linux/inet.h> 941da177e4SLinus Torvalds #include <linux/netdevice.h> 955a0e3ad6STejun Heo #include <linux/slab.h> 96c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 971da177e4SLinus Torvalds #include <linux/skbuff.h> 981da177e4SLinus Torvalds #include <linux/proc_fs.h> 991da177e4SLinus Torvalds #include <linux/seq_file.h> 100457c4cbcSEric W. Biederman #include <net/net_namespace.h> 1011da177e4SLinus Torvalds #include <net/icmp.h> 102421b3885SShawn Bohrer #include <net/inet_hashtables.h> 103e7cc0824SStefano Brivio #include <net/ip_tunnels.h> 1041da177e4SLinus Torvalds #include <net/route.h> 1051da177e4SLinus Torvalds #include <net/checksum.h> 1061da177e4SLinus Torvalds #include <net/xfrm.h> 107296f7ea7SSatoru Moriya #include <trace/events/udp.h> 108447167bfSEric Dumazet #include <linux/static_key.h> 10922911fc5SEric Dumazet #include <trace/events/skb.h> 110076bb0c8SEliezer Tamir #include <net/busy_poll.h> 111ba4e58ecSGerrit Renker #include "udp_impl.h" 112e32ea7e7SCraig Gallek #include <net/sock_reuseport.h> 113217375a0SEric Dumazet #include <net/addrconf.h> 11460fb9567SPaolo Abeni #include <net/udp_tunnel.h> 1151da177e4SLinus Torvalds 116f86dcc5aSEric Dumazet struct udp_table udp_table __read_mostly; 117645ca708SEric Dumazet EXPORT_SYMBOL(udp_table); 1181da177e4SLinus Torvalds 1198d987e5cSEric Dumazet long sysctl_udp_mem[3] __read_mostly; 12095766fffSHideo Aoki EXPORT_SYMBOL(sysctl_udp_mem); 121c482c568SEric Dumazet 1228d987e5cSEric Dumazet atomic_long_t udp_memory_allocated; 12395766fffSHideo Aoki EXPORT_SYMBOL(udp_memory_allocated); 12495766fffSHideo Aoki 125f86dcc5aSEric Dumazet #define MAX_UDP_PORTS 65536 126f86dcc5aSEric Dumazet #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN) 12798322f22SEric Dumazet 128f24d43c0SEric Dumazet static int udp_lib_lport_inuse(struct net *net, __u16 num, 129645ca708SEric Dumazet const struct udp_hslot *hslot, 13098322f22SEric Dumazet unsigned long *bitmap, 131fe38d2a1SJosef Bacik struct sock *sk, unsigned int log) 13225030a7fSGerrit Renker { 133f24d43c0SEric Dumazet struct sock *sk2; 134ba418fa3STom Herbert kuid_t uid = sock_i_uid(sk); 13525030a7fSGerrit Renker 136ca065d0cSEric Dumazet sk_for_each(sk2, &hslot->head) { 137f24d43c0SEric Dumazet if (net_eq(sock_net(sk2), net) && 138f24d43c0SEric Dumazet sk2 != sk && 139d4cada4aSEric Dumazet (bitmap || udp_sk(sk2)->udp_port_hash == num) && 140f24d43c0SEric Dumazet (!sk2->sk_reuse || !sk->sk_reuse) && 1419d4fb27dSJoe Perches (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || 1429d4fb27dSJoe Perches sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 143fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, true)) { 144df560056SEric Garver if (sk2->sk_reuseport && sk->sk_reuseport && 145df560056SEric Garver !rcu_access_pointer(sk->sk_reuseport_cb) && 146df560056SEric Garver uid_eq(uid, sock_i_uid(sk2))) { 147df560056SEric Garver if (!bitmap) 148df560056SEric Garver return 0; 149df560056SEric Garver } else { 1504243cdc2SJoe Perches if (!bitmap) 151fc038410SDavid S. Miller return 1; 152df560056SEric Garver __set_bit(udp_sk(sk2)->udp_port_hash >> log, 153df560056SEric Garver bitmap); 154df560056SEric Garver } 1554243cdc2SJoe Perches } 15698322f22SEric Dumazet } 15725030a7fSGerrit Renker return 0; 15825030a7fSGerrit Renker } 15925030a7fSGerrit Renker 16030fff923SEric Dumazet /* 16130fff923SEric Dumazet * Note: we still hold spinlock of primary hash chain, so no other writer 16230fff923SEric Dumazet * can insert/delete a socket with local_port == num 16330fff923SEric Dumazet */ 16430fff923SEric Dumazet static int udp_lib_lport_inuse2(struct net *net, __u16 num, 16530fff923SEric Dumazet struct udp_hslot *hslot2, 166fe38d2a1SJosef Bacik struct sock *sk) 16730fff923SEric Dumazet { 16830fff923SEric Dumazet struct sock *sk2; 169ba418fa3STom Herbert kuid_t uid = sock_i_uid(sk); 17030fff923SEric Dumazet int res = 0; 17130fff923SEric Dumazet 17230fff923SEric Dumazet spin_lock(&hslot2->lock); 173ca065d0cSEric Dumazet udp_portaddr_for_each_entry(sk2, &hslot2->head) { 17430fff923SEric Dumazet if (net_eq(sock_net(sk2), net) && 17530fff923SEric Dumazet sk2 != sk && 17630fff923SEric Dumazet (udp_sk(sk2)->udp_port_hash == num) && 17730fff923SEric Dumazet (!sk2->sk_reuse || !sk->sk_reuse) && 1789d4fb27dSJoe Perches (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || 1799d4fb27dSJoe Perches sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 180fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, true)) { 181df560056SEric Garver if (sk2->sk_reuseport && sk->sk_reuseport && 182df560056SEric Garver !rcu_access_pointer(sk->sk_reuseport_cb) && 183df560056SEric Garver uid_eq(uid, sock_i_uid(sk2))) { 184df560056SEric Garver res = 0; 185df560056SEric Garver } else { 18630fff923SEric Dumazet res = 1; 187df560056SEric Garver } 18830fff923SEric Dumazet break; 18930fff923SEric Dumazet } 1904243cdc2SJoe Perches } 19130fff923SEric Dumazet spin_unlock(&hslot2->lock); 19230fff923SEric Dumazet return res; 19330fff923SEric Dumazet } 19430fff923SEric Dumazet 195fe38d2a1SJosef Bacik static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot) 196e32ea7e7SCraig Gallek { 197e32ea7e7SCraig Gallek struct net *net = sock_net(sk); 198e32ea7e7SCraig Gallek kuid_t uid = sock_i_uid(sk); 199e32ea7e7SCraig Gallek struct sock *sk2; 200e32ea7e7SCraig Gallek 201ca065d0cSEric Dumazet sk_for_each(sk2, &hslot->head) { 202e32ea7e7SCraig Gallek if (net_eq(sock_net(sk2), net) && 203e32ea7e7SCraig Gallek sk2 != sk && 204e32ea7e7SCraig Gallek sk2->sk_family == sk->sk_family && 205e32ea7e7SCraig Gallek ipv6_only_sock(sk2) == ipv6_only_sock(sk) && 206e32ea7e7SCraig Gallek (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) && 207e32ea7e7SCraig Gallek (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 208e32ea7e7SCraig Gallek sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && 209fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, false)) { 2102dbb9b9eSMartin KaFai Lau return reuseport_add_sock(sk, sk2, 2112dbb9b9eSMartin KaFai Lau inet_rcv_saddr_any(sk)); 212e32ea7e7SCraig Gallek } 213e32ea7e7SCraig Gallek } 214e32ea7e7SCraig Gallek 2152dbb9b9eSMartin KaFai Lau return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); 216e32ea7e7SCraig Gallek } 217e32ea7e7SCraig Gallek 21825030a7fSGerrit Renker /** 2196ba5a3c5SPavel Emelyanov * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6 22025030a7fSGerrit Renker * 22125030a7fSGerrit Renker * @sk: socket struct in question 22225030a7fSGerrit Renker * @snum: port number to look up 22325985edcSLucas De Marchi * @hash2_nulladdr: AF-dependent hash value in secondary hash chains, 22430fff923SEric Dumazet * with NULL address 22525030a7fSGerrit Renker */ 2266ba5a3c5SPavel Emelyanov int udp_lib_get_port(struct sock *sk, unsigned short snum, 22730fff923SEric Dumazet unsigned int hash2_nulladdr) 2281da177e4SLinus Torvalds { 229512615b6SEric Dumazet struct udp_hslot *hslot, *hslot2; 230645ca708SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 23125030a7fSGerrit Renker int error = 1; 2323b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk); 2331da177e4SLinus Torvalds 23432c1da70SStephen Hemminger if (!snum) { 2359088c560SEric Dumazet int low, high, remaining; 23695c96174SEric Dumazet unsigned int rand; 23798322f22SEric Dumazet unsigned short first, last; 23898322f22SEric Dumazet DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN); 2391da177e4SLinus Torvalds 2400bbf87d8SEric W. Biederman inet_get_local_port_range(net, &low, &high); 241a25de534SAnton Arapov remaining = (high - low) + 1; 242227b60f5SStephen Hemminger 24363862b5bSAruna-Hewapathirane rand = prandom_u32(); 2448fc54f68SDaniel Borkmann first = reciprocal_scale(rand, remaining) + low; 24598322f22SEric Dumazet /* 24698322f22SEric Dumazet * force rand to be an odd multiple of UDP_HTABLE_SIZE 24798322f22SEric Dumazet */ 248f86dcc5aSEric Dumazet rand = (rand | 1) * (udptable->mask + 1); 2495781b235SEric Dumazet last = first + udptable->mask + 1; 2505781b235SEric Dumazet do { 251f86dcc5aSEric Dumazet hslot = udp_hashslot(udptable, net, first); 25298322f22SEric Dumazet bitmap_zero(bitmap, PORTS_PER_CHAIN); 253645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 25498322f22SEric Dumazet udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, 255fe38d2a1SJosef Bacik udptable->log); 25698322f22SEric Dumazet 25798322f22SEric Dumazet snum = first; 25898322f22SEric Dumazet /* 25998322f22SEric Dumazet * Iterate on all possible values of snum for this hash. 26098322f22SEric Dumazet * Using steps of an odd multiple of UDP_HTABLE_SIZE 26198322f22SEric Dumazet * give us randomization and full range coverage. 26298322f22SEric Dumazet */ 2639088c560SEric Dumazet do { 26498322f22SEric Dumazet if (low <= snum && snum <= high && 265e3826f1eSAmerigo Wang !test_bit(snum >> udptable->log, bitmap) && 266122ff243SWANG Cong !inet_is_local_reserved_port(net, snum)) 26798322f22SEric Dumazet goto found; 26898322f22SEric Dumazet snum += rand; 26998322f22SEric Dumazet } while (snum != first); 27098322f22SEric Dumazet spin_unlock_bh(&hslot->lock); 271df560056SEric Garver cond_resched(); 2725781b235SEric Dumazet } while (++first != last); 27398322f22SEric Dumazet goto fail; 274645ca708SEric Dumazet } else { 275f86dcc5aSEric Dumazet hslot = udp_hashslot(udptable, net, snum); 276645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 27730fff923SEric Dumazet if (hslot->count > 10) { 27830fff923SEric Dumazet int exist; 27930fff923SEric Dumazet unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; 28030fff923SEric Dumazet 28130fff923SEric Dumazet slot2 &= udptable->mask; 28230fff923SEric Dumazet hash2_nulladdr &= udptable->mask; 28330fff923SEric Dumazet 28430fff923SEric Dumazet hslot2 = udp_hashslot2(udptable, slot2); 28530fff923SEric Dumazet if (hslot->count < hslot2->count) 28630fff923SEric Dumazet goto scan_primary_hash; 28730fff923SEric Dumazet 288fe38d2a1SJosef Bacik exist = udp_lib_lport_inuse2(net, snum, hslot2, sk); 28930fff923SEric Dumazet if (!exist && (hash2_nulladdr != slot2)) { 29030fff923SEric Dumazet hslot2 = udp_hashslot2(udptable, hash2_nulladdr); 29130fff923SEric Dumazet exist = udp_lib_lport_inuse2(net, snum, hslot2, 292fe38d2a1SJosef Bacik sk); 29330fff923SEric Dumazet } 29430fff923SEric Dumazet if (exist) 29530fff923SEric Dumazet goto fail_unlock; 29630fff923SEric Dumazet else 29730fff923SEric Dumazet goto found; 29830fff923SEric Dumazet } 29930fff923SEric Dumazet scan_primary_hash: 300fe38d2a1SJosef Bacik if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0)) 301645ca708SEric Dumazet goto fail_unlock; 302645ca708SEric Dumazet } 30398322f22SEric Dumazet found: 304c720c7e8SEric Dumazet inet_sk(sk)->inet_num = snum; 305d4cada4aSEric Dumazet udp_sk(sk)->udp_port_hash = snum; 306d4cada4aSEric Dumazet udp_sk(sk)->udp_portaddr_hash ^= snum; 3071da177e4SLinus Torvalds if (sk_unhashed(sk)) { 308e32ea7e7SCraig Gallek if (sk->sk_reuseport && 309fe38d2a1SJosef Bacik udp_reuseport_add_sock(sk, hslot)) { 310e32ea7e7SCraig Gallek inet_sk(sk)->inet_num = 0; 311e32ea7e7SCraig Gallek udp_sk(sk)->udp_port_hash = 0; 312e32ea7e7SCraig Gallek udp_sk(sk)->udp_portaddr_hash ^= snum; 313e32ea7e7SCraig Gallek goto fail_unlock; 314e32ea7e7SCraig Gallek } 315e32ea7e7SCraig Gallek 316ca065d0cSEric Dumazet sk_add_node_rcu(sk, &hslot->head); 317fdcc8aa9SEric Dumazet hslot->count++; 318c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 319512615b6SEric Dumazet 320512615b6SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 321512615b6SEric Dumazet spin_lock(&hslot2->lock); 322d894ba18SCraig Gallek if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && 323d894ba18SCraig Gallek sk->sk_family == AF_INET6) 3241602f49bSDavid S. Miller hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node, 325d894ba18SCraig Gallek &hslot2->head); 326d894ba18SCraig Gallek else 327ca065d0cSEric Dumazet hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, 328512615b6SEric Dumazet &hslot2->head); 329512615b6SEric Dumazet hslot2->count++; 330512615b6SEric Dumazet spin_unlock(&hslot2->lock); 3311da177e4SLinus Torvalds } 332ca065d0cSEric Dumazet sock_set_flag(sk, SOCK_RCU_FREE); 33325030a7fSGerrit Renker error = 0; 334645ca708SEric Dumazet fail_unlock: 335645ca708SEric Dumazet spin_unlock_bh(&hslot->lock); 3361da177e4SLinus Torvalds fail: 33725030a7fSGerrit Renker return error; 3381da177e4SLinus Torvalds } 339c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_get_port); 3401da177e4SLinus Torvalds 3416ba5a3c5SPavel Emelyanov int udp_v4_get_port(struct sock *sk, unsigned short snum) 342db8dac20SDavid S. Miller { 34330fff923SEric Dumazet unsigned int hash2_nulladdr = 344f0b1e64cSMartin KaFai Lau ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum); 34530fff923SEric Dumazet unsigned int hash2_partial = 346f0b1e64cSMartin KaFai Lau ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); 34730fff923SEric Dumazet 348d4cada4aSEric Dumazet /* precompute partial secondary hash */ 34930fff923SEric Dumazet udp_sk(sk)->udp_portaddr_hash = hash2_partial; 350fe38d2a1SJosef Bacik return udp_lib_get_port(sk, snum, hash2_nulladdr); 351db8dac20SDavid S. Miller } 352db8dac20SDavid S. Miller 353d1e37288SSu, Xuemin static int compute_score(struct sock *sk, struct net *net, 354d1e37288SSu, Xuemin __be32 saddr, __be16 sport, 355fb74c277SDavid Ahern __be32 daddr, unsigned short hnum, 35673545373STim Beale int dif, int sdif) 357645ca708SEric Dumazet { 35860c04aecSJoe Perches int score; 35960c04aecSJoe Perches struct inet_sock *inet; 3606da5b0f0SMike Manning bool dev_match; 361645ca708SEric Dumazet 36260c04aecSJoe Perches if (!net_eq(sock_net(sk), net) || 36360c04aecSJoe Perches udp_sk(sk)->udp_port_hash != hnum || 36460c04aecSJoe Perches ipv6_only_sock(sk)) 36560c04aecSJoe Perches return -1; 366645ca708SEric Dumazet 3674cdeeee9SPeter Oskolkov if (sk->sk_rcv_saddr != daddr) 368645ca708SEric Dumazet return -1; 36960c04aecSJoe Perches 3704cdeeee9SPeter Oskolkov score = (sk->sk_family == PF_INET) ? 2 : 1; 3714cdeeee9SPeter Oskolkov 3724cdeeee9SPeter Oskolkov inet = inet_sk(sk); 373c720c7e8SEric Dumazet if (inet->inet_daddr) { 374c720c7e8SEric Dumazet if (inet->inet_daddr != saddr) 375645ca708SEric Dumazet return -1; 376ba418fa3STom Herbert score += 4; 377645ca708SEric Dumazet } 37860c04aecSJoe Perches 379c720c7e8SEric Dumazet if (inet->inet_dport) { 380c720c7e8SEric Dumazet if (inet->inet_dport != sport) 381645ca708SEric Dumazet return -1; 382ba418fa3STom Herbert score += 4; 383645ca708SEric Dumazet } 38460c04aecSJoe Perches 3856da5b0f0SMike Manning dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, 3866da5b0f0SMike Manning dif, sdif); 38769678bcdSPaolo Abeni if (!dev_match) 388645ca708SEric Dumazet return -1; 389ba418fa3STom Herbert score += 4; 390fb74c277SDavid Ahern 391*7170a977SEric Dumazet if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) 39270da268bSEric Dumazet score++; 393645ca708SEric Dumazet return score; 394645ca708SEric Dumazet } 395645ca708SEric Dumazet 3966eada011SEric Dumazet static u32 udp_ehashfn(const struct net *net, const __be32 laddr, 39765cd8033SHannes Frederic Sowa const __u16 lport, const __be32 faddr, 39865cd8033SHannes Frederic Sowa const __be16 fport) 39965cd8033SHannes Frederic Sowa { 4001bbdceefSHannes Frederic Sowa static u32 udp_ehash_secret __read_mostly; 4011bbdceefSHannes Frederic Sowa 4021bbdceefSHannes Frederic Sowa net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret)); 4031bbdceefSHannes Frederic Sowa 40465cd8033SHannes Frederic Sowa return __inet_ehashfn(laddr, lport, faddr, fport, 4051bbdceefSHannes Frederic Sowa udp_ehash_secret + net_hash_mix(net)); 40665cd8033SHannes Frederic Sowa } 40765cd8033SHannes Frederic Sowa 408d1e37288SSu, Xuemin /* called with rcu_read_lock() */ 4095051ebd2SEric Dumazet static struct sock *udp4_lib_lookup2(struct net *net, 4105051ebd2SEric Dumazet __be32 saddr, __be16 sport, 411fb74c277SDavid Ahern __be32 daddr, unsigned int hnum, 41273545373STim Beale int dif, int sdif, 413d1e37288SSu, Xuemin struct udp_hslot *hslot2, 4141134158bSCraig Gallek struct sk_buff *skb) 4155051ebd2SEric Dumazet { 4165051ebd2SEric Dumazet struct sock *sk, *result; 417e94a62f5SPaolo Abeni int score, badness; 418ba418fa3STom Herbert u32 hash = 0; 4195051ebd2SEric Dumazet 4205051ebd2SEric Dumazet result = NULL; 421ba418fa3STom Herbert badness = 0; 422ca065d0cSEric Dumazet udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { 423d1e37288SSu, Xuemin score = compute_score(sk, net, saddr, sport, 42473545373STim Beale daddr, hnum, dif, sdif); 4255051ebd2SEric Dumazet if (score > badness) { 426acdcecc6SWillem de Bruijn if (sk->sk_reuseport && 427acdcecc6SWillem de Bruijn sk->sk_state != TCP_ESTABLISHED) { 42865cd8033SHannes Frederic Sowa hash = udp_ehashfn(net, daddr, hnum, 4297c0cadc6SEric Dumazet saddr, sport); 430ca065d0cSEric Dumazet result = reuseport_select_sock(sk, hash, skb, 4311134158bSCraig Gallek sizeof(struct udphdr)); 432acdcecc6SWillem de Bruijn if (result && !reuseport_has_conns(sk, false)) 433ca065d0cSEric Dumazet return result; 434ba418fa3STom Herbert } 435ca065d0cSEric Dumazet badness = score; 436ca065d0cSEric Dumazet result = sk; 4375051ebd2SEric Dumazet } 4385051ebd2SEric Dumazet } 4395051ebd2SEric Dumazet return result; 4405051ebd2SEric Dumazet } 4415051ebd2SEric Dumazet 442db8dac20SDavid S. Miller /* UDP is nearly always wildcards out the wazoo, it makes no sense to try 443db8dac20SDavid S. Miller * harder than this. -DaveM 444db8dac20SDavid S. Miller */ 445fce82338SPavel Emelyanov struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, 446fb74c277SDavid Ahern __be16 sport, __be32 daddr, __be16 dport, int dif, 447fb74c277SDavid Ahern int sdif, struct udp_table *udptable, struct sk_buff *skb) 448db8dac20SDavid S. Miller { 4494cdeeee9SPeter Oskolkov struct sock *result; 450db8dac20SDavid S. Miller unsigned short hnum = ntohs(dport); 4514cdeeee9SPeter Oskolkov unsigned int hash2, slot2; 4524cdeeee9SPeter Oskolkov struct udp_hslot *hslot2; 453db8dac20SDavid S. Miller 454f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, daddr, hnum); 4555051ebd2SEric Dumazet slot2 = hash2 & udptable->mask; 4565051ebd2SEric Dumazet hslot2 = &udptable->hash2[slot2]; 4575051ebd2SEric Dumazet 4585051ebd2SEric Dumazet result = udp4_lib_lookup2(net, saddr, sport, 459fb74c277SDavid Ahern daddr, hnum, dif, sdif, 46073545373STim Beale hslot2, skb); 4615051ebd2SEric Dumazet if (!result) { 462f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum); 4635051ebd2SEric Dumazet slot2 = hash2 & udptable->mask; 4645051ebd2SEric Dumazet hslot2 = &udptable->hash2[slot2]; 4655051ebd2SEric Dumazet 4661223c67cSJorge Boncompte [DTI2] result = udp4_lib_lookup2(net, saddr, sport, 4674cdeeee9SPeter Oskolkov htonl(INADDR_ANY), hnum, dif, sdif, 46873545373STim Beale hslot2, skb); 4695051ebd2SEric Dumazet } 47088e235b8SEnrico Weigelt if (IS_ERR(result)) 4718217ca65SMartin KaFai Lau return NULL; 4725051ebd2SEric Dumazet return result; 4735051ebd2SEric Dumazet } 474fce82338SPavel Emelyanov EXPORT_SYMBOL_GPL(__udp4_lib_lookup); 475db8dac20SDavid S. Miller 476607c4aafSKOVACS Krisztian static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb, 477607c4aafSKOVACS Krisztian __be16 sport, __be16 dport, 478645ca708SEric Dumazet struct udp_table *udptable) 479607c4aafSKOVACS Krisztian { 480607c4aafSKOVACS Krisztian const struct iphdr *iph = ip_hdr(skb); 481607c4aafSKOVACS Krisztian 482ed7cbbceSAlexander Duyck return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, 483607c4aafSKOVACS Krisztian iph->daddr, dport, inet_iif(skb), 484fb74c277SDavid Ahern inet_sdif(skb), udptable, skb); 485607c4aafSKOVACS Krisztian } 486607c4aafSKOVACS Krisztian 48763058308STom Herbert struct sock *udp4_lib_lookup_skb(struct sk_buff *skb, 48863058308STom Herbert __be16 sport, __be16 dport) 48963058308STom Herbert { 490257a525fSMartin KaFai Lau const struct iphdr *iph = ip_hdr(skb); 491257a525fSMartin KaFai Lau 492257a525fSMartin KaFai Lau return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, 493257a525fSMartin KaFai Lau iph->daddr, dport, inet_iif(skb), 494257a525fSMartin KaFai Lau inet_sdif(skb), &udp_table, NULL); 49563058308STom Herbert } 49663058308STom Herbert EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb); 49763058308STom Herbert 498ca065d0cSEric Dumazet /* Must be called under rcu_read_lock(). 499ca065d0cSEric Dumazet * Does increment socket refcount. 500ca065d0cSEric Dumazet */ 5016e86000cSArnd Bergmann #if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4) 502bcd41303SKOVACS Krisztian struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport, 503bcd41303SKOVACS Krisztian __be32 daddr, __be16 dport, int dif) 504bcd41303SKOVACS Krisztian { 505ca065d0cSEric Dumazet struct sock *sk; 506ca065d0cSEric Dumazet 507ca065d0cSEric Dumazet sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, 508fb74c277SDavid Ahern dif, 0, &udp_table, NULL); 50941c6d650SReshetova, Elena if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) 510ca065d0cSEric Dumazet sk = NULL; 511ca065d0cSEric Dumazet return sk; 512bcd41303SKOVACS Krisztian } 513bcd41303SKOVACS Krisztian EXPORT_SYMBOL_GPL(udp4_lib_lookup); 514ca065d0cSEric Dumazet #endif 515bcd41303SKOVACS Krisztian 516421b3885SShawn Bohrer static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, 517421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 518421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 519fb74c277SDavid Ahern int dif, int sdif, unsigned short hnum) 520421b3885SShawn Bohrer { 521421b3885SShawn Bohrer struct inet_sock *inet = inet_sk(sk); 522421b3885SShawn Bohrer 523421b3885SShawn Bohrer if (!net_eq(sock_net(sk), net) || 524421b3885SShawn Bohrer udp_sk(sk)->udp_port_hash != hnum || 525421b3885SShawn Bohrer (inet->inet_daddr && inet->inet_daddr != rmt_addr) || 526421b3885SShawn Bohrer (inet->inet_dport != rmt_port && inet->inet_dport) || 527421b3885SShawn Bohrer (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) || 528421b3885SShawn Bohrer ipv6_only_sock(sk) || 52982ba25c6STim Beale !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) 530421b3885SShawn Bohrer return false; 53160d9b031SDavid Ahern if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif)) 532421b3885SShawn Bohrer return false; 533421b3885SShawn Bohrer return true; 534421b3885SShawn Bohrer } 535421b3885SShawn Bohrer 536a36e185eSStefano Brivio DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key); 537a36e185eSStefano Brivio void udp_encap_enable(void) 538a36e185eSStefano Brivio { 5399c480601SPaolo Abeni static_branch_inc(&udp_encap_needed_key); 540a36e185eSStefano Brivio } 541a36e185eSStefano Brivio EXPORT_SYMBOL(udp_encap_enable); 542a36e185eSStefano Brivio 543e7cc0824SStefano Brivio /* Handler for tunnels with arbitrary destination ports: no socket lookup, go 544e7cc0824SStefano Brivio * through error handlers in encapsulations looking for a match. 545e7cc0824SStefano Brivio */ 546e7cc0824SStefano Brivio static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info) 547e7cc0824SStefano Brivio { 548e7cc0824SStefano Brivio int i; 549e7cc0824SStefano Brivio 550e7cc0824SStefano Brivio for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) { 551e7cc0824SStefano Brivio int (*handler)(struct sk_buff *skb, u32 info); 55292b95364SPaolo Abeni const struct ip_tunnel_encap_ops *encap; 553e7cc0824SStefano Brivio 55492b95364SPaolo Abeni encap = rcu_dereference(iptun_encaps[i]); 55592b95364SPaolo Abeni if (!encap) 556e7cc0824SStefano Brivio continue; 55792b95364SPaolo Abeni handler = encap->err_handler; 558e7cc0824SStefano Brivio if (handler && !handler(skb, info)) 559e7cc0824SStefano Brivio return 0; 560e7cc0824SStefano Brivio } 561e7cc0824SStefano Brivio 562e7cc0824SStefano Brivio return -ENOENT; 563e7cc0824SStefano Brivio } 564e7cc0824SStefano Brivio 565a36e185eSStefano Brivio /* Try to match ICMP errors to UDP tunnels by looking up a socket without 566a36e185eSStefano Brivio * reversing source and destination port: this will match tunnels that force the 567a36e185eSStefano Brivio * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that 568a36e185eSStefano Brivio * lwtunnels might actually break this assumption by being configured with 569a36e185eSStefano Brivio * different destination ports on endpoints, in this case we won't be able to 570a36e185eSStefano Brivio * trace ICMP messages back to them. 571a36e185eSStefano Brivio * 572e7cc0824SStefano Brivio * If this doesn't match any socket, probe tunnels with arbitrary destination 573e7cc0824SStefano Brivio * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port 574e7cc0824SStefano Brivio * we've sent packets to won't necessarily match the local destination port. 575e7cc0824SStefano Brivio * 576a36e185eSStefano Brivio * Then ask the tunnel implementation to match the error against a valid 577a36e185eSStefano Brivio * association. 578a36e185eSStefano Brivio * 579e7cc0824SStefano Brivio * Return an error if we can't find a match, the socket if we need further 580e7cc0824SStefano Brivio * processing, zero otherwise. 581a36e185eSStefano Brivio */ 582a36e185eSStefano Brivio static struct sock *__udp4_lib_err_encap(struct net *net, 583a36e185eSStefano Brivio const struct iphdr *iph, 584a36e185eSStefano Brivio struct udphdr *uh, 585a36e185eSStefano Brivio struct udp_table *udptable, 586e7cc0824SStefano Brivio struct sk_buff *skb, u32 info) 587a36e185eSStefano Brivio { 588a36e185eSStefano Brivio int network_offset, transport_offset; 589a36e185eSStefano Brivio struct sock *sk; 590a36e185eSStefano Brivio 591a36e185eSStefano Brivio network_offset = skb_network_offset(skb); 592a36e185eSStefano Brivio transport_offset = skb_transport_offset(skb); 593a36e185eSStefano Brivio 594a36e185eSStefano Brivio /* Network header needs to point to the outer IPv4 header inside ICMP */ 595a36e185eSStefano Brivio skb_reset_network_header(skb); 596a36e185eSStefano Brivio 597a36e185eSStefano Brivio /* Transport header needs to point to the UDP header */ 598a36e185eSStefano Brivio skb_set_transport_header(skb, iph->ihl << 2); 599a36e185eSStefano Brivio 600e7cc0824SStefano Brivio sk = __udp4_lib_lookup(net, iph->daddr, uh->source, 601e7cc0824SStefano Brivio iph->saddr, uh->dest, skb->dev->ifindex, 0, 602e7cc0824SStefano Brivio udptable, NULL); 603e7cc0824SStefano Brivio if (sk) { 604e7cc0824SStefano Brivio int (*lookup)(struct sock *sk, struct sk_buff *skb); 605e7cc0824SStefano Brivio struct udp_sock *up = udp_sk(sk); 606e7cc0824SStefano Brivio 607a36e185eSStefano Brivio lookup = READ_ONCE(up->encap_err_lookup); 608a36e185eSStefano Brivio if (!lookup || lookup(sk, skb)) 609a36e185eSStefano Brivio sk = NULL; 610e7cc0824SStefano Brivio } 611e7cc0824SStefano Brivio 612e7cc0824SStefano Brivio if (!sk) 613e7cc0824SStefano Brivio sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info)); 614a36e185eSStefano Brivio 615a36e185eSStefano Brivio skb_set_transport_header(skb, transport_offset); 616a36e185eSStefano Brivio skb_set_network_header(skb, network_offset); 617a36e185eSStefano Brivio 618a36e185eSStefano Brivio return sk; 619a36e185eSStefano Brivio } 620a36e185eSStefano Brivio 621db8dac20SDavid S. Miller /* 622db8dac20SDavid S. Miller * This routine is called by the ICMP module when it gets some 623db8dac20SDavid S. Miller * sort of error condition. If err < 0 then the socket should 624db8dac20SDavid S. Miller * be closed and the error returned to the user. If err > 0 625db8dac20SDavid S. Miller * it's just the icmp type << 8 | icmp code. 626db8dac20SDavid S. Miller * Header points to the ip header of the error packet. We move 627db8dac20SDavid S. Miller * on past this. Then (as it used to claim before adjustment) 628db8dac20SDavid S. Miller * header points to the first 8 bytes of the udp header. We need 629db8dac20SDavid S. Miller * to find the appropriate port. 630db8dac20SDavid S. Miller */ 631db8dac20SDavid S. Miller 63232bbd879SStefano Brivio int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) 633db8dac20SDavid S. Miller { 634db8dac20SDavid S. Miller struct inet_sock *inet; 635b71d1d42SEric Dumazet const struct iphdr *iph = (const struct iphdr *)skb->data; 636db8dac20SDavid S. Miller struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2)); 637db8dac20SDavid S. Miller const int type = icmp_hdr(skb)->type; 638db8dac20SDavid S. Miller const int code = icmp_hdr(skb)->code; 639a36e185eSStefano Brivio bool tunnel = false; 640db8dac20SDavid S. Miller struct sock *sk; 641db8dac20SDavid S. Miller int harderr; 642db8dac20SDavid S. Miller int err; 643fd54d716SPavel Emelyanov struct net *net = dev_net(skb->dev); 644db8dac20SDavid S. Miller 645fd54d716SPavel Emelyanov sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, 646f64bf6b8SMike Manning iph->saddr, uh->source, skb->dev->ifindex, 647f64bf6b8SMike Manning inet_sdif(skb), udptable, NULL); 64851456b29SIan Morris if (!sk) { 649a36e185eSStefano Brivio /* No socket for error: try tunnels before discarding */ 650e7cc0824SStefano Brivio sk = ERR_PTR(-ENOENT); 651e7cc0824SStefano Brivio if (static_branch_unlikely(&udp_encap_needed_key)) { 652e7cc0824SStefano Brivio sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb, 653e7cc0824SStefano Brivio info); 654e7cc0824SStefano Brivio if (!sk) 655e7cc0824SStefano Brivio return 0; 656a36e185eSStefano Brivio } 657e7cc0824SStefano Brivio 658e7cc0824SStefano Brivio if (IS_ERR(sk)) { 659e7cc0824SStefano Brivio __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); 660e7cc0824SStefano Brivio return PTR_ERR(sk); 661e7cc0824SStefano Brivio } 662e7cc0824SStefano Brivio 663a36e185eSStefano Brivio tunnel = true; 664db8dac20SDavid S. Miller } 665db8dac20SDavid S. Miller 666db8dac20SDavid S. Miller err = 0; 667db8dac20SDavid S. Miller harderr = 0; 668db8dac20SDavid S. Miller inet = inet_sk(sk); 669db8dac20SDavid S. Miller 670db8dac20SDavid S. Miller switch (type) { 671db8dac20SDavid S. Miller default: 672db8dac20SDavid S. Miller case ICMP_TIME_EXCEEDED: 673db8dac20SDavid S. Miller err = EHOSTUNREACH; 674db8dac20SDavid S. Miller break; 675db8dac20SDavid S. Miller case ICMP_SOURCE_QUENCH: 676db8dac20SDavid S. Miller goto out; 677db8dac20SDavid S. Miller case ICMP_PARAMETERPROB: 678db8dac20SDavid S. Miller err = EPROTO; 679db8dac20SDavid S. Miller harderr = 1; 680db8dac20SDavid S. Miller break; 681db8dac20SDavid S. Miller case ICMP_DEST_UNREACH: 682db8dac20SDavid S. Miller if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */ 68336393395SDavid S. Miller ipv4_sk_update_pmtu(skb, sk, info); 684db8dac20SDavid S. Miller if (inet->pmtudisc != IP_PMTUDISC_DONT) { 685db8dac20SDavid S. Miller err = EMSGSIZE; 686db8dac20SDavid S. Miller harderr = 1; 687db8dac20SDavid S. Miller break; 688db8dac20SDavid S. Miller } 689db8dac20SDavid S. Miller goto out; 690db8dac20SDavid S. Miller } 691db8dac20SDavid S. Miller err = EHOSTUNREACH; 692db8dac20SDavid S. Miller if (code <= NR_ICMP_UNREACH) { 693db8dac20SDavid S. Miller harderr = icmp_err_convert[code].fatal; 694db8dac20SDavid S. Miller err = icmp_err_convert[code].errno; 695db8dac20SDavid S. Miller } 696db8dac20SDavid S. Miller break; 69755be7a9cSDavid S. Miller case ICMP_REDIRECT: 69855be7a9cSDavid S. Miller ipv4_sk_redirect(skb, sk); 6991a462d18SDuan Jiong goto out; 700db8dac20SDavid S. Miller } 701db8dac20SDavid S. Miller 702db8dac20SDavid S. Miller /* 703db8dac20SDavid S. Miller * RFC1122: OK. Passes ICMP errors back to application, as per 704db8dac20SDavid S. Miller * 4.1.3.3. 705db8dac20SDavid S. Miller */ 706a36e185eSStefano Brivio if (tunnel) { 707a36e185eSStefano Brivio /* ...not for tunnels though: we don't have a sending socket */ 708a36e185eSStefano Brivio goto out; 709a36e185eSStefano Brivio } 710db8dac20SDavid S. Miller if (!inet->recverr) { 711db8dac20SDavid S. Miller if (!harderr || sk->sk_state != TCP_ESTABLISHED) 712db8dac20SDavid S. Miller goto out; 713b1faf566SEric Dumazet } else 714db8dac20SDavid S. Miller ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1)); 715b1faf566SEric Dumazet 716db8dac20SDavid S. Miller sk->sk_err = err; 717db8dac20SDavid S. Miller sk->sk_error_report(sk); 718db8dac20SDavid S. Miller out: 71932bbd879SStefano Brivio return 0; 720db8dac20SDavid S. Miller } 721db8dac20SDavid S. Miller 72232bbd879SStefano Brivio int udp_err(struct sk_buff *skb, u32 info) 723db8dac20SDavid S. Miller { 72432bbd879SStefano Brivio return __udp4_lib_err(skb, info, &udp_table); 725db8dac20SDavid S. Miller } 726db8dac20SDavid S. Miller 727db8dac20SDavid S. Miller /* 728db8dac20SDavid S. Miller * Throw away all pending data and cancel the corking. Socket is locked. 729db8dac20SDavid S. Miller */ 73036d926b9SDenis V. Lunev void udp_flush_pending_frames(struct sock *sk) 731db8dac20SDavid S. Miller { 732db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 733db8dac20SDavid S. Miller 734db8dac20SDavid S. Miller if (up->pending) { 735db8dac20SDavid S. Miller up->len = 0; 736db8dac20SDavid S. Miller up->pending = 0; 737db8dac20SDavid S. Miller ip_flush_pending_frames(sk); 738db8dac20SDavid S. Miller } 739db8dac20SDavid S. Miller } 74036d926b9SDenis V. Lunev EXPORT_SYMBOL(udp_flush_pending_frames); 741db8dac20SDavid S. Miller 742db8dac20SDavid S. Miller /** 743f6b9664fSHerbert Xu * udp4_hwcsum - handle outgoing HW checksumming 744db8dac20SDavid S. Miller * @skb: sk_buff containing the filled-in UDP header 745db8dac20SDavid S. Miller * (checksum field must be zeroed out) 746f6b9664fSHerbert Xu * @src: source IP address 747f6b9664fSHerbert Xu * @dst: destination IP address 748db8dac20SDavid S. Miller */ 749c26bf4a5SThomas Graf void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst) 750db8dac20SDavid S. Miller { 751db8dac20SDavid S. Miller struct udphdr *uh = udp_hdr(skb); 752f6b9664fSHerbert Xu int offset = skb_transport_offset(skb); 753f6b9664fSHerbert Xu int len = skb->len - offset; 754f6b9664fSHerbert Xu int hlen = len; 755db8dac20SDavid S. Miller __wsum csum = 0; 756db8dac20SDavid S. Miller 757ebbe495fSWANG Cong if (!skb_has_frag_list(skb)) { 758db8dac20SDavid S. Miller /* 759db8dac20SDavid S. Miller * Only one fragment on the socket. 760db8dac20SDavid S. Miller */ 761db8dac20SDavid S. Miller skb->csum_start = skb_transport_header(skb) - skb->head; 762db8dac20SDavid S. Miller skb->csum_offset = offsetof(struct udphdr, check); 763f6b9664fSHerbert Xu uh->check = ~csum_tcpudp_magic(src, dst, len, 764f6b9664fSHerbert Xu IPPROTO_UDP, 0); 765db8dac20SDavid S. Miller } else { 766ebbe495fSWANG Cong struct sk_buff *frags; 767ebbe495fSWANG Cong 768db8dac20SDavid S. Miller /* 769db8dac20SDavid S. Miller * HW-checksum won't work as there are two or more 770db8dac20SDavid S. Miller * fragments on the socket so that all csums of sk_buffs 771db8dac20SDavid S. Miller * should be together 772db8dac20SDavid S. Miller */ 773ebbe495fSWANG Cong skb_walk_frags(skb, frags) { 774f6b9664fSHerbert Xu csum = csum_add(csum, frags->csum); 775f6b9664fSHerbert Xu hlen -= frags->len; 776ebbe495fSWANG Cong } 777db8dac20SDavid S. Miller 778f6b9664fSHerbert Xu csum = skb_checksum(skb, offset, hlen, csum); 779db8dac20SDavid S. Miller skb->ip_summed = CHECKSUM_NONE; 780db8dac20SDavid S. Miller 781db8dac20SDavid S. Miller uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum); 782db8dac20SDavid S. Miller if (uh->check == 0) 783db8dac20SDavid S. Miller uh->check = CSUM_MANGLED_0; 784db8dac20SDavid S. Miller } 785db8dac20SDavid S. Miller } 786c26bf4a5SThomas Graf EXPORT_SYMBOL_GPL(udp4_hwcsum); 787db8dac20SDavid S. Miller 788af5fcba7STom Herbert /* Function to set UDP checksum for an IPv4 UDP packet. This is intended 789af5fcba7STom Herbert * for the simple case like when setting the checksum for a UDP tunnel. 790af5fcba7STom Herbert */ 791af5fcba7STom Herbert void udp_set_csum(bool nocheck, struct sk_buff *skb, 792af5fcba7STom Herbert __be32 saddr, __be32 daddr, int len) 793af5fcba7STom Herbert { 794af5fcba7STom Herbert struct udphdr *uh = udp_hdr(skb); 795af5fcba7STom Herbert 796179bc67fSEdward Cree if (nocheck) { 797af5fcba7STom Herbert uh->check = 0; 798179bc67fSEdward Cree } else if (skb_is_gso(skb)) { 799af5fcba7STom Herbert uh->check = ~udp_v4_check(len, saddr, daddr, 0); 800179bc67fSEdward Cree } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 801179bc67fSEdward Cree uh->check = 0; 802179bc67fSEdward Cree uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb)); 803179bc67fSEdward Cree if (uh->check == 0) 804179bc67fSEdward Cree uh->check = CSUM_MANGLED_0; 805d75f1306SEdward Cree } else { 806af5fcba7STom Herbert skb->ip_summed = CHECKSUM_PARTIAL; 807af5fcba7STom Herbert skb->csum_start = skb_transport_header(skb) - skb->head; 808af5fcba7STom Herbert skb->csum_offset = offsetof(struct udphdr, check); 809af5fcba7STom Herbert uh->check = ~udp_v4_check(len, saddr, daddr, 0); 810af5fcba7STom Herbert } 811af5fcba7STom Herbert } 812af5fcba7STom Herbert EXPORT_SYMBOL(udp_set_csum); 813af5fcba7STom Herbert 814bec1f6f6SWillem de Bruijn static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4, 815bec1f6f6SWillem de Bruijn struct inet_cork *cork) 816f6b9664fSHerbert Xu { 817f6b9664fSHerbert Xu struct sock *sk = skb->sk; 818f6b9664fSHerbert Xu struct inet_sock *inet = inet_sk(sk); 819f6b9664fSHerbert Xu struct udphdr *uh; 820f6b9664fSHerbert Xu int err = 0; 821f6b9664fSHerbert Xu int is_udplite = IS_UDPLITE(sk); 822f6b9664fSHerbert Xu int offset = skb_transport_offset(skb); 823f6b9664fSHerbert Xu int len = skb->len - offset; 8244094871dSJosh Hunt int datalen = len - sizeof(*uh); 825f6b9664fSHerbert Xu __wsum csum = 0; 826f6b9664fSHerbert Xu 827f6b9664fSHerbert Xu /* 828f6b9664fSHerbert Xu * Create a UDP header 829f6b9664fSHerbert Xu */ 830f6b9664fSHerbert Xu uh = udp_hdr(skb); 831f6b9664fSHerbert Xu uh->source = inet->inet_sport; 83279ab0531SDavid S. Miller uh->dest = fl4->fl4_dport; 833f6b9664fSHerbert Xu uh->len = htons(len); 834f6b9664fSHerbert Xu uh->check = 0; 835f6b9664fSHerbert Xu 836bec1f6f6SWillem de Bruijn if (cork->gso_size) { 837bec1f6f6SWillem de Bruijn const int hlen = skb_network_header_len(skb) + 838bec1f6f6SWillem de Bruijn sizeof(struct udphdr); 839bec1f6f6SWillem de Bruijn 8400f149c9fSWillem de Bruijn if (hlen + cork->gso_size > cork->fragsize) { 8410f149c9fSWillem de Bruijn kfree_skb(skb); 842bec1f6f6SWillem de Bruijn return -EINVAL; 8430f149c9fSWillem de Bruijn } 8440f149c9fSWillem de Bruijn if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) { 8450f149c9fSWillem de Bruijn kfree_skb(skb); 846bec1f6f6SWillem de Bruijn return -EINVAL; 8470f149c9fSWillem de Bruijn } 8480f149c9fSWillem de Bruijn if (sk->sk_no_check_tx) { 8490f149c9fSWillem de Bruijn kfree_skb(skb); 850a8c744a8SWillem de Bruijn return -EINVAL; 8510f149c9fSWillem de Bruijn } 852ff06342cSWillem de Bruijn if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite || 8530f149c9fSWillem de Bruijn dst_xfrm(skb_dst(skb))) { 8540f149c9fSWillem de Bruijn kfree_skb(skb); 855bec1f6f6SWillem de Bruijn return -EIO; 8560f149c9fSWillem de Bruijn } 857bec1f6f6SWillem de Bruijn 8584094871dSJosh Hunt if (datalen > cork->gso_size) { 859bec1f6f6SWillem de Bruijn skb_shinfo(skb)->gso_size = cork->gso_size; 860bec1f6f6SWillem de Bruijn skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; 8614094871dSJosh Hunt skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen, 862dfec0ee2SAlexander Duyck cork->gso_size); 8634094871dSJosh Hunt } 864a8c744a8SWillem de Bruijn goto csum_partial; 865bec1f6f6SWillem de Bruijn } 866bec1f6f6SWillem de Bruijn 867f6b9664fSHerbert Xu if (is_udplite) /* UDP-Lite */ 868f6b9664fSHerbert Xu csum = udplite_csum(skb); 869f6b9664fSHerbert Xu 870ab2fb7e3SWillem de Bruijn else if (sk->sk_no_check_tx) { /* UDP csum off */ 871f6b9664fSHerbert Xu 872f6b9664fSHerbert Xu skb->ip_summed = CHECKSUM_NONE; 873f6b9664fSHerbert Xu goto send; 874f6b9664fSHerbert Xu 875f6b9664fSHerbert Xu } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */ 876a8c744a8SWillem de Bruijn csum_partial: 877f6b9664fSHerbert Xu 87879ab0531SDavid S. Miller udp4_hwcsum(skb, fl4->saddr, fl4->daddr); 879f6b9664fSHerbert Xu goto send; 880f6b9664fSHerbert Xu 881f6b9664fSHerbert Xu } else 882f6b9664fSHerbert Xu csum = udp_csum(skb); 883f6b9664fSHerbert Xu 884f6b9664fSHerbert Xu /* add protocol-dependent pseudo-header */ 88579ab0531SDavid S. Miller uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len, 886f6b9664fSHerbert Xu sk->sk_protocol, csum); 887f6b9664fSHerbert Xu if (uh->check == 0) 888f6b9664fSHerbert Xu uh->check = CSUM_MANGLED_0; 889f6b9664fSHerbert Xu 890f6b9664fSHerbert Xu send: 891b5ec8eeaSEric Dumazet err = ip_send_skb(sock_net(sk), skb); 892f6b9664fSHerbert Xu if (err) { 893f6b9664fSHerbert Xu if (err == -ENOBUFS && !inet->recverr) { 8946aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 895f6b9664fSHerbert Xu UDP_MIB_SNDBUFERRORS, is_udplite); 896f6b9664fSHerbert Xu err = 0; 897f6b9664fSHerbert Xu } 898f6b9664fSHerbert Xu } else 8996aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 900f6b9664fSHerbert Xu UDP_MIB_OUTDATAGRAMS, is_udplite); 901f6b9664fSHerbert Xu return err; 902f6b9664fSHerbert Xu } 903f6b9664fSHerbert Xu 904db8dac20SDavid S. Miller /* 905db8dac20SDavid S. Miller * Push out all pending data as one UDP datagram. Socket is locked. 906db8dac20SDavid S. Miller */ 9078822b64aSHannes Frederic Sowa int udp_push_pending_frames(struct sock *sk) 908db8dac20SDavid S. Miller { 909db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 910db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 911b6f21b26SDavid S. Miller struct flowi4 *fl4 = &inet->cork.fl.u.ip4; 912db8dac20SDavid S. Miller struct sk_buff *skb; 913db8dac20SDavid S. Miller int err = 0; 914db8dac20SDavid S. Miller 91577968b78SDavid S. Miller skb = ip_finish_skb(sk, fl4); 916f6b9664fSHerbert Xu if (!skb) 917db8dac20SDavid S. Miller goto out; 918db8dac20SDavid S. Miller 919bec1f6f6SWillem de Bruijn err = udp_send_skb(skb, fl4, &inet->cork.base); 920db8dac20SDavid S. Miller 921db8dac20SDavid S. Miller out: 922db8dac20SDavid S. Miller up->len = 0; 923db8dac20SDavid S. Miller up->pending = 0; 924db8dac20SDavid S. Miller return err; 925db8dac20SDavid S. Miller } 9268822b64aSHannes Frederic Sowa EXPORT_SYMBOL(udp_push_pending_frames); 927db8dac20SDavid S. Miller 9282e8de857SWillem de Bruijn static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size) 9292e8de857SWillem de Bruijn { 9302e8de857SWillem de Bruijn switch (cmsg->cmsg_type) { 9312e8de857SWillem de Bruijn case UDP_SEGMENT: 9322e8de857SWillem de Bruijn if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16))) 9332e8de857SWillem de Bruijn return -EINVAL; 9342e8de857SWillem de Bruijn *gso_size = *(__u16 *)CMSG_DATA(cmsg); 9352e8de857SWillem de Bruijn return 0; 9362e8de857SWillem de Bruijn default: 9372e8de857SWillem de Bruijn return -EINVAL; 9382e8de857SWillem de Bruijn } 9392e8de857SWillem de Bruijn } 9402e8de857SWillem de Bruijn 9412e8de857SWillem de Bruijn int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size) 9422e8de857SWillem de Bruijn { 9432e8de857SWillem de Bruijn struct cmsghdr *cmsg; 9442e8de857SWillem de Bruijn bool need_ip = false; 9452e8de857SWillem de Bruijn int err; 9462e8de857SWillem de Bruijn 9472e8de857SWillem de Bruijn for_each_cmsghdr(cmsg, msg) { 9482e8de857SWillem de Bruijn if (!CMSG_OK(msg, cmsg)) 9492e8de857SWillem de Bruijn return -EINVAL; 9502e8de857SWillem de Bruijn 9512e8de857SWillem de Bruijn if (cmsg->cmsg_level != SOL_UDP) { 9522e8de857SWillem de Bruijn need_ip = true; 9532e8de857SWillem de Bruijn continue; 9542e8de857SWillem de Bruijn } 9552e8de857SWillem de Bruijn 9562e8de857SWillem de Bruijn err = __udp_cmsg_send(cmsg, gso_size); 9572e8de857SWillem de Bruijn if (err) 9582e8de857SWillem de Bruijn return err; 9592e8de857SWillem de Bruijn } 9602e8de857SWillem de Bruijn 9612e8de857SWillem de Bruijn return need_ip; 9622e8de857SWillem de Bruijn } 9632e8de857SWillem de Bruijn EXPORT_SYMBOL_GPL(udp_cmsg_send); 9642e8de857SWillem de Bruijn 9651b784140SYing Xue int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 966db8dac20SDavid S. Miller { 967db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 968db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 9691cedee13SAndrey Ignatov DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); 970e474995fSDavid S. Miller struct flowi4 fl4_stack; 971b6f21b26SDavid S. Miller struct flowi4 *fl4; 972db8dac20SDavid S. Miller int ulen = len; 973db8dac20SDavid S. Miller struct ipcm_cookie ipc; 974db8dac20SDavid S. Miller struct rtable *rt = NULL; 975db8dac20SDavid S. Miller int free = 0; 976db8dac20SDavid S. Miller int connected = 0; 977db8dac20SDavid S. Miller __be32 daddr, faddr, saddr; 978db8dac20SDavid S. Miller __be16 dport; 979db8dac20SDavid S. Miller u8 tos; 980db8dac20SDavid S. Miller int err, is_udplite = IS_UDPLITE(sk); 981db8dac20SDavid S. Miller int corkreq = up->corkflag || msg->msg_flags&MSG_MORE; 982db8dac20SDavid S. Miller int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); 983903ab86dSHerbert Xu struct sk_buff *skb; 984f6d8bd05SEric Dumazet struct ip_options_data opt_copy; 985db8dac20SDavid S. Miller 986db8dac20SDavid S. Miller if (len > 0xFFFF) 987db8dac20SDavid S. Miller return -EMSGSIZE; 988db8dac20SDavid S. Miller 989db8dac20SDavid S. Miller /* 990db8dac20SDavid S. Miller * Check the flags. 991db8dac20SDavid S. Miller */ 992db8dac20SDavid S. Miller 993db8dac20SDavid S. Miller if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */ 994db8dac20SDavid S. Miller return -EOPNOTSUPP; 995db8dac20SDavid S. Miller 996903ab86dSHerbert Xu getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag; 997903ab86dSHerbert Xu 998f5fca608SDavid S. Miller fl4 = &inet->cork.fl.u.ip4; 999db8dac20SDavid S. Miller if (up->pending) { 1000db8dac20SDavid S. Miller /* 1001db8dac20SDavid S. Miller * There are pending frames. 1002db8dac20SDavid S. Miller * The socket lock must be held while it's corked. 1003db8dac20SDavid S. Miller */ 1004db8dac20SDavid S. Miller lock_sock(sk); 1005db8dac20SDavid S. Miller if (likely(up->pending)) { 1006db8dac20SDavid S. Miller if (unlikely(up->pending != AF_INET)) { 1007db8dac20SDavid S. Miller release_sock(sk); 1008db8dac20SDavid S. Miller return -EINVAL; 1009db8dac20SDavid S. Miller } 1010db8dac20SDavid S. Miller goto do_append_data; 1011db8dac20SDavid S. Miller } 1012db8dac20SDavid S. Miller release_sock(sk); 1013db8dac20SDavid S. Miller } 1014db8dac20SDavid S. Miller ulen += sizeof(struct udphdr); 1015db8dac20SDavid S. Miller 1016db8dac20SDavid S. Miller /* 1017db8dac20SDavid S. Miller * Get and verify the address. 1018db8dac20SDavid S. Miller */ 10191cedee13SAndrey Ignatov if (usin) { 1020db8dac20SDavid S. Miller if (msg->msg_namelen < sizeof(*usin)) 1021db8dac20SDavid S. Miller return -EINVAL; 1022db8dac20SDavid S. Miller if (usin->sin_family != AF_INET) { 1023db8dac20SDavid S. Miller if (usin->sin_family != AF_UNSPEC) 1024db8dac20SDavid S. Miller return -EAFNOSUPPORT; 1025db8dac20SDavid S. Miller } 1026db8dac20SDavid S. Miller 1027db8dac20SDavid S. Miller daddr = usin->sin_addr.s_addr; 1028db8dac20SDavid S. Miller dport = usin->sin_port; 1029db8dac20SDavid S. Miller if (dport == 0) 1030db8dac20SDavid S. Miller return -EINVAL; 1031db8dac20SDavid S. Miller } else { 1032db8dac20SDavid S. Miller if (sk->sk_state != TCP_ESTABLISHED) 1033db8dac20SDavid S. Miller return -EDESTADDRREQ; 1034c720c7e8SEric Dumazet daddr = inet->inet_daddr; 1035c720c7e8SEric Dumazet dport = inet->inet_dport; 1036db8dac20SDavid S. Miller /* Open fast path for connected socket. 1037db8dac20SDavid S. Miller Route will not be used, if at least one option is set. 1038db8dac20SDavid S. Miller */ 1039db8dac20SDavid S. Miller connected = 1; 1040db8dac20SDavid S. Miller } 1041c14ac945SSoheil Hassas Yeganeh 104235178206SWillem de Bruijn ipcm_init_sk(&ipc, inet); 1043bec1f6f6SWillem de Bruijn ipc.gso_size = up->gso_size; 1044bf84a010SDaniel Borkmann 1045db8dac20SDavid S. Miller if (msg->msg_controllen) { 10462e8de857SWillem de Bruijn err = udp_cmsg_send(sk, msg, &ipc.gso_size); 10472e8de857SWillem de Bruijn if (err > 0) 10482e8de857SWillem de Bruijn err = ip_cmsg_send(sk, msg, &ipc, 10492e8de857SWillem de Bruijn sk->sk_family == AF_INET6); 10502e8de857SWillem de Bruijn if (unlikely(err < 0)) { 105191948309SEric Dumazet kfree(ipc.opt); 1052db8dac20SDavid S. Miller return err; 105391948309SEric Dumazet } 1054db8dac20SDavid S. Miller if (ipc.opt) 1055db8dac20SDavid S. Miller free = 1; 1056db8dac20SDavid S. Miller connected = 0; 1057db8dac20SDavid S. Miller } 1058f6d8bd05SEric Dumazet if (!ipc.opt) { 1059f6d8bd05SEric Dumazet struct ip_options_rcu *inet_opt; 1060f6d8bd05SEric Dumazet 1061f6d8bd05SEric Dumazet rcu_read_lock(); 1062f6d8bd05SEric Dumazet inet_opt = rcu_dereference(inet->inet_opt); 1063f6d8bd05SEric Dumazet if (inet_opt) { 1064f6d8bd05SEric Dumazet memcpy(&opt_copy, inet_opt, 1065f6d8bd05SEric Dumazet sizeof(*inet_opt) + inet_opt->opt.optlen); 1066f6d8bd05SEric Dumazet ipc.opt = &opt_copy.opt; 1067f6d8bd05SEric Dumazet } 1068f6d8bd05SEric Dumazet rcu_read_unlock(); 1069f6d8bd05SEric Dumazet } 1070db8dac20SDavid S. Miller 10711cedee13SAndrey Ignatov if (cgroup_bpf_enabled && !connected) { 10721cedee13SAndrey Ignatov err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, 10731cedee13SAndrey Ignatov (struct sockaddr *)usin, &ipc.addr); 10741cedee13SAndrey Ignatov if (err) 10751cedee13SAndrey Ignatov goto out_free; 10761cedee13SAndrey Ignatov if (usin) { 10771cedee13SAndrey Ignatov if (usin->sin_port == 0) { 10781cedee13SAndrey Ignatov /* BPF program set invalid port. Reject it. */ 10791cedee13SAndrey Ignatov err = -EINVAL; 10801cedee13SAndrey Ignatov goto out_free; 10811cedee13SAndrey Ignatov } 10821cedee13SAndrey Ignatov daddr = usin->sin_addr.s_addr; 10831cedee13SAndrey Ignatov dport = usin->sin_port; 10841cedee13SAndrey Ignatov } 10851cedee13SAndrey Ignatov } 10861cedee13SAndrey Ignatov 1087db8dac20SDavid S. Miller saddr = ipc.addr; 1088db8dac20SDavid S. Miller ipc.addr = faddr = daddr; 1089db8dac20SDavid S. Miller 1090f6d8bd05SEric Dumazet if (ipc.opt && ipc.opt->opt.srr) { 10911b97013bSAndrey Ignatov if (!daddr) { 10921b97013bSAndrey Ignatov err = -EINVAL; 10931b97013bSAndrey Ignatov goto out_free; 10941b97013bSAndrey Ignatov } 1095f6d8bd05SEric Dumazet faddr = ipc.opt->opt.faddr; 1096db8dac20SDavid S. Miller connected = 0; 1097db8dac20SDavid S. Miller } 1098aa661581SFrancesco Fusco tos = get_rttos(&ipc, inet); 1099db8dac20SDavid S. Miller if (sock_flag(sk, SOCK_LOCALROUTE) || 1100db8dac20SDavid S. Miller (msg->msg_flags & MSG_DONTROUTE) || 1101f6d8bd05SEric Dumazet (ipc.opt && ipc.opt->opt.is_strictroute)) { 1102db8dac20SDavid S. Miller tos |= RTO_ONLINK; 1103db8dac20SDavid S. Miller connected = 0; 1104db8dac20SDavid S. Miller } 1105db8dac20SDavid S. Miller 1106db8dac20SDavid S. Miller if (ipv4_is_multicast(daddr)) { 1107854da991SRobert Shearman if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif)) 1108db8dac20SDavid S. Miller ipc.oif = inet->mc_index; 1109db8dac20SDavid S. Miller if (!saddr) 1110db8dac20SDavid S. Miller saddr = inet->mc_addr; 1111db8dac20SDavid S. Miller connected = 0; 11129515a2e0SDavid Ahern } else if (!ipc.oif) { 111376e21053SErich E. Hoover ipc.oif = inet->uc_index; 11149515a2e0SDavid Ahern } else if (ipv4_is_lbcast(daddr) && inet->uc_index) { 11159515a2e0SDavid Ahern /* oif is set, packet is to local broadcast and 11169515a2e0SDavid Ahern * and uc_index is set. oif is most likely set 11179515a2e0SDavid Ahern * by sk_bound_dev_if. If uc_index != oif check if the 11189515a2e0SDavid Ahern * oif is an L3 master and uc_index is an L3 slave. 11199515a2e0SDavid Ahern * If so, we want to allow the send using the uc_index. 11209515a2e0SDavid Ahern */ 11219515a2e0SDavid Ahern if (ipc.oif != inet->uc_index && 11229515a2e0SDavid Ahern ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk), 11239515a2e0SDavid Ahern inet->uc_index)) { 11249515a2e0SDavid Ahern ipc.oif = inet->uc_index; 11259515a2e0SDavid Ahern } 11269515a2e0SDavid Ahern } 1127db8dac20SDavid S. Miller 1128db8dac20SDavid S. Miller if (connected) 1129db8dac20SDavid S. Miller rt = (struct rtable *)sk_dst_check(sk, 0); 1130db8dac20SDavid S. Miller 113151456b29SIan Morris if (!rt) { 113284a3aa00SPavel Emelyanov struct net *net = sock_net(sk); 11339a24abfaSDavid Ahern __u8 flow_flags = inet_sk_flowi_flags(sk); 113484a3aa00SPavel Emelyanov 1135e474995fSDavid S. Miller fl4 = &fl4_stack; 11369a24abfaSDavid Ahern 1137c6af0c22SWillem de Bruijn flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos, 11389a24abfaSDavid Ahern RT_SCOPE_UNIVERSE, sk->sk_protocol, 11399a24abfaSDavid Ahern flow_flags, 1140e2d118a1SLorenzo Colitti faddr, saddr, dport, inet->inet_sport, 1141e2d118a1SLorenzo Colitti sk->sk_uid); 1142c0951cbcSDavid S. Miller 1143e474995fSDavid S. Miller security_sk_classify_flow(sk, flowi4_to_flowi(fl4)); 1144e474995fSDavid S. Miller rt = ip_route_output_flow(net, fl4, sk); 1145b23dd4feSDavid S. Miller if (IS_ERR(rt)) { 1146b23dd4feSDavid S. Miller err = PTR_ERR(rt); 114706dc94b1SDavid S. Miller rt = NULL; 1148db8dac20SDavid S. Miller if (err == -ENETUNREACH) 1149f1d8cba6SEric Dumazet IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 1150db8dac20SDavid S. Miller goto out; 1151db8dac20SDavid S. Miller } 1152db8dac20SDavid S. Miller 1153db8dac20SDavid S. Miller err = -EACCES; 1154db8dac20SDavid S. Miller if ((rt->rt_flags & RTCF_BROADCAST) && 1155db8dac20SDavid S. Miller !sock_flag(sk, SOCK_BROADCAST)) 1156db8dac20SDavid S. Miller goto out; 1157db8dac20SDavid S. Miller if (connected) 1158d8d1f30bSChangli Gao sk_dst_set(sk, dst_clone(&rt->dst)); 1159db8dac20SDavid S. Miller } 1160db8dac20SDavid S. Miller 1161db8dac20SDavid S. Miller if (msg->msg_flags&MSG_CONFIRM) 1162db8dac20SDavid S. Miller goto do_confirm; 1163db8dac20SDavid S. Miller back_from_confirm: 1164db8dac20SDavid S. Miller 1165e474995fSDavid S. Miller saddr = fl4->saddr; 1166db8dac20SDavid S. Miller if (!ipc.addr) 1167e474995fSDavid S. Miller daddr = ipc.addr = fl4->daddr; 1168db8dac20SDavid S. Miller 1169903ab86dSHerbert Xu /* Lockless fast path for the non-corking case. */ 1170903ab86dSHerbert Xu if (!corkreq) { 11711cd7884dSWillem de Bruijn struct inet_cork cork; 11721cd7884dSWillem de Bruijn 1173f69e6d13SAl Viro skb = ip_make_skb(sk, fl4, getfrag, msg, ulen, 1174903ab86dSHerbert Xu sizeof(struct udphdr), &ipc, &rt, 11751cd7884dSWillem de Bruijn &cork, msg->msg_flags); 1176903ab86dSHerbert Xu err = PTR_ERR(skb); 117750c3a487SYOSHIFUJI Hideaki / 吉藤英明 if (!IS_ERR_OR_NULL(skb)) 1178bec1f6f6SWillem de Bruijn err = udp_send_skb(skb, fl4, &cork); 1179903ab86dSHerbert Xu goto out; 1180903ab86dSHerbert Xu } 1181903ab86dSHerbert Xu 1182db8dac20SDavid S. Miller lock_sock(sk); 1183db8dac20SDavid S. Miller if (unlikely(up->pending)) { 1184db8dac20SDavid S. Miller /* The socket is already corked while preparing it. */ 1185db8dac20SDavid S. Miller /* ... which is an evident application bug. --ANK */ 1186db8dac20SDavid S. Miller release_sock(sk); 1187db8dac20SDavid S. Miller 1188197df02cSMatteo Croce net_dbg_ratelimited("socket already corked\n"); 1189db8dac20SDavid S. Miller err = -EINVAL; 1190db8dac20SDavid S. Miller goto out; 1191db8dac20SDavid S. Miller } 1192db8dac20SDavid S. Miller /* 1193db8dac20SDavid S. Miller * Now cork the socket to pend data. 1194db8dac20SDavid S. Miller */ 1195b6f21b26SDavid S. Miller fl4 = &inet->cork.fl.u.ip4; 1196b6f21b26SDavid S. Miller fl4->daddr = daddr; 1197b6f21b26SDavid S. Miller fl4->saddr = saddr; 11989cce96dfSDavid S. Miller fl4->fl4_dport = dport; 11999cce96dfSDavid S. Miller fl4->fl4_sport = inet->inet_sport; 1200db8dac20SDavid S. Miller up->pending = AF_INET; 1201db8dac20SDavid S. Miller 1202db8dac20SDavid S. Miller do_append_data: 1203db8dac20SDavid S. Miller up->len += ulen; 1204f69e6d13SAl Viro err = ip_append_data(sk, fl4, getfrag, msg, ulen, 12052e77d89bSEric Dumazet sizeof(struct udphdr), &ipc, &rt, 1206db8dac20SDavid S. Miller corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); 1207db8dac20SDavid S. Miller if (err) 1208db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 1209db8dac20SDavid S. Miller else if (!corkreq) 1210db8dac20SDavid S. Miller err = udp_push_pending_frames(sk); 1211db8dac20SDavid S. Miller else if (unlikely(skb_queue_empty(&sk->sk_write_queue))) 1212db8dac20SDavid S. Miller up->pending = 0; 1213db8dac20SDavid S. Miller release_sock(sk); 1214db8dac20SDavid S. Miller 1215db8dac20SDavid S. Miller out: 1216db8dac20SDavid S. Miller ip_rt_put(rt); 12171b97013bSAndrey Ignatov out_free: 1218db8dac20SDavid S. Miller if (free) 1219db8dac20SDavid S. Miller kfree(ipc.opt); 1220db8dac20SDavid S. Miller if (!err) 1221db8dac20SDavid S. Miller return len; 1222db8dac20SDavid S. Miller /* 1223db8dac20SDavid S. Miller * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting 1224db8dac20SDavid S. Miller * ENOBUFS might not be good (it's not tunable per se), but otherwise 1225db8dac20SDavid S. Miller * we don't have a good statistic (IpOutDiscards but it can be too many 1226db8dac20SDavid S. Miller * things). We could add another new stat but at least for now that 1227db8dac20SDavid S. Miller * seems like overkill. 1228db8dac20SDavid S. Miller */ 1229db8dac20SDavid S. Miller if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 12306aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1231629ca23cSPavel Emelyanov UDP_MIB_SNDBUFERRORS, is_udplite); 1232db8dac20SDavid S. Miller } 1233db8dac20SDavid S. Miller return err; 1234db8dac20SDavid S. Miller 1235db8dac20SDavid S. Miller do_confirm: 12360dec879fSJulian Anastasov if (msg->msg_flags & MSG_PROBE) 12370dec879fSJulian Anastasov dst_confirm_neigh(&rt->dst, &fl4->daddr); 1238db8dac20SDavid S. Miller if (!(msg->msg_flags&MSG_PROBE) || len) 1239db8dac20SDavid S. Miller goto back_from_confirm; 1240db8dac20SDavid S. Miller err = 0; 1241db8dac20SDavid S. Miller goto out; 1242db8dac20SDavid S. Miller } 1243c482c568SEric Dumazet EXPORT_SYMBOL(udp_sendmsg); 1244db8dac20SDavid S. Miller 1245db8dac20SDavid S. Miller int udp_sendpage(struct sock *sk, struct page *page, int offset, 1246db8dac20SDavid S. Miller size_t size, int flags) 1247db8dac20SDavid S. Miller { 1248f5fca608SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 1249db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 1250db8dac20SDavid S. Miller int ret; 1251db8dac20SDavid S. Miller 1252d3f7d56aSShawn Landden if (flags & MSG_SENDPAGE_NOTLAST) 1253d3f7d56aSShawn Landden flags |= MSG_MORE; 1254d3f7d56aSShawn Landden 1255db8dac20SDavid S. Miller if (!up->pending) { 1256db8dac20SDavid S. Miller struct msghdr msg = { .msg_flags = flags|MSG_MORE }; 1257db8dac20SDavid S. Miller 1258db8dac20SDavid S. Miller /* Call udp_sendmsg to specify destination address which 1259db8dac20SDavid S. Miller * sendpage interface can't pass. 1260db8dac20SDavid S. Miller * This will succeed only when the socket is connected. 1261db8dac20SDavid S. Miller */ 12621b784140SYing Xue ret = udp_sendmsg(sk, &msg, 0); 1263db8dac20SDavid S. Miller if (ret < 0) 1264db8dac20SDavid S. Miller return ret; 1265db8dac20SDavid S. Miller } 1266db8dac20SDavid S. Miller 1267db8dac20SDavid S. Miller lock_sock(sk); 1268db8dac20SDavid S. Miller 1269db8dac20SDavid S. Miller if (unlikely(!up->pending)) { 1270db8dac20SDavid S. Miller release_sock(sk); 1271db8dac20SDavid S. Miller 1272197df02cSMatteo Croce net_dbg_ratelimited("cork failed\n"); 1273db8dac20SDavid S. Miller return -EINVAL; 1274db8dac20SDavid S. Miller } 1275db8dac20SDavid S. Miller 1276f5fca608SDavid S. Miller ret = ip_append_page(sk, &inet->cork.fl.u.ip4, 1277f5fca608SDavid S. Miller page, offset, size, flags); 1278db8dac20SDavid S. Miller if (ret == -EOPNOTSUPP) { 1279db8dac20SDavid S. Miller release_sock(sk); 1280db8dac20SDavid S. Miller return sock_no_sendpage(sk->sk_socket, page, offset, 1281db8dac20SDavid S. Miller size, flags); 1282db8dac20SDavid S. Miller } 1283db8dac20SDavid S. Miller if (ret < 0) { 1284db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 1285db8dac20SDavid S. Miller goto out; 1286db8dac20SDavid S. Miller } 1287db8dac20SDavid S. Miller 1288db8dac20SDavid S. Miller up->len += size; 1289db8dac20SDavid S. Miller if (!(up->corkflag || (flags&MSG_MORE))) 1290db8dac20SDavid S. Miller ret = udp_push_pending_frames(sk); 1291db8dac20SDavid S. Miller if (!ret) 1292db8dac20SDavid S. Miller ret = size; 1293db8dac20SDavid S. Miller out: 1294db8dac20SDavid S. Miller release_sock(sk); 1295db8dac20SDavid S. Miller return ret; 1296db8dac20SDavid S. Miller } 1297db8dac20SDavid S. Miller 1298dce4551cSPaolo Abeni #define UDP_SKB_IS_STATELESS 0x80000000 1299dce4551cSPaolo Abeni 1300b65ac446SPaolo Abeni static void udp_set_dev_scratch(struct sk_buff *skb) 1301b65ac446SPaolo Abeni { 1302dce4551cSPaolo Abeni struct udp_dev_scratch *scratch = udp_skb_scratch(skb); 1303b65ac446SPaolo Abeni 1304b65ac446SPaolo Abeni BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long)); 1305dce4551cSPaolo Abeni scratch->_tsize_state = skb->truesize; 1306dce4551cSPaolo Abeni #if BITS_PER_LONG == 64 1307b65ac446SPaolo Abeni scratch->len = skb->len; 1308b65ac446SPaolo Abeni scratch->csum_unnecessary = !!skb_csum_unnecessary(skb); 1309b65ac446SPaolo Abeni scratch->is_linear = !skb_is_nonlinear(skb); 1310b65ac446SPaolo Abeni #endif 13113bdefdf9SPaolo Abeni /* all head states execept sp (dst, sk, nf) are always cleared by 13123bdefdf9SPaolo Abeni * udp_rcv() and we need to preserve secpath, if present, to eventually 13133bdefdf9SPaolo Abeni * process IP_CMSG_PASSSEC at recvmsg() time 13143bdefdf9SPaolo Abeni */ 13153bdefdf9SPaolo Abeni if (likely(!skb_sec_path(skb))) 1316dce4551cSPaolo Abeni scratch->_tsize_state |= UDP_SKB_IS_STATELESS; 1317dce4551cSPaolo Abeni } 1318dce4551cSPaolo Abeni 1319a793183cSEric Dumazet static void udp_skb_csum_unnecessary_set(struct sk_buff *skb) 1320a793183cSEric Dumazet { 1321a793183cSEric Dumazet /* We come here after udp_lib_checksum_complete() returned 0. 1322a793183cSEric Dumazet * This means that __skb_checksum_complete() might have 1323a793183cSEric Dumazet * set skb->csum_valid to 1. 1324a793183cSEric Dumazet * On 64bit platforms, we can set csum_unnecessary 1325a793183cSEric Dumazet * to true, but only if the skb is not shared. 1326a793183cSEric Dumazet */ 1327a793183cSEric Dumazet #if BITS_PER_LONG == 64 1328a793183cSEric Dumazet if (!skb_shared(skb)) 1329a793183cSEric Dumazet udp_skb_scratch(skb)->csum_unnecessary = true; 1330a793183cSEric Dumazet #endif 1331a793183cSEric Dumazet } 1332a793183cSEric Dumazet 1333dce4551cSPaolo Abeni static int udp_skb_truesize(struct sk_buff *skb) 1334dce4551cSPaolo Abeni { 1335dce4551cSPaolo Abeni return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS; 1336dce4551cSPaolo Abeni } 1337dce4551cSPaolo Abeni 1338dce4551cSPaolo Abeni static bool udp_skb_has_head_state(struct sk_buff *skb) 1339dce4551cSPaolo Abeni { 1340dce4551cSPaolo Abeni return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS); 1341dce4551cSPaolo Abeni } 1342b65ac446SPaolo Abeni 13437c13f97fSPaolo Abeni /* fully reclaim rmem/fwd memory allocated for skb */ 13446dfb4367SPaolo Abeni static void udp_rmem_release(struct sock *sk, int size, int partial, 13456dfb4367SPaolo Abeni bool rx_queue_lock_held) 1346f970bd9eSPaolo Abeni { 13476b229cf7SEric Dumazet struct udp_sock *up = udp_sk(sk); 13482276f58aSPaolo Abeni struct sk_buff_head *sk_queue; 1349f970bd9eSPaolo Abeni int amt; 1350f970bd9eSPaolo Abeni 13516b229cf7SEric Dumazet if (likely(partial)) { 13526b229cf7SEric Dumazet up->forward_deficit += size; 13536b229cf7SEric Dumazet size = up->forward_deficit; 13540d4a6608SPaolo Abeni if (size < (sk->sk_rcvbuf >> 2)) 13556b229cf7SEric Dumazet return; 13566b229cf7SEric Dumazet } else { 13576b229cf7SEric Dumazet size += up->forward_deficit; 13586b229cf7SEric Dumazet } 13596b229cf7SEric Dumazet up->forward_deficit = 0; 13606b229cf7SEric Dumazet 13616dfb4367SPaolo Abeni /* acquire the sk_receive_queue for fwd allocated memory scheduling, 13626dfb4367SPaolo Abeni * if the called don't held it already 13636dfb4367SPaolo Abeni */ 13642276f58aSPaolo Abeni sk_queue = &sk->sk_receive_queue; 13656dfb4367SPaolo Abeni if (!rx_queue_lock_held) 13662276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 13672276f58aSPaolo Abeni 13686dfb4367SPaolo Abeni 1369f970bd9eSPaolo Abeni sk->sk_forward_alloc += size; 1370f970bd9eSPaolo Abeni amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1); 1371f970bd9eSPaolo Abeni sk->sk_forward_alloc -= amt; 1372f970bd9eSPaolo Abeni 1373f970bd9eSPaolo Abeni if (amt) 1374f970bd9eSPaolo Abeni __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT); 137502ab0d13SEric Dumazet 137602ab0d13SEric Dumazet atomic_sub(size, &sk->sk_rmem_alloc); 13772276f58aSPaolo Abeni 13782276f58aSPaolo Abeni /* this can save us from acquiring the rx queue lock on next receive */ 13792276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, &up->reader_queue); 13802276f58aSPaolo Abeni 13816dfb4367SPaolo Abeni if (!rx_queue_lock_held) 13822276f58aSPaolo Abeni spin_unlock(&sk_queue->lock); 1383f970bd9eSPaolo Abeni } 1384f970bd9eSPaolo Abeni 13852276f58aSPaolo Abeni /* Note: called with reader_queue.lock held. 1386c84d9490SEric Dumazet * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch 1387c84d9490SEric Dumazet * This avoids a cache line miss while receive_queue lock is held. 1388c84d9490SEric Dumazet * Look at __udp_enqueue_schedule_skb() to find where this copy is done. 1389c84d9490SEric Dumazet */ 13907c13f97fSPaolo Abeni void udp_skb_destructor(struct sock *sk, struct sk_buff *skb) 1391f970bd9eSPaolo Abeni { 1392b65ac446SPaolo Abeni prefetch(&skb->data); 1393b65ac446SPaolo Abeni udp_rmem_release(sk, udp_skb_truesize(skb), 1, false); 1394f970bd9eSPaolo Abeni } 13957c13f97fSPaolo Abeni EXPORT_SYMBOL(udp_skb_destructor); 1396f970bd9eSPaolo Abeni 13976dfb4367SPaolo Abeni /* as above, but the caller held the rx queue lock, too */ 139864f5102dSColin Ian King static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb) 13996dfb4367SPaolo Abeni { 1400b65ac446SPaolo Abeni prefetch(&skb->data); 1401b65ac446SPaolo Abeni udp_rmem_release(sk, udp_skb_truesize(skb), 1, true); 14026dfb4367SPaolo Abeni } 14036dfb4367SPaolo Abeni 14044b272750SEric Dumazet /* Idea of busylocks is to let producers grab an extra spinlock 14054b272750SEric Dumazet * to relieve pressure on the receive_queue spinlock shared by consumer. 14064b272750SEric Dumazet * Under flood, this means that only one producer can be in line 14074b272750SEric Dumazet * trying to acquire the receive_queue spinlock. 14084b272750SEric Dumazet * These busylock can be allocated on a per cpu manner, instead of a 14094b272750SEric Dumazet * per socket one (that would consume a cache line per socket) 14104b272750SEric Dumazet */ 14114b272750SEric Dumazet static int udp_busylocks_log __read_mostly; 14124b272750SEric Dumazet static spinlock_t *udp_busylocks __read_mostly; 14134b272750SEric Dumazet 14144b272750SEric Dumazet static spinlock_t *busylock_acquire(void *ptr) 14154b272750SEric Dumazet { 14164b272750SEric Dumazet spinlock_t *busy; 14174b272750SEric Dumazet 14184b272750SEric Dumazet busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log); 14194b272750SEric Dumazet spin_lock(busy); 14204b272750SEric Dumazet return busy; 14214b272750SEric Dumazet } 14224b272750SEric Dumazet 14234b272750SEric Dumazet static void busylock_release(spinlock_t *busy) 14244b272750SEric Dumazet { 14254b272750SEric Dumazet if (busy) 14264b272750SEric Dumazet spin_unlock(busy); 14274b272750SEric Dumazet } 14284b272750SEric Dumazet 1429f970bd9eSPaolo Abeni int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) 1430f970bd9eSPaolo Abeni { 1431f970bd9eSPaolo Abeni struct sk_buff_head *list = &sk->sk_receive_queue; 1432f970bd9eSPaolo Abeni int rmem, delta, amt, err = -ENOMEM; 14334b272750SEric Dumazet spinlock_t *busy = NULL; 1434c8c8b127SEric Dumazet int size; 1435f970bd9eSPaolo Abeni 1436f970bd9eSPaolo Abeni /* try to avoid the costly atomic add/sub pair when the receive 1437f970bd9eSPaolo Abeni * queue is full; always allow at least a packet 1438f970bd9eSPaolo Abeni */ 1439f970bd9eSPaolo Abeni rmem = atomic_read(&sk->sk_rmem_alloc); 1440363dc73aSPaolo Abeni if (rmem > sk->sk_rcvbuf) 1441f970bd9eSPaolo Abeni goto drop; 1442f970bd9eSPaolo Abeni 1443c8c8b127SEric Dumazet /* Under mem pressure, it might be helpful to help udp_recvmsg() 1444c8c8b127SEric Dumazet * having linear skbs : 1445c8c8b127SEric Dumazet * - Reduce memory overhead and thus increase receive queue capacity 1446c8c8b127SEric Dumazet * - Less cache line misses at copyout() time 1447c8c8b127SEric Dumazet * - Less work at consume_skb() (less alien page frag freeing) 1448c8c8b127SEric Dumazet */ 14494b272750SEric Dumazet if (rmem > (sk->sk_rcvbuf >> 1)) { 1450c8c8b127SEric Dumazet skb_condense(skb); 14514b272750SEric Dumazet 14524b272750SEric Dumazet busy = busylock_acquire(sk); 14534b272750SEric Dumazet } 1454c8c8b127SEric Dumazet size = skb->truesize; 1455b65ac446SPaolo Abeni udp_set_dev_scratch(skb); 1456c8c8b127SEric Dumazet 1457f970bd9eSPaolo Abeni /* we drop only if the receive buf is full and the receive 1458f970bd9eSPaolo Abeni * queue contains some other skb 1459f970bd9eSPaolo Abeni */ 1460f970bd9eSPaolo Abeni rmem = atomic_add_return(size, &sk->sk_rmem_alloc); 1461363dc73aSPaolo Abeni if (rmem > (size + sk->sk_rcvbuf)) 1462f970bd9eSPaolo Abeni goto uncharge_drop; 1463f970bd9eSPaolo Abeni 1464f970bd9eSPaolo Abeni spin_lock(&list->lock); 1465f970bd9eSPaolo Abeni if (size >= sk->sk_forward_alloc) { 1466f970bd9eSPaolo Abeni amt = sk_mem_pages(size); 1467f970bd9eSPaolo Abeni delta = amt << SK_MEM_QUANTUM_SHIFT; 1468f970bd9eSPaolo Abeni if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) { 1469f970bd9eSPaolo Abeni err = -ENOBUFS; 1470f970bd9eSPaolo Abeni spin_unlock(&list->lock); 1471f970bd9eSPaolo Abeni goto uncharge_drop; 1472f970bd9eSPaolo Abeni } 1473f970bd9eSPaolo Abeni 1474f970bd9eSPaolo Abeni sk->sk_forward_alloc += delta; 1475f970bd9eSPaolo Abeni } 1476f970bd9eSPaolo Abeni 1477f970bd9eSPaolo Abeni sk->sk_forward_alloc -= size; 1478f970bd9eSPaolo Abeni 14797c13f97fSPaolo Abeni /* no need to setup a destructor, we will explicitly release the 14807c13f97fSPaolo Abeni * forward allocated memory on dequeue 14817c13f97fSPaolo Abeni */ 1482f970bd9eSPaolo Abeni sock_skb_set_dropcount(sk, skb); 1483f970bd9eSPaolo Abeni 1484f970bd9eSPaolo Abeni __skb_queue_tail(list, skb); 1485f970bd9eSPaolo Abeni spin_unlock(&list->lock); 1486f970bd9eSPaolo Abeni 1487f970bd9eSPaolo Abeni if (!sock_flag(sk, SOCK_DEAD)) 1488f970bd9eSPaolo Abeni sk->sk_data_ready(sk); 1489f970bd9eSPaolo Abeni 14904b272750SEric Dumazet busylock_release(busy); 1491f970bd9eSPaolo Abeni return 0; 1492f970bd9eSPaolo Abeni 1493f970bd9eSPaolo Abeni uncharge_drop: 1494f970bd9eSPaolo Abeni atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 1495f970bd9eSPaolo Abeni 1496f970bd9eSPaolo Abeni drop: 1497f970bd9eSPaolo Abeni atomic_inc(&sk->sk_drops); 14984b272750SEric Dumazet busylock_release(busy); 1499f970bd9eSPaolo Abeni return err; 1500f970bd9eSPaolo Abeni } 1501f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb); 1502f970bd9eSPaolo Abeni 1503c915fe13SPaolo Abeni void udp_destruct_sock(struct sock *sk) 1504f970bd9eSPaolo Abeni { 1505f970bd9eSPaolo Abeni /* reclaim completely the forward allocated memory */ 15062276f58aSPaolo Abeni struct udp_sock *up = udp_sk(sk); 15077c13f97fSPaolo Abeni unsigned int total = 0; 15087c13f97fSPaolo Abeni struct sk_buff *skb; 15097c13f97fSPaolo Abeni 15102276f58aSPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue); 15112276f58aSPaolo Abeni while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) { 15127c13f97fSPaolo Abeni total += skb->truesize; 15137c13f97fSPaolo Abeni kfree_skb(skb); 15147c13f97fSPaolo Abeni } 15156dfb4367SPaolo Abeni udp_rmem_release(sk, total, 0, true); 15167c13f97fSPaolo Abeni 1517f970bd9eSPaolo Abeni inet_sock_destruct(sk); 1518f970bd9eSPaolo Abeni } 1519c915fe13SPaolo Abeni EXPORT_SYMBOL_GPL(udp_destruct_sock); 1520f970bd9eSPaolo Abeni 1521f970bd9eSPaolo Abeni int udp_init_sock(struct sock *sk) 1522f970bd9eSPaolo Abeni { 15232276f58aSPaolo Abeni skb_queue_head_init(&udp_sk(sk)->reader_queue); 1524f970bd9eSPaolo Abeni sk->sk_destruct = udp_destruct_sock; 1525f970bd9eSPaolo Abeni return 0; 1526f970bd9eSPaolo Abeni } 1527f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(udp_init_sock); 1528f970bd9eSPaolo Abeni 1529f970bd9eSPaolo Abeni void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) 1530f970bd9eSPaolo Abeni { 1531f970bd9eSPaolo Abeni if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { 1532f970bd9eSPaolo Abeni bool slow = lock_sock_fast(sk); 1533f970bd9eSPaolo Abeni 1534f970bd9eSPaolo Abeni sk_peek_offset_bwd(sk, len); 1535f970bd9eSPaolo Abeni unlock_sock_fast(sk, slow); 1536f970bd9eSPaolo Abeni } 15370a463c78SPaolo Abeni 1538ca2c1418SPaolo Abeni if (!skb_unref(skb)) 1539ca2c1418SPaolo Abeni return; 1540ca2c1418SPaolo Abeni 1541dce4551cSPaolo Abeni /* In the more common cases we cleared the head states previously, 1542dce4551cSPaolo Abeni * see __udp_queue_rcv_skb(). 15430ddf3fb2SPaolo Abeni */ 1544dce4551cSPaolo Abeni if (unlikely(udp_skb_has_head_state(skb))) 15450ddf3fb2SPaolo Abeni skb_release_head_state(skb); 1546ca2c1418SPaolo Abeni __consume_stateless_skb(skb); 1547f970bd9eSPaolo Abeni } 1548f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(skb_consume_udp); 1549f970bd9eSPaolo Abeni 15502276f58aSPaolo Abeni static struct sk_buff *__first_packet_length(struct sock *sk, 15512276f58aSPaolo Abeni struct sk_buff_head *rcvq, 15522276f58aSPaolo Abeni int *total) 15532276f58aSPaolo Abeni { 15542276f58aSPaolo Abeni struct sk_buff *skb; 15552276f58aSPaolo Abeni 15569bd780f5SPaolo Abeni while ((skb = skb_peek(rcvq)) != NULL) { 15579bd780f5SPaolo Abeni if (udp_lib_checksum_complete(skb)) { 15582276f58aSPaolo Abeni __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, 15592276f58aSPaolo Abeni IS_UDPLITE(sk)); 15602276f58aSPaolo Abeni __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, 15612276f58aSPaolo Abeni IS_UDPLITE(sk)); 15622276f58aSPaolo Abeni atomic_inc(&sk->sk_drops); 15632276f58aSPaolo Abeni __skb_unlink(skb, rcvq); 15642276f58aSPaolo Abeni *total += skb->truesize; 15652276f58aSPaolo Abeni kfree_skb(skb); 15669bd780f5SPaolo Abeni } else { 1567a793183cSEric Dumazet udp_skb_csum_unnecessary_set(skb); 15689bd780f5SPaolo Abeni break; 15699bd780f5SPaolo Abeni } 15702276f58aSPaolo Abeni } 15712276f58aSPaolo Abeni return skb; 15722276f58aSPaolo Abeni } 15732276f58aSPaolo Abeni 157485584672SEric Dumazet /** 157585584672SEric Dumazet * first_packet_length - return length of first packet in receive queue 157685584672SEric Dumazet * @sk: socket 157785584672SEric Dumazet * 157885584672SEric Dumazet * Drops all bad checksum frames, until a valid one is found. 1579e83c6744SEric Dumazet * Returns the length of found skb, or -1 if none is found. 158085584672SEric Dumazet */ 1581e83c6744SEric Dumazet static int first_packet_length(struct sock *sk) 158285584672SEric Dumazet { 15832276f58aSPaolo Abeni struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue; 15842276f58aSPaolo Abeni struct sk_buff_head *sk_queue = &sk->sk_receive_queue; 158585584672SEric Dumazet struct sk_buff *skb; 15867c13f97fSPaolo Abeni int total = 0; 1587e83c6744SEric Dumazet int res; 158885584672SEric Dumazet 158985584672SEric Dumazet spin_lock_bh(&rcvq->lock); 15902276f58aSPaolo Abeni skb = __first_packet_length(sk, rcvq, &total); 1591137a0dbeSEric Dumazet if (!skb && !skb_queue_empty_lockless(sk_queue)) { 15922276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 15932276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, rcvq); 15942276f58aSPaolo Abeni spin_unlock(&sk_queue->lock); 15952276f58aSPaolo Abeni 15962276f58aSPaolo Abeni skb = __first_packet_length(sk, rcvq, &total); 159785584672SEric Dumazet } 1598e83c6744SEric Dumazet res = skb ? skb->len : -1; 15997c13f97fSPaolo Abeni if (total) 16006dfb4367SPaolo Abeni udp_rmem_release(sk, total, 1, false); 160185584672SEric Dumazet spin_unlock_bh(&rcvq->lock); 160285584672SEric Dumazet return res; 160385584672SEric Dumazet } 160485584672SEric Dumazet 16051da177e4SLinus Torvalds /* 16061da177e4SLinus Torvalds * IOCTL requests applicable to the UDP protocol 16071da177e4SLinus Torvalds */ 16081da177e4SLinus Torvalds 16091da177e4SLinus Torvalds int udp_ioctl(struct sock *sk, int cmd, unsigned long arg) 16101da177e4SLinus Torvalds { 16116516c655SStephen Hemminger switch (cmd) { 16121da177e4SLinus Torvalds case SIOCOUTQ: 16131da177e4SLinus Torvalds { 161431e6d363SEric Dumazet int amount = sk_wmem_alloc_get(sk); 161531e6d363SEric Dumazet 16161da177e4SLinus Torvalds return put_user(amount, (int __user *)arg); 16171da177e4SLinus Torvalds } 16181da177e4SLinus Torvalds 16191da177e4SLinus Torvalds case SIOCINQ: 16201da177e4SLinus Torvalds { 1621e83c6744SEric Dumazet int amount = max_t(int, 0, first_packet_length(sk)); 16221da177e4SLinus Torvalds 16231da177e4SLinus Torvalds return put_user(amount, (int __user *)arg); 16241da177e4SLinus Torvalds } 16251da177e4SLinus Torvalds 16261da177e4SLinus Torvalds default: 16271da177e4SLinus Torvalds return -ENOIOCTLCMD; 16281da177e4SLinus Torvalds } 16296516c655SStephen Hemminger 16306516c655SStephen Hemminger return 0; 16311da177e4SLinus Torvalds } 1632c482c568SEric Dumazet EXPORT_SYMBOL(udp_ioctl); 16331da177e4SLinus Torvalds 16342276f58aSPaolo Abeni struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags, 1635fd69c399SPaolo Abeni int noblock, int *off, int *err) 16362276f58aSPaolo Abeni { 16372276f58aSPaolo Abeni struct sk_buff_head *sk_queue = &sk->sk_receive_queue; 16382276f58aSPaolo Abeni struct sk_buff_head *queue; 16392276f58aSPaolo Abeni struct sk_buff *last; 16402276f58aSPaolo Abeni long timeo; 16412276f58aSPaolo Abeni int error; 16422276f58aSPaolo Abeni 16432276f58aSPaolo Abeni queue = &udp_sk(sk)->reader_queue; 16442276f58aSPaolo Abeni flags |= noblock ? MSG_DONTWAIT : 0; 16452276f58aSPaolo Abeni timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 16462276f58aSPaolo Abeni do { 16472276f58aSPaolo Abeni struct sk_buff *skb; 16482276f58aSPaolo Abeni 16492276f58aSPaolo Abeni error = sock_error(sk); 16502276f58aSPaolo Abeni if (error) 16512276f58aSPaolo Abeni break; 16522276f58aSPaolo Abeni 16532276f58aSPaolo Abeni error = -EAGAIN; 16542276f58aSPaolo Abeni do { 16552276f58aSPaolo Abeni spin_lock_bh(&queue->lock); 16562276f58aSPaolo Abeni skb = __skb_try_recv_from_queue(sk, queue, flags, 16572276f58aSPaolo Abeni udp_skb_destructor, 1658fd69c399SPaolo Abeni off, err, &last); 16592276f58aSPaolo Abeni if (skb) { 16602276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 16612276f58aSPaolo Abeni return skb; 16622276f58aSPaolo Abeni } 16632276f58aSPaolo Abeni 1664137a0dbeSEric Dumazet if (skb_queue_empty_lockless(sk_queue)) { 16652276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 16662276f58aSPaolo Abeni goto busy_check; 16672276f58aSPaolo Abeni } 16682276f58aSPaolo Abeni 16696dfb4367SPaolo Abeni /* refill the reader queue and walk it again 16706dfb4367SPaolo Abeni * keep both queues locked to avoid re-acquiring 16716dfb4367SPaolo Abeni * the sk_receive_queue lock if fwd memory scheduling 16726dfb4367SPaolo Abeni * is needed. 16736dfb4367SPaolo Abeni */ 16742276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 16752276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, queue); 16762276f58aSPaolo Abeni 16772276f58aSPaolo Abeni skb = __skb_try_recv_from_queue(sk, queue, flags, 16786dfb4367SPaolo Abeni udp_skb_dtor_locked, 1679fd69c399SPaolo Abeni off, err, &last); 16806dfb4367SPaolo Abeni spin_unlock(&sk_queue->lock); 16812276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 1682de321ed3SAndrey Vagin if (skb) 16832276f58aSPaolo Abeni return skb; 16842276f58aSPaolo Abeni 16852276f58aSPaolo Abeni busy_check: 16862276f58aSPaolo Abeni if (!sk_can_busy_loop(sk)) 16872276f58aSPaolo Abeni break; 16882276f58aSPaolo Abeni 16892276f58aSPaolo Abeni sk_busy_loop(sk, flags & MSG_DONTWAIT); 1690137a0dbeSEric Dumazet } while (!skb_queue_empty_lockless(sk_queue)); 16912276f58aSPaolo Abeni 16922276f58aSPaolo Abeni /* sk_queue is empty, reader_queue may contain peeked packets */ 16932276f58aSPaolo Abeni } while (timeo && 16942276f58aSPaolo Abeni !__skb_wait_for_more_packets(sk, &error, &timeo, 16952276f58aSPaolo Abeni (struct sk_buff *)sk_queue)); 16962276f58aSPaolo Abeni 16972276f58aSPaolo Abeni *err = error; 16982276f58aSPaolo Abeni return NULL; 16992276f58aSPaolo Abeni } 17007e823644SJiri Kosina EXPORT_SYMBOL(__skb_recv_udp); 17012276f58aSPaolo Abeni 1702db8dac20SDavid S. Miller /* 1703db8dac20SDavid S. Miller * This should be easy, if there is something there we 1704db8dac20SDavid S. Miller * return it, otherwise we block. 1705db8dac20SDavid S. Miller */ 1706db8dac20SDavid S. Miller 17071b784140SYing Xue int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, 17081b784140SYing Xue int flags, int *addr_len) 1709db8dac20SDavid S. Miller { 1710db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 1711342dfc30SSteffen Hurrle DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); 1712db8dac20SDavid S. Miller struct sk_buff *skb; 171359c2cdaeSDavid S. Miller unsigned int ulen, copied; 1714fd69c399SPaolo Abeni int off, err, peeking = flags & MSG_PEEK; 1715db8dac20SDavid S. Miller int is_udplite = IS_UDPLITE(sk); 1716197c949eSEric Dumazet bool checksum_valid = false; 1717db8dac20SDavid S. Miller 1718db8dac20SDavid S. Miller if (flags & MSG_ERRQUEUE) 171985fbaa75SHannes Frederic Sowa return ip_recv_error(sk, msg, len, addr_len); 1720db8dac20SDavid S. Miller 1721db8dac20SDavid S. Miller try_again: 1722a0917e0bSMatthew Dawson off = sk_peek_offset(sk, flags); 1723fd69c399SPaolo Abeni skb = __skb_recv_udp(sk, flags, noblock, &off, &err); 1724db8dac20SDavid S. Miller if (!skb) 1725627d2d6bSsamanthakumar return err; 1726db8dac20SDavid S. Miller 1727b65ac446SPaolo Abeni ulen = udp_skb_len(skb); 172859c2cdaeSDavid S. Miller copied = len; 1729627d2d6bSsamanthakumar if (copied > ulen - off) 1730627d2d6bSsamanthakumar copied = ulen - off; 173159c2cdaeSDavid S. Miller else if (copied < ulen) 1732db8dac20SDavid S. Miller msg->msg_flags |= MSG_TRUNC; 1733db8dac20SDavid S. Miller 1734db8dac20SDavid S. Miller /* 1735db8dac20SDavid S. Miller * If checksum is needed at all, try to do it while copying the 1736db8dac20SDavid S. Miller * data. If the data is truncated, or if we only want a partial 1737db8dac20SDavid S. Miller * coverage checksum (UDP-Lite), do it before the copy. 1738db8dac20SDavid S. Miller */ 1739db8dac20SDavid S. Miller 1740d21dbdfeSEric Dumazet if (copied < ulen || peeking || 1741d21dbdfeSEric Dumazet (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { 1742b65ac446SPaolo Abeni checksum_valid = udp_skb_csum_unnecessary(skb) || 1743b65ac446SPaolo Abeni !__udp_lib_checksum_complete(skb); 1744197c949eSEric Dumazet if (!checksum_valid) 1745db8dac20SDavid S. Miller goto csum_copy_err; 1746db8dac20SDavid S. Miller } 1747db8dac20SDavid S. Miller 1748b65ac446SPaolo Abeni if (checksum_valid || udp_skb_csum_unnecessary(skb)) { 1749b65ac446SPaolo Abeni if (udp_skb_is_linear(skb)) 1750b65ac446SPaolo Abeni err = copy_linear_skb(skb, copied, off, &msg->msg_iter); 1751b65ac446SPaolo Abeni else 1752627d2d6bSsamanthakumar err = skb_copy_datagram_msg(skb, off, msg, copied); 1753b65ac446SPaolo Abeni } else { 1754627d2d6bSsamanthakumar err = skb_copy_and_csum_datagram_msg(skb, off, msg); 1755db8dac20SDavid S. Miller 1756db8dac20SDavid S. Miller if (err == -EINVAL) 1757db8dac20SDavid S. Miller goto csum_copy_err; 1758db8dac20SDavid S. Miller } 1759db8dac20SDavid S. Miller 176022911fc5SEric Dumazet if (unlikely(err)) { 1761fd69c399SPaolo Abeni if (!peeking) { 1762979402b1SEric Dumazet atomic_inc(&sk->sk_drops); 17636aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1764979402b1SEric Dumazet UDP_MIB_INERRORS, is_udplite); 1765979402b1SEric Dumazet } 1766850cbaddSPaolo Abeni kfree_skb(skb); 1767627d2d6bSsamanthakumar return err; 176822911fc5SEric Dumazet } 1769db8dac20SDavid S. Miller 1770fd69c399SPaolo Abeni if (!peeking) 17716aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1772629ca23cSPavel Emelyanov UDP_MIB_INDATAGRAMS, is_udplite); 1773db8dac20SDavid S. Miller 17743b885787SNeil Horman sock_recv_ts_and_drops(msg, sk, skb); 1775db8dac20SDavid S. Miller 1776db8dac20SDavid S. Miller /* Copy the address. */ 1777c482c568SEric Dumazet if (sin) { 1778db8dac20SDavid S. Miller sin->sin_family = AF_INET; 1779db8dac20SDavid S. Miller sin->sin_port = udp_hdr(skb)->source; 1780db8dac20SDavid S. Miller sin->sin_addr.s_addr = ip_hdr(skb)->saddr; 1781db8dac20SDavid S. Miller memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); 1782bceaa902SHannes Frederic Sowa *addr_len = sizeof(*sin); 1783983695faSDaniel Borkmann 1784983695faSDaniel Borkmann if (cgroup_bpf_enabled) 1785983695faSDaniel Borkmann BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, 1786983695faSDaniel Borkmann (struct sockaddr *)sin); 1787db8dac20SDavid S. Miller } 1788bcd1665eSPaolo Abeni 1789bcd1665eSPaolo Abeni if (udp_sk(sk)->gro_enabled) 1790bcd1665eSPaolo Abeni udp_cmsg_recv(msg, sk, skb); 1791bcd1665eSPaolo Abeni 1792db8dac20SDavid S. Miller if (inet->cmsg_flags) 1793ad959036SPaolo Abeni ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); 1794db8dac20SDavid S. Miller 179559c2cdaeSDavid S. Miller err = copied; 1796db8dac20SDavid S. Miller if (flags & MSG_TRUNC) 1797db8dac20SDavid S. Miller err = ulen; 1798db8dac20SDavid S. Miller 1799850cbaddSPaolo Abeni skb_consume_udp(sk, skb, peeking ? -err : err); 1800db8dac20SDavid S. Miller return err; 1801db8dac20SDavid S. Miller 1802db8dac20SDavid S. Miller csum_copy_err: 18032276f58aSPaolo Abeni if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, 18042276f58aSPaolo Abeni udp_skb_destructor)) { 18056aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); 18066aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 18076a5dc9e5SEric Dumazet } 1808850cbaddSPaolo Abeni kfree_skb(skb); 1809db8dac20SDavid S. Miller 1810beb39db5SEric Dumazet /* starting over for a new packet, but check if we need to yield */ 1811beb39db5SEric Dumazet cond_resched(); 18129cfaa8deSXufeng Zhang msg->msg_flags &= ~MSG_TRUNC; 1813db8dac20SDavid S. Miller goto try_again; 1814db8dac20SDavid S. Miller } 1815db8dac20SDavid S. Miller 1816d74bad4eSAndrey Ignatov int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) 1817d74bad4eSAndrey Ignatov { 1818d74bad4eSAndrey Ignatov /* This check is replicated from __ip4_datagram_connect() and 1819d74bad4eSAndrey Ignatov * intended to prevent BPF program called below from accessing bytes 1820d74bad4eSAndrey Ignatov * that are out of the bound specified by user in addr_len. 1821d74bad4eSAndrey Ignatov */ 1822d74bad4eSAndrey Ignatov if (addr_len < sizeof(struct sockaddr_in)) 1823d74bad4eSAndrey Ignatov return -EINVAL; 1824d74bad4eSAndrey Ignatov 1825d74bad4eSAndrey Ignatov return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr); 1826d74bad4eSAndrey Ignatov } 1827d74bad4eSAndrey Ignatov EXPORT_SYMBOL(udp_pre_connect); 1828d74bad4eSAndrey Ignatov 1829286c72deSEric Dumazet int __udp_disconnect(struct sock *sk, int flags) 18301da177e4SLinus Torvalds { 18311da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 18321da177e4SLinus Torvalds /* 18331da177e4SLinus Torvalds * 1003.1g - break association. 18341da177e4SLinus Torvalds */ 18351da177e4SLinus Torvalds 18361da177e4SLinus Torvalds sk->sk_state = TCP_CLOSE; 1837c720c7e8SEric Dumazet inet->inet_daddr = 0; 1838c720c7e8SEric Dumazet inet->inet_dport = 0; 1839bdeab991STom Herbert sock_rps_reset_rxhash(sk); 18401da177e4SLinus Torvalds sk->sk_bound_dev_if = 0; 18411da177e4SLinus Torvalds if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 18421da177e4SLinus Torvalds inet_reset_saddr(sk); 18431da177e4SLinus Torvalds 18441da177e4SLinus Torvalds if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) { 18451da177e4SLinus Torvalds sk->sk_prot->unhash(sk); 1846c720c7e8SEric Dumazet inet->inet_sport = 0; 18471da177e4SLinus Torvalds } 18481da177e4SLinus Torvalds sk_dst_reset(sk); 18491da177e4SLinus Torvalds return 0; 18501da177e4SLinus Torvalds } 1851286c72deSEric Dumazet EXPORT_SYMBOL(__udp_disconnect); 1852286c72deSEric Dumazet 1853286c72deSEric Dumazet int udp_disconnect(struct sock *sk, int flags) 1854286c72deSEric Dumazet { 1855286c72deSEric Dumazet lock_sock(sk); 1856286c72deSEric Dumazet __udp_disconnect(sk, flags); 1857286c72deSEric Dumazet release_sock(sk); 1858286c72deSEric Dumazet return 0; 1859286c72deSEric Dumazet } 1860c482c568SEric Dumazet EXPORT_SYMBOL(udp_disconnect); 18611da177e4SLinus Torvalds 1862645ca708SEric Dumazet void udp_lib_unhash(struct sock *sk) 1863645ca708SEric Dumazet { 1864723b4610SEric Dumazet if (sk_hashed(sk)) { 1865645ca708SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 1866512615b6SEric Dumazet struct udp_hslot *hslot, *hslot2; 1867512615b6SEric Dumazet 1868512615b6SEric Dumazet hslot = udp_hashslot(udptable, sock_net(sk), 1869d4cada4aSEric Dumazet udp_sk(sk)->udp_port_hash); 1870512615b6SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 1871645ca708SEric Dumazet 1872c8db3fecSEric Dumazet spin_lock_bh(&hslot->lock); 1873e32ea7e7SCraig Gallek if (rcu_access_pointer(sk->sk_reuseport_cb)) 1874e32ea7e7SCraig Gallek reuseport_detach_sock(sk); 1875ca065d0cSEric Dumazet if (sk_del_node_init_rcu(sk)) { 1876fdcc8aa9SEric Dumazet hslot->count--; 1877c720c7e8SEric Dumazet inet_sk(sk)->inet_num = 0; 1878645ca708SEric Dumazet sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 1879512615b6SEric Dumazet 1880512615b6SEric Dumazet spin_lock(&hslot2->lock); 1881ca065d0cSEric Dumazet hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); 1882512615b6SEric Dumazet hslot2->count--; 1883512615b6SEric Dumazet spin_unlock(&hslot2->lock); 1884645ca708SEric Dumazet } 1885c8db3fecSEric Dumazet spin_unlock_bh(&hslot->lock); 1886645ca708SEric Dumazet } 1887723b4610SEric Dumazet } 1888645ca708SEric Dumazet EXPORT_SYMBOL(udp_lib_unhash); 1889645ca708SEric Dumazet 1890719f8358SEric Dumazet /* 1891719f8358SEric Dumazet * inet_rcv_saddr was changed, we must rehash secondary hash 1892719f8358SEric Dumazet */ 1893719f8358SEric Dumazet void udp_lib_rehash(struct sock *sk, u16 newhash) 1894719f8358SEric Dumazet { 1895719f8358SEric Dumazet if (sk_hashed(sk)) { 1896719f8358SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 1897719f8358SEric Dumazet struct udp_hslot *hslot, *hslot2, *nhslot2; 1898719f8358SEric Dumazet 1899719f8358SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 1900719f8358SEric Dumazet nhslot2 = udp_hashslot2(udptable, newhash); 1901719f8358SEric Dumazet udp_sk(sk)->udp_portaddr_hash = newhash; 1902e32ea7e7SCraig Gallek 1903e32ea7e7SCraig Gallek if (hslot2 != nhslot2 || 1904e32ea7e7SCraig Gallek rcu_access_pointer(sk->sk_reuseport_cb)) { 1905719f8358SEric Dumazet hslot = udp_hashslot(udptable, sock_net(sk), 1906719f8358SEric Dumazet udp_sk(sk)->udp_port_hash); 1907719f8358SEric Dumazet /* we must lock primary chain too */ 1908719f8358SEric Dumazet spin_lock_bh(&hslot->lock); 1909e32ea7e7SCraig Gallek if (rcu_access_pointer(sk->sk_reuseport_cb)) 1910e32ea7e7SCraig Gallek reuseport_detach_sock(sk); 1911719f8358SEric Dumazet 1912e32ea7e7SCraig Gallek if (hslot2 != nhslot2) { 1913719f8358SEric Dumazet spin_lock(&hslot2->lock); 1914ca065d0cSEric Dumazet hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); 1915719f8358SEric Dumazet hslot2->count--; 1916719f8358SEric Dumazet spin_unlock(&hslot2->lock); 1917719f8358SEric Dumazet 1918719f8358SEric Dumazet spin_lock(&nhslot2->lock); 1919ca065d0cSEric Dumazet hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, 1920719f8358SEric Dumazet &nhslot2->head); 1921719f8358SEric Dumazet nhslot2->count++; 1922719f8358SEric Dumazet spin_unlock(&nhslot2->lock); 1923e32ea7e7SCraig Gallek } 1924719f8358SEric Dumazet 1925719f8358SEric Dumazet spin_unlock_bh(&hslot->lock); 1926719f8358SEric Dumazet } 1927719f8358SEric Dumazet } 1928719f8358SEric Dumazet } 1929719f8358SEric Dumazet EXPORT_SYMBOL(udp_lib_rehash); 1930719f8358SEric Dumazet 19318f6b5392SAlexey Kodanev void udp_v4_rehash(struct sock *sk) 1932719f8358SEric Dumazet { 1933f0b1e64cSMartin KaFai Lau u16 new_hash = ipv4_portaddr_hash(sock_net(sk), 1934719f8358SEric Dumazet inet_sk(sk)->inet_rcv_saddr, 1935719f8358SEric Dumazet inet_sk(sk)->inet_num); 1936719f8358SEric Dumazet udp_lib_rehash(sk, new_hash); 1937719f8358SEric Dumazet } 1938719f8358SEric Dumazet 1939a3f96c47SPaolo Abeni static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 194093821778SHerbert Xu { 1941fec5e652STom Herbert int rc; 194293821778SHerbert Xu 1943005ec974SShawn Bohrer if (inet_sk(sk)->inet_daddr) { 1944bdeab991STom Herbert sock_rps_save_rxhash(sk, skb); 1945005ec974SShawn Bohrer sk_mark_napi_id(sk, skb); 19462c8c56e1SEric Dumazet sk_incoming_cpu_update(sk); 1947e68b6e50SEric Dumazet } else { 1948e68b6e50SEric Dumazet sk_mark_napi_id_once(sk, skb); 1949005ec974SShawn Bohrer } 1950fec5e652STom Herbert 1951850cbaddSPaolo Abeni rc = __udp_enqueue_schedule_skb(sk, skb); 1952766e9037SEric Dumazet if (rc < 0) { 1953766e9037SEric Dumazet int is_udplite = IS_UDPLITE(sk); 1954766e9037SEric Dumazet 195593821778SHerbert Xu /* Note that an ENOMEM error is charged twice */ 1956766e9037SEric Dumazet if (rc == -ENOMEM) 1957e61da9e2SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, 195893821778SHerbert Xu is_udplite); 1959e61da9e2SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 1960766e9037SEric Dumazet kfree_skb(skb); 1961296f7ea7SSatoru Moriya trace_udp_fail_queue_rcv_skb(rc, sk); 1962766e9037SEric Dumazet return -1; 196393821778SHerbert Xu } 196493821778SHerbert Xu 196593821778SHerbert Xu return 0; 196693821778SHerbert Xu } 196793821778SHerbert Xu 1968db8dac20SDavid S. Miller /* returns: 1969db8dac20SDavid S. Miller * -1: error 1970db8dac20SDavid S. Miller * 0: success 1971db8dac20SDavid S. Miller * >0: "udp encap" protocol resubmission 1972db8dac20SDavid S. Miller * 1973db8dac20SDavid S. Miller * Note that in the success and error cases, the skb is assumed to 1974db8dac20SDavid S. Miller * have either been requeued or freed. 1975db8dac20SDavid S. Miller */ 1976cf329aa4SPaolo Abeni static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) 1977db8dac20SDavid S. Miller { 1978db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 1979db8dac20SDavid S. Miller int is_udplite = IS_UDPLITE(sk); 1980db8dac20SDavid S. Miller 1981db8dac20SDavid S. Miller /* 1982db8dac20SDavid S. Miller * Charge it to the socket, dropping if the queue is full. 1983db8dac20SDavid S. Miller */ 1984db8dac20SDavid S. Miller if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) 1985db8dac20SDavid S. Miller goto drop; 1986895b5c9fSFlorian Westphal nf_reset_ct(skb); 1987db8dac20SDavid S. Miller 198888ab3108SDavidlohr Bueso if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) { 19890ad92ad0SEric Dumazet int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); 19900ad92ad0SEric Dumazet 1991db8dac20SDavid S. Miller /* 1992db8dac20SDavid S. Miller * This is an encapsulation socket so pass the skb to 1993db8dac20SDavid S. Miller * the socket's udp_encap_rcv() hook. Otherwise, just 1994db8dac20SDavid S. Miller * fall through and pass this up the UDP socket. 1995db8dac20SDavid S. Miller * up->encap_rcv() returns the following value: 1996db8dac20SDavid S. Miller * =0 if skb was successfully passed to the encap 1997db8dac20SDavid S. Miller * handler or was discarded by it. 1998db8dac20SDavid S. Miller * >0 if skb should be passed on to UDP. 1999db8dac20SDavid S. Miller * <0 if skb should be resubmitted as proto -N 2000db8dac20SDavid S. Miller */ 2001db8dac20SDavid S. Miller 2002db8dac20SDavid S. Miller /* if we're overly short, let UDP handle it */ 20036aa7de05SMark Rutland encap_rcv = READ_ONCE(up->encap_rcv); 2004e5aed006SHannes Frederic Sowa if (encap_rcv) { 2005db8dac20SDavid S. Miller int ret; 2006db8dac20SDavid S. Miller 20070a80966bSTom Herbert /* Verify checksum before giving to encap */ 20080a80966bSTom Herbert if (udp_lib_checksum_complete(skb)) 20090a80966bSTom Herbert goto csum_error; 20100a80966bSTom Herbert 20110ad92ad0SEric Dumazet ret = encap_rcv(sk, skb); 2012db8dac20SDavid S. Miller if (ret <= 0) { 201302c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), 20140283328eSPavel Emelyanov UDP_MIB_INDATAGRAMS, 2015db8dac20SDavid S. Miller is_udplite); 2016db8dac20SDavid S. Miller return -ret; 2017db8dac20SDavid S. Miller } 2018db8dac20SDavid S. Miller } 2019db8dac20SDavid S. Miller 2020db8dac20SDavid S. Miller /* FALLTHROUGH -- it's a UDP Packet */ 2021db8dac20SDavid S. Miller } 2022db8dac20SDavid S. Miller 2023db8dac20SDavid S. Miller /* 2024db8dac20SDavid S. Miller * UDP-Lite specific tests, ignored on UDP sockets 2025db8dac20SDavid S. Miller */ 2026db8dac20SDavid S. Miller if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) { 2027db8dac20SDavid S. Miller 2028db8dac20SDavid S. Miller /* 2029db8dac20SDavid S. Miller * MIB statistics other than incrementing the error count are 2030db8dac20SDavid S. Miller * disabled for the following two types of errors: these depend 2031db8dac20SDavid S. Miller * on the application settings, not on the functioning of the 2032db8dac20SDavid S. Miller * protocol stack as such. 2033db8dac20SDavid S. Miller * 2034db8dac20SDavid S. Miller * RFC 3828 here recommends (sec 3.3): "There should also be a 2035db8dac20SDavid S. Miller * way ... to ... at least let the receiving application block 2036db8dac20SDavid S. Miller * delivery of packets with coverage values less than a value 2037db8dac20SDavid S. Miller * provided by the application." 2038db8dac20SDavid S. Miller */ 2039db8dac20SDavid S. Miller if (up->pcrlen == 0) { /* full coverage was set */ 2040ba7a46f1SJoe Perches net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n", 2041db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov, skb->len); 2042db8dac20SDavid S. Miller goto drop; 2043db8dac20SDavid S. Miller } 2044db8dac20SDavid S. Miller /* The next case involves violating the min. coverage requested 2045db8dac20SDavid S. Miller * by the receiver. This is subtle: if receiver wants x and x is 2046db8dac20SDavid S. Miller * greater than the buffersize/MTU then receiver will complain 2047db8dac20SDavid S. Miller * that it wants x while sender emits packets of smaller size y. 2048db8dac20SDavid S. Miller * Therefore the above ...()->partial_cov statement is essential. 2049db8dac20SDavid S. Miller */ 2050db8dac20SDavid S. Miller if (UDP_SKB_CB(skb)->cscov < up->pcrlen) { 2051ba7a46f1SJoe Perches net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n", 2052db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov, up->pcrlen); 2053db8dac20SDavid S. Miller goto drop; 2054db8dac20SDavid S. Miller } 2055db8dac20SDavid S. Miller } 2056db8dac20SDavid S. Miller 2057dd99e425SPaolo Abeni prefetch(&sk->sk_rmem_alloc); 2058ce25d66aSEric Dumazet if (rcu_access_pointer(sk->sk_filter) && 2059ce25d66aSEric Dumazet udp_lib_checksum_complete(skb)) 20606a5dc9e5SEric Dumazet goto csum_error; 2061ce25d66aSEric Dumazet 2062ba66bbe5SDaniel Borkmann if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) 2063a6127697SMichal Kubeček goto drop; 2064db8dac20SDavid S. Miller 2065e6afc8acSsamanthakumar udp_csum_pull_header(skb); 2066db8dac20SDavid S. Miller 2067fbf8866dSShawn Bohrer ipv4_pktinfo_prepare(sk, skb); 2068850cbaddSPaolo Abeni return __udp_queue_rcv_skb(sk, skb); 2069db8dac20SDavid S. Miller 20706a5dc9e5SEric Dumazet csum_error: 207102c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); 2072db8dac20SDavid S. Miller drop: 207302c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 20748edf19c2SEric Dumazet atomic_inc(&sk->sk_drops); 2075db8dac20SDavid S. Miller kfree_skb(skb); 2076db8dac20SDavid S. Miller return -1; 2077db8dac20SDavid S. Miller } 2078db8dac20SDavid S. Miller 2079cf329aa4SPaolo Abeni static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 2080cf329aa4SPaolo Abeni { 2081cf329aa4SPaolo Abeni struct sk_buff *next, *segs; 2082cf329aa4SPaolo Abeni int ret; 2083cf329aa4SPaolo Abeni 2084cf329aa4SPaolo Abeni if (likely(!udp_unexpected_gso(sk, skb))) 2085cf329aa4SPaolo Abeni return udp_queue_rcv_one_skb(sk, skb); 2086cf329aa4SPaolo Abeni 2087cf329aa4SPaolo Abeni BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_SGO_CB_OFFSET); 2088cf329aa4SPaolo Abeni __skb_push(skb, -skb_mac_offset(skb)); 2089cf329aa4SPaolo Abeni segs = udp_rcv_segment(sk, skb, true); 2090cf329aa4SPaolo Abeni for (skb = segs; skb; skb = next) { 2091cf329aa4SPaolo Abeni next = skb->next; 2092cf329aa4SPaolo Abeni __skb_pull(skb, skb_transport_offset(skb)); 2093cf329aa4SPaolo Abeni ret = udp_queue_rcv_one_skb(sk, skb); 2094cf329aa4SPaolo Abeni if (ret > 0) 2095cf329aa4SPaolo Abeni ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret); 2096cf329aa4SPaolo Abeni } 2097cf329aa4SPaolo Abeni return 0; 2098cf329aa4SPaolo Abeni } 2099cf329aa4SPaolo Abeni 210097502231SEric Dumazet /* For TCP sockets, sk_rx_dst is protected by socket lock 2101e47eb5dfSEric Dumazet * For UDP, we use xchg() to guard against concurrent changes. 210297502231SEric Dumazet */ 210364f0f5d1SPaolo Abeni bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst) 2104421b3885SShawn Bohrer { 210597502231SEric Dumazet struct dst_entry *old; 2106421b3885SShawn Bohrer 2107d24406c8SWei Wang if (dst_hold_safe(dst)) { 2108e47eb5dfSEric Dumazet old = xchg(&sk->sk_rx_dst, dst); 210997502231SEric Dumazet dst_release(old); 211064f0f5d1SPaolo Abeni return old != dst; 211197502231SEric Dumazet } 211264f0f5d1SPaolo Abeni return false; 2113d24406c8SWei Wang } 2114c9f2c1aeSPaolo Abeni EXPORT_SYMBOL(udp_sk_rx_dst_set); 2115421b3885SShawn Bohrer 2116db8dac20SDavid S. Miller /* 2117db8dac20SDavid S. Miller * Multicasts and broadcasts go to each listener. 2118db8dac20SDavid S. Miller * 21191240d137SEric Dumazet * Note: called only from the BH handler context. 2120db8dac20SDavid S. Miller */ 2121e3163493SPavel Emelyanov static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 2122db8dac20SDavid S. Miller struct udphdr *uh, 2123db8dac20SDavid S. Miller __be32 saddr, __be32 daddr, 212436cbb245SRick Jones struct udp_table *udptable, 212536cbb245SRick Jones int proto) 2126db8dac20SDavid S. Miller { 2127ca065d0cSEric Dumazet struct sock *sk, *first = NULL; 21285cf3d461SDavid Held unsigned short hnum = ntohs(uh->dest); 21295cf3d461SDavid Held struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); 21302dc41cffSDavid Held unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); 2131ca065d0cSEric Dumazet unsigned int offset = offsetof(typeof(*sk), sk_node); 2132ca065d0cSEric Dumazet int dif = skb->dev->ifindex; 2133fb74c277SDavid Ahern int sdif = inet_sdif(skb); 2134ca065d0cSEric Dumazet struct hlist_node *node; 2135ca065d0cSEric Dumazet struct sk_buff *nskb; 21362dc41cffSDavid Held 21372dc41cffSDavid Held if (use_hash2) { 2138f0b1e64cSMartin KaFai Lau hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) & 213973e2d5e3SPablo Neira udptable->mask; 2140f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask; 21412dc41cffSDavid Held start_lookup: 214273e2d5e3SPablo Neira hslot = &udptable->hash2[hash2]; 21432dc41cffSDavid Held offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); 21442dc41cffSDavid Held } 2145db8dac20SDavid S. Miller 2146ca065d0cSEric Dumazet sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) { 2147ca065d0cSEric Dumazet if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr, 2148fb74c277SDavid Ahern uh->source, saddr, dif, sdif, hnum)) 2149ca065d0cSEric Dumazet continue; 21501240d137SEric Dumazet 2151ca065d0cSEric Dumazet if (!first) { 2152ca065d0cSEric Dumazet first = sk; 2153ca065d0cSEric Dumazet continue; 2154ca065d0cSEric Dumazet } 2155ca065d0cSEric Dumazet nskb = skb_clone(skb, GFP_ATOMIC); 2156ca065d0cSEric Dumazet 2157ca065d0cSEric Dumazet if (unlikely(!nskb)) { 2158ca065d0cSEric Dumazet atomic_inc(&sk->sk_drops); 215902c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS, 2160ca065d0cSEric Dumazet IS_UDPLITE(sk)); 216102c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_INERRORS, 2162ca065d0cSEric Dumazet IS_UDPLITE(sk)); 2163ca065d0cSEric Dumazet continue; 2164ca065d0cSEric Dumazet } 2165ca065d0cSEric Dumazet if (udp_queue_rcv_skb(sk, nskb) > 0) 2166ca065d0cSEric Dumazet consume_skb(nskb); 2167ca065d0cSEric Dumazet } 21681240d137SEric Dumazet 21692dc41cffSDavid Held /* Also lookup *:port if we are using hash2 and haven't done so yet. */ 21702dc41cffSDavid Held if (use_hash2 && hash2 != hash2_any) { 21712dc41cffSDavid Held hash2 = hash2_any; 21722dc41cffSDavid Held goto start_lookup; 21732dc41cffSDavid Held } 21742dc41cffSDavid Held 2175ca065d0cSEric Dumazet if (first) { 2176ca065d0cSEric Dumazet if (udp_queue_rcv_skb(first, skb) > 0) 2177ca065d0cSEric Dumazet consume_skb(skb); 21781240d137SEric Dumazet } else { 2179ca065d0cSEric Dumazet kfree_skb(skb); 218002c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI, 218136cbb245SRick Jones proto == IPPROTO_UDPLITE); 21821240d137SEric Dumazet } 2183db8dac20SDavid S. Miller return 0; 2184db8dac20SDavid S. Miller } 2185db8dac20SDavid S. Miller 2186db8dac20SDavid S. Miller /* Initialize UDP checksum. If exited with zero value (success), 2187db8dac20SDavid S. Miller * CHECKSUM_UNNECESSARY means, that no more checks are required. 2188666a3d6eSSu Yanjun * Otherwise, csum completion requires checksumming packet body, 2189db8dac20SDavid S. Miller * including udp header and folding it to skb->csum. 2190db8dac20SDavid S. Miller */ 2191db8dac20SDavid S. Miller static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh, 2192db8dac20SDavid S. Miller int proto) 2193db8dac20SDavid S. Miller { 2194db8dac20SDavid S. Miller int err; 2195db8dac20SDavid S. Miller 2196db8dac20SDavid S. Miller UDP_SKB_CB(skb)->partial_cov = 0; 2197db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov = skb->len; 2198db8dac20SDavid S. Miller 2199db8dac20SDavid S. Miller if (proto == IPPROTO_UDPLITE) { 2200db8dac20SDavid S. Miller err = udplite_checksum_init(skb, uh); 2201db8dac20SDavid S. Miller if (err) 2202db8dac20SDavid S. Miller return err; 220315f35d49SAlexey Kodanev 220415f35d49SAlexey Kodanev if (UDP_SKB_CB(skb)->partial_cov) { 220515f35d49SAlexey Kodanev skb->csum = inet_compute_pseudo(skb, proto); 220615f35d49SAlexey Kodanev return 0; 220715f35d49SAlexey Kodanev } 2208db8dac20SDavid S. Miller } 2209db8dac20SDavid S. Miller 2210b46d9f62SHannes Frederic Sowa /* Note, we are only interested in != 0 or == 0, thus the 2211b46d9f62SHannes Frederic Sowa * force to int. 2212b46d9f62SHannes Frederic Sowa */ 2213db4f1be3SSean Tranchetti err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check, 2214ed70fcfcSTom Herbert inet_compute_pseudo); 2215db4f1be3SSean Tranchetti if (err) 2216db4f1be3SSean Tranchetti return err; 2217db4f1be3SSean Tranchetti 2218db4f1be3SSean Tranchetti if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) { 2219db4f1be3SSean Tranchetti /* If SW calculated the value, we know it's bad */ 2220db4f1be3SSean Tranchetti if (skb->csum_complete_sw) 2221db4f1be3SSean Tranchetti return 1; 2222db4f1be3SSean Tranchetti 2223db4f1be3SSean Tranchetti /* HW says the value is bad. Let's validate that. 2224db4f1be3SSean Tranchetti * skb->csum is no longer the full packet checksum, 2225db4f1be3SSean Tranchetti * so don't treat it as such. 2226db4f1be3SSean Tranchetti */ 2227db4f1be3SSean Tranchetti skb_checksum_complete_unset(skb); 2228db4f1be3SSean Tranchetti } 2229db4f1be3SSean Tranchetti 2230db4f1be3SSean Tranchetti return 0; 2231db8dac20SDavid S. Miller } 2232db8dac20SDavid S. Miller 22332b5a9217SPaolo Abeni /* wrapper for udp_queue_rcv_skb tacking care of csum conversion and 22342b5a9217SPaolo Abeni * return code conversion for ip layer consumption 22352b5a9217SPaolo Abeni */ 22362b5a9217SPaolo Abeni static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb, 22372b5a9217SPaolo Abeni struct udphdr *uh) 22382b5a9217SPaolo Abeni { 22392b5a9217SPaolo Abeni int ret; 22402b5a9217SPaolo Abeni 22412b5a9217SPaolo Abeni if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk)) 2242e4aa33adSLi RongQing skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo); 22432b5a9217SPaolo Abeni 22442b5a9217SPaolo Abeni ret = udp_queue_rcv_skb(sk, skb); 22452b5a9217SPaolo Abeni 22462b5a9217SPaolo Abeni /* a return value > 0 means to resubmit the input, but 22472b5a9217SPaolo Abeni * it wants the return to be -protocol, or 0 22482b5a9217SPaolo Abeni */ 22492b5a9217SPaolo Abeni if (ret > 0) 22502b5a9217SPaolo Abeni return -ret; 22512b5a9217SPaolo Abeni return 0; 22522b5a9217SPaolo Abeni } 22532b5a9217SPaolo Abeni 2254db8dac20SDavid S. Miller /* 2255db8dac20SDavid S. Miller * All we need to do is get the socket, and then do a checksum. 2256db8dac20SDavid S. Miller */ 2257db8dac20SDavid S. Miller 2258645ca708SEric Dumazet int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, 2259db8dac20SDavid S. Miller int proto) 2260db8dac20SDavid S. Miller { 2261db8dac20SDavid S. Miller struct sock *sk; 22627b5e56f9SJesper Dangaard Brouer struct udphdr *uh; 2263db8dac20SDavid S. Miller unsigned short ulen; 2264adf30907SEric Dumazet struct rtable *rt = skb_rtable(skb); 22652783ef23SJesper Dangaard Brouer __be32 saddr, daddr; 22660283328eSPavel Emelyanov struct net *net = dev_net(skb->dev); 2267db8dac20SDavid S. Miller 2268db8dac20SDavid S. Miller /* 2269db8dac20SDavid S. Miller * Validate the packet. 2270db8dac20SDavid S. Miller */ 2271db8dac20SDavid S. Miller if (!pskb_may_pull(skb, sizeof(struct udphdr))) 2272db8dac20SDavid S. Miller goto drop; /* No space for header. */ 2273db8dac20SDavid S. Miller 22747b5e56f9SJesper Dangaard Brouer uh = udp_hdr(skb); 2275db8dac20SDavid S. Miller ulen = ntohs(uh->len); 2276ccc2d97cSBjørn Mork saddr = ip_hdr(skb)->saddr; 2277ccc2d97cSBjørn Mork daddr = ip_hdr(skb)->daddr; 2278ccc2d97cSBjørn Mork 2279db8dac20SDavid S. Miller if (ulen > skb->len) 2280db8dac20SDavid S. Miller goto short_packet; 2281db8dac20SDavid S. Miller 2282db8dac20SDavid S. Miller if (proto == IPPROTO_UDP) { 2283db8dac20SDavid S. Miller /* UDP validates ulen. */ 2284db8dac20SDavid S. Miller if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen)) 2285db8dac20SDavid S. Miller goto short_packet; 2286db8dac20SDavid S. Miller uh = udp_hdr(skb); 2287db8dac20SDavid S. Miller } 2288db8dac20SDavid S. Miller 2289db8dac20SDavid S. Miller if (udp4_csum_init(skb, uh, proto)) 2290db8dac20SDavid S. Miller goto csum_error; 2291db8dac20SDavid S. Miller 22928afdd99aSEric Dumazet sk = skb_steal_sock(skb); 22938afdd99aSEric Dumazet if (sk) { 229497502231SEric Dumazet struct dst_entry *dst = skb_dst(skb); 2295421b3885SShawn Bohrer int ret; 2296421b3885SShawn Bohrer 229797502231SEric Dumazet if (unlikely(sk->sk_rx_dst != dst)) 229897502231SEric Dumazet udp_sk_rx_dst_set(sk, dst); 2299421b3885SShawn Bohrer 23002b5a9217SPaolo Abeni ret = udp_unicast_rcv_skb(sk, skb, uh); 23018afdd99aSEric Dumazet sock_put(sk); 23022b5a9217SPaolo Abeni return ret; 2303c18450a5SFabian Frederick } 2304c18450a5SFabian Frederick 2305db8dac20SDavid S. Miller if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST)) 2306e3163493SPavel Emelyanov return __udp4_lib_mcast_deliver(net, skb, uh, 230736cbb245SRick Jones saddr, daddr, udptable, proto); 2308db8dac20SDavid S. Miller 2309607c4aafSKOVACS Krisztian sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable); 23102b5a9217SPaolo Abeni if (sk) 23112b5a9217SPaolo Abeni return udp_unicast_rcv_skb(sk, skb, uh); 2312db8dac20SDavid S. Miller 2313db8dac20SDavid S. Miller if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) 2314db8dac20SDavid S. Miller goto drop; 2315895b5c9fSFlorian Westphal nf_reset_ct(skb); 2316db8dac20SDavid S. Miller 2317db8dac20SDavid S. Miller /* No socket. Drop packet silently, if checksum is wrong */ 2318db8dac20SDavid S. Miller if (udp_lib_checksum_complete(skb)) 2319db8dac20SDavid S. Miller goto csum_error; 2320db8dac20SDavid S. Miller 232102c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE); 2322db8dac20SDavid S. Miller icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); 2323db8dac20SDavid S. Miller 2324db8dac20SDavid S. Miller /* 2325db8dac20SDavid S. Miller * Hmm. We got an UDP packet to a port to which we 2326db8dac20SDavid S. Miller * don't wanna listen. Ignore it. 2327db8dac20SDavid S. Miller */ 2328db8dac20SDavid S. Miller kfree_skb(skb); 2329db8dac20SDavid S. Miller return 0; 2330db8dac20SDavid S. Miller 2331db8dac20SDavid S. Miller short_packet: 2332ba7a46f1SJoe Perches net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n", 2333afd46503SJoe Perches proto == IPPROTO_UDPLITE ? "Lite" : "", 2334afd46503SJoe Perches &saddr, ntohs(uh->source), 2335afd46503SJoe Perches ulen, skb->len, 2336afd46503SJoe Perches &daddr, ntohs(uh->dest)); 2337db8dac20SDavid S. Miller goto drop; 2338db8dac20SDavid S. Miller 2339db8dac20SDavid S. Miller csum_error: 2340db8dac20SDavid S. Miller /* 2341db8dac20SDavid S. Miller * RFC1122: OK. Discards the bad packet silently (as far as 2342db8dac20SDavid S. Miller * the network is concerned, anyway) as per 4.1.3.4 (MUST). 2343db8dac20SDavid S. Miller */ 2344ba7a46f1SJoe Perches net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n", 2345afd46503SJoe Perches proto == IPPROTO_UDPLITE ? "Lite" : "", 2346afd46503SJoe Perches &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), 2347db8dac20SDavid S. Miller ulen); 234802c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE); 2349db8dac20SDavid S. Miller drop: 235002c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE); 2351db8dac20SDavid S. Miller kfree_skb(skb); 2352db8dac20SDavid S. Miller return 0; 2353db8dac20SDavid S. Miller } 2354db8dac20SDavid S. Miller 2355421b3885SShawn Bohrer /* We can only early demux multicast if there is a single matching socket. 2356421b3885SShawn Bohrer * If more than one socket found returns NULL 2357421b3885SShawn Bohrer */ 2358421b3885SShawn Bohrer static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net, 2359421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 2360421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 2361fb74c277SDavid Ahern int dif, int sdif) 2362421b3885SShawn Bohrer { 2363421b3885SShawn Bohrer struct sock *sk, *result; 2364421b3885SShawn Bohrer unsigned short hnum = ntohs(loc_port); 2365ca065d0cSEric Dumazet unsigned int slot = udp_hashfn(net, hnum, udp_table.mask); 2366421b3885SShawn Bohrer struct udp_hslot *hslot = &udp_table.hash[slot]; 2367421b3885SShawn Bohrer 236863c6f81cSEric Dumazet /* Do not bother scanning a too big list */ 236963c6f81cSEric Dumazet if (hslot->count > 10) 237063c6f81cSEric Dumazet return NULL; 237163c6f81cSEric Dumazet 2372421b3885SShawn Bohrer result = NULL; 2373ca065d0cSEric Dumazet sk_for_each_rcu(sk, &hslot->head) { 2374ca065d0cSEric Dumazet if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr, 2375fb74c277SDavid Ahern rmt_port, rmt_addr, dif, sdif, hnum)) { 2376ca065d0cSEric Dumazet if (result) 2377ca065d0cSEric Dumazet return NULL; 2378421b3885SShawn Bohrer result = sk; 2379421b3885SShawn Bohrer } 2380421b3885SShawn Bohrer } 2381421b3885SShawn Bohrer 2382421b3885SShawn Bohrer return result; 2383421b3885SShawn Bohrer } 2384421b3885SShawn Bohrer 2385421b3885SShawn Bohrer /* For unicast we should only early demux connected sockets or we can 2386421b3885SShawn Bohrer * break forwarding setups. The chains here can be long so only check 2387421b3885SShawn Bohrer * if the first socket is an exact match and if not move on. 2388421b3885SShawn Bohrer */ 2389421b3885SShawn Bohrer static struct sock *__udp4_lib_demux_lookup(struct net *net, 2390421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 2391421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 23923fa6f616SDavid Ahern int dif, int sdif) 2393421b3885SShawn Bohrer { 2394421b3885SShawn Bohrer unsigned short hnum = ntohs(loc_port); 2395f0b1e64cSMartin KaFai Lau unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum); 2396421b3885SShawn Bohrer unsigned int slot2 = hash2 & udp_table.mask; 2397421b3885SShawn Bohrer struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; 2398c7228317SJoe Perches INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); 2399421b3885SShawn Bohrer const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); 2400ca065d0cSEric Dumazet struct sock *sk; 2401421b3885SShawn Bohrer 2402ca065d0cSEric Dumazet udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { 2403ca065d0cSEric Dumazet if (INET_MATCH(sk, net, acookie, rmt_addr, 24043fa6f616SDavid Ahern loc_addr, ports, dif, sdif)) 2405ca065d0cSEric Dumazet return sk; 2406421b3885SShawn Bohrer /* Only check first socket in chain */ 2407421b3885SShawn Bohrer break; 2408421b3885SShawn Bohrer } 2409ca065d0cSEric Dumazet return NULL; 2410421b3885SShawn Bohrer } 2411421b3885SShawn Bohrer 24127487449cSPaolo Abeni int udp_v4_early_demux(struct sk_buff *skb) 2413421b3885SShawn Bohrer { 2414610438b7SEric Dumazet struct net *net = dev_net(skb->dev); 2415bc044e8dSPaolo Abeni struct in_device *in_dev = NULL; 2416610438b7SEric Dumazet const struct iphdr *iph; 2417610438b7SEric Dumazet const struct udphdr *uh; 2418ca065d0cSEric Dumazet struct sock *sk = NULL; 2419421b3885SShawn Bohrer struct dst_entry *dst; 2420421b3885SShawn Bohrer int dif = skb->dev->ifindex; 2421fb74c277SDavid Ahern int sdif = inet_sdif(skb); 24226e540309SShawn Bohrer int ours; 2423421b3885SShawn Bohrer 2424421b3885SShawn Bohrer /* validate the packet */ 2425421b3885SShawn Bohrer if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr))) 24267487449cSPaolo Abeni return 0; 2427421b3885SShawn Bohrer 2428610438b7SEric Dumazet iph = ip_hdr(skb); 2429610438b7SEric Dumazet uh = udp_hdr(skb); 2430610438b7SEric Dumazet 2431996b44fcSPaolo Abeni if (skb->pkt_type == PACKET_MULTICAST) { 2432bc044e8dSPaolo Abeni in_dev = __in_dev_get_rcu(skb->dev); 24336e540309SShawn Bohrer 24346e540309SShawn Bohrer if (!in_dev) 24357487449cSPaolo Abeni return 0; 24366e540309SShawn Bohrer 24376e540309SShawn Bohrer ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr, 24386e540309SShawn Bohrer iph->protocol); 24396e540309SShawn Bohrer if (!ours) 24407487449cSPaolo Abeni return 0; 2441ad0ea198SPaolo Abeni 2442421b3885SShawn Bohrer sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr, 2443fb74c277SDavid Ahern uh->source, iph->saddr, 2444fb74c277SDavid Ahern dif, sdif); 24456e540309SShawn Bohrer } else if (skb->pkt_type == PACKET_HOST) { 2446421b3885SShawn Bohrer sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, 24473fa6f616SDavid Ahern uh->source, iph->saddr, dif, sdif); 24486e540309SShawn Bohrer } 2449421b3885SShawn Bohrer 245041c6d650SReshetova, Elena if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 24517487449cSPaolo Abeni return 0; 2452421b3885SShawn Bohrer 2453421b3885SShawn Bohrer skb->sk = sk; 245482eabd9eSAlexander Duyck skb->destructor = sock_efree; 245510e2eb87SEric Dumazet dst = READ_ONCE(sk->sk_rx_dst); 2456421b3885SShawn Bohrer 2457421b3885SShawn Bohrer if (dst) 2458421b3885SShawn Bohrer dst = dst_check(dst, 0); 245910e2eb87SEric Dumazet if (dst) { 2460bc044e8dSPaolo Abeni u32 itag = 0; 2461bc044e8dSPaolo Abeni 2462d24406c8SWei Wang /* set noref for now. 2463d24406c8SWei Wang * any place which wants to hold dst has to call 2464d24406c8SWei Wang * dst_hold_safe() 2465d24406c8SWei Wang */ 2466421b3885SShawn Bohrer skb_dst_set_noref(skb, dst); 2467bc044e8dSPaolo Abeni 2468bc044e8dSPaolo Abeni /* for unconnected multicast sockets we need to validate 2469bc044e8dSPaolo Abeni * the source on each packet 2470bc044e8dSPaolo Abeni */ 2471bc044e8dSPaolo Abeni if (!inet_sk(sk)->inet_daddr && in_dev) 2472bc044e8dSPaolo Abeni return ip_mc_validate_source(skb, iph->daddr, 2473bc044e8dSPaolo Abeni iph->saddr, iph->tos, 2474bc044e8dSPaolo Abeni skb->dev, in_dev, &itag); 2475421b3885SShawn Bohrer } 24767487449cSPaolo Abeni return 0; 247710e2eb87SEric Dumazet } 2478421b3885SShawn Bohrer 2479db8dac20SDavid S. Miller int udp_rcv(struct sk_buff *skb) 2480db8dac20SDavid S. Miller { 2481645ca708SEric Dumazet return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP); 2482db8dac20SDavid S. Miller } 2483db8dac20SDavid S. Miller 24847d06b2e0SBrian Haley void udp_destroy_sock(struct sock *sk) 2485db8dac20SDavid S. Miller { 248644046a59STom Parkin struct udp_sock *up = udp_sk(sk); 24878a74ad60SEric Dumazet bool slow = lock_sock_fast(sk); 2488db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 24898a74ad60SEric Dumazet unlock_sock_fast(sk, slow); 249060fb9567SPaolo Abeni if (static_branch_unlikely(&udp_encap_needed_key)) { 249160fb9567SPaolo Abeni if (up->encap_type) { 249244046a59STom Parkin void (*encap_destroy)(struct sock *sk); 24936aa7de05SMark Rutland encap_destroy = READ_ONCE(up->encap_destroy); 249444046a59STom Parkin if (encap_destroy) 249544046a59STom Parkin encap_destroy(sk); 249644046a59STom Parkin } 249760fb9567SPaolo Abeni if (up->encap_enabled) 24989c480601SPaolo Abeni static_branch_dec(&udp_encap_needed_key); 249960fb9567SPaolo Abeni } 2500db8dac20SDavid S. Miller } 2501db8dac20SDavid S. Miller 25021da177e4SLinus Torvalds /* 25031da177e4SLinus Torvalds * Socket option code for UDP 25041da177e4SLinus Torvalds */ 25054c0a6cb0SGerrit Renker int udp_lib_setsockopt(struct sock *sk, int level, int optname, 2506b7058842SDavid S. Miller char __user *optval, unsigned int optlen, 25074c0a6cb0SGerrit Renker int (*push_pending_frames)(struct sock *)) 25081da177e4SLinus Torvalds { 25091da177e4SLinus Torvalds struct udp_sock *up = udp_sk(sk); 25101c19448cSTom Herbert int val, valbool; 25111da177e4SLinus Torvalds int err = 0; 2512b2bf1e26SWang Chen int is_udplite = IS_UDPLITE(sk); 25131da177e4SLinus Torvalds 25141da177e4SLinus Torvalds if (optlen < sizeof(int)) 25151da177e4SLinus Torvalds return -EINVAL; 25161da177e4SLinus Torvalds 25171da177e4SLinus Torvalds if (get_user(val, (int __user *)optval)) 25181da177e4SLinus Torvalds return -EFAULT; 25191da177e4SLinus Torvalds 25201c19448cSTom Herbert valbool = val ? 1 : 0; 25211c19448cSTom Herbert 25221da177e4SLinus Torvalds switch (optname) { 25231da177e4SLinus Torvalds case UDP_CORK: 25241da177e4SLinus Torvalds if (val != 0) { 25251da177e4SLinus Torvalds up->corkflag = 1; 25261da177e4SLinus Torvalds } else { 25271da177e4SLinus Torvalds up->corkflag = 0; 25281da177e4SLinus Torvalds lock_sock(sk); 25294243cdc2SJoe Perches push_pending_frames(sk); 25301da177e4SLinus Torvalds release_sock(sk); 25311da177e4SLinus Torvalds } 25321da177e4SLinus Torvalds break; 25331da177e4SLinus Torvalds 25341da177e4SLinus Torvalds case UDP_ENCAP: 25351da177e4SLinus Torvalds switch (val) { 25361da177e4SLinus Torvalds case 0: 25371da177e4SLinus Torvalds case UDP_ENCAP_ESPINUDP: 25381da177e4SLinus Torvalds case UDP_ENCAP_ESPINUDP_NON_IKE: 2539067b207bSJames Chapman up->encap_rcv = xfrm4_udp_encap_rcv; 2540067b207bSJames Chapman /* FALLTHROUGH */ 2541342f0234SJames Chapman case UDP_ENCAP_L2TPINUDP: 25421da177e4SLinus Torvalds up->encap_type = val; 254360fb9567SPaolo Abeni lock_sock(sk); 254460fb9567SPaolo Abeni udp_tunnel_encap_enable(sk->sk_socket); 254560fb9567SPaolo Abeni release_sock(sk); 25461da177e4SLinus Torvalds break; 25471da177e4SLinus Torvalds default: 25481da177e4SLinus Torvalds err = -ENOPROTOOPT; 25491da177e4SLinus Torvalds break; 25501da177e4SLinus Torvalds } 25511da177e4SLinus Torvalds break; 25521da177e4SLinus Torvalds 25531c19448cSTom Herbert case UDP_NO_CHECK6_TX: 25541c19448cSTom Herbert up->no_check6_tx = valbool; 25551c19448cSTom Herbert break; 25561c19448cSTom Herbert 25571c19448cSTom Herbert case UDP_NO_CHECK6_RX: 25581c19448cSTom Herbert up->no_check6_rx = valbool; 25591c19448cSTom Herbert break; 25601c19448cSTom Herbert 2561bec1f6f6SWillem de Bruijn case UDP_SEGMENT: 2562bec1f6f6SWillem de Bruijn if (val < 0 || val > USHRT_MAX) 2563bec1f6f6SWillem de Bruijn return -EINVAL; 2564bec1f6f6SWillem de Bruijn up->gso_size = val; 2565bec1f6f6SWillem de Bruijn break; 2566bec1f6f6SWillem de Bruijn 2567e20cf8d3SPaolo Abeni case UDP_GRO: 2568e20cf8d3SPaolo Abeni lock_sock(sk); 2569e20cf8d3SPaolo Abeni if (valbool) 2570e20cf8d3SPaolo Abeni udp_tunnel_encap_enable(sk->sk_socket); 2571e20cf8d3SPaolo Abeni up->gro_enabled = valbool; 2572e20cf8d3SPaolo Abeni release_sock(sk); 2573e20cf8d3SPaolo Abeni break; 2574e20cf8d3SPaolo Abeni 2575ba4e58ecSGerrit Renker /* 2576ba4e58ecSGerrit Renker * UDP-Lite's partial checksum coverage (RFC 3828). 2577ba4e58ecSGerrit Renker */ 2578ba4e58ecSGerrit Renker /* The sender sets actual checksum coverage length via this option. 2579ba4e58ecSGerrit Renker * The case coverage > packet length is handled by send module. */ 2580ba4e58ecSGerrit Renker case UDPLITE_SEND_CSCOV: 2581b2bf1e26SWang Chen if (!is_udplite) /* Disable the option on UDP sockets */ 2582ba4e58ecSGerrit Renker return -ENOPROTOOPT; 2583ba4e58ecSGerrit Renker if (val != 0 && val < 8) /* Illegal coverage: use default (8) */ 2584ba4e58ecSGerrit Renker val = 8; 25854be929beSAlexey Dobriyan else if (val > USHRT_MAX) 25864be929beSAlexey Dobriyan val = USHRT_MAX; 2587ba4e58ecSGerrit Renker up->pcslen = val; 2588ba4e58ecSGerrit Renker up->pcflag |= UDPLITE_SEND_CC; 2589ba4e58ecSGerrit Renker break; 2590ba4e58ecSGerrit Renker 2591ba4e58ecSGerrit Renker /* The receiver specifies a minimum checksum coverage value. To make 2592ba4e58ecSGerrit Renker * sense, this should be set to at least 8 (as done below). If zero is 2593ba4e58ecSGerrit Renker * used, this again means full checksum coverage. */ 2594ba4e58ecSGerrit Renker case UDPLITE_RECV_CSCOV: 2595b2bf1e26SWang Chen if (!is_udplite) /* Disable the option on UDP sockets */ 2596ba4e58ecSGerrit Renker return -ENOPROTOOPT; 2597ba4e58ecSGerrit Renker if (val != 0 && val < 8) /* Avoid silly minimal values. */ 2598ba4e58ecSGerrit Renker val = 8; 25994be929beSAlexey Dobriyan else if (val > USHRT_MAX) 26004be929beSAlexey Dobriyan val = USHRT_MAX; 2601ba4e58ecSGerrit Renker up->pcrlen = val; 2602ba4e58ecSGerrit Renker up->pcflag |= UDPLITE_RECV_CC; 2603ba4e58ecSGerrit Renker break; 2604ba4e58ecSGerrit Renker 26051da177e4SLinus Torvalds default: 26061da177e4SLinus Torvalds err = -ENOPROTOOPT; 26071da177e4SLinus Torvalds break; 26086516c655SStephen Hemminger } 26091da177e4SLinus Torvalds 26101da177e4SLinus Torvalds return err; 26111da177e4SLinus Torvalds } 2612c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_setsockopt); 26131da177e4SLinus Torvalds 2614db8dac20SDavid S. Miller int udp_setsockopt(struct sock *sk, int level, int optname, 2615b7058842SDavid S. Miller char __user *optval, unsigned int optlen) 2616db8dac20SDavid S. Miller { 2617db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 2618db8dac20SDavid S. Miller return udp_lib_setsockopt(sk, level, optname, optval, optlen, 2619db8dac20SDavid S. Miller udp_push_pending_frames); 2620db8dac20SDavid S. Miller return ip_setsockopt(sk, level, optname, optval, optlen); 2621db8dac20SDavid S. Miller } 2622db8dac20SDavid S. Miller 2623db8dac20SDavid S. Miller #ifdef CONFIG_COMPAT 2624db8dac20SDavid S. Miller int compat_udp_setsockopt(struct sock *sk, int level, int optname, 2625b7058842SDavid S. Miller char __user *optval, unsigned int optlen) 2626db8dac20SDavid S. Miller { 2627db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 2628db8dac20SDavid S. Miller return udp_lib_setsockopt(sk, level, optname, optval, optlen, 2629db8dac20SDavid S. Miller udp_push_pending_frames); 2630db8dac20SDavid S. Miller return compat_ip_setsockopt(sk, level, optname, optval, optlen); 2631db8dac20SDavid S. Miller } 2632db8dac20SDavid S. Miller #endif 2633db8dac20SDavid S. Miller 26344c0a6cb0SGerrit Renker int udp_lib_getsockopt(struct sock *sk, int level, int optname, 26351da177e4SLinus Torvalds char __user *optval, int __user *optlen) 26361da177e4SLinus Torvalds { 26371da177e4SLinus Torvalds struct udp_sock *up = udp_sk(sk); 26381da177e4SLinus Torvalds int val, len; 26391da177e4SLinus Torvalds 26401da177e4SLinus Torvalds if (get_user(len, optlen)) 26411da177e4SLinus Torvalds return -EFAULT; 26421da177e4SLinus Torvalds 26431da177e4SLinus Torvalds len = min_t(unsigned int, len, sizeof(int)); 26441da177e4SLinus Torvalds 26451da177e4SLinus Torvalds if (len < 0) 26461da177e4SLinus Torvalds return -EINVAL; 26471da177e4SLinus Torvalds 26481da177e4SLinus Torvalds switch (optname) { 26491da177e4SLinus Torvalds case UDP_CORK: 26501da177e4SLinus Torvalds val = up->corkflag; 26511da177e4SLinus Torvalds break; 26521da177e4SLinus Torvalds 26531da177e4SLinus Torvalds case UDP_ENCAP: 26541da177e4SLinus Torvalds val = up->encap_type; 26551da177e4SLinus Torvalds break; 26561da177e4SLinus Torvalds 26571c19448cSTom Herbert case UDP_NO_CHECK6_TX: 26581c19448cSTom Herbert val = up->no_check6_tx; 26591c19448cSTom Herbert break; 26601c19448cSTom Herbert 26611c19448cSTom Herbert case UDP_NO_CHECK6_RX: 26621c19448cSTom Herbert val = up->no_check6_rx; 26631c19448cSTom Herbert break; 26641c19448cSTom Herbert 2665bec1f6f6SWillem de Bruijn case UDP_SEGMENT: 2666bec1f6f6SWillem de Bruijn val = up->gso_size; 2667bec1f6f6SWillem de Bruijn break; 2668bec1f6f6SWillem de Bruijn 2669ba4e58ecSGerrit Renker /* The following two cannot be changed on UDP sockets, the return is 2670ba4e58ecSGerrit Renker * always 0 (which corresponds to the full checksum coverage of UDP). */ 2671ba4e58ecSGerrit Renker case UDPLITE_SEND_CSCOV: 2672ba4e58ecSGerrit Renker val = up->pcslen; 2673ba4e58ecSGerrit Renker break; 2674ba4e58ecSGerrit Renker 2675ba4e58ecSGerrit Renker case UDPLITE_RECV_CSCOV: 2676ba4e58ecSGerrit Renker val = up->pcrlen; 2677ba4e58ecSGerrit Renker break; 2678ba4e58ecSGerrit Renker 26791da177e4SLinus Torvalds default: 26801da177e4SLinus Torvalds return -ENOPROTOOPT; 26816516c655SStephen Hemminger } 26821da177e4SLinus Torvalds 26831da177e4SLinus Torvalds if (put_user(len, optlen)) 26841da177e4SLinus Torvalds return -EFAULT; 26851da177e4SLinus Torvalds if (copy_to_user(optval, &val, len)) 26861da177e4SLinus Torvalds return -EFAULT; 26871da177e4SLinus Torvalds return 0; 26881da177e4SLinus Torvalds } 2689c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_getsockopt); 26901da177e4SLinus Torvalds 2691db8dac20SDavid S. Miller int udp_getsockopt(struct sock *sk, int level, int optname, 2692db8dac20SDavid S. Miller char __user *optval, int __user *optlen) 2693db8dac20SDavid S. Miller { 2694db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 2695db8dac20SDavid S. Miller return udp_lib_getsockopt(sk, level, optname, optval, optlen); 2696db8dac20SDavid S. Miller return ip_getsockopt(sk, level, optname, optval, optlen); 2697db8dac20SDavid S. Miller } 2698db8dac20SDavid S. Miller 2699db8dac20SDavid S. Miller #ifdef CONFIG_COMPAT 2700db8dac20SDavid S. Miller int compat_udp_getsockopt(struct sock *sk, int level, int optname, 2701db8dac20SDavid S. Miller char __user *optval, int __user *optlen) 2702db8dac20SDavid S. Miller { 2703db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 2704db8dac20SDavid S. Miller return udp_lib_getsockopt(sk, level, optname, optval, optlen); 2705db8dac20SDavid S. Miller return compat_ip_getsockopt(sk, level, optname, optval, optlen); 2706db8dac20SDavid S. Miller } 2707db8dac20SDavid S. Miller #endif 27081da177e4SLinus Torvalds /** 27091da177e4SLinus Torvalds * udp_poll - wait for a UDP event. 27101da177e4SLinus Torvalds * @file - file struct 27111da177e4SLinus Torvalds * @sock - socket 2712a11e1d43SLinus Torvalds * @wait - poll table 27131da177e4SLinus Torvalds * 27141da177e4SLinus Torvalds * This is same as datagram poll, except for the special case of 27151da177e4SLinus Torvalds * blocking sockets. If application is using a blocking fd 27161da177e4SLinus Torvalds * and a packet with checksum error is in the queue; 27171da177e4SLinus Torvalds * then it could get return from select indicating data available 27181da177e4SLinus Torvalds * but then block when reading it. Add special case code 27191da177e4SLinus Torvalds * to work around these arguably broken applications. 27201da177e4SLinus Torvalds */ 2721a11e1d43SLinus Torvalds __poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait) 27221da177e4SLinus Torvalds { 2723a11e1d43SLinus Torvalds __poll_t mask = datagram_poll(file, sock, wait); 27241da177e4SLinus Torvalds struct sock *sk = sock->sk; 27251da177e4SLinus Torvalds 27263ef7cf57SEric Dumazet if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 2727a9a08845SLinus Torvalds mask |= EPOLLIN | EPOLLRDNORM; 27282276f58aSPaolo Abeni 27291da177e4SLinus Torvalds /* Check for false positives due to checksum errors */ 2730a11e1d43SLinus Torvalds if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) && 2731e83c6744SEric Dumazet !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1) 2732a9a08845SLinus Torvalds mask &= ~(EPOLLIN | EPOLLRDNORM); 27331da177e4SLinus Torvalds 27341da177e4SLinus Torvalds return mask; 27351da177e4SLinus Torvalds 27361da177e4SLinus Torvalds } 2737a11e1d43SLinus Torvalds EXPORT_SYMBOL(udp_poll); 27381da177e4SLinus Torvalds 27395d77dca8SDavid Ahern int udp_abort(struct sock *sk, int err) 27405d77dca8SDavid Ahern { 27415d77dca8SDavid Ahern lock_sock(sk); 27425d77dca8SDavid Ahern 27435d77dca8SDavid Ahern sk->sk_err = err; 27445d77dca8SDavid Ahern sk->sk_error_report(sk); 2745286c72deSEric Dumazet __udp_disconnect(sk, 0); 27465d77dca8SDavid Ahern 27475d77dca8SDavid Ahern release_sock(sk); 27485d77dca8SDavid Ahern 27495d77dca8SDavid Ahern return 0; 27505d77dca8SDavid Ahern } 27515d77dca8SDavid Ahern EXPORT_SYMBOL_GPL(udp_abort); 27525d77dca8SDavid Ahern 2753db8dac20SDavid S. Miller struct proto udp_prot = { 2754db8dac20SDavid S. Miller .name = "UDP", 2755db8dac20SDavid S. Miller .owner = THIS_MODULE, 2756db8dac20SDavid S. Miller .close = udp_lib_close, 2757d74bad4eSAndrey Ignatov .pre_connect = udp_pre_connect, 2758db8dac20SDavid S. Miller .connect = ip4_datagram_connect, 2759db8dac20SDavid S. Miller .disconnect = udp_disconnect, 2760db8dac20SDavid S. Miller .ioctl = udp_ioctl, 2761850cbaddSPaolo Abeni .init = udp_init_sock, 2762db8dac20SDavid S. Miller .destroy = udp_destroy_sock, 2763db8dac20SDavid S. Miller .setsockopt = udp_setsockopt, 2764db8dac20SDavid S. Miller .getsockopt = udp_getsockopt, 2765db8dac20SDavid S. Miller .sendmsg = udp_sendmsg, 2766db8dac20SDavid S. Miller .recvmsg = udp_recvmsg, 2767db8dac20SDavid S. Miller .sendpage = udp_sendpage, 27688141ed9fSSteffen Klassert .release_cb = ip4_datagram_release_cb, 2769db8dac20SDavid S. Miller .hash = udp_lib_hash, 2770db8dac20SDavid S. Miller .unhash = udp_lib_unhash, 2771719f8358SEric Dumazet .rehash = udp_v4_rehash, 2772db8dac20SDavid S. Miller .get_port = udp_v4_get_port, 2773db8dac20SDavid S. Miller .memory_allocated = &udp_memory_allocated, 2774db8dac20SDavid S. Miller .sysctl_mem = sysctl_udp_mem, 27751e802951STonghao Zhang .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min), 27761e802951STonghao Zhang .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min), 2777db8dac20SDavid S. Miller .obj_size = sizeof(struct udp_sock), 2778645ca708SEric Dumazet .h.udp_table = &udp_table, 2779db8dac20SDavid S. Miller #ifdef CONFIG_COMPAT 2780db8dac20SDavid S. Miller .compat_setsockopt = compat_udp_setsockopt, 2781db8dac20SDavid S. Miller .compat_getsockopt = compat_udp_getsockopt, 2782db8dac20SDavid S. Miller #endif 27835d77dca8SDavid Ahern .diag_destroy = udp_abort, 2784db8dac20SDavid S. Miller }; 2785c482c568SEric Dumazet EXPORT_SYMBOL(udp_prot); 27861da177e4SLinus Torvalds 27871da177e4SLinus Torvalds /* ------------------------------------------------------------------------ */ 27881da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS 27891da177e4SLinus Torvalds 2790645ca708SEric Dumazet static struct sock *udp_get_first(struct seq_file *seq, int start) 27911da177e4SLinus Torvalds { 27921da177e4SLinus Torvalds struct sock *sk; 2793a3d2599bSChristoph Hellwig struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); 27941da177e4SLinus Torvalds struct udp_iter_state *state = seq->private; 27956f191efeSDenis V. Lunev struct net *net = seq_file_net(seq); 27961da177e4SLinus Torvalds 2797a3d2599bSChristoph Hellwig for (state->bucket = start; state->bucket <= afinfo->udp_table->mask; 2798f86dcc5aSEric Dumazet ++state->bucket) { 2799a3d2599bSChristoph Hellwig struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket]; 2800f86dcc5aSEric Dumazet 2801ca065d0cSEric Dumazet if (hlist_empty(&hslot->head)) 2802f86dcc5aSEric Dumazet continue; 2803f86dcc5aSEric Dumazet 2804645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 2805ca065d0cSEric Dumazet sk_for_each(sk, &hslot->head) { 2806878628fbSYOSHIFUJI Hideaki if (!net_eq(sock_net(sk), net)) 2807a91275efSDaniel Lezcano continue; 2808a3d2599bSChristoph Hellwig if (sk->sk_family == afinfo->family) 28091da177e4SLinus Torvalds goto found; 28101da177e4SLinus Torvalds } 2811645ca708SEric Dumazet spin_unlock_bh(&hslot->lock); 28121da177e4SLinus Torvalds } 28131da177e4SLinus Torvalds sk = NULL; 28141da177e4SLinus Torvalds found: 28151da177e4SLinus Torvalds return sk; 28161da177e4SLinus Torvalds } 28171da177e4SLinus Torvalds 28181da177e4SLinus Torvalds static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) 28191da177e4SLinus Torvalds { 2820a3d2599bSChristoph Hellwig struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); 28211da177e4SLinus Torvalds struct udp_iter_state *state = seq->private; 28226f191efeSDenis V. Lunev struct net *net = seq_file_net(seq); 28231da177e4SLinus Torvalds 28241da177e4SLinus Torvalds do { 2825ca065d0cSEric Dumazet sk = sk_next(sk); 2826a3d2599bSChristoph Hellwig } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != afinfo->family)); 28271da177e4SLinus Torvalds 2828645ca708SEric Dumazet if (!sk) { 2829a3d2599bSChristoph Hellwig if (state->bucket <= afinfo->udp_table->mask) 2830a3d2599bSChristoph Hellwig spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); 2831645ca708SEric Dumazet return udp_get_first(seq, state->bucket + 1); 28321da177e4SLinus Torvalds } 28331da177e4SLinus Torvalds return sk; 28341da177e4SLinus Torvalds } 28351da177e4SLinus Torvalds 28361da177e4SLinus Torvalds static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos) 28371da177e4SLinus Torvalds { 2838645ca708SEric Dumazet struct sock *sk = udp_get_first(seq, 0); 28391da177e4SLinus Torvalds 28401da177e4SLinus Torvalds if (sk) 28411da177e4SLinus Torvalds while (pos && (sk = udp_get_next(seq, sk)) != NULL) 28421da177e4SLinus Torvalds --pos; 28431da177e4SLinus Torvalds return pos ? NULL : sk; 28441da177e4SLinus Torvalds } 28451da177e4SLinus Torvalds 2846a3d2599bSChristoph Hellwig void *udp_seq_start(struct seq_file *seq, loff_t *pos) 28471da177e4SLinus Torvalds { 284830842f29SVitaly Mayatskikh struct udp_iter_state *state = seq->private; 2849f86dcc5aSEric Dumazet state->bucket = MAX_UDP_PORTS; 285030842f29SVitaly Mayatskikh 2851b50660f1SYOSHIFUJI Hideaki return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; 28521da177e4SLinus Torvalds } 2853a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_start); 28541da177e4SLinus Torvalds 2855a3d2599bSChristoph Hellwig void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos) 28561da177e4SLinus Torvalds { 28571da177e4SLinus Torvalds struct sock *sk; 28581da177e4SLinus Torvalds 2859b50660f1SYOSHIFUJI Hideaki if (v == SEQ_START_TOKEN) 28601da177e4SLinus Torvalds sk = udp_get_idx(seq, 0); 28611da177e4SLinus Torvalds else 28621da177e4SLinus Torvalds sk = udp_get_next(seq, v); 28631da177e4SLinus Torvalds 28641da177e4SLinus Torvalds ++*pos; 28651da177e4SLinus Torvalds return sk; 28661da177e4SLinus Torvalds } 2867a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_next); 28681da177e4SLinus Torvalds 2869a3d2599bSChristoph Hellwig void udp_seq_stop(struct seq_file *seq, void *v) 28701da177e4SLinus Torvalds { 2871a3d2599bSChristoph Hellwig struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); 2872645ca708SEric Dumazet struct udp_iter_state *state = seq->private; 2873645ca708SEric Dumazet 2874a3d2599bSChristoph Hellwig if (state->bucket <= afinfo->udp_table->mask) 2875a3d2599bSChristoph Hellwig spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); 28761da177e4SLinus Torvalds } 2877a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_stop); 2878db8dac20SDavid S. Miller 2879db8dac20SDavid S. Miller /* ------------------------------------------------------------------------ */ 28805e659e4cSPavel Emelyanov static void udp4_format_sock(struct sock *sp, struct seq_file *f, 2881652586dfSTetsuo Handa int bucket) 2882db8dac20SDavid S. Miller { 2883db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sp); 2884c720c7e8SEric Dumazet __be32 dest = inet->inet_daddr; 2885c720c7e8SEric Dumazet __be32 src = inet->inet_rcv_saddr; 2886c720c7e8SEric Dumazet __u16 destp = ntohs(inet->inet_dport); 2887c720c7e8SEric Dumazet __u16 srcp = ntohs(inet->inet_sport); 2888db8dac20SDavid S. Miller 2889f86dcc5aSEric Dumazet seq_printf(f, "%5d: %08X:%04X %08X:%04X" 2890ea9a0379SPatrick Talbert " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u", 2891db8dac20SDavid S. Miller bucket, src, srcp, dest, destp, sp->sk_state, 289231e6d363SEric Dumazet sk_wmem_alloc_get(sp), 28936c206b20SPaolo Abeni udp_rqueue_get(sp), 2894a7cb5a49SEric W. Biederman 0, 0L, 0, 2895a7cb5a49SEric W. Biederman from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)), 2896a7cb5a49SEric W. Biederman 0, sock_i_ino(sp), 289741c6d650SReshetova, Elena refcount_read(&sp->sk_refcnt), sp, 2898652586dfSTetsuo Handa atomic_read(&sp->sk_drops)); 2899db8dac20SDavid S. Miller } 2900db8dac20SDavid S. Miller 2901db8dac20SDavid S. Miller int udp4_seq_show(struct seq_file *seq, void *v) 2902db8dac20SDavid S. Miller { 2903652586dfSTetsuo Handa seq_setwidth(seq, 127); 2904db8dac20SDavid S. Miller if (v == SEQ_START_TOKEN) 2905652586dfSTetsuo Handa seq_puts(seq, " sl local_address rem_address st tx_queue " 2906db8dac20SDavid S. Miller "rx_queue tr tm->when retrnsmt uid timeout " 2907cb61cb9bSEric Dumazet "inode ref pointer drops"); 2908db8dac20SDavid S. Miller else { 2909db8dac20SDavid S. Miller struct udp_iter_state *state = seq->private; 2910db8dac20SDavid S. Miller 2911652586dfSTetsuo Handa udp4_format_sock(v, seq, state->bucket); 2912db8dac20SDavid S. Miller } 2913652586dfSTetsuo Handa seq_pad(seq, '\n'); 2914db8dac20SDavid S. Miller return 0; 2915db8dac20SDavid S. Miller } 2916db8dac20SDavid S. Miller 2917c3506372SChristoph Hellwig const struct seq_operations udp_seq_ops = { 2918a3d2599bSChristoph Hellwig .start = udp_seq_start, 2919a3d2599bSChristoph Hellwig .next = udp_seq_next, 2920a3d2599bSChristoph Hellwig .stop = udp_seq_stop, 2921a3d2599bSChristoph Hellwig .show = udp4_seq_show, 2922a3d2599bSChristoph Hellwig }; 2923c3506372SChristoph Hellwig EXPORT_SYMBOL(udp_seq_ops); 292473cb88ecSArjan van de Ven 2925db8dac20SDavid S. Miller static struct udp_seq_afinfo udp4_seq_afinfo = { 2926db8dac20SDavid S. Miller .family = AF_INET, 2927645ca708SEric Dumazet .udp_table = &udp_table, 2928db8dac20SDavid S. Miller }; 2929db8dac20SDavid S. Miller 29302c8c1e72SAlexey Dobriyan static int __net_init udp4_proc_init_net(struct net *net) 293115439febSPavel Emelyanov { 2932c3506372SChristoph Hellwig if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops, 2933c3506372SChristoph Hellwig sizeof(struct udp_iter_state), &udp4_seq_afinfo)) 2934a3d2599bSChristoph Hellwig return -ENOMEM; 2935a3d2599bSChristoph Hellwig return 0; 293615439febSPavel Emelyanov } 293715439febSPavel Emelyanov 29382c8c1e72SAlexey Dobriyan static void __net_exit udp4_proc_exit_net(struct net *net) 293915439febSPavel Emelyanov { 2940a3d2599bSChristoph Hellwig remove_proc_entry("udp", net->proc_net); 294115439febSPavel Emelyanov } 294215439febSPavel Emelyanov 294315439febSPavel Emelyanov static struct pernet_operations udp4_net_ops = { 294415439febSPavel Emelyanov .init = udp4_proc_init_net, 294515439febSPavel Emelyanov .exit = udp4_proc_exit_net, 294615439febSPavel Emelyanov }; 294715439febSPavel Emelyanov 2948db8dac20SDavid S. Miller int __init udp4_proc_init(void) 2949db8dac20SDavid S. Miller { 295015439febSPavel Emelyanov return register_pernet_subsys(&udp4_net_ops); 2951db8dac20SDavid S. Miller } 2952db8dac20SDavid S. Miller 2953db8dac20SDavid S. Miller void udp4_proc_exit(void) 2954db8dac20SDavid S. Miller { 295515439febSPavel Emelyanov unregister_pernet_subsys(&udp4_net_ops); 2956db8dac20SDavid S. Miller } 29571da177e4SLinus Torvalds #endif /* CONFIG_PROC_FS */ 29581da177e4SLinus Torvalds 2959f86dcc5aSEric Dumazet static __initdata unsigned long uhash_entries; 2960f86dcc5aSEric Dumazet static int __init set_uhash_entries(char *str) 2961645ca708SEric Dumazet { 2962413c27d8SEldad Zack ssize_t ret; 2963413c27d8SEldad Zack 2964f86dcc5aSEric Dumazet if (!str) 2965f86dcc5aSEric Dumazet return 0; 2966413c27d8SEldad Zack 2967413c27d8SEldad Zack ret = kstrtoul(str, 0, &uhash_entries); 2968413c27d8SEldad Zack if (ret) 2969413c27d8SEldad Zack return 0; 2970413c27d8SEldad Zack 2971f86dcc5aSEric Dumazet if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN) 2972f86dcc5aSEric Dumazet uhash_entries = UDP_HTABLE_SIZE_MIN; 2973f86dcc5aSEric Dumazet return 1; 2974f86dcc5aSEric Dumazet } 2975f86dcc5aSEric Dumazet __setup("uhash_entries=", set_uhash_entries); 2976645ca708SEric Dumazet 2977f86dcc5aSEric Dumazet void __init udp_table_init(struct udp_table *table, const char *name) 2978f86dcc5aSEric Dumazet { 2979f86dcc5aSEric Dumazet unsigned int i; 2980f86dcc5aSEric Dumazet 2981f86dcc5aSEric Dumazet table->hash = alloc_large_system_hash(name, 2982512615b6SEric Dumazet 2 * sizeof(struct udp_hslot), 2983f86dcc5aSEric Dumazet uhash_entries, 2984f86dcc5aSEric Dumazet 21, /* one slot per 2 MB */ 2985f86dcc5aSEric Dumazet 0, 2986f86dcc5aSEric Dumazet &table->log, 2987f86dcc5aSEric Dumazet &table->mask, 298831fe62b9STim Bird UDP_HTABLE_SIZE_MIN, 2989f86dcc5aSEric Dumazet 64 * 1024); 299031fe62b9STim Bird 2991512615b6SEric Dumazet table->hash2 = table->hash + (table->mask + 1); 2992f86dcc5aSEric Dumazet for (i = 0; i <= table->mask; i++) { 2993ca065d0cSEric Dumazet INIT_HLIST_HEAD(&table->hash[i].head); 2994fdcc8aa9SEric Dumazet table->hash[i].count = 0; 2995645ca708SEric Dumazet spin_lock_init(&table->hash[i].lock); 2996645ca708SEric Dumazet } 2997512615b6SEric Dumazet for (i = 0; i <= table->mask; i++) { 2998ca065d0cSEric Dumazet INIT_HLIST_HEAD(&table->hash2[i].head); 2999512615b6SEric Dumazet table->hash2[i].count = 0; 3000512615b6SEric Dumazet spin_lock_init(&table->hash2[i].lock); 3001512615b6SEric Dumazet } 3002645ca708SEric Dumazet } 3003645ca708SEric Dumazet 3004723b8e46STom Herbert u32 udp_flow_hashrnd(void) 3005723b8e46STom Herbert { 3006723b8e46STom Herbert static u32 hashrnd __read_mostly; 3007723b8e46STom Herbert 3008723b8e46STom Herbert net_get_random_once(&hashrnd, sizeof(hashrnd)); 3009723b8e46STom Herbert 3010723b8e46STom Herbert return hashrnd; 3011723b8e46STom Herbert } 3012723b8e46STom Herbert EXPORT_SYMBOL(udp_flow_hashrnd); 3013723b8e46STom Herbert 30141e802951STonghao Zhang static void __udp_sysctl_init(struct net *net) 30151e802951STonghao Zhang { 30161e802951STonghao Zhang net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM; 30171e802951STonghao Zhang net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM; 30181e802951STonghao Zhang 30191e802951STonghao Zhang #ifdef CONFIG_NET_L3_MASTER_DEV 30201e802951STonghao Zhang net->ipv4.sysctl_udp_l3mdev_accept = 0; 30211e802951STonghao Zhang #endif 30221e802951STonghao Zhang } 30231e802951STonghao Zhang 30241e802951STonghao Zhang static int __net_init udp_sysctl_init(struct net *net) 30251e802951STonghao Zhang { 30261e802951STonghao Zhang __udp_sysctl_init(net); 30271e802951STonghao Zhang return 0; 30281e802951STonghao Zhang } 30291e802951STonghao Zhang 30301e802951STonghao Zhang static struct pernet_operations __net_initdata udp_sysctl_ops = { 30311e802951STonghao Zhang .init = udp_sysctl_init, 30321e802951STonghao Zhang }; 30331e802951STonghao Zhang 303495766fffSHideo Aoki void __init udp_init(void) 303595766fffSHideo Aoki { 3036f03d78dbSEric Dumazet unsigned long limit; 30374b272750SEric Dumazet unsigned int i; 303895766fffSHideo Aoki 3039f86dcc5aSEric Dumazet udp_table_init(&udp_table, "UDP"); 3040f03d78dbSEric Dumazet limit = nr_free_buffer_pages() / 8; 304195766fffSHideo Aoki limit = max(limit, 128UL); 304295766fffSHideo Aoki sysctl_udp_mem[0] = limit / 4 * 3; 304395766fffSHideo Aoki sysctl_udp_mem[1] = limit; 304495766fffSHideo Aoki sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2; 304595766fffSHideo Aoki 30461e802951STonghao Zhang __udp_sysctl_init(&init_net); 30474b272750SEric Dumazet 30484b272750SEric Dumazet /* 16 spinlocks per cpu */ 30494b272750SEric Dumazet udp_busylocks_log = ilog2(nr_cpu_ids) + 4; 30504b272750SEric Dumazet udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log, 30514b272750SEric Dumazet GFP_KERNEL); 30524b272750SEric Dumazet if (!udp_busylocks) 30534b272750SEric Dumazet panic("UDP: failed to alloc udp_busylocks\n"); 30544b272750SEric Dumazet for (i = 0; i < (1U << udp_busylocks_log); i++) 30554b272750SEric Dumazet spin_lock_init(udp_busylocks + i); 30561e802951STonghao Zhang 30571e802951STonghao Zhang if (register_pernet_subsys(&udp_sysctl_ops)) 30581e802951STonghao Zhang panic("UDP: failed to init sysctl parameters.\n"); 305995766fffSHideo Aoki } 3060