12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * INET An implementation of the TCP/IP protocol suite for the LINUX 41da177e4SLinus Torvalds * operating system. INET is implemented using the BSD Socket 51da177e4SLinus Torvalds * interface as the means of communication with the user level. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * The User Datagram Protocol (UDP). 81da177e4SLinus Torvalds * 902c30a84SJesper Juhl * Authors: Ross Biro 101da177e4SLinus Torvalds * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 111da177e4SLinus Torvalds * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 12113aa838SAlan Cox * Alan Cox, <alan@lxorguk.ukuu.org.uk> 131da177e4SLinus Torvalds * Hirokazu Takahashi, <taka@valinux.co.jp> 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * Fixes: 161da177e4SLinus Torvalds * Alan Cox : verify_area() calls 171da177e4SLinus Torvalds * Alan Cox : stopped close while in use off icmp 181da177e4SLinus Torvalds * messages. Not a fix but a botch that 191da177e4SLinus Torvalds * for udp at least is 'valid'. 201da177e4SLinus Torvalds * Alan Cox : Fixed icmp handling properly 211da177e4SLinus Torvalds * Alan Cox : Correct error for oversized datagrams 221da177e4SLinus Torvalds * Alan Cox : Tidied select() semantics. 231da177e4SLinus Torvalds * Alan Cox : udp_err() fixed properly, also now 241da177e4SLinus Torvalds * select and read wake correctly on errors 251da177e4SLinus Torvalds * Alan Cox : udp_send verify_area moved to avoid mem leak 261da177e4SLinus Torvalds * Alan Cox : UDP can count its memory 271da177e4SLinus Torvalds * Alan Cox : send to an unknown connection causes 281da177e4SLinus Torvalds * an ECONNREFUSED off the icmp, but 291da177e4SLinus Torvalds * does NOT close. 301da177e4SLinus Torvalds * Alan Cox : Switched to new sk_buff handlers. No more backlog! 311da177e4SLinus Torvalds * Alan Cox : Using generic datagram code. Even smaller and the PEEK 321da177e4SLinus Torvalds * bug no longer crashes it. 331da177e4SLinus Torvalds * Fred Van Kempen : Net2e support for sk->broadcast. 341da177e4SLinus Torvalds * Alan Cox : Uses skb_free_datagram 351da177e4SLinus Torvalds * Alan Cox : Added get/set sockopt support. 361da177e4SLinus Torvalds * Alan Cox : Broadcasting without option set returns EACCES. 371da177e4SLinus Torvalds * Alan Cox : No wakeup calls. Instead we now use the callbacks. 381da177e4SLinus Torvalds * Alan Cox : Use ip_tos and ip_ttl 391da177e4SLinus Torvalds * Alan Cox : SNMP Mibs 401da177e4SLinus Torvalds * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support. 411da177e4SLinus Torvalds * Matt Dillon : UDP length checks. 421da177e4SLinus Torvalds * Alan Cox : Smarter af_inet used properly. 431da177e4SLinus Torvalds * Alan Cox : Use new kernel side addressing. 441da177e4SLinus Torvalds * Alan Cox : Incorrect return on truncated datagram receive. 451da177e4SLinus Torvalds * Arnt Gulbrandsen : New udp_send and stuff 461da177e4SLinus Torvalds * Alan Cox : Cache last socket 471da177e4SLinus Torvalds * Alan Cox : Route cache 481da177e4SLinus Torvalds * Jon Peatfield : Minor efficiency fix to sendto(). 491da177e4SLinus Torvalds * Mike Shaver : RFC1122 checks. 501da177e4SLinus Torvalds * Alan Cox : Nonblocking error fix. 511da177e4SLinus Torvalds * Willy Konynenberg : Transparent proxying support. 521da177e4SLinus Torvalds * Mike McLagan : Routing by source 531da177e4SLinus Torvalds * David S. Miller : New socket lookup architecture. 541da177e4SLinus Torvalds * Last socket cache retained as it 551da177e4SLinus Torvalds * does have a high hit rate. 561da177e4SLinus Torvalds * Olaf Kirch : Don't linearise iovec on sendmsg. 571da177e4SLinus Torvalds * Andi Kleen : Some cleanups, cache destination entry 581da177e4SLinus Torvalds * for connect. 591da177e4SLinus Torvalds * Vitaly E. Lavrov : Transparent proxy revived after year coma. 601da177e4SLinus Torvalds * Melvin Smith : Check msg_name not msg_namelen in sendto(), 611da177e4SLinus Torvalds * return ENOTCONN for unconnected sockets (POSIX) 621da177e4SLinus Torvalds * Janos Farkas : don't deliver multi/broadcasts to a different 631da177e4SLinus Torvalds * bound-to-device socket 641da177e4SLinus Torvalds * Hirokazu Takahashi : HW checksumming for outgoing UDP 651da177e4SLinus Torvalds * datagrams. 661da177e4SLinus Torvalds * Hirokazu Takahashi : sendfile() on UDP works now. 671da177e4SLinus Torvalds * Arnaldo C. Melo : convert /proc/net/udp to seq_file 681da177e4SLinus Torvalds * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 691da177e4SLinus Torvalds * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind 701da177e4SLinus Torvalds * a single port at the same time. 711da177e4SLinus Torvalds * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support 72342f0234SJames Chapman * James Chapman : Add L2TP encapsulation type. 731da177e4SLinus Torvalds */ 741da177e4SLinus Torvalds 75afd46503SJoe Perches #define pr_fmt(fmt) "UDP: " fmt 76afd46503SJoe Perches 777c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 781da177e4SLinus Torvalds #include <asm/ioctls.h> 7957c8a661SMike Rapoport #include <linux/memblock.h> 808203efb3SEric Dumazet #include <linux/highmem.h> 811da177e4SLinus Torvalds #include <linux/types.h> 821da177e4SLinus Torvalds #include <linux/fcntl.h> 831da177e4SLinus Torvalds #include <linux/module.h> 841da177e4SLinus Torvalds #include <linux/socket.h> 851da177e4SLinus Torvalds #include <linux/sockios.h> 8614c85021SArnaldo Carvalho de Melo #include <linux/igmp.h> 876e540309SShawn Bohrer #include <linux/inetdevice.h> 881da177e4SLinus Torvalds #include <linux/in.h> 891da177e4SLinus Torvalds #include <linux/errno.h> 901da177e4SLinus Torvalds #include <linux/timer.h> 911da177e4SLinus Torvalds #include <linux/mm.h> 921da177e4SLinus Torvalds #include <linux/inet.h> 931da177e4SLinus Torvalds #include <linux/netdevice.h> 945a0e3ad6STejun Heo #include <linux/slab.h> 95c752f073SArnaldo Carvalho de Melo #include <net/tcp_states.h> 961da177e4SLinus Torvalds #include <linux/skbuff.h> 971da177e4SLinus Torvalds #include <linux/proc_fs.h> 981da177e4SLinus Torvalds #include <linux/seq_file.h> 99457c4cbcSEric W. Biederman #include <net/net_namespace.h> 1001da177e4SLinus Torvalds #include <net/icmp.h> 101421b3885SShawn Bohrer #include <net/inet_hashtables.h> 102e7cc0824SStefano Brivio #include <net/ip_tunnels.h> 1031da177e4SLinus Torvalds #include <net/route.h> 1041da177e4SLinus Torvalds #include <net/checksum.h> 1051da177e4SLinus Torvalds #include <net/xfrm.h> 106296f7ea7SSatoru Moriya #include <trace/events/udp.h> 107447167bfSEric Dumazet #include <linux/static_key.h> 108951cf368SYonghong Song #include <linux/btf_ids.h> 10922911fc5SEric Dumazet #include <trace/events/skb.h> 110076bb0c8SEliezer Tamir #include <net/busy_poll.h> 111ba4e58ecSGerrit Renker #include "udp_impl.h" 112e32ea7e7SCraig Gallek #include <net/sock_reuseport.h> 113217375a0SEric Dumazet #include <net/addrconf.h> 11460fb9567SPaolo Abeni #include <net/udp_tunnel.h> 1150146dca7SSabrina Dubroca #if IS_ENABLED(CONFIG_IPV6) 1160146dca7SSabrina Dubroca #include <net/ipv6_stubs.h> 1170146dca7SSabrina Dubroca #endif 1181da177e4SLinus Torvalds 119f86dcc5aSEric Dumazet struct udp_table udp_table __read_mostly; 120645ca708SEric Dumazet EXPORT_SYMBOL(udp_table); 1211da177e4SLinus Torvalds 1228d987e5cSEric Dumazet long sysctl_udp_mem[3] __read_mostly; 12395766fffSHideo Aoki EXPORT_SYMBOL(sysctl_udp_mem); 124c482c568SEric Dumazet 125*91b6d325SEric Dumazet atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp; 12695766fffSHideo Aoki EXPORT_SYMBOL(udp_memory_allocated); 12795766fffSHideo Aoki 128f86dcc5aSEric Dumazet #define MAX_UDP_PORTS 65536 129f86dcc5aSEric Dumazet #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN) 13098322f22SEric Dumazet 131f24d43c0SEric Dumazet static int udp_lib_lport_inuse(struct net *net, __u16 num, 132645ca708SEric Dumazet const struct udp_hslot *hslot, 13398322f22SEric Dumazet unsigned long *bitmap, 134fe38d2a1SJosef Bacik struct sock *sk, unsigned int log) 13525030a7fSGerrit Renker { 136f24d43c0SEric Dumazet struct sock *sk2; 137ba418fa3STom Herbert kuid_t uid = sock_i_uid(sk); 13825030a7fSGerrit Renker 139ca065d0cSEric Dumazet sk_for_each(sk2, &hslot->head) { 140f24d43c0SEric Dumazet if (net_eq(sock_net(sk2), net) && 141f24d43c0SEric Dumazet sk2 != sk && 142d4cada4aSEric Dumazet (bitmap || udp_sk(sk2)->udp_port_hash == num) && 143f24d43c0SEric Dumazet (!sk2->sk_reuse || !sk->sk_reuse) && 1449d4fb27dSJoe Perches (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || 1459d4fb27dSJoe Perches sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 146fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, true)) { 147df560056SEric Garver if (sk2->sk_reuseport && sk->sk_reuseport && 148df560056SEric Garver !rcu_access_pointer(sk->sk_reuseport_cb) && 149df560056SEric Garver uid_eq(uid, sock_i_uid(sk2))) { 150df560056SEric Garver if (!bitmap) 151df560056SEric Garver return 0; 152df560056SEric Garver } else { 1534243cdc2SJoe Perches if (!bitmap) 154fc038410SDavid S. Miller return 1; 155df560056SEric Garver __set_bit(udp_sk(sk2)->udp_port_hash >> log, 156df560056SEric Garver bitmap); 157df560056SEric Garver } 1584243cdc2SJoe Perches } 15998322f22SEric Dumazet } 16025030a7fSGerrit Renker return 0; 16125030a7fSGerrit Renker } 16225030a7fSGerrit Renker 16330fff923SEric Dumazet /* 16430fff923SEric Dumazet * Note: we still hold spinlock of primary hash chain, so no other writer 16530fff923SEric Dumazet * can insert/delete a socket with local_port == num 16630fff923SEric Dumazet */ 16730fff923SEric Dumazet static int udp_lib_lport_inuse2(struct net *net, __u16 num, 16830fff923SEric Dumazet struct udp_hslot *hslot2, 169fe38d2a1SJosef Bacik struct sock *sk) 17030fff923SEric Dumazet { 17130fff923SEric Dumazet struct sock *sk2; 172ba418fa3STom Herbert kuid_t uid = sock_i_uid(sk); 17330fff923SEric Dumazet int res = 0; 17430fff923SEric Dumazet 17530fff923SEric Dumazet spin_lock(&hslot2->lock); 176ca065d0cSEric Dumazet udp_portaddr_for_each_entry(sk2, &hslot2->head) { 17730fff923SEric Dumazet if (net_eq(sock_net(sk2), net) && 17830fff923SEric Dumazet sk2 != sk && 17930fff923SEric Dumazet (udp_sk(sk2)->udp_port_hash == num) && 18030fff923SEric Dumazet (!sk2->sk_reuse || !sk->sk_reuse) && 1819d4fb27dSJoe Perches (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || 1829d4fb27dSJoe Perches sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 183fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, true)) { 184df560056SEric Garver if (sk2->sk_reuseport && sk->sk_reuseport && 185df560056SEric Garver !rcu_access_pointer(sk->sk_reuseport_cb) && 186df560056SEric Garver uid_eq(uid, sock_i_uid(sk2))) { 187df560056SEric Garver res = 0; 188df560056SEric Garver } else { 18930fff923SEric Dumazet res = 1; 190df560056SEric Garver } 19130fff923SEric Dumazet break; 19230fff923SEric Dumazet } 1934243cdc2SJoe Perches } 19430fff923SEric Dumazet spin_unlock(&hslot2->lock); 19530fff923SEric Dumazet return res; 19630fff923SEric Dumazet } 19730fff923SEric Dumazet 198fe38d2a1SJosef Bacik static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot) 199e32ea7e7SCraig Gallek { 200e32ea7e7SCraig Gallek struct net *net = sock_net(sk); 201e32ea7e7SCraig Gallek kuid_t uid = sock_i_uid(sk); 202e32ea7e7SCraig Gallek struct sock *sk2; 203e32ea7e7SCraig Gallek 204ca065d0cSEric Dumazet sk_for_each(sk2, &hslot->head) { 205e32ea7e7SCraig Gallek if (net_eq(sock_net(sk2), net) && 206e32ea7e7SCraig Gallek sk2 != sk && 207e32ea7e7SCraig Gallek sk2->sk_family == sk->sk_family && 208e32ea7e7SCraig Gallek ipv6_only_sock(sk2) == ipv6_only_sock(sk) && 209e32ea7e7SCraig Gallek (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) && 210e32ea7e7SCraig Gallek (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && 211e32ea7e7SCraig Gallek sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && 212fe38d2a1SJosef Bacik inet_rcv_saddr_equal(sk, sk2, false)) { 2132dbb9b9eSMartin KaFai Lau return reuseport_add_sock(sk, sk2, 2142dbb9b9eSMartin KaFai Lau inet_rcv_saddr_any(sk)); 215e32ea7e7SCraig Gallek } 216e32ea7e7SCraig Gallek } 217e32ea7e7SCraig Gallek 2182dbb9b9eSMartin KaFai Lau return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); 219e32ea7e7SCraig Gallek } 220e32ea7e7SCraig Gallek 22125030a7fSGerrit Renker /** 2226ba5a3c5SPavel Emelyanov * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6 22325030a7fSGerrit Renker * 22425030a7fSGerrit Renker * @sk: socket struct in question 22525030a7fSGerrit Renker * @snum: port number to look up 22625985edcSLucas De Marchi * @hash2_nulladdr: AF-dependent hash value in secondary hash chains, 22730fff923SEric Dumazet * with NULL address 22825030a7fSGerrit Renker */ 2296ba5a3c5SPavel Emelyanov int udp_lib_get_port(struct sock *sk, unsigned short snum, 23030fff923SEric Dumazet unsigned int hash2_nulladdr) 2311da177e4SLinus Torvalds { 232512615b6SEric Dumazet struct udp_hslot *hslot, *hslot2; 233645ca708SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 23425030a7fSGerrit Renker int error = 1; 2353b1e0a65SYOSHIFUJI Hideaki struct net *net = sock_net(sk); 2361da177e4SLinus Torvalds 23732c1da70SStephen Hemminger if (!snum) { 2389088c560SEric Dumazet int low, high, remaining; 23995c96174SEric Dumazet unsigned int rand; 24098322f22SEric Dumazet unsigned short first, last; 24198322f22SEric Dumazet DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN); 2421da177e4SLinus Torvalds 2430bbf87d8SEric W. Biederman inet_get_local_port_range(net, &low, &high); 244a25de534SAnton Arapov remaining = (high - low) + 1; 245227b60f5SStephen Hemminger 24663862b5bSAruna-Hewapathirane rand = prandom_u32(); 2478fc54f68SDaniel Borkmann first = reciprocal_scale(rand, remaining) + low; 24898322f22SEric Dumazet /* 24998322f22SEric Dumazet * force rand to be an odd multiple of UDP_HTABLE_SIZE 25098322f22SEric Dumazet */ 251f86dcc5aSEric Dumazet rand = (rand | 1) * (udptable->mask + 1); 2525781b235SEric Dumazet last = first + udptable->mask + 1; 2535781b235SEric Dumazet do { 254f86dcc5aSEric Dumazet hslot = udp_hashslot(udptable, net, first); 25598322f22SEric Dumazet bitmap_zero(bitmap, PORTS_PER_CHAIN); 256645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 25798322f22SEric Dumazet udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, 258fe38d2a1SJosef Bacik udptable->log); 25998322f22SEric Dumazet 26098322f22SEric Dumazet snum = first; 26198322f22SEric Dumazet /* 26298322f22SEric Dumazet * Iterate on all possible values of snum for this hash. 26398322f22SEric Dumazet * Using steps of an odd multiple of UDP_HTABLE_SIZE 26498322f22SEric Dumazet * give us randomization and full range coverage. 26598322f22SEric Dumazet */ 2669088c560SEric Dumazet do { 26798322f22SEric Dumazet if (low <= snum && snum <= high && 268e3826f1eSAmerigo Wang !test_bit(snum >> udptable->log, bitmap) && 269122ff243SWANG Cong !inet_is_local_reserved_port(net, snum)) 27098322f22SEric Dumazet goto found; 27198322f22SEric Dumazet snum += rand; 27298322f22SEric Dumazet } while (snum != first); 27398322f22SEric Dumazet spin_unlock_bh(&hslot->lock); 274df560056SEric Garver cond_resched(); 2755781b235SEric Dumazet } while (++first != last); 27698322f22SEric Dumazet goto fail; 277645ca708SEric Dumazet } else { 278f86dcc5aSEric Dumazet hslot = udp_hashslot(udptable, net, snum); 279645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 28030fff923SEric Dumazet if (hslot->count > 10) { 28130fff923SEric Dumazet int exist; 28230fff923SEric Dumazet unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; 28330fff923SEric Dumazet 28430fff923SEric Dumazet slot2 &= udptable->mask; 28530fff923SEric Dumazet hash2_nulladdr &= udptable->mask; 28630fff923SEric Dumazet 28730fff923SEric Dumazet hslot2 = udp_hashslot2(udptable, slot2); 28830fff923SEric Dumazet if (hslot->count < hslot2->count) 28930fff923SEric Dumazet goto scan_primary_hash; 29030fff923SEric Dumazet 291fe38d2a1SJosef Bacik exist = udp_lib_lport_inuse2(net, snum, hslot2, sk); 29230fff923SEric Dumazet if (!exist && (hash2_nulladdr != slot2)) { 29330fff923SEric Dumazet hslot2 = udp_hashslot2(udptable, hash2_nulladdr); 29430fff923SEric Dumazet exist = udp_lib_lport_inuse2(net, snum, hslot2, 295fe38d2a1SJosef Bacik sk); 29630fff923SEric Dumazet } 29730fff923SEric Dumazet if (exist) 29830fff923SEric Dumazet goto fail_unlock; 29930fff923SEric Dumazet else 30030fff923SEric Dumazet goto found; 30130fff923SEric Dumazet } 30230fff923SEric Dumazet scan_primary_hash: 303fe38d2a1SJosef Bacik if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0)) 304645ca708SEric Dumazet goto fail_unlock; 305645ca708SEric Dumazet } 30698322f22SEric Dumazet found: 307c720c7e8SEric Dumazet inet_sk(sk)->inet_num = snum; 308d4cada4aSEric Dumazet udp_sk(sk)->udp_port_hash = snum; 309d4cada4aSEric Dumazet udp_sk(sk)->udp_portaddr_hash ^= snum; 3101da177e4SLinus Torvalds if (sk_unhashed(sk)) { 311e32ea7e7SCraig Gallek if (sk->sk_reuseport && 312fe38d2a1SJosef Bacik udp_reuseport_add_sock(sk, hslot)) { 313e32ea7e7SCraig Gallek inet_sk(sk)->inet_num = 0; 314e32ea7e7SCraig Gallek udp_sk(sk)->udp_port_hash = 0; 315e32ea7e7SCraig Gallek udp_sk(sk)->udp_portaddr_hash ^= snum; 316e32ea7e7SCraig Gallek goto fail_unlock; 317e32ea7e7SCraig Gallek } 318e32ea7e7SCraig Gallek 319ca065d0cSEric Dumazet sk_add_node_rcu(sk, &hslot->head); 320fdcc8aa9SEric Dumazet hslot->count++; 321c29a0bc4SPavel Emelyanov sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 322512615b6SEric Dumazet 323512615b6SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 324512615b6SEric Dumazet spin_lock(&hslot2->lock); 325d894ba18SCraig Gallek if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && 326d894ba18SCraig Gallek sk->sk_family == AF_INET6) 3271602f49bSDavid S. Miller hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node, 328d894ba18SCraig Gallek &hslot2->head); 329d894ba18SCraig Gallek else 330ca065d0cSEric Dumazet hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, 331512615b6SEric Dumazet &hslot2->head); 332512615b6SEric Dumazet hslot2->count++; 333512615b6SEric Dumazet spin_unlock(&hslot2->lock); 3341da177e4SLinus Torvalds } 335ca065d0cSEric Dumazet sock_set_flag(sk, SOCK_RCU_FREE); 33625030a7fSGerrit Renker error = 0; 337645ca708SEric Dumazet fail_unlock: 338645ca708SEric Dumazet spin_unlock_bh(&hslot->lock); 3391da177e4SLinus Torvalds fail: 34025030a7fSGerrit Renker return error; 3411da177e4SLinus Torvalds } 342c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_get_port); 3431da177e4SLinus Torvalds 3446ba5a3c5SPavel Emelyanov int udp_v4_get_port(struct sock *sk, unsigned short snum) 345db8dac20SDavid S. Miller { 34630fff923SEric Dumazet unsigned int hash2_nulladdr = 347f0b1e64cSMartin KaFai Lau ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum); 34830fff923SEric Dumazet unsigned int hash2_partial = 349f0b1e64cSMartin KaFai Lau ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); 35030fff923SEric Dumazet 351d4cada4aSEric Dumazet /* precompute partial secondary hash */ 35230fff923SEric Dumazet udp_sk(sk)->udp_portaddr_hash = hash2_partial; 353fe38d2a1SJosef Bacik return udp_lib_get_port(sk, snum, hash2_nulladdr); 354db8dac20SDavid S. Miller } 355db8dac20SDavid S. Miller 356d1e37288SSu, Xuemin static int compute_score(struct sock *sk, struct net *net, 357d1e37288SSu, Xuemin __be32 saddr, __be16 sport, 358fb74c277SDavid Ahern __be32 daddr, unsigned short hnum, 35973545373STim Beale int dif, int sdif) 360645ca708SEric Dumazet { 36160c04aecSJoe Perches int score; 36260c04aecSJoe Perches struct inet_sock *inet; 3636da5b0f0SMike Manning bool dev_match; 364645ca708SEric Dumazet 36560c04aecSJoe Perches if (!net_eq(sock_net(sk), net) || 36660c04aecSJoe Perches udp_sk(sk)->udp_port_hash != hnum || 36760c04aecSJoe Perches ipv6_only_sock(sk)) 36860c04aecSJoe Perches return -1; 369645ca708SEric Dumazet 3704cdeeee9SPeter Oskolkov if (sk->sk_rcv_saddr != daddr) 371645ca708SEric Dumazet return -1; 37260c04aecSJoe Perches 3734cdeeee9SPeter Oskolkov score = (sk->sk_family == PF_INET) ? 2 : 1; 3744cdeeee9SPeter Oskolkov 3754cdeeee9SPeter Oskolkov inet = inet_sk(sk); 376c720c7e8SEric Dumazet if (inet->inet_daddr) { 377c720c7e8SEric Dumazet if (inet->inet_daddr != saddr) 378645ca708SEric Dumazet return -1; 379ba418fa3STom Herbert score += 4; 380645ca708SEric Dumazet } 38160c04aecSJoe Perches 382c720c7e8SEric Dumazet if (inet->inet_dport) { 383c720c7e8SEric Dumazet if (inet->inet_dport != sport) 384645ca708SEric Dumazet return -1; 385ba418fa3STom Herbert score += 4; 386645ca708SEric Dumazet } 38760c04aecSJoe Perches 3886da5b0f0SMike Manning dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, 3896da5b0f0SMike Manning dif, sdif); 39069678bcdSPaolo Abeni if (!dev_match) 391645ca708SEric Dumazet return -1; 3928d6c414cSMike Manning if (sk->sk_bound_dev_if) 393ba418fa3STom Herbert score += 4; 394fb74c277SDavid Ahern 3957170a977SEric Dumazet if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) 39670da268bSEric Dumazet score++; 397645ca708SEric Dumazet return score; 398645ca708SEric Dumazet } 399645ca708SEric Dumazet 4006eada011SEric Dumazet static u32 udp_ehashfn(const struct net *net, const __be32 laddr, 40165cd8033SHannes Frederic Sowa const __u16 lport, const __be32 faddr, 40265cd8033SHannes Frederic Sowa const __be16 fport) 40365cd8033SHannes Frederic Sowa { 4041bbdceefSHannes Frederic Sowa static u32 udp_ehash_secret __read_mostly; 4051bbdceefSHannes Frederic Sowa 4061bbdceefSHannes Frederic Sowa net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret)); 4071bbdceefSHannes Frederic Sowa 40865cd8033SHannes Frederic Sowa return __inet_ehashfn(laddr, lport, faddr, fport, 4091bbdceefSHannes Frederic Sowa udp_ehash_secret + net_hash_mix(net)); 41065cd8033SHannes Frederic Sowa } 41165cd8033SHannes Frederic Sowa 412a57066b1SDavid S. Miller static struct sock *lookup_reuseport(struct net *net, struct sock *sk, 4137629c73aSJakub Sitnicki struct sk_buff *skb, 4147629c73aSJakub Sitnicki __be32 saddr, __be16 sport, 4157629c73aSJakub Sitnicki __be32 daddr, unsigned short hnum) 4167629c73aSJakub Sitnicki { 4177629c73aSJakub Sitnicki struct sock *reuse_sk = NULL; 4187629c73aSJakub Sitnicki u32 hash; 4197629c73aSJakub Sitnicki 4207629c73aSJakub Sitnicki if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) { 4217629c73aSJakub Sitnicki hash = udp_ehashfn(net, daddr, hnum, saddr, sport); 4227629c73aSJakub Sitnicki reuse_sk = reuseport_select_sock(sk, hash, skb, 4237629c73aSJakub Sitnicki sizeof(struct udphdr)); 4247629c73aSJakub Sitnicki } 4257629c73aSJakub Sitnicki return reuse_sk; 4267629c73aSJakub Sitnicki } 4277629c73aSJakub Sitnicki 428d1e37288SSu, Xuemin /* called with rcu_read_lock() */ 4295051ebd2SEric Dumazet static struct sock *udp4_lib_lookup2(struct net *net, 4305051ebd2SEric Dumazet __be32 saddr, __be16 sport, 431fb74c277SDavid Ahern __be32 daddr, unsigned int hnum, 43273545373STim Beale int dif, int sdif, 433d1e37288SSu, Xuemin struct udp_hslot *hslot2, 4341134158bSCraig Gallek struct sk_buff *skb) 4355051ebd2SEric Dumazet { 4365051ebd2SEric Dumazet struct sock *sk, *result; 437e94a62f5SPaolo Abeni int score, badness; 4385051ebd2SEric Dumazet 4395051ebd2SEric Dumazet result = NULL; 440ba418fa3STom Herbert badness = 0; 441ca065d0cSEric Dumazet udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { 442d1e37288SSu, Xuemin score = compute_score(sk, net, saddr, sport, 44373545373STim Beale daddr, hnum, dif, sdif); 4445051ebd2SEric Dumazet if (score > badness) { 4457629c73aSJakub Sitnicki result = lookup_reuseport(net, sk, skb, 4467629c73aSJakub Sitnicki saddr, sport, daddr, hnum); 447a57066b1SDavid S. Miller /* Fall back to scoring if group has connections */ 448a57066b1SDavid S. Miller if (result && !reuseport_has_conns(sk, false)) 449ca065d0cSEric Dumazet return result; 4507629c73aSJakub Sitnicki 451a57066b1SDavid S. Miller result = result ? : sk; 452ca065d0cSEric Dumazet badness = score; 4535051ebd2SEric Dumazet } 4545051ebd2SEric Dumazet } 4555051ebd2SEric Dumazet return result; 4565051ebd2SEric Dumazet } 4575051ebd2SEric Dumazet 458a57066b1SDavid S. Miller static struct sock *udp4_lookup_run_bpf(struct net *net, 45972f7e944SJakub Sitnicki struct udp_table *udptable, 46072f7e944SJakub Sitnicki struct sk_buff *skb, 46172f7e944SJakub Sitnicki __be32 saddr, __be16 sport, 462f8931565SMark Pashmfouroush __be32 daddr, u16 hnum, const int dif) 46372f7e944SJakub Sitnicki { 46472f7e944SJakub Sitnicki struct sock *sk, *reuse_sk; 46572f7e944SJakub Sitnicki bool no_reuseport; 46672f7e944SJakub Sitnicki 46772f7e944SJakub Sitnicki if (udptable != &udp_table) 46872f7e944SJakub Sitnicki return NULL; /* only UDP is supported */ 46972f7e944SJakub Sitnicki 470f8931565SMark Pashmfouroush no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP, saddr, sport, 471f8931565SMark Pashmfouroush daddr, hnum, dif, &sk); 47272f7e944SJakub Sitnicki if (no_reuseport || IS_ERR_OR_NULL(sk)) 47372f7e944SJakub Sitnicki return sk; 47472f7e944SJakub Sitnicki 47572f7e944SJakub Sitnicki reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum); 476c64c9c28SJakub Sitnicki if (reuse_sk) 47772f7e944SJakub Sitnicki sk = reuse_sk; 47872f7e944SJakub Sitnicki return sk; 47972f7e944SJakub Sitnicki } 48072f7e944SJakub Sitnicki 481db8dac20SDavid S. Miller /* UDP is nearly always wildcards out the wazoo, it makes no sense to try 482db8dac20SDavid S. Miller * harder than this. -DaveM 483db8dac20SDavid S. Miller */ 484fce82338SPavel Emelyanov struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, 485fb74c277SDavid Ahern __be16 sport, __be32 daddr, __be16 dport, int dif, 486fb74c277SDavid Ahern int sdif, struct udp_table *udptable, struct sk_buff *skb) 487db8dac20SDavid S. Miller { 488db8dac20SDavid S. Miller unsigned short hnum = ntohs(dport); 4894cdeeee9SPeter Oskolkov unsigned int hash2, slot2; 4904cdeeee9SPeter Oskolkov struct udp_hslot *hslot2; 49172f7e944SJakub Sitnicki struct sock *result, *sk; 492db8dac20SDavid S. Miller 493f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, daddr, hnum); 4945051ebd2SEric Dumazet slot2 = hash2 & udptable->mask; 4955051ebd2SEric Dumazet hslot2 = &udptable->hash2[slot2]; 4965051ebd2SEric Dumazet 49772f7e944SJakub Sitnicki /* Lookup connected or non-wildcard socket */ 4985051ebd2SEric Dumazet result = udp4_lib_lookup2(net, saddr, sport, 499fb74c277SDavid Ahern daddr, hnum, dif, sdif, 50073545373STim Beale hslot2, skb); 50172f7e944SJakub Sitnicki if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED) 50272f7e944SJakub Sitnicki goto done; 50372f7e944SJakub Sitnicki 50472f7e944SJakub Sitnicki /* Lookup redirect from BPF */ 50572f7e944SJakub Sitnicki if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { 50672f7e944SJakub Sitnicki sk = udp4_lookup_run_bpf(net, udptable, skb, 507f8931565SMark Pashmfouroush saddr, sport, daddr, hnum, dif); 50872f7e944SJakub Sitnicki if (sk) { 50972f7e944SJakub Sitnicki result = sk; 51072f7e944SJakub Sitnicki goto done; 51172f7e944SJakub Sitnicki } 51272f7e944SJakub Sitnicki } 51372f7e944SJakub Sitnicki 51472f7e944SJakub Sitnicki /* Got non-wildcard socket or error on first lookup */ 51572f7e944SJakub Sitnicki if (result) 51672f7e944SJakub Sitnicki goto done; 51772f7e944SJakub Sitnicki 51872f7e944SJakub Sitnicki /* Lookup wildcard sockets */ 519f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum); 5205051ebd2SEric Dumazet slot2 = hash2 & udptable->mask; 5215051ebd2SEric Dumazet hslot2 = &udptable->hash2[slot2]; 5225051ebd2SEric Dumazet 5231223c67cSJorge Boncompte [DTI2] result = udp4_lib_lookup2(net, saddr, sport, 5244cdeeee9SPeter Oskolkov htonl(INADDR_ANY), hnum, dif, sdif, 52573545373STim Beale hslot2, skb); 52672f7e944SJakub Sitnicki done: 52788e235b8SEnrico Weigelt if (IS_ERR(result)) 5288217ca65SMartin KaFai Lau return NULL; 5295051ebd2SEric Dumazet return result; 5305051ebd2SEric Dumazet } 531fce82338SPavel Emelyanov EXPORT_SYMBOL_GPL(__udp4_lib_lookup); 532db8dac20SDavid S. Miller 533607c4aafSKOVACS Krisztian static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb, 534607c4aafSKOVACS Krisztian __be16 sport, __be16 dport, 535645ca708SEric Dumazet struct udp_table *udptable) 536607c4aafSKOVACS Krisztian { 537607c4aafSKOVACS Krisztian const struct iphdr *iph = ip_hdr(skb); 538607c4aafSKOVACS Krisztian 539ed7cbbceSAlexander Duyck return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, 540607c4aafSKOVACS Krisztian iph->daddr, dport, inet_iif(skb), 541fb74c277SDavid Ahern inet_sdif(skb), udptable, skb); 542607c4aafSKOVACS Krisztian } 543607c4aafSKOVACS Krisztian 5447b58e63eSEric Dumazet struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb, 54563058308STom Herbert __be16 sport, __be16 dport) 54663058308STom Herbert { 547257a525fSMartin KaFai Lau const struct iphdr *iph = ip_hdr(skb); 548257a525fSMartin KaFai Lau 549257a525fSMartin KaFai Lau return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, 550257a525fSMartin KaFai Lau iph->daddr, dport, inet_iif(skb), 551257a525fSMartin KaFai Lau inet_sdif(skb), &udp_table, NULL); 55263058308STom Herbert } 55363058308STom Herbert 554ca065d0cSEric Dumazet /* Must be called under rcu_read_lock(). 555ca065d0cSEric Dumazet * Does increment socket refcount. 556ca065d0cSEric Dumazet */ 5576e86000cSArnd Bergmann #if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4) 558bcd41303SKOVACS Krisztian struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport, 559bcd41303SKOVACS Krisztian __be32 daddr, __be16 dport, int dif) 560bcd41303SKOVACS Krisztian { 561ca065d0cSEric Dumazet struct sock *sk; 562ca065d0cSEric Dumazet 563ca065d0cSEric Dumazet sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, 564fb74c277SDavid Ahern dif, 0, &udp_table, NULL); 56541c6d650SReshetova, Elena if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) 566ca065d0cSEric Dumazet sk = NULL; 567ca065d0cSEric Dumazet return sk; 568bcd41303SKOVACS Krisztian } 569bcd41303SKOVACS Krisztian EXPORT_SYMBOL_GPL(udp4_lib_lookup); 570ca065d0cSEric Dumazet #endif 571bcd41303SKOVACS Krisztian 572421b3885SShawn Bohrer static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, 573421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 574421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 575fb74c277SDavid Ahern int dif, int sdif, unsigned short hnum) 576421b3885SShawn Bohrer { 577421b3885SShawn Bohrer struct inet_sock *inet = inet_sk(sk); 578421b3885SShawn Bohrer 579421b3885SShawn Bohrer if (!net_eq(sock_net(sk), net) || 580421b3885SShawn Bohrer udp_sk(sk)->udp_port_hash != hnum || 581421b3885SShawn Bohrer (inet->inet_daddr && inet->inet_daddr != rmt_addr) || 582421b3885SShawn Bohrer (inet->inet_dport != rmt_port && inet->inet_dport) || 583421b3885SShawn Bohrer (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) || 584421b3885SShawn Bohrer ipv6_only_sock(sk) || 58582ba25c6STim Beale !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) 586421b3885SShawn Bohrer return false; 58760d9b031SDavid Ahern if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif)) 588421b3885SShawn Bohrer return false; 589421b3885SShawn Bohrer return true; 590421b3885SShawn Bohrer } 591421b3885SShawn Bohrer 592a36e185eSStefano Brivio DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key); 593a36e185eSStefano Brivio void udp_encap_enable(void) 594a36e185eSStefano Brivio { 5959c480601SPaolo Abeni static_branch_inc(&udp_encap_needed_key); 596a36e185eSStefano Brivio } 597a36e185eSStefano Brivio EXPORT_SYMBOL(udp_encap_enable); 598a36e185eSStefano Brivio 599a4a600ddSXin Long void udp_encap_disable(void) 600a4a600ddSXin Long { 601a4a600ddSXin Long static_branch_dec(&udp_encap_needed_key); 602a4a600ddSXin Long } 603a4a600ddSXin Long EXPORT_SYMBOL(udp_encap_disable); 604a4a600ddSXin Long 605e7cc0824SStefano Brivio /* Handler for tunnels with arbitrary destination ports: no socket lookup, go 606e7cc0824SStefano Brivio * through error handlers in encapsulations looking for a match. 607e7cc0824SStefano Brivio */ 608e7cc0824SStefano Brivio static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info) 609e7cc0824SStefano Brivio { 610e7cc0824SStefano Brivio int i; 611e7cc0824SStefano Brivio 612e7cc0824SStefano Brivio for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) { 613e7cc0824SStefano Brivio int (*handler)(struct sk_buff *skb, u32 info); 61492b95364SPaolo Abeni const struct ip_tunnel_encap_ops *encap; 615e7cc0824SStefano Brivio 61692b95364SPaolo Abeni encap = rcu_dereference(iptun_encaps[i]); 61792b95364SPaolo Abeni if (!encap) 618e7cc0824SStefano Brivio continue; 61992b95364SPaolo Abeni handler = encap->err_handler; 620e7cc0824SStefano Brivio if (handler && !handler(skb, info)) 621e7cc0824SStefano Brivio return 0; 622e7cc0824SStefano Brivio } 623e7cc0824SStefano Brivio 624e7cc0824SStefano Brivio return -ENOENT; 625e7cc0824SStefano Brivio } 626e7cc0824SStefano Brivio 627a36e185eSStefano Brivio /* Try to match ICMP errors to UDP tunnels by looking up a socket without 628a36e185eSStefano Brivio * reversing source and destination port: this will match tunnels that force the 629a36e185eSStefano Brivio * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that 630a36e185eSStefano Brivio * lwtunnels might actually break this assumption by being configured with 631a36e185eSStefano Brivio * different destination ports on endpoints, in this case we won't be able to 632a36e185eSStefano Brivio * trace ICMP messages back to them. 633a36e185eSStefano Brivio * 634e7cc0824SStefano Brivio * If this doesn't match any socket, probe tunnels with arbitrary destination 635e7cc0824SStefano Brivio * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port 636e7cc0824SStefano Brivio * we've sent packets to won't necessarily match the local destination port. 637e7cc0824SStefano Brivio * 638a36e185eSStefano Brivio * Then ask the tunnel implementation to match the error against a valid 639a36e185eSStefano Brivio * association. 640a36e185eSStefano Brivio * 641e7cc0824SStefano Brivio * Return an error if we can't find a match, the socket if we need further 642e7cc0824SStefano Brivio * processing, zero otherwise. 643a36e185eSStefano Brivio */ 644a36e185eSStefano Brivio static struct sock *__udp4_lib_err_encap(struct net *net, 645a36e185eSStefano Brivio const struct iphdr *iph, 646a36e185eSStefano Brivio struct udphdr *uh, 647a36e185eSStefano Brivio struct udp_table *udptable, 6489bfce73cSVadim Fedorenko struct sock *sk, 649e7cc0824SStefano Brivio struct sk_buff *skb, u32 info) 650a36e185eSStefano Brivio { 6519bfce73cSVadim Fedorenko int (*lookup)(struct sock *sk, struct sk_buff *skb); 652a36e185eSStefano Brivio int network_offset, transport_offset; 6539bfce73cSVadim Fedorenko struct udp_sock *up; 654a36e185eSStefano Brivio 655a36e185eSStefano Brivio network_offset = skb_network_offset(skb); 656a36e185eSStefano Brivio transport_offset = skb_transport_offset(skb); 657a36e185eSStefano Brivio 658a36e185eSStefano Brivio /* Network header needs to point to the outer IPv4 header inside ICMP */ 659a36e185eSStefano Brivio skb_reset_network_header(skb); 660a36e185eSStefano Brivio 661a36e185eSStefano Brivio /* Transport header needs to point to the UDP header */ 662a36e185eSStefano Brivio skb_set_transport_header(skb, iph->ihl << 2); 663a36e185eSStefano Brivio 6649bfce73cSVadim Fedorenko if (sk) { 6659bfce73cSVadim Fedorenko up = udp_sk(sk); 6669bfce73cSVadim Fedorenko 6679bfce73cSVadim Fedorenko lookup = READ_ONCE(up->encap_err_lookup); 6689bfce73cSVadim Fedorenko if (lookup && lookup(sk, skb)) 6699bfce73cSVadim Fedorenko sk = NULL; 6709bfce73cSVadim Fedorenko 6719bfce73cSVadim Fedorenko goto out; 6729bfce73cSVadim Fedorenko } 6739bfce73cSVadim Fedorenko 674e7cc0824SStefano Brivio sk = __udp4_lib_lookup(net, iph->daddr, uh->source, 675e7cc0824SStefano Brivio iph->saddr, uh->dest, skb->dev->ifindex, 0, 676e7cc0824SStefano Brivio udptable, NULL); 677e7cc0824SStefano Brivio if (sk) { 6789bfce73cSVadim Fedorenko up = udp_sk(sk); 679e7cc0824SStefano Brivio 680a36e185eSStefano Brivio lookup = READ_ONCE(up->encap_err_lookup); 681a36e185eSStefano Brivio if (!lookup || lookup(sk, skb)) 682a36e185eSStefano Brivio sk = NULL; 683e7cc0824SStefano Brivio } 684e7cc0824SStefano Brivio 6859bfce73cSVadim Fedorenko out: 686e7cc0824SStefano Brivio if (!sk) 687e7cc0824SStefano Brivio sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info)); 688a36e185eSStefano Brivio 689a36e185eSStefano Brivio skb_set_transport_header(skb, transport_offset); 690a36e185eSStefano Brivio skb_set_network_header(skb, network_offset); 691a36e185eSStefano Brivio 692a36e185eSStefano Brivio return sk; 693a36e185eSStefano Brivio } 694a36e185eSStefano Brivio 695db8dac20SDavid S. Miller /* 696db8dac20SDavid S. Miller * This routine is called by the ICMP module when it gets some 697db8dac20SDavid S. Miller * sort of error condition. If err < 0 then the socket should 698db8dac20SDavid S. Miller * be closed and the error returned to the user. If err > 0 699db8dac20SDavid S. Miller * it's just the icmp type << 8 | icmp code. 700db8dac20SDavid S. Miller * Header points to the ip header of the error packet. We move 701db8dac20SDavid S. Miller * on past this. Then (as it used to claim before adjustment) 702db8dac20SDavid S. Miller * header points to the first 8 bytes of the udp header. We need 703db8dac20SDavid S. Miller * to find the appropriate port. 704db8dac20SDavid S. Miller */ 705db8dac20SDavid S. Miller 70632bbd879SStefano Brivio int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) 707db8dac20SDavid S. Miller { 708db8dac20SDavid S. Miller struct inet_sock *inet; 709b71d1d42SEric Dumazet const struct iphdr *iph = (const struct iphdr *)skb->data; 710db8dac20SDavid S. Miller struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2)); 711db8dac20SDavid S. Miller const int type = icmp_hdr(skb)->type; 712db8dac20SDavid S. Miller const int code = icmp_hdr(skb)->code; 713a36e185eSStefano Brivio bool tunnel = false; 714db8dac20SDavid S. Miller struct sock *sk; 715db8dac20SDavid S. Miller int harderr; 716db8dac20SDavid S. Miller int err; 717fd54d716SPavel Emelyanov struct net *net = dev_net(skb->dev); 718db8dac20SDavid S. Miller 719fd54d716SPavel Emelyanov sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, 720f64bf6b8SMike Manning iph->saddr, uh->source, skb->dev->ifindex, 721f64bf6b8SMike Manning inet_sdif(skb), udptable, NULL); 7229bfce73cSVadim Fedorenko 723d26796aeSXin Long if (!sk || udp_sk(sk)->encap_type) { 724a36e185eSStefano Brivio /* No socket for error: try tunnels before discarding */ 725e7cc0824SStefano Brivio if (static_branch_unlikely(&udp_encap_needed_key)) { 7269bfce73cSVadim Fedorenko sk = __udp4_lib_err_encap(net, iph, uh, udptable, sk, skb, 727e7cc0824SStefano Brivio info); 728e7cc0824SStefano Brivio if (!sk) 729e7cc0824SStefano Brivio return 0; 7309bfce73cSVadim Fedorenko } else 7319bfce73cSVadim Fedorenko sk = ERR_PTR(-ENOENT); 732e7cc0824SStefano Brivio 733e7cc0824SStefano Brivio if (IS_ERR(sk)) { 734e7cc0824SStefano Brivio __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); 735e7cc0824SStefano Brivio return PTR_ERR(sk); 736e7cc0824SStefano Brivio } 737e7cc0824SStefano Brivio 738a36e185eSStefano Brivio tunnel = true; 739db8dac20SDavid S. Miller } 740db8dac20SDavid S. Miller 741db8dac20SDavid S. Miller err = 0; 742db8dac20SDavid S. Miller harderr = 0; 743db8dac20SDavid S. Miller inet = inet_sk(sk); 744db8dac20SDavid S. Miller 745db8dac20SDavid S. Miller switch (type) { 746db8dac20SDavid S. Miller default: 747db8dac20SDavid S. Miller case ICMP_TIME_EXCEEDED: 748db8dac20SDavid S. Miller err = EHOSTUNREACH; 749db8dac20SDavid S. Miller break; 750db8dac20SDavid S. Miller case ICMP_SOURCE_QUENCH: 751db8dac20SDavid S. Miller goto out; 752db8dac20SDavid S. Miller case ICMP_PARAMETERPROB: 753db8dac20SDavid S. Miller err = EPROTO; 754db8dac20SDavid S. Miller harderr = 1; 755db8dac20SDavid S. Miller break; 756db8dac20SDavid S. Miller case ICMP_DEST_UNREACH: 757db8dac20SDavid S. Miller if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */ 75836393395SDavid S. Miller ipv4_sk_update_pmtu(skb, sk, info); 759db8dac20SDavid S. Miller if (inet->pmtudisc != IP_PMTUDISC_DONT) { 760db8dac20SDavid S. Miller err = EMSGSIZE; 761db8dac20SDavid S. Miller harderr = 1; 762db8dac20SDavid S. Miller break; 763db8dac20SDavid S. Miller } 764db8dac20SDavid S. Miller goto out; 765db8dac20SDavid S. Miller } 766db8dac20SDavid S. Miller err = EHOSTUNREACH; 767db8dac20SDavid S. Miller if (code <= NR_ICMP_UNREACH) { 768db8dac20SDavid S. Miller harderr = icmp_err_convert[code].fatal; 769db8dac20SDavid S. Miller err = icmp_err_convert[code].errno; 770db8dac20SDavid S. Miller } 771db8dac20SDavid S. Miller break; 77255be7a9cSDavid S. Miller case ICMP_REDIRECT: 77355be7a9cSDavid S. Miller ipv4_sk_redirect(skb, sk); 7741a462d18SDuan Jiong goto out; 775db8dac20SDavid S. Miller } 776db8dac20SDavid S. Miller 777db8dac20SDavid S. Miller /* 778db8dac20SDavid S. Miller * RFC1122: OK. Passes ICMP errors back to application, as per 779db8dac20SDavid S. Miller * 4.1.3.3. 780db8dac20SDavid S. Miller */ 781a36e185eSStefano Brivio if (tunnel) { 782a36e185eSStefano Brivio /* ...not for tunnels though: we don't have a sending socket */ 783a36e185eSStefano Brivio goto out; 784a36e185eSStefano Brivio } 785db8dac20SDavid S. Miller if (!inet->recverr) { 786db8dac20SDavid S. Miller if (!harderr || sk->sk_state != TCP_ESTABLISHED) 787db8dac20SDavid S. Miller goto out; 788b1faf566SEric Dumazet } else 789db8dac20SDavid S. Miller ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1)); 790b1faf566SEric Dumazet 791db8dac20SDavid S. Miller sk->sk_err = err; 792e3ae2365SAlexander Aring sk_error_report(sk); 793db8dac20SDavid S. Miller out: 79432bbd879SStefano Brivio return 0; 795db8dac20SDavid S. Miller } 796db8dac20SDavid S. Miller 79732bbd879SStefano Brivio int udp_err(struct sk_buff *skb, u32 info) 798db8dac20SDavid S. Miller { 79932bbd879SStefano Brivio return __udp4_lib_err(skb, info, &udp_table); 800db8dac20SDavid S. Miller } 801db8dac20SDavid S. Miller 802db8dac20SDavid S. Miller /* 803db8dac20SDavid S. Miller * Throw away all pending data and cancel the corking. Socket is locked. 804db8dac20SDavid S. Miller */ 80536d926b9SDenis V. Lunev void udp_flush_pending_frames(struct sock *sk) 806db8dac20SDavid S. Miller { 807db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 808db8dac20SDavid S. Miller 809db8dac20SDavid S. Miller if (up->pending) { 810db8dac20SDavid S. Miller up->len = 0; 811db8dac20SDavid S. Miller up->pending = 0; 812db8dac20SDavid S. Miller ip_flush_pending_frames(sk); 813db8dac20SDavid S. Miller } 814db8dac20SDavid S. Miller } 81536d926b9SDenis V. Lunev EXPORT_SYMBOL(udp_flush_pending_frames); 816db8dac20SDavid S. Miller 817db8dac20SDavid S. Miller /** 818f6b9664fSHerbert Xu * udp4_hwcsum - handle outgoing HW checksumming 819db8dac20SDavid S. Miller * @skb: sk_buff containing the filled-in UDP header 820db8dac20SDavid S. Miller * (checksum field must be zeroed out) 821f6b9664fSHerbert Xu * @src: source IP address 822f6b9664fSHerbert Xu * @dst: destination IP address 823db8dac20SDavid S. Miller */ 824c26bf4a5SThomas Graf void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst) 825db8dac20SDavid S. Miller { 826db8dac20SDavid S. Miller struct udphdr *uh = udp_hdr(skb); 827f6b9664fSHerbert Xu int offset = skb_transport_offset(skb); 828f6b9664fSHerbert Xu int len = skb->len - offset; 829f6b9664fSHerbert Xu int hlen = len; 830db8dac20SDavid S. Miller __wsum csum = 0; 831db8dac20SDavid S. Miller 832ebbe495fSWANG Cong if (!skb_has_frag_list(skb)) { 833db8dac20SDavid S. Miller /* 834db8dac20SDavid S. Miller * Only one fragment on the socket. 835db8dac20SDavid S. Miller */ 836db8dac20SDavid S. Miller skb->csum_start = skb_transport_header(skb) - skb->head; 837db8dac20SDavid S. Miller skb->csum_offset = offsetof(struct udphdr, check); 838f6b9664fSHerbert Xu uh->check = ~csum_tcpudp_magic(src, dst, len, 839f6b9664fSHerbert Xu IPPROTO_UDP, 0); 840db8dac20SDavid S. Miller } else { 841ebbe495fSWANG Cong struct sk_buff *frags; 842ebbe495fSWANG Cong 843db8dac20SDavid S. Miller /* 844db8dac20SDavid S. Miller * HW-checksum won't work as there are two or more 845db8dac20SDavid S. Miller * fragments on the socket so that all csums of sk_buffs 846db8dac20SDavid S. Miller * should be together 847db8dac20SDavid S. Miller */ 848ebbe495fSWANG Cong skb_walk_frags(skb, frags) { 849f6b9664fSHerbert Xu csum = csum_add(csum, frags->csum); 850f6b9664fSHerbert Xu hlen -= frags->len; 851ebbe495fSWANG Cong } 852db8dac20SDavid S. Miller 853f6b9664fSHerbert Xu csum = skb_checksum(skb, offset, hlen, csum); 854db8dac20SDavid S. Miller skb->ip_summed = CHECKSUM_NONE; 855db8dac20SDavid S. Miller 856db8dac20SDavid S. Miller uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum); 857db8dac20SDavid S. Miller if (uh->check == 0) 858db8dac20SDavid S. Miller uh->check = CSUM_MANGLED_0; 859db8dac20SDavid S. Miller } 860db8dac20SDavid S. Miller } 861c26bf4a5SThomas Graf EXPORT_SYMBOL_GPL(udp4_hwcsum); 862db8dac20SDavid S. Miller 863af5fcba7STom Herbert /* Function to set UDP checksum for an IPv4 UDP packet. This is intended 864af5fcba7STom Herbert * for the simple case like when setting the checksum for a UDP tunnel. 865af5fcba7STom Herbert */ 866af5fcba7STom Herbert void udp_set_csum(bool nocheck, struct sk_buff *skb, 867af5fcba7STom Herbert __be32 saddr, __be32 daddr, int len) 868af5fcba7STom Herbert { 869af5fcba7STom Herbert struct udphdr *uh = udp_hdr(skb); 870af5fcba7STom Herbert 871179bc67fSEdward Cree if (nocheck) { 872af5fcba7STom Herbert uh->check = 0; 873179bc67fSEdward Cree } else if (skb_is_gso(skb)) { 874af5fcba7STom Herbert uh->check = ~udp_v4_check(len, saddr, daddr, 0); 875179bc67fSEdward Cree } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 876179bc67fSEdward Cree uh->check = 0; 877179bc67fSEdward Cree uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb)); 878179bc67fSEdward Cree if (uh->check == 0) 879179bc67fSEdward Cree uh->check = CSUM_MANGLED_0; 880d75f1306SEdward Cree } else { 881af5fcba7STom Herbert skb->ip_summed = CHECKSUM_PARTIAL; 882af5fcba7STom Herbert skb->csum_start = skb_transport_header(skb) - skb->head; 883af5fcba7STom Herbert skb->csum_offset = offsetof(struct udphdr, check); 884af5fcba7STom Herbert uh->check = ~udp_v4_check(len, saddr, daddr, 0); 885af5fcba7STom Herbert } 886af5fcba7STom Herbert } 887af5fcba7STom Herbert EXPORT_SYMBOL(udp_set_csum); 888af5fcba7STom Herbert 889bec1f6f6SWillem de Bruijn static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4, 890bec1f6f6SWillem de Bruijn struct inet_cork *cork) 891f6b9664fSHerbert Xu { 892f6b9664fSHerbert Xu struct sock *sk = skb->sk; 893f6b9664fSHerbert Xu struct inet_sock *inet = inet_sk(sk); 894f6b9664fSHerbert Xu struct udphdr *uh; 895cffb8f61SMenglong Dong int err; 896f6b9664fSHerbert Xu int is_udplite = IS_UDPLITE(sk); 897f6b9664fSHerbert Xu int offset = skb_transport_offset(skb); 898f6b9664fSHerbert Xu int len = skb->len - offset; 8994094871dSJosh Hunt int datalen = len - sizeof(*uh); 900f6b9664fSHerbert Xu __wsum csum = 0; 901f6b9664fSHerbert Xu 902f6b9664fSHerbert Xu /* 903f6b9664fSHerbert Xu * Create a UDP header 904f6b9664fSHerbert Xu */ 905f6b9664fSHerbert Xu uh = udp_hdr(skb); 906f6b9664fSHerbert Xu uh->source = inet->inet_sport; 90779ab0531SDavid S. Miller uh->dest = fl4->fl4_dport; 908f6b9664fSHerbert Xu uh->len = htons(len); 909f6b9664fSHerbert Xu uh->check = 0; 910f6b9664fSHerbert Xu 911bec1f6f6SWillem de Bruijn if (cork->gso_size) { 912bec1f6f6SWillem de Bruijn const int hlen = skb_network_header_len(skb) + 913bec1f6f6SWillem de Bruijn sizeof(struct udphdr); 914bec1f6f6SWillem de Bruijn 9150f149c9fSWillem de Bruijn if (hlen + cork->gso_size > cork->fragsize) { 9160f149c9fSWillem de Bruijn kfree_skb(skb); 917bec1f6f6SWillem de Bruijn return -EINVAL; 9180f149c9fSWillem de Bruijn } 9190f149c9fSWillem de Bruijn if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) { 9200f149c9fSWillem de Bruijn kfree_skb(skb); 921bec1f6f6SWillem de Bruijn return -EINVAL; 9220f149c9fSWillem de Bruijn } 9230f149c9fSWillem de Bruijn if (sk->sk_no_check_tx) { 9240f149c9fSWillem de Bruijn kfree_skb(skb); 925a8c744a8SWillem de Bruijn return -EINVAL; 9260f149c9fSWillem de Bruijn } 927ff06342cSWillem de Bruijn if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite || 9280f149c9fSWillem de Bruijn dst_xfrm(skb_dst(skb))) { 9290f149c9fSWillem de Bruijn kfree_skb(skb); 930bec1f6f6SWillem de Bruijn return -EIO; 9310f149c9fSWillem de Bruijn } 932bec1f6f6SWillem de Bruijn 9334094871dSJosh Hunt if (datalen > cork->gso_size) { 934bec1f6f6SWillem de Bruijn skb_shinfo(skb)->gso_size = cork->gso_size; 935bec1f6f6SWillem de Bruijn skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; 9364094871dSJosh Hunt skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen, 937dfec0ee2SAlexander Duyck cork->gso_size); 9384094871dSJosh Hunt } 939a8c744a8SWillem de Bruijn goto csum_partial; 940bec1f6f6SWillem de Bruijn } 941bec1f6f6SWillem de Bruijn 942f6b9664fSHerbert Xu if (is_udplite) /* UDP-Lite */ 943f6b9664fSHerbert Xu csum = udplite_csum(skb); 944f6b9664fSHerbert Xu 945ab2fb7e3SWillem de Bruijn else if (sk->sk_no_check_tx) { /* UDP csum off */ 946f6b9664fSHerbert Xu 947f6b9664fSHerbert Xu skb->ip_summed = CHECKSUM_NONE; 948f6b9664fSHerbert Xu goto send; 949f6b9664fSHerbert Xu 950f6b9664fSHerbert Xu } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */ 951a8c744a8SWillem de Bruijn csum_partial: 952f6b9664fSHerbert Xu 95379ab0531SDavid S. Miller udp4_hwcsum(skb, fl4->saddr, fl4->daddr); 954f6b9664fSHerbert Xu goto send; 955f6b9664fSHerbert Xu 956f6b9664fSHerbert Xu } else 957f6b9664fSHerbert Xu csum = udp_csum(skb); 958f6b9664fSHerbert Xu 959f6b9664fSHerbert Xu /* add protocol-dependent pseudo-header */ 96079ab0531SDavid S. Miller uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len, 961f6b9664fSHerbert Xu sk->sk_protocol, csum); 962f6b9664fSHerbert Xu if (uh->check == 0) 963f6b9664fSHerbert Xu uh->check = CSUM_MANGLED_0; 964f6b9664fSHerbert Xu 965f6b9664fSHerbert Xu send: 966b5ec8eeaSEric Dumazet err = ip_send_skb(sock_net(sk), skb); 967f6b9664fSHerbert Xu if (err) { 968f6b9664fSHerbert Xu if (err == -ENOBUFS && !inet->recverr) { 9696aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 970f6b9664fSHerbert Xu UDP_MIB_SNDBUFERRORS, is_udplite); 971f6b9664fSHerbert Xu err = 0; 972f6b9664fSHerbert Xu } 973f6b9664fSHerbert Xu } else 9746aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 975f6b9664fSHerbert Xu UDP_MIB_OUTDATAGRAMS, is_udplite); 976f6b9664fSHerbert Xu return err; 977f6b9664fSHerbert Xu } 978f6b9664fSHerbert Xu 979db8dac20SDavid S. Miller /* 980db8dac20SDavid S. Miller * Push out all pending data as one UDP datagram. Socket is locked. 981db8dac20SDavid S. Miller */ 9828822b64aSHannes Frederic Sowa int udp_push_pending_frames(struct sock *sk) 983db8dac20SDavid S. Miller { 984db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 985db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 986b6f21b26SDavid S. Miller struct flowi4 *fl4 = &inet->cork.fl.u.ip4; 987db8dac20SDavid S. Miller struct sk_buff *skb; 988db8dac20SDavid S. Miller int err = 0; 989db8dac20SDavid S. Miller 99077968b78SDavid S. Miller skb = ip_finish_skb(sk, fl4); 991f6b9664fSHerbert Xu if (!skb) 992db8dac20SDavid S. Miller goto out; 993db8dac20SDavid S. Miller 994bec1f6f6SWillem de Bruijn err = udp_send_skb(skb, fl4, &inet->cork.base); 995db8dac20SDavid S. Miller 996db8dac20SDavid S. Miller out: 997db8dac20SDavid S. Miller up->len = 0; 998db8dac20SDavid S. Miller up->pending = 0; 999db8dac20SDavid S. Miller return err; 1000db8dac20SDavid S. Miller } 10018822b64aSHannes Frederic Sowa EXPORT_SYMBOL(udp_push_pending_frames); 1002db8dac20SDavid S. Miller 10032e8de857SWillem de Bruijn static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size) 10042e8de857SWillem de Bruijn { 10052e8de857SWillem de Bruijn switch (cmsg->cmsg_type) { 10062e8de857SWillem de Bruijn case UDP_SEGMENT: 10072e8de857SWillem de Bruijn if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16))) 10082e8de857SWillem de Bruijn return -EINVAL; 10092e8de857SWillem de Bruijn *gso_size = *(__u16 *)CMSG_DATA(cmsg); 10102e8de857SWillem de Bruijn return 0; 10112e8de857SWillem de Bruijn default: 10122e8de857SWillem de Bruijn return -EINVAL; 10132e8de857SWillem de Bruijn } 10142e8de857SWillem de Bruijn } 10152e8de857SWillem de Bruijn 10162e8de857SWillem de Bruijn int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size) 10172e8de857SWillem de Bruijn { 10182e8de857SWillem de Bruijn struct cmsghdr *cmsg; 10192e8de857SWillem de Bruijn bool need_ip = false; 10202e8de857SWillem de Bruijn int err; 10212e8de857SWillem de Bruijn 10222e8de857SWillem de Bruijn for_each_cmsghdr(cmsg, msg) { 10232e8de857SWillem de Bruijn if (!CMSG_OK(msg, cmsg)) 10242e8de857SWillem de Bruijn return -EINVAL; 10252e8de857SWillem de Bruijn 10262e8de857SWillem de Bruijn if (cmsg->cmsg_level != SOL_UDP) { 10272e8de857SWillem de Bruijn need_ip = true; 10282e8de857SWillem de Bruijn continue; 10292e8de857SWillem de Bruijn } 10302e8de857SWillem de Bruijn 10312e8de857SWillem de Bruijn err = __udp_cmsg_send(cmsg, gso_size); 10322e8de857SWillem de Bruijn if (err) 10332e8de857SWillem de Bruijn return err; 10342e8de857SWillem de Bruijn } 10352e8de857SWillem de Bruijn 10362e8de857SWillem de Bruijn return need_ip; 10372e8de857SWillem de Bruijn } 10382e8de857SWillem de Bruijn EXPORT_SYMBOL_GPL(udp_cmsg_send); 10392e8de857SWillem de Bruijn 10401b784140SYing Xue int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1041db8dac20SDavid S. Miller { 1042db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 1043db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 10441cedee13SAndrey Ignatov DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); 1045e474995fSDavid S. Miller struct flowi4 fl4_stack; 1046b6f21b26SDavid S. Miller struct flowi4 *fl4; 1047db8dac20SDavid S. Miller int ulen = len; 1048db8dac20SDavid S. Miller struct ipcm_cookie ipc; 1049db8dac20SDavid S. Miller struct rtable *rt = NULL; 1050db8dac20SDavid S. Miller int free = 0; 1051db8dac20SDavid S. Miller int connected = 0; 1052db8dac20SDavid S. Miller __be32 daddr, faddr, saddr; 1053db8dac20SDavid S. Miller __be16 dport; 1054db8dac20SDavid S. Miller u8 tos; 1055db8dac20SDavid S. Miller int err, is_udplite = IS_UDPLITE(sk); 1056a9f59707SEric Dumazet int corkreq = READ_ONCE(up->corkflag) || msg->msg_flags&MSG_MORE; 1057db8dac20SDavid S. Miller int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); 1058903ab86dSHerbert Xu struct sk_buff *skb; 1059f6d8bd05SEric Dumazet struct ip_options_data opt_copy; 1060db8dac20SDavid S. Miller 1061db8dac20SDavid S. Miller if (len > 0xFFFF) 1062db8dac20SDavid S. Miller return -EMSGSIZE; 1063db8dac20SDavid S. Miller 1064db8dac20SDavid S. Miller /* 1065db8dac20SDavid S. Miller * Check the flags. 1066db8dac20SDavid S. Miller */ 1067db8dac20SDavid S. Miller 1068db8dac20SDavid S. Miller if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */ 1069db8dac20SDavid S. Miller return -EOPNOTSUPP; 1070db8dac20SDavid S. Miller 1071903ab86dSHerbert Xu getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag; 1072903ab86dSHerbert Xu 1073f5fca608SDavid S. Miller fl4 = &inet->cork.fl.u.ip4; 1074db8dac20SDavid S. Miller if (up->pending) { 1075db8dac20SDavid S. Miller /* 1076db8dac20SDavid S. Miller * There are pending frames. 1077db8dac20SDavid S. Miller * The socket lock must be held while it's corked. 1078db8dac20SDavid S. Miller */ 1079db8dac20SDavid S. Miller lock_sock(sk); 1080db8dac20SDavid S. Miller if (likely(up->pending)) { 1081db8dac20SDavid S. Miller if (unlikely(up->pending != AF_INET)) { 1082db8dac20SDavid S. Miller release_sock(sk); 1083db8dac20SDavid S. Miller return -EINVAL; 1084db8dac20SDavid S. Miller } 1085db8dac20SDavid S. Miller goto do_append_data; 1086db8dac20SDavid S. Miller } 1087db8dac20SDavid S. Miller release_sock(sk); 1088db8dac20SDavid S. Miller } 1089db8dac20SDavid S. Miller ulen += sizeof(struct udphdr); 1090db8dac20SDavid S. Miller 1091db8dac20SDavid S. Miller /* 1092db8dac20SDavid S. Miller * Get and verify the address. 1093db8dac20SDavid S. Miller */ 10941cedee13SAndrey Ignatov if (usin) { 1095db8dac20SDavid S. Miller if (msg->msg_namelen < sizeof(*usin)) 1096db8dac20SDavid S. Miller return -EINVAL; 1097db8dac20SDavid S. Miller if (usin->sin_family != AF_INET) { 1098db8dac20SDavid S. Miller if (usin->sin_family != AF_UNSPEC) 1099db8dac20SDavid S. Miller return -EAFNOSUPPORT; 1100db8dac20SDavid S. Miller } 1101db8dac20SDavid S. Miller 1102db8dac20SDavid S. Miller daddr = usin->sin_addr.s_addr; 1103db8dac20SDavid S. Miller dport = usin->sin_port; 1104db8dac20SDavid S. Miller if (dport == 0) 1105db8dac20SDavid S. Miller return -EINVAL; 1106db8dac20SDavid S. Miller } else { 1107db8dac20SDavid S. Miller if (sk->sk_state != TCP_ESTABLISHED) 1108db8dac20SDavid S. Miller return -EDESTADDRREQ; 1109c720c7e8SEric Dumazet daddr = inet->inet_daddr; 1110c720c7e8SEric Dumazet dport = inet->inet_dport; 1111db8dac20SDavid S. Miller /* Open fast path for connected socket. 1112db8dac20SDavid S. Miller Route will not be used, if at least one option is set. 1113db8dac20SDavid S. Miller */ 1114db8dac20SDavid S. Miller connected = 1; 1115db8dac20SDavid S. Miller } 1116c14ac945SSoheil Hassas Yeganeh 111735178206SWillem de Bruijn ipcm_init_sk(&ipc, inet); 111818a419baSEric Dumazet ipc.gso_size = READ_ONCE(up->gso_size); 1119bf84a010SDaniel Borkmann 1120db8dac20SDavid S. Miller if (msg->msg_controllen) { 11212e8de857SWillem de Bruijn err = udp_cmsg_send(sk, msg, &ipc.gso_size); 11222e8de857SWillem de Bruijn if (err > 0) 11232e8de857SWillem de Bruijn err = ip_cmsg_send(sk, msg, &ipc, 11242e8de857SWillem de Bruijn sk->sk_family == AF_INET6); 11252e8de857SWillem de Bruijn if (unlikely(err < 0)) { 112691948309SEric Dumazet kfree(ipc.opt); 1127db8dac20SDavid S. Miller return err; 112891948309SEric Dumazet } 1129db8dac20SDavid S. Miller if (ipc.opt) 1130db8dac20SDavid S. Miller free = 1; 1131db8dac20SDavid S. Miller connected = 0; 1132db8dac20SDavid S. Miller } 1133f6d8bd05SEric Dumazet if (!ipc.opt) { 1134f6d8bd05SEric Dumazet struct ip_options_rcu *inet_opt; 1135f6d8bd05SEric Dumazet 1136f6d8bd05SEric Dumazet rcu_read_lock(); 1137f6d8bd05SEric Dumazet inet_opt = rcu_dereference(inet->inet_opt); 1138f6d8bd05SEric Dumazet if (inet_opt) { 1139f6d8bd05SEric Dumazet memcpy(&opt_copy, inet_opt, 1140f6d8bd05SEric Dumazet sizeof(*inet_opt) + inet_opt->opt.optlen); 1141f6d8bd05SEric Dumazet ipc.opt = &opt_copy.opt; 1142f6d8bd05SEric Dumazet } 1143f6d8bd05SEric Dumazet rcu_read_unlock(); 1144f6d8bd05SEric Dumazet } 1145db8dac20SDavid S. Miller 11466fc88c35SDave Marchevsky if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) { 11471cedee13SAndrey Ignatov err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, 11481cedee13SAndrey Ignatov (struct sockaddr *)usin, &ipc.addr); 11491cedee13SAndrey Ignatov if (err) 11501cedee13SAndrey Ignatov goto out_free; 11511cedee13SAndrey Ignatov if (usin) { 11521cedee13SAndrey Ignatov if (usin->sin_port == 0) { 11531cedee13SAndrey Ignatov /* BPF program set invalid port. Reject it. */ 11541cedee13SAndrey Ignatov err = -EINVAL; 11551cedee13SAndrey Ignatov goto out_free; 11561cedee13SAndrey Ignatov } 11571cedee13SAndrey Ignatov daddr = usin->sin_addr.s_addr; 11581cedee13SAndrey Ignatov dport = usin->sin_port; 11591cedee13SAndrey Ignatov } 11601cedee13SAndrey Ignatov } 11611cedee13SAndrey Ignatov 1162db8dac20SDavid S. Miller saddr = ipc.addr; 1163db8dac20SDavid S. Miller ipc.addr = faddr = daddr; 1164db8dac20SDavid S. Miller 1165f6d8bd05SEric Dumazet if (ipc.opt && ipc.opt->opt.srr) { 11661b97013bSAndrey Ignatov if (!daddr) { 11671b97013bSAndrey Ignatov err = -EINVAL; 11681b97013bSAndrey Ignatov goto out_free; 11691b97013bSAndrey Ignatov } 1170f6d8bd05SEric Dumazet faddr = ipc.opt->opt.faddr; 1171db8dac20SDavid S. Miller connected = 0; 1172db8dac20SDavid S. Miller } 1173aa661581SFrancesco Fusco tos = get_rttos(&ipc, inet); 1174db8dac20SDavid S. Miller if (sock_flag(sk, SOCK_LOCALROUTE) || 1175db8dac20SDavid S. Miller (msg->msg_flags & MSG_DONTROUTE) || 1176f6d8bd05SEric Dumazet (ipc.opt && ipc.opt->opt.is_strictroute)) { 1177db8dac20SDavid S. Miller tos |= RTO_ONLINK; 1178db8dac20SDavid S. Miller connected = 0; 1179db8dac20SDavid S. Miller } 1180db8dac20SDavid S. Miller 1181db8dac20SDavid S. Miller if (ipv4_is_multicast(daddr)) { 1182854da991SRobert Shearman if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif)) 1183db8dac20SDavid S. Miller ipc.oif = inet->mc_index; 1184db8dac20SDavid S. Miller if (!saddr) 1185db8dac20SDavid S. Miller saddr = inet->mc_addr; 1186db8dac20SDavid S. Miller connected = 0; 11879515a2e0SDavid Ahern } else if (!ipc.oif) { 118876e21053SErich E. Hoover ipc.oif = inet->uc_index; 11899515a2e0SDavid Ahern } else if (ipv4_is_lbcast(daddr) && inet->uc_index) { 11909515a2e0SDavid Ahern /* oif is set, packet is to local broadcast and 11912bdcc73cSRandy Dunlap * uc_index is set. oif is most likely set 11929515a2e0SDavid Ahern * by sk_bound_dev_if. If uc_index != oif check if the 11939515a2e0SDavid Ahern * oif is an L3 master and uc_index is an L3 slave. 11949515a2e0SDavid Ahern * If so, we want to allow the send using the uc_index. 11959515a2e0SDavid Ahern */ 11969515a2e0SDavid Ahern if (ipc.oif != inet->uc_index && 11979515a2e0SDavid Ahern ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk), 11989515a2e0SDavid Ahern inet->uc_index)) { 11999515a2e0SDavid Ahern ipc.oif = inet->uc_index; 12009515a2e0SDavid Ahern } 12019515a2e0SDavid Ahern } 1202db8dac20SDavid S. Miller 1203db8dac20SDavid S. Miller if (connected) 1204db8dac20SDavid S. Miller rt = (struct rtable *)sk_dst_check(sk, 0); 1205db8dac20SDavid S. Miller 120651456b29SIan Morris if (!rt) { 120784a3aa00SPavel Emelyanov struct net *net = sock_net(sk); 12089a24abfaSDavid Ahern __u8 flow_flags = inet_sk_flowi_flags(sk); 120984a3aa00SPavel Emelyanov 1210e474995fSDavid S. Miller fl4 = &fl4_stack; 12119a24abfaSDavid Ahern 1212c6af0c22SWillem de Bruijn flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos, 12139a24abfaSDavid Ahern RT_SCOPE_UNIVERSE, sk->sk_protocol, 12149a24abfaSDavid Ahern flow_flags, 1215e2d118a1SLorenzo Colitti faddr, saddr, dport, inet->inet_sport, 1216e2d118a1SLorenzo Colitti sk->sk_uid); 1217c0951cbcSDavid S. Miller 12183df98d79SPaul Moore security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4)); 1219e474995fSDavid S. Miller rt = ip_route_output_flow(net, fl4, sk); 1220b23dd4feSDavid S. Miller if (IS_ERR(rt)) { 1221b23dd4feSDavid S. Miller err = PTR_ERR(rt); 122206dc94b1SDavid S. Miller rt = NULL; 1223db8dac20SDavid S. Miller if (err == -ENETUNREACH) 1224f1d8cba6SEric Dumazet IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 1225db8dac20SDavid S. Miller goto out; 1226db8dac20SDavid S. Miller } 1227db8dac20SDavid S. Miller 1228db8dac20SDavid S. Miller err = -EACCES; 1229db8dac20SDavid S. Miller if ((rt->rt_flags & RTCF_BROADCAST) && 1230db8dac20SDavid S. Miller !sock_flag(sk, SOCK_BROADCAST)) 1231db8dac20SDavid S. Miller goto out; 1232db8dac20SDavid S. Miller if (connected) 1233d8d1f30bSChangli Gao sk_dst_set(sk, dst_clone(&rt->dst)); 1234db8dac20SDavid S. Miller } 1235db8dac20SDavid S. Miller 1236db8dac20SDavid S. Miller if (msg->msg_flags&MSG_CONFIRM) 1237db8dac20SDavid S. Miller goto do_confirm; 1238db8dac20SDavid S. Miller back_from_confirm: 1239db8dac20SDavid S. Miller 1240e474995fSDavid S. Miller saddr = fl4->saddr; 1241db8dac20SDavid S. Miller if (!ipc.addr) 1242e474995fSDavid S. Miller daddr = ipc.addr = fl4->daddr; 1243db8dac20SDavid S. Miller 1244903ab86dSHerbert Xu /* Lockless fast path for the non-corking case. */ 1245903ab86dSHerbert Xu if (!corkreq) { 12461cd7884dSWillem de Bruijn struct inet_cork cork; 12471cd7884dSWillem de Bruijn 1248f69e6d13SAl Viro skb = ip_make_skb(sk, fl4, getfrag, msg, ulen, 1249903ab86dSHerbert Xu sizeof(struct udphdr), &ipc, &rt, 12501cd7884dSWillem de Bruijn &cork, msg->msg_flags); 1251903ab86dSHerbert Xu err = PTR_ERR(skb); 125250c3a487SYOSHIFUJI Hideaki / 吉藤英明 if (!IS_ERR_OR_NULL(skb)) 1253bec1f6f6SWillem de Bruijn err = udp_send_skb(skb, fl4, &cork); 1254903ab86dSHerbert Xu goto out; 1255903ab86dSHerbert Xu } 1256903ab86dSHerbert Xu 1257db8dac20SDavid S. Miller lock_sock(sk); 1258db8dac20SDavid S. Miller if (unlikely(up->pending)) { 1259db8dac20SDavid S. Miller /* The socket is already corked while preparing it. */ 1260db8dac20SDavid S. Miller /* ... which is an evident application bug. --ANK */ 1261db8dac20SDavid S. Miller release_sock(sk); 1262db8dac20SDavid S. Miller 1263197df02cSMatteo Croce net_dbg_ratelimited("socket already corked\n"); 1264db8dac20SDavid S. Miller err = -EINVAL; 1265db8dac20SDavid S. Miller goto out; 1266db8dac20SDavid S. Miller } 1267db8dac20SDavid S. Miller /* 1268db8dac20SDavid S. Miller * Now cork the socket to pend data. 1269db8dac20SDavid S. Miller */ 1270b6f21b26SDavid S. Miller fl4 = &inet->cork.fl.u.ip4; 1271b6f21b26SDavid S. Miller fl4->daddr = daddr; 1272b6f21b26SDavid S. Miller fl4->saddr = saddr; 12739cce96dfSDavid S. Miller fl4->fl4_dport = dport; 12749cce96dfSDavid S. Miller fl4->fl4_sport = inet->inet_sport; 1275db8dac20SDavid S. Miller up->pending = AF_INET; 1276db8dac20SDavid S. Miller 1277db8dac20SDavid S. Miller do_append_data: 1278db8dac20SDavid S. Miller up->len += ulen; 1279f69e6d13SAl Viro err = ip_append_data(sk, fl4, getfrag, msg, ulen, 12802e77d89bSEric Dumazet sizeof(struct udphdr), &ipc, &rt, 1281db8dac20SDavid S. Miller corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); 1282db8dac20SDavid S. Miller if (err) 1283db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 1284db8dac20SDavid S. Miller else if (!corkreq) 1285db8dac20SDavid S. Miller err = udp_push_pending_frames(sk); 1286db8dac20SDavid S. Miller else if (unlikely(skb_queue_empty(&sk->sk_write_queue))) 1287db8dac20SDavid S. Miller up->pending = 0; 1288db8dac20SDavid S. Miller release_sock(sk); 1289db8dac20SDavid S. Miller 1290db8dac20SDavid S. Miller out: 1291db8dac20SDavid S. Miller ip_rt_put(rt); 12921b97013bSAndrey Ignatov out_free: 1293db8dac20SDavid S. Miller if (free) 1294db8dac20SDavid S. Miller kfree(ipc.opt); 1295db8dac20SDavid S. Miller if (!err) 1296db8dac20SDavid S. Miller return len; 1297db8dac20SDavid S. Miller /* 1298db8dac20SDavid S. Miller * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting 1299db8dac20SDavid S. Miller * ENOBUFS might not be good (it's not tunable per se), but otherwise 1300db8dac20SDavid S. Miller * we don't have a good statistic (IpOutDiscards but it can be too many 1301db8dac20SDavid S. Miller * things). We could add another new stat but at least for now that 1302db8dac20SDavid S. Miller * seems like overkill. 1303db8dac20SDavid S. Miller */ 1304db8dac20SDavid S. Miller if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { 13056aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1306629ca23cSPavel Emelyanov UDP_MIB_SNDBUFERRORS, is_udplite); 1307db8dac20SDavid S. Miller } 1308db8dac20SDavid S. Miller return err; 1309db8dac20SDavid S. Miller 1310db8dac20SDavid S. Miller do_confirm: 13110dec879fSJulian Anastasov if (msg->msg_flags & MSG_PROBE) 13120dec879fSJulian Anastasov dst_confirm_neigh(&rt->dst, &fl4->daddr); 1313db8dac20SDavid S. Miller if (!(msg->msg_flags&MSG_PROBE) || len) 1314db8dac20SDavid S. Miller goto back_from_confirm; 1315db8dac20SDavid S. Miller err = 0; 1316db8dac20SDavid S. Miller goto out; 1317db8dac20SDavid S. Miller } 1318c482c568SEric Dumazet EXPORT_SYMBOL(udp_sendmsg); 1319db8dac20SDavid S. Miller 1320db8dac20SDavid S. Miller int udp_sendpage(struct sock *sk, struct page *page, int offset, 1321db8dac20SDavid S. Miller size_t size, int flags) 1322db8dac20SDavid S. Miller { 1323f5fca608SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 1324db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 1325db8dac20SDavid S. Miller int ret; 1326db8dac20SDavid S. Miller 1327d3f7d56aSShawn Landden if (flags & MSG_SENDPAGE_NOTLAST) 1328d3f7d56aSShawn Landden flags |= MSG_MORE; 1329d3f7d56aSShawn Landden 1330db8dac20SDavid S. Miller if (!up->pending) { 1331db8dac20SDavid S. Miller struct msghdr msg = { .msg_flags = flags|MSG_MORE }; 1332db8dac20SDavid S. Miller 1333db8dac20SDavid S. Miller /* Call udp_sendmsg to specify destination address which 1334db8dac20SDavid S. Miller * sendpage interface can't pass. 1335db8dac20SDavid S. Miller * This will succeed only when the socket is connected. 1336db8dac20SDavid S. Miller */ 13371b784140SYing Xue ret = udp_sendmsg(sk, &msg, 0); 1338db8dac20SDavid S. Miller if (ret < 0) 1339db8dac20SDavid S. Miller return ret; 1340db8dac20SDavid S. Miller } 1341db8dac20SDavid S. Miller 1342db8dac20SDavid S. Miller lock_sock(sk); 1343db8dac20SDavid S. Miller 1344db8dac20SDavid S. Miller if (unlikely(!up->pending)) { 1345db8dac20SDavid S. Miller release_sock(sk); 1346db8dac20SDavid S. Miller 1347197df02cSMatteo Croce net_dbg_ratelimited("cork failed\n"); 1348db8dac20SDavid S. Miller return -EINVAL; 1349db8dac20SDavid S. Miller } 1350db8dac20SDavid S. Miller 1351f5fca608SDavid S. Miller ret = ip_append_page(sk, &inet->cork.fl.u.ip4, 1352f5fca608SDavid S. Miller page, offset, size, flags); 1353db8dac20SDavid S. Miller if (ret == -EOPNOTSUPP) { 1354db8dac20SDavid S. Miller release_sock(sk); 1355db8dac20SDavid S. Miller return sock_no_sendpage(sk->sk_socket, page, offset, 1356db8dac20SDavid S. Miller size, flags); 1357db8dac20SDavid S. Miller } 1358db8dac20SDavid S. Miller if (ret < 0) { 1359db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 1360db8dac20SDavid S. Miller goto out; 1361db8dac20SDavid S. Miller } 1362db8dac20SDavid S. Miller 1363db8dac20SDavid S. Miller up->len += size; 1364a9f59707SEric Dumazet if (!(READ_ONCE(up->corkflag) || (flags&MSG_MORE))) 1365db8dac20SDavid S. Miller ret = udp_push_pending_frames(sk); 1366db8dac20SDavid S. Miller if (!ret) 1367db8dac20SDavid S. Miller ret = size; 1368db8dac20SDavid S. Miller out: 1369db8dac20SDavid S. Miller release_sock(sk); 1370db8dac20SDavid S. Miller return ret; 1371db8dac20SDavid S. Miller } 1372db8dac20SDavid S. Miller 1373dce4551cSPaolo Abeni #define UDP_SKB_IS_STATELESS 0x80000000 1374dce4551cSPaolo Abeni 1375677bf08cSFlorian Westphal /* all head states (dst, sk, nf conntrack) except skb extensions are 1376677bf08cSFlorian Westphal * cleared by udp_rcv(). 1377677bf08cSFlorian Westphal * 1378677bf08cSFlorian Westphal * We need to preserve secpath, if present, to eventually process 1379677bf08cSFlorian Westphal * IP_CMSG_PASSSEC at recvmsg() time. 1380677bf08cSFlorian Westphal * 1381677bf08cSFlorian Westphal * Other extensions can be cleared. 1382677bf08cSFlorian Westphal */ 1383677bf08cSFlorian Westphal static bool udp_try_make_stateless(struct sk_buff *skb) 1384677bf08cSFlorian Westphal { 1385677bf08cSFlorian Westphal if (!skb_has_extensions(skb)) 1386677bf08cSFlorian Westphal return true; 1387677bf08cSFlorian Westphal 1388677bf08cSFlorian Westphal if (!secpath_exists(skb)) { 1389677bf08cSFlorian Westphal skb_ext_reset(skb); 1390677bf08cSFlorian Westphal return true; 1391677bf08cSFlorian Westphal } 1392677bf08cSFlorian Westphal 1393677bf08cSFlorian Westphal return false; 1394677bf08cSFlorian Westphal } 1395677bf08cSFlorian Westphal 1396b65ac446SPaolo Abeni static void udp_set_dev_scratch(struct sk_buff *skb) 1397b65ac446SPaolo Abeni { 1398dce4551cSPaolo Abeni struct udp_dev_scratch *scratch = udp_skb_scratch(skb); 1399b65ac446SPaolo Abeni 1400b65ac446SPaolo Abeni BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long)); 1401dce4551cSPaolo Abeni scratch->_tsize_state = skb->truesize; 1402dce4551cSPaolo Abeni #if BITS_PER_LONG == 64 1403b65ac446SPaolo Abeni scratch->len = skb->len; 1404b65ac446SPaolo Abeni scratch->csum_unnecessary = !!skb_csum_unnecessary(skb); 1405b65ac446SPaolo Abeni scratch->is_linear = !skb_is_nonlinear(skb); 1406b65ac446SPaolo Abeni #endif 1407677bf08cSFlorian Westphal if (udp_try_make_stateless(skb)) 1408dce4551cSPaolo Abeni scratch->_tsize_state |= UDP_SKB_IS_STATELESS; 1409dce4551cSPaolo Abeni } 1410dce4551cSPaolo Abeni 1411a793183cSEric Dumazet static void udp_skb_csum_unnecessary_set(struct sk_buff *skb) 1412a793183cSEric Dumazet { 1413a793183cSEric Dumazet /* We come here after udp_lib_checksum_complete() returned 0. 1414a793183cSEric Dumazet * This means that __skb_checksum_complete() might have 1415a793183cSEric Dumazet * set skb->csum_valid to 1. 1416a793183cSEric Dumazet * On 64bit platforms, we can set csum_unnecessary 1417a793183cSEric Dumazet * to true, but only if the skb is not shared. 1418a793183cSEric Dumazet */ 1419a793183cSEric Dumazet #if BITS_PER_LONG == 64 1420a793183cSEric Dumazet if (!skb_shared(skb)) 1421a793183cSEric Dumazet udp_skb_scratch(skb)->csum_unnecessary = true; 1422a793183cSEric Dumazet #endif 1423a793183cSEric Dumazet } 1424a793183cSEric Dumazet 1425dce4551cSPaolo Abeni static int udp_skb_truesize(struct sk_buff *skb) 1426dce4551cSPaolo Abeni { 1427dce4551cSPaolo Abeni return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS; 1428dce4551cSPaolo Abeni } 1429dce4551cSPaolo Abeni 1430dce4551cSPaolo Abeni static bool udp_skb_has_head_state(struct sk_buff *skb) 1431dce4551cSPaolo Abeni { 1432dce4551cSPaolo Abeni return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS); 1433dce4551cSPaolo Abeni } 1434b65ac446SPaolo Abeni 14357c13f97fSPaolo Abeni /* fully reclaim rmem/fwd memory allocated for skb */ 14366dfb4367SPaolo Abeni static void udp_rmem_release(struct sock *sk, int size, int partial, 14376dfb4367SPaolo Abeni bool rx_queue_lock_held) 1438f970bd9eSPaolo Abeni { 14396b229cf7SEric Dumazet struct udp_sock *up = udp_sk(sk); 14402276f58aSPaolo Abeni struct sk_buff_head *sk_queue; 1441f970bd9eSPaolo Abeni int amt; 1442f970bd9eSPaolo Abeni 14436b229cf7SEric Dumazet if (likely(partial)) { 14446b229cf7SEric Dumazet up->forward_deficit += size; 14456b229cf7SEric Dumazet size = up->forward_deficit; 1446d39ca259SPaolo Abeni if (size < (sk->sk_rcvbuf >> 2) && 1447d39ca259SPaolo Abeni !skb_queue_empty(&up->reader_queue)) 14486b229cf7SEric Dumazet return; 14496b229cf7SEric Dumazet } else { 14506b229cf7SEric Dumazet size += up->forward_deficit; 14516b229cf7SEric Dumazet } 14526b229cf7SEric Dumazet up->forward_deficit = 0; 14536b229cf7SEric Dumazet 14546dfb4367SPaolo Abeni /* acquire the sk_receive_queue for fwd allocated memory scheduling, 14556dfb4367SPaolo Abeni * if the called don't held it already 14566dfb4367SPaolo Abeni */ 14572276f58aSPaolo Abeni sk_queue = &sk->sk_receive_queue; 14586dfb4367SPaolo Abeni if (!rx_queue_lock_held) 14592276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 14602276f58aSPaolo Abeni 14616dfb4367SPaolo Abeni 1462f970bd9eSPaolo Abeni sk->sk_forward_alloc += size; 1463f970bd9eSPaolo Abeni amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1); 1464f970bd9eSPaolo Abeni sk->sk_forward_alloc -= amt; 1465f970bd9eSPaolo Abeni 1466f970bd9eSPaolo Abeni if (amt) 1467f970bd9eSPaolo Abeni __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT); 146802ab0d13SEric Dumazet 146902ab0d13SEric Dumazet atomic_sub(size, &sk->sk_rmem_alloc); 14702276f58aSPaolo Abeni 14712276f58aSPaolo Abeni /* this can save us from acquiring the rx queue lock on next receive */ 14722276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, &up->reader_queue); 14732276f58aSPaolo Abeni 14746dfb4367SPaolo Abeni if (!rx_queue_lock_held) 14752276f58aSPaolo Abeni spin_unlock(&sk_queue->lock); 1476f970bd9eSPaolo Abeni } 1477f970bd9eSPaolo Abeni 14782276f58aSPaolo Abeni /* Note: called with reader_queue.lock held. 1479c84d9490SEric Dumazet * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch 1480c84d9490SEric Dumazet * This avoids a cache line miss while receive_queue lock is held. 1481c84d9490SEric Dumazet * Look at __udp_enqueue_schedule_skb() to find where this copy is done. 1482c84d9490SEric Dumazet */ 14837c13f97fSPaolo Abeni void udp_skb_destructor(struct sock *sk, struct sk_buff *skb) 1484f970bd9eSPaolo Abeni { 1485b65ac446SPaolo Abeni prefetch(&skb->data); 1486b65ac446SPaolo Abeni udp_rmem_release(sk, udp_skb_truesize(skb), 1, false); 1487f970bd9eSPaolo Abeni } 14887c13f97fSPaolo Abeni EXPORT_SYMBOL(udp_skb_destructor); 1489f970bd9eSPaolo Abeni 14906dfb4367SPaolo Abeni /* as above, but the caller held the rx queue lock, too */ 149164f5102dSColin Ian King static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb) 14926dfb4367SPaolo Abeni { 1493b65ac446SPaolo Abeni prefetch(&skb->data); 1494b65ac446SPaolo Abeni udp_rmem_release(sk, udp_skb_truesize(skb), 1, true); 14956dfb4367SPaolo Abeni } 14966dfb4367SPaolo Abeni 14974b272750SEric Dumazet /* Idea of busylocks is to let producers grab an extra spinlock 14984b272750SEric Dumazet * to relieve pressure on the receive_queue spinlock shared by consumer. 14994b272750SEric Dumazet * Under flood, this means that only one producer can be in line 15004b272750SEric Dumazet * trying to acquire the receive_queue spinlock. 15014b272750SEric Dumazet * These busylock can be allocated on a per cpu manner, instead of a 15024b272750SEric Dumazet * per socket one (that would consume a cache line per socket) 15034b272750SEric Dumazet */ 15044b272750SEric Dumazet static int udp_busylocks_log __read_mostly; 15054b272750SEric Dumazet static spinlock_t *udp_busylocks __read_mostly; 15064b272750SEric Dumazet 15074b272750SEric Dumazet static spinlock_t *busylock_acquire(void *ptr) 15084b272750SEric Dumazet { 15094b272750SEric Dumazet spinlock_t *busy; 15104b272750SEric Dumazet 15114b272750SEric Dumazet busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log); 15124b272750SEric Dumazet spin_lock(busy); 15134b272750SEric Dumazet return busy; 15144b272750SEric Dumazet } 15154b272750SEric Dumazet 15164b272750SEric Dumazet static void busylock_release(spinlock_t *busy) 15174b272750SEric Dumazet { 15184b272750SEric Dumazet if (busy) 15194b272750SEric Dumazet spin_unlock(busy); 15204b272750SEric Dumazet } 15214b272750SEric Dumazet 1522f970bd9eSPaolo Abeni int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) 1523f970bd9eSPaolo Abeni { 1524f970bd9eSPaolo Abeni struct sk_buff_head *list = &sk->sk_receive_queue; 1525f970bd9eSPaolo Abeni int rmem, delta, amt, err = -ENOMEM; 15264b272750SEric Dumazet spinlock_t *busy = NULL; 1527c8c8b127SEric Dumazet int size; 1528f970bd9eSPaolo Abeni 1529f970bd9eSPaolo Abeni /* try to avoid the costly atomic add/sub pair when the receive 1530f970bd9eSPaolo Abeni * queue is full; always allow at least a packet 1531f970bd9eSPaolo Abeni */ 1532f970bd9eSPaolo Abeni rmem = atomic_read(&sk->sk_rmem_alloc); 1533363dc73aSPaolo Abeni if (rmem > sk->sk_rcvbuf) 1534f970bd9eSPaolo Abeni goto drop; 1535f970bd9eSPaolo Abeni 1536c8c8b127SEric Dumazet /* Under mem pressure, it might be helpful to help udp_recvmsg() 1537c8c8b127SEric Dumazet * having linear skbs : 1538c8c8b127SEric Dumazet * - Reduce memory overhead and thus increase receive queue capacity 1539c8c8b127SEric Dumazet * - Less cache line misses at copyout() time 1540c8c8b127SEric Dumazet * - Less work at consume_skb() (less alien page frag freeing) 1541c8c8b127SEric Dumazet */ 15424b272750SEric Dumazet if (rmem > (sk->sk_rcvbuf >> 1)) { 1543c8c8b127SEric Dumazet skb_condense(skb); 15444b272750SEric Dumazet 15454b272750SEric Dumazet busy = busylock_acquire(sk); 15464b272750SEric Dumazet } 1547c8c8b127SEric Dumazet size = skb->truesize; 1548b65ac446SPaolo Abeni udp_set_dev_scratch(skb); 1549c8c8b127SEric Dumazet 1550f970bd9eSPaolo Abeni /* we drop only if the receive buf is full and the receive 1551f970bd9eSPaolo Abeni * queue contains some other skb 1552f970bd9eSPaolo Abeni */ 1553f970bd9eSPaolo Abeni rmem = atomic_add_return(size, &sk->sk_rmem_alloc); 1554feed8a4fSAntonio Messina if (rmem > (size + (unsigned int)sk->sk_rcvbuf)) 1555f970bd9eSPaolo Abeni goto uncharge_drop; 1556f970bd9eSPaolo Abeni 1557f970bd9eSPaolo Abeni spin_lock(&list->lock); 1558f970bd9eSPaolo Abeni if (size >= sk->sk_forward_alloc) { 1559f970bd9eSPaolo Abeni amt = sk_mem_pages(size); 1560f970bd9eSPaolo Abeni delta = amt << SK_MEM_QUANTUM_SHIFT; 1561f970bd9eSPaolo Abeni if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) { 1562f970bd9eSPaolo Abeni err = -ENOBUFS; 1563f970bd9eSPaolo Abeni spin_unlock(&list->lock); 1564f970bd9eSPaolo Abeni goto uncharge_drop; 1565f970bd9eSPaolo Abeni } 1566f970bd9eSPaolo Abeni 1567f970bd9eSPaolo Abeni sk->sk_forward_alloc += delta; 1568f970bd9eSPaolo Abeni } 1569f970bd9eSPaolo Abeni 1570f970bd9eSPaolo Abeni sk->sk_forward_alloc -= size; 1571f970bd9eSPaolo Abeni 15727c13f97fSPaolo Abeni /* no need to setup a destructor, we will explicitly release the 15737c13f97fSPaolo Abeni * forward allocated memory on dequeue 15747c13f97fSPaolo Abeni */ 1575f970bd9eSPaolo Abeni sock_skb_set_dropcount(sk, skb); 1576f970bd9eSPaolo Abeni 1577f970bd9eSPaolo Abeni __skb_queue_tail(list, skb); 1578f970bd9eSPaolo Abeni spin_unlock(&list->lock); 1579f970bd9eSPaolo Abeni 1580f970bd9eSPaolo Abeni if (!sock_flag(sk, SOCK_DEAD)) 1581f970bd9eSPaolo Abeni sk->sk_data_ready(sk); 1582f970bd9eSPaolo Abeni 15834b272750SEric Dumazet busylock_release(busy); 1584f970bd9eSPaolo Abeni return 0; 1585f970bd9eSPaolo Abeni 1586f970bd9eSPaolo Abeni uncharge_drop: 1587f970bd9eSPaolo Abeni atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 1588f970bd9eSPaolo Abeni 1589f970bd9eSPaolo Abeni drop: 1590f970bd9eSPaolo Abeni atomic_inc(&sk->sk_drops); 15914b272750SEric Dumazet busylock_release(busy); 1592f970bd9eSPaolo Abeni return err; 1593f970bd9eSPaolo Abeni } 1594f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb); 1595f970bd9eSPaolo Abeni 1596c915fe13SPaolo Abeni void udp_destruct_sock(struct sock *sk) 1597f970bd9eSPaolo Abeni { 1598f970bd9eSPaolo Abeni /* reclaim completely the forward allocated memory */ 15992276f58aSPaolo Abeni struct udp_sock *up = udp_sk(sk); 16007c13f97fSPaolo Abeni unsigned int total = 0; 16017c13f97fSPaolo Abeni struct sk_buff *skb; 16027c13f97fSPaolo Abeni 16032276f58aSPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue); 16042276f58aSPaolo Abeni while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) { 16057c13f97fSPaolo Abeni total += skb->truesize; 16067c13f97fSPaolo Abeni kfree_skb(skb); 16077c13f97fSPaolo Abeni } 16086dfb4367SPaolo Abeni udp_rmem_release(sk, total, 0, true); 16097c13f97fSPaolo Abeni 1610f970bd9eSPaolo Abeni inet_sock_destruct(sk); 1611f970bd9eSPaolo Abeni } 1612c915fe13SPaolo Abeni EXPORT_SYMBOL_GPL(udp_destruct_sock); 1613f970bd9eSPaolo Abeni 1614f970bd9eSPaolo Abeni int udp_init_sock(struct sock *sk) 1615f970bd9eSPaolo Abeni { 16162276f58aSPaolo Abeni skb_queue_head_init(&udp_sk(sk)->reader_queue); 1617f970bd9eSPaolo Abeni sk->sk_destruct = udp_destruct_sock; 1618f970bd9eSPaolo Abeni return 0; 1619f970bd9eSPaolo Abeni } 1620f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(udp_init_sock); 1621f970bd9eSPaolo Abeni 1622f970bd9eSPaolo Abeni void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) 1623f970bd9eSPaolo Abeni { 1624f970bd9eSPaolo Abeni if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { 1625f970bd9eSPaolo Abeni bool slow = lock_sock_fast(sk); 1626f970bd9eSPaolo Abeni 1627f970bd9eSPaolo Abeni sk_peek_offset_bwd(sk, len); 1628f970bd9eSPaolo Abeni unlock_sock_fast(sk, slow); 1629f970bd9eSPaolo Abeni } 16300a463c78SPaolo Abeni 1631ca2c1418SPaolo Abeni if (!skb_unref(skb)) 1632ca2c1418SPaolo Abeni return; 1633ca2c1418SPaolo Abeni 1634dce4551cSPaolo Abeni /* In the more common cases we cleared the head states previously, 1635dce4551cSPaolo Abeni * see __udp_queue_rcv_skb(). 16360ddf3fb2SPaolo Abeni */ 1637dce4551cSPaolo Abeni if (unlikely(udp_skb_has_head_state(skb))) 16380ddf3fb2SPaolo Abeni skb_release_head_state(skb); 1639ca2c1418SPaolo Abeni __consume_stateless_skb(skb); 1640f970bd9eSPaolo Abeni } 1641f970bd9eSPaolo Abeni EXPORT_SYMBOL_GPL(skb_consume_udp); 1642f970bd9eSPaolo Abeni 16432276f58aSPaolo Abeni static struct sk_buff *__first_packet_length(struct sock *sk, 16442276f58aSPaolo Abeni struct sk_buff_head *rcvq, 16452276f58aSPaolo Abeni int *total) 16462276f58aSPaolo Abeni { 16472276f58aSPaolo Abeni struct sk_buff *skb; 16482276f58aSPaolo Abeni 16499bd780f5SPaolo Abeni while ((skb = skb_peek(rcvq)) != NULL) { 16509bd780f5SPaolo Abeni if (udp_lib_checksum_complete(skb)) { 16512276f58aSPaolo Abeni __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, 16522276f58aSPaolo Abeni IS_UDPLITE(sk)); 16532276f58aSPaolo Abeni __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, 16542276f58aSPaolo Abeni IS_UDPLITE(sk)); 16552276f58aSPaolo Abeni atomic_inc(&sk->sk_drops); 16562276f58aSPaolo Abeni __skb_unlink(skb, rcvq); 16572276f58aSPaolo Abeni *total += skb->truesize; 16582276f58aSPaolo Abeni kfree_skb(skb); 16599bd780f5SPaolo Abeni } else { 1660a793183cSEric Dumazet udp_skb_csum_unnecessary_set(skb); 16619bd780f5SPaolo Abeni break; 16629bd780f5SPaolo Abeni } 16632276f58aSPaolo Abeni } 16642276f58aSPaolo Abeni return skb; 16652276f58aSPaolo Abeni } 16662276f58aSPaolo Abeni 166785584672SEric Dumazet /** 166885584672SEric Dumazet * first_packet_length - return length of first packet in receive queue 166985584672SEric Dumazet * @sk: socket 167085584672SEric Dumazet * 167185584672SEric Dumazet * Drops all bad checksum frames, until a valid one is found. 1672e83c6744SEric Dumazet * Returns the length of found skb, or -1 if none is found. 167385584672SEric Dumazet */ 1674e83c6744SEric Dumazet static int first_packet_length(struct sock *sk) 167585584672SEric Dumazet { 16762276f58aSPaolo Abeni struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue; 16772276f58aSPaolo Abeni struct sk_buff_head *sk_queue = &sk->sk_receive_queue; 167885584672SEric Dumazet struct sk_buff *skb; 16797c13f97fSPaolo Abeni int total = 0; 1680e83c6744SEric Dumazet int res; 168185584672SEric Dumazet 168285584672SEric Dumazet spin_lock_bh(&rcvq->lock); 16832276f58aSPaolo Abeni skb = __first_packet_length(sk, rcvq, &total); 1684137a0dbeSEric Dumazet if (!skb && !skb_queue_empty_lockless(sk_queue)) { 16852276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 16862276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, rcvq); 16872276f58aSPaolo Abeni spin_unlock(&sk_queue->lock); 16882276f58aSPaolo Abeni 16892276f58aSPaolo Abeni skb = __first_packet_length(sk, rcvq, &total); 169085584672SEric Dumazet } 1691e83c6744SEric Dumazet res = skb ? skb->len : -1; 16927c13f97fSPaolo Abeni if (total) 16936dfb4367SPaolo Abeni udp_rmem_release(sk, total, 1, false); 169485584672SEric Dumazet spin_unlock_bh(&rcvq->lock); 169585584672SEric Dumazet return res; 169685584672SEric Dumazet } 169785584672SEric Dumazet 16981da177e4SLinus Torvalds /* 16991da177e4SLinus Torvalds * IOCTL requests applicable to the UDP protocol 17001da177e4SLinus Torvalds */ 17011da177e4SLinus Torvalds 17021da177e4SLinus Torvalds int udp_ioctl(struct sock *sk, int cmd, unsigned long arg) 17031da177e4SLinus Torvalds { 17046516c655SStephen Hemminger switch (cmd) { 17051da177e4SLinus Torvalds case SIOCOUTQ: 17061da177e4SLinus Torvalds { 170731e6d363SEric Dumazet int amount = sk_wmem_alloc_get(sk); 170831e6d363SEric Dumazet 17091da177e4SLinus Torvalds return put_user(amount, (int __user *)arg); 17101da177e4SLinus Torvalds } 17111da177e4SLinus Torvalds 17121da177e4SLinus Torvalds case SIOCINQ: 17131da177e4SLinus Torvalds { 1714e83c6744SEric Dumazet int amount = max_t(int, 0, first_packet_length(sk)); 17151da177e4SLinus Torvalds 17161da177e4SLinus Torvalds return put_user(amount, (int __user *)arg); 17171da177e4SLinus Torvalds } 17181da177e4SLinus Torvalds 17191da177e4SLinus Torvalds default: 17201da177e4SLinus Torvalds return -ENOIOCTLCMD; 17211da177e4SLinus Torvalds } 17226516c655SStephen Hemminger 17236516c655SStephen Hemminger return 0; 17241da177e4SLinus Torvalds } 1725c482c568SEric Dumazet EXPORT_SYMBOL(udp_ioctl); 17261da177e4SLinus Torvalds 17272276f58aSPaolo Abeni struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags, 1728fd69c399SPaolo Abeni int noblock, int *off, int *err) 17292276f58aSPaolo Abeni { 17302276f58aSPaolo Abeni struct sk_buff_head *sk_queue = &sk->sk_receive_queue; 17312276f58aSPaolo Abeni struct sk_buff_head *queue; 17322276f58aSPaolo Abeni struct sk_buff *last; 17332276f58aSPaolo Abeni long timeo; 17342276f58aSPaolo Abeni int error; 17352276f58aSPaolo Abeni 17362276f58aSPaolo Abeni queue = &udp_sk(sk)->reader_queue; 17372276f58aSPaolo Abeni flags |= noblock ? MSG_DONTWAIT : 0; 17382276f58aSPaolo Abeni timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 17392276f58aSPaolo Abeni do { 17402276f58aSPaolo Abeni struct sk_buff *skb; 17412276f58aSPaolo Abeni 17422276f58aSPaolo Abeni error = sock_error(sk); 17432276f58aSPaolo Abeni if (error) 17442276f58aSPaolo Abeni break; 17452276f58aSPaolo Abeni 17462276f58aSPaolo Abeni error = -EAGAIN; 17472276f58aSPaolo Abeni do { 17482276f58aSPaolo Abeni spin_lock_bh(&queue->lock); 1749e427cad6SPaolo Abeni skb = __skb_try_recv_from_queue(sk, queue, flags, off, 1750e427cad6SPaolo Abeni err, &last); 17512276f58aSPaolo Abeni if (skb) { 1752e427cad6SPaolo Abeni if (!(flags & MSG_PEEK)) 1753e427cad6SPaolo Abeni udp_skb_destructor(sk, skb); 17542276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 17552276f58aSPaolo Abeni return skb; 17562276f58aSPaolo Abeni } 17572276f58aSPaolo Abeni 1758137a0dbeSEric Dumazet if (skb_queue_empty_lockless(sk_queue)) { 17592276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 17602276f58aSPaolo Abeni goto busy_check; 17612276f58aSPaolo Abeni } 17622276f58aSPaolo Abeni 17636dfb4367SPaolo Abeni /* refill the reader queue and walk it again 17646dfb4367SPaolo Abeni * keep both queues locked to avoid re-acquiring 17656dfb4367SPaolo Abeni * the sk_receive_queue lock if fwd memory scheduling 17666dfb4367SPaolo Abeni * is needed. 17676dfb4367SPaolo Abeni */ 17682276f58aSPaolo Abeni spin_lock(&sk_queue->lock); 17692276f58aSPaolo Abeni skb_queue_splice_tail_init(sk_queue, queue); 17702276f58aSPaolo Abeni 1771e427cad6SPaolo Abeni skb = __skb_try_recv_from_queue(sk, queue, flags, off, 1772e427cad6SPaolo Abeni err, &last); 1773e427cad6SPaolo Abeni if (skb && !(flags & MSG_PEEK)) 1774e427cad6SPaolo Abeni udp_skb_dtor_locked(sk, skb); 17756dfb4367SPaolo Abeni spin_unlock(&sk_queue->lock); 17762276f58aSPaolo Abeni spin_unlock_bh(&queue->lock); 1777de321ed3SAndrey Vagin if (skb) 17782276f58aSPaolo Abeni return skb; 17792276f58aSPaolo Abeni 17802276f58aSPaolo Abeni busy_check: 17812276f58aSPaolo Abeni if (!sk_can_busy_loop(sk)) 17822276f58aSPaolo Abeni break; 17832276f58aSPaolo Abeni 17842276f58aSPaolo Abeni sk_busy_loop(sk, flags & MSG_DONTWAIT); 1785137a0dbeSEric Dumazet } while (!skb_queue_empty_lockless(sk_queue)); 17862276f58aSPaolo Abeni 17872276f58aSPaolo Abeni /* sk_queue is empty, reader_queue may contain peeked packets */ 17882276f58aSPaolo Abeni } while (timeo && 1789b50b0580SSabrina Dubroca !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue, 1790b50b0580SSabrina Dubroca &error, &timeo, 17912276f58aSPaolo Abeni (struct sk_buff *)sk_queue)); 17922276f58aSPaolo Abeni 17932276f58aSPaolo Abeni *err = error; 17942276f58aSPaolo Abeni return NULL; 17952276f58aSPaolo Abeni } 17967e823644SJiri Kosina EXPORT_SYMBOL(__skb_recv_udp); 17972276f58aSPaolo Abeni 1798d7f57118SCong Wang int udp_read_sock(struct sock *sk, read_descriptor_t *desc, 1799d7f57118SCong Wang sk_read_actor_t recv_actor) 1800d7f57118SCong Wang { 1801d7f57118SCong Wang int copied = 0; 1802d7f57118SCong Wang 1803d7f57118SCong Wang while (1) { 1804d7f57118SCong Wang struct sk_buff *skb; 1805d7f57118SCong Wang int err, used; 1806d7f57118SCong Wang 1807d7f57118SCong Wang skb = skb_recv_udp(sk, 0, 1, &err); 1808d7f57118SCong Wang if (!skb) 1809d7f57118SCong Wang return err; 1810d7f57118SCong Wang used = recv_actor(desc, skb, 0, skb->len); 1811d7f57118SCong Wang if (used <= 0) { 1812d7f57118SCong Wang if (!copied) 1813d7f57118SCong Wang copied = used; 1814e00a5c33SCong Wang kfree_skb(skb); 1815d7f57118SCong Wang break; 1816d7f57118SCong Wang } else if (used <= skb->len) { 1817d7f57118SCong Wang copied += used; 1818d7f57118SCong Wang } 1819d7f57118SCong Wang 1820e00a5c33SCong Wang kfree_skb(skb); 1821d7f57118SCong Wang if (!desc->count) 1822d7f57118SCong Wang break; 1823d7f57118SCong Wang } 1824d7f57118SCong Wang 1825d7f57118SCong Wang return copied; 1826d7f57118SCong Wang } 1827d7f57118SCong Wang EXPORT_SYMBOL(udp_read_sock); 1828d7f57118SCong Wang 1829db8dac20SDavid S. Miller /* 1830db8dac20SDavid S. Miller * This should be easy, if there is something there we 1831db8dac20SDavid S. Miller * return it, otherwise we block. 1832db8dac20SDavid S. Miller */ 1833db8dac20SDavid S. Miller 18341b784140SYing Xue int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, 18351b784140SYing Xue int flags, int *addr_len) 1836db8dac20SDavid S. Miller { 1837db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sk); 1838342dfc30SSteffen Hurrle DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); 1839db8dac20SDavid S. Miller struct sk_buff *skb; 184059c2cdaeSDavid S. Miller unsigned int ulen, copied; 1841fd69c399SPaolo Abeni int off, err, peeking = flags & MSG_PEEK; 1842db8dac20SDavid S. Miller int is_udplite = IS_UDPLITE(sk); 1843197c949eSEric Dumazet bool checksum_valid = false; 1844db8dac20SDavid S. Miller 1845db8dac20SDavid S. Miller if (flags & MSG_ERRQUEUE) 184685fbaa75SHannes Frederic Sowa return ip_recv_error(sk, msg, len, addr_len); 1847db8dac20SDavid S. Miller 1848db8dac20SDavid S. Miller try_again: 1849a0917e0bSMatthew Dawson off = sk_peek_offset(sk, flags); 1850fd69c399SPaolo Abeni skb = __skb_recv_udp(sk, flags, noblock, &off, &err); 1851db8dac20SDavid S. Miller if (!skb) 1852627d2d6bSsamanthakumar return err; 1853db8dac20SDavid S. Miller 1854b65ac446SPaolo Abeni ulen = udp_skb_len(skb); 185559c2cdaeSDavid S. Miller copied = len; 1856627d2d6bSsamanthakumar if (copied > ulen - off) 1857627d2d6bSsamanthakumar copied = ulen - off; 185859c2cdaeSDavid S. Miller else if (copied < ulen) 1859db8dac20SDavid S. Miller msg->msg_flags |= MSG_TRUNC; 1860db8dac20SDavid S. Miller 1861db8dac20SDavid S. Miller /* 1862db8dac20SDavid S. Miller * If checksum is needed at all, try to do it while copying the 1863db8dac20SDavid S. Miller * data. If the data is truncated, or if we only want a partial 1864db8dac20SDavid S. Miller * coverage checksum (UDP-Lite), do it before the copy. 1865db8dac20SDavid S. Miller */ 1866db8dac20SDavid S. Miller 1867d21dbdfeSEric Dumazet if (copied < ulen || peeking || 1868d21dbdfeSEric Dumazet (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { 1869b65ac446SPaolo Abeni checksum_valid = udp_skb_csum_unnecessary(skb) || 1870b65ac446SPaolo Abeni !__udp_lib_checksum_complete(skb); 1871197c949eSEric Dumazet if (!checksum_valid) 1872db8dac20SDavid S. Miller goto csum_copy_err; 1873db8dac20SDavid S. Miller } 1874db8dac20SDavid S. Miller 1875b65ac446SPaolo Abeni if (checksum_valid || udp_skb_csum_unnecessary(skb)) { 1876b65ac446SPaolo Abeni if (udp_skb_is_linear(skb)) 1877b65ac446SPaolo Abeni err = copy_linear_skb(skb, copied, off, &msg->msg_iter); 1878b65ac446SPaolo Abeni else 1879627d2d6bSsamanthakumar err = skb_copy_datagram_msg(skb, off, msg, copied); 1880b65ac446SPaolo Abeni } else { 1881627d2d6bSsamanthakumar err = skb_copy_and_csum_datagram_msg(skb, off, msg); 1882db8dac20SDavid S. Miller 1883db8dac20SDavid S. Miller if (err == -EINVAL) 1884db8dac20SDavid S. Miller goto csum_copy_err; 1885db8dac20SDavid S. Miller } 1886db8dac20SDavid S. Miller 188722911fc5SEric Dumazet if (unlikely(err)) { 1888fd69c399SPaolo Abeni if (!peeking) { 1889979402b1SEric Dumazet atomic_inc(&sk->sk_drops); 18906aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1891979402b1SEric Dumazet UDP_MIB_INERRORS, is_udplite); 1892979402b1SEric Dumazet } 1893850cbaddSPaolo Abeni kfree_skb(skb); 1894627d2d6bSsamanthakumar return err; 189522911fc5SEric Dumazet } 1896db8dac20SDavid S. Miller 1897fd69c399SPaolo Abeni if (!peeking) 18986aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), 1899629ca23cSPavel Emelyanov UDP_MIB_INDATAGRAMS, is_udplite); 1900db8dac20SDavid S. Miller 19013b885787SNeil Horman sock_recv_ts_and_drops(msg, sk, skb); 1902db8dac20SDavid S. Miller 1903db8dac20SDavid S. Miller /* Copy the address. */ 1904c482c568SEric Dumazet if (sin) { 1905db8dac20SDavid S. Miller sin->sin_family = AF_INET; 1906db8dac20SDavid S. Miller sin->sin_port = udp_hdr(skb)->source; 1907db8dac20SDavid S. Miller sin->sin_addr.s_addr = ip_hdr(skb)->saddr; 1908db8dac20SDavid S. Miller memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); 1909bceaa902SHannes Frederic Sowa *addr_len = sizeof(*sin); 1910983695faSDaniel Borkmann 1911983695faSDaniel Borkmann BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, 1912983695faSDaniel Borkmann (struct sockaddr *)sin); 1913db8dac20SDavid S. Miller } 1914bcd1665eSPaolo Abeni 1915bcd1665eSPaolo Abeni if (udp_sk(sk)->gro_enabled) 1916bcd1665eSPaolo Abeni udp_cmsg_recv(msg, sk, skb); 1917bcd1665eSPaolo Abeni 1918db8dac20SDavid S. Miller if (inet->cmsg_flags) 1919ad959036SPaolo Abeni ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); 1920db8dac20SDavid S. Miller 192159c2cdaeSDavid S. Miller err = copied; 1922db8dac20SDavid S. Miller if (flags & MSG_TRUNC) 1923db8dac20SDavid S. Miller err = ulen; 1924db8dac20SDavid S. Miller 1925850cbaddSPaolo Abeni skb_consume_udp(sk, skb, peeking ? -err : err); 1926db8dac20SDavid S. Miller return err; 1927db8dac20SDavid S. Miller 1928db8dac20SDavid S. Miller csum_copy_err: 19292276f58aSPaolo Abeni if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, 19302276f58aSPaolo Abeni udp_skb_destructor)) { 19316aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); 19326aef70a8SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 19336a5dc9e5SEric Dumazet } 1934850cbaddSPaolo Abeni kfree_skb(skb); 1935db8dac20SDavid S. Miller 1936beb39db5SEric Dumazet /* starting over for a new packet, but check if we need to yield */ 1937beb39db5SEric Dumazet cond_resched(); 19389cfaa8deSXufeng Zhang msg->msg_flags &= ~MSG_TRUNC; 1939db8dac20SDavid S. Miller goto try_again; 1940db8dac20SDavid S. Miller } 1941db8dac20SDavid S. Miller 1942d74bad4eSAndrey Ignatov int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) 1943d74bad4eSAndrey Ignatov { 1944d74bad4eSAndrey Ignatov /* This check is replicated from __ip4_datagram_connect() and 1945d74bad4eSAndrey Ignatov * intended to prevent BPF program called below from accessing bytes 1946d74bad4eSAndrey Ignatov * that are out of the bound specified by user in addr_len. 1947d74bad4eSAndrey Ignatov */ 1948d74bad4eSAndrey Ignatov if (addr_len < sizeof(struct sockaddr_in)) 1949d74bad4eSAndrey Ignatov return -EINVAL; 1950d74bad4eSAndrey Ignatov 1951d74bad4eSAndrey Ignatov return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr); 1952d74bad4eSAndrey Ignatov } 1953d74bad4eSAndrey Ignatov EXPORT_SYMBOL(udp_pre_connect); 1954d74bad4eSAndrey Ignatov 1955286c72deSEric Dumazet int __udp_disconnect(struct sock *sk, int flags) 19561da177e4SLinus Torvalds { 19571da177e4SLinus Torvalds struct inet_sock *inet = inet_sk(sk); 19581da177e4SLinus Torvalds /* 19591da177e4SLinus Torvalds * 1003.1g - break association. 19601da177e4SLinus Torvalds */ 19611da177e4SLinus Torvalds 19621da177e4SLinus Torvalds sk->sk_state = TCP_CLOSE; 1963c720c7e8SEric Dumazet inet->inet_daddr = 0; 1964c720c7e8SEric Dumazet inet->inet_dport = 0; 1965bdeab991STom Herbert sock_rps_reset_rxhash(sk); 19661da177e4SLinus Torvalds sk->sk_bound_dev_if = 0; 1967303d0403SWillem de Bruijn if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) { 19681da177e4SLinus Torvalds inet_reset_saddr(sk); 1969303d0403SWillem de Bruijn if (sk->sk_prot->rehash && 1970303d0403SWillem de Bruijn (sk->sk_userlocks & SOCK_BINDPORT_LOCK)) 1971303d0403SWillem de Bruijn sk->sk_prot->rehash(sk); 1972303d0403SWillem de Bruijn } 19731da177e4SLinus Torvalds 19741da177e4SLinus Torvalds if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) { 19751da177e4SLinus Torvalds sk->sk_prot->unhash(sk); 1976c720c7e8SEric Dumazet inet->inet_sport = 0; 19771da177e4SLinus Torvalds } 19781da177e4SLinus Torvalds sk_dst_reset(sk); 19791da177e4SLinus Torvalds return 0; 19801da177e4SLinus Torvalds } 1981286c72deSEric Dumazet EXPORT_SYMBOL(__udp_disconnect); 1982286c72deSEric Dumazet 1983286c72deSEric Dumazet int udp_disconnect(struct sock *sk, int flags) 1984286c72deSEric Dumazet { 1985286c72deSEric Dumazet lock_sock(sk); 1986286c72deSEric Dumazet __udp_disconnect(sk, flags); 1987286c72deSEric Dumazet release_sock(sk); 1988286c72deSEric Dumazet return 0; 1989286c72deSEric Dumazet } 1990c482c568SEric Dumazet EXPORT_SYMBOL(udp_disconnect); 19911da177e4SLinus Torvalds 1992645ca708SEric Dumazet void udp_lib_unhash(struct sock *sk) 1993645ca708SEric Dumazet { 1994723b4610SEric Dumazet if (sk_hashed(sk)) { 1995645ca708SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 1996512615b6SEric Dumazet struct udp_hslot *hslot, *hslot2; 1997512615b6SEric Dumazet 1998512615b6SEric Dumazet hslot = udp_hashslot(udptable, sock_net(sk), 1999d4cada4aSEric Dumazet udp_sk(sk)->udp_port_hash); 2000512615b6SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 2001645ca708SEric Dumazet 2002c8db3fecSEric Dumazet spin_lock_bh(&hslot->lock); 2003e32ea7e7SCraig Gallek if (rcu_access_pointer(sk->sk_reuseport_cb)) 2004e32ea7e7SCraig Gallek reuseport_detach_sock(sk); 2005ca065d0cSEric Dumazet if (sk_del_node_init_rcu(sk)) { 2006fdcc8aa9SEric Dumazet hslot->count--; 2007c720c7e8SEric Dumazet inet_sk(sk)->inet_num = 0; 2008645ca708SEric Dumazet sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 2009512615b6SEric Dumazet 2010512615b6SEric Dumazet spin_lock(&hslot2->lock); 2011ca065d0cSEric Dumazet hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); 2012512615b6SEric Dumazet hslot2->count--; 2013512615b6SEric Dumazet spin_unlock(&hslot2->lock); 2014645ca708SEric Dumazet } 2015c8db3fecSEric Dumazet spin_unlock_bh(&hslot->lock); 2016645ca708SEric Dumazet } 2017723b4610SEric Dumazet } 2018645ca708SEric Dumazet EXPORT_SYMBOL(udp_lib_unhash); 2019645ca708SEric Dumazet 2020719f8358SEric Dumazet /* 2021719f8358SEric Dumazet * inet_rcv_saddr was changed, we must rehash secondary hash 2022719f8358SEric Dumazet */ 2023719f8358SEric Dumazet void udp_lib_rehash(struct sock *sk, u16 newhash) 2024719f8358SEric Dumazet { 2025719f8358SEric Dumazet if (sk_hashed(sk)) { 2026719f8358SEric Dumazet struct udp_table *udptable = sk->sk_prot->h.udp_table; 2027719f8358SEric Dumazet struct udp_hslot *hslot, *hslot2, *nhslot2; 2028719f8358SEric Dumazet 2029719f8358SEric Dumazet hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); 2030719f8358SEric Dumazet nhslot2 = udp_hashslot2(udptable, newhash); 2031719f8358SEric Dumazet udp_sk(sk)->udp_portaddr_hash = newhash; 2032e32ea7e7SCraig Gallek 2033e32ea7e7SCraig Gallek if (hslot2 != nhslot2 || 2034e32ea7e7SCraig Gallek rcu_access_pointer(sk->sk_reuseport_cb)) { 2035719f8358SEric Dumazet hslot = udp_hashslot(udptable, sock_net(sk), 2036719f8358SEric Dumazet udp_sk(sk)->udp_port_hash); 2037719f8358SEric Dumazet /* we must lock primary chain too */ 2038719f8358SEric Dumazet spin_lock_bh(&hslot->lock); 2039e32ea7e7SCraig Gallek if (rcu_access_pointer(sk->sk_reuseport_cb)) 2040e32ea7e7SCraig Gallek reuseport_detach_sock(sk); 2041719f8358SEric Dumazet 2042e32ea7e7SCraig Gallek if (hslot2 != nhslot2) { 2043719f8358SEric Dumazet spin_lock(&hslot2->lock); 2044ca065d0cSEric Dumazet hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); 2045719f8358SEric Dumazet hslot2->count--; 2046719f8358SEric Dumazet spin_unlock(&hslot2->lock); 2047719f8358SEric Dumazet 2048719f8358SEric Dumazet spin_lock(&nhslot2->lock); 2049ca065d0cSEric Dumazet hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, 2050719f8358SEric Dumazet &nhslot2->head); 2051719f8358SEric Dumazet nhslot2->count++; 2052719f8358SEric Dumazet spin_unlock(&nhslot2->lock); 2053e32ea7e7SCraig Gallek } 2054719f8358SEric Dumazet 2055719f8358SEric Dumazet spin_unlock_bh(&hslot->lock); 2056719f8358SEric Dumazet } 2057719f8358SEric Dumazet } 2058719f8358SEric Dumazet } 2059719f8358SEric Dumazet EXPORT_SYMBOL(udp_lib_rehash); 2060719f8358SEric Dumazet 20618f6b5392SAlexey Kodanev void udp_v4_rehash(struct sock *sk) 2062719f8358SEric Dumazet { 2063f0b1e64cSMartin KaFai Lau u16 new_hash = ipv4_portaddr_hash(sock_net(sk), 2064719f8358SEric Dumazet inet_sk(sk)->inet_rcv_saddr, 2065719f8358SEric Dumazet inet_sk(sk)->inet_num); 2066719f8358SEric Dumazet udp_lib_rehash(sk, new_hash); 2067719f8358SEric Dumazet } 2068719f8358SEric Dumazet 2069a3f96c47SPaolo Abeni static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 207093821778SHerbert Xu { 2071fec5e652STom Herbert int rc; 207293821778SHerbert Xu 2073005ec974SShawn Bohrer if (inet_sk(sk)->inet_daddr) { 2074bdeab991STom Herbert sock_rps_save_rxhash(sk, skb); 2075005ec974SShawn Bohrer sk_mark_napi_id(sk, skb); 20762c8c56e1SEric Dumazet sk_incoming_cpu_update(sk); 2077e68b6e50SEric Dumazet } else { 2078e68b6e50SEric Dumazet sk_mark_napi_id_once(sk, skb); 2079005ec974SShawn Bohrer } 2080fec5e652STom Herbert 2081850cbaddSPaolo Abeni rc = __udp_enqueue_schedule_skb(sk, skb); 2082766e9037SEric Dumazet if (rc < 0) { 2083766e9037SEric Dumazet int is_udplite = IS_UDPLITE(sk); 2084766e9037SEric Dumazet 208593821778SHerbert Xu /* Note that an ENOMEM error is charged twice */ 2086766e9037SEric Dumazet if (rc == -ENOMEM) 2087e61da9e2SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, 208893821778SHerbert Xu is_udplite); 2089a3ce2b10SMenglong Dong else 2090a3ce2b10SMenglong Dong UDP_INC_STATS(sock_net(sk), UDP_MIB_MEMERRORS, 2091a3ce2b10SMenglong Dong is_udplite); 2092e61da9e2SEric Dumazet UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 2093766e9037SEric Dumazet kfree_skb(skb); 2094296f7ea7SSatoru Moriya trace_udp_fail_queue_rcv_skb(rc, sk); 2095766e9037SEric Dumazet return -1; 209693821778SHerbert Xu } 209793821778SHerbert Xu 209893821778SHerbert Xu return 0; 209993821778SHerbert Xu } 210093821778SHerbert Xu 2101db8dac20SDavid S. Miller /* returns: 2102db8dac20SDavid S. Miller * -1: error 2103db8dac20SDavid S. Miller * 0: success 2104db8dac20SDavid S. Miller * >0: "udp encap" protocol resubmission 2105db8dac20SDavid S. Miller * 2106db8dac20SDavid S. Miller * Note that in the success and error cases, the skb is assumed to 2107db8dac20SDavid S. Miller * have either been requeued or freed. 2108db8dac20SDavid S. Miller */ 2109cf329aa4SPaolo Abeni static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) 2110db8dac20SDavid S. Miller { 2111db8dac20SDavid S. Miller struct udp_sock *up = udp_sk(sk); 2112db8dac20SDavid S. Miller int is_udplite = IS_UDPLITE(sk); 2113db8dac20SDavid S. Miller 2114db8dac20SDavid S. Miller /* 2115db8dac20SDavid S. Miller * Charge it to the socket, dropping if the queue is full. 2116db8dac20SDavid S. Miller */ 2117db8dac20SDavid S. Miller if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) 2118db8dac20SDavid S. Miller goto drop; 2119895b5c9fSFlorian Westphal nf_reset_ct(skb); 2120db8dac20SDavid S. Miller 212188ab3108SDavidlohr Bueso if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) { 21220ad92ad0SEric Dumazet int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); 21230ad92ad0SEric Dumazet 2124db8dac20SDavid S. Miller /* 2125db8dac20SDavid S. Miller * This is an encapsulation socket so pass the skb to 2126db8dac20SDavid S. Miller * the socket's udp_encap_rcv() hook. Otherwise, just 2127db8dac20SDavid S. Miller * fall through and pass this up the UDP socket. 2128db8dac20SDavid S. Miller * up->encap_rcv() returns the following value: 2129db8dac20SDavid S. Miller * =0 if skb was successfully passed to the encap 2130db8dac20SDavid S. Miller * handler or was discarded by it. 2131db8dac20SDavid S. Miller * >0 if skb should be passed on to UDP. 2132db8dac20SDavid S. Miller * <0 if skb should be resubmitted as proto -N 2133db8dac20SDavid S. Miller */ 2134db8dac20SDavid S. Miller 2135db8dac20SDavid S. Miller /* if we're overly short, let UDP handle it */ 21366aa7de05SMark Rutland encap_rcv = READ_ONCE(up->encap_rcv); 2137e5aed006SHannes Frederic Sowa if (encap_rcv) { 2138db8dac20SDavid S. Miller int ret; 2139db8dac20SDavid S. Miller 21400a80966bSTom Herbert /* Verify checksum before giving to encap */ 21410a80966bSTom Herbert if (udp_lib_checksum_complete(skb)) 21420a80966bSTom Herbert goto csum_error; 21430a80966bSTom Herbert 21440ad92ad0SEric Dumazet ret = encap_rcv(sk, skb); 2145db8dac20SDavid S. Miller if (ret <= 0) { 214602c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), 21470283328eSPavel Emelyanov UDP_MIB_INDATAGRAMS, 2148db8dac20SDavid S. Miller is_udplite); 2149db8dac20SDavid S. Miller return -ret; 2150db8dac20SDavid S. Miller } 2151db8dac20SDavid S. Miller } 2152db8dac20SDavid S. Miller 2153db8dac20SDavid S. Miller /* FALLTHROUGH -- it's a UDP Packet */ 2154db8dac20SDavid S. Miller } 2155db8dac20SDavid S. Miller 2156db8dac20SDavid S. Miller /* 2157db8dac20SDavid S. Miller * UDP-Lite specific tests, ignored on UDP sockets 2158db8dac20SDavid S. Miller */ 2159b0a42277SMiaohe Lin if ((up->pcflag & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) { 2160db8dac20SDavid S. Miller 2161db8dac20SDavid S. Miller /* 2162db8dac20SDavid S. Miller * MIB statistics other than incrementing the error count are 2163db8dac20SDavid S. Miller * disabled for the following two types of errors: these depend 2164db8dac20SDavid S. Miller * on the application settings, not on the functioning of the 2165db8dac20SDavid S. Miller * protocol stack as such. 2166db8dac20SDavid S. Miller * 2167db8dac20SDavid S. Miller * RFC 3828 here recommends (sec 3.3): "There should also be a 2168db8dac20SDavid S. Miller * way ... to ... at least let the receiving application block 2169db8dac20SDavid S. Miller * delivery of packets with coverage values less than a value 2170db8dac20SDavid S. Miller * provided by the application." 2171db8dac20SDavid S. Miller */ 2172db8dac20SDavid S. Miller if (up->pcrlen == 0) { /* full coverage was set */ 2173ba7a46f1SJoe Perches net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n", 2174db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov, skb->len); 2175db8dac20SDavid S. Miller goto drop; 2176db8dac20SDavid S. Miller } 2177db8dac20SDavid S. Miller /* The next case involves violating the min. coverage requested 2178db8dac20SDavid S. Miller * by the receiver. This is subtle: if receiver wants x and x is 2179db8dac20SDavid S. Miller * greater than the buffersize/MTU then receiver will complain 2180db8dac20SDavid S. Miller * that it wants x while sender emits packets of smaller size y. 2181db8dac20SDavid S. Miller * Therefore the above ...()->partial_cov statement is essential. 2182db8dac20SDavid S. Miller */ 2183db8dac20SDavid S. Miller if (UDP_SKB_CB(skb)->cscov < up->pcrlen) { 2184ba7a46f1SJoe Perches net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n", 2185db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov, up->pcrlen); 2186db8dac20SDavid S. Miller goto drop; 2187db8dac20SDavid S. Miller } 2188db8dac20SDavid S. Miller } 2189db8dac20SDavid S. Miller 2190dd99e425SPaolo Abeni prefetch(&sk->sk_rmem_alloc); 2191ce25d66aSEric Dumazet if (rcu_access_pointer(sk->sk_filter) && 2192ce25d66aSEric Dumazet udp_lib_checksum_complete(skb)) 21936a5dc9e5SEric Dumazet goto csum_error; 2194ce25d66aSEric Dumazet 2195ba66bbe5SDaniel Borkmann if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) 2196a6127697SMichal Kubeček goto drop; 2197db8dac20SDavid S. Miller 2198e6afc8acSsamanthakumar udp_csum_pull_header(skb); 2199db8dac20SDavid S. Miller 2200fbf8866dSShawn Bohrer ipv4_pktinfo_prepare(sk, skb); 2201850cbaddSPaolo Abeni return __udp_queue_rcv_skb(sk, skb); 2202db8dac20SDavid S. Miller 22036a5dc9e5SEric Dumazet csum_error: 220402c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); 2205db8dac20SDavid S. Miller drop: 220602c22347SEric Dumazet __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); 22078edf19c2SEric Dumazet atomic_inc(&sk->sk_drops); 2208db8dac20SDavid S. Miller kfree_skb(skb); 2209db8dac20SDavid S. Miller return -1; 2210db8dac20SDavid S. Miller } 2211db8dac20SDavid S. Miller 2212cf329aa4SPaolo Abeni static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 2213cf329aa4SPaolo Abeni { 2214cf329aa4SPaolo Abeni struct sk_buff *next, *segs; 2215cf329aa4SPaolo Abeni int ret; 2216cf329aa4SPaolo Abeni 2217cf329aa4SPaolo Abeni if (likely(!udp_unexpected_gso(sk, skb))) 2218cf329aa4SPaolo Abeni return udp_queue_rcv_one_skb(sk, skb); 2219cf329aa4SPaolo Abeni 2220a08e7fd9SCambda Zhu BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET); 2221cf329aa4SPaolo Abeni __skb_push(skb, -skb_mac_offset(skb)); 2222cf329aa4SPaolo Abeni segs = udp_rcv_segment(sk, skb, true); 22231a186c14SJason A. Donenfeld skb_list_walk_safe(segs, skb, next) { 2224cf329aa4SPaolo Abeni __skb_pull(skb, skb_transport_offset(skb)); 2225000ac44dSPaolo Abeni 2226000ac44dSPaolo Abeni udp_post_segment_fix_csum(skb); 2227cf329aa4SPaolo Abeni ret = udp_queue_rcv_one_skb(sk, skb); 2228cf329aa4SPaolo Abeni if (ret > 0) 222910c678bdSXin Long ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret); 2230cf329aa4SPaolo Abeni } 2231cf329aa4SPaolo Abeni return 0; 2232cf329aa4SPaolo Abeni } 2233cf329aa4SPaolo Abeni 223497502231SEric Dumazet /* For TCP sockets, sk_rx_dst is protected by socket lock 2235e47eb5dfSEric Dumazet * For UDP, we use xchg() to guard against concurrent changes. 223697502231SEric Dumazet */ 223764f0f5d1SPaolo Abeni bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst) 2238421b3885SShawn Bohrer { 223997502231SEric Dumazet struct dst_entry *old; 2240421b3885SShawn Bohrer 2241d24406c8SWei Wang if (dst_hold_safe(dst)) { 2242e47eb5dfSEric Dumazet old = xchg(&sk->sk_rx_dst, dst); 224397502231SEric Dumazet dst_release(old); 224464f0f5d1SPaolo Abeni return old != dst; 224597502231SEric Dumazet } 224664f0f5d1SPaolo Abeni return false; 2247d24406c8SWei Wang } 2248c9f2c1aeSPaolo Abeni EXPORT_SYMBOL(udp_sk_rx_dst_set); 2249421b3885SShawn Bohrer 2250db8dac20SDavid S. Miller /* 2251db8dac20SDavid S. Miller * Multicasts and broadcasts go to each listener. 2252db8dac20SDavid S. Miller * 22531240d137SEric Dumazet * Note: called only from the BH handler context. 2254db8dac20SDavid S. Miller */ 2255e3163493SPavel Emelyanov static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 2256db8dac20SDavid S. Miller struct udphdr *uh, 2257db8dac20SDavid S. Miller __be32 saddr, __be32 daddr, 225836cbb245SRick Jones struct udp_table *udptable, 225936cbb245SRick Jones int proto) 2260db8dac20SDavid S. Miller { 2261ca065d0cSEric Dumazet struct sock *sk, *first = NULL; 22625cf3d461SDavid Held unsigned short hnum = ntohs(uh->dest); 22635cf3d461SDavid Held struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); 22642dc41cffSDavid Held unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); 2265ca065d0cSEric Dumazet unsigned int offset = offsetof(typeof(*sk), sk_node); 2266ca065d0cSEric Dumazet int dif = skb->dev->ifindex; 2267fb74c277SDavid Ahern int sdif = inet_sdif(skb); 2268ca065d0cSEric Dumazet struct hlist_node *node; 2269ca065d0cSEric Dumazet struct sk_buff *nskb; 22702dc41cffSDavid Held 22712dc41cffSDavid Held if (use_hash2) { 2272f0b1e64cSMartin KaFai Lau hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) & 227373e2d5e3SPablo Neira udptable->mask; 2274f0b1e64cSMartin KaFai Lau hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask; 22752dc41cffSDavid Held start_lookup: 227673e2d5e3SPablo Neira hslot = &udptable->hash2[hash2]; 22772dc41cffSDavid Held offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); 22782dc41cffSDavid Held } 2279db8dac20SDavid S. Miller 2280ca065d0cSEric Dumazet sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) { 2281ca065d0cSEric Dumazet if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr, 2282fb74c277SDavid Ahern uh->source, saddr, dif, sdif, hnum)) 2283ca065d0cSEric Dumazet continue; 22841240d137SEric Dumazet 2285ca065d0cSEric Dumazet if (!first) { 2286ca065d0cSEric Dumazet first = sk; 2287ca065d0cSEric Dumazet continue; 2288ca065d0cSEric Dumazet } 2289ca065d0cSEric Dumazet nskb = skb_clone(skb, GFP_ATOMIC); 2290ca065d0cSEric Dumazet 2291ca065d0cSEric Dumazet if (unlikely(!nskb)) { 2292ca065d0cSEric Dumazet atomic_inc(&sk->sk_drops); 229302c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS, 2294ca065d0cSEric Dumazet IS_UDPLITE(sk)); 229502c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_INERRORS, 2296ca065d0cSEric Dumazet IS_UDPLITE(sk)); 2297ca065d0cSEric Dumazet continue; 2298ca065d0cSEric Dumazet } 2299ca065d0cSEric Dumazet if (udp_queue_rcv_skb(sk, nskb) > 0) 2300ca065d0cSEric Dumazet consume_skb(nskb); 2301ca065d0cSEric Dumazet } 23021240d137SEric Dumazet 23032dc41cffSDavid Held /* Also lookup *:port if we are using hash2 and haven't done so yet. */ 23042dc41cffSDavid Held if (use_hash2 && hash2 != hash2_any) { 23052dc41cffSDavid Held hash2 = hash2_any; 23062dc41cffSDavid Held goto start_lookup; 23072dc41cffSDavid Held } 23082dc41cffSDavid Held 2309ca065d0cSEric Dumazet if (first) { 2310ca065d0cSEric Dumazet if (udp_queue_rcv_skb(first, skb) > 0) 2311ca065d0cSEric Dumazet consume_skb(skb); 23121240d137SEric Dumazet } else { 2313ca065d0cSEric Dumazet kfree_skb(skb); 231402c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI, 231536cbb245SRick Jones proto == IPPROTO_UDPLITE); 23161240d137SEric Dumazet } 2317db8dac20SDavid S. Miller return 0; 2318db8dac20SDavid S. Miller } 2319db8dac20SDavid S. Miller 2320db8dac20SDavid S. Miller /* Initialize UDP checksum. If exited with zero value (success), 2321db8dac20SDavid S. Miller * CHECKSUM_UNNECESSARY means, that no more checks are required. 2322666a3d6eSSu Yanjun * Otherwise, csum completion requires checksumming packet body, 2323db8dac20SDavid S. Miller * including udp header and folding it to skb->csum. 2324db8dac20SDavid S. Miller */ 2325db8dac20SDavid S. Miller static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh, 2326db8dac20SDavid S. Miller int proto) 2327db8dac20SDavid S. Miller { 2328db8dac20SDavid S. Miller int err; 2329db8dac20SDavid S. Miller 2330db8dac20SDavid S. Miller UDP_SKB_CB(skb)->partial_cov = 0; 2331db8dac20SDavid S. Miller UDP_SKB_CB(skb)->cscov = skb->len; 2332db8dac20SDavid S. Miller 2333db8dac20SDavid S. Miller if (proto == IPPROTO_UDPLITE) { 2334db8dac20SDavid S. Miller err = udplite_checksum_init(skb, uh); 2335db8dac20SDavid S. Miller if (err) 2336db8dac20SDavid S. Miller return err; 233715f35d49SAlexey Kodanev 233815f35d49SAlexey Kodanev if (UDP_SKB_CB(skb)->partial_cov) { 233915f35d49SAlexey Kodanev skb->csum = inet_compute_pseudo(skb, proto); 234015f35d49SAlexey Kodanev return 0; 234115f35d49SAlexey Kodanev } 2342db8dac20SDavid S. Miller } 2343db8dac20SDavid S. Miller 2344b46d9f62SHannes Frederic Sowa /* Note, we are only interested in != 0 or == 0, thus the 2345b46d9f62SHannes Frederic Sowa * force to int. 2346b46d9f62SHannes Frederic Sowa */ 2347db4f1be3SSean Tranchetti err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check, 2348ed70fcfcSTom Herbert inet_compute_pseudo); 2349db4f1be3SSean Tranchetti if (err) 2350db4f1be3SSean Tranchetti return err; 2351db4f1be3SSean Tranchetti 2352db4f1be3SSean Tranchetti if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) { 2353db4f1be3SSean Tranchetti /* If SW calculated the value, we know it's bad */ 2354db4f1be3SSean Tranchetti if (skb->csum_complete_sw) 2355db4f1be3SSean Tranchetti return 1; 2356db4f1be3SSean Tranchetti 2357db4f1be3SSean Tranchetti /* HW says the value is bad. Let's validate that. 2358db4f1be3SSean Tranchetti * skb->csum is no longer the full packet checksum, 2359db4f1be3SSean Tranchetti * so don't treat it as such. 2360db4f1be3SSean Tranchetti */ 2361db4f1be3SSean Tranchetti skb_checksum_complete_unset(skb); 2362db4f1be3SSean Tranchetti } 2363db4f1be3SSean Tranchetti 2364db4f1be3SSean Tranchetti return 0; 2365db8dac20SDavid S. Miller } 2366db8dac20SDavid S. Miller 23672b5a9217SPaolo Abeni /* wrapper for udp_queue_rcv_skb tacking care of csum conversion and 23682b5a9217SPaolo Abeni * return code conversion for ip layer consumption 23692b5a9217SPaolo Abeni */ 23702b5a9217SPaolo Abeni static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb, 23712b5a9217SPaolo Abeni struct udphdr *uh) 23722b5a9217SPaolo Abeni { 23732b5a9217SPaolo Abeni int ret; 23742b5a9217SPaolo Abeni 23752b5a9217SPaolo Abeni if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk)) 2376e4aa33adSLi RongQing skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo); 23772b5a9217SPaolo Abeni 23782b5a9217SPaolo Abeni ret = udp_queue_rcv_skb(sk, skb); 23792b5a9217SPaolo Abeni 23802b5a9217SPaolo Abeni /* a return value > 0 means to resubmit the input, but 23812b5a9217SPaolo Abeni * it wants the return to be -protocol, or 0 23822b5a9217SPaolo Abeni */ 23832b5a9217SPaolo Abeni if (ret > 0) 23842b5a9217SPaolo Abeni return -ret; 23852b5a9217SPaolo Abeni return 0; 23862b5a9217SPaolo Abeni } 23872b5a9217SPaolo Abeni 2388db8dac20SDavid S. Miller /* 2389db8dac20SDavid S. Miller * All we need to do is get the socket, and then do a checksum. 2390db8dac20SDavid S. Miller */ 2391db8dac20SDavid S. Miller 2392645ca708SEric Dumazet int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, 2393db8dac20SDavid S. Miller int proto) 2394db8dac20SDavid S. Miller { 2395db8dac20SDavid S. Miller struct sock *sk; 23967b5e56f9SJesper Dangaard Brouer struct udphdr *uh; 2397db8dac20SDavid S. Miller unsigned short ulen; 2398adf30907SEric Dumazet struct rtable *rt = skb_rtable(skb); 23992783ef23SJesper Dangaard Brouer __be32 saddr, daddr; 24000283328eSPavel Emelyanov struct net *net = dev_net(skb->dev); 240171489e21SJoe Stringer bool refcounted; 2402db8dac20SDavid S. Miller 2403db8dac20SDavid S. Miller /* 2404db8dac20SDavid S. Miller * Validate the packet. 2405db8dac20SDavid S. Miller */ 2406db8dac20SDavid S. Miller if (!pskb_may_pull(skb, sizeof(struct udphdr))) 2407db8dac20SDavid S. Miller goto drop; /* No space for header. */ 2408db8dac20SDavid S. Miller 24097b5e56f9SJesper Dangaard Brouer uh = udp_hdr(skb); 2410db8dac20SDavid S. Miller ulen = ntohs(uh->len); 2411ccc2d97cSBjørn Mork saddr = ip_hdr(skb)->saddr; 2412ccc2d97cSBjørn Mork daddr = ip_hdr(skb)->daddr; 2413ccc2d97cSBjørn Mork 2414db8dac20SDavid S. Miller if (ulen > skb->len) 2415db8dac20SDavid S. Miller goto short_packet; 2416db8dac20SDavid S. Miller 2417db8dac20SDavid S. Miller if (proto == IPPROTO_UDP) { 2418db8dac20SDavid S. Miller /* UDP validates ulen. */ 2419db8dac20SDavid S. Miller if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen)) 2420db8dac20SDavid S. Miller goto short_packet; 2421db8dac20SDavid S. Miller uh = udp_hdr(skb); 2422db8dac20SDavid S. Miller } 2423db8dac20SDavid S. Miller 2424db8dac20SDavid S. Miller if (udp4_csum_init(skb, uh, proto)) 2425db8dac20SDavid S. Miller goto csum_error; 2426db8dac20SDavid S. Miller 242771489e21SJoe Stringer sk = skb_steal_sock(skb, &refcounted); 24288afdd99aSEric Dumazet if (sk) { 242997502231SEric Dumazet struct dst_entry *dst = skb_dst(skb); 2430421b3885SShawn Bohrer int ret; 2431421b3885SShawn Bohrer 243297502231SEric Dumazet if (unlikely(sk->sk_rx_dst != dst)) 243397502231SEric Dumazet udp_sk_rx_dst_set(sk, dst); 2434421b3885SShawn Bohrer 24352b5a9217SPaolo Abeni ret = udp_unicast_rcv_skb(sk, skb, uh); 243671489e21SJoe Stringer if (refcounted) 24378afdd99aSEric Dumazet sock_put(sk); 24382b5a9217SPaolo Abeni return ret; 2439c18450a5SFabian Frederick } 2440c18450a5SFabian Frederick 2441db8dac20SDavid S. Miller if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST)) 2442e3163493SPavel Emelyanov return __udp4_lib_mcast_deliver(net, skb, uh, 244336cbb245SRick Jones saddr, daddr, udptable, proto); 2444db8dac20SDavid S. Miller 2445607c4aafSKOVACS Krisztian sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable); 24462b5a9217SPaolo Abeni if (sk) 24472b5a9217SPaolo Abeni return udp_unicast_rcv_skb(sk, skb, uh); 2448db8dac20SDavid S. Miller 2449db8dac20SDavid S. Miller if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) 2450db8dac20SDavid S. Miller goto drop; 2451895b5c9fSFlorian Westphal nf_reset_ct(skb); 2452db8dac20SDavid S. Miller 2453db8dac20SDavid S. Miller /* No socket. Drop packet silently, if checksum is wrong */ 2454db8dac20SDavid S. Miller if (udp_lib_checksum_complete(skb)) 2455db8dac20SDavid S. Miller goto csum_error; 2456db8dac20SDavid S. Miller 245702c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE); 2458db8dac20SDavid S. Miller icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); 2459db8dac20SDavid S. Miller 2460db8dac20SDavid S. Miller /* 2461db8dac20SDavid S. Miller * Hmm. We got an UDP packet to a port to which we 2462db8dac20SDavid S. Miller * don't wanna listen. Ignore it. 2463db8dac20SDavid S. Miller */ 2464db8dac20SDavid S. Miller kfree_skb(skb); 2465db8dac20SDavid S. Miller return 0; 2466db8dac20SDavid S. Miller 2467db8dac20SDavid S. Miller short_packet: 2468ba7a46f1SJoe Perches net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n", 2469afd46503SJoe Perches proto == IPPROTO_UDPLITE ? "Lite" : "", 2470afd46503SJoe Perches &saddr, ntohs(uh->source), 2471afd46503SJoe Perches ulen, skb->len, 2472afd46503SJoe Perches &daddr, ntohs(uh->dest)); 2473db8dac20SDavid S. Miller goto drop; 2474db8dac20SDavid S. Miller 2475db8dac20SDavid S. Miller csum_error: 2476db8dac20SDavid S. Miller /* 2477db8dac20SDavid S. Miller * RFC1122: OK. Discards the bad packet silently (as far as 2478db8dac20SDavid S. Miller * the network is concerned, anyway) as per 4.1.3.4 (MUST). 2479db8dac20SDavid S. Miller */ 2480ba7a46f1SJoe Perches net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n", 2481afd46503SJoe Perches proto == IPPROTO_UDPLITE ? "Lite" : "", 2482afd46503SJoe Perches &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), 2483db8dac20SDavid S. Miller ulen); 248402c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE); 2485db8dac20SDavid S. Miller drop: 248602c22347SEric Dumazet __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE); 2487db8dac20SDavid S. Miller kfree_skb(skb); 2488db8dac20SDavid S. Miller return 0; 2489db8dac20SDavid S. Miller } 2490db8dac20SDavid S. Miller 2491421b3885SShawn Bohrer /* We can only early demux multicast if there is a single matching socket. 2492421b3885SShawn Bohrer * If more than one socket found returns NULL 2493421b3885SShawn Bohrer */ 2494421b3885SShawn Bohrer static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net, 2495421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 2496421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 2497fb74c277SDavid Ahern int dif, int sdif) 2498421b3885SShawn Bohrer { 2499421b3885SShawn Bohrer struct sock *sk, *result; 2500421b3885SShawn Bohrer unsigned short hnum = ntohs(loc_port); 2501ca065d0cSEric Dumazet unsigned int slot = udp_hashfn(net, hnum, udp_table.mask); 2502421b3885SShawn Bohrer struct udp_hslot *hslot = &udp_table.hash[slot]; 2503421b3885SShawn Bohrer 250463c6f81cSEric Dumazet /* Do not bother scanning a too big list */ 250563c6f81cSEric Dumazet if (hslot->count > 10) 250663c6f81cSEric Dumazet return NULL; 250763c6f81cSEric Dumazet 2508421b3885SShawn Bohrer result = NULL; 2509ca065d0cSEric Dumazet sk_for_each_rcu(sk, &hslot->head) { 2510ca065d0cSEric Dumazet if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr, 2511fb74c277SDavid Ahern rmt_port, rmt_addr, dif, sdif, hnum)) { 2512ca065d0cSEric Dumazet if (result) 2513ca065d0cSEric Dumazet return NULL; 2514421b3885SShawn Bohrer result = sk; 2515421b3885SShawn Bohrer } 2516421b3885SShawn Bohrer } 2517421b3885SShawn Bohrer 2518421b3885SShawn Bohrer return result; 2519421b3885SShawn Bohrer } 2520421b3885SShawn Bohrer 2521421b3885SShawn Bohrer /* For unicast we should only early demux connected sockets or we can 2522421b3885SShawn Bohrer * break forwarding setups. The chains here can be long so only check 2523421b3885SShawn Bohrer * if the first socket is an exact match and if not move on. 2524421b3885SShawn Bohrer */ 2525421b3885SShawn Bohrer static struct sock *__udp4_lib_demux_lookup(struct net *net, 2526421b3885SShawn Bohrer __be16 loc_port, __be32 loc_addr, 2527421b3885SShawn Bohrer __be16 rmt_port, __be32 rmt_addr, 25283fa6f616SDavid Ahern int dif, int sdif) 2529421b3885SShawn Bohrer { 2530421b3885SShawn Bohrer unsigned short hnum = ntohs(loc_port); 2531f0b1e64cSMartin KaFai Lau unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum); 2532421b3885SShawn Bohrer unsigned int slot2 = hash2 & udp_table.mask; 2533421b3885SShawn Bohrer struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; 2534c7228317SJoe Perches INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); 2535421b3885SShawn Bohrer const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); 2536ca065d0cSEric Dumazet struct sock *sk; 2537421b3885SShawn Bohrer 2538ca065d0cSEric Dumazet udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { 2539ca065d0cSEric Dumazet if (INET_MATCH(sk, net, acookie, rmt_addr, 25403fa6f616SDavid Ahern loc_addr, ports, dif, sdif)) 2541ca065d0cSEric Dumazet return sk; 2542421b3885SShawn Bohrer /* Only check first socket in chain */ 2543421b3885SShawn Bohrer break; 2544421b3885SShawn Bohrer } 2545ca065d0cSEric Dumazet return NULL; 2546421b3885SShawn Bohrer } 2547421b3885SShawn Bohrer 25487487449cSPaolo Abeni int udp_v4_early_demux(struct sk_buff *skb) 2549421b3885SShawn Bohrer { 2550610438b7SEric Dumazet struct net *net = dev_net(skb->dev); 2551bc044e8dSPaolo Abeni struct in_device *in_dev = NULL; 2552610438b7SEric Dumazet const struct iphdr *iph; 2553610438b7SEric Dumazet const struct udphdr *uh; 2554ca065d0cSEric Dumazet struct sock *sk = NULL; 2555421b3885SShawn Bohrer struct dst_entry *dst; 2556421b3885SShawn Bohrer int dif = skb->dev->ifindex; 2557fb74c277SDavid Ahern int sdif = inet_sdif(skb); 25586e540309SShawn Bohrer int ours; 2559421b3885SShawn Bohrer 2560421b3885SShawn Bohrer /* validate the packet */ 2561421b3885SShawn Bohrer if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr))) 25627487449cSPaolo Abeni return 0; 2563421b3885SShawn Bohrer 2564610438b7SEric Dumazet iph = ip_hdr(skb); 2565610438b7SEric Dumazet uh = udp_hdr(skb); 2566610438b7SEric Dumazet 2567996b44fcSPaolo Abeni if (skb->pkt_type == PACKET_MULTICAST) { 2568bc044e8dSPaolo Abeni in_dev = __in_dev_get_rcu(skb->dev); 25696e540309SShawn Bohrer 25706e540309SShawn Bohrer if (!in_dev) 25717487449cSPaolo Abeni return 0; 25726e540309SShawn Bohrer 25736e540309SShawn Bohrer ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr, 25746e540309SShawn Bohrer iph->protocol); 25756e540309SShawn Bohrer if (!ours) 25767487449cSPaolo Abeni return 0; 2577ad0ea198SPaolo Abeni 2578421b3885SShawn Bohrer sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr, 2579fb74c277SDavid Ahern uh->source, iph->saddr, 2580fb74c277SDavid Ahern dif, sdif); 25816e540309SShawn Bohrer } else if (skb->pkt_type == PACKET_HOST) { 2582421b3885SShawn Bohrer sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, 25833fa6f616SDavid Ahern uh->source, iph->saddr, dif, sdif); 25846e540309SShawn Bohrer } 2585421b3885SShawn Bohrer 258641c6d650SReshetova, Elena if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 25877487449cSPaolo Abeni return 0; 2588421b3885SShawn Bohrer 2589421b3885SShawn Bohrer skb->sk = sk; 259082eabd9eSAlexander Duyck skb->destructor = sock_efree; 259110e2eb87SEric Dumazet dst = READ_ONCE(sk->sk_rx_dst); 2592421b3885SShawn Bohrer 2593421b3885SShawn Bohrer if (dst) 2594421b3885SShawn Bohrer dst = dst_check(dst, 0); 259510e2eb87SEric Dumazet if (dst) { 2596bc044e8dSPaolo Abeni u32 itag = 0; 2597bc044e8dSPaolo Abeni 2598d24406c8SWei Wang /* set noref for now. 2599d24406c8SWei Wang * any place which wants to hold dst has to call 2600d24406c8SWei Wang * dst_hold_safe() 2601d24406c8SWei Wang */ 2602421b3885SShawn Bohrer skb_dst_set_noref(skb, dst); 2603bc044e8dSPaolo Abeni 2604bc044e8dSPaolo Abeni /* for unconnected multicast sockets we need to validate 2605bc044e8dSPaolo Abeni * the source on each packet 2606bc044e8dSPaolo Abeni */ 2607bc044e8dSPaolo Abeni if (!inet_sk(sk)->inet_daddr && in_dev) 2608bc044e8dSPaolo Abeni return ip_mc_validate_source(skb, iph->daddr, 26098d2b51b0SGuillaume Nault iph->saddr, 26108d2b51b0SGuillaume Nault iph->tos & IPTOS_RT_MASK, 2611bc044e8dSPaolo Abeni skb->dev, in_dev, &itag); 2612421b3885SShawn Bohrer } 26137487449cSPaolo Abeni return 0; 261410e2eb87SEric Dumazet } 2615421b3885SShawn Bohrer 2616db8dac20SDavid S. Miller int udp_rcv(struct sk_buff *skb) 2617db8dac20SDavid S. Miller { 2618645ca708SEric Dumazet return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP); 2619db8dac20SDavid S. Miller } 2620db8dac20SDavid S. Miller 26217d06b2e0SBrian Haley void udp_destroy_sock(struct sock *sk) 2622db8dac20SDavid S. Miller { 262344046a59STom Parkin struct udp_sock *up = udp_sk(sk); 26248a74ad60SEric Dumazet bool slow = lock_sock_fast(sk); 2625a8b897c7SPaolo Abeni 2626a8b897c7SPaolo Abeni /* protects from races with udp_abort() */ 2627a8b897c7SPaolo Abeni sock_set_flag(sk, SOCK_DEAD); 2628db8dac20SDavid S. Miller udp_flush_pending_frames(sk); 26298a74ad60SEric Dumazet unlock_sock_fast(sk, slow); 263060fb9567SPaolo Abeni if (static_branch_unlikely(&udp_encap_needed_key)) { 263160fb9567SPaolo Abeni if (up->encap_type) { 263244046a59STom Parkin void (*encap_destroy)(struct sock *sk); 26336aa7de05SMark Rutland encap_destroy = READ_ONCE(up->encap_destroy); 263444046a59STom Parkin if (encap_destroy) 263544046a59STom Parkin encap_destroy(sk); 263644046a59STom Parkin } 263760fb9567SPaolo Abeni if (up->encap_enabled) 26389c480601SPaolo Abeni static_branch_dec(&udp_encap_needed_key); 263960fb9567SPaolo Abeni } 2640db8dac20SDavid S. Miller } 2641db8dac20SDavid S. Miller 26421da177e4SLinus Torvalds /* 26431da177e4SLinus Torvalds * Socket option code for UDP 26441da177e4SLinus Torvalds */ 26454c0a6cb0SGerrit Renker int udp_lib_setsockopt(struct sock *sk, int level, int optname, 264691ac1ccaSChristoph Hellwig sockptr_t optval, unsigned int optlen, 26474c0a6cb0SGerrit Renker int (*push_pending_frames)(struct sock *)) 26481da177e4SLinus Torvalds { 26491da177e4SLinus Torvalds struct udp_sock *up = udp_sk(sk); 26501c19448cSTom Herbert int val, valbool; 26511da177e4SLinus Torvalds int err = 0; 2652b2bf1e26SWang Chen int is_udplite = IS_UDPLITE(sk); 26531da177e4SLinus Torvalds 26541da177e4SLinus Torvalds if (optlen < sizeof(int)) 26551da177e4SLinus Torvalds return -EINVAL; 26561da177e4SLinus Torvalds 265791ac1ccaSChristoph Hellwig if (copy_from_sockptr(&val, optval, sizeof(val))) 26581da177e4SLinus Torvalds return -EFAULT; 26591da177e4SLinus Torvalds 26601c19448cSTom Herbert valbool = val ? 1 : 0; 26611c19448cSTom Herbert 26621da177e4SLinus Torvalds switch (optname) { 26631da177e4SLinus Torvalds case UDP_CORK: 26641da177e4SLinus Torvalds if (val != 0) { 2665a9f59707SEric Dumazet WRITE_ONCE(up->corkflag, 1); 26661da177e4SLinus Torvalds } else { 2667a9f59707SEric Dumazet WRITE_ONCE(up->corkflag, 0); 26681da177e4SLinus Torvalds lock_sock(sk); 26694243cdc2SJoe Perches push_pending_frames(sk); 26701da177e4SLinus Torvalds release_sock(sk); 26711da177e4SLinus Torvalds } 26721da177e4SLinus Torvalds break; 26731da177e4SLinus Torvalds 26741da177e4SLinus Torvalds case UDP_ENCAP: 26751da177e4SLinus Torvalds switch (val) { 26761da177e4SLinus Torvalds case 0: 2677fd1ac07fSAlexey Dobriyan #ifdef CONFIG_XFRM 26781da177e4SLinus Torvalds case UDP_ENCAP_ESPINUDP: 26791da177e4SLinus Torvalds case UDP_ENCAP_ESPINUDP_NON_IKE: 26800146dca7SSabrina Dubroca #if IS_ENABLED(CONFIG_IPV6) 26810146dca7SSabrina Dubroca if (sk->sk_family == AF_INET6) 26820146dca7SSabrina Dubroca up->encap_rcv = ipv6_stub->xfrm6_udp_encap_rcv; 26830146dca7SSabrina Dubroca else 26840146dca7SSabrina Dubroca #endif 2685067b207bSJames Chapman up->encap_rcv = xfrm4_udp_encap_rcv; 2686fd1ac07fSAlexey Dobriyan #endif 2687a8eceea8SJoe Perches fallthrough; 2688342f0234SJames Chapman case UDP_ENCAP_L2TPINUDP: 26891da177e4SLinus Torvalds up->encap_type = val; 269060fb9567SPaolo Abeni lock_sock(sk); 269160fb9567SPaolo Abeni udp_tunnel_encap_enable(sk->sk_socket); 269260fb9567SPaolo Abeni release_sock(sk); 26931da177e4SLinus Torvalds break; 26941da177e4SLinus Torvalds default: 26951da177e4SLinus Torvalds err = -ENOPROTOOPT; 26961da177e4SLinus Torvalds break; 26971da177e4SLinus Torvalds } 26981da177e4SLinus Torvalds break; 26991da177e4SLinus Torvalds 27001c19448cSTom Herbert case UDP_NO_CHECK6_TX: 27011c19448cSTom Herbert up->no_check6_tx = valbool; 27021c19448cSTom Herbert break; 27031c19448cSTom Herbert 27041c19448cSTom Herbert case UDP_NO_CHECK6_RX: 27051c19448cSTom Herbert up->no_check6_rx = valbool; 27061c19448cSTom Herbert break; 27071c19448cSTom Herbert 2708bec1f6f6SWillem de Bruijn case UDP_SEGMENT: 2709bec1f6f6SWillem de Bruijn if (val < 0 || val > USHRT_MAX) 2710bec1f6f6SWillem de Bruijn return -EINVAL; 271118a419baSEric Dumazet WRITE_ONCE(up->gso_size, val); 2712bec1f6f6SWillem de Bruijn break; 2713bec1f6f6SWillem de Bruijn 2714e20cf8d3SPaolo Abeni case UDP_GRO: 2715e20cf8d3SPaolo Abeni lock_sock(sk); 271678352f73SPaolo Abeni 271778352f73SPaolo Abeni /* when enabling GRO, accept the related GSO packet type */ 2718e20cf8d3SPaolo Abeni if (valbool) 2719e20cf8d3SPaolo Abeni udp_tunnel_encap_enable(sk->sk_socket); 2720e20cf8d3SPaolo Abeni up->gro_enabled = valbool; 272178352f73SPaolo Abeni up->accept_udp_l4 = valbool; 2722e20cf8d3SPaolo Abeni release_sock(sk); 2723e20cf8d3SPaolo Abeni break; 2724e20cf8d3SPaolo Abeni 2725ba4e58ecSGerrit Renker /* 2726ba4e58ecSGerrit Renker * UDP-Lite's partial checksum coverage (RFC 3828). 2727ba4e58ecSGerrit Renker */ 2728ba4e58ecSGerrit Renker /* The sender sets actual checksum coverage length via this option. 2729ba4e58ecSGerrit Renker * The case coverage > packet length is handled by send module. */ 2730ba4e58ecSGerrit Renker case UDPLITE_SEND_CSCOV: 2731b2bf1e26SWang Chen if (!is_udplite) /* Disable the option on UDP sockets */ 2732ba4e58ecSGerrit Renker return -ENOPROTOOPT; 2733ba4e58ecSGerrit Renker if (val != 0 && val < 8) /* Illegal coverage: use default (8) */ 2734ba4e58ecSGerrit Renker val = 8; 27354be929beSAlexey Dobriyan else if (val > USHRT_MAX) 27364be929beSAlexey Dobriyan val = USHRT_MAX; 2737ba4e58ecSGerrit Renker up->pcslen = val; 2738ba4e58ecSGerrit Renker up->pcflag |= UDPLITE_SEND_CC; 2739ba4e58ecSGerrit Renker break; 2740ba4e58ecSGerrit Renker 2741ba4e58ecSGerrit Renker /* The receiver specifies a minimum checksum coverage value. To make 2742ba4e58ecSGerrit Renker * sense, this should be set to at least 8 (as done below). If zero is 2743ba4e58ecSGerrit Renker * used, this again means full checksum coverage. */ 2744ba4e58ecSGerrit Renker case UDPLITE_RECV_CSCOV: 2745b2bf1e26SWang Chen if (!is_udplite) /* Disable the option on UDP sockets */ 2746ba4e58ecSGerrit Renker return -ENOPROTOOPT; 2747ba4e58ecSGerrit Renker if (val != 0 && val < 8) /* Avoid silly minimal values. */ 2748ba4e58ecSGerrit Renker val = 8; 27494be929beSAlexey Dobriyan else if (val > USHRT_MAX) 27504be929beSAlexey Dobriyan val = USHRT_MAX; 2751ba4e58ecSGerrit Renker up->pcrlen = val; 2752ba4e58ecSGerrit Renker up->pcflag |= UDPLITE_RECV_CC; 2753ba4e58ecSGerrit Renker break; 2754ba4e58ecSGerrit Renker 27551da177e4SLinus Torvalds default: 27561da177e4SLinus Torvalds err = -ENOPROTOOPT; 27571da177e4SLinus Torvalds break; 27586516c655SStephen Hemminger } 27591da177e4SLinus Torvalds 27601da177e4SLinus Torvalds return err; 27611da177e4SLinus Torvalds } 2762c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_setsockopt); 27631da177e4SLinus Torvalds 2764a7b75c5aSChristoph Hellwig int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 2765a7b75c5aSChristoph Hellwig unsigned int optlen) 2766db8dac20SDavid S. Miller { 2767db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 276891ac1ccaSChristoph Hellwig return udp_lib_setsockopt(sk, level, optname, 2769a7b75c5aSChristoph Hellwig optval, optlen, 2770db8dac20SDavid S. Miller udp_push_pending_frames); 2771db8dac20SDavid S. Miller return ip_setsockopt(sk, level, optname, optval, optlen); 2772db8dac20SDavid S. Miller } 2773db8dac20SDavid S. Miller 27744c0a6cb0SGerrit Renker int udp_lib_getsockopt(struct sock *sk, int level, int optname, 27751da177e4SLinus Torvalds char __user *optval, int __user *optlen) 27761da177e4SLinus Torvalds { 27771da177e4SLinus Torvalds struct udp_sock *up = udp_sk(sk); 27781da177e4SLinus Torvalds int val, len; 27791da177e4SLinus Torvalds 27801da177e4SLinus Torvalds if (get_user(len, optlen)) 27811da177e4SLinus Torvalds return -EFAULT; 27821da177e4SLinus Torvalds 27831da177e4SLinus Torvalds len = min_t(unsigned int, len, sizeof(int)); 27841da177e4SLinus Torvalds 27851da177e4SLinus Torvalds if (len < 0) 27861da177e4SLinus Torvalds return -EINVAL; 27871da177e4SLinus Torvalds 27881da177e4SLinus Torvalds switch (optname) { 27891da177e4SLinus Torvalds case UDP_CORK: 2790a9f59707SEric Dumazet val = READ_ONCE(up->corkflag); 27911da177e4SLinus Torvalds break; 27921da177e4SLinus Torvalds 27931da177e4SLinus Torvalds case UDP_ENCAP: 27941da177e4SLinus Torvalds val = up->encap_type; 27951da177e4SLinus Torvalds break; 27961da177e4SLinus Torvalds 27971c19448cSTom Herbert case UDP_NO_CHECK6_TX: 27981c19448cSTom Herbert val = up->no_check6_tx; 27991c19448cSTom Herbert break; 28001c19448cSTom Herbert 28011c19448cSTom Herbert case UDP_NO_CHECK6_RX: 28021c19448cSTom Herbert val = up->no_check6_rx; 28031c19448cSTom Herbert break; 28041c19448cSTom Herbert 2805bec1f6f6SWillem de Bruijn case UDP_SEGMENT: 280618a419baSEric Dumazet val = READ_ONCE(up->gso_size); 2807bec1f6f6SWillem de Bruijn break; 2808bec1f6f6SWillem de Bruijn 280998184612SNorman Maurer case UDP_GRO: 281098184612SNorman Maurer val = up->gro_enabled; 281198184612SNorman Maurer break; 281298184612SNorman Maurer 2813ba4e58ecSGerrit Renker /* The following two cannot be changed on UDP sockets, the return is 2814ba4e58ecSGerrit Renker * always 0 (which corresponds to the full checksum coverage of UDP). */ 2815ba4e58ecSGerrit Renker case UDPLITE_SEND_CSCOV: 2816ba4e58ecSGerrit Renker val = up->pcslen; 2817ba4e58ecSGerrit Renker break; 2818ba4e58ecSGerrit Renker 2819ba4e58ecSGerrit Renker case UDPLITE_RECV_CSCOV: 2820ba4e58ecSGerrit Renker val = up->pcrlen; 2821ba4e58ecSGerrit Renker break; 2822ba4e58ecSGerrit Renker 28231da177e4SLinus Torvalds default: 28241da177e4SLinus Torvalds return -ENOPROTOOPT; 28256516c655SStephen Hemminger } 28261da177e4SLinus Torvalds 28271da177e4SLinus Torvalds if (put_user(len, optlen)) 28281da177e4SLinus Torvalds return -EFAULT; 28291da177e4SLinus Torvalds if (copy_to_user(optval, &val, len)) 28301da177e4SLinus Torvalds return -EFAULT; 28311da177e4SLinus Torvalds return 0; 28321da177e4SLinus Torvalds } 2833c482c568SEric Dumazet EXPORT_SYMBOL(udp_lib_getsockopt); 28341da177e4SLinus Torvalds 2835db8dac20SDavid S. Miller int udp_getsockopt(struct sock *sk, int level, int optname, 2836db8dac20SDavid S. Miller char __user *optval, int __user *optlen) 2837db8dac20SDavid S. Miller { 2838db8dac20SDavid S. Miller if (level == SOL_UDP || level == SOL_UDPLITE) 2839db8dac20SDavid S. Miller return udp_lib_getsockopt(sk, level, optname, optval, optlen); 2840db8dac20SDavid S. Miller return ip_getsockopt(sk, level, optname, optval, optlen); 2841db8dac20SDavid S. Miller } 2842db8dac20SDavid S. Miller 28431da177e4SLinus Torvalds /** 28441da177e4SLinus Torvalds * udp_poll - wait for a UDP event. 28453628e3cbSAndrew Lunn * @file: - file struct 28463628e3cbSAndrew Lunn * @sock: - socket 28473628e3cbSAndrew Lunn * @wait: - poll table 28481da177e4SLinus Torvalds * 28491da177e4SLinus Torvalds * This is same as datagram poll, except for the special case of 28501da177e4SLinus Torvalds * blocking sockets. If application is using a blocking fd 28511da177e4SLinus Torvalds * and a packet with checksum error is in the queue; 28521da177e4SLinus Torvalds * then it could get return from select indicating data available 28531da177e4SLinus Torvalds * but then block when reading it. Add special case code 28541da177e4SLinus Torvalds * to work around these arguably broken applications. 28551da177e4SLinus Torvalds */ 2856a11e1d43SLinus Torvalds __poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait) 28571da177e4SLinus Torvalds { 2858a11e1d43SLinus Torvalds __poll_t mask = datagram_poll(file, sock, wait); 28591da177e4SLinus Torvalds struct sock *sk = sock->sk; 28601da177e4SLinus Torvalds 28613ef7cf57SEric Dumazet if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 2862a9a08845SLinus Torvalds mask |= EPOLLIN | EPOLLRDNORM; 28632276f58aSPaolo Abeni 28641da177e4SLinus Torvalds /* Check for false positives due to checksum errors */ 2865a11e1d43SLinus Torvalds if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) && 2866e83c6744SEric Dumazet !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1) 2867a9a08845SLinus Torvalds mask &= ~(EPOLLIN | EPOLLRDNORM); 28681da177e4SLinus Torvalds 2869af493388SCong Wang /* psock ingress_msg queue should not contain any bad checksum frames */ 2870af493388SCong Wang if (sk_is_readable(sk)) 2871af493388SCong Wang mask |= EPOLLIN | EPOLLRDNORM; 28721da177e4SLinus Torvalds return mask; 28731da177e4SLinus Torvalds 28741da177e4SLinus Torvalds } 2875a11e1d43SLinus Torvalds EXPORT_SYMBOL(udp_poll); 28761da177e4SLinus Torvalds 28775d77dca8SDavid Ahern int udp_abort(struct sock *sk, int err) 28785d77dca8SDavid Ahern { 28795d77dca8SDavid Ahern lock_sock(sk); 28805d77dca8SDavid Ahern 2881a8b897c7SPaolo Abeni /* udp{v6}_destroy_sock() sets it under the sk lock, avoid racing 2882a8b897c7SPaolo Abeni * with close() 2883a8b897c7SPaolo Abeni */ 2884a8b897c7SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 2885a8b897c7SPaolo Abeni goto out; 2886a8b897c7SPaolo Abeni 28875d77dca8SDavid Ahern sk->sk_err = err; 2888e3ae2365SAlexander Aring sk_error_report(sk); 2889286c72deSEric Dumazet __udp_disconnect(sk, 0); 28905d77dca8SDavid Ahern 2891a8b897c7SPaolo Abeni out: 28925d77dca8SDavid Ahern release_sock(sk); 28935d77dca8SDavid Ahern 28945d77dca8SDavid Ahern return 0; 28955d77dca8SDavid Ahern } 28965d77dca8SDavid Ahern EXPORT_SYMBOL_GPL(udp_abort); 28975d77dca8SDavid Ahern 2898db8dac20SDavid S. Miller struct proto udp_prot = { 2899db8dac20SDavid S. Miller .name = "UDP", 2900db8dac20SDavid S. Miller .owner = THIS_MODULE, 2901db8dac20SDavid S. Miller .close = udp_lib_close, 2902d74bad4eSAndrey Ignatov .pre_connect = udp_pre_connect, 2903db8dac20SDavid S. Miller .connect = ip4_datagram_connect, 2904db8dac20SDavid S. Miller .disconnect = udp_disconnect, 2905db8dac20SDavid S. Miller .ioctl = udp_ioctl, 2906850cbaddSPaolo Abeni .init = udp_init_sock, 2907db8dac20SDavid S. Miller .destroy = udp_destroy_sock, 2908db8dac20SDavid S. Miller .setsockopt = udp_setsockopt, 2909db8dac20SDavid S. Miller .getsockopt = udp_getsockopt, 2910db8dac20SDavid S. Miller .sendmsg = udp_sendmsg, 2911db8dac20SDavid S. Miller .recvmsg = udp_recvmsg, 2912db8dac20SDavid S. Miller .sendpage = udp_sendpage, 29138141ed9fSSteffen Klassert .release_cb = ip4_datagram_release_cb, 2914db8dac20SDavid S. Miller .hash = udp_lib_hash, 2915db8dac20SDavid S. Miller .unhash = udp_lib_unhash, 2916719f8358SEric Dumazet .rehash = udp_v4_rehash, 2917db8dac20SDavid S. Miller .get_port = udp_v4_get_port, 29188a59f9d1SCong Wang #ifdef CONFIG_BPF_SYSCALL 29198a59f9d1SCong Wang .psock_update_sk_prot = udp_bpf_update_proto, 29208a59f9d1SCong Wang #endif 2921db8dac20SDavid S. Miller .memory_allocated = &udp_memory_allocated, 2922db8dac20SDavid S. Miller .sysctl_mem = sysctl_udp_mem, 29231e802951STonghao Zhang .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min), 29241e802951STonghao Zhang .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min), 2925db8dac20SDavid S. Miller .obj_size = sizeof(struct udp_sock), 2926645ca708SEric Dumazet .h.udp_table = &udp_table, 29275d77dca8SDavid Ahern .diag_destroy = udp_abort, 2928db8dac20SDavid S. Miller }; 2929c482c568SEric Dumazet EXPORT_SYMBOL(udp_prot); 29301da177e4SLinus Torvalds 29311da177e4SLinus Torvalds /* ------------------------------------------------------------------------ */ 29321da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS 29331da177e4SLinus Torvalds 2934645ca708SEric Dumazet static struct sock *udp_get_first(struct seq_file *seq, int start) 29351da177e4SLinus Torvalds { 29361da177e4SLinus Torvalds struct sock *sk; 29379e8ca27aSYonghong Song struct udp_seq_afinfo *afinfo; 29381da177e4SLinus Torvalds struct udp_iter_state *state = seq->private; 29396f191efeSDenis V. Lunev struct net *net = seq_file_net(seq); 29401da177e4SLinus Torvalds 29419e8ca27aSYonghong Song if (state->bpf_seq_afinfo) 29429e8ca27aSYonghong Song afinfo = state->bpf_seq_afinfo; 29439e8ca27aSYonghong Song else 29449e8ca27aSYonghong Song afinfo = PDE_DATA(file_inode(seq->file)); 29459e8ca27aSYonghong Song 2946a3d2599bSChristoph Hellwig for (state->bucket = start; state->bucket <= afinfo->udp_table->mask; 2947f86dcc5aSEric Dumazet ++state->bucket) { 2948a3d2599bSChristoph Hellwig struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket]; 2949f86dcc5aSEric Dumazet 2950ca065d0cSEric Dumazet if (hlist_empty(&hslot->head)) 2951f86dcc5aSEric Dumazet continue; 2952f86dcc5aSEric Dumazet 2953645ca708SEric Dumazet spin_lock_bh(&hslot->lock); 2954ca065d0cSEric Dumazet sk_for_each(sk, &hslot->head) { 2955878628fbSYOSHIFUJI Hideaki if (!net_eq(sock_net(sk), net)) 2956a91275efSDaniel Lezcano continue; 29579e8ca27aSYonghong Song if (afinfo->family == AF_UNSPEC || 29589e8ca27aSYonghong Song sk->sk_family == afinfo->family) 29591da177e4SLinus Torvalds goto found; 29601da177e4SLinus Torvalds } 2961645ca708SEric Dumazet spin_unlock_bh(&hslot->lock); 29621da177e4SLinus Torvalds } 29631da177e4SLinus Torvalds sk = NULL; 29641da177e4SLinus Torvalds found: 29651da177e4SLinus Torvalds return sk; 29661da177e4SLinus Torvalds } 29671da177e4SLinus Torvalds 29681da177e4SLinus Torvalds static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) 29691da177e4SLinus Torvalds { 29709e8ca27aSYonghong Song struct udp_seq_afinfo *afinfo; 29711da177e4SLinus Torvalds struct udp_iter_state *state = seq->private; 29726f191efeSDenis V. Lunev struct net *net = seq_file_net(seq); 29731da177e4SLinus Torvalds 29749e8ca27aSYonghong Song if (state->bpf_seq_afinfo) 29759e8ca27aSYonghong Song afinfo = state->bpf_seq_afinfo; 29769e8ca27aSYonghong Song else 29779e8ca27aSYonghong Song afinfo = PDE_DATA(file_inode(seq->file)); 29789e8ca27aSYonghong Song 29791da177e4SLinus Torvalds do { 2980ca065d0cSEric Dumazet sk = sk_next(sk); 29819e8ca27aSYonghong Song } while (sk && (!net_eq(sock_net(sk), net) || 29829e8ca27aSYonghong Song (afinfo->family != AF_UNSPEC && 29839e8ca27aSYonghong Song sk->sk_family != afinfo->family))); 29841da177e4SLinus Torvalds 2985645ca708SEric Dumazet if (!sk) { 2986a3d2599bSChristoph Hellwig if (state->bucket <= afinfo->udp_table->mask) 2987a3d2599bSChristoph Hellwig spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); 2988645ca708SEric Dumazet return udp_get_first(seq, state->bucket + 1); 29891da177e4SLinus Torvalds } 29901da177e4SLinus Torvalds return sk; 29911da177e4SLinus Torvalds } 29921da177e4SLinus Torvalds 29931da177e4SLinus Torvalds static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos) 29941da177e4SLinus Torvalds { 2995645ca708SEric Dumazet struct sock *sk = udp_get_first(seq, 0); 29961da177e4SLinus Torvalds 29971da177e4SLinus Torvalds if (sk) 29981da177e4SLinus Torvalds while (pos && (sk = udp_get_next(seq, sk)) != NULL) 29991da177e4SLinus Torvalds --pos; 30001da177e4SLinus Torvalds return pos ? NULL : sk; 30011da177e4SLinus Torvalds } 30021da177e4SLinus Torvalds 3003a3d2599bSChristoph Hellwig void *udp_seq_start(struct seq_file *seq, loff_t *pos) 30041da177e4SLinus Torvalds { 300530842f29SVitaly Mayatskikh struct udp_iter_state *state = seq->private; 3006f86dcc5aSEric Dumazet state->bucket = MAX_UDP_PORTS; 300730842f29SVitaly Mayatskikh 3008b50660f1SYOSHIFUJI Hideaki return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; 30091da177e4SLinus Torvalds } 3010a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_start); 30111da177e4SLinus Torvalds 3012a3d2599bSChristoph Hellwig void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos) 30131da177e4SLinus Torvalds { 30141da177e4SLinus Torvalds struct sock *sk; 30151da177e4SLinus Torvalds 3016b50660f1SYOSHIFUJI Hideaki if (v == SEQ_START_TOKEN) 30171da177e4SLinus Torvalds sk = udp_get_idx(seq, 0); 30181da177e4SLinus Torvalds else 30191da177e4SLinus Torvalds sk = udp_get_next(seq, v); 30201da177e4SLinus Torvalds 30211da177e4SLinus Torvalds ++*pos; 30221da177e4SLinus Torvalds return sk; 30231da177e4SLinus Torvalds } 3024a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_next); 30251da177e4SLinus Torvalds 3026a3d2599bSChristoph Hellwig void udp_seq_stop(struct seq_file *seq, void *v) 30271da177e4SLinus Torvalds { 30289e8ca27aSYonghong Song struct udp_seq_afinfo *afinfo; 3029645ca708SEric Dumazet struct udp_iter_state *state = seq->private; 3030645ca708SEric Dumazet 30319e8ca27aSYonghong Song if (state->bpf_seq_afinfo) 30329e8ca27aSYonghong Song afinfo = state->bpf_seq_afinfo; 30339e8ca27aSYonghong Song else 30349e8ca27aSYonghong Song afinfo = PDE_DATA(file_inode(seq->file)); 30359e8ca27aSYonghong Song 3036a3d2599bSChristoph Hellwig if (state->bucket <= afinfo->udp_table->mask) 3037a3d2599bSChristoph Hellwig spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); 30381da177e4SLinus Torvalds } 3039a3d2599bSChristoph Hellwig EXPORT_SYMBOL(udp_seq_stop); 3040db8dac20SDavid S. Miller 3041db8dac20SDavid S. Miller /* ------------------------------------------------------------------------ */ 30425e659e4cSPavel Emelyanov static void udp4_format_sock(struct sock *sp, struct seq_file *f, 3043652586dfSTetsuo Handa int bucket) 3044db8dac20SDavid S. Miller { 3045db8dac20SDavid S. Miller struct inet_sock *inet = inet_sk(sp); 3046c720c7e8SEric Dumazet __be32 dest = inet->inet_daddr; 3047c720c7e8SEric Dumazet __be32 src = inet->inet_rcv_saddr; 3048c720c7e8SEric Dumazet __u16 destp = ntohs(inet->inet_dport); 3049c720c7e8SEric Dumazet __u16 srcp = ntohs(inet->inet_sport); 3050db8dac20SDavid S. Miller 3051f86dcc5aSEric Dumazet seq_printf(f, "%5d: %08X:%04X %08X:%04X" 3052ea9a0379SPatrick Talbert " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u", 3053db8dac20SDavid S. Miller bucket, src, srcp, dest, destp, sp->sk_state, 305431e6d363SEric Dumazet sk_wmem_alloc_get(sp), 30556c206b20SPaolo Abeni udp_rqueue_get(sp), 3056a7cb5a49SEric W. Biederman 0, 0L, 0, 3057a7cb5a49SEric W. Biederman from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)), 3058a7cb5a49SEric W. Biederman 0, sock_i_ino(sp), 305941c6d650SReshetova, Elena refcount_read(&sp->sk_refcnt), sp, 3060652586dfSTetsuo Handa atomic_read(&sp->sk_drops)); 3061db8dac20SDavid S. Miller } 3062db8dac20SDavid S. Miller 3063db8dac20SDavid S. Miller int udp4_seq_show(struct seq_file *seq, void *v) 3064db8dac20SDavid S. Miller { 3065652586dfSTetsuo Handa seq_setwidth(seq, 127); 3066db8dac20SDavid S. Miller if (v == SEQ_START_TOKEN) 3067652586dfSTetsuo Handa seq_puts(seq, " sl local_address rem_address st tx_queue " 3068db8dac20SDavid S. Miller "rx_queue tr tm->when retrnsmt uid timeout " 3069cb61cb9bSEric Dumazet "inode ref pointer drops"); 3070db8dac20SDavid S. Miller else { 3071db8dac20SDavid S. Miller struct udp_iter_state *state = seq->private; 3072db8dac20SDavid S. Miller 3073652586dfSTetsuo Handa udp4_format_sock(v, seq, state->bucket); 3074db8dac20SDavid S. Miller } 3075652586dfSTetsuo Handa seq_pad(seq, '\n'); 3076db8dac20SDavid S. Miller return 0; 3077db8dac20SDavid S. Miller } 3078db8dac20SDavid S. Miller 30795788b3a0SYonghong Song #ifdef CONFIG_BPF_SYSCALL 30805788b3a0SYonghong Song struct bpf_iter__udp { 30815788b3a0SYonghong Song __bpf_md_ptr(struct bpf_iter_meta *, meta); 30825788b3a0SYonghong Song __bpf_md_ptr(struct udp_sock *, udp_sk); 30835788b3a0SYonghong Song uid_t uid __aligned(8); 30845788b3a0SYonghong Song int bucket __aligned(8); 30855788b3a0SYonghong Song }; 30865788b3a0SYonghong Song 30875788b3a0SYonghong Song static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta, 30885788b3a0SYonghong Song struct udp_sock *udp_sk, uid_t uid, int bucket) 30895788b3a0SYonghong Song { 30905788b3a0SYonghong Song struct bpf_iter__udp ctx; 30915788b3a0SYonghong Song 30925788b3a0SYonghong Song meta->seq_num--; /* skip SEQ_START_TOKEN */ 30935788b3a0SYonghong Song ctx.meta = meta; 30945788b3a0SYonghong Song ctx.udp_sk = udp_sk; 30955788b3a0SYonghong Song ctx.uid = uid; 30965788b3a0SYonghong Song ctx.bucket = bucket; 30975788b3a0SYonghong Song return bpf_iter_run_prog(prog, &ctx); 30985788b3a0SYonghong Song } 30995788b3a0SYonghong Song 31005788b3a0SYonghong Song static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v) 31015788b3a0SYonghong Song { 31025788b3a0SYonghong Song struct udp_iter_state *state = seq->private; 31035788b3a0SYonghong Song struct bpf_iter_meta meta; 31045788b3a0SYonghong Song struct bpf_prog *prog; 31055788b3a0SYonghong Song struct sock *sk = v; 31065788b3a0SYonghong Song uid_t uid; 31075788b3a0SYonghong Song 31085788b3a0SYonghong Song if (v == SEQ_START_TOKEN) 31095788b3a0SYonghong Song return 0; 31105788b3a0SYonghong Song 31115788b3a0SYonghong Song uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk)); 31125788b3a0SYonghong Song meta.seq = seq; 31135788b3a0SYonghong Song prog = bpf_iter_get_info(&meta, false); 31145788b3a0SYonghong Song return udp_prog_seq_show(prog, &meta, v, uid, state->bucket); 31155788b3a0SYonghong Song } 31165788b3a0SYonghong Song 31175788b3a0SYonghong Song static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v) 31185788b3a0SYonghong Song { 31195788b3a0SYonghong Song struct bpf_iter_meta meta; 31205788b3a0SYonghong Song struct bpf_prog *prog; 31215788b3a0SYonghong Song 31225788b3a0SYonghong Song if (!v) { 31235788b3a0SYonghong Song meta.seq = seq; 31245788b3a0SYonghong Song prog = bpf_iter_get_info(&meta, true); 31255788b3a0SYonghong Song if (prog) 31265788b3a0SYonghong Song (void)udp_prog_seq_show(prog, &meta, v, 0, 0); 31275788b3a0SYonghong Song } 31285788b3a0SYonghong Song 31295788b3a0SYonghong Song udp_seq_stop(seq, v); 31305788b3a0SYonghong Song } 31315788b3a0SYonghong Song 31325788b3a0SYonghong Song static const struct seq_operations bpf_iter_udp_seq_ops = { 31335788b3a0SYonghong Song .start = udp_seq_start, 31345788b3a0SYonghong Song .next = udp_seq_next, 31355788b3a0SYonghong Song .stop = bpf_iter_udp_seq_stop, 31365788b3a0SYonghong Song .show = bpf_iter_udp_seq_show, 31375788b3a0SYonghong Song }; 31385788b3a0SYonghong Song #endif 31395788b3a0SYonghong Song 3140c3506372SChristoph Hellwig const struct seq_operations udp_seq_ops = { 3141a3d2599bSChristoph Hellwig .start = udp_seq_start, 3142a3d2599bSChristoph Hellwig .next = udp_seq_next, 3143a3d2599bSChristoph Hellwig .stop = udp_seq_stop, 3144a3d2599bSChristoph Hellwig .show = udp4_seq_show, 3145a3d2599bSChristoph Hellwig }; 3146c3506372SChristoph Hellwig EXPORT_SYMBOL(udp_seq_ops); 314773cb88ecSArjan van de Ven 3148db8dac20SDavid S. Miller static struct udp_seq_afinfo udp4_seq_afinfo = { 3149db8dac20SDavid S. Miller .family = AF_INET, 3150645ca708SEric Dumazet .udp_table = &udp_table, 3151db8dac20SDavid S. Miller }; 3152db8dac20SDavid S. Miller 31532c8c1e72SAlexey Dobriyan static int __net_init udp4_proc_init_net(struct net *net) 315415439febSPavel Emelyanov { 3155c3506372SChristoph Hellwig if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops, 3156c3506372SChristoph Hellwig sizeof(struct udp_iter_state), &udp4_seq_afinfo)) 3157a3d2599bSChristoph Hellwig return -ENOMEM; 3158a3d2599bSChristoph Hellwig return 0; 315915439febSPavel Emelyanov } 316015439febSPavel Emelyanov 31612c8c1e72SAlexey Dobriyan static void __net_exit udp4_proc_exit_net(struct net *net) 316215439febSPavel Emelyanov { 3163a3d2599bSChristoph Hellwig remove_proc_entry("udp", net->proc_net); 316415439febSPavel Emelyanov } 316515439febSPavel Emelyanov 316615439febSPavel Emelyanov static struct pernet_operations udp4_net_ops = { 316715439febSPavel Emelyanov .init = udp4_proc_init_net, 316815439febSPavel Emelyanov .exit = udp4_proc_exit_net, 316915439febSPavel Emelyanov }; 317015439febSPavel Emelyanov 3171db8dac20SDavid S. Miller int __init udp4_proc_init(void) 3172db8dac20SDavid S. Miller { 317315439febSPavel Emelyanov return register_pernet_subsys(&udp4_net_ops); 3174db8dac20SDavid S. Miller } 3175db8dac20SDavid S. Miller 3176db8dac20SDavid S. Miller void udp4_proc_exit(void) 3177db8dac20SDavid S. Miller { 317815439febSPavel Emelyanov unregister_pernet_subsys(&udp4_net_ops); 3179db8dac20SDavid S. Miller } 31801da177e4SLinus Torvalds #endif /* CONFIG_PROC_FS */ 31811da177e4SLinus Torvalds 3182f86dcc5aSEric Dumazet static __initdata unsigned long uhash_entries; 3183f86dcc5aSEric Dumazet static int __init set_uhash_entries(char *str) 3184645ca708SEric Dumazet { 3185413c27d8SEldad Zack ssize_t ret; 3186413c27d8SEldad Zack 3187f86dcc5aSEric Dumazet if (!str) 3188f86dcc5aSEric Dumazet return 0; 3189413c27d8SEldad Zack 3190413c27d8SEldad Zack ret = kstrtoul(str, 0, &uhash_entries); 3191413c27d8SEldad Zack if (ret) 3192413c27d8SEldad Zack return 0; 3193413c27d8SEldad Zack 3194f86dcc5aSEric Dumazet if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN) 3195f86dcc5aSEric Dumazet uhash_entries = UDP_HTABLE_SIZE_MIN; 3196f86dcc5aSEric Dumazet return 1; 3197f86dcc5aSEric Dumazet } 3198f86dcc5aSEric Dumazet __setup("uhash_entries=", set_uhash_entries); 3199645ca708SEric Dumazet 3200f86dcc5aSEric Dumazet void __init udp_table_init(struct udp_table *table, const char *name) 3201f86dcc5aSEric Dumazet { 3202f86dcc5aSEric Dumazet unsigned int i; 3203f86dcc5aSEric Dumazet 3204f86dcc5aSEric Dumazet table->hash = alloc_large_system_hash(name, 3205512615b6SEric Dumazet 2 * sizeof(struct udp_hslot), 3206f86dcc5aSEric Dumazet uhash_entries, 3207f86dcc5aSEric Dumazet 21, /* one slot per 2 MB */ 3208f86dcc5aSEric Dumazet 0, 3209f86dcc5aSEric Dumazet &table->log, 3210f86dcc5aSEric Dumazet &table->mask, 321131fe62b9STim Bird UDP_HTABLE_SIZE_MIN, 3212f86dcc5aSEric Dumazet 64 * 1024); 321331fe62b9STim Bird 3214512615b6SEric Dumazet table->hash2 = table->hash + (table->mask + 1); 3215f86dcc5aSEric Dumazet for (i = 0; i <= table->mask; i++) { 3216ca065d0cSEric Dumazet INIT_HLIST_HEAD(&table->hash[i].head); 3217fdcc8aa9SEric Dumazet table->hash[i].count = 0; 3218645ca708SEric Dumazet spin_lock_init(&table->hash[i].lock); 3219645ca708SEric Dumazet } 3220512615b6SEric Dumazet for (i = 0; i <= table->mask; i++) { 3221ca065d0cSEric Dumazet INIT_HLIST_HEAD(&table->hash2[i].head); 3222512615b6SEric Dumazet table->hash2[i].count = 0; 3223512615b6SEric Dumazet spin_lock_init(&table->hash2[i].lock); 3224512615b6SEric Dumazet } 3225645ca708SEric Dumazet } 3226645ca708SEric Dumazet 3227723b8e46STom Herbert u32 udp_flow_hashrnd(void) 3228723b8e46STom Herbert { 3229723b8e46STom Herbert static u32 hashrnd __read_mostly; 3230723b8e46STom Herbert 3231723b8e46STom Herbert net_get_random_once(&hashrnd, sizeof(hashrnd)); 3232723b8e46STom Herbert 3233723b8e46STom Herbert return hashrnd; 3234723b8e46STom Herbert } 3235723b8e46STom Herbert EXPORT_SYMBOL(udp_flow_hashrnd); 3236723b8e46STom Herbert 32371e802951STonghao Zhang static void __udp_sysctl_init(struct net *net) 32381e802951STonghao Zhang { 32391e802951STonghao Zhang net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM; 32401e802951STonghao Zhang net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM; 32411e802951STonghao Zhang 32421e802951STonghao Zhang #ifdef CONFIG_NET_L3_MASTER_DEV 32431e802951STonghao Zhang net->ipv4.sysctl_udp_l3mdev_accept = 0; 32441e802951STonghao Zhang #endif 32451e802951STonghao Zhang } 32461e802951STonghao Zhang 32471e802951STonghao Zhang static int __net_init udp_sysctl_init(struct net *net) 32481e802951STonghao Zhang { 32491e802951STonghao Zhang __udp_sysctl_init(net); 32501e802951STonghao Zhang return 0; 32511e802951STonghao Zhang } 32521e802951STonghao Zhang 32531e802951STonghao Zhang static struct pernet_operations __net_initdata udp_sysctl_ops = { 32541e802951STonghao Zhang .init = udp_sysctl_init, 32551e802951STonghao Zhang }; 32561e802951STonghao Zhang 32575788b3a0SYonghong Song #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 32585788b3a0SYonghong Song DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta, 32595788b3a0SYonghong Song struct udp_sock *udp_sk, uid_t uid, int bucket) 32605788b3a0SYonghong Song 3261f9c79272SYonghong Song static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux) 32625788b3a0SYonghong Song { 32635788b3a0SYonghong Song struct udp_iter_state *st = priv_data; 32645788b3a0SYonghong Song struct udp_seq_afinfo *afinfo; 32655788b3a0SYonghong Song int ret; 32665788b3a0SYonghong Song 32675788b3a0SYonghong Song afinfo = kmalloc(sizeof(*afinfo), GFP_USER | __GFP_NOWARN); 32685788b3a0SYonghong Song if (!afinfo) 32695788b3a0SYonghong Song return -ENOMEM; 32705788b3a0SYonghong Song 32715788b3a0SYonghong Song afinfo->family = AF_UNSPEC; 32725788b3a0SYonghong Song afinfo->udp_table = &udp_table; 32735788b3a0SYonghong Song st->bpf_seq_afinfo = afinfo; 3274f9c79272SYonghong Song ret = bpf_iter_init_seq_net(priv_data, aux); 32755788b3a0SYonghong Song if (ret) 32765788b3a0SYonghong Song kfree(afinfo); 32775788b3a0SYonghong Song return ret; 32785788b3a0SYonghong Song } 32795788b3a0SYonghong Song 32805788b3a0SYonghong Song static void bpf_iter_fini_udp(void *priv_data) 32815788b3a0SYonghong Song { 32825788b3a0SYonghong Song struct udp_iter_state *st = priv_data; 32835788b3a0SYonghong Song 32845788b3a0SYonghong Song kfree(st->bpf_seq_afinfo); 32855788b3a0SYonghong Song bpf_iter_fini_seq_net(priv_data); 32865788b3a0SYonghong Song } 32875788b3a0SYonghong Song 328814fc6bd6SYonghong Song static const struct bpf_iter_seq_info udp_seq_info = { 32895788b3a0SYonghong Song .seq_ops = &bpf_iter_udp_seq_ops, 32905788b3a0SYonghong Song .init_seq_private = bpf_iter_init_udp, 32915788b3a0SYonghong Song .fini_seq_private = bpf_iter_fini_udp, 32925788b3a0SYonghong Song .seq_priv_size = sizeof(struct udp_iter_state), 329314fc6bd6SYonghong Song }; 329414fc6bd6SYonghong Song 329514fc6bd6SYonghong Song static struct bpf_iter_reg udp_reg_info = { 329614fc6bd6SYonghong Song .target = "udp", 32975788b3a0SYonghong Song .ctx_arg_info_size = 1, 32985788b3a0SYonghong Song .ctx_arg_info = { 32995788b3a0SYonghong Song { offsetof(struct bpf_iter__udp, udp_sk), 33005788b3a0SYonghong Song PTR_TO_BTF_ID_OR_NULL }, 33015788b3a0SYonghong Song }, 330214fc6bd6SYonghong Song .seq_info = &udp_seq_info, 33035788b3a0SYonghong Song }; 33045788b3a0SYonghong Song 33055788b3a0SYonghong Song static void __init bpf_iter_register(void) 33065788b3a0SYonghong Song { 3307951cf368SYonghong Song udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP]; 33085788b3a0SYonghong Song if (bpf_iter_reg_target(&udp_reg_info)) 33095788b3a0SYonghong Song pr_warn("Warning: could not register bpf iterator udp\n"); 33105788b3a0SYonghong Song } 33115788b3a0SYonghong Song #endif 33125788b3a0SYonghong Song 331395766fffSHideo Aoki void __init udp_init(void) 331495766fffSHideo Aoki { 3315f03d78dbSEric Dumazet unsigned long limit; 33164b272750SEric Dumazet unsigned int i; 331795766fffSHideo Aoki 3318f86dcc5aSEric Dumazet udp_table_init(&udp_table, "UDP"); 3319f03d78dbSEric Dumazet limit = nr_free_buffer_pages() / 8; 332095766fffSHideo Aoki limit = max(limit, 128UL); 332195766fffSHideo Aoki sysctl_udp_mem[0] = limit / 4 * 3; 332295766fffSHideo Aoki sysctl_udp_mem[1] = limit; 332395766fffSHideo Aoki sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2; 332495766fffSHideo Aoki 33251e802951STonghao Zhang __udp_sysctl_init(&init_net); 33264b272750SEric Dumazet 33274b272750SEric Dumazet /* 16 spinlocks per cpu */ 33284b272750SEric Dumazet udp_busylocks_log = ilog2(nr_cpu_ids) + 4; 33294b272750SEric Dumazet udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log, 33304b272750SEric Dumazet GFP_KERNEL); 33314b272750SEric Dumazet if (!udp_busylocks) 33324b272750SEric Dumazet panic("UDP: failed to alloc udp_busylocks\n"); 33334b272750SEric Dumazet for (i = 0; i < (1U << udp_busylocks_log); i++) 33344b272750SEric Dumazet spin_lock_init(udp_busylocks + i); 33351e802951STonghao Zhang 33361e802951STonghao Zhang if (register_pernet_subsys(&udp_sysctl_ops)) 33371e802951STonghao Zhang panic("UDP: failed to init sysctl parameters.\n"); 33385788b3a0SYonghong Song 33395788b3a0SYonghong Song #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 33405788b3a0SYonghong Song bpf_iter_register(); 33415788b3a0SYonghong Song #endif 334295766fffSHideo Aoki } 3343