1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the UDP protocol.
8 *
9 * Version: @(#)udp.h 1.0.2 04/28/93
10 *
11 * Author: Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 */
13 #ifndef _LINUX_UDP_H
14 #define _LINUX_UDP_H
15
16 #include <net/inet_sock.h>
17 #include <linux/skbuff.h>
18 #include <net/netns/hash.h>
19 #include <uapi/linux/udp.h>
20
udp_hdr(const struct sk_buff * skb)21 static inline struct udphdr *udp_hdr(const struct sk_buff *skb)
22 {
23 return (struct udphdr *)skb_transport_header(skb);
24 }
25
udp_get_len(const struct sk_buff * skb,const struct udphdr * uh,unsigned int dataoff)26 static inline unsigned int udp_get_len(const struct sk_buff *skb,
27 const struct udphdr *uh,
28 unsigned int dataoff)
29 {
30 if (uh->len)
31 return ntohs(uh->len);
32 if (skb_is_gso(skb)) /* BIG TCP */
33 return skb->len - dataoff;
34 return 0;
35 }
36
udp_get_len_short(const struct udphdr * uh)37 static inline unsigned int udp_get_len_short(const struct udphdr *uh)
38 {
39 return ntohs(uh->len);
40 }
41
udp_set_len(struct udphdr * uh,unsigned int len)42 static inline void udp_set_len(struct udphdr *uh, unsigned int len)
43 {
44 uh->len = len < GRO_LEGACY_MAX_SIZE ? htons(len) : 0;
45 }
46
udp_set_len_short(struct udphdr * uh,unsigned int len)47 static inline void udp_set_len_short(struct udphdr *uh, unsigned int len)
48 {
49 DEBUG_NET_WARN_ON_ONCE(len >= GRO_LEGACY_MAX_SIZE);
50 uh->len = htons(len);
51 }
52
53 #define UDP_HTABLE_SIZE_MIN_PERNET 128
54 #define UDP_HTABLE_SIZE_MIN (IS_ENABLED(CONFIG_BASE_SMALL) ? 128 : 256)
55 #define UDP_HTABLE_SIZE_MAX 65536
56
udp_hashfn(const struct net * net,u32 num,u32 mask)57 static inline u32 udp_hashfn(const struct net *net, u32 num, u32 mask)
58 {
59 return (num + net_hash_mix(net)) & mask;
60 }
61
62 enum {
63 UDP_FLAGS_CORK, /* Cork is required */
64 UDP_FLAGS_NO_CHECK6_TX, /* Send zero UDP6 checksums on TX? */
65 UDP_FLAGS_NO_CHECK6_RX, /* Allow zero UDP6 checksums on RX? */
66 UDP_FLAGS_GRO_ENABLED, /* Request GRO aggregation */
67 UDP_FLAGS_ACCEPT_FRAGLIST,
68 UDP_FLAGS_ACCEPT_L4,
69 UDP_FLAGS_ENCAP_ENABLED, /* This socket enabled encap */
70 };
71
72 /* per NUMA structure for lockless producer usage. */
73 struct udp_prod_queue {
74 struct llist_head ll_root ____cacheline_aligned_in_smp;
75 atomic_t rmem_alloc;
76 };
77
78 struct udp_sock {
79 /* inet_sock has to be the first member */
80 struct inet_sock inet;
81 #define udp_port_hash inet.sk.__sk_common.skc_u16hashes[0]
82 #define udp_portaddr_hash inet.sk.__sk_common.skc_u16hashes[1]
83 #define udp_portaddr_node inet.sk.__sk_common.skc_portaddr_node
84
85 unsigned long udp_flags;
86
87 int pending; /* Any pending frames ? */
88 __u8 encap_type; /* Is this an Encapsulation socket? */
89
90 #if !IS_ENABLED(CONFIG_BASE_SMALL)
91 /* For UDP 4-tuple hash */
92 __u16 udp_lrpa_hash;
93 struct hlist_nulls_node udp_lrpa_node;
94 #endif
95
96 /*
97 * Following member retains the information to create a UDP header
98 * when the socket is uncorked.
99 */
100 __u16 len; /* total length of pending frames */
101 __u16 gso_size;
102
103 /*
104 * For encapsulation sockets.
105 */
106 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
107 void (*encap_err_rcv)(struct sock *sk, struct sk_buff *skb, int err,
108 __be16 port, u32 info, u8 *payload);
109 int (*encap_err_lookup)(struct sock *sk, struct sk_buff *skb);
110 void (*encap_destroy)(struct sock *sk);
111
112 /* GRO functions for UDP socket */
113 struct sk_buff * (*gro_receive)(struct sock *sk,
114 struct list_head *head,
115 struct sk_buff *skb);
116 int (*gro_complete)(struct sock *sk,
117 struct sk_buff *skb,
118 int nhoff);
119
120 struct udp_prod_queue *udp_prod_queue;
121
122 /* udp_recvmsg try to use this before splicing sk_receive_queue */
123 struct sk_buff_head reader_queue ____cacheline_aligned_in_smp;
124
125 /* This field is dirtied by udp_recvmsg() */
126 int forward_deficit;
127
128 /* This fields follows rcvbuf value, and is touched by udp_recvmsg */
129 int forward_threshold;
130
131 /* Cache friendly copy of sk->sk_peek_off >= 0 */
132 bool peeking_with_offset;
133
134 /*
135 * Accounting for the tunnel GRO fastpath.
136 * Unprotected by compilers guard, as it uses space available in
137 * the last UDP socket cacheline.
138 */
139 struct hlist_node tunnel_list;
140 struct numa_drop_counters drop_counters;
141 };
142
143 #define udp_test_bit(nr, sk) \
144 test_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
145 #define udp_set_bit(nr, sk) \
146 set_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
147 #define udp_test_and_set_bit(nr, sk) \
148 test_and_set_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
149 #define udp_clear_bit(nr, sk) \
150 clear_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
151 #define udp_assign_bit(nr, sk, val) \
152 assign_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags, val)
153
154 #define UDP_MAX_SEGMENTS (1 << 7UL)
155
156 #define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk)
157
udp_set_peek_off(struct sock * sk,int val)158 static inline int udp_set_peek_off(struct sock *sk, int val)
159 {
160 sk_set_peek_off(sk, val);
161 WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0);
162 return 0;
163 }
164
udp_set_no_check6_tx(struct sock * sk,bool val)165 static inline void udp_set_no_check6_tx(struct sock *sk, bool val)
166 {
167 udp_assign_bit(NO_CHECK6_TX, sk, val);
168 }
169
udp_set_no_check6_rx(struct sock * sk,bool val)170 static inline void udp_set_no_check6_rx(struct sock *sk, bool val)
171 {
172 udp_assign_bit(NO_CHECK6_RX, sk, val);
173 }
174
udp_get_no_check6_tx(const struct sock * sk)175 static inline bool udp_get_no_check6_tx(const struct sock *sk)
176 {
177 return udp_test_bit(NO_CHECK6_TX, sk);
178 }
179
udp_get_no_check6_rx(const struct sock * sk)180 static inline bool udp_get_no_check6_rx(const struct sock *sk)
181 {
182 return udp_test_bit(NO_CHECK6_RX, sk);
183 }
184
udp_cmsg_recv(struct msghdr * msg,struct sock * sk,struct sk_buff * skb)185 static inline void udp_cmsg_recv(struct msghdr *msg, struct sock *sk,
186 struct sk_buff *skb)
187 {
188 int gso_size;
189
190 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
191 gso_size = skb_shinfo(skb)->gso_size;
192 put_cmsg(msg, SOL_UDP, UDP_GRO, sizeof(gso_size), &gso_size);
193 }
194 }
195
196 DECLARE_STATIC_KEY_FALSE(udp_encap_needed_key);
197 #if IS_ENABLED(CONFIG_IPV6)
198 DECLARE_STATIC_KEY_FALSE(udpv6_encap_needed_key);
199 #endif
200
udp_encap_needed(void)201 static inline bool udp_encap_needed(void)
202 {
203 if (static_branch_unlikely(&udp_encap_needed_key))
204 return true;
205
206 #if IS_ENABLED(CONFIG_IPV6)
207 if (static_branch_unlikely(&udpv6_encap_needed_key))
208 return true;
209 #endif
210
211 return false;
212 }
213
udp_unexpected_gso(struct sock * sk,struct sk_buff * skb)214 static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb)
215 {
216 if (!skb_is_gso(skb))
217 return false;
218
219 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 &&
220 !udp_test_bit(ACCEPT_L4, sk))
221 return true;
222
223 if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST &&
224 !udp_test_bit(ACCEPT_FRAGLIST, sk))
225 return true;
226
227 /* GSO packets lacking the SKB_GSO_UDP_TUNNEL/_CSUM bits might still
228 * land in a tunnel as the socket check in udp_gro_receive cannot be
229 * foolproof.
230 */
231 if (udp_encap_needed() &&
232 READ_ONCE(udp_sk(sk)->encap_rcv) &&
233 !(skb_shinfo(skb)->gso_type &
234 (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)))
235 return true;
236
237 return false;
238 }
239
udp_allow_gso(struct sock * sk)240 static inline void udp_allow_gso(struct sock *sk)
241 {
242 udp_set_bit(ACCEPT_L4, sk);
243 udp_set_bit(ACCEPT_FRAGLIST, sk);
244 }
245
246 #define udp_portaddr_for_each_entry(__sk, list) \
247 hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node)
248
249 #define udp_portaddr_for_each_entry_from(__sk) \
250 hlist_for_each_entry_from(__sk, __sk_common.skc_portaddr_node)
251
252 #define udp_portaddr_for_each_entry_rcu(__sk, list) \
253 hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node)
254
255 #if !IS_ENABLED(CONFIG_BASE_SMALL)
256 #define udp_lrpa_for_each_entry_rcu(__up, node, list) \
257 hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
258 #endif
259
udp_tunnel_sk(const struct net * net,bool is_ipv6)260 static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
261 {
262 #if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
263 return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);
264 #else
265 return NULL;
266 #endif
267 }
268
269 #endif /* _LINUX_UDP_H */
270