1edc6741cSLorenz Bauer // SPDX-License-Identifier: GPL-2.0
2edc6741cSLorenz Bauer /* Copyright (c) 2020 Cloudflare Ltd https://cloudflare.com */
3edc6741cSLorenz Bauer
4edc6741cSLorenz Bauer #include <linux/skmsg.h>
5edc6741cSLorenz Bauer #include <net/sock.h>
6edc6741cSLorenz Bauer #include <net/udp.h>
71f5be6b3SCong Wang #include <net/inet_common.h>
81f5be6b3SCong Wang
91f5be6b3SCong Wang #include "udp_impl.h"
101f5be6b3SCong Wang
111f5be6b3SCong Wang static struct proto *udpv6_prot_saved __read_mostly;
121f5be6b3SCong Wang
sk_udp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int flags,int * addr_len)131f5be6b3SCong Wang static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
14ec095263SOliver Hartkopp int flags, int *addr_len)
151f5be6b3SCong Wang {
161f5be6b3SCong Wang #if IS_ENABLED(CONFIG_IPV6)
171f5be6b3SCong Wang if (sk->sk_family == AF_INET6)
18ec095263SOliver Hartkopp return udpv6_prot_saved->recvmsg(sk, msg, len, flags, addr_len);
191f5be6b3SCong Wang #endif
20ec095263SOliver Hartkopp return udp_prot.recvmsg(sk, msg, len, flags, addr_len);
211f5be6b3SCong Wang }
221f5be6b3SCong Wang
udp_sk_has_data(struct sock * sk)239f2470fbSCong Wang static bool udp_sk_has_data(struct sock *sk)
249f2470fbSCong Wang {
259f2470fbSCong Wang return !skb_queue_empty(&udp_sk(sk)->reader_queue) ||
269f2470fbSCong Wang !skb_queue_empty(&sk->sk_receive_queue);
279f2470fbSCong Wang }
289f2470fbSCong Wang
psock_has_data(struct sk_psock * psock)299f2470fbSCong Wang static bool psock_has_data(struct sk_psock *psock)
309f2470fbSCong Wang {
319f2470fbSCong Wang return !skb_queue_empty(&psock->ingress_skb) ||
329f2470fbSCong Wang !sk_psock_queue_empty(psock);
339f2470fbSCong Wang }
349f2470fbSCong Wang
359f2470fbSCong Wang #define udp_msg_has_data(__sk, __psock) \
369f2470fbSCong Wang ({ udp_sk_has_data(__sk) || psock_has_data(__psock); })
379f2470fbSCong Wang
udp_msg_wait_data(struct sock * sk,struct sk_psock * psock,long timeo)38b6df0078SJakub Kicinski static int udp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
39b6df0078SJakub Kicinski long timeo)
409f2470fbSCong Wang {
419f2470fbSCong Wang DEFINE_WAIT_FUNC(wait, woken_wake_function);
429f2470fbSCong Wang int ret = 0;
439f2470fbSCong Wang
449f2470fbSCong Wang if (sk->sk_shutdown & RCV_SHUTDOWN)
459f2470fbSCong Wang return 1;
469f2470fbSCong Wang
479f2470fbSCong Wang if (!timeo)
489f2470fbSCong Wang return ret;
499f2470fbSCong Wang
509f2470fbSCong Wang add_wait_queue(sk_sleep(sk), &wait);
519f2470fbSCong Wang sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
529f2470fbSCong Wang ret = udp_msg_has_data(sk, psock);
539f2470fbSCong Wang if (!ret) {
549f2470fbSCong Wang wait_woken(&wait, TASK_INTERRUPTIBLE, timeo);
559f2470fbSCong Wang ret = udp_msg_has_data(sk, psock);
569f2470fbSCong Wang }
579f2470fbSCong Wang sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
589f2470fbSCong Wang remove_wait_queue(sk_sleep(sk), &wait);
599f2470fbSCong Wang return ret;
609f2470fbSCong Wang }
619f2470fbSCong Wang
udp_bpf_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int flags,int * addr_len)621f5be6b3SCong Wang static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
63ec095263SOliver Hartkopp int flags, int *addr_len)
641f5be6b3SCong Wang {
651f5be6b3SCong Wang struct sk_psock *psock;
661f5be6b3SCong Wang int copied, ret;
671f5be6b3SCong Wang
681f5be6b3SCong Wang if (unlikely(flags & MSG_ERRQUEUE))
691f5be6b3SCong Wang return inet_recv_error(sk, msg, len, addr_len);
701f5be6b3SCong Wang
71*d900f3d2SLiu Jian if (!len)
72*d900f3d2SLiu Jian return 0;
73*d900f3d2SLiu Jian
741f5be6b3SCong Wang psock = sk_psock_get(sk);
751f5be6b3SCong Wang if (unlikely(!psock))
76ec095263SOliver Hartkopp return sk_udp_recvmsg(sk, msg, len, flags, addr_len);
771f5be6b3SCong Wang
789f2470fbSCong Wang if (!psock_has_data(psock)) {
79ec095263SOliver Hartkopp ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
801f5be6b3SCong Wang goto out;
811f5be6b3SCong Wang }
821f5be6b3SCong Wang
831f5be6b3SCong Wang msg_bytes_ready:
841f5be6b3SCong Wang copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
851f5be6b3SCong Wang if (!copied) {
861f5be6b3SCong Wang long timeo;
87c49661aaSCong Wang int data;
881f5be6b3SCong Wang
89ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
90b6df0078SJakub Kicinski data = udp_msg_wait_data(sk, psock, timeo);
911f5be6b3SCong Wang if (data) {
929f2470fbSCong Wang if (psock_has_data(psock))
931f5be6b3SCong Wang goto msg_bytes_ready;
94ec095263SOliver Hartkopp ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
951f5be6b3SCong Wang goto out;
961f5be6b3SCong Wang }
971f5be6b3SCong Wang copied = -EAGAIN;
981f5be6b3SCong Wang }
991f5be6b3SCong Wang ret = copied;
1001f5be6b3SCong Wang out:
1011f5be6b3SCong Wang sk_psock_put(sk, psock);
1021f5be6b3SCong Wang return ret;
1031f5be6b3SCong Wang }
104edc6741cSLorenz Bauer
105edc6741cSLorenz Bauer enum {
106edc6741cSLorenz Bauer UDP_BPF_IPV4,
107edc6741cSLorenz Bauer UDP_BPF_IPV6,
108edc6741cSLorenz Bauer UDP_BPF_NUM_PROTS,
109edc6741cSLorenz Bauer };
110edc6741cSLorenz Bauer
111edc6741cSLorenz Bauer static DEFINE_SPINLOCK(udpv6_prot_lock);
112edc6741cSLorenz Bauer static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
113edc6741cSLorenz Bauer
udp_bpf_rebuild_protos(struct proto * prot,const struct proto * base)114edc6741cSLorenz Bauer static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base)
115edc6741cSLorenz Bauer {
116edc6741cSLorenz Bauer *prot = *base;
117edc6741cSLorenz Bauer prot->close = sock_map_close;
1181f5be6b3SCong Wang prot->recvmsg = udp_bpf_recvmsg;
119af493388SCong Wang prot->sock_is_readable = sk_msg_is_readable;
120edc6741cSLorenz Bauer }
121edc6741cSLorenz Bauer
udp_bpf_check_v6_needs_rebuild(struct proto * ops)1227b219da4SLorenz Bauer static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
123edc6741cSLorenz Bauer {
1247b219da4SLorenz Bauer if (unlikely(ops != smp_load_acquire(&udpv6_prot_saved))) {
125edc6741cSLorenz Bauer spin_lock_bh(&udpv6_prot_lock);
126edc6741cSLorenz Bauer if (likely(ops != udpv6_prot_saved)) {
127edc6741cSLorenz Bauer udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV6], ops);
128edc6741cSLorenz Bauer smp_store_release(&udpv6_prot_saved, ops);
129edc6741cSLorenz Bauer }
130edc6741cSLorenz Bauer spin_unlock_bh(&udpv6_prot_lock);
131edc6741cSLorenz Bauer }
132edc6741cSLorenz Bauer }
133edc6741cSLorenz Bauer
udp_bpf_v4_build_proto(void)134edc6741cSLorenz Bauer static int __init udp_bpf_v4_build_proto(void)
135edc6741cSLorenz Bauer {
136edc6741cSLorenz Bauer udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
137edc6741cSLorenz Bauer return 0;
138edc6741cSLorenz Bauer }
13954ea2f49SJakub Sitnicki late_initcall(udp_bpf_v4_build_proto);
140edc6741cSLorenz Bauer
udp_bpf_update_proto(struct sock * sk,struct sk_psock * psock,bool restore)14151e0158aSCong Wang int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
142edc6741cSLorenz Bauer {
143edc6741cSLorenz Bauer int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
1448a59f9d1SCong Wang
1458a59f9d1SCong Wang if (restore) {
1468a59f9d1SCong Wang sk->sk_write_space = psock->saved_write_space;
147fee9ac06SPavel Begunkov sock_replace_proto(sk, psock->sk_proto);
1488a59f9d1SCong Wang return 0;
1498a59f9d1SCong Wang }
150edc6741cSLorenz Bauer
1517b219da4SLorenz Bauer if (sk->sk_family == AF_INET6)
1527b219da4SLorenz Bauer udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
153edc6741cSLorenz Bauer
154fee9ac06SPavel Begunkov sock_replace_proto(sk, &udp_bpf_prots[family]);
1558a59f9d1SCong Wang return 0;
156edc6741cSLorenz Bauer }
1578a59f9d1SCong Wang EXPORT_SYMBOL_GPL(udp_bpf_update_proto);
158