1604326b4SDaniel Borkmann // SPDX-License-Identifier: GPL-2.0
2604326b4SDaniel Borkmann /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3604326b4SDaniel Borkmann
4604326b4SDaniel Borkmann #include <linux/skmsg.h>
5604326b4SDaniel Borkmann #include <linux/skbuff.h>
6604326b4SDaniel Borkmann #include <linux/scatterlist.h>
7604326b4SDaniel Borkmann
8604326b4SDaniel Borkmann #include <net/sock.h>
9604326b4SDaniel Borkmann #include <net/tcp.h>
10e91de6afSJohn Fastabend #include <net/tls.h>
1140e0b090SPeilin Ye #include <trace/events/sock.h>
12604326b4SDaniel Borkmann
sk_msg_try_coalesce_ok(struct sk_msg * msg,int elem_first_coalesce)13604326b4SDaniel Borkmann static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce)
14604326b4SDaniel Borkmann {
15604326b4SDaniel Borkmann if (msg->sg.end > msg->sg.start &&
16604326b4SDaniel Borkmann elem_first_coalesce < msg->sg.end)
17604326b4SDaniel Borkmann return true;
18604326b4SDaniel Borkmann
19604326b4SDaniel Borkmann if (msg->sg.end < msg->sg.start &&
20604326b4SDaniel Borkmann (elem_first_coalesce > msg->sg.start ||
21604326b4SDaniel Borkmann elem_first_coalesce < msg->sg.end))
22604326b4SDaniel Borkmann return true;
23604326b4SDaniel Borkmann
24604326b4SDaniel Borkmann return false;
25604326b4SDaniel Borkmann }
26604326b4SDaniel Borkmann
sk_msg_alloc(struct sock * sk,struct sk_msg * msg,int len,int elem_first_coalesce)27604326b4SDaniel Borkmann int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
28604326b4SDaniel Borkmann int elem_first_coalesce)
29604326b4SDaniel Borkmann {
30604326b4SDaniel Borkmann struct page_frag *pfrag = sk_page_frag(sk);
319c34e38cSWang Yufen u32 osize = msg->sg.size;
32604326b4SDaniel Borkmann int ret = 0;
33604326b4SDaniel Borkmann
34604326b4SDaniel Borkmann len -= msg->sg.size;
35604326b4SDaniel Borkmann while (len > 0) {
36604326b4SDaniel Borkmann struct scatterlist *sge;
37604326b4SDaniel Borkmann u32 orig_offset;
38604326b4SDaniel Borkmann int use, i;
39604326b4SDaniel Borkmann
409c34e38cSWang Yufen if (!sk_page_frag_refill(sk, pfrag)) {
419c34e38cSWang Yufen ret = -ENOMEM;
429c34e38cSWang Yufen goto msg_trim;
439c34e38cSWang Yufen }
44604326b4SDaniel Borkmann
45604326b4SDaniel Borkmann orig_offset = pfrag->offset;
46604326b4SDaniel Borkmann use = min_t(int, len, pfrag->size - orig_offset);
479c34e38cSWang Yufen if (!sk_wmem_schedule(sk, use)) {
489c34e38cSWang Yufen ret = -ENOMEM;
499c34e38cSWang Yufen goto msg_trim;
509c34e38cSWang Yufen }
51604326b4SDaniel Borkmann
52604326b4SDaniel Borkmann i = msg->sg.end;
53604326b4SDaniel Borkmann sk_msg_iter_var_prev(i);
54604326b4SDaniel Borkmann sge = &msg->sg.data[i];
55604326b4SDaniel Borkmann
56604326b4SDaniel Borkmann if (sk_msg_try_coalesce_ok(msg, elem_first_coalesce) &&
57604326b4SDaniel Borkmann sg_page(sge) == pfrag->page &&
58604326b4SDaniel Borkmann sge->offset + sge->length == orig_offset) {
59604326b4SDaniel Borkmann sge->length += use;
60604326b4SDaniel Borkmann } else {
61604326b4SDaniel Borkmann if (sk_msg_full(msg)) {
62604326b4SDaniel Borkmann ret = -ENOSPC;
63604326b4SDaniel Borkmann break;
64604326b4SDaniel Borkmann }
65604326b4SDaniel Borkmann
66604326b4SDaniel Borkmann sge = &msg->sg.data[msg->sg.end];
67604326b4SDaniel Borkmann sg_unmark_end(sge);
68604326b4SDaniel Borkmann sg_set_page(sge, pfrag->page, use, orig_offset);
69604326b4SDaniel Borkmann get_page(pfrag->page);
70604326b4SDaniel Borkmann sk_msg_iter_next(msg, end);
71604326b4SDaniel Borkmann }
72604326b4SDaniel Borkmann
73604326b4SDaniel Borkmann sk_mem_charge(sk, use);
74604326b4SDaniel Borkmann msg->sg.size += use;
75604326b4SDaniel Borkmann pfrag->offset += use;
76604326b4SDaniel Borkmann len -= use;
77604326b4SDaniel Borkmann }
78604326b4SDaniel Borkmann
79604326b4SDaniel Borkmann return ret;
809c34e38cSWang Yufen
819c34e38cSWang Yufen msg_trim:
829c34e38cSWang Yufen sk_msg_trim(sk, msg, osize);
839c34e38cSWang Yufen return ret;
84604326b4SDaniel Borkmann }
85604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_alloc);
86604326b4SDaniel Borkmann
sk_msg_clone(struct sock * sk,struct sk_msg * dst,struct sk_msg * src,u32 off,u32 len)87d829e9c4SDaniel Borkmann int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
88d829e9c4SDaniel Borkmann u32 off, u32 len)
89d829e9c4SDaniel Borkmann {
90d829e9c4SDaniel Borkmann int i = src->sg.start;
91d829e9c4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(src, i);
92fda497e5SVakul Garg struct scatterlist *sgd = NULL;
93d829e9c4SDaniel Borkmann u32 sge_len, sge_off;
94d829e9c4SDaniel Borkmann
95d829e9c4SDaniel Borkmann while (off) {
96d829e9c4SDaniel Borkmann if (sge->length > off)
97d829e9c4SDaniel Borkmann break;
98d829e9c4SDaniel Borkmann off -= sge->length;
99d829e9c4SDaniel Borkmann sk_msg_iter_var_next(i);
100d829e9c4SDaniel Borkmann if (i == src->sg.end && off)
101d829e9c4SDaniel Borkmann return -ENOSPC;
102d829e9c4SDaniel Borkmann sge = sk_msg_elem(src, i);
103d829e9c4SDaniel Borkmann }
104d829e9c4SDaniel Borkmann
105d829e9c4SDaniel Borkmann while (len) {
106d829e9c4SDaniel Borkmann sge_len = sge->length - off;
107d829e9c4SDaniel Borkmann if (sge_len > len)
108d829e9c4SDaniel Borkmann sge_len = len;
109fda497e5SVakul Garg
110fda497e5SVakul Garg if (dst->sg.end)
111fda497e5SVakul Garg sgd = sk_msg_elem(dst, dst->sg.end - 1);
112fda497e5SVakul Garg
113fda497e5SVakul Garg if (sgd &&
114fda497e5SVakul Garg (sg_page(sge) == sg_page(sgd)) &&
115fda497e5SVakul Garg (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) {
116fda497e5SVakul Garg sgd->length += sge_len;
117fda497e5SVakul Garg dst->sg.size += sge_len;
118fda497e5SVakul Garg } else if (!sk_msg_full(dst)) {
119fda497e5SVakul Garg sge_off = sge->offset + off;
120fda497e5SVakul Garg sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off);
121fda497e5SVakul Garg } else {
122fda497e5SVakul Garg return -ENOSPC;
123fda497e5SVakul Garg }
124fda497e5SVakul Garg
125d829e9c4SDaniel Borkmann off = 0;
126d829e9c4SDaniel Borkmann len -= sge_len;
127d829e9c4SDaniel Borkmann sk_mem_charge(sk, sge_len);
128d829e9c4SDaniel Borkmann sk_msg_iter_var_next(i);
129d829e9c4SDaniel Borkmann if (i == src->sg.end && len)
130d829e9c4SDaniel Borkmann return -ENOSPC;
131d829e9c4SDaniel Borkmann sge = sk_msg_elem(src, i);
132d829e9c4SDaniel Borkmann }
133d829e9c4SDaniel Borkmann
134d829e9c4SDaniel Borkmann return 0;
135d829e9c4SDaniel Borkmann }
136d829e9c4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_clone);
137d829e9c4SDaniel Borkmann
sk_msg_return_zero(struct sock * sk,struct sk_msg * msg,int bytes)138604326b4SDaniel Borkmann void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
139604326b4SDaniel Borkmann {
140604326b4SDaniel Borkmann int i = msg->sg.start;
141604326b4SDaniel Borkmann
142604326b4SDaniel Borkmann do {
143604326b4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg, i);
144604326b4SDaniel Borkmann
145604326b4SDaniel Borkmann if (bytes < sge->length) {
146604326b4SDaniel Borkmann sge->length -= bytes;
147604326b4SDaniel Borkmann sge->offset += bytes;
148604326b4SDaniel Borkmann sk_mem_uncharge(sk, bytes);
149604326b4SDaniel Borkmann break;
150604326b4SDaniel Borkmann }
151604326b4SDaniel Borkmann
152604326b4SDaniel Borkmann sk_mem_uncharge(sk, sge->length);
153604326b4SDaniel Borkmann bytes -= sge->length;
154604326b4SDaniel Borkmann sge->length = 0;
155604326b4SDaniel Borkmann sge->offset = 0;
156604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
157604326b4SDaniel Borkmann } while (bytes && i != msg->sg.end);
158604326b4SDaniel Borkmann msg->sg.start = i;
159604326b4SDaniel Borkmann }
160604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_return_zero);
161604326b4SDaniel Borkmann
sk_msg_return(struct sock * sk,struct sk_msg * msg,int bytes)162604326b4SDaniel Borkmann void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes)
163604326b4SDaniel Borkmann {
164604326b4SDaniel Borkmann int i = msg->sg.start;
165604326b4SDaniel Borkmann
166604326b4SDaniel Borkmann do {
167604326b4SDaniel Borkmann struct scatterlist *sge = &msg->sg.data[i];
168604326b4SDaniel Borkmann int uncharge = (bytes < sge->length) ? bytes : sge->length;
169604326b4SDaniel Borkmann
170604326b4SDaniel Borkmann sk_mem_uncharge(sk, uncharge);
171604326b4SDaniel Borkmann bytes -= uncharge;
172604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
173604326b4SDaniel Borkmann } while (i != msg->sg.end);
174604326b4SDaniel Borkmann }
175604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_return);
176604326b4SDaniel Borkmann
sk_msg_free_elem(struct sock * sk,struct sk_msg * msg,u32 i,bool charge)177604326b4SDaniel Borkmann static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i,
178604326b4SDaniel Borkmann bool charge)
179604326b4SDaniel Borkmann {
180604326b4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg, i);
181604326b4SDaniel Borkmann u32 len = sge->length;
182604326b4SDaniel Borkmann
18336cd0e69SJohn Fastabend /* When the skb owns the memory we free it from consume_skb path. */
18436cd0e69SJohn Fastabend if (!msg->skb) {
185604326b4SDaniel Borkmann if (charge)
186604326b4SDaniel Borkmann sk_mem_uncharge(sk, len);
187604326b4SDaniel Borkmann put_page(sg_page(sge));
18836cd0e69SJohn Fastabend }
189604326b4SDaniel Borkmann memset(sge, 0, sizeof(*sge));
190604326b4SDaniel Borkmann return len;
191604326b4SDaniel Borkmann }
192604326b4SDaniel Borkmann
__sk_msg_free(struct sock * sk,struct sk_msg * msg,u32 i,bool charge)193604326b4SDaniel Borkmann static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i,
194604326b4SDaniel Borkmann bool charge)
195604326b4SDaniel Borkmann {
196604326b4SDaniel Borkmann struct scatterlist *sge = sk_msg_elem(msg, i);
197604326b4SDaniel Borkmann int freed = 0;
198604326b4SDaniel Borkmann
199604326b4SDaniel Borkmann while (msg->sg.size) {
200604326b4SDaniel Borkmann msg->sg.size -= sge->length;
201604326b4SDaniel Borkmann freed += sk_msg_free_elem(sk, msg, i, charge);
202604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
203604326b4SDaniel Borkmann sk_msg_check_to_free(msg, i, msg->sg.size);
204604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i);
205604326b4SDaniel Borkmann }
206604326b4SDaniel Borkmann consume_skb(msg->skb);
207604326b4SDaniel Borkmann sk_msg_init(msg);
208604326b4SDaniel Borkmann return freed;
209604326b4SDaniel Borkmann }
210604326b4SDaniel Borkmann
sk_msg_free_nocharge(struct sock * sk,struct sk_msg * msg)211604326b4SDaniel Borkmann int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg)
212604326b4SDaniel Borkmann {
213604326b4SDaniel Borkmann return __sk_msg_free(sk, msg, msg->sg.start, false);
214604326b4SDaniel Borkmann }
215604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_free_nocharge);
216604326b4SDaniel Borkmann
sk_msg_free(struct sock * sk,struct sk_msg * msg)217604326b4SDaniel Borkmann int sk_msg_free(struct sock *sk, struct sk_msg *msg)
218604326b4SDaniel Borkmann {
219604326b4SDaniel Borkmann return __sk_msg_free(sk, msg, msg->sg.start, true);
220604326b4SDaniel Borkmann }
221604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_free);
222604326b4SDaniel Borkmann
__sk_msg_free_partial(struct sock * sk,struct sk_msg * msg,u32 bytes,bool charge)223604326b4SDaniel Borkmann static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg,
224604326b4SDaniel Borkmann u32 bytes, bool charge)
225604326b4SDaniel Borkmann {
226604326b4SDaniel Borkmann struct scatterlist *sge;
227604326b4SDaniel Borkmann u32 i = msg->sg.start;
228604326b4SDaniel Borkmann
229604326b4SDaniel Borkmann while (bytes) {
230604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i);
231604326b4SDaniel Borkmann if (!sge->length)
232604326b4SDaniel Borkmann break;
233604326b4SDaniel Borkmann if (bytes < sge->length) {
234604326b4SDaniel Borkmann if (charge)
235604326b4SDaniel Borkmann sk_mem_uncharge(sk, bytes);
236604326b4SDaniel Borkmann sge->length -= bytes;
237604326b4SDaniel Borkmann sge->offset += bytes;
238604326b4SDaniel Borkmann msg->sg.size -= bytes;
239604326b4SDaniel Borkmann break;
240604326b4SDaniel Borkmann }
241604326b4SDaniel Borkmann
242604326b4SDaniel Borkmann msg->sg.size -= sge->length;
243604326b4SDaniel Borkmann bytes -= sge->length;
244604326b4SDaniel Borkmann sk_msg_free_elem(sk, msg, i, charge);
245604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
246604326b4SDaniel Borkmann sk_msg_check_to_free(msg, i, bytes);
247604326b4SDaniel Borkmann }
248604326b4SDaniel Borkmann msg->sg.start = i;
249604326b4SDaniel Borkmann }
250604326b4SDaniel Borkmann
sk_msg_free_partial(struct sock * sk,struct sk_msg * msg,u32 bytes)251604326b4SDaniel Borkmann void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes)
252604326b4SDaniel Borkmann {
253604326b4SDaniel Borkmann __sk_msg_free_partial(sk, msg, bytes, true);
254604326b4SDaniel Borkmann }
255604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_free_partial);
256604326b4SDaniel Borkmann
sk_msg_free_partial_nocharge(struct sock * sk,struct sk_msg * msg,u32 bytes)257604326b4SDaniel Borkmann void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
258604326b4SDaniel Borkmann u32 bytes)
259604326b4SDaniel Borkmann {
260604326b4SDaniel Borkmann __sk_msg_free_partial(sk, msg, bytes, false);
261604326b4SDaniel Borkmann }
262604326b4SDaniel Borkmann
sk_msg_trim(struct sock * sk,struct sk_msg * msg,int len)263604326b4SDaniel Borkmann void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len)
264604326b4SDaniel Borkmann {
265604326b4SDaniel Borkmann int trim = msg->sg.size - len;
266604326b4SDaniel Borkmann u32 i = msg->sg.end;
267604326b4SDaniel Borkmann
268604326b4SDaniel Borkmann if (trim <= 0) {
269604326b4SDaniel Borkmann WARN_ON(trim < 0);
270604326b4SDaniel Borkmann return;
271604326b4SDaniel Borkmann }
272604326b4SDaniel Borkmann
273604326b4SDaniel Borkmann sk_msg_iter_var_prev(i);
274604326b4SDaniel Borkmann msg->sg.size = len;
275604326b4SDaniel Borkmann while (msg->sg.data[i].length &&
276604326b4SDaniel Borkmann trim >= msg->sg.data[i].length) {
277604326b4SDaniel Borkmann trim -= msg->sg.data[i].length;
278604326b4SDaniel Borkmann sk_msg_free_elem(sk, msg, i, true);
279604326b4SDaniel Borkmann sk_msg_iter_var_prev(i);
280604326b4SDaniel Borkmann if (!trim)
281604326b4SDaniel Borkmann goto out;
282604326b4SDaniel Borkmann }
283604326b4SDaniel Borkmann
284604326b4SDaniel Borkmann msg->sg.data[i].length -= trim;
285604326b4SDaniel Borkmann sk_mem_uncharge(sk, trim);
286683916f6SJakub Kicinski /* Adjust copybreak if it falls into the trimmed part of last buf */
287683916f6SJakub Kicinski if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length)
288683916f6SJakub Kicinski msg->sg.copybreak = msg->sg.data[i].length;
289604326b4SDaniel Borkmann out:
290683916f6SJakub Kicinski sk_msg_iter_var_next(i);
291683916f6SJakub Kicinski msg->sg.end = i;
292683916f6SJakub Kicinski
293683916f6SJakub Kicinski /* If we trim data a full sg elem before curr pointer update
294683916f6SJakub Kicinski * copybreak and current so that any future copy operations
295683916f6SJakub Kicinski * start at new copy location.
296*731733c6SSimon Horman * However trimmed data that has not yet been used in a copy op
297604326b4SDaniel Borkmann * does not require an update.
298604326b4SDaniel Borkmann */
299683916f6SJakub Kicinski if (!msg->sg.size) {
300683916f6SJakub Kicinski msg->sg.curr = msg->sg.start;
301683916f6SJakub Kicinski msg->sg.copybreak = 0;
302683916f6SJakub Kicinski } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >=
303683916f6SJakub Kicinski sk_msg_iter_dist(msg->sg.start, msg->sg.end)) {
304683916f6SJakub Kicinski sk_msg_iter_var_prev(i);
305604326b4SDaniel Borkmann msg->sg.curr = i;
306604326b4SDaniel Borkmann msg->sg.copybreak = msg->sg.data[i].length;
307604326b4SDaniel Borkmann }
308604326b4SDaniel Borkmann }
309604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_trim);
310604326b4SDaniel Borkmann
sk_msg_zerocopy_from_iter(struct sock * sk,struct iov_iter * from,struct sk_msg * msg,u32 bytes)311604326b4SDaniel Borkmann int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
312604326b4SDaniel Borkmann struct sk_msg *msg, u32 bytes)
313604326b4SDaniel Borkmann {
314604326b4SDaniel Borkmann int i, maxpages, ret = 0, num_elems = sk_msg_elem_used(msg);
315604326b4SDaniel Borkmann const int to_max_pages = MAX_MSG_FRAGS;
316604326b4SDaniel Borkmann struct page *pages[MAX_MSG_FRAGS];
317604326b4SDaniel Borkmann ssize_t orig, copied, use, offset;
318604326b4SDaniel Borkmann
319604326b4SDaniel Borkmann orig = msg->sg.size;
320604326b4SDaniel Borkmann while (bytes > 0) {
321604326b4SDaniel Borkmann i = 0;
322604326b4SDaniel Borkmann maxpages = to_max_pages - num_elems;
323604326b4SDaniel Borkmann if (maxpages == 0) {
324604326b4SDaniel Borkmann ret = -EFAULT;
325604326b4SDaniel Borkmann goto out;
326604326b4SDaniel Borkmann }
327604326b4SDaniel Borkmann
3281ef255e2SAl Viro copied = iov_iter_get_pages2(from, pages, bytes, maxpages,
329604326b4SDaniel Borkmann &offset);
330604326b4SDaniel Borkmann if (copied <= 0) {
331604326b4SDaniel Borkmann ret = -EFAULT;
332604326b4SDaniel Borkmann goto out;
333604326b4SDaniel Borkmann }
334604326b4SDaniel Borkmann
335604326b4SDaniel Borkmann bytes -= copied;
336604326b4SDaniel Borkmann msg->sg.size += copied;
337604326b4SDaniel Borkmann
338604326b4SDaniel Borkmann while (copied) {
339604326b4SDaniel Borkmann use = min_t(int, copied, PAGE_SIZE - offset);
340604326b4SDaniel Borkmann sg_set_page(&msg->sg.data[msg->sg.end],
341604326b4SDaniel Borkmann pages[i], use, offset);
342604326b4SDaniel Borkmann sg_unmark_end(&msg->sg.data[msg->sg.end]);
343604326b4SDaniel Borkmann sk_mem_charge(sk, use);
344604326b4SDaniel Borkmann
345604326b4SDaniel Borkmann offset = 0;
346604326b4SDaniel Borkmann copied -= use;
347604326b4SDaniel Borkmann sk_msg_iter_next(msg, end);
348604326b4SDaniel Borkmann num_elems++;
349604326b4SDaniel Borkmann i++;
350604326b4SDaniel Borkmann }
351604326b4SDaniel Borkmann /* When zerocopy is mixed with sk_msg_*copy* operations we
352604326b4SDaniel Borkmann * may have a copybreak set in this case clear and prefer
353604326b4SDaniel Borkmann * zerocopy remainder when possible.
354604326b4SDaniel Borkmann */
355604326b4SDaniel Borkmann msg->sg.copybreak = 0;
356604326b4SDaniel Borkmann msg->sg.curr = msg->sg.end;
357604326b4SDaniel Borkmann }
358604326b4SDaniel Borkmann out:
359604326b4SDaniel Borkmann /* Revert iov_iter updates, msg will need to use 'trim' later if it
360604326b4SDaniel Borkmann * also needs to be cleared.
361604326b4SDaniel Borkmann */
362604326b4SDaniel Borkmann if (ret)
363604326b4SDaniel Borkmann iov_iter_revert(from, msg->sg.size - orig);
364604326b4SDaniel Borkmann return ret;
365604326b4SDaniel Borkmann }
366604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_zerocopy_from_iter);
367604326b4SDaniel Borkmann
sk_msg_memcopy_from_iter(struct sock * sk,struct iov_iter * from,struct sk_msg * msg,u32 bytes)368604326b4SDaniel Borkmann int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
369604326b4SDaniel Borkmann struct sk_msg *msg, u32 bytes)
370604326b4SDaniel Borkmann {
371604326b4SDaniel Borkmann int ret = -ENOSPC, i = msg->sg.curr;
372604326b4SDaniel Borkmann struct scatterlist *sge;
373604326b4SDaniel Borkmann u32 copy, buf_size;
374604326b4SDaniel Borkmann void *to;
375604326b4SDaniel Borkmann
376604326b4SDaniel Borkmann do {
377604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i);
378604326b4SDaniel Borkmann /* This is possible if a trim operation shrunk the buffer */
379604326b4SDaniel Borkmann if (msg->sg.copybreak >= sge->length) {
380604326b4SDaniel Borkmann msg->sg.copybreak = 0;
381604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
382604326b4SDaniel Borkmann if (i == msg->sg.end)
383604326b4SDaniel Borkmann break;
384604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i);
385604326b4SDaniel Borkmann }
386604326b4SDaniel Borkmann
387604326b4SDaniel Borkmann buf_size = sge->length - msg->sg.copybreak;
388604326b4SDaniel Borkmann copy = (buf_size > bytes) ? bytes : buf_size;
389604326b4SDaniel Borkmann to = sg_virt(sge) + msg->sg.copybreak;
390604326b4SDaniel Borkmann msg->sg.copybreak += copy;
391604326b4SDaniel Borkmann if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY)
392604326b4SDaniel Borkmann ret = copy_from_iter_nocache(to, copy, from);
393604326b4SDaniel Borkmann else
394604326b4SDaniel Borkmann ret = copy_from_iter(to, copy, from);
395604326b4SDaniel Borkmann if (ret != copy) {
396604326b4SDaniel Borkmann ret = -EFAULT;
397604326b4SDaniel Borkmann goto out;
398604326b4SDaniel Borkmann }
399604326b4SDaniel Borkmann bytes -= copy;
400604326b4SDaniel Borkmann if (!bytes)
401604326b4SDaniel Borkmann break;
402604326b4SDaniel Borkmann msg->sg.copybreak = 0;
403604326b4SDaniel Borkmann sk_msg_iter_var_next(i);
404604326b4SDaniel Borkmann } while (i != msg->sg.end);
405604326b4SDaniel Borkmann out:
406604326b4SDaniel Borkmann msg->sg.curr = i;
407604326b4SDaniel Borkmann return ret;
408604326b4SDaniel Borkmann }
409604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
410604326b4SDaniel Borkmann
4112bc793e3SCong Wang /* Receive sk_msg from psock->ingress_msg to @msg. */
sk_msg_recvmsg(struct sock * sk,struct sk_psock * psock,struct msghdr * msg,int len,int flags)4122bc793e3SCong Wang int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
4132bc793e3SCong Wang int len, int flags)
4142bc793e3SCong Wang {
4152bc793e3SCong Wang struct iov_iter *iter = &msg->msg_iter;
4162bc793e3SCong Wang int peek = flags & MSG_PEEK;
4172bc793e3SCong Wang struct sk_msg *msg_rx;
4182bc793e3SCong Wang int i, copied = 0;
4192bc793e3SCong Wang
4202bc793e3SCong Wang msg_rx = sk_psock_peek_msg(psock);
4212bc793e3SCong Wang while (copied != len) {
4222bc793e3SCong Wang struct scatterlist *sge;
4232bc793e3SCong Wang
4242bc793e3SCong Wang if (unlikely(!msg_rx))
4252bc793e3SCong Wang break;
4262bc793e3SCong Wang
4272bc793e3SCong Wang i = msg_rx->sg.start;
4282bc793e3SCong Wang do {
4292bc793e3SCong Wang struct page *page;
4302bc793e3SCong Wang int copy;
4312bc793e3SCong Wang
4322bc793e3SCong Wang sge = sk_msg_elem(msg_rx, i);
4332bc793e3SCong Wang copy = sge->length;
4342bc793e3SCong Wang page = sg_page(sge);
4352bc793e3SCong Wang if (copied + copy > len)
4362bc793e3SCong Wang copy = len - copied;
437f0c18025SGeliang Tang if (copy)
4382bc793e3SCong Wang copy = copy_page_to_iter(page, sge->offset, copy, iter);
439bec21719SLiu Jian if (!copy) {
440bec21719SLiu Jian copied = copied ? copied : -EFAULT;
441bec21719SLiu Jian goto out;
442bec21719SLiu Jian }
4432bc793e3SCong Wang
4442bc793e3SCong Wang copied += copy;
4452bc793e3SCong Wang if (likely(!peek)) {
4462bc793e3SCong Wang sge->offset += copy;
4472bc793e3SCong Wang sge->length -= copy;
4482bc793e3SCong Wang if (!msg_rx->skb)
4492bc793e3SCong Wang sk_mem_uncharge(sk, copy);
4502bc793e3SCong Wang msg_rx->sg.size -= copy;
4512bc793e3SCong Wang
4522bc793e3SCong Wang if (!sge->length) {
4532bc793e3SCong Wang sk_msg_iter_var_next(i);
4542bc793e3SCong Wang if (!msg_rx->skb)
4552bc793e3SCong Wang put_page(page);
4562bc793e3SCong Wang }
4572bc793e3SCong Wang } else {
4582bc793e3SCong Wang /* Lets not optimize peek case if copy_page_to_iter
4592bc793e3SCong Wang * didn't copy the entire length lets just break.
4602bc793e3SCong Wang */
4612bc793e3SCong Wang if (copy != sge->length)
462bec21719SLiu Jian goto out;
4632bc793e3SCong Wang sk_msg_iter_var_next(i);
4642bc793e3SCong Wang }
4652bc793e3SCong Wang
4662bc793e3SCong Wang if (copied == len)
4672bc793e3SCong Wang break;
468583585e4SLiu Jian } while ((i != msg_rx->sg.end) && !sg_is_last(sge));
4692bc793e3SCong Wang
4702bc793e3SCong Wang if (unlikely(peek)) {
4712bc793e3SCong Wang msg_rx = sk_psock_next_msg(psock, msg_rx);
4722bc793e3SCong Wang if (!msg_rx)
4732bc793e3SCong Wang break;
4742bc793e3SCong Wang continue;
4752bc793e3SCong Wang }
4762bc793e3SCong Wang
4772bc793e3SCong Wang msg_rx->sg.start = i;
478583585e4SLiu Jian if (!sge->length && (i == msg_rx->sg.end || sg_is_last(sge))) {
4792bc793e3SCong Wang msg_rx = sk_psock_dequeue_msg(psock);
4802bc793e3SCong Wang kfree_sk_msg(msg_rx);
4812bc793e3SCong Wang }
4822bc793e3SCong Wang msg_rx = sk_psock_peek_msg(psock);
4832bc793e3SCong Wang }
484bec21719SLiu Jian out:
4852bc793e3SCong Wang return copied;
4862bc793e3SCong Wang }
4872bc793e3SCong Wang EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
4882bc793e3SCong Wang
sk_msg_is_readable(struct sock * sk)489fb4e0a5eSCong Wang bool sk_msg_is_readable(struct sock *sk)
490fb4e0a5eSCong Wang {
491fb4e0a5eSCong Wang struct sk_psock *psock;
492fb4e0a5eSCong Wang bool empty = true;
493fb4e0a5eSCong Wang
494fb4e0a5eSCong Wang rcu_read_lock();
495fb4e0a5eSCong Wang psock = sk_psock(sk);
496fb4e0a5eSCong Wang if (likely(psock))
497fb4e0a5eSCong Wang empty = list_empty(&psock->ingress_msg);
498fb4e0a5eSCong Wang rcu_read_unlock();
499fb4e0a5eSCong Wang return !empty;
500fb4e0a5eSCong Wang }
501fb4e0a5eSCong Wang EXPORT_SYMBOL_GPL(sk_msg_is_readable);
502fb4e0a5eSCong Wang
alloc_sk_msg(gfp_t gfp)5032d1f274bSEric Dumazet static struct sk_msg *alloc_sk_msg(gfp_t gfp)
504604326b4SDaniel Borkmann {
505604326b4SDaniel Borkmann struct sk_msg *msg;
506604326b4SDaniel Borkmann
5072d1f274bSEric Dumazet msg = kzalloc(sizeof(*msg), gfp | __GFP_NOWARN);
50843312915SCong Wang if (unlikely(!msg))
50943312915SCong Wang return NULL;
51043312915SCong Wang sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
51143312915SCong Wang return msg;
51243312915SCong Wang }
51343312915SCong Wang
sk_psock_create_ingress_msg(struct sock * sk,struct sk_buff * skb)51443312915SCong Wang static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
51543312915SCong Wang struct sk_buff *skb)
51643312915SCong Wang {
51736cd0e69SJohn Fastabend if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
5186fa9201aSJohn Fastabend return NULL;
5196fa9201aSJohn Fastabend
5206fa9201aSJohn Fastabend if (!sk_rmem_schedule(sk, skb, skb->truesize))
5216fa9201aSJohn Fastabend return NULL;
52236cd0e69SJohn Fastabend
5232d1f274bSEric Dumazet return alloc_sk_msg(GFP_KERNEL);
5246fa9201aSJohn Fastabend }
5256fa9201aSJohn Fastabend
sk_psock_skb_ingress_enqueue(struct sk_buff * skb,u32 off,u32 len,struct sk_psock * psock,struct sock * sk,struct sk_msg * msg)5266fa9201aSJohn Fastabend static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
5277303524eSLiu Jian u32 off, u32 len,
5286fa9201aSJohn Fastabend struct sk_psock *psock,
5296fa9201aSJohn Fastabend struct sock *sk,
5306fa9201aSJohn Fastabend struct sk_msg *msg)
5316fa9201aSJohn Fastabend {
5324363023dSJohn Fastabend int num_sge, copied;
5336fa9201aSJohn Fastabend
5343527bfe6SLiu Jian num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
5353527bfe6SLiu Jian if (num_sge < 0) {
5364363023dSJohn Fastabend /* skb linearize may fail with ENOMEM, but lets simply try again
5374363023dSJohn Fastabend * later if this happens. Under memory pressure we don't want to
5384363023dSJohn Fastabend * drop the skb. We need to linearize the skb so that the mapping
5394363023dSJohn Fastabend * in skb_to_sgvec can not error.
5404363023dSJohn Fastabend */
5414363023dSJohn Fastabend if (skb_linearize(skb))
5424363023dSJohn Fastabend return -EAGAIN;
5433527bfe6SLiu Jian
5447303524eSLiu Jian num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
5457e6b27a6SJohn Fastabend if (unlikely(num_sge < 0))
546604326b4SDaniel Borkmann return num_sge;
5473527bfe6SLiu Jian }
548604326b4SDaniel Borkmann
5497303524eSLiu Jian copied = len;
550604326b4SDaniel Borkmann msg->sg.start = 0;
551cabede8bSJohn Fastabend msg->sg.size = copied;
552031097d9SJakub Kicinski msg->sg.end = num_sge;
553604326b4SDaniel Borkmann msg->skb = skb;
554604326b4SDaniel Borkmann
555604326b4SDaniel Borkmann sk_psock_queue_msg(psock, msg);
556552de910SJohn Fastabend sk_psock_data_ready(sk, psock);
557604326b4SDaniel Borkmann return copied;
558604326b4SDaniel Borkmann }
559604326b4SDaniel Borkmann
5607303524eSLiu Jian static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
5617303524eSLiu Jian u32 off, u32 len);
5622443ca66SJohn Fastabend
sk_psock_skb_ingress(struct sk_psock * psock,struct sk_buff * skb,u32 off,u32 len)5637303524eSLiu Jian static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
5647303524eSLiu Jian u32 off, u32 len)
5656fa9201aSJohn Fastabend {
5666fa9201aSJohn Fastabend struct sock *sk = psock->sk;
5676fa9201aSJohn Fastabend struct sk_msg *msg;
5687e6b27a6SJohn Fastabend int err;
5696fa9201aSJohn Fastabend
5702443ca66SJohn Fastabend /* If we are receiving on the same sock skb->sk is already assigned,
5712443ca66SJohn Fastabend * skip memory accounting and owner transition seeing it already set
5722443ca66SJohn Fastabend * correctly.
5732443ca66SJohn Fastabend */
5742443ca66SJohn Fastabend if (unlikely(skb->sk == sk))
5757303524eSLiu Jian return sk_psock_skb_ingress_self(psock, skb, off, len);
5766fa9201aSJohn Fastabend msg = sk_psock_create_ingress_msg(sk, skb);
5776fa9201aSJohn Fastabend if (!msg)
5786fa9201aSJohn Fastabend return -EAGAIN;
5796fa9201aSJohn Fastabend
5806fa9201aSJohn Fastabend /* This will transition ownership of the data from the socket where
5816fa9201aSJohn Fastabend * the BPF program was run initiating the redirect to the socket
5826fa9201aSJohn Fastabend * we will eventually receive this data on. The data will be released
5836fa9201aSJohn Fastabend * from skb_consume found in __tcp_bpf_recvmsg() after its been copied
5846fa9201aSJohn Fastabend * into user buffers.
5856fa9201aSJohn Fastabend */
5866fa9201aSJohn Fastabend skb_set_owner_r(skb, sk);
5877303524eSLiu Jian err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
5887e6b27a6SJohn Fastabend if (err < 0)
5897e6b27a6SJohn Fastabend kfree(msg);
5907e6b27a6SJohn Fastabend return err;
5916fa9201aSJohn Fastabend }
5926fa9201aSJohn Fastabend
5936fa9201aSJohn Fastabend /* Puts an skb on the ingress queue of the socket already assigned to the
5946fa9201aSJohn Fastabend * skb. In this case we do not need to check memory limits or skb_set_owner_r
5956fa9201aSJohn Fastabend * because the skb is already accounted for here.
5966fa9201aSJohn Fastabend */
sk_psock_skb_ingress_self(struct sk_psock * psock,struct sk_buff * skb,u32 off,u32 len)5977303524eSLiu Jian static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
5987303524eSLiu Jian u32 off, u32 len)
5996fa9201aSJohn Fastabend {
6002d1f274bSEric Dumazet struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC);
6016fa9201aSJohn Fastabend struct sock *sk = psock->sk;
6027e6b27a6SJohn Fastabend int err;
6036fa9201aSJohn Fastabend
6046fa9201aSJohn Fastabend if (unlikely(!msg))
6056fa9201aSJohn Fastabend return -EAGAIN;
606144748ebSJohn Fastabend skb_set_owner_r(skb, sk);
6077303524eSLiu Jian err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
6087e6b27a6SJohn Fastabend if (err < 0)
6097e6b27a6SJohn Fastabend kfree(msg);
6107e6b27a6SJohn Fastabend return err;
6116fa9201aSJohn Fastabend }
6126fa9201aSJohn Fastabend
sk_psock_handle_skb(struct sk_psock * psock,struct sk_buff * skb,u32 off,u32 len,bool ingress)613604326b4SDaniel Borkmann static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
614604326b4SDaniel Borkmann u32 off, u32 len, bool ingress)
615604326b4SDaniel Borkmann {
616a454d84eSJohn Fastabend int err = 0;
617a454d84eSJohn Fastabend
6189047f19eSJohn Fastabend if (!ingress) {
6199047f19eSJohn Fastabend if (!sock_writeable(psock->sk))
6209047f19eSJohn Fastabend return -EAGAIN;
621799aa7f9SCong Wang return skb_send_sock(psock->sk, skb, off, len);
622604326b4SDaniel Borkmann }
623a454d84eSJohn Fastabend skb_get(skb);
624a454d84eSJohn Fastabend err = sk_psock_skb_ingress(psock, skb, off, len);
625a454d84eSJohn Fastabend if (err < 0)
626a454d84eSJohn Fastabend kfree_skb(skb);
627a454d84eSJohn Fastabend return err;
6289047f19eSJohn Fastabend }
629604326b4SDaniel Borkmann
sk_psock_skb_state(struct sk_psock * psock,struct sk_psock_work_state * state,int len,int off)630476d9801SJohn Fastabend static void sk_psock_skb_state(struct sk_psock *psock,
631476d9801SJohn Fastabend struct sk_psock_work_state *state,
632476d9801SJohn Fastabend int len, int off)
633476d9801SJohn Fastabend {
634476d9801SJohn Fastabend spin_lock_bh(&psock->ingress_lock);
635476d9801SJohn Fastabend if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
636476d9801SJohn Fastabend state->len = len;
637476d9801SJohn Fastabend state->off = off;
638476d9801SJohn Fastabend }
639476d9801SJohn Fastabend spin_unlock_bh(&psock->ingress_lock);
640476d9801SJohn Fastabend }
641476d9801SJohn Fastabend
sk_psock_backlog(struct work_struct * work)642604326b4SDaniel Borkmann static void sk_psock_backlog(struct work_struct *work)
643604326b4SDaniel Borkmann {
64429173d07SJohn Fastabend struct delayed_work *dwork = to_delayed_work(work);
64529173d07SJohn Fastabend struct sk_psock *psock = container_of(dwork, struct sk_psock, work);
646604326b4SDaniel Borkmann struct sk_psock_work_state *state = &psock->work_state;
647476d9801SJohn Fastabend struct sk_buff *skb = NULL;
648405df89dSJohn Fastabend u32 len = 0, off = 0;
649604326b4SDaniel Borkmann bool ingress;
650604326b4SDaniel Borkmann int ret;
651604326b4SDaniel Borkmann
652799aa7f9SCong Wang mutex_lock(&psock->work_mutex);
653405df89dSJohn Fastabend if (unlikely(state->len)) {
654604326b4SDaniel Borkmann len = state->len;
655604326b4SDaniel Borkmann off = state->off;
656604326b4SDaniel Borkmann }
657604326b4SDaniel Borkmann
658405df89dSJohn Fastabend while ((skb = skb_peek(&psock->ingress_skb))) {
659604326b4SDaniel Borkmann len = skb->len;
660604326b4SDaniel Borkmann off = 0;
6617303524eSLiu Jian if (skb_bpf_strparser(skb)) {
6627303524eSLiu Jian struct strp_msg *stm = strp_msg(skb);
6637303524eSLiu Jian
6647303524eSLiu Jian off = stm->offset;
6657303524eSLiu Jian len = stm->full_len;
6667303524eSLiu Jian }
667e3526bb9SCong Wang ingress = skb_bpf_ingress(skb);
668e3526bb9SCong Wang skb_bpf_redirect_clear(skb);
669604326b4SDaniel Borkmann do {
670604326b4SDaniel Borkmann ret = -EIO;
671799aa7f9SCong Wang if (!sock_flag(psock->sk, SOCK_DEAD))
672604326b4SDaniel Borkmann ret = sk_psock_handle_skb(psock, skb, off,
673604326b4SDaniel Borkmann len, ingress);
674604326b4SDaniel Borkmann if (ret <= 0) {
675604326b4SDaniel Borkmann if (ret == -EAGAIN) {
676405df89dSJohn Fastabend sk_psock_skb_state(psock, state, len, off);
67729173d07SJohn Fastabend
67829173d07SJohn Fastabend /* Delay slightly to prioritize any
67929173d07SJohn Fastabend * other work that might be here.
68029173d07SJohn Fastabend */
68129173d07SJohn Fastabend if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
68229173d07SJohn Fastabend schedule_delayed_work(&psock->work, 1);
683604326b4SDaniel Borkmann goto end;
684604326b4SDaniel Borkmann }
685604326b4SDaniel Borkmann /* Hard errors break pipe and stop xmit. */
686604326b4SDaniel Borkmann sk_psock_report_error(psock, ret ? -ret : EPIPE);
687604326b4SDaniel Borkmann sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
688604326b4SDaniel Borkmann goto end;
689604326b4SDaniel Borkmann }
690604326b4SDaniel Borkmann off += ret;
691604326b4SDaniel Borkmann len -= ret;
692604326b4SDaniel Borkmann } while (len);
693604326b4SDaniel Borkmann
694405df89dSJohn Fastabend skb = skb_dequeue(&psock->ingress_skb);
695604326b4SDaniel Borkmann kfree_skb(skb);
696604326b4SDaniel Borkmann }
697604326b4SDaniel Borkmann end:
698799aa7f9SCong Wang mutex_unlock(&psock->work_mutex);
699604326b4SDaniel Borkmann }
700604326b4SDaniel Borkmann
sk_psock_init(struct sock * sk,int node)701604326b4SDaniel Borkmann struct sk_psock *sk_psock_init(struct sock *sk, int node)
702604326b4SDaniel Borkmann {
7037b219da4SLorenz Bauer struct sk_psock *psock;
7047b219da4SLorenz Bauer struct proto *prot;
705604326b4SDaniel Borkmann
7067b219da4SLorenz Bauer write_lock_bh(&sk->sk_callback_lock);
7077b219da4SLorenz Bauer
708e34a07c0SJakub Kicinski if (sk_is_inet(sk) && inet_csk_has_ulp(sk)) {
709e34a07c0SJakub Kicinski psock = ERR_PTR(-EINVAL);
710e34a07c0SJakub Kicinski goto out;
711e34a07c0SJakub Kicinski }
712e34a07c0SJakub Kicinski
7137b219da4SLorenz Bauer if (sk->sk_user_data) {
7147b219da4SLorenz Bauer psock = ERR_PTR(-EBUSY);
7157b219da4SLorenz Bauer goto out;
7167b219da4SLorenz Bauer }
7177b219da4SLorenz Bauer
7187b219da4SLorenz Bauer psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);
7197b219da4SLorenz Bauer if (!psock) {
7207b219da4SLorenz Bauer psock = ERR_PTR(-ENOMEM);
7217b219da4SLorenz Bauer goto out;
7227b219da4SLorenz Bauer }
7237b219da4SLorenz Bauer
7247b219da4SLorenz Bauer prot = READ_ONCE(sk->sk_prot);
725604326b4SDaniel Borkmann psock->sk = sk;
726604326b4SDaniel Borkmann psock->eval = __SK_NONE;
7277b219da4SLorenz Bauer psock->sk_proto = prot;
7287b219da4SLorenz Bauer psock->saved_unhash = prot->unhash;
729d8616ee2SWang Yufen psock->saved_destroy = prot->destroy;
7307b219da4SLorenz Bauer psock->saved_close = prot->close;
7317b219da4SLorenz Bauer psock->saved_write_space = sk->sk_write_space;
732604326b4SDaniel Borkmann
733604326b4SDaniel Borkmann INIT_LIST_HEAD(&psock->link);
734604326b4SDaniel Borkmann spin_lock_init(&psock->link_lock);
735604326b4SDaniel Borkmann
73629173d07SJohn Fastabend INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);
737799aa7f9SCong Wang mutex_init(&psock->work_mutex);
738604326b4SDaniel Borkmann INIT_LIST_HEAD(&psock->ingress_msg);
739b01fd6e8SCong Wang spin_lock_init(&psock->ingress_lock);
740604326b4SDaniel Borkmann skb_queue_head_init(&psock->ingress_skb);
741604326b4SDaniel Borkmann
742604326b4SDaniel Borkmann sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
743604326b4SDaniel Borkmann refcount_set(&psock->refcnt, 1);
744604326b4SDaniel Borkmann
7452a013372SHawkins Jiawei __rcu_assign_sk_user_data_with_flags(sk, psock,
7462a013372SHawkins Jiawei SK_USER_DATA_NOCOPY |
7472a013372SHawkins Jiawei SK_USER_DATA_PSOCK);
748604326b4SDaniel Borkmann sock_hold(sk);
749604326b4SDaniel Borkmann
7507b219da4SLorenz Bauer out:
7517b219da4SLorenz Bauer write_unlock_bh(&sk->sk_callback_lock);
752604326b4SDaniel Borkmann return psock;
753604326b4SDaniel Borkmann }
754604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_psock_init);
755604326b4SDaniel Borkmann
sk_psock_link_pop(struct sk_psock * psock)756604326b4SDaniel Borkmann struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
757604326b4SDaniel Borkmann {
758604326b4SDaniel Borkmann struct sk_psock_link *link;
759604326b4SDaniel Borkmann
760604326b4SDaniel Borkmann spin_lock_bh(&psock->link_lock);
761604326b4SDaniel Borkmann link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
762604326b4SDaniel Borkmann list);
763604326b4SDaniel Borkmann if (link)
764604326b4SDaniel Borkmann list_del(&link->list);
765604326b4SDaniel Borkmann spin_unlock_bh(&psock->link_lock);
766604326b4SDaniel Borkmann return link;
767604326b4SDaniel Borkmann }
768604326b4SDaniel Borkmann
__sk_psock_purge_ingress_msg(struct sk_psock * psock)769cd81cefbSCong Wang static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
770604326b4SDaniel Borkmann {
771604326b4SDaniel Borkmann struct sk_msg *msg, *tmp;
772604326b4SDaniel Borkmann
773604326b4SDaniel Borkmann list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
774604326b4SDaniel Borkmann list_del(&msg->list);
775604326b4SDaniel Borkmann sk_msg_free(psock->sk, msg);
776604326b4SDaniel Borkmann kfree(msg);
777604326b4SDaniel Borkmann }
778604326b4SDaniel Borkmann }
779604326b4SDaniel Borkmann
__sk_psock_zap_ingress(struct sk_psock * psock)780799aa7f9SCong Wang static void __sk_psock_zap_ingress(struct sk_psock *psock)
781604326b4SDaniel Borkmann {
782e3526bb9SCong Wang struct sk_buff *skb;
783e3526bb9SCong Wang
78437f0e514SCong Wang while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
785e3526bb9SCong Wang skb_bpf_redirect_clear(skb);
786781dd043SCong Wang sock_drop(psock->sk, skb);
787e3526bb9SCong Wang }
788604326b4SDaniel Borkmann __sk_psock_purge_ingress_msg(psock);
789604326b4SDaniel Borkmann }
790604326b4SDaniel Borkmann
sk_psock_link_destroy(struct sk_psock * psock)791604326b4SDaniel Borkmann static void sk_psock_link_destroy(struct sk_psock *psock)
792604326b4SDaniel Borkmann {
793604326b4SDaniel Borkmann struct sk_psock_link *link, *tmp;
794604326b4SDaniel Borkmann
795604326b4SDaniel Borkmann list_for_each_entry_safe(link, tmp, &psock->link, list) {
796604326b4SDaniel Borkmann list_del(&link->list);
797604326b4SDaniel Borkmann sk_psock_free_link(link);
798604326b4SDaniel Borkmann }
799604326b4SDaniel Borkmann }
800604326b4SDaniel Borkmann
sk_psock_stop(struct sk_psock * psock)8018bbabb3fSCong Wang void sk_psock_stop(struct sk_psock *psock)
802799aa7f9SCong Wang {
803799aa7f9SCong Wang spin_lock_bh(&psock->ingress_lock);
804799aa7f9SCong Wang sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
805799aa7f9SCong Wang sk_psock_cork_free(psock);
806799aa7f9SCong Wang spin_unlock_bh(&psock->ingress_lock);
807799aa7f9SCong Wang }
808799aa7f9SCong Wang
80988759609SCong Wang static void sk_psock_done_strp(struct sk_psock *psock);
81088759609SCong Wang
sk_psock_destroy(struct work_struct * work)8117786dfc4SCong Wang static void sk_psock_destroy(struct work_struct *work)
812604326b4SDaniel Borkmann {
8137786dfc4SCong Wang struct sk_psock *psock = container_of(to_rcu_work(work),
8147786dfc4SCong Wang struct sk_psock, rwork);
815604326b4SDaniel Borkmann /* No sk_callback_lock since already detached. */
81601489436SJohn Fastabend
81788759609SCong Wang sk_psock_done_strp(psock);
818604326b4SDaniel Borkmann
81929173d07SJohn Fastabend cancel_delayed_work_sync(&psock->work);
820405df89dSJohn Fastabend __sk_psock_zap_ingress(psock);
821799aa7f9SCong Wang mutex_destroy(&psock->work_mutex);
822604326b4SDaniel Borkmann
823604326b4SDaniel Borkmann psock_progs_drop(&psock->progs);
824604326b4SDaniel Borkmann
825604326b4SDaniel Borkmann sk_psock_link_destroy(psock);
826604326b4SDaniel Borkmann sk_psock_cork_free(psock);
827604326b4SDaniel Borkmann
828604326b4SDaniel Borkmann if (psock->sk_redir)
829604326b4SDaniel Borkmann sock_put(psock->sk_redir);
8308866730aSJohn Fastabend if (psock->sk_pair)
8318866730aSJohn Fastabend sock_put(psock->sk_pair);
832604326b4SDaniel Borkmann sock_put(psock->sk);
833604326b4SDaniel Borkmann kfree(psock);
834604326b4SDaniel Borkmann }
835604326b4SDaniel Borkmann
sk_psock_drop(struct sock * sk,struct sk_psock * psock)836604326b4SDaniel Borkmann void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
837604326b4SDaniel Borkmann {
838604326b4SDaniel Borkmann write_lock_bh(&sk->sk_callback_lock);
83995fa1454SJohn Fastabend sk_psock_restore_proto(sk, psock);
84095fa1454SJohn Fastabend rcu_assign_sk_user_data(sk, NULL);
841ae8b8332SCong Wang if (psock->progs.stream_parser)
842604326b4SDaniel Borkmann sk_psock_stop_strp(sk, psock);
843a7ba4558SCong Wang else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
844ef565928SJohn Fastabend sk_psock_stop_verdict(sk, psock);
845604326b4SDaniel Borkmann write_unlock_bh(&sk->sk_callback_lock);
846604326b4SDaniel Borkmann
8478bbabb3fSCong Wang sk_psock_stop(psock);
848343597d5SJohn Fastabend
8497786dfc4SCong Wang INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
8507786dfc4SCong Wang queue_rcu_work(system_wq, &psock->rwork);
851604326b4SDaniel Borkmann }
852604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_psock_drop);
853604326b4SDaniel Borkmann
sk_psock_map_verd(int verdict,bool redir)854604326b4SDaniel Borkmann static int sk_psock_map_verd(int verdict, bool redir)
855604326b4SDaniel Borkmann {
856604326b4SDaniel Borkmann switch (verdict) {
857604326b4SDaniel Borkmann case SK_PASS:
858604326b4SDaniel Borkmann return redir ? __SK_REDIRECT : __SK_PASS;
859604326b4SDaniel Borkmann case SK_DROP:
860604326b4SDaniel Borkmann default:
861604326b4SDaniel Borkmann break;
862604326b4SDaniel Borkmann }
863604326b4SDaniel Borkmann
864604326b4SDaniel Borkmann return __SK_DROP;
865604326b4SDaniel Borkmann }
866604326b4SDaniel Borkmann
sk_psock_msg_verdict(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg)867604326b4SDaniel Borkmann int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
868604326b4SDaniel Borkmann struct sk_msg *msg)
869604326b4SDaniel Borkmann {
870604326b4SDaniel Borkmann struct bpf_prog *prog;
871604326b4SDaniel Borkmann int ret;
872604326b4SDaniel Borkmann
873604326b4SDaniel Borkmann rcu_read_lock();
874604326b4SDaniel Borkmann prog = READ_ONCE(psock->progs.msg_parser);
875604326b4SDaniel Borkmann if (unlikely(!prog)) {
876604326b4SDaniel Borkmann ret = __SK_PASS;
877604326b4SDaniel Borkmann goto out;
878604326b4SDaniel Borkmann }
879604326b4SDaniel Borkmann
880604326b4SDaniel Borkmann sk_msg_compute_data_pointers(msg);
881604326b4SDaniel Borkmann msg->sk = sk;
8823d9f773cSDavid Miller ret = bpf_prog_run_pin_on_cpu(prog, msg);
883604326b4SDaniel Borkmann ret = sk_psock_map_verd(ret, msg->sk_redir);
884604326b4SDaniel Borkmann psock->apply_bytes = msg->apply_bytes;
885604326b4SDaniel Borkmann if (ret == __SK_REDIRECT) {
886a351d608SPengcheng Yang if (psock->sk_redir) {
887604326b4SDaniel Borkmann sock_put(psock->sk_redir);
888a351d608SPengcheng Yang psock->sk_redir = NULL;
889a351d608SPengcheng Yang }
890a351d608SPengcheng Yang if (!msg->sk_redir) {
891604326b4SDaniel Borkmann ret = __SK_DROP;
892604326b4SDaniel Borkmann goto out;
893604326b4SDaniel Borkmann }
894a351d608SPengcheng Yang psock->redir_ingress = sk_msg_to_ingress(msg);
895a351d608SPengcheng Yang psock->sk_redir = msg->sk_redir;
896604326b4SDaniel Borkmann sock_hold(psock->sk_redir);
897604326b4SDaniel Borkmann }
898604326b4SDaniel Borkmann out:
899604326b4SDaniel Borkmann rcu_read_unlock();
900604326b4SDaniel Borkmann return ret;
901604326b4SDaniel Borkmann }
902604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(sk_psock_msg_verdict);
903604326b4SDaniel Borkmann
sk_psock_skb_redirect(struct sk_psock * from,struct sk_buff * skb)90442830571SCong Wang static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb)
905604326b4SDaniel Borkmann {
906604326b4SDaniel Borkmann struct sk_psock *psock_other;
907604326b4SDaniel Borkmann struct sock *sk_other;
908604326b4SDaniel Borkmann
909e3526bb9SCong Wang sk_other = skb_bpf_redirect_fetch(skb);
9109047f19eSJohn Fastabend /* This error is a buggy BPF program, it returned a redirect
9119047f19eSJohn Fastabend * return code, but then didn't set a redirect interface.
9129047f19eSJohn Fastabend */
913ca2f5f21SJohn Fastabend if (unlikely(!sk_other)) {
9147303524eSLiu Jian skb_bpf_redirect_clear(skb);
915781dd043SCong Wang sock_drop(from->sk, skb);
9161581a6c1SCong Wang return -EIO;
917ca2f5f21SJohn Fastabend }
918ca2f5f21SJohn Fastabend psock_other = sk_psock(sk_other);
9199047f19eSJohn Fastabend /* This error indicates the socket is being torn down or had another
9209047f19eSJohn Fastabend * error that caused the pipe to break. We can't send a packet on
9219047f19eSJohn Fastabend * a socket that is in this state so we drop the skb.
9229047f19eSJohn Fastabend */
923799aa7f9SCong Wang if (!psock_other || sock_flag(sk_other, SOCK_DEAD)) {
92430b9c54aSCong Wang skb_bpf_redirect_clear(skb);
925781dd043SCong Wang sock_drop(from->sk, skb);
9261581a6c1SCong Wang return -EIO;
927799aa7f9SCong Wang }
928799aa7f9SCong Wang spin_lock_bh(&psock_other->ingress_lock);
929799aa7f9SCong Wang if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
930799aa7f9SCong Wang spin_unlock_bh(&psock_other->ingress_lock);
93130b9c54aSCong Wang skb_bpf_redirect_clear(skb);
932781dd043SCong Wang sock_drop(from->sk, skb);
9331581a6c1SCong Wang return -EIO;
934ca2f5f21SJohn Fastabend }
935ca2f5f21SJohn Fastabend
936ca2f5f21SJohn Fastabend skb_queue_tail(&psock_other->ingress_skb, skb);
93729173d07SJohn Fastabend schedule_delayed_work(&psock_other->work, 0);
938799aa7f9SCong Wang spin_unlock_bh(&psock_other->ingress_lock);
9391581a6c1SCong Wang return 0;
940ca2f5f21SJohn Fastabend }
941ca2f5f21SJohn Fastabend
sk_psock_tls_verdict_apply(struct sk_buff * skb,struct sk_psock * from,int verdict)94242830571SCong Wang static void sk_psock_tls_verdict_apply(struct sk_buff *skb,
94342830571SCong Wang struct sk_psock *from, int verdict)
944e91de6afSJohn Fastabend {
945e91de6afSJohn Fastabend switch (verdict) {
946e91de6afSJohn Fastabend case __SK_REDIRECT:
94742830571SCong Wang sk_psock_skb_redirect(from, skb);
948e91de6afSJohn Fastabend break;
949e91de6afSJohn Fastabend case __SK_PASS:
950e91de6afSJohn Fastabend case __SK_DROP:
951e91de6afSJohn Fastabend default:
952e91de6afSJohn Fastabend break;
953e91de6afSJohn Fastabend }
954e91de6afSJohn Fastabend }
955e91de6afSJohn Fastabend
sk_psock_tls_strp_read(struct sk_psock * psock,struct sk_buff * skb)956e91de6afSJohn Fastabend int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
957e91de6afSJohn Fastabend {
958e91de6afSJohn Fastabend struct bpf_prog *prog;
959e91de6afSJohn Fastabend int ret = __SK_PASS;
960e91de6afSJohn Fastabend
961e91de6afSJohn Fastabend rcu_read_lock();
962ae8b8332SCong Wang prog = READ_ONCE(psock->progs.stream_verdict);
963e91de6afSJohn Fastabend if (likely(prog)) {
9640b17ad25SJohn Fastabend skb->sk = psock->sk;
965e3526bb9SCong Wang skb_dst_drop(skb);
966e3526bb9SCong Wang skb_bpf_redirect_clear(skb);
96753334232SCong Wang ret = bpf_prog_run_pin_on_cpu(prog, skb);
968e3526bb9SCong Wang ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
9690b17ad25SJohn Fastabend skb->sk = NULL;
970e91de6afSJohn Fastabend }
97142830571SCong Wang sk_psock_tls_verdict_apply(skb, psock, ret);
972e91de6afSJohn Fastabend rcu_read_unlock();
973e91de6afSJohn Fastabend return ret;
974e91de6afSJohn Fastabend }
975e91de6afSJohn Fastabend EXPORT_SYMBOL_GPL(sk_psock_tls_strp_read);
976e91de6afSJohn Fastabend
sk_psock_verdict_apply(struct sk_psock * psock,struct sk_buff * skb,int verdict)9771581a6c1SCong Wang static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
9781581a6c1SCong Wang int verdict)
979ca2f5f21SJohn Fastabend {
980ca2f5f21SJohn Fastabend struct sock *sk_other;
9811581a6c1SCong Wang int err = 0;
9827303524eSLiu Jian u32 len, off;
983ca2f5f21SJohn Fastabend
984604326b4SDaniel Borkmann switch (verdict) {
98551199405SJohn Fastabend case __SK_PASS:
9861581a6c1SCong Wang err = -EIO;
98751199405SJohn Fastabend sk_other = psock->sk;
98851199405SJohn Fastabend if (sock_flag(sk_other, SOCK_DEAD) ||
989e5c6de5fSJohn Fastabend !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
99051199405SJohn Fastabend goto out_free;
99151199405SJohn Fastabend
992e3526bb9SCong Wang skb_bpf_set_ingress(skb);
9939ecbfb06SJohn Fastabend
9949ecbfb06SJohn Fastabend /* If the queue is empty then we can submit directly
9959ecbfb06SJohn Fastabend * into the msg queue. If its not empty we have to
9969ecbfb06SJohn Fastabend * queue work otherwise we may get OOO data. Otherwise,
9979ecbfb06SJohn Fastabend * if sk_psock_skb_ingress errors will be handled by
9989ecbfb06SJohn Fastabend * retrying later from workqueue.
9999ecbfb06SJohn Fastabend */
10009ecbfb06SJohn Fastabend if (skb_queue_empty(&psock->ingress_skb)) {
10017303524eSLiu Jian len = skb->len;
10027303524eSLiu Jian off = 0;
10037303524eSLiu Jian if (skb_bpf_strparser(skb)) {
10047303524eSLiu Jian struct strp_msg *stm = strp_msg(skb);
10057303524eSLiu Jian
10067303524eSLiu Jian off = stm->offset;
10077303524eSLiu Jian len = stm->full_len;
10087303524eSLiu Jian }
10097303524eSLiu Jian err = sk_psock_skb_ingress_self(psock, skb, off, len);
10109ecbfb06SJohn Fastabend }
10119ecbfb06SJohn Fastabend if (err < 0) {
1012799aa7f9SCong Wang spin_lock_bh(&psock->ingress_lock);
1013799aa7f9SCong Wang if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
101451199405SJohn Fastabend skb_queue_tail(&psock->ingress_skb, skb);
101529173d07SJohn Fastabend schedule_delayed_work(&psock->work, 0);
10160cf6672bSCong Wang err = 0;
10179ecbfb06SJohn Fastabend }
1018799aa7f9SCong Wang spin_unlock_bh(&psock->ingress_lock);
1019e5c6de5fSJohn Fastabend if (err < 0)
10200cf6672bSCong Wang goto out_free;
10210cf6672bSCong Wang }
102251199405SJohn Fastabend break;
1023604326b4SDaniel Borkmann case __SK_REDIRECT:
1024e5c6de5fSJohn Fastabend tcp_eat_skb(psock->sk, skb);
102542830571SCong Wang err = sk_psock_skb_redirect(psock, skb);
1026604326b4SDaniel Borkmann break;
1027604326b4SDaniel Borkmann case __SK_DROP:
1028604326b4SDaniel Borkmann default:
1029604326b4SDaniel Borkmann out_free:
1030e5c6de5fSJohn Fastabend skb_bpf_redirect_clear(skb);
1031e5c6de5fSJohn Fastabend tcp_eat_skb(psock->sk, skb);
1032781dd043SCong Wang sock_drop(psock->sk, skb);
1033604326b4SDaniel Borkmann }
10341581a6c1SCong Wang
10351581a6c1SCong Wang return err;
1036604326b4SDaniel Borkmann }
1037604326b4SDaniel Borkmann
sk_psock_write_space(struct sock * sk)103888759609SCong Wang static void sk_psock_write_space(struct sock *sk)
103988759609SCong Wang {
104088759609SCong Wang struct sk_psock *psock;
104188759609SCong Wang void (*write_space)(struct sock *sk) = NULL;
104288759609SCong Wang
104388759609SCong Wang rcu_read_lock();
104488759609SCong Wang psock = sk_psock(sk);
104588759609SCong Wang if (likely(psock)) {
104688759609SCong Wang if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
104729173d07SJohn Fastabend schedule_delayed_work(&psock->work, 0);
104888759609SCong Wang write_space = psock->saved_write_space;
104988759609SCong Wang }
105088759609SCong Wang rcu_read_unlock();
105188759609SCong Wang if (write_space)
105288759609SCong Wang write_space(sk);
105388759609SCong Wang }
105488759609SCong Wang
105588759609SCong Wang #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
sk_psock_strp_read(struct strparser * strp,struct sk_buff * skb)1056604326b4SDaniel Borkmann static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
1057604326b4SDaniel Borkmann {
10588025751dSJohn Fastabend struct sk_psock *psock;
1059604326b4SDaniel Borkmann struct bpf_prog *prog;
1060604326b4SDaniel Borkmann int ret = __SK_DROP;
10618025751dSJohn Fastabend struct sock *sk;
1062604326b4SDaniel Borkmann
1063604326b4SDaniel Borkmann rcu_read_lock();
10648025751dSJohn Fastabend sk = strp->sk;
10658025751dSJohn Fastabend psock = sk_psock(sk);
10668025751dSJohn Fastabend if (unlikely(!psock)) {
1067781dd043SCong Wang sock_drop(sk, skb);
10688025751dSJohn Fastabend goto out;
10698025751dSJohn Fastabend }
1070ae8b8332SCong Wang prog = READ_ONCE(psock->progs.stream_verdict);
1071604326b4SDaniel Borkmann if (likely(prog)) {
1072144748ebSJohn Fastabend skb->sk = sk;
1073e3526bb9SCong Wang skb_dst_drop(skb);
1074e3526bb9SCong Wang skb_bpf_redirect_clear(skb);
107553334232SCong Wang ret = bpf_prog_run_pin_on_cpu(prog, skb);
10767303524eSLiu Jian skb_bpf_set_strparser(skb);
1077e3526bb9SCong Wang ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
1078144748ebSJohn Fastabend skb->sk = NULL;
1079604326b4SDaniel Borkmann }
1080604326b4SDaniel Borkmann sk_psock_verdict_apply(psock, skb, ret);
10818025751dSJohn Fastabend out:
108293dd5f18SJohn Fastabend rcu_read_unlock();
1083604326b4SDaniel Borkmann }
1084604326b4SDaniel Borkmann
sk_psock_strp_read_done(struct strparser * strp,int err)1085604326b4SDaniel Borkmann static int sk_psock_strp_read_done(struct strparser *strp, int err)
1086604326b4SDaniel Borkmann {
1087604326b4SDaniel Borkmann return err;
1088604326b4SDaniel Borkmann }
1089604326b4SDaniel Borkmann
sk_psock_strp_parse(struct strparser * strp,struct sk_buff * skb)1090604326b4SDaniel Borkmann static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
1091604326b4SDaniel Borkmann {
10925a685cd9SCong Wang struct sk_psock *psock = container_of(strp, struct sk_psock, strp);
1093604326b4SDaniel Borkmann struct bpf_prog *prog;
1094604326b4SDaniel Borkmann int ret = skb->len;
1095604326b4SDaniel Borkmann
1096604326b4SDaniel Borkmann rcu_read_lock();
1097ae8b8332SCong Wang prog = READ_ONCE(psock->progs.stream_parser);
10980b17ad25SJohn Fastabend if (likely(prog)) {
10990b17ad25SJohn Fastabend skb->sk = psock->sk;
110053334232SCong Wang ret = bpf_prog_run_pin_on_cpu(prog, skb);
11010b17ad25SJohn Fastabend skb->sk = NULL;
11020b17ad25SJohn Fastabend }
1103604326b4SDaniel Borkmann rcu_read_unlock();
1104604326b4SDaniel Borkmann return ret;
1105604326b4SDaniel Borkmann }
1106604326b4SDaniel Borkmann
1107604326b4SDaniel Borkmann /* Called with socket lock held. */
sk_psock_strp_data_ready(struct sock * sk)1108552de910SJohn Fastabend static void sk_psock_strp_data_ready(struct sock *sk)
1109604326b4SDaniel Borkmann {
1110604326b4SDaniel Borkmann struct sk_psock *psock;
1111604326b4SDaniel Borkmann
111240e0b090SPeilin Ye trace_sk_data_ready(sk);
111340e0b090SPeilin Ye
1114604326b4SDaniel Borkmann rcu_read_lock();
1115604326b4SDaniel Borkmann psock = sk_psock(sk);
1116604326b4SDaniel Borkmann if (likely(psock)) {
1117e91de6afSJohn Fastabend if (tls_sw_has_ctx_rx(sk)) {
11185a685cd9SCong Wang psock->saved_data_ready(sk);
1119e91de6afSJohn Fastabend } else {
1120604326b4SDaniel Borkmann write_lock_bh(&sk->sk_callback_lock);
11215a685cd9SCong Wang strp_data_ready(&psock->strp);
1122604326b4SDaniel Borkmann write_unlock_bh(&sk->sk_callback_lock);
1123604326b4SDaniel Borkmann }
1124e91de6afSJohn Fastabend }
1125604326b4SDaniel Borkmann rcu_read_unlock();
1126604326b4SDaniel Borkmann }
1127604326b4SDaniel Borkmann
sk_psock_init_strp(struct sock * sk,struct sk_psock * psock)112888759609SCong Wang int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
112988759609SCong Wang {
1130809e4dc7SXu Kuohai int ret;
1131809e4dc7SXu Kuohai
113288759609SCong Wang static const struct strp_callbacks cb = {
113388759609SCong Wang .rcv_msg = sk_psock_strp_read,
113488759609SCong Wang .read_sock_done = sk_psock_strp_read_done,
113588759609SCong Wang .parse_msg = sk_psock_strp_parse,
113688759609SCong Wang };
113788759609SCong Wang
1138809e4dc7SXu Kuohai ret = strp_init(&psock->strp, sk, &cb);
1139809e4dc7SXu Kuohai if (!ret)
1140809e4dc7SXu Kuohai sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);
1141809e4dc7SXu Kuohai
1142809e4dc7SXu Kuohai return ret;
114388759609SCong Wang }
114488759609SCong Wang
sk_psock_start_strp(struct sock * sk,struct sk_psock * psock)114588759609SCong Wang void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
114688759609SCong Wang {
11475a685cd9SCong Wang if (psock->saved_data_ready)
114888759609SCong Wang return;
114988759609SCong Wang
11505a685cd9SCong Wang psock->saved_data_ready = sk->sk_data_ready;
115188759609SCong Wang sk->sk_data_ready = sk_psock_strp_data_ready;
115288759609SCong Wang sk->sk_write_space = sk_psock_write_space;
115388759609SCong Wang }
115488759609SCong Wang
sk_psock_stop_strp(struct sock * sk,struct sk_psock * psock)115588759609SCong Wang void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
115688759609SCong Wang {
1157c0d95d33SJohn Fastabend psock_set_prog(&psock->progs.stream_parser, NULL);
1158c0d95d33SJohn Fastabend
11595a685cd9SCong Wang if (!psock->saved_data_ready)
116088759609SCong Wang return;
116188759609SCong Wang
11625a685cd9SCong Wang sk->sk_data_ready = psock->saved_data_ready;
11635a685cd9SCong Wang psock->saved_data_ready = NULL;
11645a685cd9SCong Wang strp_stop(&psock->strp);
116588759609SCong Wang }
116688759609SCong Wang
sk_psock_done_strp(struct sk_psock * psock)116788759609SCong Wang static void sk_psock_done_strp(struct sk_psock *psock)
116888759609SCong Wang {
116988759609SCong Wang /* Parser has been stopped */
1170809e4dc7SXu Kuohai if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))
11715a685cd9SCong Wang strp_done(&psock->strp);
117288759609SCong Wang }
117388759609SCong Wang #else
sk_psock_done_strp(struct sk_psock * psock)117488759609SCong Wang static void sk_psock_done_strp(struct sk_psock *psock)
117588759609SCong Wang {
117688759609SCong Wang }
117788759609SCong Wang #endif /* CONFIG_BPF_STREAM_PARSER */
117888759609SCong Wang
sk_psock_verdict_recv(struct sock * sk,struct sk_buff * skb)1179965b57b4SCong Wang static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
1180ef565928SJohn Fastabend {
1181ef565928SJohn Fastabend struct sk_psock *psock;
1182ef565928SJohn Fastabend struct bpf_prog *prog;
1183ef565928SJohn Fastabend int ret = __SK_DROP;
1184965b57b4SCong Wang int len = skb->len;
1185ef565928SJohn Fastabend
1186ef565928SJohn Fastabend rcu_read_lock();
1187ef565928SJohn Fastabend psock = sk_psock(sk);
1188ef565928SJohn Fastabend if (unlikely(!psock)) {
1189ef565928SJohn Fastabend len = 0;
1190e5c6de5fSJohn Fastabend tcp_eat_skb(sk, skb);
1191781dd043SCong Wang sock_drop(sk, skb);
1192ef565928SJohn Fastabend goto out;
1193ef565928SJohn Fastabend }
1194ae8b8332SCong Wang prog = READ_ONCE(psock->progs.stream_verdict);
1195a7ba4558SCong Wang if (!prog)
1196a7ba4558SCong Wang prog = READ_ONCE(psock->progs.skb_verdict);
1197ef565928SJohn Fastabend if (likely(prog)) {
1198e3526bb9SCong Wang skb_dst_drop(skb);
1199e3526bb9SCong Wang skb_bpf_redirect_clear(skb);
120053334232SCong Wang ret = bpf_prog_run_pin_on_cpu(prog, skb);
1201e3526bb9SCong Wang ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
1202ef565928SJohn Fastabend }
12032e23acd9SCong Wang ret = sk_psock_verdict_apply(psock, skb, ret);
12042e23acd9SCong Wang if (ret < 0)
12052e23acd9SCong Wang len = ret;
1206ef565928SJohn Fastabend out:
1207ef565928SJohn Fastabend rcu_read_unlock();
1208ef565928SJohn Fastabend return len;
1209ef565928SJohn Fastabend }
1210ef565928SJohn Fastabend
sk_psock_verdict_data_ready(struct sock * sk)1211ef565928SJohn Fastabend static void sk_psock_verdict_data_ready(struct sock *sk)
1212ef565928SJohn Fastabend {
1213ef565928SJohn Fastabend struct socket *sock = sk->sk_socket;
12141ded5e5aSEric Dumazet const struct proto_ops *ops;
12156df7f764SJohn Fastabend int copied;
1216ef565928SJohn Fastabend
121740e0b090SPeilin Ye trace_sk_data_ready(sk);
121840e0b090SPeilin Ye
12191ded5e5aSEric Dumazet if (unlikely(!sock))
1220ef565928SJohn Fastabend return;
12211ded5e5aSEric Dumazet ops = READ_ONCE(sock->ops);
12221ded5e5aSEric Dumazet if (!ops || !ops->read_skb)
12231ded5e5aSEric Dumazet return;
12241ded5e5aSEric Dumazet copied = ops->read_skb(sk, sk_psock_verdict_recv);
12256df7f764SJohn Fastabend if (copied >= 0) {
12266df7f764SJohn Fastabend struct sk_psock *psock;
12276df7f764SJohn Fastabend
12286df7f764SJohn Fastabend rcu_read_lock();
12296df7f764SJohn Fastabend psock = sk_psock(sk);
12306648e613SJason Xing if (psock)
12314cd12c60SShigeru Yoshida sk_psock_data_ready(sk, psock);
12326df7f764SJohn Fastabend rcu_read_unlock();
12336df7f764SJohn Fastabend }
1234ef565928SJohn Fastabend }
1235ef565928SJohn Fastabend
sk_psock_start_verdict(struct sock * sk,struct sk_psock * psock)1236ef565928SJohn Fastabend void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1237ef565928SJohn Fastabend {
12385a685cd9SCong Wang if (psock->saved_data_ready)
1239ef565928SJohn Fastabend return;
1240ef565928SJohn Fastabend
12415a685cd9SCong Wang psock->saved_data_ready = sk->sk_data_ready;
1242ef565928SJohn Fastabend sk->sk_data_ready = sk_psock_verdict_data_ready;
1243ef565928SJohn Fastabend sk->sk_write_space = sk_psock_write_space;
1244ef565928SJohn Fastabend }
1245ef565928SJohn Fastabend
sk_psock_stop_verdict(struct sock * sk,struct sk_psock * psock)1246ef565928SJohn Fastabend void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1247ef565928SJohn Fastabend {
1248c0d95d33SJohn Fastabend psock_set_prog(&psock->progs.stream_verdict, NULL);
1249c0d95d33SJohn Fastabend psock_set_prog(&psock->progs.skb_verdict, NULL);
1250c0d95d33SJohn Fastabend
12515a685cd9SCong Wang if (!psock->saved_data_ready)
1252ef565928SJohn Fastabend return;
1253ef565928SJohn Fastabend
12545a685cd9SCong Wang sk->sk_data_ready = psock->saved_data_ready;
12555a685cd9SCong Wang psock->saved_data_ready = NULL;
1256ef565928SJohn Fastabend }
1257