Lines Matching +full:default +full:- +full:trim
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
15 if (msg->sg.end > msg->sg.start &&
16 elem_first_coalesce < msg->sg.end)
19 if (msg->sg.end < msg->sg.start &&
20 (elem_first_coalesce > msg->sg.start ||
21 elem_first_coalesce < msg->sg.end))
31 u32 osize = msg->sg.size;
34 len -= msg->sg.size;
41 ret = -ENOMEM;
45 orig_offset = pfrag->offset;
46 use = min_t(int, len, pfrag->size - orig_offset);
48 ret = -ENOMEM;
52 i = msg->sg.end;
54 sge = &msg->sg.data[i];
57 sg_page(sge) == pfrag->page &&
58 sge->offset + sge->length == orig_offset) {
59 sge->length += use;
62 ret = -ENOSPC;
66 sge = &msg->sg.data[msg->sg.end];
68 sg_set_page(sge, pfrag->page, use, orig_offset);
69 get_page(pfrag->page);
74 msg->sg.size += use;
75 pfrag->offset += use;
76 len -= use;
90 int i = src->sg.start;
96 if (sge->length > off)
98 off -= sge->length;
100 if (i == src->sg.end && off)
101 return -ENOSPC;
106 sge_len = sge->length - off;
110 if (dst->sg.end)
111 sgd = sk_msg_elem(dst, dst->sg.end - 1);
115 (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) {
116 sgd->length += sge_len;
117 dst->sg.size += sge_len;
119 sge_off = sge->offset + off;
122 return -ENOSPC;
126 len -= sge_len;
129 if (i == src->sg.end && len)
130 return -ENOSPC;
140 int i = msg->sg.start;
145 if (bytes < sge->length) {
146 sge->length -= bytes;
147 sge->offset += bytes;
152 sk_mem_uncharge(sk, sge->length);
153 bytes -= sge->length;
154 sge->length = 0;
155 sge->offset = 0;
157 } while (bytes && i != msg->sg.end);
158 msg->sg.start = i;
164 int i = msg->sg.start;
167 struct scatterlist *sge = &msg->sg.data[i];
168 int uncharge = (bytes < sge->length) ? bytes : sge->length;
171 bytes -= uncharge;
173 } while (i != msg->sg.end);
181 u32 len = sge->length;
184 if (!msg->skb) {
199 while (msg->sg.size) {
200 msg->sg.size -= sge->length;
203 sk_msg_check_to_free(msg, i, msg->sg.size);
206 consume_skb(msg->skb);
213 return __sk_msg_free(sk, msg, msg->sg.start, false);
219 return __sk_msg_free(sk, msg, msg->sg.start, true);
227 u32 i = msg->sg.start;
231 if (!sge->length)
233 if (bytes < sge->length) {
236 sge->length -= bytes;
237 sge->offset += bytes;
238 msg->sg.size -= bytes;
242 msg->sg.size -= sge->length;
243 bytes -= sge->length;
248 msg->sg.start = i;
265 int trim = msg->sg.size - len;
266 u32 i = msg->sg.end;
268 if (trim <= 0) {
269 WARN_ON(trim < 0);
274 msg->sg.size = len;
275 while (msg->sg.data[i].length &&
276 trim >= msg->sg.data[i].length) {
277 trim -= msg->sg.data[i].length;
280 if (!trim)
284 msg->sg.data[i].length -= trim;
285 sk_mem_uncharge(sk, trim);
287 if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length)
288 msg->sg.copybreak = msg->sg.data[i].length;
291 msg->sg.end = i;
293 /* If we trim data a full sg elem before curr pointer update
299 if (!msg->sg.size) {
300 msg->sg.curr = msg->sg.start;
301 msg->sg.copybreak = 0;
302 } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >=
303 sk_msg_iter_dist(msg->sg.start, msg->sg.end)) {
305 msg->sg.curr = i;
306 msg->sg.copybreak = msg->sg.data[i].length;
319 orig = msg->sg.size;
322 maxpages = to_max_pages - num_elems;
324 ret = -EFAULT;
331 ret = -EFAULT;
335 bytes -= copied;
336 msg->sg.size += copied;
339 use = min_t(int, copied, PAGE_SIZE - offset);
340 sg_set_page(&msg->sg.data[msg->sg.end],
342 sg_unmark_end(&msg->sg.data[msg->sg.end]);
346 copied -= use;
355 msg->sg.copybreak = 0;
356 msg->sg.curr = msg->sg.end;
359 /* Revert iov_iter updates, msg will need to use 'trim' later if it
363 iov_iter_revert(from, msg->sg.size - orig);
371 int ret = -ENOSPC, i = msg->sg.curr;
378 /* This is possible if a trim operation shrunk the buffer */
379 if (msg->sg.copybreak >= sge->length) {
380 msg->sg.copybreak = 0;
382 if (i == msg->sg.end)
387 buf_size = sge->length - msg->sg.copybreak;
389 to = sg_virt(sge) + msg->sg.copybreak;
390 msg->sg.copybreak += copy;
391 if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY)
396 ret = -EFAULT;
399 bytes -= copy;
403 msg->sg.copybreak = 0;
405 } while (i != msg->sg.end);
407 msg->sg.curr = i;
412 /* Receive sk_msg from psock->ingress_msg to @msg. */
416 struct iov_iter *iter = &msg->msg_iter;
428 i = msg_rx->sg.start;
434 copy = sge->length;
437 copy = len - copied;
439 copy = copy_page_to_iter(page, sge->offset, copy, iter);
441 copied = copied ? copied : -EFAULT;
447 sge->offset += copy;
448 sge->length -= copy;
449 if (!msg_rx->skb) {
451 atomic_sub(copy, &sk->sk_rmem_alloc);
453 msg_rx->sg.size -= copy;
455 if (!sge->length) {
457 if (!msg_rx->skb)
464 if (copy != sge->length)
471 } while ((i != msg_rx->sg.end) && !sg_is_last(sge));
480 msg_rx->sg.start = i;
481 if (!sge->length && (i == msg_rx->sg.end || sg_is_last(sge))) {
500 empty = list_empty(&psock->ingress_msg);
513 sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
520 if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
523 if (!sk_rmem_schedule(sk, skb, skb->truesize))
542 num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
551 return -EAGAIN;
553 num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
559 psock->ingress_bytes += len;
562 msg->sg.start = 0;
563 msg->sg.size = copied;
564 msg->sg.end = num_sge;
565 msg->skb = take_ref ? skb_get(skb) : skb;
578 struct sock *sk = psock->sk;
582 /* If we are receiving on the same sock skb->sk is already assigned,
586 if (unlikely(skb->sk == sk))
590 return -EAGAIN;
613 struct sock *sk = psock->sk;
617 return -EAGAIN;
629 if (!sock_writeable(psock->sk))
630 return -EAGAIN;
631 return skb_send_sock(psock->sk, skb, off, len);
641 spin_lock_bh(&psock->ingress_lock);
643 state->len = len;
644 state->off = off;
646 spin_unlock_bh(&psock->ingress_lock);
653 struct sk_psock_work_state *state = &psock->work_state;
671 if (!sk_psock_get(psock->sk))
673 mutex_lock(&psock->work_mutex);
674 while ((skb = skb_peek(&psock->ingress_skb))) {
675 len = skb->len;
680 off = stm->offset;
681 len = stm->full_len;
685 if (unlikely(state->len)) {
686 len = state->len;
687 off = state->off;
693 ret = -EIO;
694 if (!sock_flag(psock->sk, SOCK_DEAD))
698 if (ret == -EAGAIN) {
701 skb_bpf_set_redir(skb, psock->sk, ingress);
706 schedule_delayed_work(&psock->work, 1);
710 sk_psock_report_error(psock, ret ? -ret : EPIPE);
715 len -= ret;
720 skb = skb_dequeue(&psock->ingress_skb);
724 mutex_unlock(&psock->work_mutex);
725 sk_psock_put(psock->sk, psock);
733 write_lock_bh(&sk->sk_callback_lock);
736 psock = ERR_PTR(-EINVAL);
740 if (sk->sk_user_data) {
741 psock = ERR_PTR(-EBUSY);
747 psock = ERR_PTR(-ENOMEM);
751 prot = READ_ONCE(sk->sk_prot);
752 psock->sk = sk;
753 psock->eval = __SK_NONE;
754 psock->sk_proto = prot;
755 psock->saved_unhash = prot->unhash;
756 psock->saved_destroy = prot->destroy;
757 psock->saved_close = prot->close;
758 psock->saved_write_space = sk->sk_write_space;
760 INIT_LIST_HEAD(&psock->link);
761 spin_lock_init(&psock->link_lock);
763 INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);
764 mutex_init(&psock->work_mutex);
765 INIT_LIST_HEAD(&psock->ingress_msg);
766 spin_lock_init(&psock->ingress_lock);
767 skb_queue_head_init(&psock->ingress_skb);
770 refcount_set(&psock->refcnt, 1);
778 write_unlock_bh(&sk->sk_callback_lock);
787 spin_lock_bh(&psock->link_lock);
788 link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
791 list_del(&link->list);
792 spin_unlock_bh(&psock->link_lock);
800 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
801 list_del(&msg->list);
802 if (!msg->skb)
803 atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc);
804 sk_msg_free(psock->sk, msg);
813 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
815 sock_drop(psock->sk, skb);
824 list_for_each_entry_safe(link, tmp, &psock->link, list) {
825 list_del(&link->list);
832 spin_lock_bh(&psock->ingress_lock);
835 spin_unlock_bh(&psock->ingress_lock);
848 cancel_delayed_work_sync(&psock->work);
850 mutex_destroy(&psock->work_mutex);
852 psock_progs_drop(&psock->progs);
857 if (psock->sk_redir)
858 sock_put(psock->sk_redir);
859 if (psock->sk_pair)
860 sock_put(psock->sk_pair);
861 sock_put(psock->sk);
867 write_lock_bh(&sk->sk_callback_lock);
870 if (psock->progs.stream_parser)
872 else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
874 write_unlock_bh(&sk->sk_callback_lock);
878 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
879 queue_rcu_work(system_percpu_wq, &psock->rwork);
889 default:
903 prog = READ_ONCE(psock->progs.msg_parser);
910 msg->sk = sk;
912 ret = sk_psock_map_verd(ret, msg->sk_redir);
913 psock->apply_bytes = msg->apply_bytes;
915 if (psock->sk_redir) {
916 sock_put(psock->sk_redir);
917 psock->sk_redir = NULL;
919 if (!msg->sk_redir) {
923 psock->redir_ingress = sk_msg_to_ingress(msg);
924 psock->sk_redir = msg->sk_redir;
925 sock_hold(psock->sk_redir);
944 sock_drop(from->sk, skb);
945 return -EIO;
954 sock_drop(from->sk, skb);
955 return -EIO;
957 spin_lock_bh(&psock_other->ingress_lock);
959 spin_unlock_bh(&psock_other->ingress_lock);
961 sock_drop(from->sk, skb);
962 return -EIO;
965 skb_queue_tail(&psock_other->ingress_skb, skb);
966 schedule_delayed_work(&psock_other->work, 0);
967 spin_unlock_bh(&psock_other->ingress_lock);
980 default:
991 prog = READ_ONCE(psock->progs.stream_verdict);
993 skb->sk = psock->sk;
998 skb->sk = NULL;
1015 err = -EIO;
1016 sk_other = psock->sk;
1029 if (skb_queue_empty(&psock->ingress_skb)) {
1030 len = skb->len;
1035 off = stm->offset;
1036 len = stm->full_len;
1041 spin_lock_bh(&psock->ingress_lock);
1043 skb_queue_tail(&psock->ingress_skb, skb);
1044 schedule_delayed_work(&psock->work, 0);
1047 spin_unlock_bh(&psock->ingress_lock);
1053 tcp_eat_skb(psock->sk, skb);
1057 default:
1060 tcp_eat_skb(psock->sk, skb);
1061 sock_drop(psock->sk, skb);
1076 schedule_delayed_work(&psock->work, 0);
1077 write_space = psock->saved_write_space;
1093 sk = strp->sk;
1099 prog = READ_ONCE(psock->progs.stream_verdict);
1101 skb->sk = sk;
1107 skb->sk = NULL;
1123 int ret = skb->len;
1126 prog = READ_ONCE(psock->progs.stream_parser);
1128 skb->sk = psock->sk;
1130 skb->sk = NULL;
1147 psock->saved_data_ready(sk);
1149 read_lock_bh(&sk->sk_callback_lock);
1150 strp_data_ready(&psock->strp);
1151 read_unlock_bh(&sk->sk_callback_lock);
1167 ret = strp_init(&psock->strp, sk, &cb);
1172 psock->strp.cb.read_sock = tcp_bpf_strp_read_sock;
1173 psock->copied_seq = tcp_sk(sk)->copied_seq;
1180 if (psock->saved_data_ready)
1183 psock->saved_data_ready = sk->sk_data_ready;
1184 sk->sk_data_ready = sk_psock_strp_data_ready;
1185 sk->sk_write_space = sk_psock_write_space;
1190 psock_set_prog(&psock->progs.stream_parser, NULL);
1192 if (!psock->saved_data_ready)
1195 sk->sk_data_ready = psock->saved_data_ready;
1196 psock->saved_data_ready = NULL;
1197 strp_stop(&psock->strp);
1204 strp_done(&psock->strp);
1217 int len = skb->len;
1227 prog = READ_ONCE(psock->progs.stream_verdict);
1229 prog = READ_ONCE(psock->progs.skb_verdict);
1246 struct socket *sock = sk->sk_socket;
1254 ops = READ_ONCE(sock->ops);
1255 if (!ops || !ops->read_skb)
1257 copied = ops->read_skb(sk, sk_psock_verdict_recv);
1271 if (psock->saved_data_ready)
1274 psock->saved_data_ready = sk->sk_data_ready;
1275 sk->sk_data_ready = sk_psock_verdict_data_ready;
1276 sk->sk_write_space = sk_psock_write_space;
1281 psock_set_prog(&psock->progs.stream_verdict, NULL);
1282 psock_set_prog(&psock->progs.skb_verdict, NULL);
1284 if (!psock->saved_data_ready)
1287 sk->sk_data_ready = psock->saved_data_ready;
1288 psock->saved_data_ready = NULL;