Lines Matching full:zc
963 /* skb changing from pure zc to mixed, must charge zc */
1061 int zc = 0; in tcp_sendmsg_locked() local
1070 zc = MSG_ZEROCOPY; in tcp_sendmsg_locked()
1079 zc = MSG_ZEROCOPY; in tcp_sendmsg_locked()
1085 zc = MSG_SPLICE_PAGES; in tcp_sendmsg_locked()
1193 if (zc == 0) { in tcp_sendmsg_locked()
1238 } else if (zc == MSG_ZEROCOPY) { in tcp_sendmsg_locked()
1259 } else if (zc == MSG_SPLICE_PAGES) { in tcp_sendmsg_locked()
1852 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_set_hint_for_skb() argument
1860 zc->recv_skip_hint = skb->len - offset; in tcp_zerocopy_set_hint_for_skb()
1876 zc->recv_skip_hint -= partial_frag_remainder; in tcp_zerocopy_set_hint_for_skb()
1884 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); in tcp_zerocopy_set_hint_for_skb()
1885 zc->recv_skip_hint = mappable_offset + partial_frag_remainder; in tcp_zerocopy_set_hint_for_skb()
1892 struct tcp_zerocopy_receive *zc, int inq, in receive_fallback_to_copy() argument
1895 unsigned long copy_address = (unsigned long)zc->copybuf_address; in receive_fallback_to_copy()
1899 zc->length = 0; in receive_fallback_to_copy()
1900 zc->recv_skip_hint = 0; in receive_fallback_to_copy()
1902 if (copy_address != zc->copybuf_address) in receive_fallback_to_copy()
1911 tss, &zc->msg_flags); in receive_fallback_to_copy()
1915 zc->copybuf_len = err; in receive_fallback_to_copy()
1916 if (likely(zc->copybuf_len)) { in receive_fallback_to_copy()
1922 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset); in receive_fallback_to_copy()
1927 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, in tcp_copy_straggler_data() argument
1931 unsigned long copy_address = (unsigned long)zc->copybuf_address; in tcp_copy_straggler_data()
1935 if (copy_address != zc->copybuf_address) in tcp_copy_straggler_data()
1945 zc->recv_skip_hint -= copylen; in tcp_copy_straggler_data()
1951 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, in tcp_zc_handle_leftover() argument
1958 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); in tcp_zc_handle_leftover()
1969 zc->msg_flags |= TCP_CMSG_TS; in tcp_zc_handle_leftover()
1973 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, in tcp_zc_handle_leftover()
1975 return zc->copybuf_len < 0 ? 0 : copylen; in tcp_zc_handle_leftover()
1984 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_vm_insert_batch_error() argument
1990 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) { in tcp_zerocopy_vm_insert_batch_error()
2021 zc->recv_skip_hint += bytes_not_mapped; in tcp_zerocopy_vm_insert_batch_error()
2032 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_vm_insert_batch() argument
2054 pages_remaining, address, length, seq, zc, total_bytes_to_map, in tcp_zerocopy_vm_insert_batch()
2060 struct tcp_zerocopy_receive *zc, in tcp_zc_finalize_rx_tstamp() argument
2066 msg_control_addr = (unsigned long)zc->msg_control; in tcp_zc_finalize_rx_tstamp()
2069 (__kernel_size_t)zc->msg_controllen; in tcp_zc_finalize_rx_tstamp()
2073 zc->msg_flags = 0; in tcp_zc_finalize_rx_tstamp()
2074 if (zc->msg_control == msg_control_addr && in tcp_zc_finalize_rx_tstamp()
2075 zc->msg_controllen == cmsg_dummy.msg_controllen) { in tcp_zc_finalize_rx_tstamp()
2077 zc->msg_control = (__u64) in tcp_zc_finalize_rx_tstamp()
2079 zc->msg_controllen = in tcp_zc_finalize_rx_tstamp()
2081 zc->msg_flags = (__u32)cmsg_dummy.msg_flags; in tcp_zc_finalize_rx_tstamp()
2112 struct tcp_zerocopy_receive *zc, in tcp_zerocopy_receive() argument
2116 unsigned long address = (unsigned long)zc->address; in tcp_zerocopy_receive()
2118 s32 copybuf_len = zc->copybuf_len; in tcp_zerocopy_receive()
2130 zc->copybuf_len = 0; in tcp_zerocopy_receive()
2131 zc->msg_flags = 0; in tcp_zerocopy_receive()
2133 if (address & (PAGE_SIZE - 1) || address != zc->address) in tcp_zerocopy_receive()
2142 return receive_fallback_to_copy(sk, zc, inq, tss); in tcp_zerocopy_receive()
2145 zc->length = 0; in tcp_zerocopy_receive()
2146 zc->recv_skip_hint = inq; in tcp_zerocopy_receive()
2156 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); in tcp_zerocopy_receive()
2160 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT)) in tcp_zerocopy_receive()
2163 zc->length = total_bytes_to_map; in tcp_zerocopy_receive()
2164 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
2166 zc->length = avail_len; in tcp_zerocopy_receive()
2167 zc->recv_skip_hint = avail_len; in tcp_zerocopy_receive()
2170 while (length + PAGE_SIZE <= zc->length) { in tcp_zerocopy_receive()
2174 if (zc->recv_skip_hint < PAGE_SIZE) { in tcp_zerocopy_receive()
2178 if (zc->recv_skip_hint > 0) in tcp_zerocopy_receive()
2191 zc->msg_flags |= TCP_CMSG_TS; in tcp_zerocopy_receive()
2193 zc->recv_skip_hint = skb->len - offset; in tcp_zerocopy_receive()
2200 zc->recv_skip_hint); in tcp_zerocopy_receive()
2202 zc->recv_skip_hint = mappable_offset; in tcp_zerocopy_receive()
2212 zc->recv_skip_hint -= PAGE_SIZE; in tcp_zerocopy_receive()
2215 zc->recv_skip_hint < PAGE_SIZE) { in tcp_zerocopy_receive()
2222 &seq, zc, in tcp_zerocopy_receive()
2232 zc, total_bytes_to_map); in tcp_zerocopy_receive()
2241 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss); in tcp_zerocopy_receive()
2251 if (length == zc->length) in tcp_zerocopy_receive()
2252 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
2254 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) in tcp_zerocopy_receive()
2257 zc->length = length; in tcp_zerocopy_receive()
4572 struct tcp_zerocopy_receive zc = {}; in do_tcp_getsockopt() local
4580 if (unlikely(len > sizeof(zc))) { in do_tcp_getsockopt()
4581 err = check_zeroed_sockptr(optval, sizeof(zc), in do_tcp_getsockopt()
4582 len - sizeof(zc)); in do_tcp_getsockopt()
4585 len = sizeof(zc); in do_tcp_getsockopt()
4589 if (copy_from_sockptr(&zc, optval, len)) in do_tcp_getsockopt()
4591 if (zc.reserved) in do_tcp_getsockopt()
4593 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) in do_tcp_getsockopt()
4596 err = tcp_zerocopy_receive(sk, &zc, &tss); in do_tcp_getsockopt()
4598 &zc, &len, err); in do_tcp_getsockopt()
4619 if (zc.msg_flags & TCP_CMSG_TS) in do_tcp_getsockopt()
4620 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss); in do_tcp_getsockopt()
4622 zc.msg_flags = 0; in do_tcp_getsockopt()
4625 zc.err = sock_error(sk); in do_tcp_getsockopt()
4627 zc.inq = tcp_inq_hint(sk); in do_tcp_getsockopt()
4629 if (!err && copy_to_sockptr(optval, &zc, len)) in do_tcp_getsockopt()