Lines Matching refs:zc
988 /* skb changing from pure zc to mixed, must charge zc */
1088 int zc = 0;
1105 zc = MSG_ZEROCOPY;
1115 zc = MSG_ZEROCOPY;
1130 zc = MSG_SPLICE_PAGES;
1242 if (zc == 0) {
1287 } else if (zc == MSG_ZEROCOPY) {
1309 } else if (zc == MSG_SPLICE_PAGES) {
1911 struct tcp_zerocopy_receive *zc,
1919 zc->recv_skip_hint = skb->len - offset;
1935 zc->recv_skip_hint -= partial_frag_remainder;
1943 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);
1944 zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
1951 struct tcp_zerocopy_receive *zc, int inq,
1954 unsigned long copy_address = (unsigned long)zc->copybuf_address;
1958 zc->length = 0;
1959 zc->recv_skip_hint = 0;
1961 if (copy_address != zc->copybuf_address)
1970 tss, &zc->msg_flags);
1974 zc->copybuf_len = err;
1975 if (likely(zc->copybuf_len)) {
1981 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
1986 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
1990 unsigned long copy_address = (unsigned long)zc->copybuf_address;
1994 if (copy_address != zc->copybuf_address)
2004 zc->recv_skip_hint -= copylen;
2010 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
2017 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
2028 zc->msg_flags |= TCP_CMSG_TS;
2032 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
2034 return zc->copybuf_len < 0 ? 0 : copylen;
2043 struct tcp_zerocopy_receive *zc,
2049 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
2080 zc->recv_skip_hint += bytes_not_mapped;
2091 struct tcp_zerocopy_receive *zc,
2113 pages_remaining, address, length, seq, zc, total_bytes_to_map,
2119 struct tcp_zerocopy_receive *zc,
2125 msg_control_addr = (unsigned long)zc->msg_control;
2128 (__kernel_size_t)zc->msg_controllen;
2132 zc->msg_flags = 0;
2133 if (zc->msg_control == msg_control_addr &&
2134 zc->msg_controllen == cmsg_dummy.msg_controllen) {
2136 zc->msg_control = (__u64)
2138 zc->msg_controllen =
2140 zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
2171 struct tcp_zerocopy_receive *zc,
2175 unsigned long address = (unsigned long)zc->address;
2177 s32 copybuf_len = zc->copybuf_len;
2189 zc->copybuf_len = 0;
2190 zc->msg_flags = 0;
2192 if (address & (PAGE_SIZE - 1) || address != zc->address)
2201 return receive_fallback_to_copy(sk, zc, inq, tss);
2204 zc->length = 0;
2205 zc->recv_skip_hint = inq;
2215 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
2219 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
2222 zc->length = total_bytes_to_map;
2223 zc->recv_skip_hint = 0;
2225 zc->length = avail_len;
2226 zc->recv_skip_hint = avail_len;
2229 while (length + PAGE_SIZE <= zc->length) {
2233 if (zc->recv_skip_hint < PAGE_SIZE) {
2237 if (zc->recv_skip_hint > 0)
2250 zc->msg_flags |= TCP_CMSG_TS;
2252 zc->recv_skip_hint = skb->len - offset;
2259 zc->recv_skip_hint);
2261 zc->recv_skip_hint = mappable_offset;
2271 zc->recv_skip_hint -= PAGE_SIZE;
2274 zc->recv_skip_hint < PAGE_SIZE) {
2281 &seq, zc,
2291 zc, total_bytes_to_map);
2300 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
2310 if (length == zc->length)
2311 zc->recv_skip_hint = 0;
2313 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
2316 zc->length = length;
4713 struct tcp_zerocopy_receive zc = {};
4721 if (unlikely(len > sizeof(zc))) {
4722 err = check_zeroed_sockptr(optval, sizeof(zc),
4723 len - sizeof(zc));
4726 len = sizeof(zc);
4730 if (copy_from_sockptr(&zc, optval, len))
4732 if (zc.reserved)
4734 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS))
4737 err = tcp_zerocopy_receive(sk, &zc, &tss);
4739 &zc, &len, err);
4760 if (zc.msg_flags & TCP_CMSG_TS)
4761 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);
4763 zc.msg_flags = 0;
4766 zc.err = sock_error(sk);
4768 zc.inq = tcp_inq_hint(sk);
4770 if (!err && copy_to_sockptr(optval, &zc, len))