Lines Matching +full:rs +full:-
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
47 refcount_set(&inc->i_refcount, 1); in rds_inc_init()
48 INIT_LIST_HEAD(&inc->i_item); in rds_inc_init()
49 inc->i_conn = conn; in rds_inc_init()
50 inc->i_saddr = *saddr; in rds_inc_init()
51 inc->i_usercopy.rdma_cookie = 0; in rds_inc_init()
52 inc->i_usercopy.rx_tstamp = ktime_set(0, 0); in rds_inc_init()
54 memset(inc->i_rx_lat_trace, 0, sizeof(inc->i_rx_lat_trace)); in rds_inc_init()
61 refcount_set(&inc->i_refcount, 1); in rds_inc_path_init()
62 INIT_LIST_HEAD(&inc->i_item); in rds_inc_path_init()
63 inc->i_conn = cp->cp_conn; in rds_inc_path_init()
64 inc->i_conn_path = cp; in rds_inc_path_init()
65 inc->i_saddr = *saddr; in rds_inc_path_init()
66 inc->i_usercopy.rdma_cookie = 0; in rds_inc_path_init()
67 inc->i_usercopy.rx_tstamp = ktime_set(0, 0); in rds_inc_path_init()
73 rdsdebug("addref inc %p ref %d\n", inc, refcount_read(&inc->i_refcount)); in rds_inc_addref()
74 refcount_inc(&inc->i_refcount); in rds_inc_addref()
79 rdsdebug("put inc %p ref %d\n", inc, refcount_read(&inc->i_refcount)); in rds_inc_put()
80 if (refcount_dec_and_test(&inc->i_refcount)) { in rds_inc_put()
81 BUG_ON(!list_empty(&inc->i_item)); in rds_inc_put()
83 inc->i_conn->c_trans->inc_free(inc); in rds_inc_put()
88 static void rds_recv_rcvbuf_delta(struct rds_sock *rs, struct sock *sk, in rds_recv_rcvbuf_delta() argument
97 rs->rs_rcv_bytes += delta; in rds_recv_rcvbuf_delta()
101 rds_stats_add(s_recv_bytes_removed_from_socket, -delta); in rds_recv_rcvbuf_delta()
104 if (rs->rs_transport->t_type == RDS_TRANS_LOOP) in rds_recv_rcvbuf_delta()
107 now_congested = rs->rs_rcv_bytes > rds_sk_rcvbuf(rs); in rds_recv_rcvbuf_delta()
109 rdsdebug("rs %p (%pI6c:%u) recv bytes %d buf %d " in rds_recv_rcvbuf_delta()
111 rs, &rs->rs_bound_addr, in rds_recv_rcvbuf_delta()
112 ntohs(rs->rs_bound_port), rs->rs_rcv_bytes, in rds_recv_rcvbuf_delta()
113 rds_sk_rcvbuf(rs), now_congested, delta); in rds_recv_rcvbuf_delta()
115 /* wasn't -> am congested */ in rds_recv_rcvbuf_delta()
116 if (!rs->rs_congested && now_congested) { in rds_recv_rcvbuf_delta()
117 rs->rs_congested = 1; in rds_recv_rcvbuf_delta()
121 /* was -> aren't congested */ in rds_recv_rcvbuf_delta()
124 else if (rs->rs_congested && (rs->rs_rcv_bytes < (rds_sk_rcvbuf(rs)/2))) { in rds_recv_rcvbuf_delta()
125 rs->rs_congested = 0; in rds_recv_rcvbuf_delta()
140 WARN_ON(conn->c_trans->t_type != RDS_TRANS_TCP); in rds_conn_peer_gen_update()
142 if (conn->c_peer_gen_num != 0 && in rds_conn_peer_gen_update()
143 peer_gen_num != conn->c_peer_gen_num) { in rds_conn_peer_gen_update()
147 cp = &conn->c_path[i]; in rds_conn_peer_gen_update()
148 spin_lock_irqsave(&cp->cp_lock, flags); in rds_conn_peer_gen_update()
149 cp->cp_next_tx_seq = 1; in rds_conn_peer_gen_update()
150 cp->cp_next_rx_seq = 0; in rds_conn_peer_gen_update()
152 &cp->cp_retrans, in rds_conn_peer_gen_update()
154 set_bit(RDS_MSG_FLUSH, &rm->m_flags); in rds_conn_peer_gen_update()
156 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_conn_peer_gen_update()
159 conn->c_peer_gen_num = peer_gen_num; in rds_conn_peer_gen_update()
166 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs) in rds_recv_incoming_exthdrs() argument
168 struct rds_header *hdr = &inc->i_hdr; in rds_recv_incoming_exthdrs()
184 rds_rdma_unuse(rs, be32_to_cpu(buffer.rdma.h_rdma_rkey), 0); in rds_recv_incoming_exthdrs()
190 inc->i_usercopy.rdma_cookie = rds_rdma_make_cookie( in rds_recv_incoming_exthdrs()
218 conn->c_npaths = min_t(int, RDS_MPATH_WORKERS, in rds_recv_hs_exthdrs()
229 /* if RDS_EXTHDR_NPATHS was not found, default to a single-path */ in rds_recv_hs_exthdrs()
230 conn->c_npaths = max_t(int, conn->c_npaths, 1); in rds_recv_hs_exthdrs()
231 conn->c_ping_triggered = 0; in rds_recv_hs_exthdrs()
240 * 2. rcvr of probe-ping knows the mprds_paths = min(s_npaths, r_npaths). It
241 * sends back a probe-pong with r_npaths. After that, if rcvr is the
246 * called after reception of the probe-pong on all mprds_paths.
247 * Otherwise (sender of probe-ping is not the smaller ip addr): just call
258 if (conn->c_npaths > 1 && in rds_start_mprds()
259 rds_addr_cmp(&conn->c_laddr, &conn->c_faddr) < 0) { in rds_start_mprds()
260 for (i = 0; i < conn->c_npaths; i++) { in rds_start_mprds()
261 cp = &conn->c_path[i]; in rds_start_mprds()
273 * instead of per-flow which means that we don't have to go digging through
277 * is only to reduce the memory consumption of pre-posted buffers.
287 struct rds_sock *rs = NULL; in rds_recv_incoming() local
292 inc->i_conn = conn; in rds_recv_incoming()
293 inc->i_rx_jiffies = jiffies; in rds_recv_incoming()
294 if (conn->c_trans->t_mp_capable) in rds_recv_incoming()
295 cp = inc->i_conn_path; in rds_recv_incoming()
297 cp = &conn->c_path[0]; in rds_recv_incoming()
301 (unsigned long long)cp->cp_next_rx_seq, in rds_recv_incoming()
303 (unsigned long long)be64_to_cpu(inc->i_hdr.h_sequence), in rds_recv_incoming()
304 be32_to_cpu(inc->i_hdr.h_len), in rds_recv_incoming()
305 be16_to_cpu(inc->i_hdr.h_sport), in rds_recv_incoming()
306 be16_to_cpu(inc->i_hdr.h_dport), in rds_recv_incoming()
307 inc->i_hdr.h_flags, in rds_recv_incoming()
308 inc->i_rx_jiffies); in rds_recv_incoming()
330 if (be64_to_cpu(inc->i_hdr.h_sequence) < cp->cp_next_rx_seq && in rds_recv_incoming()
331 (inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) { in rds_recv_incoming()
335 cp->cp_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1; in rds_recv_incoming()
337 if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) { in rds_recv_incoming()
338 if (inc->i_hdr.h_sport == 0) { in rds_recv_incoming()
344 rds_send_pong(cp, inc->i_hdr.h_sport); in rds_recv_incoming()
346 if (RDS_HS_PROBE(be16_to_cpu(inc->i_hdr.h_sport), in rds_recv_incoming()
347 be16_to_cpu(inc->i_hdr.h_dport))) { in rds_recv_incoming()
348 rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn); in rds_recv_incoming()
349 rds_start_mprds(cp->cp_conn); in rds_recv_incoming()
354 if (be16_to_cpu(inc->i_hdr.h_dport) == RDS_FLAG_PROBE_PORT && in rds_recv_incoming()
355 inc->i_hdr.h_sport == 0) { in rds_recv_incoming()
356 rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn); in rds_recv_incoming()
358 rds_start_mprds(cp->cp_conn); in rds_recv_incoming()
359 wake_up(&cp->cp_conn->c_hs_waitq); in rds_recv_incoming()
363 rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if); in rds_recv_incoming()
364 if (!rs) { in rds_recv_incoming()
370 rds_recv_incoming_exthdrs(inc, rs); in rds_recv_incoming()
373 sk = rds_rs_to_sk(rs); in rds_recv_incoming()
375 /* serialize with rds_release -> sock_orphan */ in rds_recv_incoming()
376 write_lock_irqsave(&rs->rs_recv_lock, flags); in rds_recv_incoming()
378 rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs); in rds_recv_incoming()
380 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_recv_incoming()
381 be32_to_cpu(inc->i_hdr.h_len), in rds_recv_incoming()
382 inc->i_hdr.h_dport); in rds_recv_incoming()
384 inc->i_usercopy.rx_tstamp = ktime_get_real(); in rds_recv_incoming()
386 inc->i_rx_lat_trace[RDS_MSG_RX_END] = local_clock(); in rds_recv_incoming()
387 list_add_tail(&inc->i_item, &rs->rs_recv_queue); in rds_recv_incoming()
392 write_unlock_irqrestore(&rs->rs_recv_lock, flags); in rds_recv_incoming()
395 if (rs) in rds_recv_incoming()
396 rds_sock_put(rs); in rds_recv_incoming()
404 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc) in rds_next_incoming() argument
409 read_lock_irqsave(&rs->rs_recv_lock, flags); in rds_next_incoming()
410 if (!list_empty(&rs->rs_recv_queue)) { in rds_next_incoming()
411 *inc = list_entry(rs->rs_recv_queue.next, in rds_next_incoming()
416 read_unlock_irqrestore(&rs->rs_recv_lock, flags); in rds_next_incoming()
422 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc, in rds_still_queued() argument
425 struct sock *sk = rds_rs_to_sk(rs); in rds_still_queued()
430 write_lock_irqsave(&rs->rs_recv_lock, flags); in rds_still_queued()
431 if (!list_empty(&inc->i_item)) { in rds_still_queued()
435 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_still_queued()
436 -be32_to_cpu(inc->i_hdr.h_len), in rds_still_queued()
437 inc->i_hdr.h_dport); in rds_still_queued()
438 list_del_init(&inc->i_item); in rds_still_queued()
442 write_unlock_irqrestore(&rs->rs_recv_lock, flags); in rds_still_queued()
447 rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop); in rds_still_queued()
455 int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msghdr) in rds_notify_queue_get() argument
469 * losing notifications - except when the buffer is so small that it wouldn't in rds_notify_queue_get()
474 max_messages = msghdr->msg_controllen / CMSG_SPACE(sizeof(cmsg)); in rds_notify_queue_get()
479 spin_lock_irqsave(&rs->rs_lock, flags); in rds_notify_queue_get()
480 while (!list_empty(&rs->rs_notify_queue) && count < max_messages) { in rds_notify_queue_get()
481 notifier = list_entry(rs->rs_notify_queue.next, in rds_notify_queue_get()
483 list_move(¬ifier->n_list, ©); in rds_notify_queue_get()
486 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_notify_queue_get()
495 cmsg.user_token = notifier->n_user_token; in rds_notify_queue_get()
496 cmsg.status = notifier->n_status; in rds_notify_queue_get()
504 list_del_init(¬ifier->n_list); in rds_notify_queue_get()
512 spin_lock_irqsave(&rs->rs_lock, flags); in rds_notify_queue_get()
513 list_splice(©, &rs->rs_notify_queue); in rds_notify_queue_get()
514 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_notify_queue_get()
523 static int rds_notify_cong(struct rds_sock *rs, struct msghdr *msghdr) in rds_notify_cong() argument
525 uint64_t notify = rs->rs_cong_notify; in rds_notify_cong()
534 spin_lock_irqsave(&rs->rs_lock, flags); in rds_notify_cong()
535 rs->rs_cong_notify &= ~notify; in rds_notify_cong()
536 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_notify_cong()
545 struct rds_sock *rs) in rds_cmsg_recv() argument
549 if (inc->i_usercopy.rdma_cookie) { in rds_cmsg_recv()
551 sizeof(inc->i_usercopy.rdma_cookie), in rds_cmsg_recv()
552 &inc->i_usercopy.rdma_cookie); in rds_cmsg_recv()
557 if ((inc->i_usercopy.rx_tstamp != 0) && in rds_cmsg_recv()
558 sock_flag(rds_rs_to_sk(rs), SOCK_RCVTSTAMP)) { in rds_cmsg_recv()
560 ns_to_kernel_old_timeval(inc->i_usercopy.rx_tstamp); in rds_cmsg_recv()
562 if (!sock_flag(rds_rs_to_sk(rs), SOCK_TSTAMP_NEW)) { in rds_cmsg_recv()
579 if (rs->rs_rx_traces) { in rds_cmsg_recv()
584 inc->i_rx_lat_trace[RDS_MSG_RX_CMSG] = local_clock(); in rds_cmsg_recv()
585 t.rx_traces = rs->rs_rx_traces; in rds_cmsg_recv()
586 for (i = 0; i < rs->rs_rx_traces; i++) { in rds_cmsg_recv()
587 j = rs->rs_rx_trace[i]; in rds_cmsg_recv()
589 t.rx_trace[i] = inc->i_rx_lat_trace[j + 1] - in rds_cmsg_recv()
590 inc->i_rx_lat_trace[j]; in rds_cmsg_recv()
603 static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg) in rds_recvmsg_zcookie() argument
605 struct rds_msg_zcopy_queue *q = &rs->rs_zcookie_queue; in rds_recvmsg_zcookie()
610 if (!msg->msg_control) in rds_recvmsg_zcookie()
613 if (!sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY) || in rds_recvmsg_zcookie()
614 msg->msg_controllen < CMSG_SPACE(sizeof(*done))) in rds_recvmsg_zcookie()
617 spin_lock_irqsave(&q->lock, flags); in rds_recvmsg_zcookie()
618 if (!list_empty(&q->zcookie_head)) { in rds_recvmsg_zcookie()
619 info = list_entry(q->zcookie_head.next, in rds_recvmsg_zcookie()
621 list_del(&info->rs_zcookie_next); in rds_recvmsg_zcookie()
623 spin_unlock_irqrestore(&q->lock, flags); in rds_recvmsg_zcookie()
626 done = &info->zcookies; in rds_recvmsg_zcookie()
629 spin_lock_irqsave(&q->lock, flags); in rds_recvmsg_zcookie()
630 list_add(&info->rs_zcookie_next, &q->zcookie_head); in rds_recvmsg_zcookie()
631 spin_unlock_irqrestore(&q->lock, flags); in rds_recvmsg_zcookie()
641 struct sock *sk = sock->sk; in rds_recvmsg()
642 struct rds_sock *rs = rds_sk_to_rs(sk); in rds_recvmsg() local
645 DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name); in rds_recvmsg()
646 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); in rds_recvmsg()
649 /* udp_recvmsg()->sock_recvtimeo() gets away without locking too.. */ in rds_recvmsg()
660 /* If there are pending notifications, do those - and nothing else */ in rds_recvmsg()
661 if (!list_empty(&rs->rs_notify_queue)) { in rds_recvmsg()
662 ret = rds_notify_queue_get(rs, msg); in rds_recvmsg()
666 if (rs->rs_cong_notify) { in rds_recvmsg()
667 ret = rds_notify_cong(rs, msg); in rds_recvmsg()
671 if (!rds_next_incoming(rs, &inc)) { in rds_recvmsg()
673 bool reaped = rds_recvmsg_zcookie(rs, msg); in rds_recvmsg()
675 ret = reaped ? 0 : -EAGAIN; in rds_recvmsg()
680 (!list_empty(&rs->rs_notify_queue) || in rds_recvmsg()
681 rs->rs_cong_notify || in rds_recvmsg()
682 rds_next_incoming(rs, &inc)), timeo); in rds_recvmsg()
690 ret = -ETIMEDOUT; in rds_recvmsg()
695 &inc->i_conn->c_faddr, in rds_recvmsg()
696 ntohs(inc->i_hdr.h_sport)); in rds_recvmsg()
697 ret = inc->i_conn->c_trans->inc_copy_to_user(inc, &msg->msg_iter); in rds_recvmsg()
706 if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) { in rds_recvmsg()
710 iov_iter_revert(&msg->msg_iter, ret); in rds_recvmsg()
714 if (ret < be32_to_cpu(inc->i_hdr.h_len)) { in rds_recvmsg()
716 ret = be32_to_cpu(inc->i_hdr.h_len); in rds_recvmsg()
717 msg->msg_flags |= MSG_TRUNC; in rds_recvmsg()
720 if (rds_cmsg_recv(inc, msg, rs)) { in rds_recvmsg()
721 ret = -EFAULT; in rds_recvmsg()
724 rds_recvmsg_zcookie(rs, msg); in rds_recvmsg()
728 if (msg->msg_name) { in rds_recvmsg()
729 if (ipv6_addr_v4mapped(&inc->i_saddr)) { in rds_recvmsg()
730 sin->sin_family = AF_INET; in rds_recvmsg()
731 sin->sin_port = inc->i_hdr.h_sport; in rds_recvmsg()
732 sin->sin_addr.s_addr = in rds_recvmsg()
733 inc->i_saddr.s6_addr32[3]; in rds_recvmsg()
734 memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); in rds_recvmsg()
735 msg->msg_namelen = sizeof(*sin); in rds_recvmsg()
737 sin6->sin6_family = AF_INET6; in rds_recvmsg()
738 sin6->sin6_port = inc->i_hdr.h_sport; in rds_recvmsg()
739 sin6->sin6_addr = inc->i_saddr; in rds_recvmsg()
740 sin6->sin6_flowinfo = 0; in rds_recvmsg()
741 sin6->sin6_scope_id = rs->rs_bound_scope_id; in rds_recvmsg()
742 msg->msg_namelen = sizeof(*sin6); in rds_recvmsg()
760 void rds_clear_recv_queue(struct rds_sock *rs) in rds_clear_recv_queue() argument
762 struct sock *sk = rds_rs_to_sk(rs); in rds_clear_recv_queue()
767 write_lock_irqsave(&rs->rs_recv_lock, flags); in rds_clear_recv_queue()
768 list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) { in rds_clear_recv_queue()
769 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_clear_recv_queue()
770 -be32_to_cpu(inc->i_hdr.h_len), in rds_clear_recv_queue()
771 inc->i_hdr.h_dport); in rds_clear_recv_queue()
772 list_move(&inc->i_item, &to_drop); in rds_clear_recv_queue()
774 write_unlock_irqrestore(&rs->rs_recv_lock, flags); in rds_clear_recv_queue()
777 list_del_init(&inc->i_item); in rds_clear_recv_queue()
783 * inc->i_saddr isn't used here because it is only set in the receive
792 minfo.seq = be64_to_cpu(inc->i_hdr.h_sequence); in rds_inc_info_copy()
793 minfo.len = be32_to_cpu(inc->i_hdr.h_len); in rds_inc_info_copy()
794 minfo.tos = inc->i_conn->c_tos; in rds_inc_info_copy()
799 minfo.lport = inc->i_hdr.h_dport; in rds_inc_info_copy()
800 minfo.fport = inc->i_hdr.h_sport; in rds_inc_info_copy()
804 minfo.lport = inc->i_hdr.h_sport; in rds_inc_info_copy()
805 minfo.fport = inc->i_hdr.h_dport; in rds_inc_info_copy()
821 minfo6.seq = be64_to_cpu(inc->i_hdr.h_sequence); in rds6_inc_info_copy()
822 minfo6.len = be32_to_cpu(inc->i_hdr.h_len); in rds6_inc_info_copy()
823 minfo6.tos = inc->i_conn->c_tos; in rds6_inc_info_copy()
828 minfo6.lport = inc->i_hdr.h_dport; in rds6_inc_info_copy()
829 minfo6.fport = inc->i_hdr.h_sport; in rds6_inc_info_copy()
833 minfo6.lport = inc->i_hdr.h_sport; in rds6_inc_info_copy()
834 minfo6.fport = inc->i_hdr.h_dport; in rds6_inc_info_copy()