Lines Matching refs:rs
532 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
536 assert_spin_locked(&rs->rs_lock); in rds_send_sndbuf_remove()
538 BUG_ON(rs->rs_snd_bytes < len); in rds_send_sndbuf_remove()
539 rs->rs_snd_bytes -= len; in rds_send_sndbuf_remove()
541 if (rs->rs_snd_bytes == 0) in rds_send_sndbuf_remove()
561 struct rds_sock *rs = NULL; in rds_rdma_send_complete() local
572 rs = rm->m_rs; in rds_rdma_send_complete()
573 sock_hold(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
576 spin_lock(&rs->rs_lock); in rds_rdma_send_complete()
577 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_rdma_send_complete()
578 spin_unlock(&rs->rs_lock); in rds_rdma_send_complete()
585 if (rs) { in rds_rdma_send_complete()
586 rds_wake_sk_sleep(rs); in rds_rdma_send_complete()
587 sock_put(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
597 struct rds_sock *rs = NULL; in rds_atomic_send_complete() local
608 rs = rm->m_rs; in rds_atomic_send_complete()
609 sock_hold(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
612 spin_lock(&rs->rs_lock); in rds_atomic_send_complete()
613 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_atomic_send_complete()
614 spin_unlock(&rs->rs_lock); in rds_atomic_send_complete()
621 if (rs) { in rds_atomic_send_complete()
622 rds_wake_sk_sleep(rs); in rds_atomic_send_complete()
623 sock_put(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
634 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
642 list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
649 list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
667 struct rds_sock *rs = NULL; in rds_send_remove_from_sock() local
691 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
692 if (rs) { in rds_send_remove_from_sock()
693 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
694 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
696 rs = rm->m_rs; in rds_send_remove_from_sock()
697 if (rs) in rds_send_remove_from_sock()
698 sock_hold(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
700 if (!rs) in rds_send_remove_from_sock()
702 spin_lock(&rs->rs_lock); in rds_send_remove_from_sock()
709 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
715 &rs->rs_notify_queue); in rds_send_remove_from_sock()
722 spin_unlock(&rs->rs_lock); in rds_send_remove_from_sock()
731 if (rs) { in rds_send_remove_from_sock()
732 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
733 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
781 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest) in rds_send_drop_to() argument
790 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_drop_to()
792 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
799 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
806 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_drop_to()
839 spin_lock(&rs->rs_lock); in rds_send_drop_to()
840 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
841 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
848 rds_wake_sk_sleep(rs); in rds_send_drop_to()
862 spin_lock(&rs->rs_lock); in rds_send_drop_to()
863 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
864 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
877 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn, in rds_send_queue_rm() argument
892 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_queue_rm()
902 if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) { in rds_send_queue_rm()
903 rs->rs_snd_bytes += len; in rds_send_queue_rm()
910 if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2) in rds_send_queue_rm()
913 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
916 sock_hold(rds_rs_to_sk(rs)); in rds_send_queue_rm()
917 rm->m_rs = rs; in rds_send_queue_rm()
933 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
939 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_queue_rm()
1030 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
1043 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
1064 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]); in rds_cmsg_send()
1069 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
1073 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
1086 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
1090 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1130 struct rds_sock *rs = rds_sk_to_rs(sk); in rds_sendmsg() local
1145 sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY)); in rds_sendmsg()
1230 daddr = rs->rs_conn_addr; in rds_sendmsg()
1231 dport = rs->rs_conn_port; in rds_sendmsg()
1232 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1237 if (ipv6_addr_any(&rs->rs_bound_addr) || ipv6_addr_any(&daddr)) { in rds_sendmsg()
1247 ipv6_addr_v4mapped(&rs->rs_bound_addr)) { in rds_sendmsg()
1256 if (scope_id != rs->rs_bound_scope_id) { in rds_sendmsg()
1258 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1259 } else if (rs->rs_bound_scope_id) { in rds_sendmsg()
1277 if (payload_len > rds_sk_sndbuf(rs)) { in rds_sendmsg()
1283 if (rs->rs_transport->t_type != RDS_TRANS_TCP) { in rds_sendmsg()
1317 if (rs->rs_conn && ipv6_addr_equal(&rs->rs_conn->c_faddr, &daddr) && in rds_sendmsg()
1318 rs->rs_tos == rs->rs_conn->c_tos) { in rds_sendmsg()
1319 conn = rs->rs_conn; in rds_sendmsg()
1322 &rs->rs_bound_addr, &daddr, in rds_sendmsg()
1323 rs->rs_transport, rs->rs_tos, in rds_sendmsg()
1330 rs->rs_conn = conn; in rds_sendmsg()
1337 cpath = &conn->c_path[RDS_MPATH_HASH(rs, conn->c_npaths ? : 1)]; in rds_sendmsg()
1363 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct); in rds_sendmsg()
1389 ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs); in rds_sendmsg()
1391 rs->rs_seen_congestion = 1; in rds_sendmsg()
1394 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1404 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1405 rs->rs_bound_port, in rds_sendmsg()
1456 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()