Lines Matching full:rs
472 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
476 assert_spin_locked(&rs->rs_lock); in rds_send_sndbuf_remove()
478 BUG_ON(rs->rs_snd_bytes < len); in rds_send_sndbuf_remove()
479 rs->rs_snd_bytes -= len; in rds_send_sndbuf_remove()
481 if (rs->rs_snd_bytes == 0) in rds_send_sndbuf_remove()
501 struct rds_sock *rs = NULL; in rds_rdma_send_complete() local
512 rs = rm->m_rs; in rds_rdma_send_complete()
513 sock_hold(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
516 spin_lock(&rs->rs_lock); in rds_rdma_send_complete()
517 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_rdma_send_complete()
518 spin_unlock(&rs->rs_lock); in rds_rdma_send_complete()
525 if (rs) { in rds_rdma_send_complete()
526 rds_wake_sk_sleep(rs); in rds_rdma_send_complete()
527 sock_put(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
537 struct rds_sock *rs = NULL; in rds_atomic_send_complete() local
548 rs = rm->m_rs; in rds_atomic_send_complete()
549 sock_hold(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
552 spin_lock(&rs->rs_lock); in rds_atomic_send_complete()
553 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_atomic_send_complete()
554 spin_unlock(&rs->rs_lock); in rds_atomic_send_complete()
561 if (rs) { in rds_atomic_send_complete()
562 rds_wake_sk_sleep(rs); in rds_atomic_send_complete()
563 sock_put(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
574 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
582 list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
589 list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
607 struct rds_sock *rs = NULL; in rds_send_remove_from_sock() local
631 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
632 if (rs) { in rds_send_remove_from_sock()
633 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
634 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
636 rs = rm->m_rs; in rds_send_remove_from_sock()
637 if (rs) in rds_send_remove_from_sock()
638 sock_hold(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
640 if (!rs) in rds_send_remove_from_sock()
642 spin_lock(&rs->rs_lock); in rds_send_remove_from_sock()
649 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
655 &rs->rs_notify_queue); in rds_send_remove_from_sock()
662 spin_unlock(&rs->rs_lock); in rds_send_remove_from_sock()
671 if (rs) { in rds_send_remove_from_sock()
672 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
673 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
721 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest) in rds_send_drop_to() argument
729 /* get all the messages we're dropping under the rs lock */ in rds_send_drop_to()
730 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_drop_to()
732 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
739 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
743 /* order flag updates with the rs lock */ in rds_send_drop_to()
746 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_drop_to()
779 spin_lock(&rs->rs_lock); in rds_send_drop_to()
780 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
781 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
788 rds_wake_sk_sleep(rs); in rds_send_drop_to()
802 spin_lock(&rs->rs_lock); in rds_send_drop_to()
803 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
804 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
817 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn, in rds_send_queue_rm() argument
832 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_queue_rm()
842 if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) { in rds_send_queue_rm()
843 rs->rs_snd_bytes += len; in rds_send_queue_rm()
850 if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2) in rds_send_queue_rm()
853 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
856 sock_hold(rds_rs_to_sk(rs)); in rds_send_queue_rm()
857 rm->m_rs = rs; in rds_send_queue_rm()
872 rdsdebug("queued msg %p len %d, rs %p bytes %d seq %llu\n", in rds_send_queue_rm()
873 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
879 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_queue_rm()
970 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
983 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
1004 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]); in rds_cmsg_send()
1009 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
1013 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
1026 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
1030 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1044 static int rds_send_mprds_hash(struct rds_sock *rs, in rds_send_mprds_hash() argument
1050 hash = RDS_MPATH_HASH(rs, RDS_MPATH_WORKERS); in rds_send_mprds_hash()
1052 hash = RDS_MPATH_HASH(rs, conn->c_npaths); in rds_send_mprds_hash()
1103 struct rds_sock *rs = rds_sk_to_rs(sk); in rds_sendmsg() local
1118 sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY)); in rds_sendmsg()
1203 daddr = rs->rs_conn_addr; in rds_sendmsg()
1204 dport = rs->rs_conn_port; in rds_sendmsg()
1205 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1210 if (ipv6_addr_any(&rs->rs_bound_addr) || ipv6_addr_any(&daddr)) { in rds_sendmsg()
1220 ipv6_addr_v4mapped(&rs->rs_bound_addr)) { in rds_sendmsg()
1229 if (scope_id != rs->rs_bound_scope_id) { in rds_sendmsg()
1231 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1232 } else if (rs->rs_bound_scope_id) { in rds_sendmsg()
1250 if (payload_len > rds_sk_sndbuf(rs)) { in rds_sendmsg()
1256 if (rs->rs_transport->t_type != RDS_TRANS_TCP) { in rds_sendmsg()
1290 if (rs->rs_conn && ipv6_addr_equal(&rs->rs_conn->c_faddr, &daddr) && in rds_sendmsg()
1291 rs->rs_tos == rs->rs_conn->c_tos) { in rds_sendmsg()
1292 conn = rs->rs_conn; in rds_sendmsg()
1295 &rs->rs_bound_addr, &daddr, in rds_sendmsg()
1296 rs->rs_transport, rs->rs_tos, in rds_sendmsg()
1303 rs->rs_conn = conn; in rds_sendmsg()
1307 cpath = &conn->c_path[rds_send_mprds_hash(rs, conn, nonblock)]; in rds_sendmsg()
1314 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct); in rds_sendmsg()
1340 ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs); in rds_sendmsg()
1342 rs->rs_seen_congestion = 1; in rds_sendmsg()
1345 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1355 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1356 rs->rs_bound_port, in rds_sendmsg()
1405 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()