97 	struct rds_sock *rs = mr->r_sock;  in rds_destroy_mr()  local
104 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
106 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_destroy_mr()
109 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_destroy_mr()
127 void rds_rdma_drop_keys(struct rds_sock *rs)  in rds_rdma_drop_keys()  argument
134 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
135 	while ((node = rb_first(&rs->rs_rdma_keys))) {  in rds_rdma_drop_keys()
137 		if (mr->r_trans == rs->rs_transport)  in rds_rdma_drop_keys()
139 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_drop_keys()
141 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
143 		spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
145 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_drop_keys()
147 	if (rs->rs_transport && rs->rs_transport->flush_mrs)  in rds_rdma_drop_keys()
148 		rs->rs_transport->flush_mrs();  in rds_rdma_drop_keys()
172 static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args,  in __rds_rdma_map()  argument
188 	if (ipv6_addr_any(&rs->rs_bound_addr) || !rs->rs_transport) {  in __rds_rdma_map()
193 	if (!rs->rs_transport->get_mr) {  in __rds_rdma_map()
245 	mr->r_trans = rs->rs_transport;  in __rds_rdma_map()
246 	mr->r_sock = rs;  in __rds_rdma_map()
290 	trans_private = rs->rs_transport->get_mr(  in __rds_rdma_map()
291 		sg, nents, rs, &mr->r_key, cp ? cp->cp_conn : NULL,  in __rds_rdma_map()
339 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
340 	found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);  in __rds_rdma_map()
341 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in __rds_rdma_map()
359 int rds_get_mr(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_get_mr()  argument
369 	return __rds_rdma_map(rs, &args, NULL, NULL, NULL);  in rds_get_mr()
372 int rds_get_mr_for_dest(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_get_mr_for_dest()  argument
393 	return __rds_rdma_map(rs, &new_args, NULL, NULL, NULL);  in rds_get_mr_for_dest()
399 int rds_free_mr(struct rds_sock *rs, sockptr_t optval, int optlen)  in rds_free_mr()  argument
413 		if (!rs->rs_transport || !rs->rs_transport->flush_mrs)  in rds_free_mr()
415 		rs->rs_transport->flush_mrs();  in rds_free_mr()
423 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_free_mr()
424 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
426 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_free_mr()
431 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_free_mr()
445 void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force)  in rds_rdma_unuse()  argument
451 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
452 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_rdma_unuse()
456 		spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
469 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_unuse()
473 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_rdma_unuse()
611 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_args()  argument
631 	if (ipv6_addr_any(&rs->rs_bound_addr)) {  in rds_cmsg_rdma_args()
668 	op->op_recverr = rs->rs_recverr;  in rds_cmsg_rdma_args()
715 		rs->rs_user_addr = iov->addr;  in rds_cmsg_rdma_args()
716 		rs->rs_user_bytes = iov->bytes;  in rds_cmsg_rdma_args()
729 			if (!rs->rs_transport->get_mr) {  in rds_cmsg_rdma_args()
741 			local_odp_mr->r_trans = rs->rs_transport;  in rds_cmsg_rdma_args()
742 			local_odp_mr->r_sock = rs;  in rds_cmsg_rdma_args()
744 				rs->rs_transport->get_mr(  in rds_cmsg_rdma_args()
745 					NULL, 0, rs, &local_odp_mr->r_key, NULL,  in rds_cmsg_rdma_args()
811 int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_dest()  argument
832 	spin_lock_irqsave(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
833 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_cmsg_rdma_dest()
838 	spin_unlock_irqrestore(&rs->rs_rdma_lock, flags);  in rds_cmsg_rdma_dest()
854 int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_map()  argument
861 	return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie,  in rds_cmsg_rdma_map()
868 int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_atomic()  argument
914 	rm->atomic.op_recverr = rs->rs_recverr;  in rds_cmsg_atomic()