Lines Matching refs:rm
67 struct rds_message *rm, *tmp;
71 rm = cp->cp_xmit_rm;
77 rds_message_unmapped(rm);
78 rds_message_put(rm);
95 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
96 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
97 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags);
190 struct rds_message *rm;
252 rm = cp->cp_xmit_rm;
254 if (!rm) {
269 if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) {
270 rm = rds_cong_update_alloc(conn);
271 if (IS_ERR(rm)) {
272 ret = PTR_ERR(rm);
275 rm->data.op_active = 1;
276 rm->m_inc.i_conn_path = cp;
277 rm->m_inc.i_conn = cp->cp_conn;
279 cp->cp_xmit_rm = rm;
289 if (!rm) {
313 rm = list_entry(cp->cp_send_queue.next,
316 rds_message_addref(rm);
322 list_move_tail(&rm->m_conn_item,
328 if (!rm)
338 if (test_bit(RDS_MSG_FLUSH, &rm->m_flags) ||
339 (rm->rdma.op_active &&
340 test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))) {
342 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags))
343 list_move(&rm->m_conn_item, &to_be_dropped);
349 len = ntohl(rm->m_inc.i_hdr.h_len);
352 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
364 cp->cp_xmit_rm = rm;
368 if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) {
369 rm->m_final_op = &rm->rdma;
373 set_bit(RDS_MSG_MAPPED, &rm->m_flags);
374 ret = conn->c_trans->xmit_rdma(conn, &rm->rdma);
376 clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
377 wake_up_interruptible(&rm->m_flush_wait);
384 if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) {
385 rm->m_final_op = &rm->atomic;
389 set_bit(RDS_MSG_MAPPED, &rm->m_flags);
390 ret = conn->c_trans->xmit_atomic(conn, &rm->atomic);
392 clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
393 wake_up_interruptible(&rm->m_flush_wait);
407 if (rm->data.op_nents == 0) {
411 ops_present = (rm->atomic.op_active || rm->rdma.op_active);
412 if (rm->atomic.op_active && !rm->atomic.op_silent)
414 if (rm->rdma.op_active && !rm->rdma.op_silent)
418 && !rm->m_rdma_cookie)
419 rm->data.op_active = 0;
422 if (rm->data.op_active && !cp->cp_xmit_data_sent) {
423 rm->m_final_op = &rm->data;
425 ret = conn->c_trans->xmit(conn, rm,
440 sg = &rm->data.op_sg[cp->cp_xmit_sg];
451 rm->data.op_nents);
456 (cp->cp_xmit_sg == rm->data.op_nents))
461 * A rm will only take multiple times through this loop
463 * none), then we're done with the rm.
465 if (!rm->data.op_active || cp->cp_xmit_data_sent) {
474 rds_message_put(rm);
486 list_for_each_entry(rm, &to_be_dropped, m_conn_item)
487 rds_message_put(rm);
532 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)
534 u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len);
545 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack,
549 return is_acked(rm, ack);
550 return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack;
559 void rds_rdma_send_complete(struct rds_message *rm, int status)
566 spin_lock_irqsave(&rm->m_rs_lock, flags);
568 ro = &rm->rdma;
569 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) &&
572 rs = rm->m_rs;
583 spin_unlock_irqrestore(&rm->m_rs_lock, flags);
595 void rds_atomic_send_complete(struct rds_message *rm, int status)
602 spin_lock_irqsave(&rm->m_rs_lock, flags);
604 ao = &rm->atomic;
605 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)
608 rs = rm->m_rs;
619 spin_unlock_irqrestore(&rm->m_rs_lock, flags);
634 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)
639 ro = &rm->rdma;
646 ao = &rm->atomic;
668 struct rds_message *rm;
673 rm = list_entry(messages->next, struct rds_message,
675 list_del_init(&rm->m_conn_item);
683 * The message spinlock makes sure nobody clears rm->m_rs
687 spin_lock_irqsave(&rm->m_rs_lock, flags);
688 if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags))
691 if (rs != rm->m_rs) {
696 rs = rm->m_rs;
704 if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) {
705 struct rm_rdma_op *ro = &rm->rdma;
708 list_del_init(&rm->m_sock_item);
709 rds_send_sndbuf_remove(rs, rm);
718 rm->rdma.op_notifier = NULL;
725 spin_unlock_irqrestore(&rm->m_rs_lock, flags);
726 rds_message_put(rm);
728 rds_message_put(rm);
748 struct rds_message *rm, *tmp;
754 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
755 if (!rds_send_is_acked(rm, ack, is_acked))
758 list_move(&rm->m_conn_item, &list);
759 clear_bit(RDS_MSG_ON_CONN, &rm->m_flags);
783 struct rds_message *rm, *tmp;
792 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) {
794 (!ipv6_addr_equal(&dest->sin6_addr, &rm->m_daddr) ||
795 dest->sin6_port != rm->m_inc.i_hdr.h_dport))
798 list_move(&rm->m_sock_item, &list);
799 rds_send_sndbuf_remove(rs, rm);
800 clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags);
812 list_for_each_entry(rm, &list, m_sock_item) {
814 conn = rm->m_inc.i_conn;
816 cp = rm->m_inc.i_conn_path;
822 * Maybe someone else beat us to removing rm from the conn.
826 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) {
830 list_del_init(&rm->m_conn_item);
837 spin_lock_irqsave(&rm->m_rs_lock, flags);
840 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED);
843 spin_unlock_irqrestore(&rm->m_rs_lock, flags);
845 rds_message_put(rm);
851 rm = list_entry(list.next, struct rds_message, m_sock_item);
852 list_del_init(&rm->m_sock_item);
853 rds_message_wait(rm);
860 spin_lock_irqsave(&rm->m_rs_lock, flags);
863 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED);
866 spin_unlock_irqrestore(&rm->m_rs_lock, flags);
868 rds_message_put(rm);
879 struct rds_message *rm, __be16 sport,
888 len = be32_to_cpu(rm->m_inc.i_hdr.h_len);
911 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
913 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue);
914 set_bit(RDS_MSG_ON_SOCK, &rm->m_flags);
915 rds_message_addref(rm);
917 rm->m_rs = rs;
921 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0);
922 rm->m_inc.i_conn = conn;
923 rm->m_inc.i_conn_path = cp;
924 rds_message_addref(rm);
927 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++);
928 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
929 set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
933 rm, len, rs, rs->rs_snd_bytes,
934 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence));
1030 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm,
1036 !rm->data.op_mmp_znotifier)
1039 rm->data.op_mmp_znotifier->z_cookie = *cookie;
1043 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,
1058 * rm->rdma.m_rdma_cookie and rm->rdma.m_rdma_mr.
1064 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]);
1069 ret = rds_cmsg_rdma_dest(rs, rm, cmsg);
1073 ret = rds_cmsg_rdma_map(rs, rm, cmsg);
1086 ret = rds_cmsg_atomic(rs, rm, cmsg);
1090 ret = rds_cmsg_zcopy(rs, rm, cmsg);
1134 struct rds_message *rm = NULL;
1289 /* size of rm including all sgs */
1294 rm = rds_message_alloc(ret, GFP_KERNEL);
1295 if (!rm) {
1300 /* Attach data to the rm */
1302 rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);
1303 if (IS_ERR(rm->data.op_sg)) {
1304 ret = PTR_ERR(rm->data.op_sg);
1307 ret = rds_message_copy_from_user(rm, &msg->msg_iter, zcopy);
1311 rm->data.op_active = 1;
1313 rm->m_daddr = daddr;
1360 rm->m_conn_path = cpath;
1363 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct);
1367 if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) {
1369 &rm->rdma, conn->c_trans->xmit_rdma);
1374 if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) {
1376 &rm->atomic, conn->c_trans->xmit_atomic);
1394 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port,
1404 rds_send_queue_rm(rs, conn, cpath, rm,
1439 rds_message_put(rm);
1456 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1);
1458 if (rm)
1459 rds_message_put(rm);
1475 struct rds_message *rm;
1479 rm = rds_message_alloc(0, GFP_ATOMIC);
1480 if (!rm) {
1485 rm->m_daddr = cp->cp_conn->c_faddr;
1486 rm->data.op_active = 1;
1495 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
1496 set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
1497 rds_message_addref(rm);
1498 rm->m_inc.i_conn = cp->cp_conn;
1499 rm->m_inc.i_conn_path = cp;
1501 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport,
1503 rm->m_inc.i_hdr.h_flags |= h_flags;
1512 rds_message_add_extension(&rm->m_inc.i_hdr,
1514 rds_message_add_extension(&rm->m_inc.i_hdr,
1517 rds_message_add_extension(&rm->m_inc.i_hdr,
1532 rds_message_put(rm);
1536 if (rm)
1537 rds_message_put(rm);