Lines Matching refs:cp
65 void rds_send_path_reset(struct rds_conn_path *cp) in rds_send_path_reset() argument
70 if (cp->cp_xmit_rm) { in rds_send_path_reset()
71 rm = cp->cp_xmit_rm; in rds_send_path_reset()
72 cp->cp_xmit_rm = NULL; in rds_send_path_reset()
81 cp->cp_xmit_sg = 0; in rds_send_path_reset()
82 cp->cp_xmit_hdr_off = 0; in rds_send_path_reset()
83 cp->cp_xmit_data_off = 0; in rds_send_path_reset()
84 cp->cp_xmit_atomic_sent = 0; in rds_send_path_reset()
85 cp->cp_xmit_rdma_sent = 0; in rds_send_path_reset()
86 cp->cp_xmit_data_sent = 0; in rds_send_path_reset()
88 cp->cp_conn->c_map_queued = 0; in rds_send_path_reset()
90 cp->cp_unacked_packets = rds_sysctl_max_unacked_packets; in rds_send_path_reset()
91 cp->cp_unacked_bytes = rds_sysctl_max_unacked_bytes; in rds_send_path_reset()
94 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_path_reset()
95 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_reset()
99 list_splice_init(&cp->cp_retrans, &cp->cp_send_queue); in rds_send_path_reset()
100 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_path_reset()
104 static int acquire_in_xmit(struct rds_conn_path *cp) in acquire_in_xmit() argument
106 return test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags) == 0; in acquire_in_xmit()
109 static void release_in_xmit(struct rds_conn_path *cp) in release_in_xmit() argument
111 clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); in release_in_xmit()
118 if (waitqueue_active(&cp->cp_waitq)) in release_in_xmit()
119 wake_up_all(&cp->cp_waitq); in release_in_xmit()
187 int rds_send_xmit(struct rds_conn_path *cp) in rds_send_xmit() argument
189 struct rds_connection *conn = cp->cp_conn; in rds_send_xmit()
210 if (!acquire_in_xmit(cp)) { in rds_send_xmit()
216 if (rds_destroy_pending(cp->cp_conn)) { in rds_send_xmit()
217 release_in_xmit(cp); in rds_send_xmit()
230 send_gen = READ_ONCE(cp->cp_send_gen) + 1; in rds_send_xmit()
231 WRITE_ONCE(cp->cp_send_gen, send_gen); in rds_send_xmit()
237 if (!rds_conn_path_up(cp)) { in rds_send_xmit()
238 release_in_xmit(cp); in rds_send_xmit()
244 conn->c_trans->xmit_path_prepare(cp); in rds_send_xmit()
252 rm = cp->cp_xmit_rm; in rds_send_xmit()
276 rm->m_inc.i_conn_path = cp; in rds_send_xmit()
277 rm->m_inc.i_conn = cp->cp_conn; in rds_send_xmit()
279 cp->cp_xmit_rm = rm; in rds_send_xmit()
305 if (cp->cp_index > 0 && rds_mprds_cp0_catchup(conn)) { in rds_send_xmit()
310 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_xmit()
312 if (!list_empty(&cp->cp_send_queue)) { in rds_send_xmit()
313 rm = list_entry(cp->cp_send_queue.next, in rds_send_xmit()
323 &cp->cp_retrans); in rds_send_xmit()
326 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_xmit()
341 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_xmit()
344 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_xmit()
350 if (cp->cp_unacked_packets == 0 || in rds_send_xmit()
351 cp->cp_unacked_bytes < len) { in rds_send_xmit()
354 cp->cp_unacked_packets = in rds_send_xmit()
356 cp->cp_unacked_bytes = in rds_send_xmit()
360 cp->cp_unacked_bytes -= len; in rds_send_xmit()
361 cp->cp_unacked_packets--; in rds_send_xmit()
364 cp->cp_xmit_rm = rm; in rds_send_xmit()
368 if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) { in rds_send_xmit()
380 cp->cp_xmit_rdma_sent = 1; in rds_send_xmit()
384 if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) { in rds_send_xmit()
396 cp->cp_xmit_atomic_sent = 1; in rds_send_xmit()
422 if (rm->data.op_active && !cp->cp_xmit_data_sent) { in rds_send_xmit()
426 cp->cp_xmit_hdr_off, in rds_send_xmit()
427 cp->cp_xmit_sg, in rds_send_xmit()
428 cp->cp_xmit_data_off); in rds_send_xmit()
432 if (cp->cp_xmit_hdr_off < sizeof(struct rds_header)) { in rds_send_xmit()
435 cp->cp_xmit_hdr_off); in rds_send_xmit()
436 cp->cp_xmit_hdr_off += tmp; in rds_send_xmit()
440 sg = &rm->data.op_sg[cp->cp_xmit_sg]; in rds_send_xmit()
443 cp->cp_xmit_data_off); in rds_send_xmit()
444 cp->cp_xmit_data_off += tmp; in rds_send_xmit()
446 if (cp->cp_xmit_data_off == sg->length) { in rds_send_xmit()
447 cp->cp_xmit_data_off = 0; in rds_send_xmit()
449 cp->cp_xmit_sg++; in rds_send_xmit()
450 BUG_ON(ret != 0 && cp->cp_xmit_sg == in rds_send_xmit()
455 if (cp->cp_xmit_hdr_off == sizeof(struct rds_header) && in rds_send_xmit()
456 (cp->cp_xmit_sg == rm->data.op_nents)) in rds_send_xmit()
457 cp->cp_xmit_data_sent = 1; in rds_send_xmit()
465 if (!rm->data.op_active || cp->cp_xmit_data_sent) { in rds_send_xmit()
466 cp->cp_xmit_rm = NULL; in rds_send_xmit()
467 cp->cp_xmit_sg = 0; in rds_send_xmit()
468 cp->cp_xmit_hdr_off = 0; in rds_send_xmit()
469 cp->cp_xmit_data_off = 0; in rds_send_xmit()
470 cp->cp_xmit_rdma_sent = 0; in rds_send_xmit()
471 cp->cp_xmit_atomic_sent = 0; in rds_send_xmit()
472 cp->cp_xmit_data_sent = 0; in rds_send_xmit()
480 conn->c_trans->xmit_path_complete(cp); in rds_send_xmit()
481 release_in_xmit(cp); in rds_send_xmit()
510 raced = send_gen != READ_ONCE(cp->cp_send_gen); in rds_send_xmit()
513 !list_empty(&cp->cp_send_queue)) && !raced) { in rds_send_xmit()
517 if (rds_destroy_pending(cp->cp_conn)) in rds_send_xmit()
520 queue_delayed_work(cp->cp_wq, in rds_send_xmit()
521 &cp->cp_send_w, 1); in rds_send_xmit()
745 void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack, in rds_send_path_drop_acked() argument
752 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_path_drop_acked()
754 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_drop_acked()
766 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_path_drop_acked()
785 struct rds_conn_path *cp; in rds_send_drop_to() local
816 cp = rm->m_inc.i_conn_path; in rds_send_drop_to()
818 cp = &conn->c_path[0]; in rds_send_drop_to()
820 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_drop_to()
827 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_drop_to()
831 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_drop_to()
878 struct rds_conn_path *cp, in rds_send_queue_rm() argument
923 rm->m_inc.i_conn_path = cp; in rds_send_queue_rm()
926 spin_lock(&cp->cp_lock); in rds_send_queue_rm()
927 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++); in rds_send_queue_rm()
928 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_queue_rm()
930 spin_unlock(&cp->cp_lock); in rds_send_queue_rm()
1472 rds_send_probe(struct rds_conn_path *cp, __be16 sport, in rds_send_probe() argument
1485 rm->m_daddr = cp->cp_conn->c_faddr; in rds_send_probe()
1488 rds_conn_path_connect_if_down(cp); in rds_send_probe()
1490 ret = rds_cong_wait(cp->cp_conn->c_fcong, dport, 1, NULL); in rds_send_probe()
1494 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_probe()
1495 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_probe()
1498 rm->m_inc.i_conn = cp->cp_conn; in rds_send_probe()
1499 rm->m_inc.i_conn_path = cp; in rds_send_probe()
1502 cp->cp_next_tx_seq); in rds_send_probe()
1504 cp->cp_next_tx_seq++; in rds_send_probe()
1507 cp->cp_conn->c_trans->t_mp_capable) { in rds_send_probe()
1509 __be32 my_gen_num = cpu_to_be32(cp->cp_conn->c_my_gen_num); in rds_send_probe()
1521 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_probe()
1528 if (!rds_destroy_pending(cp->cp_conn)) in rds_send_probe()
1529 queue_delayed_work(cp->cp_wq, &cp->cp_send_w, 1); in rds_send_probe()
1542 rds_send_pong(struct rds_conn_path *cp, __be16 dport) in rds_send_pong() argument
1544 return rds_send_probe(cp, 0, dport, 0); in rds_send_pong()
1551 struct rds_conn_path *cp = &conn->c_path[cp_index]; in rds_send_ping() local
1553 spin_lock_irqsave(&cp->cp_lock, flags); in rds_send_ping()
1555 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_ping()
1559 spin_unlock_irqrestore(&cp->cp_lock, flags); in rds_send_ping()
1560 rds_send_probe(cp, cpu_to_be16(RDS_FLAG_PROBE_PORT), 0, 0); in rds_send_ping()