48 static void rxrpc_tx_backoff(struct rxrpc_call *call, int ret)  in rxrpc_tx_backoff()  argument
51 		if (call->tx_backoff < 1000)  in rxrpc_tx_backoff()
52 			call->tx_backoff += 100;  in rxrpc_tx_backoff()
54 		call->tx_backoff = 0;  in rxrpc_tx_backoff()
60  * lets the far side know we're still interested in this call and helps keep
66 static void rxrpc_set_keepalive(struct rxrpc_call *call, ktime_t now)  in rxrpc_set_keepalive()  argument
68 	ktime_t delay = ms_to_ktime(READ_ONCE(call->next_rx_timo) / 6);  in rxrpc_set_keepalive()
70 	call->keepalive_at = ktime_add(ktime_get_real(), delay);  in rxrpc_set_keepalive()
71 	trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_keepalive);  in rxrpc_set_keepalive()
77 static int rxrpc_alloc_ack(struct rxrpc_call *call, size_t sack_size)  in rxrpc_alloc_ack()  argument
82 	struct kvec *kv = call->local->kvec;  in rxrpc_alloc_ack()
87 	buf = page_frag_alloc(&call->local->tx_alloc,  in rxrpc_alloc_ack()
93 		buf2 = page_frag_alloc(&call->local->tx_alloc, sack_size, gfp);  in rxrpc_alloc_ack()
114 static void rxrpc_free_ack(struct rxrpc_call *call)  in rxrpc_free_ack()  argument
116 	page_frag_free(call->local->kvec[0].iov_base);  in rxrpc_free_ack()
117 	if (call->local->kvec[1].iov_base)  in rxrpc_free_ack()
118 		page_frag_free(call->local->kvec[1].iov_base);  in rxrpc_free_ack()
124 static void rxrpc_begin_rtt_probe(struct rxrpc_call *call, rxrpc_serial_t serial,  in rxrpc_begin_rtt_probe()  argument
127 	unsigned long avail = call->rtt_avail;  in rxrpc_begin_rtt_probe()
134 	if (!test_and_clear_bit(rtt_slot, &call->rtt_avail))  in rxrpc_begin_rtt_probe()
137 	call->rtt_serial[rtt_slot] = serial;  in rxrpc_begin_rtt_probe()
138 	call->rtt_sent_at[rtt_slot] = now;  in rxrpc_begin_rtt_probe()
140 	set_bit(rtt_slot + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);  in rxrpc_begin_rtt_probe()
142 	trace_rxrpc_rtt_tx(call, why, rtt_slot, serial);  in rxrpc_begin_rtt_probe()
146 	trace_rxrpc_rtt_tx(call, rxrpc_rtt_tx_no_slot, rtt_slot, serial);  in rxrpc_begin_rtt_probe()
152 static int rxrpc_fill_out_ack(struct rxrpc_call *call, int nr_kv, u8 ack_reason,  in rxrpc_fill_out_ack()  argument
155 	struct kvec *kv = call->local->kvec;  in rxrpc_fill_out_ack()
166 	rxrpc_inc_stat(call->rxnet, stat_tx_ack_fill);  in rxrpc_fill_out_ack()
168 	window = call->ackr_window;  in rxrpc_fill_out_ack()
169 	wtop   = call->ackr_wtop;  in rxrpc_fill_out_ack()
170 	sack   = call->ackr_sack_base % RXRPC_SACK_SIZE;  in rxrpc_fill_out_ack()
172 	*_ack_serial = rxrpc_get_next_serial(call->conn);  in rxrpc_fill_out_ack()
174 	whdr->epoch		= htonl(call->conn->proto.epoch);  in rxrpc_fill_out_ack()
175 	whdr->cid		= htonl(call->cid);  in rxrpc_fill_out_ack()
176 	whdr->callNumber	= htonl(call->call_id);  in rxrpc_fill_out_ack()
180 	whdr->flags		= call->conn->out_clientflag | RXRPC_SLOW_START_OK;  in rxrpc_fill_out_ack()
182 	whdr->securityIndex	= call->security_ix;  in rxrpc_fill_out_ack()
184 	whdr->serviceId		= htons(call->dest_srx.srx_service);  in rxrpc_fill_out_ack()
189 	ack->previousPacket	= htonl(call->rx_highest_seq);  in rxrpc_fill_out_ack()
207 			memcpy(sackp, call->ackr_sack_table + sack, ack->nAcks);  in rxrpc_fill_out_ack()
209 			memcpy(sackp, call->ackr_sack_table + sack, wrap);  in rxrpc_fill_out_ack()
210 			memcpy(sackp + wrap, call->ackr_sack_table, to - wrap);  in rxrpc_fill_out_ack()
218 	qsize = (window - 1) - call->rx_consumed;  in rxrpc_fill_out_ack()
219 	rsize = max_t(int, call->rx_winsize - qsize, 0);  in rxrpc_fill_out_ack()
221 	if_mtu = call->peer->if_mtu - call->peer->hdrsize;  in rxrpc_fill_out_ack()
222 	if (call->peer->ackr_adv_pmtud) {  in rxrpc_fill_out_ack()
223 		max_mtu = umax(call->peer->max_data, rxrpc_rx_mtu);  in rxrpc_fill_out_ack()
235 		rxrpc_begin_rtt_probe(call, *_ack_serial, now, rxrpc_rtt_tx_ping);  in rxrpc_fill_out_ack()
237 		call->rtt_last_req = now;  in rxrpc_fill_out_ack()
238 	rxrpc_set_keepalive(call, now);  in rxrpc_fill_out_ack()
245 static void rxrpc_send_ack_packet(struct rxrpc_call *call, int nr_kv, size_t len,  in rxrpc_send_ack_packet()  argument
248 	struct kvec *kv = call->local->kvec;  in rxrpc_send_ack_packet()
256 	if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))  in rxrpc_send_ack_packet()
259 	conn = call->conn;  in rxrpc_send_ack_packet()
261 	msg.msg_name	= &call->peer->srx.transport;  in rxrpc_send_ack_packet()
262 	msg.msg_namelen	= call->peer->srx.transport_len;  in rxrpc_send_ack_packet()
267 	trace_rxrpc_tx_ack(call->debug_id, serial,  in rxrpc_send_ack_packet()
272 	rxrpc_inc_stat(call->rxnet, stat_tx_ack_send);  in rxrpc_send_ack_packet()
278 	call->peer->last_tx_at = ktime_get_seconds();  in rxrpc_send_ack_packet()
280 		trace_rxrpc_tx_fail(call->debug_id, serial, ret,  in rxrpc_send_ack_packet()
286 		trace_rxrpc_tx_packet(call->debug_id, whdr,  in rxrpc_send_ack_packet()
289 			call->peer->pmtud_pending = false;  in rxrpc_send_ack_packet()
290 			call->peer->pmtud_probing = true;  in rxrpc_send_ack_packet()
291 			call->conn->pmtud_probe = serial;  in rxrpc_send_ack_packet()
292 			call->conn->pmtud_call = call->debug_id;  in rxrpc_send_ack_packet()
293 			trace_rxrpc_pmtud_tx(call);  in rxrpc_send_ack_packet()
296 	rxrpc_tx_backoff(call, ret);  in rxrpc_send_ack_packet()
302 void rxrpc_send_ACK(struct rxrpc_call *call, u8 ack_reason,  in rxrpc_send_ACK()  argument
305 	struct kvec *kv = call->local->kvec;  in rxrpc_send_ACK()
310 	if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))  in rxrpc_send_ACK()
313 	rxrpc_inc_stat(call->rxnet, stat_tx_acks[ack_reason]);  in rxrpc_send_ACK()
315 	nr_kv = rxrpc_alloc_ack(call, call->ackr_wtop - call->ackr_window);  in rxrpc_send_ACK()
321 	nr_kv = rxrpc_fill_out_ack(call, nr_kv, ack_reason, serial_to_ack, &ack_serial);  in rxrpc_send_ACK()
328 		size_t probe_mtu = call->peer->pmtud_trial + sizeof(struct rxrpc_wire_header);  in rxrpc_send_ACK()
342 	call->ackr_nr_unacked = 0;  in rxrpc_send_ACK()
343 	atomic_set(&call->ackr_nr_consumed, 0);  in rxrpc_send_ACK()
344 	clear_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags);  in rxrpc_send_ACK()
346 	trace_rxrpc_send_ack(call, why, ack_reason, ack_serial);  in rxrpc_send_ACK()
347 	rxrpc_send_ack_packet(call, nr_kv, len, ack_serial, why);  in rxrpc_send_ACK()
349 	rxrpc_free_ack(call);  in rxrpc_send_ACK()
355 void rxrpc_send_probe_for_pmtud(struct rxrpc_call *call)  in rxrpc_send_probe_for_pmtud()  argument
357 	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,  in rxrpc_send_probe_for_pmtud()
362  * Send an ABORT call packet.
364 int rxrpc_send_abort_packet(struct rxrpc_call *call)  in rxrpc_send_abort_packet()  argument
373 	/* Don't bother sending aborts for a client call once the server has  in rxrpc_send_abort_packet()
376 	 * the reply, it's not worth it if we can send a new call on the same  in rxrpc_send_abort_packet()
377 	 * channel instead, thereby closing off this call.  in rxrpc_send_abort_packet()
379 	if (rxrpc_is_client_call(call) &&  in rxrpc_send_abort_packet()
380 	    test_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags))  in rxrpc_send_abort_packet()
383 	if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))  in rxrpc_send_abort_packet()
386 	conn = call->conn;  in rxrpc_send_abort_packet()
388 	msg.msg_name	= &call->peer->srx.transport;  in rxrpc_send_abort_packet()
389 	msg.msg_namelen	= call->peer->srx.transport_len;  in rxrpc_send_abort_packet()
395 	pkt.whdr.cid		= htonl(call->cid);  in rxrpc_send_abort_packet()
396 	pkt.whdr.callNumber	= htonl(call->call_id);  in rxrpc_send_abort_packet()
401 	pkt.whdr.securityIndex	= call->security_ix;  in rxrpc_send_abort_packet()
403 	pkt.whdr.serviceId	= htons(call->dest_srx.srx_service);  in rxrpc_send_abort_packet()
404 	pkt.abort_code		= htonl(call->abort_code);  in rxrpc_send_abort_packet()
416 		trace_rxrpc_tx_fail(call->debug_id, serial, ret,  in rxrpc_send_abort_packet()
419 		trace_rxrpc_tx_packet(call->debug_id, &pkt.whdr,  in rxrpc_send_abort_packet()
421 	rxrpc_tx_backoff(call, ret);  in rxrpc_send_abort_packet()
428 static size_t rxrpc_prepare_data_subpacket(struct rxrpc_call *call,  in rxrpc_prepare_data_subpacket()  argument
436 	struct rxrpc_connection *conn = call->conn;  in rxrpc_prepare_data_subpacket()
437 	struct kvec *kv = &call->local->kvec[1 + subpkt];  in rxrpc_prepare_data_subpacket()
463 	 * service call, lest OpenAFS incorrectly send us an ACK with some  in rxrpc_prepare_data_subpacket()
468 	else if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events))  in rxrpc_prepare_data_subpacket()
472 	else if (call->cong_ca_state == RXRPC_CA_SLOW_START && call->cong_cwnd <= RXRPC_MIN_CWND)  in rxrpc_prepare_data_subpacket()
474 	else if (call->tx_winsize <= 2)  in rxrpc_prepare_data_subpacket()
476 	else if (call->rtt_count < 3)  in rxrpc_prepare_data_subpacket()
478 	else if (ktime_before(ktime_add_ms(call->rtt_last_req, 1000), ktime_get_real()))  in rxrpc_prepare_data_subpacket()
480 	else if (!last && !after(READ_ONCE(call->send_top), txb->seq))  in rxrpc_prepare_data_subpacket()
485 	rxrpc_inc_stat(call->rxnet, stat_why_req_ack[why]);  in rxrpc_prepare_data_subpacket()
486 	trace_rxrpc_req_ack(call->debug_id, txb->seq, why);  in rxrpc_prepare_data_subpacket()
489 		trace_rxrpc_rtt_tx(call, rxrpc_rtt_tx_data, -1, serial);  in rxrpc_prepare_data_subpacket()
490 		call->rtt_last_req = req->now;  in rxrpc_prepare_data_subpacket()
511 	trace_rxrpc_tx_data(call, txb->seq, txb->serial, flags, req->trace);  in rxrpc_prepare_data_subpacket()
535 static size_t rxrpc_prepare_data_packet(struct rxrpc_call *call,  in rxrpc_prepare_data_packet()  argument
546 	trace_rxrpc_tq(call, tq, seq, rxrpc_tq_transmit);  in rxrpc_prepare_data_packet()
549 	serial = rxrpc_get_next_serials(call->conn, req->n);  in rxrpc_prepare_data_packet()
551 	whdr->epoch		= htonl(call->conn->proto.epoch);  in rxrpc_prepare_data_packet()
552 	whdr->cid		= htonl(call->cid);  in rxrpc_prepare_data_packet()
553 	whdr->callNumber	= htonl(call->call_id);  in rxrpc_prepare_data_packet()
559 	whdr->securityIndex	= call->security_ix;  in rxrpc_prepare_data_packet()
561 	whdr->serviceId		= htons(call->conn->service_id);  in rxrpc_prepare_data_packet()
563 	call->tx_last_serial = serial + req->n - 1;  in rxrpc_prepare_data_packet()
564 	call->tx_last_sent = req->now;  in rxrpc_prepare_data_packet()
576 			call->tx_nr_lost--;  in rxrpc_prepare_data_packet()
580 			call->tx_nr_resent++;  in rxrpc_prepare_data_packet()
582 			call->tx_nr_sent++;  in rxrpc_prepare_data_packet()
590 		len += rxrpc_prepare_data_subpacket(call, req, txb, whdr, serial, i);  in rxrpc_prepare_data_packet()
598 			trace_rxrpc_tq(call, tq, seq, rxrpc_tq_transmit_advance);  in rxrpc_prepare_data_packet()
606 		call->tlp_serial = serial - 1;  in rxrpc_prepare_data_packet()
607 		call->tlp_seq = seq - 1;  in rxrpc_prepare_data_packet()
612 		if (!test_bit(RXRPC_CALL_BEGAN_RX_TIMER, &call->flags))  in rxrpc_prepare_data_packet()
616 			pto = rxrpc_tlp_calc_pto(call, req->now);  in rxrpc_prepare_data_packet()
618 		call->rack_timer_mode = RXRPC_CALL_RACKTIMER_TLP_PTO;  in rxrpc_prepare_data_packet()
619 		call->rack_timo_at = ktime_add(req->now, pto);  in rxrpc_prepare_data_packet()
620 		trace_rxrpc_rack_timer(call, pto, false);  in rxrpc_prepare_data_packet()
621 		trace_rxrpc_timer_set(call, pto, rxrpc_timer_trace_rack_tlp_pto);  in rxrpc_prepare_data_packet()
624 	if (!test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER, &call->flags)) {  in rxrpc_prepare_data_packet()
625 		ktime_t delay = ms_to_ktime(READ_ONCE(call->next_rx_timo));  in rxrpc_prepare_data_packet()
627 		call->expect_rx_by = ktime_add(req->now, delay);  in rxrpc_prepare_data_packet()
628 		trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_expect_rx);  in rxrpc_prepare_data_packet()
631 	rxrpc_set_keepalive(call, req->now);  in rxrpc_prepare_data_packet()
639 void rxrpc_send_data_packet(struct rxrpc_call *call, struct rxrpc_send_data_req *req)  in rxrpc_send_data_packet()  argument
642 	struct rxrpc_connection *conn = call->conn;  in rxrpc_send_data_packet()
649 	bool new_call = test_bit(RXRPC_CALL_BEGAN_RX_TIMER, &call->flags);  in rxrpc_send_data_packet()
654 	whdr = page_frag_alloc(&call->local->tx_alloc, sizeof(*whdr), GFP_NOFS);  in rxrpc_send_data_packet()
658 	call->local->kvec[0].iov_base = whdr;  in rxrpc_send_data_packet()
659 	call->local->kvec[0].iov_len = sizeof(*whdr);  in rxrpc_send_data_packet()
661 	stat_ix = umin(req->n, ARRAY_SIZE(call->rxnet->stat_tx_jumbo)) - 1;  in rxrpc_send_data_packet()
662 	atomic_inc(&call->rxnet->stat_tx_jumbo[stat_ix]);  in rxrpc_send_data_packet()
664 	len += rxrpc_prepare_data_packet(call, req, whdr);  in rxrpc_send_data_packet()
667 	iov_iter_kvec(&msg.msg_iter, WRITE, call->local->kvec, 1 + req->n, len);  in rxrpc_send_data_packet()
669 	msg.msg_name	= &call->peer->srx.transport;  in rxrpc_send_data_packet()
670 	msg.msg_namelen	= call->peer->srx.transport_len;  in rxrpc_send_data_packet()
678 	if (seq == call->tx_transmitted + 1 &&  in rxrpc_send_data_packet()
679 	    len >= sizeof(struct rxrpc_wire_header) + call->peer->max_data) {  in rxrpc_send_data_packet()
691 	if (seq == call->tx_transmitted + 1)  in rxrpc_send_data_packet()
692 		call->tx_transmitted = seq + req->n - 1;  in rxrpc_send_data_packet()
699 			trace_rxrpc_tx_data(call, txb->seq, txb->serial, txb->flags,  in rxrpc_send_data_packet()
712 	rxrpc_inc_stat(call->rxnet, stat_tx_data_send);  in rxrpc_send_data_packet()
717 		rxrpc_inc_stat(call->rxnet, stat_tx_data_send_msgsize);  in rxrpc_send_data_packet()
718 		trace_rxrpc_tx_packet(call->debug_id, whdr, frag);  in rxrpc_send_data_packet()
721 		rxrpc_inc_stat(call->rxnet, stat_tx_data_send_fail);  in rxrpc_send_data_packet()
722 		trace_rxrpc_tx_fail(call->debug_id, txb->serial, ret, frag);  in rxrpc_send_data_packet()
724 		trace_rxrpc_tx_packet(call->debug_id, whdr, frag);  in rxrpc_send_data_packet()
727 	rxrpc_tx_backoff(call, ret);  in rxrpc_send_data_packet()
730 		/* Cancel the call if the initial transmission fails or if we  in rxrpc_send_data_packet()
739 			rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR,  in rxrpc_send_data_packet()
744 	_leave(" = %d [%u]", ret, call->peer->max_data);  in rxrpc_send_data_packet()