102a19356SDavid Howells // SPDX-License-Identifier: GPL-2.0-or-later
202a19356SDavid Howells /* RxRPC Tx data buffering.
302a19356SDavid Howells *
402a19356SDavid Howells * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
502a19356SDavid Howells * Written by David Howells (dhowells@redhat.com)
602a19356SDavid Howells */
702a19356SDavid Howells
802a19356SDavid Howells #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
902a19356SDavid Howells
1002a19356SDavid Howells #include <linux/slab.h>
1102a19356SDavid Howells #include "ar-internal.h"
1202a19356SDavid Howells
1302a19356SDavid Howells static atomic_t rxrpc_txbuf_debug_ids;
1402a19356SDavid Howells atomic_t rxrpc_nr_txbuf;
1502a19356SDavid Howells
1602a19356SDavid Howells /*
1749489bb0SDavid Howells * Allocate and partially initialise a data transmission buffer.
1802a19356SDavid Howells */
rxrpc_alloc_data_txbuf(struct rxrpc_call * call,size_t data_size,size_t data_align,gfp_t gfp)1949489bb0SDavid Howells struct rxrpc_txbuf *rxrpc_alloc_data_txbuf(struct rxrpc_call *call, size_t data_size,
2049489bb0SDavid Howells size_t data_align, gfp_t gfp)
2102a19356SDavid Howells {
228985f2b0SDavid Howells struct rxrpc_wire_header *whdr;
2302a19356SDavid Howells struct rxrpc_txbuf *txb;
24*9f8eeea1SYunsheng Lin size_t total, hoff;
2549489bb0SDavid Howells void *buf;
2602a19356SDavid Howells
2702a19356SDavid Howells txb = kmalloc(sizeof(*txb), gfp);
2849489bb0SDavid Howells if (!txb)
2949489bb0SDavid Howells return NULL;
3049489bb0SDavid Howells
3149489bb0SDavid Howells hoff = round_up(sizeof(*whdr), data_align) - sizeof(*whdr);
3249489bb0SDavid Howells total = hoff + sizeof(*whdr) + data_size;
3349489bb0SDavid Howells
34*9f8eeea1SYunsheng Lin data_align = umax(data_align, L1_CACHE_BYTES);
3549489bb0SDavid Howells mutex_lock(&call->conn->tx_data_alloc_lock);
36*9f8eeea1SYunsheng Lin buf = page_frag_alloc_align(&call->conn->tx_data_alloc, total, gfp,
37*9f8eeea1SYunsheng Lin data_align);
3849489bb0SDavid Howells mutex_unlock(&call->conn->tx_data_alloc_lock);
3949489bb0SDavid Howells if (!buf) {
4049489bb0SDavid Howells kfree(txb);
4149489bb0SDavid Howells return NULL;
4249489bb0SDavid Howells }
4349489bb0SDavid Howells
4449489bb0SDavid Howells whdr = buf + hoff;
458985f2b0SDavid Howells
4602a19356SDavid Howells INIT_LIST_HEAD(&txb->call_link);
4702a19356SDavid Howells INIT_LIST_HEAD(&txb->tx_link);
4802a19356SDavid Howells refcount_set(&txb->ref, 1);
4949489bb0SDavid Howells txb->last_sent = KTIME_MIN;
5002a19356SDavid Howells txb->call_debug_id = call->debug_id;
5102a19356SDavid Howells txb->debug_id = atomic_inc_return(&rxrpc_txbuf_debug_ids);
5249489bb0SDavid Howells txb->space = data_size;
5302a19356SDavid Howells txb->len = 0;
5449489bb0SDavid Howells txb->offset = sizeof(*whdr);
5512bdff73SDavid Howells txb->flags = call->conn->out_clientflag;
5672f0c6fbSDavid Howells txb->ack_why = 0;
57cf37b598SDavid Howells txb->seq = call->tx_prepared + 1;
58ba132d84SDavid Howells txb->serial = 0;
5941b8debbSDavid Howells txb->cksum = 0;
60ff342bdcSDavid Howells txb->nr_kvec = 1;
618985f2b0SDavid Howells txb->kvec[0].iov_base = whdr;
628985f2b0SDavid Howells txb->kvec[0].iov_len = sizeof(*whdr);
6349489bb0SDavid Howells
648985f2b0SDavid Howells whdr->epoch = htonl(call->conn->proto.epoch);
658985f2b0SDavid Howells whdr->cid = htonl(call->cid);
668985f2b0SDavid Howells whdr->callNumber = htonl(call->call_id);
678985f2b0SDavid Howells whdr->seq = htonl(txb->seq);
6849489bb0SDavid Howells whdr->type = RXRPC_PACKET_TYPE_DATA;
698985f2b0SDavid Howells whdr->flags = 0;
708985f2b0SDavid Howells whdr->userStatus = 0;
718985f2b0SDavid Howells whdr->securityIndex = call->security_ix;
728985f2b0SDavid Howells whdr->_rsvd = 0;
738985f2b0SDavid Howells whdr->serviceId = htons(call->dest_srx.srx_service);
7402a19356SDavid Howells
7549489bb0SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 1,
7649489bb0SDavid Howells rxrpc_txbuf_alloc_data);
7749489bb0SDavid Howells
7802a19356SDavid Howells atomic_inc(&rxrpc_nr_txbuf);
7949489bb0SDavid Howells return txb;
8002a19356SDavid Howells }
8102a19356SDavid Howells
8249489bb0SDavid Howells /*
8349489bb0SDavid Howells * Allocate and partially initialise an ACK packet.
8449489bb0SDavid Howells */
rxrpc_alloc_ack_txbuf(struct rxrpc_call * call,size_t sack_size)8549489bb0SDavid Howells struct rxrpc_txbuf *rxrpc_alloc_ack_txbuf(struct rxrpc_call *call, size_t sack_size)
8649489bb0SDavid Howells {
8749489bb0SDavid Howells struct rxrpc_wire_header *whdr;
8849489bb0SDavid Howells struct rxrpc_acktrailer *trailer;
8949489bb0SDavid Howells struct rxrpc_ackpacket *ack;
9049489bb0SDavid Howells struct rxrpc_txbuf *txb;
9149489bb0SDavid Howells gfp_t gfp = rcu_read_lock_held() ? GFP_ATOMIC | __GFP_NOWARN : GFP_NOFS;
9249489bb0SDavid Howells void *buf, *buf2 = NULL;
9349489bb0SDavid Howells u8 *filler;
9449489bb0SDavid Howells
9549489bb0SDavid Howells txb = kmalloc(sizeof(*txb), gfp);
9649489bb0SDavid Howells if (!txb)
9749489bb0SDavid Howells return NULL;
9849489bb0SDavid Howells
9949489bb0SDavid Howells buf = page_frag_alloc(&call->local->tx_alloc,
10049489bb0SDavid Howells sizeof(*whdr) + sizeof(*ack) + 1 + 3 + sizeof(*trailer), gfp);
10149489bb0SDavid Howells if (!buf) {
10249489bb0SDavid Howells kfree(txb);
10349489bb0SDavid Howells return NULL;
10449489bb0SDavid Howells }
10549489bb0SDavid Howells
10649489bb0SDavid Howells if (sack_size) {
10749489bb0SDavid Howells buf2 = page_frag_alloc(&call->local->tx_alloc, sack_size, gfp);
10849489bb0SDavid Howells if (!buf2) {
10949489bb0SDavid Howells page_frag_free(buf);
11049489bb0SDavid Howells kfree(txb);
11149489bb0SDavid Howells return NULL;
11249489bb0SDavid Howells }
11349489bb0SDavid Howells }
11449489bb0SDavid Howells
11549489bb0SDavid Howells whdr = buf;
11649489bb0SDavid Howells ack = buf + sizeof(*whdr);
11749489bb0SDavid Howells filler = buf + sizeof(*whdr) + sizeof(*ack) + 1;
11849489bb0SDavid Howells trailer = buf + sizeof(*whdr) + sizeof(*ack) + 1 + 3;
11949489bb0SDavid Howells
12049489bb0SDavid Howells INIT_LIST_HEAD(&txb->call_link);
12149489bb0SDavid Howells INIT_LIST_HEAD(&txb->tx_link);
12249489bb0SDavid Howells refcount_set(&txb->ref, 1);
12349489bb0SDavid Howells txb->call_debug_id = call->debug_id;
12449489bb0SDavid Howells txb->debug_id = atomic_inc_return(&rxrpc_txbuf_debug_ids);
12549489bb0SDavid Howells txb->space = 0;
12649489bb0SDavid Howells txb->len = sizeof(*whdr) + sizeof(*ack) + 3 + sizeof(*trailer);
12749489bb0SDavid Howells txb->offset = 0;
12849489bb0SDavid Howells txb->flags = call->conn->out_clientflag;
12949489bb0SDavid Howells txb->ack_rwind = 0;
13049489bb0SDavid Howells txb->seq = 0;
13149489bb0SDavid Howells txb->serial = 0;
13249489bb0SDavid Howells txb->cksum = 0;
13349489bb0SDavid Howells txb->nr_kvec = 3;
13449489bb0SDavid Howells txb->kvec[0].iov_base = whdr;
13549489bb0SDavid Howells txb->kvec[0].iov_len = sizeof(*whdr) + sizeof(*ack);
13649489bb0SDavid Howells txb->kvec[1].iov_base = buf2;
13749489bb0SDavid Howells txb->kvec[1].iov_len = sack_size;
13849489bb0SDavid Howells txb->kvec[2].iov_base = filler;
13949489bb0SDavid Howells txb->kvec[2].iov_len = 3 + sizeof(*trailer);
14049489bb0SDavid Howells
14149489bb0SDavid Howells whdr->epoch = htonl(call->conn->proto.epoch);
14249489bb0SDavid Howells whdr->cid = htonl(call->cid);
14349489bb0SDavid Howells whdr->callNumber = htonl(call->call_id);
14449489bb0SDavid Howells whdr->seq = 0;
14549489bb0SDavid Howells whdr->type = RXRPC_PACKET_TYPE_ACK;
14649489bb0SDavid Howells whdr->flags = 0;
14749489bb0SDavid Howells whdr->userStatus = 0;
14849489bb0SDavid Howells whdr->securityIndex = call->security_ix;
14949489bb0SDavid Howells whdr->_rsvd = 0;
15049489bb0SDavid Howells whdr->serviceId = htons(call->dest_srx.srx_service);
15149489bb0SDavid Howells
15249489bb0SDavid Howells get_page(virt_to_head_page(trailer));
15349489bb0SDavid Howells
15449489bb0SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 1,
15549489bb0SDavid Howells rxrpc_txbuf_alloc_ack);
15649489bb0SDavid Howells atomic_inc(&rxrpc_nr_txbuf);
15702a19356SDavid Howells return txb;
15802a19356SDavid Howells }
15902a19356SDavid Howells
rxrpc_get_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)16002a19356SDavid Howells void rxrpc_get_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
16102a19356SDavid Howells {
16202a19356SDavid Howells int r;
16302a19356SDavid Howells
16402a19356SDavid Howells __refcount_inc(&txb->ref, &r);
16502a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r + 1, what);
16602a19356SDavid Howells }
16702a19356SDavid Howells
rxrpc_see_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)16802a19356SDavid Howells void rxrpc_see_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
16902a19356SDavid Howells {
17002a19356SDavid Howells int r = refcount_read(&txb->ref);
17102a19356SDavid Howells
17202a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r, what);
17302a19356SDavid Howells }
17402a19356SDavid Howells
rxrpc_free_txbuf(struct rxrpc_txbuf * txb)17549489bb0SDavid Howells static void rxrpc_free_txbuf(struct rxrpc_txbuf *txb)
17602a19356SDavid Howells {
17749489bb0SDavid Howells int i;
17802a19356SDavid Howells
17902a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 0,
18002a19356SDavid Howells rxrpc_txbuf_free);
18149489bb0SDavid Howells for (i = 0; i < txb->nr_kvec; i++)
18249489bb0SDavid Howells if (txb->kvec[i].iov_base)
18349489bb0SDavid Howells page_frag_free(txb->kvec[i].iov_base);
18402a19356SDavid Howells kfree(txb);
18502a19356SDavid Howells atomic_dec(&rxrpc_nr_txbuf);
18602a19356SDavid Howells }
18702a19356SDavid Howells
rxrpc_put_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)18802a19356SDavid Howells void rxrpc_put_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
18902a19356SDavid Howells {
19002a19356SDavid Howells unsigned int debug_id, call_debug_id;
19102a19356SDavid Howells rxrpc_seq_t seq;
19202a19356SDavid Howells bool dead;
19302a19356SDavid Howells int r;
19402a19356SDavid Howells
19502a19356SDavid Howells if (txb) {
19602a19356SDavid Howells debug_id = txb->debug_id;
19702a19356SDavid Howells call_debug_id = txb->call_debug_id;
19802a19356SDavid Howells seq = txb->seq;
19902a19356SDavid Howells dead = __refcount_dec_and_test(&txb->ref, &r);
20002a19356SDavid Howells trace_rxrpc_txbuf(debug_id, call_debug_id, seq, r - 1, what);
20102a19356SDavid Howells if (dead)
20249489bb0SDavid Howells rxrpc_free_txbuf(txb);
20302a19356SDavid Howells }
20402a19356SDavid Howells }
205a4ea4c47SDavid Howells
206a4ea4c47SDavid Howells /*
207a4ea4c47SDavid Howells * Shrink the transmit buffer.
208a4ea4c47SDavid Howells */
rxrpc_shrink_call_tx_buffer(struct rxrpc_call * call)209a4ea4c47SDavid Howells void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call)
210a4ea4c47SDavid Howells {
211a4ea4c47SDavid Howells struct rxrpc_txbuf *txb;
212a4ea4c47SDavid Howells rxrpc_seq_t hard_ack = smp_load_acquire(&call->acks_hard_ack);
213cf37b598SDavid Howells bool wake = false;
214a4ea4c47SDavid Howells
215a4ea4c47SDavid Howells _enter("%x/%x/%x", call->tx_bottom, call->acks_hard_ack, call->tx_top);
216a4ea4c47SDavid Howells
217b30d61f4SDavid Howells while ((txb = list_first_entry_or_null(&call->tx_buffer,
218b30d61f4SDavid Howells struct rxrpc_txbuf, call_link))) {
219a4ea4c47SDavid Howells hard_ack = smp_load_acquire(&call->acks_hard_ack);
220a4ea4c47SDavid Howells if (before(hard_ack, txb->seq))
221a4ea4c47SDavid Howells break;
222a4ea4c47SDavid Howells
2233feda9d6SDavid Howells if (txb->seq != call->tx_bottom + 1)
2243feda9d6SDavid Howells rxrpc_see_txbuf(txb, rxrpc_txbuf_see_out_of_step);
225a4ea4c47SDavid Howells ASSERTCMP(txb->seq, ==, call->tx_bottom + 1);
226cf37b598SDavid Howells smp_store_release(&call->tx_bottom, call->tx_bottom + 1);
227a4ea4c47SDavid Howells list_del_rcu(&txb->call_link);
228a4ea4c47SDavid Howells
229a4ea4c47SDavid Howells trace_rxrpc_txqueue(call, rxrpc_txqueue_dequeue);
230a4ea4c47SDavid Howells
231a4ea4c47SDavid Howells rxrpc_put_txbuf(txb, rxrpc_txbuf_put_rotated);
232cf37b598SDavid Howells if (after(call->acks_hard_ack, call->tx_bottom + 128))
233cf37b598SDavid Howells wake = true;
234a4ea4c47SDavid Howells }
235a4ea4c47SDavid Howells
236cf37b598SDavid Howells if (wake)
237cf37b598SDavid Howells wake_up(&call->waitq);
238a4ea4c47SDavid Howells }
239