1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Broadcom NetXtreme-C/E network driver.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation.
7 */
8
9 #include <linux/pci.h>
10 #include <linux/netdevice.h>
11 #include <linux/skbuff.h>
12 #include <net/netdev_queues.h>
13 #include <net/ip.h>
14 #include <net/ipv6.h>
15 #include <net/udp.h>
16 #include <net/tso.h>
17 #include <linux/bnxt/hsi.h>
18
19 #include "bnxt.h"
20 #include "bnxt_gso.h"
21
bnxt_sw_gso_lhint(unsigned int len)22 static u32 bnxt_sw_gso_lhint(unsigned int len)
23 {
24 if (len <= 512)
25 return TX_BD_FLAGS_LHINT_512_AND_SMALLER;
26 else if (len <= 1023)
27 return TX_BD_FLAGS_LHINT_512_TO_1023;
28 else if (len <= 2047)
29 return TX_BD_FLAGS_LHINT_1024_TO_2047;
30 else
31 return TX_BD_FLAGS_LHINT_2048_AND_LARGER;
32 }
33
34 /* Transmit an skb requiring software UDP segmentation.
35 *
36 * Returns 1 if the skb was queued and new BDs were produced, 0 if the skb
37 * was dropped, or -1 if the ring is full and the skb should be retried.
38 * The caller owns the doorbell for all three cases.
39 */
bnxt_sw_udp_gso_xmit(struct bnxt * bp,struct bnxt_tx_ring_info * txr,struct netdev_queue * txq,struct sk_buff * skb)40 int bnxt_sw_udp_gso_xmit(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
41 struct netdev_queue *txq, struct sk_buff *skb)
42 {
43 unsigned int last_unmap_len __maybe_unused = 0;
44 dma_addr_t last_unmap_addr __maybe_unused = 0;
45 struct bnxt_sw_tx_bd *last_unmap_buf = NULL;
46 unsigned int hdr_len, mss, num_segs;
47 struct pci_dev *pdev = bp->pdev;
48 unsigned int total_payload;
49 struct tso_dma_map map;
50 u32 vlan_tag_flags = 0;
51 int i, bds_needed;
52 struct tso_t tso;
53 u16 cfa_action;
54 __le32 csum;
55 u16 prod;
56
57 hdr_len = tso_start(skb, &tso);
58 mss = skb_shinfo(skb)->gso_size;
59 total_payload = skb->len - hdr_len;
60 num_segs = DIV_ROUND_UP(total_payload, mss);
61
62 if (unlikely(num_segs <= 1))
63 goto drop;
64
65 /* Upper bound on the number of descriptors needed.
66 *
67 * Each segment uses 1 long BD + 1 ext BD + payload BDs, which is
68 * at most num_segs + nr_frags (each frag boundary crossing adds at
69 * most 1 extra BD).
70 */
71 bds_needed = 3 * num_segs + skb_shinfo(skb)->nr_frags + 1;
72
73 if (unlikely(bnxt_tx_avail(bp, txr) < bds_needed)) {
74 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
75 bp->tx_wake_thresh);
76 return -1;
77 }
78
79 /* BD backpressure alone cannot prevent overwriting in-flight
80 * headers in the inline buffer. Check slot availability directly.
81 */
82 if (!netif_txq_maybe_stop(txq, bnxt_inline_avail(txr),
83 num_segs, num_segs))
84 return -1;
85
86 if (unlikely(tso_dma_map_init(&map, &pdev->dev, skb, hdr_len)))
87 goto drop;
88
89 cfa_action = bnxt_xmit_get_cfa_action(skb);
90 if (skb_vlan_tag_present(skb)) {
91 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
92 skb_vlan_tag_get(skb);
93 if (skb->vlan_proto == htons(ETH_P_8021Q))
94 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
95 }
96
97 csum = cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
98 if (!tso.ipv6)
99 csum |= cpu_to_le32(TX_BD_FLAGS_IP_CKSUM);
100
101 prod = txr->tx_prod;
102
103 for (i = 0; i < num_segs; i++) {
104 unsigned int seg_payload = min_t(unsigned int, mss,
105 total_payload - i * mss);
106 u16 slot = (txr->tx_inline_prod + i) &
107 (BNXT_SW_USO_MAX_SEGS - 1);
108 struct bnxt_sw_tx_bd *tx_buf;
109 unsigned int mapping_len;
110 dma_addr_t this_hdr_dma;
111 unsigned int chunk_len;
112 unsigned int offset;
113 dma_addr_t dma_addr;
114 struct tx_bd *txbd;
115 struct udphdr *uh;
116 void *this_hdr;
117 int bd_count;
118 bool last;
119 u32 flags;
120
121 last = (i == num_segs - 1);
122 offset = slot * TSO_HEADER_SIZE;
123 this_hdr = txr->tx_inline_buf + offset;
124 this_hdr_dma = txr->tx_inline_dma + offset;
125
126 tso_build_hdr(skb, this_hdr, &tso, seg_payload, last);
127
128 /* Zero stale csum fields copied from the original skb;
129 * HW offload recomputes from scratch.
130 */
131 uh = this_hdr + skb_transport_offset(skb);
132 uh->check = 0;
133 if (!tso.ipv6) {
134 struct iphdr *iph = this_hdr + skb_network_offset(skb);
135
136 iph->check = 0;
137 }
138
139 dma_sync_single_for_device(&pdev->dev, this_hdr_dma,
140 hdr_len, DMA_TO_DEVICE);
141
142 bd_count = tso_dma_map_count(&map, seg_payload);
143
144 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
145 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
146
147 tx_buf->skb = skb;
148 tx_buf->nr_frags = bd_count;
149 tx_buf->is_push = 0;
150 tx_buf->is_ts_pkt = 0;
151
152 dma_unmap_addr_set(tx_buf, mapping, this_hdr_dma);
153 dma_unmap_len_set(tx_buf, len, 0);
154
155 if (last) {
156 tx_buf->is_sw_gso = BNXT_SW_GSO_LAST;
157 tso_dma_map_completion_save(&map, &tx_buf->sw_gso_cstate);
158 } else {
159 tx_buf->is_sw_gso = BNXT_SW_GSO_MID;
160 }
161
162 flags = (hdr_len << TX_BD_LEN_SHIFT) |
163 TX_BD_TYPE_LONG_TX_BD |
164 TX_BD_CNT(2 + bd_count);
165
166 flags |= bnxt_sw_gso_lhint(hdr_len + seg_payload);
167
168 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
169 txbd->tx_bd_haddr = cpu_to_le64(this_hdr_dma);
170 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod,
171 2 + bd_count);
172
173 prod = NEXT_TX(prod);
174 bnxt_init_ext_bd(bp, txr, prod, csum,
175 vlan_tag_flags, cfa_action);
176
177 /* set dma_unmap_len on the LAST BD touching each
178 * region. Since completions are in-order, the last segment
179 * completes after all earlier ones, so the unmap is safe.
180 */
181 while (tso_dma_map_next(&map, &dma_addr, &chunk_len,
182 &mapping_len, seg_payload)) {
183 prod = NEXT_TX(prod);
184 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
185 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
186
187 txbd->tx_bd_haddr = cpu_to_le64(dma_addr);
188 dma_unmap_addr_set(tx_buf, mapping, dma_addr);
189 dma_unmap_len_set(tx_buf, len, 0);
190 tx_buf->skb = NULL;
191 tx_buf->is_sw_gso = 0;
192
193 if (mapping_len) {
194 if (last_unmap_buf) {
195 dma_unmap_addr_set(last_unmap_buf,
196 mapping,
197 last_unmap_addr);
198 dma_unmap_len_set(last_unmap_buf,
199 len,
200 last_unmap_len);
201 }
202 last_unmap_addr = dma_addr;
203 last_unmap_len = mapping_len;
204 }
205 last_unmap_buf = tx_buf;
206
207 flags = chunk_len << TX_BD_LEN_SHIFT;
208 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
209 txbd->tx_bd_opaque = 0;
210
211 seg_payload -= chunk_len;
212 }
213
214 txbd->tx_bd_len_flags_type |=
215 cpu_to_le32(TX_BD_FLAGS_PACKET_END);
216
217 prod = NEXT_TX(prod);
218 }
219
220 if (last_unmap_buf) {
221 dma_unmap_addr_set(last_unmap_buf, mapping, last_unmap_addr);
222 dma_unmap_len_set(last_unmap_buf, len, last_unmap_len);
223 }
224
225 txr->tx_inline_prod += num_segs;
226
227 netdev_tx_sent_queue(txq, skb->len);
228
229 WRITE_ONCE(txr->tx_prod, prod);
230
231 if (unlikely(bnxt_tx_avail(bp, txr) <= bp->tx_wake_thresh))
232 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
233 bp->tx_wake_thresh);
234
235 return 1;
236
237 drop:
238 dev_kfree_skb_any(skb);
239 dev_core_stats_tx_dropped_inc(bp->dev);
240 return 0;
241 }
242