1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
34 */
35
36 /* Theory of operation:
37 *
38 * Tx queues allocation and mapping on Siena
39 *
40 * One Tx queue with enabled checksum offload is allocated per Rx channel
41 * (event queue). Also 2 Tx queues (one without checksum offload and one
42 * with IP checksum offload only) are allocated and bound to event queue 0.
43 * sfxge_txq_type is used as Tx queue label.
44 *
45 * So, event queue plus label mapping to Tx queue index is:
46 * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES)
47 * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1
48 * See sfxge_get_txq_by_label() sfxge_ev.c
49 *
50 * Tx queue allocation and mapping on EF10
51 *
52 * One Tx queue with enabled checksum offload is allocated per Rx
53 * channel (event queue). Checksum offload on all Tx queues is enabled or
54 * disabled dynamically by inserting option descriptors, so the additional
55 * queues used on Siena are not required.
56 *
57 * TxQ label is always set to zero on EF10 hardware.
58 * So, event queue to Tx queue mapping is simple:
59 * TxQ-index = EvQ-index
60 */
61
62 #include <sys/cdefs.h>
63 #include "opt_rss.h"
64
65 #include <sys/param.h>
66 #include <sys/malloc.h>
67 #include <sys/mbuf.h>
68 #include <sys/smp.h>
69 #include <sys/socket.h>
70 #include <sys/sysctl.h>
71 #include <sys/syslog.h>
72 #include <sys/limits.h>
73
74 #include <net/bpf.h>
75 #include <net/ethernet.h>
76 #include <net/if.h>
77 #include <net/if_vlan_var.h>
78
79 #include <netinet/in.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip6.h>
82 #include <netinet/tcp.h>
83
84 #ifdef RSS
85 #include <net/rss_config.h>
86 #endif
87
88 #include "common/efx.h"
89
90 #include "sfxge.h"
91 #include "sfxge_tx.h"
92
93 #define SFXGE_PARAM_TX_DPL_GET_MAX SFXGE_PARAM(tx_dpl_get_max)
94 static int sfxge_tx_dpl_get_max = SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT;
95 TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_MAX, &sfxge_tx_dpl_get_max);
96 SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_max, CTLFLAG_RDTUN,
97 &sfxge_tx_dpl_get_max, 0,
98 "Maximum number of any packets in deferred packet get-list");
99
100 #define SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX \
101 SFXGE_PARAM(tx_dpl_get_non_tcp_max)
102 static int sfxge_tx_dpl_get_non_tcp_max =
103 SFXGE_TX_DPL_GET_NON_TCP_PKT_LIMIT_DEFAULT;
104 TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX, &sfxge_tx_dpl_get_non_tcp_max);
105 SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_non_tcp_max, CTLFLAG_RDTUN,
106 &sfxge_tx_dpl_get_non_tcp_max, 0,
107 "Maximum number of non-TCP packets in deferred packet get-list");
108
109 #define SFXGE_PARAM_TX_DPL_PUT_MAX SFXGE_PARAM(tx_dpl_put_max)
110 static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT;
111 TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max);
112 SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN,
113 &sfxge_tx_dpl_put_max, 0,
114 "Maximum number of any packets in deferred packet put-list");
115
116 #define SFXGE_PARAM_TSO_FW_ASSISTED SFXGE_PARAM(tso_fw_assisted)
117 static int sfxge_tso_fw_assisted = (SFXGE_FATSOV1 | SFXGE_FATSOV2);
118 TUNABLE_INT(SFXGE_PARAM_TSO_FW_ASSISTED, &sfxge_tso_fw_assisted);
119 SYSCTL_INT(_hw_sfxge, OID_AUTO, tso_fw_assisted, CTLFLAG_RDTUN,
120 &sfxge_tso_fw_assisted, 0,
121 "Bitmask of FW-assisted TSO allowed to use if supported by NIC firmware");
122
123 static const struct {
124 const char *name;
125 size_t offset;
126 } sfxge_tx_stats[] = {
127 #define SFXGE_TX_STAT(name, member) \
128 { #name, offsetof(struct sfxge_txq, member) }
129 SFXGE_TX_STAT(tso_bursts, tso_bursts),
130 SFXGE_TX_STAT(tso_packets, tso_packets),
131 SFXGE_TX_STAT(tso_long_headers, tso_long_headers),
132 SFXGE_TX_STAT(tso_pdrop_too_many, tso_pdrop_too_many),
133 SFXGE_TX_STAT(tso_pdrop_no_rsrc, tso_pdrop_no_rsrc),
134 SFXGE_TX_STAT(tx_collapses, collapses),
135 SFXGE_TX_STAT(tx_drops, drops),
136 SFXGE_TX_STAT(tx_get_overflow, get_overflow),
137 SFXGE_TX_STAT(tx_get_non_tcp_overflow, get_non_tcp_overflow),
138 SFXGE_TX_STAT(tx_put_overflow, put_overflow),
139 SFXGE_TX_STAT(tx_netdown_drops, netdown_drops),
140 };
141
142 /* Forward declarations. */
143 static void sfxge_tx_qdpl_service(struct sfxge_txq *txq);
144 static void sfxge_tx_qlist_post(struct sfxge_txq *txq);
145 static void sfxge_tx_qunblock(struct sfxge_txq *txq);
146 static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf,
147 const bus_dma_segment_t *dma_seg, int n_dma_seg,
148 int n_extra_descs);
149
150 static inline void
sfxge_next_stmp(struct sfxge_txq * txq,struct sfxge_tx_mapping ** pstmp)151 sfxge_next_stmp(struct sfxge_txq *txq, struct sfxge_tx_mapping **pstmp)
152 {
153 KASSERT((*pstmp)->flags == 0, ("stmp flags are not 0"));
154 if (__predict_false(*pstmp ==
155 &txq->stmp[txq->ptr_mask]))
156 *pstmp = &txq->stmp[0];
157 else
158 (*pstmp)++;
159 }
160
161 static int
sfxge_tx_maybe_toggle_cksum_offload(struct sfxge_txq * txq,struct mbuf * mbuf,struct sfxge_tx_mapping ** pstmp)162 sfxge_tx_maybe_toggle_cksum_offload(struct sfxge_txq *txq, struct mbuf *mbuf,
163 struct sfxge_tx_mapping **pstmp)
164 {
165 uint16_t new_hw_cksum_flags;
166 efx_desc_t *desc;
167
168 if (mbuf->m_pkthdr.csum_flags &
169 (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6 | CSUM_TSO)) {
170 /*
171 * We always set EFX_TXQ_CKSUM_IPV4 here because this
172 * configuration is the most useful, and this won't
173 * cause any trouble in case of IPv6 traffic anyway.
174 */
175 new_hw_cksum_flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
176 } else if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
177 new_hw_cksum_flags = EFX_TXQ_CKSUM_IPV4;
178 } else {
179 new_hw_cksum_flags = 0;
180 }
181
182 if (new_hw_cksum_flags == txq->hw_cksum_flags)
183 return (0);
184
185 desc = &txq->pend_desc[txq->n_pend_desc];
186 efx_tx_qdesc_checksum_create(txq->common, new_hw_cksum_flags, desc);
187 txq->hw_cksum_flags = new_hw_cksum_flags;
188 txq->n_pend_desc++;
189
190 sfxge_next_stmp(txq, pstmp);
191
192 return (1);
193 }
194
195 static int
sfxge_tx_maybe_insert_tag(struct sfxge_txq * txq,struct mbuf * mbuf,struct sfxge_tx_mapping ** pstmp)196 sfxge_tx_maybe_insert_tag(struct sfxge_txq *txq, struct mbuf *mbuf,
197 struct sfxge_tx_mapping **pstmp)
198 {
199 uint16_t this_tag = ((mbuf->m_flags & M_VLANTAG) ?
200 mbuf->m_pkthdr.ether_vtag :
201 0);
202 efx_desc_t *desc;
203
204 if (this_tag == txq->hw_vlan_tci)
205 return (0);
206
207 desc = &txq->pend_desc[txq->n_pend_desc];
208 efx_tx_qdesc_vlantci_create(txq->common, bswap16(this_tag), desc);
209 txq->hw_vlan_tci = this_tag;
210 txq->n_pend_desc++;
211
212 sfxge_next_stmp(txq, pstmp);
213
214 return (1);
215 }
216
217 void
sfxge_tx_qcomplete(struct sfxge_txq * txq,struct sfxge_evq * evq)218 sfxge_tx_qcomplete(struct sfxge_txq *txq, struct sfxge_evq *evq)
219 {
220 unsigned int completed;
221
222 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
223
224 completed = txq->completed;
225 while (completed != txq->pending) {
226 struct sfxge_tx_mapping *stmp;
227 unsigned int id;
228
229 id = completed++ & txq->ptr_mask;
230
231 stmp = &txq->stmp[id];
232 if (stmp->flags & TX_BUF_UNMAP) {
233 bus_dmamap_unload(txq->packet_dma_tag, stmp->map);
234 if (stmp->flags & TX_BUF_MBUF) {
235 struct mbuf *m = stmp->u.mbuf;
236 do
237 m = m_free(m);
238 while (m != NULL);
239 } else {
240 free(stmp->u.heap_buf, M_SFXGE);
241 }
242 stmp->flags = 0;
243 }
244 }
245 txq->completed = completed;
246
247 /* Check whether we need to unblock the queue. */
248 mb();
249 if (txq->blocked) {
250 unsigned int level;
251
252 level = txq->added - txq->completed;
253 if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries))
254 sfxge_tx_qunblock(txq);
255 }
256 }
257
258 static unsigned int
sfxge_is_mbuf_non_tcp(struct mbuf * mbuf)259 sfxge_is_mbuf_non_tcp(struct mbuf *mbuf)
260 {
261 /* Absence of TCP checksum flags does not mean that it is non-TCP
262 * but it should be true if user wants to achieve high throughput.
263 */
264 return (!(mbuf->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)));
265 }
266
267 /*
268 * Reorder the put list and append it to the get list.
269 */
270 static void
sfxge_tx_qdpl_swizzle(struct sfxge_txq * txq)271 sfxge_tx_qdpl_swizzle(struct sfxge_txq *txq)
272 {
273 struct sfxge_tx_dpl *stdp;
274 struct mbuf *mbuf, *get_next, **get_tailp;
275 volatile uintptr_t *putp;
276 uintptr_t put;
277 unsigned int count;
278 unsigned int non_tcp_count;
279
280 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
281
282 stdp = &txq->dpl;
283
284 /* Acquire the put list. */
285 putp = &stdp->std_put;
286 put = atomic_readandclear_ptr(putp);
287 mbuf = (void *)put;
288
289 if (mbuf == NULL)
290 return;
291
292 /* Reverse the put list. */
293 get_tailp = &mbuf->m_nextpkt;
294 get_next = NULL;
295
296 count = 0;
297 non_tcp_count = 0;
298 do {
299 struct mbuf *put_next;
300
301 non_tcp_count += sfxge_is_mbuf_non_tcp(mbuf);
302 put_next = mbuf->m_nextpkt;
303 mbuf->m_nextpkt = get_next;
304 get_next = mbuf;
305 mbuf = put_next;
306
307 count++;
308 } while (mbuf != NULL);
309
310 if (count > stdp->std_put_hiwat)
311 stdp->std_put_hiwat = count;
312
313 /* Append the reversed put list to the get list. */
314 KASSERT(*get_tailp == NULL, ("*get_tailp != NULL"));
315 *stdp->std_getp = get_next;
316 stdp->std_getp = get_tailp;
317 stdp->std_get_count += count;
318 stdp->std_get_non_tcp_count += non_tcp_count;
319 }
320
321 static void
sfxge_tx_qreap(struct sfxge_txq * txq)322 sfxge_tx_qreap(struct sfxge_txq *txq)
323 {
324 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
325
326 txq->reaped = txq->completed;
327 }
328
329 static void
sfxge_tx_qlist_post(struct sfxge_txq * txq)330 sfxge_tx_qlist_post(struct sfxge_txq *txq)
331 {
332 unsigned int old_added __diagused;
333 unsigned int block_level;
334 unsigned int level;
335 int rc __diagused;
336
337 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
338
339 KASSERT(txq->n_pend_desc != 0, ("txq->n_pend_desc == 0"));
340 KASSERT(txq->n_pend_desc <= txq->max_pkt_desc,
341 ("txq->n_pend_desc too large"));
342 KASSERT(!txq->blocked, ("txq->blocked"));
343
344 old_added = txq->added;
345
346 /* Post the fragment list. */
347 rc = efx_tx_qdesc_post(txq->common, txq->pend_desc, txq->n_pend_desc,
348 txq->reaped, &txq->added);
349 KASSERT(rc == 0, ("efx_tx_qdesc_post() failed"));
350
351 /* If efx_tx_qdesc_post() had to refragment, our information about
352 * buffers to free may be associated with the wrong
353 * descriptors.
354 */
355 KASSERT(txq->added - old_added == txq->n_pend_desc,
356 ("efx_tx_qdesc_post() refragmented descriptors"));
357
358 level = txq->added - txq->reaped;
359 KASSERT(level <= txq->entries, ("overfilled TX queue"));
360
361 /* Clear the fragment list. */
362 txq->n_pend_desc = 0;
363
364 /*
365 * Set the block level to ensure there is space to generate a
366 * large number of descriptors for TSO.
367 */
368 block_level = EFX_TXQ_LIMIT(txq->entries) - txq->max_pkt_desc;
369
370 /* Have we reached the block level? */
371 if (level < block_level)
372 return;
373
374 /* Reap, and check again */
375 sfxge_tx_qreap(txq);
376 level = txq->added - txq->reaped;
377 if (level < block_level)
378 return;
379
380 txq->blocked = 1;
381
382 /*
383 * Avoid a race with completion interrupt handling that could leave
384 * the queue blocked.
385 */
386 mb();
387 sfxge_tx_qreap(txq);
388 level = txq->added - txq->reaped;
389 if (level < block_level) {
390 mb();
391 txq->blocked = 0;
392 }
393 }
394
sfxge_tx_queue_mbuf(struct sfxge_txq * txq,struct mbuf * mbuf)395 static int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf)
396 {
397 bus_dmamap_t *used_map;
398 bus_dmamap_t map;
399 bus_dma_segment_t dma_seg[SFXGE_TX_MAPPING_MAX_SEG];
400 unsigned int id;
401 struct sfxge_tx_mapping *stmp;
402 efx_desc_t *desc;
403 int n_dma_seg;
404 int rc;
405 int i;
406 int eop;
407 uint16_t hw_cksum_flags_prev;
408 uint16_t hw_vlan_tci_prev;
409 int n_extra_descs;
410
411 KASSERT(!txq->blocked, ("txq->blocked"));
412
413 #if SFXGE_TX_PARSE_EARLY
414 /*
415 * If software TSO is used, we still need to copy packet header,
416 * even if we have already parsed it early before enqueue.
417 */
418 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) &&
419 (txq->tso_fw_assisted == 0))
420 prefetch_read_many(mbuf->m_data);
421 #else
422 /*
423 * Prefetch packet header since we need to parse it and extract
424 * IP ID, TCP sequence number and flags.
425 */
426 if (mbuf->m_pkthdr.csum_flags & CSUM_TSO)
427 prefetch_read_many(mbuf->m_data);
428 #endif
429
430 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) {
431 rc = EINTR;
432 goto reject;
433 }
434
435 /* Load the packet for DMA. */
436 id = txq->added & txq->ptr_mask;
437 stmp = &txq->stmp[id];
438 rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, stmp->map,
439 mbuf, dma_seg, &n_dma_seg, 0);
440 if (rc == EFBIG) {
441 /* Try again. */
442 struct mbuf *new_mbuf = m_collapse(mbuf, M_NOWAIT,
443 SFXGE_TX_MAPPING_MAX_SEG);
444 if (new_mbuf == NULL)
445 goto reject;
446 ++txq->collapses;
447 mbuf = new_mbuf;
448 rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag,
449 stmp->map, mbuf,
450 dma_seg, &n_dma_seg, 0);
451 }
452 if (rc != 0)
453 goto reject;
454
455 /* Make the packet visible to the hardware. */
456 bus_dmamap_sync(txq->packet_dma_tag, stmp->map, BUS_DMASYNC_PREWRITE);
457
458 used_map = &stmp->map;
459
460 hw_cksum_flags_prev = txq->hw_cksum_flags;
461 hw_vlan_tci_prev = txq->hw_vlan_tci;
462
463 /*
464 * The order of option descriptors, which are used to leverage VLAN tag
465 * and checksum offloads, might be important. Changing checksum offload
466 * between VLAN option and packet descriptors probably does not work.
467 */
468 n_extra_descs = sfxge_tx_maybe_toggle_cksum_offload(txq, mbuf, &stmp);
469 n_extra_descs += sfxge_tx_maybe_insert_tag(txq, mbuf, &stmp);
470
471 if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) {
472 rc = sfxge_tx_queue_tso(txq, mbuf, dma_seg, n_dma_seg,
473 n_extra_descs);
474 if (rc < 0)
475 goto reject_mapped;
476 stmp = &txq->stmp[(rc - 1) & txq->ptr_mask];
477 } else {
478 /* Add the mapping to the fragment list, and set flags
479 * for the buffer.
480 */
481
482 i = 0;
483 for (;;) {
484 desc = &txq->pend_desc[i + n_extra_descs];
485 eop = (i == n_dma_seg - 1);
486 efx_tx_qdesc_dma_create(txq->common,
487 dma_seg[i].ds_addr,
488 dma_seg[i].ds_len,
489 eop,
490 desc);
491 if (eop)
492 break;
493 i++;
494 sfxge_next_stmp(txq, &stmp);
495 }
496 txq->n_pend_desc = n_dma_seg + n_extra_descs;
497 }
498
499 /*
500 * If the mapping required more than one descriptor
501 * then we need to associate the DMA map with the last
502 * descriptor, not the first.
503 */
504 if (used_map != &stmp->map) {
505 map = stmp->map;
506 stmp->map = *used_map;
507 *used_map = map;
508 }
509
510 stmp->u.mbuf = mbuf;
511 stmp->flags = TX_BUF_UNMAP | TX_BUF_MBUF;
512
513 /* Post the fragment list. */
514 sfxge_tx_qlist_post(txq);
515
516 return (0);
517
518 reject_mapped:
519 txq->hw_vlan_tci = hw_vlan_tci_prev;
520 txq->hw_cksum_flags = hw_cksum_flags_prev;
521 bus_dmamap_unload(txq->packet_dma_tag, *used_map);
522 reject:
523 /* Drop the packet on the floor. */
524 m_freem(mbuf);
525 ++txq->drops;
526
527 return (rc);
528 }
529
530 /*
531 * Drain the deferred packet list into the transmit queue.
532 */
533 static void
sfxge_tx_qdpl_drain(struct sfxge_txq * txq)534 sfxge_tx_qdpl_drain(struct sfxge_txq *txq)
535 {
536 struct sfxge_softc *sc;
537 struct sfxge_tx_dpl *stdp;
538 struct mbuf *mbuf, *next;
539 unsigned int count;
540 unsigned int non_tcp_count;
541 unsigned int pushed;
542 int rc;
543
544 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
545
546 sc = txq->sc;
547 stdp = &txq->dpl;
548 pushed = txq->added;
549
550 if (__predict_true(txq->init_state == SFXGE_TXQ_STARTED)) {
551 prefetch_read_many(sc->enp);
552 prefetch_read_many(txq->common);
553 }
554
555 mbuf = stdp->std_get;
556 count = stdp->std_get_count;
557 non_tcp_count = stdp->std_get_non_tcp_count;
558
559 if (count > stdp->std_get_hiwat)
560 stdp->std_get_hiwat = count;
561
562 while (count != 0) {
563 KASSERT(mbuf != NULL, ("mbuf == NULL"));
564
565 next = mbuf->m_nextpkt;
566 mbuf->m_nextpkt = NULL;
567
568 ETHER_BPF_MTAP(sc->ifnet, mbuf); /* packet capture */
569
570 if (next != NULL)
571 prefetch_read_many(next);
572
573 rc = sfxge_tx_queue_mbuf(txq, mbuf);
574 --count;
575 non_tcp_count -= sfxge_is_mbuf_non_tcp(mbuf);
576 mbuf = next;
577 if (rc != 0)
578 continue;
579
580 if (txq->blocked)
581 break;
582
583 /* Push the fragments to the hardware in batches. */
584 if (txq->added - pushed >= SFXGE_TX_BATCH) {
585 efx_tx_qpush(txq->common, txq->added, pushed);
586 pushed = txq->added;
587 }
588 }
589
590 if (count == 0) {
591 KASSERT(mbuf == NULL, ("mbuf != NULL"));
592 KASSERT(non_tcp_count == 0,
593 ("inconsistent TCP/non-TCP detection"));
594 stdp->std_get = NULL;
595 stdp->std_get_count = 0;
596 stdp->std_get_non_tcp_count = 0;
597 stdp->std_getp = &stdp->std_get;
598 } else {
599 stdp->std_get = mbuf;
600 stdp->std_get_count = count;
601 stdp->std_get_non_tcp_count = non_tcp_count;
602 }
603
604 if (txq->added != pushed)
605 efx_tx_qpush(txq->common, txq->added, pushed);
606
607 KASSERT(txq->blocked || stdp->std_get_count == 0,
608 ("queue unblocked but count is non-zero"));
609 }
610
611 #define SFXGE_TX_QDPL_PENDING(_txq) ((_txq)->dpl.std_put != 0)
612
613 /*
614 * Service the deferred packet list.
615 *
616 * NOTE: drops the txq mutex!
617 */
618 static void
sfxge_tx_qdpl_service(struct sfxge_txq * txq)619 sfxge_tx_qdpl_service(struct sfxge_txq *txq)
620 {
621 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
622
623 do {
624 if (SFXGE_TX_QDPL_PENDING(txq))
625 sfxge_tx_qdpl_swizzle(txq);
626
627 if (!txq->blocked)
628 sfxge_tx_qdpl_drain(txq);
629
630 SFXGE_TXQ_UNLOCK(txq);
631 } while (SFXGE_TX_QDPL_PENDING(txq) &&
632 SFXGE_TXQ_TRYLOCK(txq));
633 }
634
635 /*
636 * Put a packet on the deferred packet get-list.
637 */
638 static int
sfxge_tx_qdpl_put_locked(struct sfxge_txq * txq,struct mbuf * mbuf)639 sfxge_tx_qdpl_put_locked(struct sfxge_txq *txq, struct mbuf *mbuf)
640 {
641 struct sfxge_tx_dpl *stdp;
642
643 stdp = &txq->dpl;
644
645 KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL"));
646
647 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq);
648
649 if (stdp->std_get_count >= stdp->std_get_max) {
650 txq->get_overflow++;
651 return (ENOBUFS);
652 }
653 if (sfxge_is_mbuf_non_tcp(mbuf)) {
654 if (stdp->std_get_non_tcp_count >=
655 stdp->std_get_non_tcp_max) {
656 txq->get_non_tcp_overflow++;
657 return (ENOBUFS);
658 }
659 stdp->std_get_non_tcp_count++;
660 }
661
662 *(stdp->std_getp) = mbuf;
663 stdp->std_getp = &mbuf->m_nextpkt;
664 stdp->std_get_count++;
665
666 return (0);
667 }
668
669 /*
670 * Put a packet on the deferred packet put-list.
671 *
672 * We overload the csum_data field in the mbuf to keep track of this length
673 * because there is no cheap alternative to avoid races.
674 */
675 static int
sfxge_tx_qdpl_put_unlocked(struct sfxge_txq * txq,struct mbuf * mbuf)676 sfxge_tx_qdpl_put_unlocked(struct sfxge_txq *txq, struct mbuf *mbuf)
677 {
678 struct sfxge_tx_dpl *stdp;
679 volatile uintptr_t *putp;
680 uintptr_t old;
681 uintptr_t new;
682 unsigned int put_count;
683
684 KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL"));
685
686 SFXGE_TXQ_LOCK_ASSERT_NOTOWNED(txq);
687
688 stdp = &txq->dpl;
689 putp = &stdp->std_put;
690 new = (uintptr_t)mbuf;
691
692 do {
693 old = *putp;
694 if (old != 0) {
695 struct mbuf *mp = (struct mbuf *)old;
696 put_count = mp->m_pkthdr.csum_data;
697 } else
698 put_count = 0;
699 if (put_count >= stdp->std_put_max) {
700 atomic_add_long(&txq->put_overflow, 1);
701 return (ENOBUFS);
702 }
703 mbuf->m_pkthdr.csum_data = put_count + 1;
704 mbuf->m_nextpkt = (void *)old;
705 } while (atomic_cmpset_ptr(putp, old, new) == 0);
706
707 return (0);
708 }
709
710 /*
711 * Called from if_transmit - will try to grab the txq lock and enqueue to the
712 * put list if it succeeds, otherwise try to push onto the defer list if space.
713 */
714 static int
sfxge_tx_packet_add(struct sfxge_txq * txq,struct mbuf * m)715 sfxge_tx_packet_add(struct sfxge_txq *txq, struct mbuf *m)
716 {
717 int rc;
718
719 if (!SFXGE_LINK_UP(txq->sc)) {
720 atomic_add_long(&txq->netdown_drops, 1);
721 return (ENETDOWN);
722 }
723
724 /*
725 * Try to grab the txq lock. If we are able to get the lock,
726 * the packet will be appended to the "get list" of the deferred
727 * packet list. Otherwise, it will be pushed on the "put list".
728 */
729 if (SFXGE_TXQ_TRYLOCK(txq)) {
730 /* First swizzle put-list to get-list to keep order */
731 sfxge_tx_qdpl_swizzle(txq);
732
733 rc = sfxge_tx_qdpl_put_locked(txq, m);
734
735 /* Try to service the list. */
736 sfxge_tx_qdpl_service(txq);
737 /* Lock has been dropped. */
738 } else {
739 rc = sfxge_tx_qdpl_put_unlocked(txq, m);
740
741 /*
742 * Try to grab the lock again.
743 *
744 * If we are able to get the lock, we need to process
745 * the deferred packet list. If we are not able to get
746 * the lock, another thread is processing the list.
747 */
748 if ((rc == 0) && SFXGE_TXQ_TRYLOCK(txq)) {
749 sfxge_tx_qdpl_service(txq);
750 /* Lock has been dropped. */
751 }
752 }
753
754 SFXGE_TXQ_LOCK_ASSERT_NOTOWNED(txq);
755
756 return (rc);
757 }
758
759 static void
sfxge_tx_qdpl_flush(struct sfxge_txq * txq)760 sfxge_tx_qdpl_flush(struct sfxge_txq *txq)
761 {
762 struct sfxge_tx_dpl *stdp = &txq->dpl;
763 struct mbuf *mbuf, *next;
764
765 SFXGE_TXQ_LOCK(txq);
766
767 sfxge_tx_qdpl_swizzle(txq);
768 for (mbuf = stdp->std_get; mbuf != NULL; mbuf = next) {
769 next = mbuf->m_nextpkt;
770 m_freem(mbuf);
771 }
772 stdp->std_get = NULL;
773 stdp->std_get_count = 0;
774 stdp->std_get_non_tcp_count = 0;
775 stdp->std_getp = &stdp->std_get;
776
777 SFXGE_TXQ_UNLOCK(txq);
778 }
779
780 void
sfxge_if_qflush(if_t ifp)781 sfxge_if_qflush(if_t ifp)
782 {
783 struct sfxge_softc *sc;
784 unsigned int i;
785
786 sc = if_getsoftc(ifp);
787
788 for (i = 0; i < sc->txq_count; i++)
789 sfxge_tx_qdpl_flush(sc->txq[i]);
790 }
791
792 #if SFXGE_TX_PARSE_EARLY
793
794 /* There is little space for user data in mbuf pkthdr, so we
795 * use l*hlen fields which are not used by the driver otherwise
796 * to store header offsets.
797 * The fields are 8-bit, but it's ok, no header may be longer than 255 bytes.
798 */
799
800 #define TSO_MBUF_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[0])
801 /* We abuse l5hlen here because PH_loc can hold only 64 bits of data */
802 #define TSO_MBUF_FLAGS(_mbuf) ((_mbuf)->m_pkthdr.l5hlen)
803 #define TSO_MBUF_PACKETID(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[1])
804 #define TSO_MBUF_SEQNUM(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.thirtytwo[1])
805
sfxge_parse_tx_packet(struct mbuf * mbuf)806 static void sfxge_parse_tx_packet(struct mbuf *mbuf)
807 {
808 struct ether_header *eh = mtod(mbuf, struct ether_header *);
809 const struct tcphdr *th;
810 struct tcphdr th_copy;
811
812 /* Find network protocol and header */
813 TSO_MBUF_PROTO(mbuf) = eh->ether_type;
814 if (TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_VLAN)) {
815 struct ether_vlan_header *veh =
816 mtod(mbuf, struct ether_vlan_header *);
817 TSO_MBUF_PROTO(mbuf) = veh->evl_proto;
818 mbuf->m_pkthdr.l2hlen = sizeof(*veh);
819 } else {
820 mbuf->m_pkthdr.l2hlen = sizeof(*eh);
821 }
822
823 /* Find TCP header */
824 if (TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_IP)) {
825 const struct ip *iph = (const struct ip *)mtodo(mbuf, mbuf->m_pkthdr.l2hlen);
826
827 KASSERT(iph->ip_p == IPPROTO_TCP,
828 ("TSO required on non-TCP packet"));
829 mbuf->m_pkthdr.l3hlen = mbuf->m_pkthdr.l2hlen + 4 * iph->ip_hl;
830 TSO_MBUF_PACKETID(mbuf) = iph->ip_id;
831 } else {
832 KASSERT(TSO_MBUF_PROTO(mbuf) == htons(ETHERTYPE_IPV6),
833 ("TSO required on non-IP packet"));
834 KASSERT(((const struct ip6_hdr *)mtodo(mbuf, mbuf->m_pkthdr.l2hlen))->ip6_nxt ==
835 IPPROTO_TCP,
836 ("TSO required on non-TCP packet"));
837 mbuf->m_pkthdr.l3hlen = mbuf->m_pkthdr.l2hlen + sizeof(struct ip6_hdr);
838 TSO_MBUF_PACKETID(mbuf) = 0;
839 }
840
841 KASSERT(mbuf->m_len >= mbuf->m_pkthdr.l3hlen,
842 ("network header is fragmented in mbuf"));
843
844 /* We need TCP header including flags (window is the next) */
845 if (mbuf->m_len < mbuf->m_pkthdr.l3hlen + offsetof(struct tcphdr, th_win)) {
846 m_copydata(mbuf, mbuf->m_pkthdr.l3hlen, sizeof(th_copy),
847 (caddr_t)&th_copy);
848 th = &th_copy;
849 } else {
850 th = (const struct tcphdr *)mtodo(mbuf, mbuf->m_pkthdr.l3hlen);
851 }
852
853 mbuf->m_pkthdr.l4hlen = mbuf->m_pkthdr.l3hlen + 4 * th->th_off;
854 TSO_MBUF_SEQNUM(mbuf) = ntohl(th->th_seq);
855
856 /* These flags must not be duplicated */
857 /*
858 * RST should not be duplicated as well, but FreeBSD kernel
859 * generates TSO packets with RST flag. So, do not assert
860 * its absence.
861 */
862 KASSERT(!(th->th_flags & (TH_URG | TH_SYN)),
863 ("incompatible TCP flag 0x%x on TSO packet",
864 th->th_flags & (TH_URG | TH_SYN)));
865 TSO_MBUF_FLAGS(mbuf) = th->th_flags;
866 }
867 #endif
868
869 /*
870 * TX start -- called by the stack.
871 */
872 int
sfxge_if_transmit(if_t ifp,struct mbuf * m)873 sfxge_if_transmit(if_t ifp, struct mbuf *m)
874 {
875 struct sfxge_softc *sc;
876 struct sfxge_txq *txq;
877 int rc;
878
879 sc = (struct sfxge_softc *)if_getsoftc(ifp);
880
881 /*
882 * Transmit may be called when interface is up from the kernel
883 * point of view, but not yet up (in progress) from the driver
884 * point of view. I.e. link aggregation bring up.
885 * Transmit may be called when interface is up from the driver
886 * point of view, but already down from the kernel point of
887 * view. I.e. Rx when interface shutdown is in progress.
888 */
889 KASSERT((if_getflags(ifp) & IFF_UP) || (sc->if_flags & IFF_UP),
890 ("interface not up"));
891
892 /* Pick the desired transmit queue. */
893 if (sc->txq_dynamic_cksum_toggle_supported |
894 (m->m_pkthdr.csum_flags &
895 (CSUM_DELAY_DATA | CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | CSUM_TSO))) {
896 int index = 0;
897
898 #ifdef RSS
899 uint32_t bucket_id;
900
901 /*
902 * Select a TX queue which matches the corresponding
903 * RX queue for the hash in order to assign both
904 * TX and RX parts of the flow to the same CPU
905 */
906 if (rss_m2bucket(m, &bucket_id) == 0)
907 index = bucket_id % (sc->txq_count - (SFXGE_TXQ_NTYPES - 1));
908 #else
909 /* check if flowid is set */
910 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
911 uint32_t hash = m->m_pkthdr.flowid;
912 uint32_t idx = hash % nitems(sc->rx_indir_table);
913
914 index = sc->rx_indir_table[idx];
915 }
916 #endif
917 #if SFXGE_TX_PARSE_EARLY
918 if (m->m_pkthdr.csum_flags & CSUM_TSO)
919 sfxge_parse_tx_packet(m);
920 #endif
921 index += (sc->txq_dynamic_cksum_toggle_supported == B_FALSE) ?
922 SFXGE_TXQ_IP_TCP_UDP_CKSUM : 0;
923 txq = sc->txq[index];
924 } else if (m->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
925 txq = sc->txq[SFXGE_TXQ_IP_CKSUM];
926 } else {
927 txq = sc->txq[SFXGE_TXQ_NON_CKSUM];
928 }
929
930 rc = sfxge_tx_packet_add(txq, m);
931 if (rc != 0)
932 m_freem(m);
933
934 return (rc);
935 }
936
937 /*
938 * Software "TSO". Not quite as good as doing it in hardware, but
939 * still faster than segmenting in the stack.
940 */
941
942 struct sfxge_tso_state {
943 /* Output position */
944 unsigned out_len; /* Remaining length in current segment */
945 unsigned seqnum; /* Current sequence number */
946 unsigned packet_space; /* Remaining space in current packet */
947 unsigned segs_space; /* Remaining number of DMA segments
948 for the packet (FATSOv2 only) */
949
950 /* Input position */
951 uint64_t dma_addr; /* DMA address of current position */
952 unsigned in_len; /* Remaining length in current mbuf */
953
954 const struct mbuf *mbuf; /* Input mbuf (head of chain) */
955 u_short protocol; /* Network protocol (after VLAN decap) */
956 ssize_t nh_off; /* Offset of network header */
957 ssize_t tcph_off; /* Offset of TCP header */
958 unsigned header_len; /* Number of bytes of header */
959 unsigned seg_size; /* TCP segment size */
960 int fw_assisted; /* Use FW-assisted TSO */
961 u_short packet_id; /* IPv4 packet ID from the original packet */
962 uint8_t tcp_flags; /* TCP flags */
963 efx_desc_t header_desc; /* Precomputed header descriptor for
964 * FW-assisted TSO */
965 };
966
967 #if !SFXGE_TX_PARSE_EARLY
tso_iph(const struct sfxge_tso_state * tso)968 static const struct ip *tso_iph(const struct sfxge_tso_state *tso)
969 {
970 KASSERT(tso->protocol == htons(ETHERTYPE_IP),
971 ("tso_iph() in non-IPv4 state"));
972 return (const struct ip *)(tso->mbuf->m_data + tso->nh_off);
973 }
974
tso_ip6h(const struct sfxge_tso_state * tso)975 static __unused const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso)
976 {
977 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
978 ("tso_ip6h() in non-IPv6 state"));
979 return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off);
980 }
981
tso_tcph(const struct sfxge_tso_state * tso)982 static const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso)
983 {
984 return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off);
985 }
986 #endif
987
988 /* Size of preallocated TSO header buffers. Larger blocks must be
989 * allocated from the heap.
990 */
991 #define TSOH_STD_SIZE 128
992
993 /* At most half the descriptors in the queue at any time will refer to
994 * a TSO header buffer, since they must always be followed by a
995 * payload descriptor referring to an mbuf.
996 */
997 #define TSOH_COUNT(_txq_entries) ((_txq_entries) / 2u)
998 #define TSOH_PER_PAGE (PAGE_SIZE / TSOH_STD_SIZE)
999 #define TSOH_PAGE_COUNT(_txq_entries) \
1000 howmany(TSOH_COUNT(_txq_entries), TSOH_PER_PAGE)
1001
tso_init(struct sfxge_txq * txq)1002 static int tso_init(struct sfxge_txq *txq)
1003 {
1004 struct sfxge_softc *sc = txq->sc;
1005 unsigned int tsoh_page_count = TSOH_PAGE_COUNT(sc->txq_entries);
1006 int i, rc;
1007
1008 /* Allocate TSO header buffers */
1009 txq->tsoh_buffer = malloc(tsoh_page_count * sizeof(txq->tsoh_buffer[0]),
1010 M_SFXGE, M_WAITOK);
1011
1012 for (i = 0; i < tsoh_page_count; i++) {
1013 rc = sfxge_dma_alloc(sc, PAGE_SIZE, &txq->tsoh_buffer[i]);
1014 if (rc != 0)
1015 goto fail;
1016 }
1017
1018 return (0);
1019
1020 fail:
1021 while (i-- > 0)
1022 sfxge_dma_free(&txq->tsoh_buffer[i]);
1023 free(txq->tsoh_buffer, M_SFXGE);
1024 txq->tsoh_buffer = NULL;
1025 return (rc);
1026 }
1027
tso_fini(struct sfxge_txq * txq)1028 static void tso_fini(struct sfxge_txq *txq)
1029 {
1030 int i;
1031
1032 if (txq->tsoh_buffer != NULL) {
1033 for (i = 0; i < TSOH_PAGE_COUNT(txq->sc->txq_entries); i++)
1034 sfxge_dma_free(&txq->tsoh_buffer[i]);
1035 free(txq->tsoh_buffer, M_SFXGE);
1036 }
1037 }
1038
tso_start(struct sfxge_txq * txq,struct sfxge_tso_state * tso,const bus_dma_segment_t * hdr_dma_seg,struct mbuf * mbuf)1039 static void tso_start(struct sfxge_txq *txq, struct sfxge_tso_state *tso,
1040 const bus_dma_segment_t *hdr_dma_seg,
1041 struct mbuf *mbuf)
1042 {
1043 const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->sc->enp);
1044 #if !SFXGE_TX_PARSE_EARLY
1045 struct ether_header *eh = mtod(mbuf, struct ether_header *);
1046 const struct tcphdr *th;
1047 struct tcphdr th_copy;
1048 #endif
1049
1050 tso->fw_assisted = txq->tso_fw_assisted;
1051 tso->mbuf = mbuf;
1052
1053 /* Find network protocol and header */
1054 #if !SFXGE_TX_PARSE_EARLY
1055 tso->protocol = eh->ether_type;
1056 if (tso->protocol == htons(ETHERTYPE_VLAN)) {
1057 struct ether_vlan_header *veh =
1058 mtod(mbuf, struct ether_vlan_header *);
1059 tso->protocol = veh->evl_proto;
1060 tso->nh_off = sizeof(*veh);
1061 } else {
1062 tso->nh_off = sizeof(*eh);
1063 }
1064 #else
1065 tso->protocol = TSO_MBUF_PROTO(mbuf);
1066 tso->nh_off = mbuf->m_pkthdr.l2hlen;
1067 tso->tcph_off = mbuf->m_pkthdr.l3hlen;
1068 tso->packet_id = ntohs(TSO_MBUF_PACKETID(mbuf));
1069 #endif
1070
1071 #if !SFXGE_TX_PARSE_EARLY
1072 /* Find TCP header */
1073 if (tso->protocol == htons(ETHERTYPE_IP)) {
1074 KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP,
1075 ("TSO required on non-TCP packet"));
1076 tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl;
1077 tso->packet_id = ntohs(tso_iph(tso)->ip_id);
1078 } else {
1079 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6),
1080 ("TSO required on non-IP packet"));
1081 KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP,
1082 ("TSO required on non-TCP packet"));
1083 tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr);
1084 tso->packet_id = 0;
1085 }
1086 #endif
1087
1088 if (tso->fw_assisted &&
1089 __predict_false(tso->tcph_off >
1090 encp->enc_tx_tso_tcp_header_offset_limit)) {
1091 tso->fw_assisted = 0;
1092 }
1093
1094 #if !SFXGE_TX_PARSE_EARLY
1095 KASSERT(mbuf->m_len >= tso->tcph_off,
1096 ("network header is fragmented in mbuf"));
1097 /* We need TCP header including flags (window is the next) */
1098 if (mbuf->m_len < tso->tcph_off + offsetof(struct tcphdr, th_win)) {
1099 m_copydata(tso->mbuf, tso->tcph_off, sizeof(th_copy),
1100 (caddr_t)&th_copy);
1101 th = &th_copy;
1102 } else {
1103 th = tso_tcph(tso);
1104 }
1105 tso->header_len = tso->tcph_off + 4 * th->th_off;
1106 #else
1107 tso->header_len = mbuf->m_pkthdr.l4hlen;
1108 #endif
1109 tso->seg_size = mbuf->m_pkthdr.tso_segsz;
1110
1111 #if !SFXGE_TX_PARSE_EARLY
1112 tso->seqnum = ntohl(th->th_seq);
1113
1114 /* These flags must not be duplicated */
1115 /*
1116 * RST should not be duplicated as well, but FreeBSD kernel
1117 * generates TSO packets with RST flag. So, do not assert
1118 * its absence.
1119 */
1120 KASSERT(!(th->th_flags & (TH_URG | TH_SYN)),
1121 ("incompatible TCP flag 0x%x on TSO packet",
1122 th->th_flags & (TH_URG | TH_SYN)));
1123 tso->tcp_flags = th->th_flags;
1124 #else
1125 tso->seqnum = TSO_MBUF_SEQNUM(mbuf);
1126 tso->tcp_flags = TSO_MBUF_FLAGS(mbuf);
1127 #endif
1128
1129 tso->out_len = mbuf->m_pkthdr.len - tso->header_len;
1130
1131 if (tso->fw_assisted) {
1132 if (hdr_dma_seg->ds_len >= tso->header_len)
1133 efx_tx_qdesc_dma_create(txq->common,
1134 hdr_dma_seg->ds_addr,
1135 tso->header_len,
1136 B_FALSE,
1137 &tso->header_desc);
1138 else
1139 tso->fw_assisted = 0;
1140 }
1141 }
1142
1143 /*
1144 * tso_fill_packet_with_fragment - form descriptors for the current fragment
1145 *
1146 * Form descriptors for the current fragment, until we reach the end
1147 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
1148 * space.
1149 */
tso_fill_packet_with_fragment(struct sfxge_txq * txq,struct sfxge_tso_state * tso)1150 static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
1151 struct sfxge_tso_state *tso)
1152 {
1153 efx_desc_t *desc;
1154 int n;
1155 uint64_t dma_addr = tso->dma_addr;
1156 boolean_t eop;
1157
1158 if (tso->in_len == 0 || tso->packet_space == 0)
1159 return;
1160
1161 KASSERT(tso->in_len > 0, ("TSO input length went negative"));
1162 KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
1163
1164 if (tso->fw_assisted & SFXGE_FATSOV2) {
1165 n = tso->in_len;
1166 tso->out_len -= n;
1167 tso->seqnum += n;
1168 tso->in_len = 0;
1169 if (n < tso->packet_space) {
1170 tso->packet_space -= n;
1171 tso->segs_space--;
1172 } else {
1173 tso->packet_space = tso->seg_size -
1174 (n - tso->packet_space) % tso->seg_size;
1175 tso->segs_space =
1176 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1 -
1177 (tso->packet_space != tso->seg_size);
1178 }
1179 } else {
1180 n = min(tso->in_len, tso->packet_space);
1181 tso->packet_space -= n;
1182 tso->out_len -= n;
1183 tso->dma_addr += n;
1184 tso->in_len -= n;
1185 }
1186
1187 /*
1188 * It is OK to use binary OR below to avoid extra branching
1189 * since all conditions may always be checked.
1190 */
1191 eop = (tso->out_len == 0) | (tso->packet_space == 0) |
1192 (tso->segs_space == 0);
1193
1194 desc = &txq->pend_desc[txq->n_pend_desc++];
1195 efx_tx_qdesc_dma_create(txq->common, dma_addr, n, eop, desc);
1196 }
1197
1198 /* Callback from bus_dmamap_load() for long TSO headers. */
tso_map_long_header(void * dma_addr_ret,bus_dma_segment_t * segs,int nseg,int error)1199 static void tso_map_long_header(void *dma_addr_ret,
1200 bus_dma_segment_t *segs, int nseg,
1201 int error)
1202 {
1203 *(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) &&
1204 __predict_true(nseg == 1)) ?
1205 segs->ds_addr : 0);
1206 }
1207
1208 /*
1209 * tso_start_new_packet - generate a new header and prepare for the new packet
1210 *
1211 * Generate a new header and prepare for the new packet. Return 0 on
1212 * success, or an error code if failed to alloc header.
1213 */
tso_start_new_packet(struct sfxge_txq * txq,struct sfxge_tso_state * tso,unsigned int * idp)1214 static int tso_start_new_packet(struct sfxge_txq *txq,
1215 struct sfxge_tso_state *tso,
1216 unsigned int *idp)
1217 {
1218 unsigned int id = *idp;
1219 struct tcphdr *tsoh_th;
1220 unsigned ip_length;
1221 caddr_t header;
1222 uint64_t dma_addr;
1223 bus_dmamap_t map;
1224 efx_desc_t *desc;
1225 int rc;
1226
1227 if (tso->fw_assisted) {
1228 if (tso->fw_assisted & SFXGE_FATSOV2) {
1229 /* Add 2 FATSOv2 option descriptors */
1230 desc = &txq->pend_desc[txq->n_pend_desc];
1231 efx_tx_qdesc_tso2_create(txq->common,
1232 tso->packet_id,
1233 0,
1234 tso->seqnum,
1235 tso->seg_size,
1236 desc,
1237 EFX_TX_FATSOV2_OPT_NDESCS);
1238 desc += EFX_TX_FATSOV2_OPT_NDESCS;
1239 txq->n_pend_desc += EFX_TX_FATSOV2_OPT_NDESCS;
1240 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1241 id = (id + EFX_TX_FATSOV2_OPT_NDESCS) & txq->ptr_mask;
1242
1243 tso->segs_space =
1244 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1;
1245 } else {
1246 uint8_t tcp_flags = tso->tcp_flags;
1247
1248 if (tso->out_len > tso->seg_size)
1249 tcp_flags &= ~(TH_FIN | TH_PUSH);
1250
1251 /* Add FATSOv1 option descriptor */
1252 desc = &txq->pend_desc[txq->n_pend_desc++];
1253 efx_tx_qdesc_tso_create(txq->common,
1254 tso->packet_id,
1255 tso->seqnum,
1256 tcp_flags,
1257 desc++);
1258 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1259 id = (id + 1) & txq->ptr_mask;
1260
1261 tso->seqnum += tso->seg_size;
1262 tso->segs_space = UINT_MAX;
1263 }
1264
1265 /* Header DMA descriptor */
1266 *desc = tso->header_desc;
1267 txq->n_pend_desc++;
1268 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1269 id = (id + 1) & txq->ptr_mask;
1270 } else {
1271 /* Allocate a DMA-mapped header buffer. */
1272 if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
1273 unsigned int page_index = (id / 2) / TSOH_PER_PAGE;
1274 unsigned int buf_index = (id / 2) % TSOH_PER_PAGE;
1275
1276 header = (txq->tsoh_buffer[page_index].esm_base +
1277 buf_index * TSOH_STD_SIZE);
1278 dma_addr = (txq->tsoh_buffer[page_index].esm_addr +
1279 buf_index * TSOH_STD_SIZE);
1280 map = txq->tsoh_buffer[page_index].esm_map;
1281
1282 KASSERT(txq->stmp[id].flags == 0,
1283 ("stmp flags are not 0"));
1284 } else {
1285 struct sfxge_tx_mapping *stmp = &txq->stmp[id];
1286
1287 /* We cannot use bus_dmamem_alloc() as that may sleep */
1288 header = malloc(tso->header_len, M_SFXGE, M_NOWAIT);
1289 if (__predict_false(!header))
1290 return (ENOMEM);
1291 rc = bus_dmamap_load(txq->packet_dma_tag, stmp->map,
1292 header, tso->header_len,
1293 tso_map_long_header, &dma_addr,
1294 BUS_DMA_NOWAIT);
1295 if (__predict_false(dma_addr == 0)) {
1296 if (rc == 0) {
1297 /* Succeeded but got >1 segment */
1298 bus_dmamap_unload(txq->packet_dma_tag,
1299 stmp->map);
1300 rc = EINVAL;
1301 }
1302 free(header, M_SFXGE);
1303 return (rc);
1304 }
1305 map = stmp->map;
1306
1307 txq->tso_long_headers++;
1308 stmp->u.heap_buf = header;
1309 stmp->flags = TX_BUF_UNMAP;
1310 }
1311
1312 tsoh_th = (struct tcphdr *)(header + tso->tcph_off);
1313
1314 /* Copy and update the headers. */
1315 m_copydata(tso->mbuf, 0, tso->header_len, header);
1316
1317 tsoh_th->th_seq = htonl(tso->seqnum);
1318 tso->seqnum += tso->seg_size;
1319 if (tso->out_len > tso->seg_size) {
1320 /* This packet will not finish the TSO burst. */
1321 ip_length = tso->header_len - tso->nh_off + tso->seg_size;
1322 tsoh_th->th_flags &= ~(TH_FIN | TH_PUSH);
1323 } else {
1324 /* This packet will be the last in the TSO burst. */
1325 ip_length = tso->header_len - tso->nh_off + tso->out_len;
1326 }
1327
1328 if (tso->protocol == htons(ETHERTYPE_IP)) {
1329 struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off);
1330 tsoh_iph->ip_len = htons(ip_length);
1331 /* XXX We should increment ip_id, but FreeBSD doesn't
1332 * currently allocate extra IDs for multiple segments.
1333 */
1334 } else {
1335 struct ip6_hdr *tsoh_iph =
1336 (struct ip6_hdr *)(header + tso->nh_off);
1337 tsoh_iph->ip6_plen = htons(ip_length - sizeof(*tsoh_iph));
1338 }
1339
1340 /* Make the header visible to the hardware. */
1341 bus_dmamap_sync(txq->packet_dma_tag, map, BUS_DMASYNC_PREWRITE);
1342
1343 /* Form a descriptor for this header. */
1344 desc = &txq->pend_desc[txq->n_pend_desc++];
1345 efx_tx_qdesc_dma_create(txq->common,
1346 dma_addr,
1347 tso->header_len,
1348 0,
1349 desc);
1350 id = (id + 1) & txq->ptr_mask;
1351
1352 tso->segs_space = UINT_MAX;
1353 }
1354 tso->packet_space = tso->seg_size;
1355 txq->tso_packets++;
1356 *idp = id;
1357
1358 return (0);
1359 }
1360
1361 static int
sfxge_tx_queue_tso(struct sfxge_txq * txq,struct mbuf * mbuf,const bus_dma_segment_t * dma_seg,int n_dma_seg,int n_extra_descs)1362 sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf,
1363 const bus_dma_segment_t *dma_seg, int n_dma_seg,
1364 int n_extra_descs)
1365 {
1366 struct sfxge_tso_state tso;
1367 unsigned int id;
1368 unsigned skipped = 0;
1369
1370 tso_start(txq, &tso, dma_seg, mbuf);
1371
1372 while (dma_seg->ds_len + skipped <= tso.header_len) {
1373 skipped += dma_seg->ds_len;
1374 --n_dma_seg;
1375 KASSERT(n_dma_seg, ("no payload found in TSO packet"));
1376 ++dma_seg;
1377 }
1378 tso.in_len = dma_seg->ds_len - (tso.header_len - skipped);
1379 tso.dma_addr = dma_seg->ds_addr + (tso.header_len - skipped);
1380
1381 id = (txq->added + n_extra_descs) & txq->ptr_mask;
1382 if (__predict_false(tso_start_new_packet(txq, &tso, &id)))
1383 return (-1);
1384
1385 while (1) {
1386 tso_fill_packet_with_fragment(txq, &tso);
1387 /* Exactly one DMA descriptor is added */
1388 KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
1389 id = (id + 1) & txq->ptr_mask;
1390
1391 /* Move onto the next fragment? */
1392 if (tso.in_len == 0) {
1393 --n_dma_seg;
1394 if (n_dma_seg == 0)
1395 break;
1396 ++dma_seg;
1397 tso.in_len = dma_seg->ds_len;
1398 tso.dma_addr = dma_seg->ds_addr;
1399 }
1400
1401 /* End of packet? */
1402 if ((tso.packet_space == 0) | (tso.segs_space == 0)) {
1403 unsigned int n_fatso_opt_desc =
1404 (tso.fw_assisted & SFXGE_FATSOV2) ?
1405 EFX_TX_FATSOV2_OPT_NDESCS :
1406 (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0;
1407
1408 /* If the queue is now full due to tiny MSS,
1409 * or we can't create another header, discard
1410 * the remainder of the input mbuf but do not
1411 * roll back the work we have done.
1412 */
1413 if (txq->n_pend_desc + n_fatso_opt_desc +
1414 1 /* header */ + n_dma_seg > txq->max_pkt_desc) {
1415 txq->tso_pdrop_too_many++;
1416 break;
1417 }
1418 if (__predict_false(tso_start_new_packet(txq, &tso,
1419 &id))) {
1420 txq->tso_pdrop_no_rsrc++;
1421 break;
1422 }
1423 }
1424 }
1425
1426 txq->tso_bursts++;
1427 return (id);
1428 }
1429
1430 static void
sfxge_tx_qunblock(struct sfxge_txq * txq)1431 sfxge_tx_qunblock(struct sfxge_txq *txq)
1432 {
1433 struct sfxge_softc *sc;
1434 struct sfxge_evq *evq __diagused;
1435
1436 sc = txq->sc;
1437 evq = sc->evq[txq->evq_index];
1438
1439 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
1440
1441 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
1442 return;
1443
1444 SFXGE_TXQ_LOCK(txq);
1445
1446 if (txq->blocked) {
1447 unsigned int level;
1448
1449 level = txq->added - txq->completed;
1450 if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries)) {
1451 /* reaped must be in sync with blocked */
1452 sfxge_tx_qreap(txq);
1453 txq->blocked = 0;
1454 }
1455 }
1456
1457 sfxge_tx_qdpl_service(txq);
1458 /* note: lock has been dropped */
1459 }
1460
1461 void
sfxge_tx_qflush_done(struct sfxge_txq * txq)1462 sfxge_tx_qflush_done(struct sfxge_txq *txq)
1463 {
1464
1465 txq->flush_state = SFXGE_FLUSH_DONE;
1466 }
1467
1468 static void
sfxge_tx_qstop(struct sfxge_softc * sc,unsigned int index)1469 sfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index)
1470 {
1471 struct sfxge_txq *txq;
1472 struct sfxge_evq *evq;
1473 unsigned int count;
1474
1475 SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc);
1476
1477 txq = sc->txq[index];
1478 evq = sc->evq[txq->evq_index];
1479
1480 SFXGE_EVQ_LOCK(evq);
1481 SFXGE_TXQ_LOCK(txq);
1482
1483 KASSERT(txq->init_state == SFXGE_TXQ_STARTED,
1484 ("txq->init_state != SFXGE_TXQ_STARTED"));
1485
1486 txq->init_state = SFXGE_TXQ_INITIALIZED;
1487
1488 if (txq->flush_state != SFXGE_FLUSH_DONE) {
1489 txq->flush_state = SFXGE_FLUSH_PENDING;
1490
1491 SFXGE_EVQ_UNLOCK(evq);
1492 SFXGE_TXQ_UNLOCK(txq);
1493
1494 /* Flush the transmit queue. */
1495 if (efx_tx_qflush(txq->common) != 0) {
1496 log(LOG_ERR, "%s: Flushing Tx queue %u failed\n",
1497 device_get_nameunit(sc->dev), index);
1498 txq->flush_state = SFXGE_FLUSH_DONE;
1499 } else {
1500 count = 0;
1501 do {
1502 /* Spin for 100ms. */
1503 DELAY(100000);
1504 if (txq->flush_state != SFXGE_FLUSH_PENDING)
1505 break;
1506 } while (++count < 20);
1507 }
1508 SFXGE_EVQ_LOCK(evq);
1509 SFXGE_TXQ_LOCK(txq);
1510
1511 KASSERT(txq->flush_state != SFXGE_FLUSH_FAILED,
1512 ("txq->flush_state == SFXGE_FLUSH_FAILED"));
1513
1514 if (txq->flush_state != SFXGE_FLUSH_DONE) {
1515 /* Flush timeout */
1516 log(LOG_ERR, "%s: Cannot flush Tx queue %u\n",
1517 device_get_nameunit(sc->dev), index);
1518 txq->flush_state = SFXGE_FLUSH_DONE;
1519 }
1520 }
1521
1522 txq->blocked = 0;
1523 txq->pending = txq->added;
1524
1525 sfxge_tx_qcomplete(txq, evq);
1526 KASSERT(txq->completed == txq->added,
1527 ("txq->completed != txq->added"));
1528
1529 sfxge_tx_qreap(txq);
1530 KASSERT(txq->reaped == txq->completed,
1531 ("txq->reaped != txq->completed"));
1532
1533 txq->added = 0;
1534 txq->pending = 0;
1535 txq->completed = 0;
1536 txq->reaped = 0;
1537
1538 /* Destroy the common code transmit queue. */
1539 efx_tx_qdestroy(txq->common);
1540 txq->common = NULL;
1541
1542 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1543 EFX_TXQ_NBUFS(sc->txq_entries));
1544
1545 txq->hw_cksum_flags = 0;
1546
1547 SFXGE_EVQ_UNLOCK(evq);
1548 SFXGE_TXQ_UNLOCK(txq);
1549 }
1550
1551 /*
1552 * Estimate maximum number of Tx descriptors required for TSO packet.
1553 * With minimum MSS and maximum mbuf length we might need more (even
1554 * than a ring-ful of descriptors), but this should not happen in
1555 * practice except due to deliberate attack. In that case we will
1556 * truncate the output at a packet boundary.
1557 */
1558 static unsigned int
sfxge_tx_max_pkt_desc(const struct sfxge_softc * sc,enum sfxge_txq_type type,unsigned int tso_fw_assisted)1559 sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type,
1560 unsigned int tso_fw_assisted)
1561 {
1562 /* One descriptor for every input fragment */
1563 unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG;
1564 unsigned int sw_tso_max_descs;
1565 unsigned int fa_tso_v1_max_descs = 0;
1566 unsigned int fa_tso_v2_max_descs = 0;
1567
1568 /* Checksum offload Tx option descriptor may be required */
1569 if (sc->txq_dynamic_cksum_toggle_supported)
1570 max_descs++;
1571
1572 /* VLAN tagging Tx option descriptor may be required */
1573 if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled)
1574 max_descs++;
1575
1576 if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) {
1577 /*
1578 * Plus header and payload descriptor for each output segment.
1579 * Minus one since header fragment is already counted.
1580 * Even if FATSO is used, we should be ready to fallback
1581 * to do it in the driver.
1582 */
1583 sw_tso_max_descs = SFXGE_TSO_MAX_SEGS * 2 - 1;
1584
1585 /* FW assisted TSOv1 requires one more descriptor per segment
1586 * in comparison to SW TSO */
1587 if (tso_fw_assisted & SFXGE_FATSOV1)
1588 fa_tso_v1_max_descs =
1589 sw_tso_max_descs + SFXGE_TSO_MAX_SEGS;
1590
1591 /* FW assisted TSOv2 requires 3 (2 FATSO plus header) extra
1592 * descriptors per superframe limited by number of DMA fetches
1593 * per packet. The first packet header is already counted.
1594 */
1595 if (tso_fw_assisted & SFXGE_FATSOV2) {
1596 fa_tso_v2_max_descs =
1597 howmany(SFXGE_TX_MAPPING_MAX_SEG,
1598 EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1) *
1599 (EFX_TX_FATSOV2_OPT_NDESCS + 1) - 1;
1600 }
1601
1602 max_descs += MAX(sw_tso_max_descs,
1603 MAX(fa_tso_v1_max_descs, fa_tso_v2_max_descs));
1604 }
1605
1606 return (max_descs);
1607 }
1608
1609 static int
sfxge_tx_qstart(struct sfxge_softc * sc,unsigned int index)1610 sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
1611 {
1612 struct sfxge_txq *txq;
1613 efsys_mem_t *esmp;
1614 uint16_t flags;
1615 unsigned int tso_fw_assisted;
1616 unsigned int label;
1617 struct sfxge_evq *evq;
1618 unsigned int desc_index;
1619 int rc;
1620
1621 SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc);
1622
1623 txq = sc->txq[index];
1624 esmp = &txq->mem;
1625 evq = sc->evq[txq->evq_index];
1626
1627 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
1628 ("txq->init_state != SFXGE_TXQ_INITIALIZED"));
1629 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
1630 ("evq->init_state != SFXGE_EVQ_STARTED"));
1631
1632 /* Program the buffer table. */
1633 if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp,
1634 EFX_TXQ_NBUFS(sc->txq_entries))) != 0)
1635 return (rc);
1636
1637 /* Determine the kind of queue we are creating. */
1638 tso_fw_assisted = 0;
1639 switch (txq->type) {
1640 case SFXGE_TXQ_NON_CKSUM:
1641 flags = 0;
1642 break;
1643 case SFXGE_TXQ_IP_CKSUM:
1644 flags = EFX_TXQ_CKSUM_IPV4;
1645 break;
1646 case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
1647 flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
1648 tso_fw_assisted = sc->tso_fw_assisted;
1649 if (tso_fw_assisted & SFXGE_FATSOV2)
1650 flags |= EFX_TXQ_FATSOV2;
1651 break;
1652 default:
1653 KASSERT(0, ("Impossible TX queue"));
1654 flags = 0;
1655 break;
1656 }
1657
1658 label = (sc->txq_dynamic_cksum_toggle_supported) ? 0 : txq->type;
1659
1660 /* Create the common code transmit queue. */
1661 if ((rc = efx_tx_qcreate(sc->enp, index, label, esmp,
1662 sc->txq_entries, txq->buf_base_id, flags, evq->common,
1663 &txq->common, &desc_index)) != 0) {
1664 /* Retry if no FATSOv2 resources, otherwise fail */
1665 if ((rc != ENOSPC) || (~flags & EFX_TXQ_FATSOV2))
1666 goto fail;
1667
1668 /* Looks like all FATSOv2 contexts are used */
1669 flags &= ~EFX_TXQ_FATSOV2;
1670 tso_fw_assisted &= ~SFXGE_FATSOV2;
1671 if ((rc = efx_tx_qcreate(sc->enp, index, label, esmp,
1672 sc->txq_entries, txq->buf_base_id, flags, evq->common,
1673 &txq->common, &desc_index)) != 0)
1674 goto fail;
1675 }
1676
1677 /* Initialise queue descriptor indexes */
1678 txq->added = txq->pending = txq->completed = txq->reaped = desc_index;
1679
1680 SFXGE_TXQ_LOCK(txq);
1681
1682 /* Enable the transmit queue. */
1683 efx_tx_qenable(txq->common);
1684
1685 txq->init_state = SFXGE_TXQ_STARTED;
1686 txq->flush_state = SFXGE_FLUSH_REQUIRED;
1687 txq->tso_fw_assisted = tso_fw_assisted;
1688
1689 txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, txq->type,
1690 tso_fw_assisted);
1691
1692 txq->hw_vlan_tci = 0;
1693
1694 txq->hw_cksum_flags = flags &
1695 (EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP);
1696
1697 SFXGE_TXQ_UNLOCK(txq);
1698
1699 return (0);
1700
1701 fail:
1702 efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id,
1703 EFX_TXQ_NBUFS(sc->txq_entries));
1704 return (rc);
1705 }
1706
1707 void
sfxge_tx_stop(struct sfxge_softc * sc)1708 sfxge_tx_stop(struct sfxge_softc *sc)
1709 {
1710 int index;
1711
1712 index = sc->txq_count;
1713 while (--index >= 0)
1714 sfxge_tx_qstop(sc, index);
1715
1716 /* Tear down the transmit module */
1717 efx_tx_fini(sc->enp);
1718 }
1719
1720 int
sfxge_tx_start(struct sfxge_softc * sc)1721 sfxge_tx_start(struct sfxge_softc *sc)
1722 {
1723 int index;
1724 int rc;
1725
1726 /* Initialize the common code transmit module. */
1727 if ((rc = efx_tx_init(sc->enp)) != 0)
1728 return (rc);
1729
1730 for (index = 0; index < sc->txq_count; index++) {
1731 if ((rc = sfxge_tx_qstart(sc, index)) != 0)
1732 goto fail;
1733 }
1734
1735 return (0);
1736
1737 fail:
1738 while (--index >= 0)
1739 sfxge_tx_qstop(sc, index);
1740
1741 efx_tx_fini(sc->enp);
1742
1743 return (rc);
1744 }
1745
1746 static int
sfxge_txq_stat_init(struct sfxge_txq * txq,struct sysctl_oid * txq_node)1747 sfxge_txq_stat_init(struct sfxge_txq *txq, struct sysctl_oid *txq_node)
1748 {
1749 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(txq->sc->dev);
1750 struct sysctl_oid *stat_node;
1751 unsigned int id;
1752
1753 stat_node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(txq_node), OID_AUTO,
1754 "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Tx queue statistics");
1755 if (stat_node == NULL)
1756 return (ENOMEM);
1757
1758 for (id = 0; id < nitems(sfxge_tx_stats); id++) {
1759 SYSCTL_ADD_ULONG(
1760 ctx, SYSCTL_CHILDREN(stat_node), OID_AUTO,
1761 sfxge_tx_stats[id].name, CTLFLAG_RD | CTLFLAG_STATS,
1762 (unsigned long *)((caddr_t)txq + sfxge_tx_stats[id].offset),
1763 "");
1764 }
1765
1766 return (0);
1767 }
1768
1769 /**
1770 * Destroy a transmit queue.
1771 */
1772 static void
sfxge_tx_qfini(struct sfxge_softc * sc,unsigned int index)1773 sfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index)
1774 {
1775 struct sfxge_txq *txq;
1776 unsigned int nmaps;
1777
1778 txq = sc->txq[index];
1779
1780 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
1781 ("txq->init_state != SFXGE_TXQ_INITIALIZED"));
1782
1783 if (txq->type == SFXGE_TXQ_IP_TCP_UDP_CKSUM)
1784 tso_fini(txq);
1785
1786 /* Free the context arrays. */
1787 free(txq->pend_desc, M_SFXGE);
1788 nmaps = sc->txq_entries;
1789 while (nmaps-- != 0)
1790 bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map);
1791 free(txq->stmp, M_SFXGE);
1792
1793 /* Release DMA memory mapping. */
1794 sfxge_dma_free(&txq->mem);
1795
1796 sc->txq[index] = NULL;
1797
1798 SFXGE_TXQ_LOCK_DESTROY(txq);
1799
1800 free(txq, M_SFXGE);
1801 }
1802
1803 static int
sfxge_tx_qinit(struct sfxge_softc * sc,unsigned int txq_index,enum sfxge_txq_type type,unsigned int evq_index)1804 sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
1805 enum sfxge_txq_type type, unsigned int evq_index)
1806 {
1807 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp);
1808 char name[16];
1809 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
1810 struct sysctl_oid *txq_node;
1811 struct sfxge_txq *txq;
1812 struct sfxge_tx_dpl *stdp;
1813 struct sysctl_oid *dpl_node;
1814 efsys_mem_t *esmp;
1815 unsigned int nmaps;
1816 int rc;
1817
1818 txq = malloc(sizeof(struct sfxge_txq), M_SFXGE, M_ZERO | M_WAITOK);
1819 txq->sc = sc;
1820 txq->entries = sc->txq_entries;
1821 txq->ptr_mask = txq->entries - 1;
1822
1823 sc->txq[txq_index] = txq;
1824 esmp = &txq->mem;
1825
1826 /* Allocate and zero DMA space for the descriptor ring. */
1827 if ((rc = sfxge_dma_alloc(sc, EFX_TXQ_SIZE(sc->txq_entries), esmp)) != 0)
1828 return (rc);
1829
1830 /* Allocate buffer table entries. */
1831 sfxge_sram_buf_tbl_alloc(sc, EFX_TXQ_NBUFS(sc->txq_entries),
1832 &txq->buf_base_id);
1833
1834 /* Create a DMA tag for packet mappings. */
1835 if (bus_dma_tag_create(sc->parent_dma_tag, 1,
1836 encp->enc_tx_dma_desc_boundary,
1837 MIN(0x3FFFFFFFFFFFUL, BUS_SPACE_MAXADDR), BUS_SPACE_MAXADDR, NULL,
1838 NULL, 0x11000, SFXGE_TX_MAPPING_MAX_SEG,
1839 encp->enc_tx_dma_desc_size_max, 0, NULL, NULL,
1840 &txq->packet_dma_tag) != 0) {
1841 device_printf(sc->dev, "Couldn't allocate txq DMA tag\n");
1842 rc = ENOMEM;
1843 goto fail;
1844 }
1845
1846 /* Allocate pending descriptor array for batching writes. */
1847 txq->pend_desc = malloc(sizeof(efx_desc_t) * sc->txq_entries,
1848 M_SFXGE, M_ZERO | M_WAITOK);
1849
1850 /* Allocate and initialise mbuf DMA mapping array. */
1851 txq->stmp = malloc(sizeof(struct sfxge_tx_mapping) * sc->txq_entries,
1852 M_SFXGE, M_ZERO | M_WAITOK);
1853 for (nmaps = 0; nmaps < sc->txq_entries; nmaps++) {
1854 rc = bus_dmamap_create(txq->packet_dma_tag, 0,
1855 &txq->stmp[nmaps].map);
1856 if (rc != 0)
1857 goto fail2;
1858 }
1859
1860 snprintf(name, sizeof(name), "%u", txq_index);
1861 txq_node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(sc->txqs_node),
1862 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
1863 if (txq_node == NULL) {
1864 rc = ENOMEM;
1865 goto fail_txq_node;
1866 }
1867
1868 if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM &&
1869 (rc = tso_init(txq)) != 0)
1870 goto fail3;
1871
1872 /* Initialize the deferred packet list. */
1873 stdp = &txq->dpl;
1874 stdp->std_put_max = sfxge_tx_dpl_put_max;
1875 stdp->std_get_max = sfxge_tx_dpl_get_max;
1876 stdp->std_get_non_tcp_max = sfxge_tx_dpl_get_non_tcp_max;
1877 stdp->std_getp = &stdp->std_get;
1878
1879 SFXGE_TXQ_LOCK_INIT(txq, device_get_nameunit(sc->dev), txq_index);
1880
1881 dpl_node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(txq_node), OID_AUTO,
1882 "dpl", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1883 "Deferred packet list statistics");
1884 if (dpl_node == NULL) {
1885 rc = ENOMEM;
1886 goto fail_dpl_node;
1887 }
1888
1889 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(dpl_node), OID_AUTO,
1890 "get_count", CTLFLAG_RD | CTLFLAG_STATS,
1891 &stdp->std_get_count, 0, "");
1892 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(dpl_node), OID_AUTO,
1893 "get_non_tcp_count", CTLFLAG_RD | CTLFLAG_STATS,
1894 &stdp->std_get_non_tcp_count, 0, "");
1895 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(dpl_node), OID_AUTO,
1896 "get_hiwat", CTLFLAG_RD | CTLFLAG_STATS,
1897 &stdp->std_get_hiwat, 0, "");
1898 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(dpl_node), OID_AUTO,
1899 "put_hiwat", CTLFLAG_RD | CTLFLAG_STATS,
1900 &stdp->std_put_hiwat, 0, "");
1901
1902 rc = sfxge_txq_stat_init(txq, txq_node);
1903 if (rc != 0)
1904 goto fail_txq_stat_init;
1905
1906 txq->type = type;
1907 txq->evq_index = evq_index;
1908 txq->init_state = SFXGE_TXQ_INITIALIZED;
1909
1910 return (0);
1911
1912 fail_txq_stat_init:
1913 fail_dpl_node:
1914 fail3:
1915 fail_txq_node:
1916 free(txq->pend_desc, M_SFXGE);
1917 fail2:
1918 while (nmaps-- != 0)
1919 bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map);
1920 free(txq->stmp, M_SFXGE);
1921 bus_dma_tag_destroy(txq->packet_dma_tag);
1922
1923 fail:
1924 sfxge_dma_free(esmp);
1925
1926 return (rc);
1927 }
1928
1929 static int
sfxge_tx_stat_handler(SYSCTL_HANDLER_ARGS)1930 sfxge_tx_stat_handler(SYSCTL_HANDLER_ARGS)
1931 {
1932 struct sfxge_softc *sc = arg1;
1933 unsigned int id = arg2;
1934 unsigned long sum;
1935 unsigned int index;
1936
1937 /* Sum across all TX queues */
1938 sum = 0;
1939 for (index = 0; index < sc->txq_count; index++)
1940 sum += *(unsigned long *)((caddr_t)sc->txq[index] +
1941 sfxge_tx_stats[id].offset);
1942
1943 return (SYSCTL_OUT(req, &sum, sizeof(sum)));
1944 }
1945
1946 static void
sfxge_tx_stat_init(struct sfxge_softc * sc)1947 sfxge_tx_stat_init(struct sfxge_softc *sc)
1948 {
1949 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
1950 struct sysctl_oid_list *stat_list;
1951 unsigned int id;
1952
1953 stat_list = SYSCTL_CHILDREN(sc->stats_node);
1954
1955 for (id = 0; id < nitems(sfxge_tx_stats); id++) {
1956 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO,
1957 sfxge_tx_stats[id].name,
1958 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
1959 sc, id, sfxge_tx_stat_handler, "LU", "");
1960 }
1961 }
1962
1963 uint64_t
sfxge_tx_get_drops(struct sfxge_softc * sc)1964 sfxge_tx_get_drops(struct sfxge_softc *sc)
1965 {
1966 unsigned int index;
1967 uint64_t drops = 0;
1968 struct sfxge_txq *txq;
1969
1970 /* Sum across all TX queues */
1971 for (index = 0; index < sc->txq_count; index++) {
1972 txq = sc->txq[index];
1973 /*
1974 * In theory, txq->put_overflow and txq->netdown_drops
1975 * should use atomic operation and other should be
1976 * obtained under txq lock, but it is just statistics.
1977 */
1978 drops += txq->drops + txq->get_overflow +
1979 txq->get_non_tcp_overflow +
1980 txq->put_overflow + txq->netdown_drops +
1981 txq->tso_pdrop_too_many + txq->tso_pdrop_no_rsrc;
1982 }
1983 return (drops);
1984 }
1985
1986 void
sfxge_tx_fini(struct sfxge_softc * sc)1987 sfxge_tx_fini(struct sfxge_softc *sc)
1988 {
1989 int index;
1990
1991 index = sc->txq_count;
1992 while (--index >= 0)
1993 sfxge_tx_qfini(sc, index);
1994
1995 sc->txq_count = 0;
1996 }
1997
1998 int
sfxge_tx_init(struct sfxge_softc * sc)1999 sfxge_tx_init(struct sfxge_softc *sc)
2000 {
2001 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp);
2002 struct sfxge_intr *intr __diagused;
2003 int index;
2004 int rc;
2005
2006 intr = &sc->intr;
2007
2008 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
2009 ("intr->state != SFXGE_INTR_INITIALIZED"));
2010
2011 if (sfxge_tx_dpl_get_max <= 0) {
2012 log(LOG_ERR, "%s=%d must be greater than 0",
2013 SFXGE_PARAM_TX_DPL_GET_MAX, sfxge_tx_dpl_get_max);
2014 rc = EINVAL;
2015 goto fail_tx_dpl_get_max;
2016 }
2017 if (sfxge_tx_dpl_get_non_tcp_max <= 0) {
2018 log(LOG_ERR, "%s=%d must be greater than 0",
2019 SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX,
2020 sfxge_tx_dpl_get_non_tcp_max);
2021 rc = EINVAL;
2022 goto fail_tx_dpl_get_non_tcp_max;
2023 }
2024 if (sfxge_tx_dpl_put_max < 0) {
2025 log(LOG_ERR, "%s=%d must be greater or equal to 0",
2026 SFXGE_PARAM_TX_DPL_PUT_MAX, sfxge_tx_dpl_put_max);
2027 rc = EINVAL;
2028 goto fail_tx_dpl_put_max;
2029 }
2030
2031 sc->txq_count = SFXGE_EVQ0_N_TXQ(sc) - 1 + sc->intr.n_alloc;
2032
2033 sc->tso_fw_assisted = sfxge_tso_fw_assisted;
2034 if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) ||
2035 (!encp->enc_fw_assisted_tso_enabled))
2036 sc->tso_fw_assisted &= ~SFXGE_FATSOV1;
2037 if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO_V2) ||
2038 (!encp->enc_fw_assisted_tso_v2_enabled))
2039 sc->tso_fw_assisted &= ~SFXGE_FATSOV2;
2040
2041 sc->txqs_node = SYSCTL_ADD_NODE(device_get_sysctl_ctx(sc->dev),
2042 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO,
2043 "txq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Tx queues");
2044 if (sc->txqs_node == NULL) {
2045 rc = ENOMEM;
2046 goto fail_txq_node;
2047 }
2048
2049 /* Initialize the transmit queues */
2050 if (sc->txq_dynamic_cksum_toggle_supported == B_FALSE) {
2051 if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_NON_CKSUM,
2052 SFXGE_TXQ_NON_CKSUM, 0)) != 0)
2053 goto fail;
2054
2055 if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_CKSUM,
2056 SFXGE_TXQ_IP_CKSUM, 0)) != 0)
2057 goto fail2;
2058 }
2059
2060 for (index = 0;
2061 index < sc->txq_count - SFXGE_EVQ0_N_TXQ(sc) + 1;
2062 index++) {
2063 if ((rc = sfxge_tx_qinit(sc, SFXGE_EVQ0_N_TXQ(sc) - 1 + index,
2064 SFXGE_TXQ_IP_TCP_UDP_CKSUM, index)) != 0)
2065 goto fail3;
2066 }
2067
2068 sfxge_tx_stat_init(sc);
2069
2070 return (0);
2071
2072 fail3:
2073 while (--index >= 0)
2074 sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index);
2075
2076 sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM);
2077
2078 fail2:
2079 sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM);
2080
2081 fail:
2082 fail_txq_node:
2083 sc->txq_count = 0;
2084 fail_tx_dpl_put_max:
2085 fail_tx_dpl_get_non_tcp_max:
2086 fail_tx_dpl_get_max:
2087 return (rc);
2088 }
2089