1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2013 Nexenta Inc. All rights reserved.
14 * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
15 * Copyright 2021 Joyent, Inc.
16 * Copyright 2019 Joshua M. Clulow <josh@sysmgr.org>
17 * Copyright 2025 Hans Rosenfeld
18 * Copyright 2026 Oxide Computer Company
19 */
20
21 /* Based on the NetBSD virtio driver by Minoura Makoto. */
22 /*
23 * Copyright (c) 2010 Minoura Makoto.
24 * All rights reserved.
25 *
26 * Redistribution and use in source and binary forms, with or without
27 * modification, are permitted provided that the following conditions
28 * are met:
29 * 1. Redistributions of source code must retain the above copyright
30 * notice, this list of conditions and the following disclaimer.
31 * 2. Redistributions in binary form must reproduce the above copyright
32 * notice, this list of conditions and the following disclaimer in the
33 * documentation and/or other materials provided with the distribution.
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
36 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
37 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
38 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
39 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
40 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
44 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 */
46
47 /*
48 * VIRTIO NETWORK DRIVER: TRANSMIT PATH
49 *
50 * Transmit buffer management, frame transmission and descriptor reclamation,
51 * including the transmit ring entrypoints that we provide to MAC.
52 */
53
54 #include <sys/types.h>
55 #include <sys/errno.h>
56 #include <sys/param.h>
57 #include <sys/stropts.h>
58 #include <sys/stream.h>
59 #include <sys/strsubr.h>
60 #include <sys/strsun.h>
61 #include <sys/kmem.h>
62 #include <sys/ksynch.h>
63 #include <sys/debug.h>
64 #include <sys/ethernet.h>
65 #include <sys/vlan.h>
66 #include <sys/sysmacros.h>
67 #include <sys/byteorder.h>
68 #include <sys/list.h>
69 #include <sys/pattr.h>
70 #include <sys/ddi.h>
71 #include <sys/sunddi.h>
72
73 #include <inet/tcp.h>
74
75 #include <sys/mac.h>
76 #include <sys/mac_provider.h>
77
78 #include "virtio.h"
79 #include "vioif.h"
80
81 /*
82 * Interval for the periodic TX reclaim.
83 */
84 uint_t vioif_reclaim_ms = 200;
85
86 /*
87 * DMA attributes for mapping larger transmit buffers from the networking
88 * stack. The requirements are quite loose, but note that the SGL entry length
89 * field is 32-bit.
90 */
91 static ddi_dma_attr_t vioif_dma_attr_external = {
92 .dma_attr_version = DMA_ATTR_V0,
93 .dma_attr_addr_lo = 0x0000000000000000,
94 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFF,
95 .dma_attr_count_max = 0x00000000FFFFFFFF,
96 .dma_attr_align = 1,
97 .dma_attr_burstsizes = 1,
98 .dma_attr_minxfer = 1,
99 .dma_attr_maxxfer = 0x00000000FFFFFFFF,
100 .dma_attr_seg = 0x00000000FFFFFFFF,
101 .dma_attr_sgllen = VIOIF_MAX_SEGS - 1,
102 .dma_attr_granular = 1,
103 .dma_attr_flags = 0
104 };
105
106 /*
107 * Result of an attempt to map a message for transmission by reference.
108 */
109 typedef enum vioif_txext_result {
110 VIOIF_TXEXT_OK, /* mapped and appended to the chain */
111 VIOIF_TXEXT_FAIL, /* failed; the message has been freed */
112 VIOIF_TXEXT_TOOBIG /* too many segments; message retained */
113 } vioif_txext_result_t;
114
115 static void vioif_reclaim_restart(vioif_txq_t *);
116
117 static vioif_txbuf_t *
vioif_txbuf_alloc(vioif_txq_t * txq)118 vioif_txbuf_alloc(vioif_txq_t *txq)
119 {
120 vioif_txbuf_t *tb;
121
122 VERIFY(MUTEX_HELD(&txq->vtq_mutex));
123
124 if ((tb = list_remove_head(&txq->vtq_bufs)) != NULL) {
125 txq->vtq_nbufs_alloc++;
126 }
127
128 return (tb);
129 }
130
131 static void
vioif_txbuf_free(vioif_txq_t * txq,vioif_txbuf_t * tb)132 vioif_txbuf_free(vioif_txq_t *txq, vioif_txbuf_t *tb)
133 {
134 VERIFY(MUTEX_HELD(&txq->vtq_mutex));
135
136 VERIFY3U(txq->vtq_nbufs_alloc, >, 0);
137 txq->vtq_nbufs_alloc--;
138
139 virtio_chain_clear(tb->tb_chain);
140 list_insert_head(&txq->vtq_bufs, tb);
141 }
142
143 int
vioif_alloc_txq_bufs(vioif_txq_t * txq)144 vioif_alloc_txq_bufs(vioif_txq_t *txq)
145 {
146 vioif_t *vif = txq->vtq_vif;
147
148 /*
149 * Allocate one contiguous chunk of memory for the transmit buffer
150 * tracking objects. If the ring is unusually small, we'll reduce
151 * our target buffer count accordingly.
152 */
153 txq->vtq_bufs_capacity = MIN(VIRTIO_NET_TX_BUFS,
154 virtio_queue_size(txq->vtq_vq));
155 txq->vtq_bufs_mem = kmem_zalloc(
156 sizeof (vioif_txbuf_t) * txq->vtq_bufs_capacity, KM_SLEEP);
157 list_create(&txq->vtq_bufs, sizeof (vioif_txbuf_t),
158 offsetof(vioif_txbuf_t, tb_link));
159
160 /*
161 * Put everything in the free list straight away in order to simplify
162 * the use of vioif_free_txq_bufs() for cleanup on allocation failure.
163 */
164 for (uint_t i = 0; i < txq->vtq_bufs_capacity; i++) {
165 list_insert_tail(&txq->vtq_bufs, &txq->vtq_bufs_mem[i]);
166 }
167
168 /*
169 * The transmit inline buffer is small (less than a page), so it's
170 * reasonable to request a single cookie.
171 */
172 ddi_dma_attr_t attr = vioif_dma_attr_bufs;
173 attr.dma_attr_sgllen = 1;
174
175 for (vioif_txbuf_t *tb = list_head(&txq->vtq_bufs); tb != NULL;
176 tb = list_next(&txq->vtq_bufs, tb)) {
177 if ((tb->tb_dma = virtio_dma_alloc(vif->vif_virtio,
178 VIOIF_TX_INLINE_SIZE, &attr,
179 DDI_DMA_STREAMING | DDI_DMA_WRITE, KM_SLEEP)) == NULL) {
180 return (ENOMEM);
181 }
182 VERIFY3U(virtio_dma_ncookies(tb->tb_dma), ==, 1);
183
184 if ((tb->tb_chain = virtio_chain_alloc(txq->vtq_vq,
185 KM_SLEEP)) == NULL) {
186 return (ENOMEM);
187 }
188 virtio_chain_data_set(tb->tb_chain, tb);
189
190 tb->tb_dmaext_capacity = VIOIF_MAX_SEGS - 1;
191 tb->tb_dmaext = kmem_zalloc(
192 sizeof (virtio_dma_t *) * tb->tb_dmaext_capacity,
193 KM_SLEEP);
194 }
195
196 return (0);
197 }
198
199 void
vioif_free_txq_bufs(vioif_txq_t * txq)200 vioif_free_txq_bufs(vioif_txq_t *txq)
201 {
202 if (txq->vtq_bufs_mem == NULL)
203 return;
204
205 VERIFY3U(txq->vtq_nbufs_alloc, ==, 0);
206 for (uint_t i = 0; i < txq->vtq_bufs_capacity; i++) {
207 vioif_txbuf_t *tb = &txq->vtq_bufs_mem[i];
208
209 /*
210 * Ensure that this txbuf is now in the free list:
211 */
212 VERIFY(list_link_active(&tb->tb_link));
213 list_remove(&txq->vtq_bufs, tb);
214
215 /*
216 * We should not have an mblk chain at this point.
217 */
218 VERIFY3P(tb->tb_mp, ==, NULL);
219
220 if (tb->tb_dma != NULL) {
221 virtio_dma_free(tb->tb_dma);
222 tb->tb_dma = NULL;
223 }
224
225 if (tb->tb_chain != NULL) {
226 virtio_chain_free(tb->tb_chain);
227 tb->tb_chain = NULL;
228 }
229
230 if (tb->tb_dmaext != NULL) {
231 for (uint_t j = 0; j < tb->tb_dmaext_capacity; j++) {
232 if (tb->tb_dmaext[j] != NULL) {
233 virtio_dma_free(tb->tb_dmaext[j]);
234 tb->tb_dmaext[j] = NULL;
235 }
236 }
237
238 kmem_free(tb->tb_dmaext,
239 sizeof (virtio_dma_t *) * tb->tb_dmaext_capacity);
240 tb->tb_dmaext = NULL;
241 tb->tb_dmaext_capacity = 0;
242 }
243 }
244 VERIFY(list_is_empty(&txq->vtq_bufs));
245 list_destroy(&txq->vtq_bufs);
246
247 kmem_free(txq->vtq_bufs_mem,
248 sizeof (vioif_txbuf_t) * txq->vtq_bufs_capacity);
249 txq->vtq_bufs_mem = NULL;
250 txq->vtq_bufs_capacity = 0;
251 }
252
253 static uint_t
vioif_reclaim_used_tx(vioif_txq_t * txq)254 vioif_reclaim_used_tx(vioif_txq_t *txq)
255 {
256 virtio_chain_t *vic;
257 uint_t num_reclaimed = 0;
258
259 VERIFY(MUTEX_NOT_HELD(&txq->vtq_mutex));
260
261 while ((vic = virtio_queue_poll(txq->vtq_vq)) != NULL) {
262 vioif_txbuf_t *tb = virtio_chain_data(vic);
263
264 if (tb->tb_mp != NULL) {
265 /*
266 * Unbind the external mapping.
267 */
268 for (uint_t i = 0; i < tb->tb_dmaext_capacity; i++) {
269 if (tb->tb_dmaext[i] == NULL) {
270 continue;
271 }
272
273 virtio_dma_unbind(tb->tb_dmaext[i]);
274 }
275
276 freemsg(tb->tb_mp);
277 tb->tb_mp = NULL;
278 }
279
280 /*
281 * Return this transmit buffer to the free list for reuse.
282 */
283 mutex_enter(&txq->vtq_mutex);
284 vioif_txbuf_free(txq, tb);
285 mutex_exit(&txq->vtq_mutex);
286
287 num_reclaimed++;
288 }
289
290 /* Return ring to transmitting state if descriptors were reclaimed. */
291 if (num_reclaimed > 0) {
292 boolean_t do_update = B_FALSE;
293
294 mutex_enter(&txq->vtq_mutex);
295 txq->vtq_stat_tx_reclaim += num_reclaimed;
296 if (txq->vtq_corked) {
297 /*
298 * TX was corked on a lack of available descriptors.
299 * That dire state has passed so the TX interrupt can
300 * be disabled and MAC can be notified that
301 * transmission is possible again.
302 */
303 txq->vtq_corked = false;
304 virtio_queue_no_interrupt(txq->vtq_vq, B_TRUE);
305 do_update = B_TRUE;
306 }
307
308 mutex_exit(&txq->vtq_mutex);
309 if (do_update) {
310 mac_tx_ring_update(txq->vtq_vif->vif_mac_handle,
311 txq->vtq_ringh);
312 }
313 }
314
315 return (num_reclaimed);
316 }
317
318 static void
vioif_reclaim_periodic(void * arg)319 vioif_reclaim_periodic(void *arg)
320 {
321 vioif_txq_t *txq = arg;
322 uint_t num_reclaimed;
323
324 num_reclaimed = vioif_reclaim_used_tx(txq);
325
326 mutex_enter(&txq->vtq_mutex);
327 txq->vtq_reclaim_tid = 0;
328 /*
329 * If used descriptors were reclaimed or TX descriptors appear to be
330 * outstanding, the ring is considered active and periodic reclamation
331 * is necessary for now.
332 */
333 if (num_reclaimed != 0 || virtio_queue_nactive(txq->vtq_vq) != 0) {
334 /* Do not reschedule if the ring is being drained. */
335 if (!txq->vtq_drain) {
336 vioif_reclaim_restart(txq);
337 }
338 }
339 mutex_exit(&txq->vtq_mutex);
340 }
341
342 static void
vioif_reclaim_restart(vioif_txq_t * txq)343 vioif_reclaim_restart(vioif_txq_t *txq)
344 {
345 VERIFY(MUTEX_HELD(&txq->vtq_mutex));
346 VERIFY(!txq->vtq_drain);
347
348 if (txq->vtq_reclaim_tid == 0) {
349 txq->vtq_reclaim_tid = timeout(vioif_reclaim_periodic, txq,
350 MSEC_TO_TICK_ROUNDUP(vioif_reclaim_ms));
351 }
352 }
353
354 void
vioif_tx_drain(vioif_txq_t * txq)355 vioif_tx_drain(vioif_txq_t *txq)
356 {
357 VERIFY(MUTEX_HELD(&txq->vtq_mutex));
358
359 txq->vtq_drain = true;
360 /* Put a stop to the periodic reclaim if it is running */
361 if (txq->vtq_reclaim_tid != 0) {
362 timeout_id_t tid = txq->vtq_reclaim_tid;
363
364 /*
365 * With vtq_drain set, there is no risk that a racing
366 * vioif_reclaim_periodic() call will reschedule itself.
367 *
368 * Being part of the mc_stop hook also guarantees that
369 * vioif_ring_tx() will not be called to restart it.
370 */
371 txq->vtq_reclaim_tid = 0;
372 mutex_exit(&txq->vtq_mutex);
373 (void) untimeout(tid);
374 mutex_enter(&txq->vtq_mutex);
375 }
376 virtio_queue_no_interrupt(txq->vtq_vq, B_TRUE);
377
378 /*
379 * Wait for all of the TX descriptors to be processed by the host so
380 * they can be reclaimed.
381 */
382 while (txq->vtq_nbufs_alloc > 0) {
383 mutex_exit(&txq->vtq_mutex);
384 (void) vioif_reclaim_used_tx(txq);
385 delay(5);
386 mutex_enter(&txq->vtq_mutex);
387 }
388 VERIFY(!txq->vtq_corked);
389 VERIFY3U(txq->vtq_reclaim_tid, ==, 0);
390 VERIFY3U(virtio_queue_nactive(txq->vtq_vq), ==, 0);
391 }
392
393 /*
394 * Add the descriptor entry for the virtio net header, which is held at the
395 * start of the transmit buffer's inline DMA memory. For legacy devices, and
396 * those that have not negotiated VIRTIO_F_ANY_LAYOUT, the header must appear
397 * in a separate descriptor entry to the rest of the buffer. We do that for
398 * modern devices too.
399 */
400 static int
vioif_tx_header_append(vioif_t * vif,vioif_txbuf_t * tb)401 vioif_tx_header_append(vioif_t *vif, vioif_txbuf_t *tb)
402 {
403 return (virtio_chain_append(tb->tb_chain,
404 virtio_dma_cookie_pa(tb->tb_dma, 0), vif->vif_rxbuf_hdrlen,
405 VIRTIO_DIR_DEVICE_READS));
406 }
407
408 static int
vioif_tx_inline(vioif_txq_t * txq,vioif_txbuf_t * tb,mblk_t * mp,size_t msg_size)409 vioif_tx_inline(vioif_txq_t *txq, vioif_txbuf_t *tb, mblk_t *mp,
410 size_t msg_size)
411 {
412 VERIFY(MUTEX_NOT_HELD(&txq->vtq_mutex));
413
414 VERIFY3U(msg_size, <=, virtio_dma_size(tb->tb_dma) - VIOIF_HEADER_SKIP);
415
416 /*
417 * Copy the message into the inline buffer and then free the message.
418 */
419 mcopymsg(mp, virtio_dma_va(tb->tb_dma, VIOIF_HEADER_SKIP));
420
421 if (virtio_chain_append(tb->tb_chain,
422 virtio_dma_cookie_pa(tb->tb_dma, 0) + VIOIF_HEADER_SKIP,
423 msg_size, VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) {
424 return (DDI_FAILURE);
425 }
426
427 return (DDI_SUCCESS);
428 }
429
430 /*
431 * Map a message for transmission by reference, binding each of its mblks for
432 * DMA and appending the resulting cookies to the transmit buffer's
433 * descriptor chain. Returns VIOIF_TXEXT_TOOBIG, with the message retained
434 * and the chain cleared, if it has more disjoint segments than a chain can
435 * carry. The caller may pull the message up into contiguous storage and try
436 * again. On any other failure the message is freed.
437 */
438 static vioif_txext_result_t
vioif_tx_external(vioif_txq_t * txq,vioif_txbuf_t * tb,mblk_t * mp,size_t msg_size)439 vioif_tx_external(vioif_txq_t *txq, vioif_txbuf_t *tb, mblk_t *mp,
440 size_t msg_size)
441 {
442 vioif_t *vif = txq->vtq_vif;
443 vioif_txext_result_t res = VIOIF_TXEXT_FAIL;
444
445 VERIFY(MUTEX_NOT_HELD(&txq->vtq_mutex));
446
447 mblk_t *nmp = mp;
448 tb->tb_ndmaext = 0;
449
450 while (nmp != NULL) {
451 size_t len;
452
453 if ((len = MBLKL(nmp)) == 0) {
454 /*
455 * Skip any zero-length entries in the chain.
456 */
457 nmp = nmp->b_cont;
458 continue;
459 }
460
461 if (tb->tb_ndmaext >= tb->tb_dmaext_capacity) {
462 mutex_enter(&txq->vtq_mutex);
463 txq->vtq_txfail_indirect_limit++;
464 mutex_exit(&txq->vtq_mutex);
465 res = VIOIF_TXEXT_TOOBIG;
466 goto fail;
467 }
468
469 if (tb->tb_dmaext[tb->tb_ndmaext] == NULL) {
470 /*
471 * Allocate a DMA handle for this slot.
472 */
473 if ((tb->tb_dmaext[tb->tb_ndmaext] =
474 virtio_dma_alloc_nomem(vif->vif_virtio,
475 &vioif_dma_attr_external, KM_SLEEP)) == NULL) {
476 mutex_enter(&txq->vtq_mutex);
477 txq->vtq_notxbuf++;
478 mutex_exit(&txq->vtq_mutex);
479 goto fail;
480 }
481 }
482 virtio_dma_t *extdma = tb->tb_dmaext[tb->tb_ndmaext++];
483
484 if (virtio_dma_bind(extdma, nmp->b_rptr, len,
485 DDI_DMA_WRITE | DDI_DMA_STREAMING, KM_SLEEP) !=
486 DDI_SUCCESS) {
487 mutex_enter(&txq->vtq_mutex);
488 txq->vtq_txfail_dma_bind++;
489 mutex_exit(&txq->vtq_mutex);
490 goto fail;
491 }
492
493 for (uint_t n = 0; n < virtio_dma_ncookies(extdma); n++) {
494 uint64_t pa = virtio_dma_cookie_pa(extdma, n);
495 size_t sz = virtio_dma_cookie_size(extdma, n);
496
497 if (virtio_chain_append(tb->tb_chain, pa, sz,
498 VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) {
499 mutex_enter(&txq->vtq_mutex);
500 txq->vtq_txfail_indirect_limit++;
501 mutex_exit(&txq->vtq_mutex);
502 res = VIOIF_TXEXT_TOOBIG;
503 goto fail;
504 }
505 }
506
507 nmp = nmp->b_cont;
508 }
509
510 /*
511 * We need to keep the message around until we reclaim the buffer from
512 * the device before freeing it.
513 */
514 tb->tb_mp = mp;
515
516 return (VIOIF_TXEXT_OK);
517
518 fail:
519 for (uint_t n = 0; n < tb->tb_ndmaext; n++) {
520 if (tb->tb_dmaext[n] != NULL) {
521 virtio_dma_unbind(tb->tb_dmaext[n]);
522 }
523 }
524 tb->tb_ndmaext = 0;
525
526 if (res == VIOIF_TXEXT_TOOBIG) {
527 /*
528 * Clear the chain, which will have entries for the virtio net
529 * header and any segments appended before the capacity was
530 * reached, so that the caller can rebuild it and retry with a
531 * pulled up copy of the message. The other failure paths can
532 * leave the chain as it is, since the buffer is going back to
533 * the free list, which clears the chain as part of returning
534 * it.
535 */
536 virtio_chain_clear(tb->tb_chain);
537 } else {
538 freemsg(mp);
539 }
540
541 return (res);
542 }
543
544 /*
545 * Attempt to transmit a single message on a transmit ring. Returns B_TRUE
546 * if the message was consumed, whether it was transmitted or dropped as
547 * untransmittable. Returns B_FALSE, with the message retained, only when no
548 * transmit buffer is available. The caller may retry the same message once
549 * descriptors have been reclaimed.
550 */
551 static boolean_t
vioif_send(vioif_txq_t * txq,mblk_t * mp)552 vioif_send(vioif_txq_t *txq, mblk_t *mp)
553 {
554 vioif_t *vif = txq->vtq_vif;
555
556 VERIFY(MUTEX_NOT_HELD(&txq->vtq_mutex));
557
558 vioif_txbuf_t *tb = NULL;
559 struct virtio_net_hdr *vnh = NULL;
560 size_t msg_size = 0;
561 uint32_t csum_start;
562 uint32_t csum_stuff;
563 uint32_t csum_flags;
564 uint32_t lso_flags;
565 uint32_t lso_mss;
566 mblk_t *nmp;
567 int ret;
568 boolean_t lso_required = B_FALSE;
569 struct ether_header *ether = (void *)mp->b_rptr;
570
571 for (nmp = mp; nmp; nmp = nmp->b_cont)
572 msg_size += MBLKL(nmp);
573
574 if (vif->vif_tx_tso4 || vif->vif_tx_tso6) {
575 mac_lso_get(mp, &lso_mss, &lso_flags);
576 lso_required = (lso_flags & HW_LSO) != 0;
577 }
578
579 mutex_enter(&txq->vtq_mutex);
580 if ((tb = vioif_txbuf_alloc(txq)) == NULL) {
581 txq->vtq_notxbuf++;
582 mutex_exit(&txq->vtq_mutex);
583 return (B_FALSE);
584 }
585 mutex_exit(&txq->vtq_mutex);
586
587 /*
588 * Use the inline buffer for the virtio net header. Zero the portion
589 * of our DMA allocation prior to the packet data.
590 */
591 vnh = virtio_dma_va(tb->tb_dma, 0);
592 bzero(vnh, VIOIF_HEADER_SKIP);
593
594 if (vioif_tx_header_append(vif, tb) != DDI_SUCCESS)
595 goto drop;
596
597 mac_hcksum_get(mp, &csum_start, &csum_stuff, NULL, NULL, &csum_flags);
598
599 /*
600 * They want us to do the TCP/UDP csum calculation.
601 */
602 if (csum_flags & HCK_PARTIALCKSUM) {
603 int eth_hsize;
604
605 /*
606 * Did we ask for it?
607 */
608 ASSERT(vif->vif_tx_csum);
609
610 /*
611 * We only asked for partial csum packets.
612 */
613 ASSERT(!(csum_flags & HCK_IPV4_HDRCKSUM));
614 ASSERT(!(csum_flags & HCK_FULLCKSUM));
615
616 if (ether->ether_type == htons(ETHERTYPE_VLAN)) {
617 eth_hsize = sizeof (struct ether_vlan_header);
618 } else {
619 eth_hsize = sizeof (struct ether_header);
620 }
621
622 vnh->vnh_flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
623 vnh->vnh_csum_start = eth_hsize + csum_start;
624 vnh->vnh_csum_offset = csum_stuff - csum_start;
625 }
626
627 /*
628 * Setup LSO fields if required.
629 */
630 if (lso_required) {
631 const mac_ether_offload_flags_t needed =
632 MEOI_L2INFO_SET | MEOI_L3INFO_SET | MEOI_L4INFO_SET;
633 mac_ether_offload_info_t meo;
634 uint32_t cksum;
635 size_t len, hdrs_len;
636 tcpha_t *tcpha;
637
638 mac_ether_offload_info(mp, &meo);
639 if ((meo.meoi_flags & needed) != needed) {
640 goto drop;
641 }
642
643 if (meo.meoi_l4proto != IPPROTO_TCP) {
644 goto drop;
645 }
646
647 if (meo.meoi_l3proto == ETHERTYPE_IP && vif->vif_tx_tso4) {
648 vnh->vnh_gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
649 } else if (meo.meoi_l3proto == ETHERTYPE_IPV6 &&
650 vif->vif_tx_tso6) {
651 vnh->vnh_gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
652 } else {
653 goto drop;
654 }
655
656 /*
657 * The TCP stack does not include the length in the TCP
658 * pseudo-header when it is performing LSO since hardware
659 * generally asks for it to be removed (as it'll change).
660 * Unfortunately, for virtio, we actually need it. This means we
661 * need to go through and calculate the actual length and fix
662 * things up. Because the virtio spec cares about the ECN flag
663 * and indicating that, at least this means we'll have that
664 * available as well.
665 */
666 hdrs_len = meo.meoi_l2hlen + meo.meoi_l3hlen + meo.meoi_l4hlen;
667 if (MBLKL(mp) < hdrs_len) {
668 mblk_t *pullmp;
669
670 /*
671 * The headers do not all sit within the first mblk.
672 * Replace the message with an equivalent one in which
673 * they do, so that the checksum manipulation below
674 * lands in the message that is transmitted.
675 */
676 if ((pullmp = msgpullup(mp, hdrs_len)) == NULL)
677 goto drop;
678 freemsg(mp);
679 mp = pullmp;
680 ether = (void *)mp->b_rptr;
681 }
682 tcpha = (tcpha_t *)(mp->b_rptr + meo.meoi_l2hlen +
683 meo.meoi_l3hlen);
684
685 len = meo.meoi_len - meo.meoi_l2hlen - meo.meoi_l3hlen;
686 cksum = ntohs(tcpha->tha_sum) + len;
687 cksum = (cksum >> 16) + (cksum & 0xffff);
688 cksum = (cksum >> 16) + (cksum & 0xffff);
689 tcpha->tha_sum = htons(cksum);
690
691 if (tcpha->tha_flags & TH_CWR) {
692 vnh->vnh_gso_type |= VIRTIO_NET_HDR_GSO_ECN;
693 }
694 vnh->vnh_gso_size = (uint16_t)lso_mss;
695 vnh->vnh_hdr_len = hdrs_len;
696 }
697
698 /*
699 * The device does not maintain its own statistics about broadcast or
700 * multicast packets, so we have to check the destination address
701 * ourselves.
702 */
703 if ((ether->ether_dhost.ether_addr_octet[0] & 0x01) != 0) {
704 mutex_enter(&txq->vtq_mutex);
705 if (ether_cmp(ðer->ether_dhost, vioif_broadcast) == 0) {
706 txq->vtq_brdcstxmt++;
707 } else {
708 txq->vtq_multixmt++;
709 }
710 mutex_exit(&txq->vtq_mutex);
711 }
712
713 /*
714 * For small packets, copy into the preallocated inline buffer rather
715 * than incur the overhead of mapping. Note that both of these
716 * functions ensure that "mp" is freed before returning.
717 */
718 if (msg_size < vif->vif_txcopy_thresh) {
719 ret = vioif_tx_inline(txq, tb, mp, msg_size);
720 mp = NULL;
721 } else {
722 vioif_txext_result_t xres;
723
724 xres = vioif_tx_external(txq, tb, mp, msg_size);
725 if (xres == VIOIF_TXEXT_TOOBIG) {
726 mblk_t *pulled;
727
728 /*
729 * The message is spread across more disjoint pieces
730 * of memory than a descriptor chain can carry.
731 * Rebuild it as a single contiguous message, re-add
732 * the header descriptor that was lost when the chain
733 * was cleared, and try once more. The copy is
734 * expensive, but better than dropping the frame.
735 */
736 if ((pulled = msgpullup(mp, -1)) == NULL)
737 goto drop;
738 freemsg(mp);
739 mp = pulled;
740
741 if (vioif_tx_header_append(vif, tb) != DDI_SUCCESS)
742 goto drop;
743 xres = vioif_tx_external(txq, tb, mp, msg_size);
744 }
745
746 switch (xres) {
747 case VIOIF_TXEXT_OK:
748 ret = DDI_SUCCESS;
749 /*
750 * The message is held in `tb_mp` until reclaim and
751 * we no longer own it.
752 */
753 mp = NULL;
754 break;
755 case VIOIF_TXEXT_FAIL:
756 /* The message has already been freed. */
757 ret = DDI_FAILURE;
758 mp = NULL;
759 break;
760 case VIOIF_TXEXT_TOOBIG:
761 default:
762 /*
763 * A single contiguous message cannot exceed the chain
764 * capacity, so this should not recur after the pullup
765 * above. Drop the frame if it somehow does.
766 */
767 goto drop;
768 }
769 }
770
771 if (ret != DDI_SUCCESS) {
772 goto drop;
773 }
774
775 mutex_enter(&txq->vtq_mutex);
776 txq->vtq_opackets++;
777 txq->vtq_obytes += msg_size;
778 mutex_exit(&txq->vtq_mutex);
779
780 virtio_dma_sync(tb->tb_dma, DDI_DMA_SYNC_FORDEV);
781 virtio_chain_submit(tb->tb_chain, B_TRUE);
782
783 return (B_TRUE);
784
785 drop:
786 /*
787 * This message cannot be transmitted. Consume and drop it, counting
788 * it against the ring's error statistic. Returning it to MAC would
789 * block the ring behind a condition that descriptor reclamation
790 * cannot resolve.
791 */
792 freemsg(mp);
793
794 mutex_enter(&txq->vtq_mutex);
795 txq->vtq_oerrors++;
796 vioif_txbuf_free(txq, tb);
797 mutex_exit(&txq->vtq_mutex);
798
799 return (B_TRUE);
800 }
801
802 /*
803 * This is the MAC transmit entrypoint for a single transmit ring; MAC hands
804 * us one frame at a time. Returning NULL indicates the frame was consumed,
805 * whether it was transmitted or dropped; returning the message blocks the
806 * ring until we call mac_tx_ring_update().
807 */
808 static mblk_t *
vioif_ring_tx(void * arg,mblk_t * mp)809 vioif_ring_tx(void *arg, mblk_t *mp)
810 {
811 vioif_txq_t *txq = arg;
812
813 VERIFY3P(mp->b_next, ==, NULL);
814
815 /*
816 * Prior to attempting to send any more frames, do a reclaim to pick up
817 * any descriptors which have been processed by the host.
818 */
819 if (virtio_queue_nactive(txq->vtq_vq) != 0) {
820 (void) vioif_reclaim_used_tx(txq);
821 }
822
823 for (;;) {
824 if (vioif_send(txq, mp)) {
825 break;
826 }
827
828 /*
829 * If there are no descriptors available, try to reclaim some,
830 * allowing a retry of the send if some are found.
831 */
832 if (vioif_reclaim_used_tx(txq) != 0) {
833 continue;
834 }
835
836 mutex_enter(&txq->vtq_mutex);
837 if (!list_is_empty(&txq->vtq_bufs)) {
838 /*
839 * A racing reclaim returned buffers to the free list
840 * after our own reclaim found nothing. Retry the
841 * send.
842 */
843 mutex_exit(&txq->vtq_mutex);
844 continue;
845 }
846
847 /*
848 * Otherwise, enable the TX ring interrupt so that as soon as
849 * a descriptor becomes available, transmission can begin
850 * again. For safety, make sure the periodic reclaim is
851 * running as well.
852 */
853 txq->vtq_corked = true;
854 virtio_queue_no_interrupt(txq->vtq_vq, B_FALSE);
855 vioif_reclaim_restart(txq);
856 mutex_exit(&txq->vtq_mutex);
857
858 /*
859 * Descriptors returned by the device before the interrupt was
860 * enabled above will never raise it. Reclaim once more now.
861 * If anything is found this also clears the cork and notifies
862 * MAC, and the send can be retried immediately.
863 */
864 if (vioif_reclaim_used_tx(txq) != 0) {
865 continue;
866 }
867
868 return (mp);
869 }
870
871 /* Ensure the periodic reclaim has been started. */
872 mutex_enter(&txq->vtq_mutex);
873 vioif_reclaim_restart(txq);
874 mutex_exit(&txq->vtq_mutex);
875
876 return (NULL);
877 }
878
879 uint_t
vioif_tx_handler(caddr_t arg0,caddr_t arg1 __unused)880 vioif_tx_handler(caddr_t arg0, caddr_t arg1 __unused)
881 {
882 vioif_txq_t *txq = (vioif_txq_t *)arg0;
883
884 /*
885 * The TX interrupt could race with other reclamation activity, so
886 * interpreting the return value is unimportant.
887 */
888 (void) vioif_reclaim_used_tx(txq);
889
890 return (DDI_INTR_CLAIMED);
891 }
892
893 static int
vioif_tx_ring_stat(mac_ring_driver_t rh,uint_t stat,uint64_t * val)894 vioif_tx_ring_stat(mac_ring_driver_t rh, uint_t stat, uint64_t *val)
895 {
896 vioif_txq_t *txq = (vioif_txq_t *)rh;
897
898 switch (stat) {
899 case MAC_STAT_OBYTES:
900 *val = txq->vtq_obytes;
901 break;
902 case MAC_STAT_OPACKETS:
903 *val = txq->vtq_opackets;
904 break;
905 default:
906 *val = 0;
907 return (ENOTSUP);
908 }
909
910 return (0);
911 }
912
913 void
vioif_fill_tx_ring(void * arg,mac_ring_type_t rtype,const int group_index,const int ring_index,mac_ring_info_t * infop,mac_ring_handle_t rh)914 vioif_fill_tx_ring(void *arg, mac_ring_type_t rtype, const int group_index,
915 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
916 {
917 vioif_t *vif = arg;
918 vioif_txq_t *txq;
919
920 VERIFY3S(rtype, ==, MAC_RING_TYPE_TX);
921 /*
922 * We do not provide transmit groups, so the group index here is
923 * expected to be -1.
924 */
925 VERIFY3S(group_index, ==, -1);
926 VERIFY3S(ring_index, >=, 0);
927 VERIFY3U(ring_index, <, vif->vif_nqpairs);
928
929 txq = &vif->vif_txqs[ring_index];
930 txq->vtq_ringh = rh;
931
932 infop->mri_driver = (mac_ring_driver_t)txq;
933 infop->mri_start = NULL;
934 infop->mri_stop = NULL;
935 infop->mri_tx = vioif_ring_tx;
936 infop->mri_stat = vioif_tx_ring_stat;
937
938 /*
939 * Provide the interrupt handle for the MSI-X vector servicing this
940 * queue, if it has one, so that MAC can retarget the interrupt to
941 * the CPU associated with the ring.
942 */
943 infop->mri_intr.mi_ddi_handle = virtio_queue_intr_handle(txq->vtq_vq);
944 }
945