1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #ifndef _IF_VTNETVAR_H
30 #define _IF_VTNETVAR_H
31
32 #define VTNET_ALTQ_CAPABLE (0)
33 #ifdef ALTQ
34 #undef VTNET_ALTQ_CAPABLE
35 #define VTNET_ALTQ_CAPABLE (1)
36 #endif
37
38 struct vtnet_softc;
39
40 struct vtnet_statistics {
41 uint64_t mbuf_alloc_failed;
42
43 uint64_t rx_frame_too_large;
44 uint64_t rx_enq_replacement_failed;
45 uint64_t rx_mergeable_failed;
46 uint64_t rx_csum_bad_ethtype;
47 uint64_t rx_csum_bad_ipproto;
48 uint64_t rx_csum_bad_offset;
49 uint64_t rx_csum_inaccessible_ipproto;
50 uint64_t tx_csum_unknown_ethtype;
51 uint64_t tx_csum_proto_mismatch;
52 uint64_t tx_tso_not_tcp;
53 uint64_t tx_tso_without_csum;
54 uint64_t tx_defragged;
55 uint64_t tx_defrag_failed;
56
57 /*
58 * These are accumulated from each Rx/Tx queue.
59 */
60 uint64_t rx_csum_failed;
61 uint64_t rx_csum_offloaded;
62 uint64_t rx_task_rescheduled;
63 uint64_t tx_csum_offloaded;
64 uint64_t tx_tso_offloaded;
65 uint64_t tx_task_rescheduled;
66 };
67
68 struct vtnet_rxq_stats {
69 uint64_t vrxs_ipackets; /* if_ipackets */
70 uint64_t vrxs_ibytes; /* if_ibytes */
71 uint64_t vrxs_iqdrops; /* if_iqdrops */
72 uint64_t vrxs_ierrors; /* if_ierrors */
73 uint64_t vrxs_csum;
74 uint64_t vrxs_csum_failed;
75 uint64_t vrxs_host_lro;
76 uint64_t vrxs_rescheduled;
77 };
78
79 struct vtnet_rxq {
80 struct mtx vtnrx_mtx;
81 struct vtnet_softc *vtnrx_sc;
82 struct virtqueue *vtnrx_vq;
83 struct sglist *vtnrx_sg;
84 int vtnrx_id;
85 struct vtnet_rxq_stats vtnrx_stats;
86 struct taskqueue *vtnrx_tq;
87 struct task vtnrx_intrtask;
88 struct lro_ctrl vtnrx_lro;
89 #ifdef DEV_NETMAP
90 uint32_t vtnrx_nm_refill;
91 struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;
92 #endif /* DEV_NETMAP */
93 char vtnrx_name[16];
94 } __aligned(CACHE_LINE_SIZE);
95
96 #define VTNET_RXQ_LOCK(_rxq) mtx_lock(&(_rxq)->vtnrx_mtx)
97 #define VTNET_RXQ_UNLOCK(_rxq) mtx_unlock(&(_rxq)->vtnrx_mtx)
98 #define VTNET_RXQ_LOCK_ASSERT(_rxq) \
99 mtx_assert(&(_rxq)->vtnrx_mtx, MA_OWNED)
100 #define VTNET_RXQ_LOCK_ASSERT_NOTOWNED(_rxq) \
101 mtx_assert(&(_rxq)->vtnrx_mtx, MA_NOTOWNED)
102
103 struct vtnet_txq_stats {
104 uint64_t vtxs_opackets; /* if_opackets */
105 uint64_t vtxs_obytes; /* if_obytes */
106 uint64_t vtxs_omcasts; /* if_omcasts */
107 uint64_t vtxs_csum;
108 uint64_t vtxs_tso;
109 uint64_t vtxs_rescheduled;
110 };
111
112 struct vtnet_txq {
113 struct mtx vtntx_mtx;
114 struct vtnet_softc *vtntx_sc;
115 struct virtqueue *vtntx_vq;
116 struct sglist *vtntx_sg;
117 struct buf_ring *vtntx_br;
118 int vtntx_id;
119 int vtntx_watchdog;
120 int vtntx_intr_threshold;
121 struct vtnet_txq_stats vtntx_stats;
122 struct taskqueue *vtntx_tq;
123 struct task vtntx_intrtask;
124 struct task vtntx_defrtask;
125 #ifdef DEV_NETMAP
126 struct virtio_net_hdr_mrg_rxbuf vtntx_shrhdr;
127 #endif /* DEV_NETMAP */
128 char vtntx_name[16];
129 } __aligned(CACHE_LINE_SIZE);
130
131 #define VTNET_TXQ_LOCK(_txq) mtx_lock(&(_txq)->vtntx_mtx)
132 #define VTNET_TXQ_TRYLOCK(_txq) mtx_trylock(&(_txq)->vtntx_mtx)
133 #define VTNET_TXQ_UNLOCK(_txq) mtx_unlock(&(_txq)->vtntx_mtx)
134 #define VTNET_TXQ_LOCK_ASSERT(_txq) \
135 mtx_assert(&(_txq)->vtntx_mtx, MA_OWNED)
136 #define VTNET_TXQ_LOCK_ASSERT_NOTOWNED(_txq) \
137 mtx_assert(&(_txq)->vtntx_mtx, MA_NOTOWNED)
138
139 struct vtnet_softc {
140 device_t vtnet_dev;
141 if_t vtnet_ifp;
142 struct vtnet_rxq *vtnet_rxqs;
143 struct vtnet_txq *vtnet_txqs;
144 pfil_head_t vtnet_pfil;
145 uint64_t vtnet_features;
146
147 uint32_t vtnet_flags;
148 #define VTNET_FLAG_MODERN 0x0001
149 #define VTNET_FLAG_MAC 0x0002
150 #define VTNET_FLAG_CTRL_VQ 0x0004
151 #define VTNET_FLAG_CTRL_RX 0x0008
152 #define VTNET_FLAG_CTRL_MAC 0x0010
153 #define VTNET_FLAG_VLAN_FILTER 0x0020
154 #define VTNET_FLAG_TSO_ECN 0x0040
155 #define VTNET_FLAG_MRG_RXBUFS 0x0080
156 #define VTNET_FLAG_LRO_NOMRG 0x0100
157 #define VTNET_FLAG_MQ 0x0200
158 #define VTNET_FLAG_INDIRECT 0x0400
159 #define VTNET_FLAG_EVENT_IDX 0x0800
160 #define VTNET_FLAG_SUSPENDED 0x1000
161 #define VTNET_FLAG_FIXUP_NEEDS_CSUM 0x2000
162 #define VTNET_FLAG_SW_LRO 0x4000
163
164 u_int vtnet_hdr_size;
165 int vtnet_rx_nmbufs;
166 int vtnet_rx_clustersz;
167 int vtnet_rx_nsegs;
168 int vtnet_rx_process_limit;
169 int vtnet_link_active;
170 int vtnet_act_vq_pairs;
171 int vtnet_req_vq_pairs;
172 int vtnet_max_vq_pairs;
173 int vtnet_tx_nsegs;
174 int vtnet_if_flags;
175 u_int vtnet_max_mtu;
176 int vtnet_lro_entry_count;
177 int vtnet_lro_mbufq_depth;
178
179 struct virtqueue *vtnet_ctrl_vq;
180 struct vtnet_mac_filter *vtnet_mac_filter;
181 uint32_t *vtnet_vlan_filter;
182
183 uint64_t vtnet_negotiated_features;
184 struct vtnet_statistics vtnet_stats;
185 struct callout vtnet_tick_ch;
186 struct task vtnet_announce_task;
187 struct ifmedia vtnet_media;
188 eventhandler_tag vtnet_vlan_attach;
189 eventhandler_tag vtnet_vlan_detach;
190
191 struct mtx vtnet_mtx;
192 char vtnet_mtx_name[16];
193 uint8_t vtnet_hwaddr[ETHER_ADDR_LEN];
194
195 bus_dma_tag_t vtnet_rx_dmat;
196 struct mtx vtnet_rx_mtx;
197
198 bus_dma_tag_t vtnet_tx_dmat;
199 struct mtx vtnet_tx_mtx;
200
201 bus_dma_tag_t vtnet_hdr_dmat;
202 struct mtx vtnet_hdr_mtx;
203
204 bus_dma_tag_t vtnet_ack_dmat;
205 struct mtx vtnet_ack_mtx;
206 };
207 /* vtnet flag descriptions for use with printf(9) %b identifier. */
208 #define VTNET_FLAGS_BITS \
209 "\20\1MODERN\2MAC\3CTRL_VQ\4CTRL_RX\5CTRL_MAC\6VLAN_FILTER\7TSO_ECN" \
210 "\10MRG_RXBUFS\11LRO_NOMRG\12MQ\13INDIRECT\14EVENT_IDX\15SUSPENDED" \
211 "\16FIXUP_NEEDS_CSUM\17SW_LRO"
212
213 static bool
vtnet_modern(struct vtnet_softc * sc)214 vtnet_modern(struct vtnet_softc *sc)
215 {
216 return ((sc->vtnet_flags & VTNET_FLAG_MODERN) != 0);
217 }
218
219 static bool
vtnet_software_lro(struct vtnet_softc * sc)220 vtnet_software_lro(struct vtnet_softc *sc)
221 {
222 return ((sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0);
223 }
224
225 /*
226 * Maximum number of queue pairs we will autoconfigure to.
227 */
228 #define VTNET_MAX_QUEUE_PAIRS 32
229
230 /*
231 * Additional completed entries can appear in a virtqueue before we can
232 * reenable interrupts. Number of times to retry before scheduling the
233 * taskqueue to process the completed entries.
234 */
235 #define VTNET_INTR_DISABLE_RETRIES 4
236
237 /*
238 * Similarly, additional completed entries can appear in a virtqueue
239 * between when lasted checked and before notifying the host. Number
240 * of times to retry before scheduling the taskqueue to process the
241 * queue.
242 */
243 #define VTNET_NOTIFY_RETRIES 4
244
245 /*
246 * Number of words to allocate for the VLAN shadow table. There is one
247 * bit for each VLAN.
248 */
249 #define VTNET_VLAN_FILTER_NWORDS (4096 / 32)
250
251 /*
252 * We depend on all of the hdr structures being even, and matching the standard
253 * length. As well, we depend on two being identally sized (with the same
254 * layout).
255 */
256 CTASSERT(sizeof(struct virtio_net_hdr_v1) == 12);
257 CTASSERT(sizeof(struct virtio_net_hdr) == 10);
258 CTASSERT(sizeof(struct virtio_net_hdr_mrg_rxbuf) ==
259 sizeof(struct virtio_net_hdr_v1));
260
261 /*
262 * In legacy VirtIO when mergeable buffers are not negotiated, this structure
263 * is placed at the beginning of the mbuf data. Use 4 bytes of pad to keep
264 * both the VirtIO header and the data non-contiguous and the frame's payload
265 * 4 byte aligned. Note this padding would not be necessary if the
266 * VIRTIO_F_ANY_LAYOUT feature was negotiated (but we don't support that yet).
267 *
268 * In modern VirtIO or when mergeable buffers are negotiated, the host puts
269 * the VirtIO header in the beginning of the first mbuf's data.
270 */
271 #define VTNET_RX_HEADER_PAD 4
272 struct vtnet_rx_header {
273 struct virtio_net_hdr vrh_hdr;
274 char vrh_pad[VTNET_RX_HEADER_PAD];
275 } __packed;
276
277 /*
278 * For each outgoing frame, the vtnet_tx_header below is allocated from
279 * the vtnet_tx_header_zone.
280 */
281 struct vtnet_tx_header {
282 union {
283 struct virtio_net_hdr hdr;
284 struct virtio_net_hdr_mrg_rxbuf mhdr;
285 struct virtio_net_hdr_v1 v1hdr;
286 } vth_uhdr;
287
288 struct mbuf *vth_mbuf;
289
290 bus_dmamap_t dmap;
291
292 bus_dmamap_t hdr_dmap;
293 };
294
295 /*
296 * The VirtIO specification does not place a limit on the number of MAC
297 * addresses the guest driver may request to be filtered. In practice,
298 * the host is constrained by available resources. To simplify this driver,
299 * impose a reasonably high limit of MAC addresses we will filter before
300 * falling back to promiscuous or all-multicast modes.
301 */
302 #define VTNET_MAX_MAC_ENTRIES 128
303
304 /*
305 * The driver version of struct virtio_net_ctrl_mac but with our predefined
306 * number of MAC addresses allocated. This structure is shared with the host,
307 * so nentries field is in the correct VirtIO endianness.
308 */
309 struct vtnet_mac_table {
310 uint32_t nentries;
311 uint8_t macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
312 } __packed;
313
314 struct vtnet_mac_filter {
315 struct vtnet_mac_table vmf_unicast;
316 uint32_t vmf_pad; /* Make tables non-contiguous. */
317 struct vtnet_mac_table vmf_multicast;
318 };
319
320 /*
321 * The MAC filter table is malloc(9)'d when needed. Ensure it will
322 * always fit in one segment.
323 */
324 CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
325
326 #define VTNET_TX_TIMEOUT 5
327 #define VTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP)
328 #define VTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
329
330 #define VTNET_CSUM_ALL_OFFLOAD \
331 (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6 | CSUM_TSO)
332
333 #define VTNET_COMMON_FEATURES \
334 (VIRTIO_NET_F_MAC | \
335 VIRTIO_NET_F_STATUS | \
336 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS | \
337 VIRTIO_NET_F_MTU | \
338 VIRTIO_NET_F_CTRL_VQ | \
339 VIRTIO_NET_F_CTRL_RX | \
340 VIRTIO_NET_F_CTRL_RX_EXTRA | \
341 VIRTIO_NET_F_CTRL_MAC_ADDR | \
342 VIRTIO_NET_F_CTRL_VLAN | \
343 VIRTIO_NET_F_GUEST_ANNOUNCE | \
344 VIRTIO_NET_F_CSUM | \
345 VIRTIO_NET_F_HOST_TSO4 | \
346 VIRTIO_NET_F_HOST_TSO6 | \
347 VIRTIO_NET_F_HOST_ECN | \
348 VIRTIO_NET_F_GUEST_CSUM | \
349 VIRTIO_NET_F_GUEST_TSO4 | \
350 VIRTIO_NET_F_GUEST_TSO6 | \
351 VIRTIO_NET_F_GUEST_ECN | \
352 VIRTIO_NET_F_MRG_RXBUF | \
353 VIRTIO_NET_F_MQ | \
354 VIRTIO_NET_F_SPEED_DUPLEX | \
355 VIRTIO_RING_F_EVENT_IDX | \
356 VIRTIO_RING_F_INDIRECT_DESC)
357
358 #define VTNET_MODERN_FEATURES (VTNET_COMMON_FEATURES)
359 #define VTNET_LEGACY_FEATURES (VTNET_COMMON_FEATURES | VIRTIO_NET_F_GSO)
360
361 /*
362 * The VIRTIO_NET_F_HOST_TSO[46] features permit us to send the host
363 * frames larger than 1514 bytes.
364 */
365 #define VTNET_TSO_FEATURES (VIRTIO_NET_F_GSO | VIRTIO_NET_F_HOST_TSO4 | \
366 VIRTIO_NET_F_HOST_TSO6 | VIRTIO_NET_F_HOST_ECN)
367
368 /*
369 * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
370 * frames larger than 1514 bytes.
371 */
372 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
373 VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
374
375 /*
376 * Union of the offload-related features offered by the driver. As per spec,
377 * a device is permitted to reject an otherwise valid subset of its offered
378 * features by failing FEATURES_OK (v1.3 §2.2.2). Offloads are where this
379 * happens in practice, so feature negotiation retries without this entire
380 * group when the device rejects the first feature set.
381 *
382 * Must cover every offload-related bit in VTNET_COMMON_FEATURES.
383 */
384 #define VTNET_OFFLOAD_FEATURES \
385 (VIRTIO_NET_F_CSUM | \
386 VIRTIO_NET_F_GUEST_CSUM | \
387 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS | \
388 VTNET_TSO_FEATURES | \
389 VTNET_LRO_FEATURES)
390
391 #define VTNET_MIN_MTU 68
392 #define VTNET_MAX_MTU 65536
393 #define VTNET_MAX_RX_SIZE 65550
394
395 /*
396 * Used to preallocate the VQ indirect descriptors. Modern and mergeable
397 * buffers do not required one segment for the VirtIO header since it is
398 * placed inline at the beginning of the receive buffer.
399 */
400 #define VTNET_RX_SEGS_HDR_INLINE 1
401 #define VTNET_RX_SEGS_HDR_SEPARATE 2
402 #define VTNET_RX_SEGS_LRO_NOMRG 34
403 #define VTNET_TX_SEGS_MIN 32
404 #define VTNET_TX_SEGS_MAX 64
405
406 CTASSERT(((VTNET_RX_SEGS_LRO_NOMRG - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
407 CTASSERT(((VTNET_TX_SEGS_MAX - 1) * MCLBYTES) >= VTNET_MAX_MTU);
408
409 /*
410 * Number of slots in the Tx bufrings. This value matches most other
411 * multiqueue drivers.
412 */
413 #define VTNET_DEFAULT_BUFRING_SIZE 4096
414
415 #define VTNET_CORE_MTX(_sc) (&(_sc)->vtnet_mtx)
416 #define VTNET_CORE_LOCK(_sc) mtx_lock(VTNET_CORE_MTX((_sc)))
417 #define VTNET_CORE_UNLOCK(_sc) mtx_unlock(VTNET_CORE_MTX((_sc)))
418 #define VTNET_CORE_LOCK_DESTROY(_sc) mtx_destroy(VTNET_CORE_MTX((_sc)))
419 #define VTNET_CORE_LOCK_ASSERT(_sc) \
420 mtx_assert(VTNET_CORE_MTX((_sc)), MA_OWNED)
421 #define VTNET_CORE_LOCK_ASSERT_NOTOWNED(_sc) \
422 mtx_assert(VTNET_CORE_MTX((_sc)), MA_NOTOWNED)
423
424 #define VTNET_CORE_LOCK_INIT(_sc) do { \
425 snprintf((_sc)->vtnet_mtx_name, sizeof((_sc)->vtnet_mtx_name), \
426 "%s", device_get_nameunit((_sc)->vtnet_dev)); \
427 mtx_init(VTNET_CORE_MTX((_sc)), (_sc)->vtnet_mtx_name, \
428 "VTNET Core Lock", MTX_DEF); \
429 } while (0)
430
431 /*
432 * Values for the init_mode argument of vtnet_init_locked().
433 */
434 #define VTNET_INIT_NETMAP_ENTER 1
435 #define VTNET_INIT_NETMAP_EXIT 2
436
437 #endif /* _IF_VTNETVAR_H */
438