1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * This header is BSD licensed so anyone can use the definitions to implement
5 * compatible drivers/servers.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of IBM nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #ifndef _VIRTIO_NET_H
32 #define _VIRTIO_NET_H
33
34 #include "opt_inet.h"
35 #include "opt_inet6.h"
36
37 #include <sys/endian.h>
38
39 /* The feature bitmap for virtio net */
40 #define VIRTIO_NET_F_CSUM (1ULL << 0) /* Host handles pkts w/ partial csum */
41 #define VIRTIO_NET_F_GUEST_CSUM (1ULL << 1) /* Guest handles pkts w/ partial csum*/
42 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS (1ULL << 2) /* Dynamic offload configuration. */
43 #define VIRTIO_NET_F_MTU (1ULL << 3) /* Initial MTU advice */
44 #define VIRTIO_NET_F_MAC (1ULL << 5) /* Host has given MAC address. */
45 #define VIRTIO_NET_F_GSO (1ULL << 6) /* Host handles pkts w/ any GSO type */
46 #define VIRTIO_NET_F_GUEST_TSO4 (1ULL << 7) /* Guest can handle TSOv4 in. */
47 #define VIRTIO_NET_F_GUEST_TSO6 (1ULL << 8) /* Guest can handle TSOv6 in. */
48 #define VIRTIO_NET_F_GUEST_ECN (1ULL << 9) /* Guest can handle TSO[6] w/ ECN in. */
49 #define VIRTIO_NET_F_GUEST_UFO (1ULL << 10) /* Guest can handle UFO in. */
50 #define VIRTIO_NET_F_HOST_TSO4 (1ULL << 11) /* Host can handle TSOv4 in. */
51 #define VIRTIO_NET_F_HOST_TSO6 (1ULL << 12) /* Host can handle TSOv6 in. */
52 #define VIRTIO_NET_F_HOST_ECN (1ULL << 13) /* Host can handle TSO[6] w/ ECN in. */
53 #define VIRTIO_NET_F_HOST_UFO (1ULL << 14) /* Host can handle UFO in. */
54 #define VIRTIO_NET_F_MRG_RXBUF (1ULL << 15) /* Host can merge receive buffers. */
55 #define VIRTIO_NET_F_STATUS (1ULL << 16) /* virtio_net_config.status available*/
56 #define VIRTIO_NET_F_CTRL_VQ (1ULL << 17) /* Control channel available */
57 #define VIRTIO_NET_F_CTRL_RX (1ULL << 18) /* Control channel RX mode support */
58 #define VIRTIO_NET_F_CTRL_VLAN (1ULL << 19) /* Control channel VLAN filtering */
59 #define VIRTIO_NET_F_CTRL_RX_EXTRA (1ULL << 20) /* Extra RX mode control support */
60 #define VIRTIO_NET_F_GUEST_ANNOUNCE (1ULL << 21) /* Announce device on network */
61 #define VIRTIO_NET_F_MQ (1ULL << 22) /* Device supports Receive Flow Steering */
62 #define VIRTIO_NET_F_CTRL_MAC_ADDR (1ULL << 23) /* Set MAC address */
63 #define VIRTIO_NET_F_SPEED_DUPLEX (1ULL << 63) /* Device set linkspeed and duplex */
64
65 /* virtio net feature flag descriptions for use with printf(9) %b identifier. */
66 #define VIRTIO_NET_FEATURE_BITS \
67 "\20\200CSUM\201GUEST_CSUM\202CTRL_GUEST_OFFLOADS\203MTU\205MAC\206GSO" \
68 "\207GUEST_TSO4\210GUEST_TSO6\211GUEST_ECN\212GUEST_UFO\213HOST_TSO4" \
69 "\214HOST_TSO6\215HOST_ECN\216HOST_UFO\217MRG_RXBUF\220STATUS\221CTRL_VQ" \
70 "\222CTRL_RX\223CTRL_VLAN\224CTRL_RX_EXTRA\225GUEST_ANNOUNCE\226MQ" \
71 "\227CTRL_MAC_ADDR\277SPEED_DUPLEX"
72
73 #define VIRTIO_NET_S_LINK_UP 1 /* Link is up */
74 #define VIRTIO_NET_S_ANNOUNCE 2 /* Announcement is needed */
75
76 struct virtio_net_config {
77 /* The config defining mac address (if VIRTIO_NET_F_MAC) */
78 uint8_t mac[ETHER_ADDR_LEN];
79 /* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
80 uint16_t status;
81 /* Maximum number of each of transmit and receive queues;
82 * see VIRTIO_NET_F_MQ and VIRTIO_NET_CTRL_MQ.
83 * Legal values are between 1 and 0x8000.
84 */
85 uint16_t max_virtqueue_pairs;
86 /* Default maximum transmit unit advice */
87 uint16_t mtu;
88 /*
89 * speed, in units of 1Mb. All values 0 to INT_MAX are legal.
90 * Any other value stands for unknown.
91 */
92 uint32_t speed;
93 /*
94 * 0x00 - half duplex
95 * 0x01 - full duplex
96 * Any other value stands for unknown.
97 */
98 uint8_t duplex;
99 } __packed;
100
101 /*
102 * This header comes first in the scatter-gather list. If you don't
103 * specify GSO or CSUM features, you can simply ignore the header.
104 *
105 * This is bitwise-equivalent to the legacy struct virtio_net_hdr_mrg_rxbuf,
106 * only flattened.
107 */
108 struct virtio_net_hdr_v1 {
109 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */
110 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
111 uint8_t flags;
112 #define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */
113 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */
114 #define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */
115 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */
116 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */
117 uint8_t gso_type;
118 uint16_t hdr_len; /* Ethernet + IP + tcp/udp hdrs */
119 uint16_t gso_size; /* Bytes to append to hdr_len per frame */
120 uint16_t csum_start; /* Position to start checksumming from */
121 uint16_t csum_offset; /* Offset after that to place checksum */
122 uint16_t num_buffers; /* Number of merged rx buffers */
123 };
124
125 /*
126 * This header comes first in the scatter-gather list.
127 * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, it must
128 * be the first element of the scatter-gather list. If you don't
129 * specify GSO or CSUM features, you can simply ignore the header.
130 */
131 struct virtio_net_hdr {
132 /* See VIRTIO_NET_HDR_F_* */
133 uint8_t flags;
134 /* See VIRTIO_NET_HDR_GSO_* */
135 uint8_t gso_type;
136 uint16_t hdr_len; /* Ethernet + IP + tcp/udp hdrs */
137 uint16_t gso_size; /* Bytes to append to hdr_len per frame */
138 uint16_t csum_start; /* Position to start checksumming from */
139 uint16_t csum_offset; /* Offset after that to place checksum */
140 };
141
142 /*
143 * This is the version of the header to use when the MRG_RXBUF
144 * feature has been negotiated.
145 */
146 struct virtio_net_hdr_mrg_rxbuf {
147 struct virtio_net_hdr hdr;
148 uint16_t num_buffers; /* Number of merged rx buffers */
149 };
150
151 /*
152 * Control virtqueue data structures
153 *
154 * The control virtqueue expects a header in the first sg entry
155 * and an ack/status response in the last entry. Data for the
156 * command goes in between.
157 */
158 struct virtio_net_ctrl_hdr {
159 uint8_t class;
160 uint8_t cmd;
161 } __packed;
162
163 #define VIRTIO_NET_OK 0
164 #define VIRTIO_NET_ERR 1
165
166 /*
167 * Control the RX mode, ie. promiscuous, allmulti, etc...
168 * All commands require an "out" sg entry containing a 1 byte
169 * state value, zero = disable, non-zero = enable. Commands
170 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
171 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
172 */
173 #define VIRTIO_NET_CTRL_RX 0
174 #define VIRTIO_NET_CTRL_RX_PROMISC 0
175 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1
176 #define VIRTIO_NET_CTRL_RX_ALLUNI 2
177 #define VIRTIO_NET_CTRL_RX_NOMULTI 3
178 #define VIRTIO_NET_CTRL_RX_NOUNI 4
179 #define VIRTIO_NET_CTRL_RX_NOBCAST 5
180
181 /*
182 * Control the MAC filter table.
183 *
184 * The MAC filter table is managed by the hypervisor, the guest should
185 * assume the size is infinite. Filtering should be considered
186 * non-perfect, ie. based on hypervisor resources, the guest may
187 * received packets from sources not specified in the filter list.
188 *
189 * In addition to the class/cmd header, the TABLE_SET command requires
190 * two out scatterlists. Each contains a 4 byte count of entries followed
191 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The
192 * first sg list contains unicast addresses, the second is for multicast.
193 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
194 * is available.
195 *
196 * The ADDR_SET command requests one out scatterlist, it contains a
197 * 6 bytes MAC address. This functionality is present if the
198 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
199 */
200 struct virtio_net_ctrl_mac {
201 uint32_t entries;
202 uint8_t macs[][ETHER_ADDR_LEN];
203 } __packed;
204
205 #define VIRTIO_NET_CTRL_MAC 1
206 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
207 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
208
209 /*
210 * Control VLAN filtering
211 *
212 * The VLAN filter table is controlled via a simple ADD/DEL interface.
213 * VLAN IDs not added may be filtered by the hypervisor. Del is the
214 * opposite of add. Both commands expect an out entry containing a 2
215 * byte VLAN ID. VLAN filtering is available with the
216 * VIRTIO_NET_F_CTRL_VLAN feature bit.
217 */
218 #define VIRTIO_NET_CTRL_VLAN 2
219 #define VIRTIO_NET_CTRL_VLAN_ADD 0
220 #define VIRTIO_NET_CTRL_VLAN_DEL 1
221
222 /*
223 * Control link announce acknowledgement
224 *
225 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that
226 * driver has received the notification; device would clear the
227 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives
228 * this command.
229 */
230 #define VIRTIO_NET_CTRL_ANNOUNCE 3
231 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0
232
233 /*
234 * Control Receive Flow Steering
235 *
236 * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET enables Receive Flow
237 * Steering, specifying the number of the transmit and receive queues
238 * that will be used. After the command is consumed and acked by the
239 * device, the device will not steer new packets on receive virtqueues
240 * other than specified nor read from transmit virtqueues other than
241 * specified. Accordingly, driver should not transmit new packets on
242 * virtqueues other than specified.
243 */
244 struct virtio_net_ctrl_mq {
245 uint16_t virtqueue_pairs;
246 } __packed;
247
248 #define VIRTIO_NET_CTRL_MQ 4
249 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
250 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
251 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
252
253 /*
254 * Control network offloads
255 *
256 * Reconfigures the network offloads that Guest can handle.
257 *
258 * Available with the VIRTIO_NET_F_CTRL_GUEST_OFFLOADS feature bit.
259 *
260 * Command data format matches the feature bit mask exactly.
261 *
262 * See VIRTIO_NET_F_GUEST_* for the list of offloads
263 * that can be enabled/disabled.
264 */
265 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5
266 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0
267
268 #if defined(INET) || defined(INET6)
269 static inline void
virtio_net_rx_csum_needs_csum(struct mbuf * m,bool isipv6,int protocol,struct virtio_net_hdr * hdr)270 virtio_net_rx_csum_needs_csum(struct mbuf *m, bool isipv6, int protocol,
271 struct virtio_net_hdr *hdr)
272 {
273 /*
274 * The packet is likely from another VM on the same host or from the
275 * host that itself performed checksum offloading so Tx/Rx is basically
276 * a memcpy and the checksum has little value so far.
277 */
278
279 KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP,
280 ("%s: unsupported IP protocol %d", __func__, protocol));
281
282 /*
283 * Just forward the order to compute the checksum by setting
284 * the corresponding mbuf flag (e.g., CSUM_TCP).
285 */
286 switch (protocol) {
287 case IPPROTO_TCP:
288 m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP);
289 break;
290 case IPPROTO_UDP:
291 m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP);
292 break;
293 }
294 m->m_pkthdr.csum_data = hdr->csum_offset;
295 }
296
297 static inline void
virtio_net_rx_csum_data_valid(struct mbuf * m,int protocol)298 virtio_net_rx_csum_data_valid(struct mbuf *m, int protocol)
299 {
300 KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP,
301 ("%s: unsupported IP protocol %d", __func__, protocol));
302
303 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
304 m->m_pkthdr.csum_data = 0xFFFF;
305 }
306
307 #define VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO 1
308 #define VIRTIO_NET_RX_CSUM_BAD_ETHTYPE 2
309 #define VIRTIO_NET_RX_CSUM_BAD_IPPROTO 3
310
311 /*
312 * For a packet received over the VirtIO channel, it checks the given
313 * VirtIO header and sets the appropriate CSUM_* flags in the given mbuf.
314 *
315 * Unfortunately, the information provided is not directly useful to us. The
316 * VirtIO header gives the offset of the checksum, which is all Linux needs, but
317 * this is not how FreeBSD does things. We are forced to peek inside the packet
318 * a bit.
319 *
320 * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
321 * could accept the offsets and let the stack figure it out.
322 *
323 * @param m mbuf of the packet where CSUM_* flags might need to be set.
324 * @param hdr VirtIO header of the received packet that needs to be checked
325 * with its field values stored in the byte order this machine
326 * uses (i.e., readable without a byte swap).
327 *
328 * @return 0 on success, or one of the VIRTIO_NET_RX_CSUM_* error codes.
329 */
330 static inline int
virtio_net_rx_csum(struct mbuf * m,struct virtio_net_hdr * hdr)331 virtio_net_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr)
332 {
333 const struct ether_header *eh;
334 int hoff, protocol;
335 uint16_t etype;
336 bool isipv6;
337
338 KASSERT(hdr->flags &
339 (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID),
340 ("%s: missing checksum offloading flag %x", __func__, hdr->flags));
341
342 eh = mtod(m, const struct ether_header *);
343 etype = ntohs(eh->ether_type);
344 if (etype == ETHERTYPE_VLAN) {
345 /* TODO BMV: Handle QinQ. */
346 const struct ether_vlan_header *evh =
347 mtod(m, const struct ether_vlan_header *);
348 etype = ntohs(evh->evl_proto);
349 hoff = sizeof(struct ether_vlan_header);
350 } else
351 hoff = sizeof(struct ether_header);
352
353 /* Check whether ethernet type is IP or IPv6, and get protocol. */
354 switch (etype) {
355 #if defined(INET)
356 case ETHERTYPE_IP:
357 if (__predict_false(m->m_len < hoff + sizeof(struct ip))) {
358 return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO);
359 } else {
360 struct ip *ip = (struct ip *)(m->m_data + hoff);
361 protocol = ip->ip_p;
362 }
363 isipv6 = false;
364 break;
365 #endif
366 #if defined(INET6)
367 case ETHERTYPE_IPV6:
368 if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr))
369 || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) {
370 return (VIRTIO_NET_RX_CSUM_INACCESSIBLE_IPPROTO);
371 }
372 isipv6 = true;
373 break;
374 #endif
375 default:
376 return (VIRTIO_NET_RX_CSUM_BAD_ETHTYPE);
377 }
378
379 /* Check whether protocol is TCP or UDP. */
380 switch (protocol) {
381 case IPPROTO_TCP:
382 case IPPROTO_UDP:
383 break;
384 default:
385 /*
386 * FreeBSD does not support checksum offloading of this
387 * protocol here.
388 */
389 return (VIRTIO_NET_RX_CSUM_BAD_IPPROTO);
390 }
391
392 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
393 virtio_net_rx_csum_needs_csum(m, isipv6, protocol, hdr);
394 else /* VIRTIO_NET_HDR_F_DATA_VALID */
395 virtio_net_rx_csum_data_valid(m, protocol);
396
397 return (0);
398 }
399
400 #define VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE 1
401 #define VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH 2
402 #define VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP 3
403 #define VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM 4
404 #define VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED 5
405
406 #define VIRTIO_NET_TX_MODERN_LE(modern, val) (modern ? htole16(val) : val)
407
408 /*
409 * BMV: This can go away once we finally have offsets in the mbuf header.
410 */
411 static inline int
virtio_net_tx_offload_ctx(struct mbuf * m,int * etype,int * proto,int * start)412 virtio_net_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
413 {
414 struct ether_vlan_header *evh;
415 #if defined(INET) || defined(INET6)
416 int offset;
417 #endif
418
419 evh = mtod(m, struct ether_vlan_header *);
420 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
421 /* BMV: We should handle nested VLAN tags too. */
422 *etype = ntohs(evh->evl_proto);
423 #if defined(INET) || defined(INET6)
424 offset = sizeof(struct ether_vlan_header);
425 #endif
426 } else {
427 *etype = ntohs(evh->evl_encap_proto);
428 #if defined(INET) || defined(INET6)
429 offset = sizeof(struct ether_header);
430 #endif
431 }
432
433 switch (*etype) {
434 #if defined(INET)
435 case ETHERTYPE_IP: {
436 struct ip *ip, iphdr;
437 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
438 m_copydata(m, offset, sizeof(struct ip),
439 (caddr_t) &iphdr);
440 ip = &iphdr;
441 } else
442 ip = (struct ip *)(m->m_data + offset);
443 *proto = ip->ip_p;
444 *start = offset + (ip->ip_hl << 2);
445 break;
446 }
447 #endif
448 #if defined(INET6)
449 case ETHERTYPE_IPV6:
450 *proto = -1;
451 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
452 /* Assert the network stack sent us a valid packet. */
453 KASSERT(*start > offset,
454 ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
455 *start, offset, *proto));
456 break;
457 #endif
458 default:
459 return (VIRTIO_NET_TX_OFFLOAD_UNKNOWN_ETHTYPE);
460 }
461
462 return (0);
463 }
464
465 static inline int
virtio_net_tx_offload_tso(struct ifnet * ifp,struct mbuf * m,int eth_type,int offset,struct virtio_net_hdr * hdr,bool tso_ecn,bool modern)466 virtio_net_tx_offload_tso(struct ifnet *ifp, struct mbuf *m, int eth_type,
467 int offset, struct virtio_net_hdr *hdr, bool tso_ecn, bool modern)
468 {
469 static struct timeval lastecn;
470 static int curecn;
471 struct tcphdr *tcp, tcphdr;
472
473 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
474 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
475 tcp = &tcphdr;
476 } else
477 tcp = (struct tcphdr *)(m->m_data + offset);
478
479 /*
480 * Set VirtIO header fields with the correct byte order.
481 * In modern mode, this is little endian (LE).
482 * In legacy mode, this is the endianness of the guest, which means a
483 * FreeBSD guest can use its native endianness and a host must use the
484 * guests endianness. However, since a FreeBSD host with bhyve runs only
485 * on LE systems and supports only LE guests, no conversion is required.
486 */
487 hdr->hdr_len = VIRTIO_NET_TX_MODERN_LE(modern,
488 offset + (tcp->th_off << 2));
489 hdr->gso_size = VIRTIO_NET_TX_MODERN_LE(modern, m->m_pkthdr.tso_segsz);
490 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
491 VIRTIO_NET_HDR_GSO_TCPV6;
492
493 if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) {
494 /*
495 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In
496 * FreeBSD, ECN support is not on a per-interface basis,
497 * but globally via the net.inet.tcp.ecn.enable sysctl
498 * knob. The default is off.
499 */
500 if (!tso_ecn) {
501 if (ppsratecheck(&lastecn, &curecn, 1))
502 if_printf(ifp,
503 "TSO with ECN not negotiated with host\n");
504 return (VIRTIO_NET_TX_OFFLOAD_TSO_ECN_UNEXPECTED);
505 }
506 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
507 }
508
509 return (0);
510 }
511
512 /*
513 * For a packet to be transmitted over the VirtIO channel, it checks the
514 * CSUM_* flags in the mbuf and sets the appropriate flags in the VirtIO header.
515 * In case of an error, it frees the mbuf and sets the pointer referenced by mp
516 * to NULL.
517 *
518 * @param ifp ifnet struct of outgoing interface.
519 * @param mp mbuf on which the CSUM_* flags needs to be checked.
520 * @param hdr VirtIO header to be filled for the outgoing packet.
521 * @param tso_ecn true if ECN has been negotiated between host and guest.
522 * @param modern true if VirtIO modern mode is used.
523 *
524 * @return 0 on success, or one of the VIRTIO_NET_TX_OFFLOAD_* error codes.
525 */
526 static inline int
virtio_net_tx_offload(struct ifnet * ifp,struct mbuf ** mp,struct virtio_net_hdr * hdr,bool tso_ecn,bool modern)527 virtio_net_tx_offload(struct ifnet *ifp, struct mbuf **mp,
528 struct virtio_net_hdr *hdr, bool tso_ecn, bool modern)
529 {
530 int flags, etype, csum_start, proto, error;
531 struct mbuf *m;
532
533 m = *mp;
534 flags = m->m_pkthdr.csum_flags;
535
536 error = virtio_net_tx_offload_ctx(m, &etype, &proto, &csum_start);
537 if (error != 0)
538 goto drop;
539
540 if (flags & (CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) {
541 /* Sanity check the parsed mbuf matches the offload flags. */
542 if (__predict_false((flags & (CSUM_TCP | CSUM_UDP) &&
543 etype != ETHERTYPE_IP) ||
544 (flags & (CSUM_TCP_IPV6 | CSUM_UDP_IPV6) &&
545 etype != ETHERTYPE_IPV6))) {
546 error = VIRTIO_NET_TX_OFFLOAD_PROTO_MISMATCH;
547 goto drop;
548 }
549
550 /*
551 * Set VirtIO header fields with the correct byte order.
552 * See comment in virtio_net_tx_offload_tso()
553 */
554 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
555 hdr->csum_start = VIRTIO_NET_TX_MODERN_LE(modern, csum_start);
556 hdr->csum_offset = VIRTIO_NET_TX_MODERN_LE(modern,
557 m->m_pkthdr.csum_data);
558 }
559
560 if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) {
561 /*
562 * Sanity check the parsed mbuf IP protocol is TCP, and
563 * VirtIO TSO reqires the checksum offloading above.
564 */
565 if (__predict_false(proto != IPPROTO_TCP)) {
566 error = VIRTIO_NET_TX_OFFLOAD_TSO_NOT_TCP;
567 goto drop;
568 } else if (__predict_false((hdr->flags &
569 VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
570 error = VIRTIO_NET_TX_OFFLOAD_TSO_WITHOUT_CSUM;
571 goto drop;
572 }
573
574 error = virtio_net_tx_offload_tso(ifp, m, etype, csum_start,
575 hdr, tso_ecn, modern);
576 if (error != 0)
577 goto drop;
578 }
579
580 return (error);
581
582 drop:
583 m_freem(m);
584 *mp = NULL;
585 return (error);
586 }
587 #endif /* defined(INET) || defined(INET6) */
588
589 #endif /* _VIRTIO_NET_H */
590