1 /* 2 * Copyright (c) 2013 Chris Torek <torek @ torek net> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 /* 27 * This file and its contents are supplied under the terms of the 28 * Common Development and Distribution License ("CDDL"), version 1.0. 29 * You may only use this file in accordance with the terms of version 30 * 1.0 of the CDDL. 31 * 32 * A full copy of the text of the CDDL should have accompanied this 33 * source. A copy of the CDDL is also available via the Internet at 34 * http://www.illumos.org/license/CDDL. 35 * 36 * Copyright 2015 Pluribus Networks Inc. 37 * Copyright 2019 Joyent, Inc. 38 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 39 * Copyright 2023 Oxide Computer Company 40 */ 41 42 #ifndef _VIONA_IMPL_H 43 #define _VIONA_IMPL_H 44 45 #include <sys/ddi.h> 46 #include <sys/list.h> 47 #include <sys/sunddi.h> 48 #include <sys/sunndi.h> 49 #include <sys/strsun.h> 50 #include <sys/sysmacros.h> 51 #include <sys/uio.h> 52 53 #include <sys/mac_client.h> 54 #include <sys/mac_provider.h> 55 #include <sys/mac_client_priv.h> 56 #include <sys/neti.h> 57 #include <inet/ip.h> 58 #include <inet/tcp.h> 59 60 #include <sys/vmm_drv.h> 61 #include <sys/viona_io.h> 62 63 struct viona_link; 64 typedef struct viona_link viona_link_t; 65 struct viona_desb; 66 typedef struct viona_desb viona_desb_t; 67 struct viona_net; 68 typedef struct viona_neti viona_neti_t; 69 70 enum viona_ring_state { 71 VRS_RESET = 0x0, /* just allocated or reset */ 72 VRS_SETUP = 0x1, /* addrs setup and starting worker thread */ 73 VRS_INIT = 0x2, /* worker thread started & waiting to run */ 74 VRS_RUN = 0x3, /* running work routine */ 75 VRS_STOP = 0x4, /* worker is exiting */ 76 }; 77 enum viona_ring_state_flags { 78 VRSF_REQ_START = 0x1, /* start running from INIT state */ 79 VRSF_REQ_STOP = 0x2, /* stop running, clean up, goto RESET state */ 80 VRSF_REQ_PAUSE = 0x4, /* stop running, goto INIT state */ 81 VRSF_RENEW = 0x8, /* ring renewing lease */ 82 }; 83 84 typedef struct viona_vring { 85 viona_link_t *vr_link; 86 87 kmutex_t vr_lock; 88 kcondvar_t vr_cv; 89 uint16_t vr_state; 90 uint16_t vr_state_flags; 91 uint_t vr_xfer_outstanding; 92 kthread_t *vr_worker_thread; 93 vmm_lease_t *vr_lease; 94 95 /* ring-sized resources for TX activity */ 96 viona_desb_t *vr_txdesb; 97 struct iovec *vr_txiov; 98 99 uint_t vr_intr_enabled; 100 uint64_t vr_msi_addr; 101 uint64_t vr_msi_msg; 102 103 /* Internal ring-related state */ 104 kmutex_t vr_a_mutex; /* sync consumers of 'avail' */ 105 kmutex_t vr_u_mutex; /* sync consumers of 'used' */ 106 uint64_t vr_pa; 107 uint16_t vr_size; 108 uint16_t vr_mask; /* cached from vr_size */ 109 uint16_t vr_cur_aidx; /* trails behind 'avail_idx' */ 110 uint16_t vr_cur_uidx; /* drives 'used_idx' */ 111 112 /* Reference to guest pages holding virtqueue */ 113 void **vr_map_pages; 114 vmm_page_t *vr_map_hold; 115 116 /* Per-ring error condition statistics */ 117 struct viona_ring_stats { 118 uint64_t rs_ndesc_too_high; 119 uint64_t rs_bad_idx; 120 uint64_t rs_indir_bad_len; 121 uint64_t rs_indir_bad_nest; 122 uint64_t rs_indir_bad_next; 123 uint64_t rs_no_space; 124 uint64_t rs_too_many_desc; 125 uint64_t rs_desc_bad_len; 126 127 uint64_t rs_bad_ring_addr; 128 129 uint64_t rs_fail_hcksum; 130 uint64_t rs_fail_hcksum6; 131 uint64_t rs_fail_hcksum_proto; 132 133 uint64_t rs_bad_rx_frame; 134 uint64_t rs_rx_merge_overrun; 135 uint64_t rs_rx_merge_underrun; 136 uint64_t rs_rx_pad_short; 137 uint64_t rs_rx_mcast_check; 138 uint64_t rs_too_short; 139 uint64_t rs_tx_absent; 140 141 uint64_t rs_rx_hookdrop; 142 uint64_t rs_tx_hookdrop; 143 } vr_stats; 144 } viona_vring_t; 145 146 struct viona_link { 147 vmm_hold_t *l_vm_hold; 148 boolean_t l_destroyed; 149 150 viona_vring_t l_vrings[VIONA_VQ_MAX]; 151 152 uint32_t l_features; 153 uint32_t l_features_hw; 154 uint32_t l_cap_csum; 155 156 uint16_t l_notify_ioport; 157 void *l_notify_cookie; 158 159 datalink_id_t l_linkid; 160 mac_handle_t l_mh; 161 mac_client_handle_t l_mch; 162 mac_promisc_handle_t l_mph; 163 mac_unicast_handle_t l_muh; 164 viona_promisc_t l_promisc; 165 166 pollhead_t l_pollhead; 167 168 viona_neti_t *l_neti; 169 }; 170 171 typedef struct viona_nethook { 172 net_handle_t vnh_neti; 173 hook_family_t vnh_family; 174 hook_event_t vnh_event_in; 175 hook_event_t vnh_event_out; 176 hook_event_token_t vnh_token_in; 177 hook_event_token_t vnh_token_out; 178 boolean_t vnh_hooked; 179 } viona_nethook_t; 180 181 struct viona_neti { 182 list_node_t vni_node; 183 184 netid_t vni_netid; 185 zoneid_t vni_zid; 186 187 viona_nethook_t vni_nethook; 188 189 kmutex_t vni_lock; /* Protects remaining members */ 190 kcondvar_t vni_ref_change; /* Protected by vni_lock */ 191 uint_t vni_ref; /* Protected by vni_lock */ 192 list_t vni_dev_list; /* Protected by vni_lock */ 193 }; 194 195 typedef struct used_elem { 196 uint16_t id; 197 uint32_t len; 198 } used_elem_t; 199 200 typedef struct viona_soft_state { 201 kmutex_t ss_lock; 202 viona_link_t *ss_link; 203 list_node_t ss_node; 204 } viona_soft_state_t; 205 206 #pragma pack(1) 207 struct virtio_desc { 208 uint64_t vd_addr; 209 uint32_t vd_len; 210 uint16_t vd_flags; 211 uint16_t vd_next; 212 }; 213 214 struct virtio_used { 215 uint32_t vu_idx; 216 uint32_t vu_tlen; 217 }; 218 219 struct virtio_net_mrgrxhdr { 220 uint8_t vrh_flags; 221 uint8_t vrh_gso_type; 222 uint16_t vrh_hdr_len; 223 uint16_t vrh_gso_size; 224 uint16_t vrh_csum_start; 225 uint16_t vrh_csum_offset; 226 uint16_t vrh_bufs; 227 }; 228 229 struct virtio_net_hdr { 230 uint8_t vrh_flags; 231 uint8_t vrh_gso_type; 232 uint16_t vrh_hdr_len; 233 uint16_t vrh_gso_size; 234 uint16_t vrh_csum_start; 235 uint16_t vrh_csum_offset; 236 }; 237 #pragma pack() 238 239 #define VNETHOOK_INTERESTED_IN(neti) \ 240 (neti)->vni_nethook.vnh_event_in.he_interested 241 #define VNETHOOK_INTERESTED_OUT(neti) \ 242 (neti)->vni_nethook.vnh_event_out.he_interested 243 244 245 #define VIONA_PROBE(name) DTRACE_PROBE(viona__##name) 246 #define VIONA_PROBE1(name, arg1, arg2) \ 247 DTRACE_PROBE1(viona__##name, arg1, arg2) 248 #define VIONA_PROBE2(name, arg1, arg2, arg3, arg4) \ 249 DTRACE_PROBE2(viona__##name, arg1, arg2, arg3, arg4) 250 #define VIONA_PROBE3(name, arg1, arg2, arg3, arg4, arg5, arg6) \ 251 DTRACE_PROBE3(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6) 252 #define VIONA_PROBE4(name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) \ 253 DTRACE_PROBE4(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, \ 254 arg8) 255 #define VIONA_PROBE5(name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, \ 256 arg9, arg10) \ 257 DTRACE_PROBE5(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, \ 258 arg8, arg9, arg10) 259 #define VIONA_PROBE_BAD_RING_ADDR(r, a) \ 260 VIONA_PROBE2(bad_ring_addr, viona_vring_t *, r, void *, (void *)(a)) 261 262 #define VIONA_RING_STAT_INCR(r, name) \ 263 (((r)->vr_stats.rs_ ## name)++) 264 265 266 #define VIONA_MAX_HDRS_LEN (sizeof (struct ether_vlan_header) + \ 267 IP_MAX_HDR_LENGTH + TCP_MAX_HDR_LENGTH) 268 269 #define VRING_AVAIL_F_NO_INTERRUPT 1 270 #define VRING_USED_F_NO_NOTIFY 1 271 272 #define VRING_DESC_F_NEXT (1 << 0) 273 #define VRING_DESC_F_WRITE (1 << 1) 274 #define VRING_DESC_F_INDIRECT (1 << 2) 275 276 #define VIRTIO_NET_HDR_F_NEEDS_CSUM (1 << 0) 277 #define VIRTIO_NET_HDR_F_DATA_VALID (1 << 1) 278 279 #define VIRTIO_NET_HDR_GSO_NONE 0 280 #define VIRTIO_NET_HDR_GSO_TCPV4 1 281 282 #define VIRTIO_NET_F_CSUM (1 << 0) 283 #define VIRTIO_NET_F_GUEST_CSUM (1 << 1) 284 #define VIRTIO_NET_F_MAC (1 << 5) /* host supplies MAC */ 285 #define VIRTIO_NET_F_GUEST_TSO4 (1 << 7) /* guest can accept TSO */ 286 #define VIRTIO_NET_F_HOST_TSO4 (1 << 11) /* host can accept TSO */ 287 #define VIRTIO_NET_F_MRG_RXBUF (1 << 15) /* host can merge RX bufs */ 288 #define VIRTIO_NET_F_STATUS (1 << 16) /* cfg status field present */ 289 #define VIRTIO_F_RING_NOTIFY_ON_EMPTY (1 << 24) 290 #define VIRTIO_F_RING_INDIRECT_DESC (1 << 28) 291 #define VIRTIO_F_RING_EVENT_IDX (1 << 29) 292 293 struct viona_ring_params { 294 uint64_t vrp_pa; 295 uint16_t vrp_size; 296 uint16_t vrp_avail_idx; 297 uint16_t vrp_used_idx; 298 }; 299 300 void viona_ring_alloc(viona_link_t *, viona_vring_t *); 301 void viona_ring_free(viona_vring_t *); 302 int viona_ring_get_state(viona_link_t *, uint16_t, struct viona_ring_params *); 303 int viona_ring_set_state(viona_link_t *, uint16_t, 304 const struct viona_ring_params *); 305 int viona_ring_reset(viona_vring_t *, boolean_t); 306 int viona_ring_init(viona_link_t *, uint16_t, const struct viona_ring_params *); 307 boolean_t viona_ring_lease_renew(viona_vring_t *); 308 bool vring_need_bail(const viona_vring_t *); 309 int viona_ring_pause(viona_vring_t *); 310 311 int vq_popchain(viona_vring_t *, struct iovec *, uint_t, uint16_t *, 312 vmm_page_t **); 313 void vq_pushchain(viona_vring_t *, uint32_t, uint16_t); 314 void vq_pushchain_many(viona_vring_t *, uint_t, used_elem_t *); 315 316 void viona_intr_ring(viona_vring_t *ring, boolean_t); 317 void viona_ring_set_no_notify(viona_vring_t *, boolean_t); 318 void viona_ring_disable_notify(viona_vring_t *); 319 void viona_ring_enable_notify(viona_vring_t *); 320 uint16_t viona_ring_num_avail(viona_vring_t *); 321 322 323 void viona_rx_init(void); 324 void viona_rx_fini(void); 325 int viona_rx_set(viona_link_t *, viona_promisc_t); 326 void viona_rx_clear(viona_link_t *); 327 void viona_worker_rx(viona_vring_t *, viona_link_t *); 328 329 extern kmutex_t viona_force_copy_lock; 330 void viona_worker_tx(viona_vring_t *, viona_link_t *); 331 void viona_tx_ring_alloc(viona_vring_t *, const uint16_t); 332 void viona_tx_ring_free(viona_vring_t *, const uint16_t); 333 334 void viona_neti_attach(void); 335 void viona_neti_detach(void); 336 viona_neti_t *viona_neti_lookup_by_zid(zoneid_t); 337 void viona_neti_rele(viona_neti_t *); 338 int viona_hook(viona_link_t *, viona_vring_t *, mblk_t **, boolean_t); 339 340 #endif /* _VIONA_IMPL_H */ 341