1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019 Joyent, Inc. 14 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 15 * Copyright 2026 Oxide Computer Company 16 */ 17 18 #ifndef _VIRTIO_IMPL_H 19 #define _VIRTIO_IMPL_H 20 21 /* 22 * VIRTIO FRAMEWORK: FRAMEWORK-PRIVATE DEFINITIONS 23 * 24 * For design and usage documentation, see the comments in "virtio.h". 25 * 26 * NOTE: Client drivers should not use definitions from this file. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/dditypes.h> 31 #include <sys/list.h> 32 #include <sys/ccompile.h> 33 #include <sys/stdbool.h> 34 35 #include "virtio.h" 36 #include "virtio_spec.h" 37 38 #ifdef __cplusplus 39 extern "C" { 40 #endif 41 42 extern ddi_device_acc_attr_t virtio_acc_attr; 43 extern ddi_dma_attr_t virtio_dma_attr; 44 45 extern void virtio_acquireq(virtio_t *, uint16_t); 46 extern void virtio_releaseq(virtio_t *); 47 48 int virtio_dma_init(virtio_t *, virtio_dma_t *, size_t, const ddi_dma_attr_t *, 49 int, int); 50 void virtio_dma_fini(virtio_dma_t *); 51 52 typedef enum virtio_dma_level { 53 VIRTIO_DMALEVEL_HANDLE_ALLOC = (1ULL << 0), 54 VIRTIO_DMALEVEL_MEMORY_ALLOC = (1ULL << 1), 55 VIRTIO_DMALEVEL_HANDLE_BOUND = (1ULL << 2), 56 VIRTIO_DMALEVEL_COOKIE_ARRAY = (1ULL << 3), 57 } virtio_dma_level_t; 58 59 struct virtio_dma { 60 virtio_dma_level_t vidma_level; 61 virtio_t *vidma_virtio; 62 caddr_t vidma_va; 63 size_t vidma_size; 64 size_t vidma_real_size; 65 ddi_dma_handle_t vidma_dma_handle; 66 ddi_acc_handle_t vidma_acc_handle; 67 uint_t vidma_dma_ncookies; 68 ddi_dma_cookie_t *vidma_dma_cookies; 69 }; 70 71 typedef enum virtio_initlevel { 72 VIRTIO_INITLEVEL_REGS = (1ULL << 0), 73 VIRTIO_INITLEVEL_PROVIDER = (1ULL << 1), 74 VIRTIO_INITLEVEL_INT_ALLOC = (1ULL << 2), 75 VIRTIO_INITLEVEL_INT_ADDED = (1ULL << 3), 76 VIRTIO_INITLEVEL_INT_ENABLED = (1ULL << 4), 77 VIRTIO_INITLEVEL_SHUTDOWN = (1ULL << 5), 78 } virtio_initlevel_t; 79 80 typedef struct virtio_pci_cap { 81 virtio_pci_cap_type_t vpc_type; 82 uint8_t vpc_baridx; 83 uint64_t vpc_offset; 84 uint64_t vpc_size; 85 86 ddi_acc_handle_t vpc_barh; 87 caddr_t vpc_bar; 88 } virtio_pci_cap_t; 89 90 typedef enum virtio_mode { 91 VIRTIO_MODE_LEGACY = 1, /* A pure "legacy" device */ 92 VIRTIO_MODE_TRANSITIONAL = 2, /* A "transitional" device */ 93 VIRTIO_MODE_MODERN = 3, /* A pure "modern" device */ 94 } virtio_mode_t; 95 96 typedef struct virtio_ops { 97 uint64_t (*vop_device_get_features)(virtio_t *); 98 bool (*vop_device_set_features)(virtio_t *, uint64_t); 99 void (*vop_set_status_locked)(virtio_t *, uint8_t); 100 uint8_t (*vop_get_status)(virtio_t *); 101 void (*vop_device_reset_locked)(virtio_t *); 102 uint8_t (*vop_isr_status)(virtio_t *); 103 void (*vop_msix_config_set)(virtio_t *, uint16_t); 104 uint16_t (*vop_msix_config_get)(virtio_t *); 105 void (*vop_queue_notify)(virtio_queue_t *); 106 107 void (*vop_queue_select)(virtio_t *, uint16_t); 108 uint16_t (*vop_queue_size_get)(virtio_t *, uint16_t); 109 void (*vop_queue_size_set)(virtio_t *, uint16_t, uint16_t); 110 uint64_t (*vop_queue_noff_get)(virtio_t *, uint16_t); 111 bool (*vop_queue_enable_get)(virtio_t *, uint16_t); 112 void (*vop_queue_enable_set)(virtio_t *, uint16_t, bool); 113 void (*vop_queue_addr_set)(virtio_t *, uint16_t, uint64_t, 114 uint64_t, uint64_t); 115 void (*vop_msix_queue_set)(virtio_t *, uint16_t, uint16_t); 116 uint16_t (*vop_msix_queue_get)(virtio_t *, uint16_t); 117 118 uint8_t (*vop_device_cfg_gen)(virtio_t *); 119 uint8_t (*vop_device_cfg_get8)(virtio_t *, uintptr_t); 120 uint16_t (*vop_device_cfg_get16)(virtio_t *, uintptr_t); 121 uint32_t (*vop_device_cfg_get32)(virtio_t *, uintptr_t); 122 uint64_t (*vop_device_cfg_get64)(virtio_t *, uintptr_t); 123 void (*vop_device_cfg_put8)(virtio_t *, uintptr_t, uint8_t); 124 void (*vop_device_cfg_put16)(virtio_t *, uintptr_t, 125 uint16_t); 126 void (*vop_device_cfg_put32)(virtio_t *, uintptr_t, 127 uint32_t); 128 } virtio_ops_t; 129 130 extern virtio_ops_t virtio_legacy_ops, virtio_modern_ops; 131 132 struct virtio { 133 dev_info_t *vio_dip; 134 135 kmutex_t vio_mutex; 136 137 virtio_initlevel_t vio_initlevel; 138 139 virtio_mode_t vio_mode; 140 virtio_ops_t *vio_ops; 141 142 list_t vio_queues; 143 kmutex_t vio_qlock; 144 uint16_t vio_qcur; 145 146 virtio_pci_cap_t vio_cap_common; 147 virtio_pci_cap_t vio_cap_notify; 148 virtio_pci_cap_t vio_cap_isr; 149 virtio_pci_cap_t vio_cap_device; 150 151 /* Notification multiplier used with the modern interface */ 152 uint32_t vio_multiplier; 153 154 ddi_acc_handle_t vio_barh; 155 caddr_t vio_bar; 156 uint_t vio_legacy_cfg_offset; 157 158 uint64_t vio_features; 159 uint64_t vio_features_device; 160 161 ddi_intr_handle_t *vio_interrupts; 162 int vio_ninterrupts; 163 int vio_interrupt_type; 164 int vio_interrupt_cap; 165 uint_t vio_interrupt_priority; 166 167 ddi_intr_handler_t *vio_cfgchange_handler; 168 void *vio_cfgchange_handlerarg; 169 boolean_t vio_cfgchange_handler_added; 170 uint_t vio_cfgchange_handler_index; 171 }; 172 173 struct virtio_queue { 174 virtio_t *viq_virtio; 175 kmutex_t viq_mutex; 176 const char *viq_name; 177 list_node_t viq_link; 178 179 boolean_t viq_shutdown; 180 boolean_t viq_indirect; 181 uint_t viq_max_segs; 182 183 /* 184 * Set when VIRTIO_F_RING_EVENT_IDX was negotiated for the device, in 185 * which case interrupt suppression and device notification decisions 186 * use the "used_event" and "avail_event" ring entries rather than the 187 * suppression flags. 188 */ 189 boolean_t viq_event_idx; 190 191 /* 192 * The most recent interrupt disposition requested by the client 193 * driver, protected by "viq_mutex". With the event index mechanism 194 * this is needed when returned chains are collected, to decide 195 * whether to request an event for the next chain. 196 */ 197 boolean_t viq_no_interrupt; 198 199 /* 200 * Each Virtio device type has some set of queues for data transfer to 201 * and from the host. This index is described in the specification for 202 * the particular device and queue type, and written to QUEUE_SELECT to 203 * allow interaction with the queue. For example, a network device has 204 * at least a receive queue with index 0, and a transmit queue with 205 * index 1. 206 */ 207 uint16_t viq_index; 208 209 /* 210 * Modern devices use a BAR region for notifications with each queue 211 * potentially having its own offset within that region. We store the 212 * offset for this queue here. 213 */ 214 uint64_t viq_noff; 215 216 /* 217 * For legacy Virtio devices, the size and shape of the queue is 218 * determined entirely by the number of queue entries. 219 */ 220 uint16_t viq_size; 221 id_space_t *viq_descmap; 222 223 /* 224 * The memory shared between the device and the driver is allocated as 225 * a large phyisically contiguous chunk. Access to this area is 226 * through three pointers to packed structures. 227 */ 228 virtio_dma_t viq_dma; 229 virtio_vq_desc_t *viq_dma_descs; 230 virtio_vq_driver_t *viq_dma_driver; 231 virtio_vq_device_t *viq_dma_device; 232 233 uint16_t viq_device_index; 234 uint16_t viq_driver_index; 235 236 /* 237 * Interrupt handler function, or NULL if not provided. 238 */ 239 ddi_intr_handler_t *viq_func; 240 void *viq_funcarg; 241 boolean_t viq_handler_added; 242 uint_t viq_handler_index; 243 244 /* 245 * MSI-X vector sharing. A queue that has been directed to share the 246 * vector assigned to another queue points at that queue through 247 * "viq_intr_owner", and is linked onto the owning queue's 248 * "viq_intr_sharers" list. The list is walked without locks from the 249 * owning vector's interrupt handler. It can only change during the 250 * initialisation phase, before any handler has been added, and 251 * during teardown, after every handler has been removed. 252 */ 253 virtio_queue_t *viq_intr_owner; 254 list_t viq_intr_sharers; 255 list_node_t viq_intr_sharers_link; 256 257 /* 258 * When a chain is submitted to the queue, it is also stored in this 259 * AVL tree keyed by the index of the first descriptor in the chain. 260 */ 261 avl_tree_t viq_inflight; 262 }; 263 264 struct virtio_chain { 265 virtio_queue_t *vic_vq; 266 avl_node_t vic_node; 267 268 void *vic_data; 269 270 uint16_t vic_head; 271 uint32_t vic_received_length; 272 273 virtio_dma_t vic_indirect_dma; 274 uint_t vic_indirect_capacity; 275 uint_t vic_indirect_used; 276 277 uint_t vic_direct_capacity; 278 uint_t vic_direct_used; 279 uint16_t vic_direct[]; 280 }; 281 282 /* 283 * When laying out queues for use over the modern interface we choose to align 284 * all queue components using the most restrictive alignment requirement, that 285 * of the descriptor part of the ring. 286 */ 287 #define MODERN_VQ_ALIGN MODERN_VQ_ALIGN_DESC 288 289 /* 290 * DMA SYNCHRONISATION WRAPPERS 291 */ 292 293 /* 294 * The offset of the device-owned portion within the queue DMA allocation, 295 * which also serves as the length of the driver-owned portion that precedes 296 * it. 297 */ 298 #define VIRTQ_DMA_DEVICE_OFFSET(viq) \ 299 ((uintptr_t)(viq)->viq_dma_device - \ 300 (uintptr_t)(viq)->viq_dma_descs) 301 302 /* 303 * Synchronise the driver-owned portion of the queue so that the device can see 304 * our writes. This covers the memory accessed via the "viq_dma_descs" and 305 * "viq_dma_driver" members. 306 */ 307 #define VIRTQ_DMA_SYNC_FORDEV(viq) VERIFY0(ddi_dma_sync( \ 308 (viq)->viq_dma.vidma_dma_handle, \ 309 0, \ 310 VIRTQ_DMA_DEVICE_OFFSET(viq), \ 311 DDI_DMA_SYNC_FORDEV)) 312 313 /* 314 * Synchronise the device-owned portion of the queue so that we can see any 315 * writes from the device. This covers the memory accessed via the 316 * "viq_dma_device" member. 317 */ 318 #define VIRTQ_DMA_SYNC_FORKERNEL(viq) VERIFY0(ddi_dma_sync( \ 319 (viq)->viq_dma.vidma_dma_handle, \ 320 VIRTQ_DMA_DEVICE_OFFSET(viq), \ 321 (viq)->viq_dma.vidma_size - \ 322 VIRTQ_DMA_DEVICE_OFFSET(viq), \ 323 DDI_DMA_SYNC_FORKERNEL)) 324 325 #ifdef __cplusplus 326 } 327 #endif 328 329 #endif /* _VIRTIO_IMPL_H */ 330