1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019 Joyent, Inc. 14 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 15 * Copyright 2025 Oxide Computer Company 16 */ 17 18 #ifndef _VIRTIO_IMPL_H 19 #define _VIRTIO_IMPL_H 20 21 /* 22 * VIRTIO FRAMEWORK: FRAMEWORK-PRIVATE DEFINITIONS 23 * 24 * For design and usage documentation, see the comments in "virtio.h". 25 * 26 * NOTE: Client drivers should not use definitions from this file. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/dditypes.h> 31 #include <sys/list.h> 32 #include <sys/ccompile.h> 33 #include <sys/stdbool.h> 34 35 #include "virtio.h" 36 #include "virtio_spec.h" 37 38 #ifdef __cplusplus 39 extern "C" { 40 #endif 41 42 extern ddi_device_acc_attr_t virtio_acc_attr; 43 extern ddi_dma_attr_t virtio_dma_attr; 44 45 extern void virtio_acquireq(virtio_t *, uint16_t); 46 extern void virtio_releaseq(virtio_t *); 47 48 int virtio_dma_init(virtio_t *, virtio_dma_t *, size_t, const ddi_dma_attr_t *, 49 int, int); 50 void virtio_dma_fini(virtio_dma_t *); 51 52 typedef enum virtio_dma_level { 53 VIRTIO_DMALEVEL_HANDLE_ALLOC = (1ULL << 0), 54 VIRTIO_DMALEVEL_MEMORY_ALLOC = (1ULL << 1), 55 VIRTIO_DMALEVEL_HANDLE_BOUND = (1ULL << 2), 56 VIRTIO_DMALEVEL_COOKIE_ARRAY = (1ULL << 3), 57 } virtio_dma_level_t; 58 59 struct virtio_dma { 60 virtio_dma_level_t vidma_level; 61 virtio_t *vidma_virtio; 62 caddr_t vidma_va; 63 size_t vidma_size; 64 size_t vidma_real_size; 65 ddi_dma_handle_t vidma_dma_handle; 66 ddi_acc_handle_t vidma_acc_handle; 67 uint_t vidma_dma_ncookies; 68 ddi_dma_cookie_t *vidma_dma_cookies; 69 }; 70 71 typedef enum virtio_initlevel { 72 VIRTIO_INITLEVEL_REGS = (1ULL << 0), 73 VIRTIO_INITLEVEL_PROVIDER = (1ULL << 1), 74 VIRTIO_INITLEVEL_INT_ALLOC = (1ULL << 2), 75 VIRTIO_INITLEVEL_INT_ADDED = (1ULL << 3), 76 VIRTIO_INITLEVEL_INT_ENABLED = (1ULL << 4), 77 VIRTIO_INITLEVEL_SHUTDOWN = (1ULL << 5), 78 } virtio_initlevel_t; 79 80 typedef struct virtio_pci_cap { 81 virtio_pci_cap_type_t vpc_type; 82 uint8_t vpc_baridx; 83 uint64_t vpc_offset; 84 uint64_t vpc_size; 85 86 ddi_acc_handle_t vpc_barh; 87 caddr_t vpc_bar; 88 } virtio_pci_cap_t; 89 90 typedef enum virtio_mode { 91 VIRTIO_MODE_LEGACY = 1, /* A pure "legacy" device */ 92 VIRTIO_MODE_TRANSITIONAL = 2, /* A "transitional" device */ 93 VIRTIO_MODE_MODERN = 3, /* A pure "modern" device */ 94 } virtio_mode_t; 95 96 typedef struct virtio_ops { 97 uint64_t (*vop_device_get_features)(virtio_t *); 98 bool (*vop_device_set_features)(virtio_t *, uint64_t); 99 void (*vop_set_status_locked)(virtio_t *, uint8_t); 100 uint8_t (*vop_get_status)(virtio_t *); 101 void (*vop_device_reset_locked)(virtio_t *); 102 uint8_t (*vop_isr_status)(virtio_t *); 103 void (*vop_msix_config_set)(virtio_t *, uint16_t); 104 uint16_t (*vop_msix_config_get)(virtio_t *); 105 void (*vop_queue_notify)(virtio_queue_t *); 106 107 void (*vop_queue_select)(virtio_t *, uint16_t); 108 uint16_t (*vop_queue_size_get)(virtio_t *, uint16_t); 109 void (*vop_queue_size_set)(virtio_t *, uint16_t, uint16_t); 110 uint64_t (*vop_queue_noff_get)(virtio_t *, uint16_t); 111 bool (*vop_queue_enable_get)(virtio_t *, uint16_t); 112 void (*vop_queue_enable_set)(virtio_t *, uint16_t, bool); 113 void (*vop_queue_addr_set)(virtio_t *, uint16_t, uint64_t, 114 uint64_t, uint64_t); 115 void (*vop_msix_queue_set)(virtio_t *, uint16_t, uint16_t); 116 uint16_t (*vop_msix_queue_get)(virtio_t *, uint16_t); 117 118 uint8_t (*vop_device_cfg_gen)(virtio_t *); 119 uint8_t (*vop_device_cfg_get8)(virtio_t *, uintptr_t); 120 uint16_t (*vop_device_cfg_get16)(virtio_t *, uintptr_t); 121 uint32_t (*vop_device_cfg_get32)(virtio_t *, uintptr_t); 122 uint64_t (*vop_device_cfg_get64)(virtio_t *, uintptr_t); 123 void (*vop_device_cfg_put8)(virtio_t *, uintptr_t, uint8_t); 124 void (*vop_device_cfg_put16)(virtio_t *, uintptr_t, 125 uint16_t); 126 void (*vop_device_cfg_put32)(virtio_t *, uintptr_t, 127 uint32_t); 128 } virtio_ops_t; 129 130 extern virtio_ops_t virtio_legacy_ops, virtio_modern_ops; 131 132 struct virtio { 133 dev_info_t *vio_dip; 134 135 kmutex_t vio_mutex; 136 137 virtio_initlevel_t vio_initlevel; 138 139 virtio_mode_t vio_mode; 140 virtio_ops_t *vio_ops; 141 142 list_t vio_queues; 143 kmutex_t vio_qlock; 144 uint16_t vio_qcur; 145 146 virtio_pci_cap_t vio_cap_common; 147 virtio_pci_cap_t vio_cap_notify; 148 virtio_pci_cap_t vio_cap_isr; 149 virtio_pci_cap_t vio_cap_device; 150 151 /* Notification multiplier used with the modern interface */ 152 uint32_t vio_multiplier; 153 154 ddi_acc_handle_t vio_barh; 155 caddr_t vio_bar; 156 uint_t vio_legacy_cfg_offset; 157 158 uint64_t vio_features; 159 uint64_t vio_features_device; 160 161 ddi_intr_handle_t *vio_interrupts; 162 int vio_ninterrupts; 163 int vio_interrupt_type; 164 int vio_interrupt_cap; 165 uint_t vio_interrupt_priority; 166 167 ddi_intr_handler_t *vio_cfgchange_handler; 168 void *vio_cfgchange_handlerarg; 169 boolean_t vio_cfgchange_handler_added; 170 uint_t vio_cfgchange_handler_index; 171 }; 172 173 struct virtio_queue { 174 virtio_t *viq_virtio; 175 kmutex_t viq_mutex; 176 const char *viq_name; 177 list_node_t viq_link; 178 179 boolean_t viq_shutdown; 180 boolean_t viq_indirect; 181 uint_t viq_max_segs; 182 183 /* 184 * Each Virtio device type has some set of queues for data transfer to 185 * and from the host. This index is described in the specification for 186 * the particular device and queue type, and written to QUEUE_SELECT to 187 * allow interaction with the queue. For example, a network device has 188 * at least a receive queue with index 0, and a transmit queue with 189 * index 1. 190 */ 191 uint16_t viq_index; 192 193 /* 194 * Modern devices use a BAR region for notifications with each queue 195 * potentially having its own offset within that region. We store the 196 * offset for this queue here. 197 */ 198 uint64_t viq_noff; 199 200 /* 201 * For legacy Virtio devices, the size and shape of the queue is 202 * determined entirely by the number of queue entries. 203 */ 204 uint16_t viq_size; 205 id_space_t *viq_descmap; 206 207 /* 208 * The memory shared between the device and the driver is allocated as 209 * a large phyisically contiguous chunk. Access to this area is 210 * through three pointers to packed structures. 211 */ 212 virtio_dma_t viq_dma; 213 virtio_vq_desc_t *viq_dma_descs; 214 virtio_vq_driver_t *viq_dma_driver; 215 virtio_vq_device_t *viq_dma_device; 216 217 uint16_t viq_device_index; 218 uint16_t viq_driver_index; 219 220 /* 221 * Interrupt handler function, or NULL if not provided. 222 */ 223 ddi_intr_handler_t *viq_func; 224 void *viq_funcarg; 225 boolean_t viq_handler_added; 226 uint_t viq_handler_index; 227 228 /* 229 * When a chain is submitted to the queue, it is also stored in this 230 * AVL tree keyed by the index of the first descriptor in the chain. 231 */ 232 avl_tree_t viq_inflight; 233 }; 234 235 struct virtio_chain { 236 virtio_queue_t *vic_vq; 237 avl_node_t vic_node; 238 239 void *vic_data; 240 241 uint16_t vic_head; 242 uint32_t vic_received_length; 243 244 virtio_dma_t vic_indirect_dma; 245 uint_t vic_indirect_capacity; 246 uint_t vic_indirect_used; 247 248 uint_t vic_direct_capacity; 249 uint_t vic_direct_used; 250 uint16_t vic_direct[]; 251 }; 252 253 /* 254 * When laying out queues for use over the modern interface we choose to align 255 * all queue components using the most restrictive alignment requirement, that 256 * of the descriptor part of the ring. 257 */ 258 #define MODERN_VQ_ALIGN MODERN_VQ_ALIGN_DESC 259 260 /* 261 * DMA SYNCHRONISATION WRAPPERS 262 */ 263 264 /* 265 * Synchronise the driver-owned portion of the queue so that the device can see 266 * our writes. This covers the memory accessed via the "viq_dma_descs" and 267 * "viq_dma_driver" members. 268 */ 269 #define VIRTQ_DMA_SYNC_FORDEV(viq) VERIFY0(ddi_dma_sync( \ 270 (viq)->viq_dma.vidma_dma_handle, \ 271 0, \ 272 (uintptr_t)(viq)->viq_dma_device - \ 273 (uintptr_t)(viq)->viq_dma_descs, \ 274 DDI_DMA_SYNC_FORDEV)) 275 276 /* 277 * Synchronise the device-owned portion of the queue so that we can see any 278 * writes from the device. This covers the memory accessed via the 279 * "viq_dma_device" member. 280 */ 281 #define VIRTQ_DMA_SYNC_FORKERNEL(viq) VERIFY0(ddi_dma_sync( \ 282 (viq)->viq_dma.vidma_dma_handle, \ 283 (uintptr_t)(viq)->viq_dma_device - \ 284 (uintptr_t)(viq)->viq_dma_descs, \ 285 (viq)->viq_dma.vidma_size - \ 286 (uintptr_t)(viq)->viq_dma_device - \ 287 (uintptr_t)(viq)->viq_dma_descs, \ 288 DDI_DMA_SYNC_FORKERNEL)) 289 290 #ifdef __cplusplus 291 } 292 #endif 293 294 #endif /* _VIRTIO_IMPL_H */ 295