1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019 Joyent, Inc. 14 */ 15 16 #ifndef _VIRTIO_IMPL_H 17 #define _VIRTIO_IMPL_H 18 19 /* 20 * VIRTIO FRAMEWORK: FRAMEWORK-PRIVATE DEFINITIONS 21 * 22 * For design and usage documentation, see the comments in "virtio.h". 23 * 24 * NOTE: Client drivers should not use definitions from this file. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/dditypes.h> 29 #include <sys/list.h> 30 #include <sys/ccompile.h> 31 32 #include "virtio.h" 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 extern ddi_device_acc_attr_t virtio_acc_attr; 39 extern ddi_dma_attr_t virtio_dma_attr; 40 41 typedef struct virtio_vq_desc virtio_vq_desc_t; 42 typedef struct virtio_vq_driver virtio_vq_driver_t; 43 typedef struct virtio_vq_device virtio_vq_device_t; 44 typedef struct virtio_vq_elem virtio_vq_elem_t; 45 46 int virtio_dma_init(virtio_t *, virtio_dma_t *, size_t, const ddi_dma_attr_t *, 47 int, int); 48 void virtio_dma_fini(virtio_dma_t *); 49 50 51 52 typedef enum virtio_dma_level { 53 VIRTIO_DMALEVEL_HANDLE_ALLOC = (1ULL << 0), 54 VIRTIO_DMALEVEL_MEMORY_ALLOC = (1ULL << 1), 55 VIRTIO_DMALEVEL_HANDLE_BOUND = (1ULL << 2), 56 VIRTIO_DMALEVEL_COOKIE_ARRAY = (1ULL << 3), 57 } virtio_dma_level_t; 58 59 struct virtio_dma { 60 virtio_dma_level_t vidma_level; 61 virtio_t *vidma_virtio; 62 caddr_t vidma_va; 63 size_t vidma_size; 64 size_t vidma_real_size; 65 ddi_dma_handle_t vidma_dma_handle; 66 ddi_acc_handle_t vidma_acc_handle; 67 uint_t vidma_dma_ncookies; 68 ddi_dma_cookie_t *vidma_dma_cookies; 69 }; 70 71 typedef enum virtio_initlevel { 72 VIRTIO_INITLEVEL_REGS = (1ULL << 0), 73 VIRTIO_INITLEVEL_PROVIDER = (1ULL << 1), 74 VIRTIO_INITLEVEL_INT_ALLOC = (1ULL << 2), 75 VIRTIO_INITLEVEL_INT_ADDED = (1ULL << 3), 76 VIRTIO_INITLEVEL_INT_ENABLED = (1ULL << 4), 77 VIRTIO_INITLEVEL_SHUTDOWN = (1ULL << 5), 78 } virtio_initlevel_t; 79 80 struct virtio { 81 dev_info_t *vio_dip; 82 83 kmutex_t vio_mutex; 84 85 virtio_initlevel_t vio_initlevel; 86 87 list_t vio_queues; 88 89 ddi_acc_handle_t vio_barh; 90 caddr_t vio_bar; 91 uint_t vio_config_offset; 92 93 uint32_t vio_features; 94 uint32_t vio_features_device; 95 96 ddi_intr_handle_t *vio_interrupts; 97 int vio_ninterrupts; 98 int vio_interrupt_type; 99 int vio_interrupt_cap; 100 uint_t vio_interrupt_priority; 101 }; 102 103 struct virtio_queue { 104 virtio_t *viq_virtio; 105 kmutex_t viq_mutex; 106 const char *viq_name; 107 list_node_t viq_link; 108 109 boolean_t viq_shutdown; 110 boolean_t viq_indirect; 111 uint_t viq_max_segs; 112 113 /* 114 * Each Virtio device type has some set of queues for data transfer to 115 * and from the host. This index is described in the specification for 116 * the particular device and queue type, and written to QUEUE_SELECT to 117 * allow interaction with the queue. For example, a network device has 118 * at least a receive queue with index 0, and a transmit queue with 119 * index 1. 120 */ 121 uint16_t viq_index; 122 123 /* 124 * For legacy Virtio devices, the size and shape of the queue is 125 * determined entirely by the number of queue entries. 126 */ 127 uint16_t viq_size; 128 id_space_t *viq_descmap; 129 130 /* 131 * The memory shared between the device and the driver is allocated as 132 * a large phyisically contiguous chunk. Access to this area is 133 * through three pointers to packed structures. 134 */ 135 virtio_dma_t viq_dma; 136 virtio_vq_desc_t *viq_dma_descs; 137 virtio_vq_driver_t *viq_dma_driver; 138 virtio_vq_device_t *viq_dma_device; 139 140 uint16_t viq_device_index; 141 uint16_t viq_driver_index; 142 143 /* 144 * Interrupt handler function, or NULL if not provided. 145 */ 146 ddi_intr_handler_t *viq_func; 147 void *viq_funcarg; 148 boolean_t viq_handler_added; 149 uint_t viq_handler_index; 150 151 /* 152 * When a chain is submitted to the queue, it is also stored in this 153 * AVL tree keyed by the index of the first descriptor in the chain. 154 */ 155 avl_tree_t viq_inflight; 156 }; 157 158 struct virtio_chain { 159 virtio_queue_t *vic_vq; 160 avl_node_t vic_node; 161 162 void *vic_data; 163 164 uint16_t vic_head; 165 uint32_t vic_received_length; 166 167 virtio_dma_t vic_indirect_dma; 168 uint_t vic_indirect_capacity; 169 uint_t vic_indirect_used; 170 171 uint_t vic_direct_capacity; 172 uint_t vic_direct_used; 173 uint16_t vic_direct[]; 174 }; 175 176 /* 177 * PACKED STRUCTS FOR DEVICE ACCESS 178 */ 179 180 struct virtio_vq_desc { 181 /* 182 * Buffer physical address and length. 183 */ 184 uint64_t vqd_addr; 185 uint32_t vqd_len; 186 187 /* 188 * Flags. Use with the VIRTQ_DESC_F_* family of constants. See below. 189 */ 190 uint16_t vqd_flags; 191 192 /* 193 * If VIRTQ_DESC_F_NEXT is set in flags, this refers to the next 194 * descriptor in the chain by table index. 195 */ 196 uint16_t vqd_next; 197 } __packed; 198 199 /* 200 * VIRTIO DESCRIPTOR FLAGS (vqd_flags) 201 */ 202 203 /* 204 * NEXT: 205 * Signals that this descriptor (direct or indirect) is part of a chain. 206 * If populated, "vqd_next" names the next descriptor in the chain by its 207 * table index. 208 */ 209 #define VIRTQ_DESC_F_NEXT (1 << 0) 210 211 /* 212 * WRITE: 213 * Determines whether this buffer is to be written by the device (WRITE is 214 * set) or by the driver (WRITE is not set). 215 */ 216 #define VIRTQ_DESC_F_WRITE (1 << 1) 217 218 /* 219 * INDIRECT: 220 * This bit signals that a direct descriptor refers to an indirect 221 * descriptor list, rather than directly to a buffer. This bit may only 222 * be used in a direct descriptor; indirect descriptors are not allowed to 223 * refer to additional layers of indirect tables. If this bit is set, 224 * NEXT must be clear; indirect descriptors may not be chained. 225 */ 226 #define VIRTQ_DESC_F_INDIRECT (1 << 2) 227 228 /* 229 * This structure is variously known as the "available" or "avail" ring, or the 230 * driver-owned portion of the queue structure. It is used by the driver to 231 * submit descriptor chains to the device. 232 */ 233 struct virtio_vq_driver { 234 uint16_t vqdr_flags; 235 uint16_t vqdr_index; 236 uint16_t vqdr_ring[]; 237 } __packed; 238 239 #define VIRTQ_AVAIL_F_NO_INTERRUPT (1 << 0) 240 241 /* 242 * We use the sizeof operator on this packed struct to calculate the offset of 243 * subsequent structs. Ensure the compiler is not adding any padding to the 244 * end of the struct. 245 */ 246 CTASSERT(sizeof (virtio_vq_driver_t) == 247 offsetof(virtio_vq_driver_t, vqdr_ring)); 248 249 struct virtio_vq_elem { 250 /* 251 * The device returns chains of descriptors by specifying the table 252 * index of the first descriptor in the chain. 253 */ 254 uint32_t vqe_start; 255 uint32_t vqe_len; 256 } __packed; 257 258 /* 259 * This structure is variously known as the "used" ring, or the device-owned 260 * portion of the queue structure. It is used by the device to return 261 * completed descriptor chains to the driver. 262 */ 263 struct virtio_vq_device { 264 uint16_t vqde_flags; 265 uint16_t vqde_index; 266 virtio_vq_elem_t vqde_ring[]; 267 } __packed; 268 269 #define VIRTQ_USED_F_NO_NOTIFY (1 << 0) 270 271 /* 272 * BASIC CONFIGURATION 273 * 274 * Legacy devices expose both their generic and their device-specific 275 * configuration through PCI BAR0. This is the second entry in the register 276 * address space set for these devices. 277 */ 278 #define VIRTIO_LEGACY_PCI_BAR0 1 279 280 /* 281 * These are offsets into the base configuration space available through the 282 * virtio_get*() and virtio_put*() family of functions. These offsets are for 283 * what the specification describes as the "legacy" mode of device operation. 284 */ 285 #define VIRTIO_LEGACY_FEATURES_DEVICE 0x00 /* 32 R */ 286 #define VIRTIO_LEGACY_FEATURES_DRIVER 0x04 /* 32 R/W */ 287 #define VIRTIO_LEGACY_QUEUE_ADDRESS 0x08 /* 32 R/W */ 288 #define VIRTIO_LEGACY_QUEUE_SIZE 0x0C /* 16 R */ 289 #define VIRTIO_LEGACY_QUEUE_SELECT 0x0E /* 16 R/W */ 290 #define VIRTIO_LEGACY_QUEUE_NOTIFY 0x10 /* 16 R/W */ 291 #define VIRTIO_LEGACY_DEVICE_STATUS 0x12 /* 8 R/W */ 292 #define VIRTIO_LEGACY_ISR_STATUS 0x13 /* 8 R */ 293 294 #define VIRTIO_LEGACY_MSIX_CONFIG 0x14 /* 16 R/W */ 295 #define VIRTIO_LEGACY_MSIX_QUEUE 0x16 /* 16 R/W */ 296 297 #define VIRTIO_LEGACY_CFG_OFFSET (VIRTIO_LEGACY_ISR_STATUS + 1) 298 #define VIRTIO_LEGACY_CFG_OFFSET_MSIX (VIRTIO_LEGACY_MSIX_QUEUE + 2) 299 300 #define VIRTIO_LEGACY_MSI_NO_VECTOR 0xFFFF 301 302 /* 303 * Bits in the Device Status byte (VIRTIO_LEGACY_DEVICE_STATUS): 304 */ 305 #define VIRTIO_STATUS_RESET 0 306 #define VIRTIO_STATUS_ACKNOWLEDGE (1 << 0) 307 #define VIRTIO_STATUS_DRIVER (1 << 1) 308 #define VIRTIO_STATUS_DRIVER_OK (1 << 2) 309 #define VIRTIO_STATUS_FAILED (1 << 7) 310 311 /* 312 * Bits in the Interrupt Service Routine Status byte 313 * (VIRTIO_LEGACY_ISR_STATUS): 314 */ 315 #define VIRTIO_ISR_CHECK_QUEUES (1 << 0) 316 #define VIRTIO_ISR_CHECK_CONFIG (1 << 1) 317 318 /* 319 * Bits in the Features fields (VIRTIO_LEGACY_FEATURES_DEVICE, 320 * VIRTIO_LEGACY_FEATURES_DRIVER): 321 */ 322 #define VIRTIO_F_RING_INDIRECT_DESC (1ULL << 28) 323 324 /* 325 * For devices operating in the legacy mode, virtqueues must be aligned on a 326 * "page size" of 4096 bytes; this is also called the "Queue Align" value in 327 * newer versions of the specification. 328 */ 329 #define VIRTIO_PAGE_SHIFT 12 330 #define VIRTIO_PAGE_SIZE (1 << VIRTIO_PAGE_SHIFT) 331 CTASSERT(VIRTIO_PAGE_SIZE == 4096); 332 CTASSERT(ISP2(VIRTIO_PAGE_SIZE)); 333 334 /* 335 * DMA SYNCHRONISATION WRAPPERS 336 */ 337 338 /* 339 * Synchronise the driver-owned portion of the queue so that the device can see 340 * our writes. This covers the memory accessed via the "viq_dma_descs" and 341 * "viq_dma_device" members. 342 */ 343 #define VIRTQ_DMA_SYNC_FORDEV(viq) VERIFY0(ddi_dma_sync( \ 344 (viq)->viq_dma.vidma_dma_handle, \ 345 0, \ 346 (uintptr_t)(viq)->viq_dma_device - \ 347 (uintptr_t)(viq)->viq_dma_descs, \ 348 DDI_DMA_SYNC_FORDEV)) 349 350 /* 351 * Synchronise the device-owned portion of the queue so that we can see any 352 * writes from the device. This covers the memory accessed via the 353 * "viq_dma_device" member. 354 */ 355 #define VIRTQ_DMA_SYNC_FORKERNEL(viq) VERIFY0(ddi_dma_sync( \ 356 (viq)->viq_dma.vidma_dma_handle, \ 357 (uintptr_t)(viq)->viq_dma_device - \ 358 (uintptr_t)(viq)->viq_dma_descs, \ 359 (viq)->viq_dma.vidma_size - \ 360 (uintptr_t)(viq)->viq_dma_device - \ 361 (uintptr_t)(viq)->viq_dma_descs, \ 362 DDI_DMA_SYNC_FORKERNEL)) 363 364 #ifdef __cplusplus 365 } 366 #endif 367 368 #endif /* _VIRTIO_IMPL_H */ 369