xref: /illumos-gate/usr/src/uts/common/io/virtio/virtio_impl.h (revision 66582b606a8194f7f3ba5b3a3a6dca5b0d346361)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2019 Joyent, Inc.
14  */
15 
16 #ifndef _VIRTIO_IMPL_H
17 #define	_VIRTIO_IMPL_H
18 
19 /*
20  * VIRTIO FRAMEWORK: FRAMEWORK-PRIVATE DEFINITIONS
21  *
22  * For design and usage documentation, see the comments in "virtio.h".
23  *
24  * NOTE: Client drivers should not use definitions from this file.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/dditypes.h>
29 #include <sys/list.h>
30 #include <sys/ccompile.h>
31 
32 #include "virtio.h"
33 
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37 
38 extern ddi_device_acc_attr_t virtio_acc_attr;
39 extern ddi_dma_attr_t virtio_dma_attr;
40 
41 typedef struct virtio_vq_desc virtio_vq_desc_t;
42 typedef struct virtio_vq_driver virtio_vq_driver_t;
43 typedef struct virtio_vq_device virtio_vq_device_t;
44 typedef struct virtio_vq_elem virtio_vq_elem_t;
45 
46 int virtio_dma_init(virtio_t *, virtio_dma_t *, size_t, const ddi_dma_attr_t *,
47     int, int);
48 void virtio_dma_fini(virtio_dma_t *);
49 
50 
51 
52 typedef enum virtio_dma_level {
53 	VIRTIO_DMALEVEL_HANDLE_ALLOC =	(1ULL << 0),
54 	VIRTIO_DMALEVEL_MEMORY_ALLOC =	(1ULL << 1),
55 	VIRTIO_DMALEVEL_HANDLE_BOUND =	(1ULL << 2),
56 	VIRTIO_DMALEVEL_COOKIE_ARRAY =	(1ULL << 3),
57 } virtio_dma_level_t;
58 
59 struct virtio_dma {
60 	virtio_dma_level_t		vidma_level;
61 	virtio_t			*vidma_virtio;
62 	caddr_t				vidma_va;
63 	size_t				vidma_size;
64 	size_t				vidma_real_size;
65 	ddi_dma_handle_t		vidma_dma_handle;
66 	ddi_acc_handle_t		vidma_acc_handle;
67 	uint_t				vidma_dma_ncookies;
68 	ddi_dma_cookie_t		*vidma_dma_cookies;
69 };
70 
71 typedef enum virtio_initlevel {
72 	VIRTIO_INITLEVEL_REGS =		(1ULL << 0),
73 	VIRTIO_INITLEVEL_PROVIDER =	(1ULL << 1),
74 	VIRTIO_INITLEVEL_INT_ALLOC =	(1ULL << 2),
75 	VIRTIO_INITLEVEL_INT_ADDED =	(1ULL << 3),
76 	VIRTIO_INITLEVEL_INT_ENABLED =	(1ULL << 4),
77 	VIRTIO_INITLEVEL_SHUTDOWN =	(1ULL << 5),
78 } virtio_initlevel_t;
79 
80 struct virtio {
81 	dev_info_t			*vio_dip;
82 
83 	kmutex_t			vio_mutex;
84 
85 	virtio_initlevel_t		vio_initlevel;
86 
87 	list_t				vio_queues;
88 
89 	ddi_acc_handle_t		vio_barh;
90 	caddr_t				vio_bar;
91 	uint_t				vio_config_offset;
92 
93 	uint32_t			vio_features;
94 	uint32_t			vio_features_device;
95 
96 	ddi_intr_handle_t		*vio_interrupts;
97 	int				vio_ninterrupts;
98 	int				vio_interrupt_type;
99 	int				vio_interrupt_cap;
100 	uint_t				vio_interrupt_priority;
101 };
102 
103 struct virtio_queue {
104 	virtio_t			*viq_virtio;
105 	kmutex_t			viq_mutex;
106 	const char			*viq_name;
107 	list_node_t			viq_link;
108 
109 	boolean_t			viq_shutdown;
110 	boolean_t			viq_indirect;
111 	uint_t				viq_max_segs;
112 
113 	/*
114 	 * Each Virtio device type has some set of queues for data transfer to
115 	 * and from the host.  This index is described in the specification for
116 	 * the particular device and queue type, and written to QUEUE_SELECT to
117 	 * allow interaction with the queue.  For example, a network device has
118 	 * at least a receive queue with index 0, and a transmit queue with
119 	 * index 1.
120 	 */
121 	uint16_t			viq_index;
122 
123 	/*
124 	 * For legacy Virtio devices, the size and shape of the queue is
125 	 * determined entirely by the number of queue entries.
126 	 */
127 	uint16_t			viq_size;
128 	id_space_t			*viq_descmap;
129 
130 	/*
131 	 * The memory shared between the device and the driver is allocated as
132 	 * a large phyisically contiguous chunk.  Access to this area is
133 	 * through three pointers to packed structures.
134 	 */
135 	virtio_dma_t			viq_dma;
136 	virtio_vq_desc_t		*viq_dma_descs;
137 	virtio_vq_driver_t		*viq_dma_driver;
138 	virtio_vq_device_t		*viq_dma_device;
139 
140 	uint16_t			viq_device_index;
141 	uint16_t			viq_driver_index;
142 
143 	/*
144 	 * Interrupt handler function, or NULL if not provided.
145 	 */
146 	ddi_intr_handler_t		*viq_func;
147 	void				*viq_funcarg;
148 	boolean_t			viq_handler_added;
149 	uint_t				viq_handler_index;
150 
151 	/*
152 	 * When a chain is submitted to the queue, it is also stored in this
153 	 * AVL tree keyed by the index of the first descriptor in the chain.
154 	 */
155 	avl_tree_t			viq_inflight;
156 };
157 
158 struct virtio_chain {
159 	virtio_queue_t			*vic_vq;
160 	avl_node_t			vic_node;
161 
162 	void				*vic_data;
163 
164 	uint16_t			vic_head;
165 	uint32_t			vic_received_length;
166 
167 	virtio_dma_t			vic_indirect_dma;
168 	uint_t				vic_indirect_capacity;
169 	uint_t				vic_indirect_used;
170 
171 	uint_t				vic_direct_capacity;
172 	uint_t				vic_direct_used;
173 	uint16_t			vic_direct[];
174 };
175 
176 /*
177  * PACKED STRUCTS FOR DEVICE ACCESS
178  */
179 
180 struct virtio_vq_desc {
181 	/*
182 	 * Buffer physical address and length.
183 	 */
184 	uint64_t			vqd_addr;
185 	uint32_t			vqd_len;
186 
187 	/*
188 	 * Flags.  Use with the VIRTQ_DESC_F_* family of constants.  See below.
189 	 */
190 	uint16_t			vqd_flags;
191 
192 	/*
193 	 * If VIRTQ_DESC_F_NEXT is set in flags, this refers to the next
194 	 * descriptor in the chain by table index.
195 	 */
196 	uint16_t			vqd_next;
197 } __packed;
198 
199 /*
200  * VIRTIO DESCRIPTOR FLAGS (vqd_flags)
201  */
202 
203 /*
204  * NEXT:
205  *	Signals that this descriptor (direct or indirect) is part of a chain.
206  *	If populated, "vqd_next" names the next descriptor in the chain by its
207  *	table index.
208  */
209 #define	VIRTQ_DESC_F_NEXT		(1 << 0)
210 
211 /*
212  * WRITE:
213  *	Determines whether this buffer is to be written by the device (WRITE is
214  *	set) or by the driver (WRITE is not set).
215  */
216 #define	VIRTQ_DESC_F_WRITE		(1 << 1)
217 
218 /*
219  * INDIRECT:
220  *	This bit signals that a direct descriptor refers to an indirect
221  *	descriptor list, rather than directly to a buffer.  This bit may only
222  *	be used in a direct descriptor; indirect descriptors are not allowed to
223  *	refer to additional layers of indirect tables.  If this bit is set,
224  *	NEXT must be clear; indirect descriptors may not be chained.
225  */
226 #define	VIRTQ_DESC_F_INDIRECT		(1 << 2)
227 
228 /*
229  * This structure is variously known as the "available" or "avail" ring, or the
230  * driver-owned portion of the queue structure.  It is used by the driver to
231  * submit descriptor chains to the device.
232  */
233 struct virtio_vq_driver {
234 	uint16_t			vqdr_flags;
235 	uint16_t			vqdr_index;
236 	uint16_t			vqdr_ring[];
237 } __packed;
238 
239 #define	VIRTQ_AVAIL_F_NO_INTERRUPT	(1 << 0)
240 
241 /*
242  * We use the sizeof operator on this packed struct to calculate the offset of
243  * subsequent structs.  Ensure the compiler is not adding any padding to the
244  * end of the struct.
245  */
246 CTASSERT(sizeof (virtio_vq_driver_t) ==
247     offsetof(virtio_vq_driver_t, vqdr_ring));
248 
249 struct virtio_vq_elem {
250 	/*
251 	 * The device returns chains of descriptors by specifying the table
252 	 * index of the first descriptor in the chain.
253 	 */
254 	uint32_t			vqe_start;
255 	uint32_t			vqe_len;
256 } __packed;
257 
258 /*
259  * This structure is variously known as the "used" ring, or the device-owned
260  * portion of the queue structure.  It is used by the device to return
261  * completed descriptor chains to the driver.
262  */
263 struct virtio_vq_device {
264 	uint16_t			vqde_flags;
265 	uint16_t			vqde_index;
266 	virtio_vq_elem_t		vqde_ring[];
267 } __packed;
268 
269 #define	VIRTQ_USED_F_NO_NOTIFY		(1 << 0)
270 
271 /*
272  * BASIC CONFIGURATION
273  *
274  * Legacy devices expose both their generic and their device-specific
275  * configuration through PCI BAR0.  This is the second entry in the register
276  * address space set for these devices.
277  */
278 #define	VIRTIO_LEGACY_PCI_BAR0		1
279 
280 /*
281  * These are offsets into the base configuration space available through the
282  * virtio_get*() and virtio_put*() family of functions.  These offsets are for
283  * what the specification describes as the "legacy" mode of device operation.
284  */
285 #define	VIRTIO_LEGACY_FEATURES_DEVICE	0x00	/* 32 R   */
286 #define	VIRTIO_LEGACY_FEATURES_DRIVER	0x04	/* 32 R/W */
287 #define	VIRTIO_LEGACY_QUEUE_ADDRESS	0x08	/* 32 R/W */
288 #define	VIRTIO_LEGACY_QUEUE_SIZE	0x0C	/* 16 R   */
289 #define	VIRTIO_LEGACY_QUEUE_SELECT	0x0E	/* 16 R/W */
290 #define	VIRTIO_LEGACY_QUEUE_NOTIFY	0x10	/* 16 R/W */
291 #define	VIRTIO_LEGACY_DEVICE_STATUS	0x12	/* 8  R/W */
292 #define	VIRTIO_LEGACY_ISR_STATUS	0x13	/* 8  R   */
293 
294 #define	VIRTIO_LEGACY_MSIX_CONFIG	0x14	/* 16 R/W */
295 #define	VIRTIO_LEGACY_MSIX_QUEUE	0x16	/* 16 R/W */
296 
297 #define	VIRTIO_LEGACY_CFG_OFFSET	(VIRTIO_LEGACY_ISR_STATUS + 1)
298 #define	VIRTIO_LEGACY_CFG_OFFSET_MSIX	(VIRTIO_LEGACY_MSIX_QUEUE + 2)
299 
300 #define	VIRTIO_LEGACY_MSI_NO_VECTOR	0xFFFF
301 
302 /*
303  * Bits in the Device Status byte (VIRTIO_LEGACY_DEVICE_STATUS):
304  */
305 #define	VIRTIO_STATUS_RESET		0
306 #define	VIRTIO_STATUS_ACKNOWLEDGE	(1 << 0)
307 #define	VIRTIO_STATUS_DRIVER		(1 << 1)
308 #define	VIRTIO_STATUS_DRIVER_OK		(1 << 2)
309 #define	VIRTIO_STATUS_FAILED		(1 << 7)
310 
311 /*
312  * Bits in the Interrupt Service Routine Status byte
313  * (VIRTIO_LEGACY_ISR_STATUS):
314  */
315 #define	VIRTIO_ISR_CHECK_QUEUES		(1 << 0)
316 #define	VIRTIO_ISR_CHECK_CONFIG		(1 << 1)
317 
318 /*
319  * Bits in the Features fields (VIRTIO_LEGACY_FEATURES_DEVICE,
320  * VIRTIO_LEGACY_FEATURES_DRIVER):
321  */
322 #define	VIRTIO_F_RING_INDIRECT_DESC	(1ULL << 28)
323 
324 /*
325  * For devices operating in the legacy mode, virtqueues must be aligned on a
326  * "page size" of 4096 bytes; this is also called the "Queue Align" value in
327  * newer versions of the specification.
328  */
329 #define	VIRTIO_PAGE_SHIFT		12
330 #define	VIRTIO_PAGE_SIZE		(1 << VIRTIO_PAGE_SHIFT)
331 CTASSERT(VIRTIO_PAGE_SIZE == 4096);
332 CTASSERT(ISP2(VIRTIO_PAGE_SIZE));
333 
334 /*
335  * DMA SYNCHRONISATION WRAPPERS
336  */
337 
338 /*
339  * Synchronise the driver-owned portion of the queue so that the device can see
340  * our writes.  This covers the memory accessed via the "viq_dma_descs" and
341  * "viq_dma_device" members.
342  */
343 #define	VIRTQ_DMA_SYNC_FORDEV(viq)	VERIFY0(ddi_dma_sync( \
344 					    (viq)->viq_dma.vidma_dma_handle, \
345 					    0, \
346 					    (uintptr_t)(viq)->viq_dma_device - \
347 					    (uintptr_t)(viq)->viq_dma_descs, \
348 					    DDI_DMA_SYNC_FORDEV))
349 
350 /*
351  * Synchronise the device-owned portion of the queue so that we can see any
352  * writes from the device.  This covers the memory accessed via the
353  * "viq_dma_device" member.
354  */
355 #define	VIRTQ_DMA_SYNC_FORKERNEL(viq)	VERIFY0(ddi_dma_sync( \
356 					    (viq)->viq_dma.vidma_dma_handle, \
357 					    (uintptr_t)(viq)->viq_dma_device - \
358 					    (uintptr_t)(viq)->viq_dma_descs, \
359 					    (viq)->viq_dma.vidma_size - \
360 					    (uintptr_t)(viq)->viq_dma_device - \
361 					    (uintptr_t)(viq)->viq_dma_descs, \
362 					    DDI_DMA_SYNC_FORKERNEL))
363 
364 #ifdef __cplusplus
365 }
366 #endif
367 
368 #endif /* _VIRTIO_IMPL_H */
369