1f8296c60SJoshua M. Clulow /*
2f8296c60SJoshua M. Clulow * This file and its contents are supplied under the terms of the
3f8296c60SJoshua M. Clulow * Common Development and Distribution License ("CDDL"), version 1.0.
4f8296c60SJoshua M. Clulow * You may only use this file in accordance with the terms of version
5f8296c60SJoshua M. Clulow * 1.0 of the CDDL.
6f8296c60SJoshua M. Clulow *
7f8296c60SJoshua M. Clulow * A full copy of the text of the CDDL should have accompanied this
8f8296c60SJoshua M. Clulow * source. A copy of the CDDL is also available via the Internet at
9f8296c60SJoshua M. Clulow * http://www.illumos.org/license/CDDL.
10f8296c60SJoshua M. Clulow */
11f8296c60SJoshua M. Clulow
12f8296c60SJoshua M. Clulow /*
13f8296c60SJoshua M. Clulow * Copyright 2019 Joyent, Inc.
14*64439ec0SJoshua M. Clulow * Copyright 2022 Oxide Computer Company
15f8296c60SJoshua M. Clulow */
16f8296c60SJoshua M. Clulow
17f8296c60SJoshua M. Clulow /*
18f8296c60SJoshua M. Clulow * VIRTIO FRAMEWORK: DMA ROUTINES
19f8296c60SJoshua M. Clulow *
20f8296c60SJoshua M. Clulow * For design and usage documentation, see the comments in "virtio.h".
21f8296c60SJoshua M. Clulow */
22f8296c60SJoshua M. Clulow
23f8296c60SJoshua M. Clulow #include <sys/conf.h>
24f8296c60SJoshua M. Clulow #include <sys/kmem.h>
25f8296c60SJoshua M. Clulow #include <sys/debug.h>
26f8296c60SJoshua M. Clulow #include <sys/modctl.h>
27f8296c60SJoshua M. Clulow #include <sys/autoconf.h>
28f8296c60SJoshua M. Clulow #include <sys/ddi_impldefs.h>
29f8296c60SJoshua M. Clulow #include <sys/ddi.h>
30f8296c60SJoshua M. Clulow #include <sys/sunddi.h>
31f8296c60SJoshua M. Clulow #include <sys/sunndi.h>
32f8296c60SJoshua M. Clulow #include <sys/avintr.h>
33f8296c60SJoshua M. Clulow #include <sys/spl.h>
34f8296c60SJoshua M. Clulow #include <sys/promif.h>
35f8296c60SJoshua M. Clulow #include <sys/list.h>
36f8296c60SJoshua M. Clulow #include <sys/bootconf.h>
37f8296c60SJoshua M. Clulow #include <sys/bootsvcs.h>
38f8296c60SJoshua M. Clulow #include <sys/sysmacros.h>
39f8296c60SJoshua M. Clulow #include <sys/pci.h>
40f8296c60SJoshua M. Clulow
41f8296c60SJoshua M. Clulow #include "virtio.h"
42f8296c60SJoshua M. Clulow #include "virtio_impl.h"
43f8296c60SJoshua M. Clulow
44*64439ec0SJoshua M. Clulow typedef int (dma_wait_t)(caddr_t);
45f8296c60SJoshua M. Clulow
46*64439ec0SJoshua M. Clulow static dma_wait_t *
virtio_dma_wait_from_kmflags(int kmflags)47*64439ec0SJoshua M. Clulow virtio_dma_wait_from_kmflags(int kmflags)
48*64439ec0SJoshua M. Clulow {
49*64439ec0SJoshua M. Clulow switch (kmflags) {
50*64439ec0SJoshua M. Clulow case KM_SLEEP:
51*64439ec0SJoshua M. Clulow return (DDI_DMA_SLEEP);
52*64439ec0SJoshua M. Clulow case KM_NOSLEEP:
53*64439ec0SJoshua M. Clulow case KM_NOSLEEP_LAZY:
54*64439ec0SJoshua M. Clulow return (DDI_DMA_DONTWAIT);
55*64439ec0SJoshua M. Clulow default:
56*64439ec0SJoshua M. Clulow panic("unexpected kmflags value 0x%x", kmflags);
57*64439ec0SJoshua M. Clulow }
58*64439ec0SJoshua M. Clulow }
59f8296c60SJoshua M. Clulow
60f8296c60SJoshua M. Clulow void
virtio_dma_sync(virtio_dma_t * vidma,int flag)61f8296c60SJoshua M. Clulow virtio_dma_sync(virtio_dma_t *vidma, int flag)
62f8296c60SJoshua M. Clulow {
63f8296c60SJoshua M. Clulow VERIFY0(ddi_dma_sync(vidma->vidma_dma_handle, 0, 0, flag));
64f8296c60SJoshua M. Clulow }
65f8296c60SJoshua M. Clulow
66f8296c60SJoshua M. Clulow uint_t
virtio_dma_ncookies(virtio_dma_t * vidma)67f8296c60SJoshua M. Clulow virtio_dma_ncookies(virtio_dma_t *vidma)
68f8296c60SJoshua M. Clulow {
69f8296c60SJoshua M. Clulow return (vidma->vidma_dma_ncookies);
70f8296c60SJoshua M. Clulow }
71f8296c60SJoshua M. Clulow
72f8296c60SJoshua M. Clulow size_t
virtio_dma_size(virtio_dma_t * vidma)73f8296c60SJoshua M. Clulow virtio_dma_size(virtio_dma_t *vidma)
74f8296c60SJoshua M. Clulow {
75f8296c60SJoshua M. Clulow return (vidma->vidma_size);
76f8296c60SJoshua M. Clulow }
77f8296c60SJoshua M. Clulow
78f8296c60SJoshua M. Clulow void *
virtio_dma_va(virtio_dma_t * vidma,size_t offset)79f8296c60SJoshua M. Clulow virtio_dma_va(virtio_dma_t *vidma, size_t offset)
80f8296c60SJoshua M. Clulow {
81f8296c60SJoshua M. Clulow VERIFY3U(offset, <, vidma->vidma_size);
82f8296c60SJoshua M. Clulow
83f8296c60SJoshua M. Clulow return (vidma->vidma_va + offset);
84f8296c60SJoshua M. Clulow }
85f8296c60SJoshua M. Clulow
86f8296c60SJoshua M. Clulow uint64_t
virtio_dma_cookie_pa(virtio_dma_t * vidma,uint_t cookie)87f8296c60SJoshua M. Clulow virtio_dma_cookie_pa(virtio_dma_t *vidma, uint_t cookie)
88f8296c60SJoshua M. Clulow {
89f8296c60SJoshua M. Clulow VERIFY3U(cookie, <, vidma->vidma_dma_ncookies);
90f8296c60SJoshua M. Clulow
91f8296c60SJoshua M. Clulow return (vidma->vidma_dma_cookies[cookie].dmac_laddress);
92f8296c60SJoshua M. Clulow }
93f8296c60SJoshua M. Clulow
94f8296c60SJoshua M. Clulow size_t
virtio_dma_cookie_size(virtio_dma_t * vidma,uint_t cookie)95f8296c60SJoshua M. Clulow virtio_dma_cookie_size(virtio_dma_t *vidma, uint_t cookie)
96f8296c60SJoshua M. Clulow {
97f8296c60SJoshua M. Clulow VERIFY3U(cookie, <, vidma->vidma_dma_ncookies);
98f8296c60SJoshua M. Clulow
99f8296c60SJoshua M. Clulow return (vidma->vidma_dma_cookies[cookie].dmac_size);
100f8296c60SJoshua M. Clulow }
101f8296c60SJoshua M. Clulow
102f8296c60SJoshua M. Clulow int
virtio_dma_init_handle(virtio_t * vio,virtio_dma_t * vidma,const ddi_dma_attr_t * attr,int kmflags)103f8296c60SJoshua M. Clulow virtio_dma_init_handle(virtio_t *vio, virtio_dma_t *vidma,
104f8296c60SJoshua M. Clulow const ddi_dma_attr_t *attr, int kmflags)
105f8296c60SJoshua M. Clulow {
106f8296c60SJoshua M. Clulow int r;
107f8296c60SJoshua M. Clulow dev_info_t *dip = vio->vio_dip;
108*64439ec0SJoshua M. Clulow int (*dma_wait)(caddr_t) = virtio_dma_wait_from_kmflags(kmflags);
109f8296c60SJoshua M. Clulow
110f8296c60SJoshua M. Clulow vidma->vidma_virtio = vio;
111f8296c60SJoshua M. Clulow
112f8296c60SJoshua M. Clulow /*
113f8296c60SJoshua M. Clulow * Ensure we don't try to allocate a second time using the same
114f8296c60SJoshua M. Clulow * tracking object.
115f8296c60SJoshua M. Clulow */
116f8296c60SJoshua M. Clulow VERIFY0(vidma->vidma_level);
117f8296c60SJoshua M. Clulow
118f8296c60SJoshua M. Clulow if ((r = ddi_dma_alloc_handle(dip, (ddi_dma_attr_t *)attr, dma_wait,
119f8296c60SJoshua M. Clulow NULL, &vidma->vidma_dma_handle)) != DDI_SUCCESS) {
120f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "DMA handle allocation failed (%x)", r);
121f8296c60SJoshua M. Clulow goto fail;
122f8296c60SJoshua M. Clulow }
123f8296c60SJoshua M. Clulow vidma->vidma_level |= VIRTIO_DMALEVEL_HANDLE_ALLOC;
124f8296c60SJoshua M. Clulow
125f8296c60SJoshua M. Clulow return (DDI_SUCCESS);
126f8296c60SJoshua M. Clulow
127f8296c60SJoshua M. Clulow fail:
128f8296c60SJoshua M. Clulow virtio_dma_fini(vidma);
129f8296c60SJoshua M. Clulow return (DDI_FAILURE);
130f8296c60SJoshua M. Clulow }
131f8296c60SJoshua M. Clulow
132f8296c60SJoshua M. Clulow int
virtio_dma_init(virtio_t * vio,virtio_dma_t * vidma,size_t sz,const ddi_dma_attr_t * attr,int dmaflags,int kmflags)133f8296c60SJoshua M. Clulow virtio_dma_init(virtio_t *vio, virtio_dma_t *vidma, size_t sz,
134f8296c60SJoshua M. Clulow const ddi_dma_attr_t *attr, int dmaflags, int kmflags)
135f8296c60SJoshua M. Clulow {
136f8296c60SJoshua M. Clulow int r;
137f8296c60SJoshua M. Clulow dev_info_t *dip = vio->vio_dip;
138f8296c60SJoshua M. Clulow caddr_t va = NULL;
139*64439ec0SJoshua M. Clulow int (*dma_wait)(caddr_t) = virtio_dma_wait_from_kmflags(kmflags);
140f8296c60SJoshua M. Clulow
141f8296c60SJoshua M. Clulow if (virtio_dma_init_handle(vio, vidma, attr, kmflags) !=
142f8296c60SJoshua M. Clulow DDI_SUCCESS) {
143f8296c60SJoshua M. Clulow goto fail;
144f8296c60SJoshua M. Clulow }
145f8296c60SJoshua M. Clulow
146f8296c60SJoshua M. Clulow if ((r = ddi_dma_mem_alloc(vidma->vidma_dma_handle, sz,
147f8296c60SJoshua M. Clulow &virtio_acc_attr,
148f8296c60SJoshua M. Clulow dmaflags & (DDI_DMA_STREAMING | DDI_DMA_CONSISTENT),
149f8296c60SJoshua M. Clulow dma_wait, NULL, &va, &vidma->vidma_real_size,
150f8296c60SJoshua M. Clulow &vidma->vidma_acc_handle)) != DDI_SUCCESS) {
151f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "DMA memory allocation failed (%x)", r);
152f8296c60SJoshua M. Clulow goto fail;
153f8296c60SJoshua M. Clulow }
154f8296c60SJoshua M. Clulow vidma->vidma_level |= VIRTIO_DMALEVEL_MEMORY_ALLOC;
155f8296c60SJoshua M. Clulow
156f8296c60SJoshua M. Clulow /*
157f8296c60SJoshua M. Clulow * Zero the memory to avoid accidental exposure of arbitrary kernel
158f8296c60SJoshua M. Clulow * memory.
159f8296c60SJoshua M. Clulow */
160f8296c60SJoshua M. Clulow bzero(va, vidma->vidma_real_size);
161f8296c60SJoshua M. Clulow
162f8296c60SJoshua M. Clulow if (virtio_dma_bind(vidma, va, sz, dmaflags, kmflags) != DDI_SUCCESS) {
163f8296c60SJoshua M. Clulow goto fail;
164f8296c60SJoshua M. Clulow }
165f8296c60SJoshua M. Clulow
166f8296c60SJoshua M. Clulow return (DDI_SUCCESS);
167f8296c60SJoshua M. Clulow
168f8296c60SJoshua M. Clulow fail:
169f8296c60SJoshua M. Clulow virtio_dma_fini(vidma);
170f8296c60SJoshua M. Clulow return (DDI_FAILURE);
171f8296c60SJoshua M. Clulow }
172f8296c60SJoshua M. Clulow
173f8296c60SJoshua M. Clulow int
virtio_dma_bind(virtio_dma_t * vidma,void * va,size_t sz,int dmaflags,int kmflags)174f8296c60SJoshua M. Clulow virtio_dma_bind(virtio_dma_t *vidma, void *va, size_t sz, int dmaflags,
175f8296c60SJoshua M. Clulow int kmflags)
176f8296c60SJoshua M. Clulow {
177f8296c60SJoshua M. Clulow int r;
178f8296c60SJoshua M. Clulow dev_info_t *dip = vidma->vidma_virtio->vio_dip;
179f8296c60SJoshua M. Clulow ddi_dma_cookie_t dmac;
180*64439ec0SJoshua M. Clulow int (*dma_wait)(caddr_t) = virtio_dma_wait_from_kmflags(kmflags);
181f8296c60SJoshua M. Clulow
182f8296c60SJoshua M. Clulow VERIFY(vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_ALLOC);
183f8296c60SJoshua M. Clulow VERIFY(!(vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_BOUND));
184f8296c60SJoshua M. Clulow
185f8296c60SJoshua M. Clulow vidma->vidma_va = va;
186f8296c60SJoshua M. Clulow vidma->vidma_size = sz;
187f8296c60SJoshua M. Clulow
188f8296c60SJoshua M. Clulow if ((r = ddi_dma_addr_bind_handle(vidma->vidma_dma_handle, NULL,
189f8296c60SJoshua M. Clulow vidma->vidma_va, vidma->vidma_size, dmaflags, dma_wait, NULL,
190f8296c60SJoshua M. Clulow &dmac, &vidma->vidma_dma_ncookies)) != DDI_DMA_MAPPED) {
191f8296c60SJoshua M. Clulow VERIFY3S(r, !=, DDI_DMA_PARTIAL_MAP);
192f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "DMA handle bind failed (%x)", r);
193f8296c60SJoshua M. Clulow goto fail;
194f8296c60SJoshua M. Clulow }
195f8296c60SJoshua M. Clulow vidma->vidma_level |= VIRTIO_DMALEVEL_HANDLE_BOUND;
196f8296c60SJoshua M. Clulow
197f8296c60SJoshua M. Clulow if ((vidma->vidma_dma_cookies = kmem_alloc(
198f8296c60SJoshua M. Clulow vidma->vidma_dma_ncookies * sizeof (ddi_dma_cookie_t),
199f8296c60SJoshua M. Clulow kmflags)) == NULL) {
200f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "DMA cookie array allocation failure");
201f8296c60SJoshua M. Clulow goto fail;
202f8296c60SJoshua M. Clulow }
203f8296c60SJoshua M. Clulow vidma->vidma_level |= VIRTIO_DMALEVEL_COOKIE_ARRAY;
204f8296c60SJoshua M. Clulow
205f8296c60SJoshua M. Clulow vidma->vidma_dma_cookies[0] = dmac;
206f8296c60SJoshua M. Clulow for (uint_t n = 1; n < vidma->vidma_dma_ncookies; n++) {
207f8296c60SJoshua M. Clulow ddi_dma_nextcookie(vidma->vidma_dma_handle,
208f8296c60SJoshua M. Clulow &vidma->vidma_dma_cookies[n]);
209f8296c60SJoshua M. Clulow }
210f8296c60SJoshua M. Clulow
211f8296c60SJoshua M. Clulow return (DDI_SUCCESS);
212f8296c60SJoshua M. Clulow
213f8296c60SJoshua M. Clulow fail:
214f8296c60SJoshua M. Clulow virtio_dma_unbind(vidma);
215f8296c60SJoshua M. Clulow return (DDI_FAILURE);
216f8296c60SJoshua M. Clulow }
217f8296c60SJoshua M. Clulow
218f8296c60SJoshua M. Clulow virtio_dma_t *
virtio_dma_alloc(virtio_t * vio,size_t sz,const ddi_dma_attr_t * attr,int dmaflags,int kmflags)219f8296c60SJoshua M. Clulow virtio_dma_alloc(virtio_t *vio, size_t sz, const ddi_dma_attr_t *attr,
220f8296c60SJoshua M. Clulow int dmaflags, int kmflags)
221f8296c60SJoshua M. Clulow {
222f8296c60SJoshua M. Clulow virtio_dma_t *vidma;
223f8296c60SJoshua M. Clulow
224f8296c60SJoshua M. Clulow if ((vidma = kmem_zalloc(sizeof (*vidma), kmflags)) == NULL) {
225f8296c60SJoshua M. Clulow return (NULL);
226f8296c60SJoshua M. Clulow }
227f8296c60SJoshua M. Clulow
228f8296c60SJoshua M. Clulow if (virtio_dma_init(vio, vidma, sz, attr, dmaflags, kmflags) !=
229f8296c60SJoshua M. Clulow DDI_SUCCESS) {
230f8296c60SJoshua M. Clulow kmem_free(vidma, sizeof (*vidma));
231f8296c60SJoshua M. Clulow return (NULL);
232f8296c60SJoshua M. Clulow }
233f8296c60SJoshua M. Clulow
234f8296c60SJoshua M. Clulow return (vidma);
235f8296c60SJoshua M. Clulow }
236f8296c60SJoshua M. Clulow
237f8296c60SJoshua M. Clulow virtio_dma_t *
virtio_dma_alloc_nomem(virtio_t * vio,const ddi_dma_attr_t * attr,int kmflags)238f8296c60SJoshua M. Clulow virtio_dma_alloc_nomem(virtio_t *vio, const ddi_dma_attr_t *attr, int kmflags)
239f8296c60SJoshua M. Clulow {
240f8296c60SJoshua M. Clulow virtio_dma_t *vidma;
241f8296c60SJoshua M. Clulow
242f8296c60SJoshua M. Clulow if ((vidma = kmem_zalloc(sizeof (*vidma), kmflags)) == NULL) {
243f8296c60SJoshua M. Clulow return (NULL);
244f8296c60SJoshua M. Clulow }
245f8296c60SJoshua M. Clulow
246f8296c60SJoshua M. Clulow if (virtio_dma_init_handle(vio, vidma, attr, kmflags) != DDI_SUCCESS) {
247f8296c60SJoshua M. Clulow kmem_free(vidma, sizeof (*vidma));
248f8296c60SJoshua M. Clulow return (NULL);
249f8296c60SJoshua M. Clulow }
250f8296c60SJoshua M. Clulow
251f8296c60SJoshua M. Clulow return (vidma);
252f8296c60SJoshua M. Clulow }
253f8296c60SJoshua M. Clulow
254f8296c60SJoshua M. Clulow void
virtio_dma_fini(virtio_dma_t * vidma)255f8296c60SJoshua M. Clulow virtio_dma_fini(virtio_dma_t *vidma)
256f8296c60SJoshua M. Clulow {
257f8296c60SJoshua M. Clulow virtio_dma_unbind(vidma);
258f8296c60SJoshua M. Clulow
259f8296c60SJoshua M. Clulow if (vidma->vidma_level & VIRTIO_DMALEVEL_MEMORY_ALLOC) {
260f8296c60SJoshua M. Clulow ddi_dma_mem_free(&vidma->vidma_acc_handle);
261f8296c60SJoshua M. Clulow
262f8296c60SJoshua M. Clulow vidma->vidma_level &= ~VIRTIO_DMALEVEL_MEMORY_ALLOC;
263f8296c60SJoshua M. Clulow }
264f8296c60SJoshua M. Clulow
265f8296c60SJoshua M. Clulow if (vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_ALLOC) {
266f8296c60SJoshua M. Clulow ddi_dma_free_handle(&vidma->vidma_dma_handle);
267f8296c60SJoshua M. Clulow
268f8296c60SJoshua M. Clulow vidma->vidma_level &= ~VIRTIO_DMALEVEL_HANDLE_ALLOC;
269f8296c60SJoshua M. Clulow }
270f8296c60SJoshua M. Clulow
271f8296c60SJoshua M. Clulow VERIFY0(vidma->vidma_level);
272f8296c60SJoshua M. Clulow bzero(vidma, sizeof (*vidma));
273f8296c60SJoshua M. Clulow }
274f8296c60SJoshua M. Clulow
275f8296c60SJoshua M. Clulow void
virtio_dma_unbind(virtio_dma_t * vidma)276f8296c60SJoshua M. Clulow virtio_dma_unbind(virtio_dma_t *vidma)
277f8296c60SJoshua M. Clulow {
278f8296c60SJoshua M. Clulow if (vidma->vidma_level & VIRTIO_DMALEVEL_COOKIE_ARRAY) {
279f8296c60SJoshua M. Clulow kmem_free(vidma->vidma_dma_cookies,
280f8296c60SJoshua M. Clulow vidma->vidma_dma_ncookies * sizeof (ddi_dma_cookie_t));
281f8296c60SJoshua M. Clulow
282f8296c60SJoshua M. Clulow vidma->vidma_level &= ~VIRTIO_DMALEVEL_COOKIE_ARRAY;
283f8296c60SJoshua M. Clulow }
284f8296c60SJoshua M. Clulow
285f8296c60SJoshua M. Clulow if (vidma->vidma_level & VIRTIO_DMALEVEL_HANDLE_BOUND) {
286f8296c60SJoshua M. Clulow VERIFY3U(ddi_dma_unbind_handle(vidma->vidma_dma_handle), ==,
287f8296c60SJoshua M. Clulow DDI_SUCCESS);
288f8296c60SJoshua M. Clulow
289f8296c60SJoshua M. Clulow vidma->vidma_level &= ~VIRTIO_DMALEVEL_HANDLE_BOUND;
290f8296c60SJoshua M. Clulow }
291f8296c60SJoshua M. Clulow
292f8296c60SJoshua M. Clulow vidma->vidma_va = 0;
293f8296c60SJoshua M. Clulow vidma->vidma_size = 0;
294f8296c60SJoshua M. Clulow }
295f8296c60SJoshua M. Clulow
296f8296c60SJoshua M. Clulow void
virtio_dma_free(virtio_dma_t * vidma)297f8296c60SJoshua M. Clulow virtio_dma_free(virtio_dma_t *vidma)
298f8296c60SJoshua M. Clulow {
299f8296c60SJoshua M. Clulow virtio_dma_fini(vidma);
300f8296c60SJoshua M. Clulow kmem_free(vidma, sizeof (*vidma));
301f8296c60SJoshua M. Clulow }
302