1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
24 * Copyright 2012 Alexey Zaytsev <alexey.zaytsev@gmail.com>
25 */
26
27 /* Based on the NetBSD virtio driver by Minoura Makoto. */
28 /*
29 * Copyright (c) 2010 Minoura Makoto.
30 * All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
42 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
43 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
44 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
45 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
47 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
48 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
49 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
50 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
51 *
52 */
53
54 #include <sys/conf.h>
55 #include <sys/kmem.h>
56 #include <sys/debug.h>
57 #include <sys/modctl.h>
58 #include <sys/autoconf.h>
59 #include <sys/ddi_impldefs.h>
60 #include <sys/ddi.h>
61 #include <sys/sunddi.h>
62 #include <sys/sunndi.h>
63 #include <sys/avintr.h>
64 #include <sys/spl.h>
65 #include <sys/promif.h>
66 #include <sys/list.h>
67 #include <sys/bootconf.h>
68 #include <sys/bootsvcs.h>
69 #include <sys/sysmacros.h>
70 #include <sys/pci.h>
71
72 #include "virtiovar.h"
73 #include "virtioreg.h"
74
75 #define NDEVNAMES (sizeof (virtio_device_name) / sizeof (char *))
76 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */
77 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1)) & \
78 ~(VIRTIO_PAGE_SIZE-1))
79
80 void
virtio_set_status(struct virtio_softc * sc,unsigned int status)81 virtio_set_status(struct virtio_softc *sc, unsigned int status)
82 {
83 int old = 0;
84
85 if (status != 0) {
86 old = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
87 VIRTIO_CONFIG_DEVICE_STATUS));
88 }
89
90 ddi_put8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
91 VIRTIO_CONFIG_DEVICE_STATUS), status | old);
92 }
93
94 /*
95 * Negotiate features, save the result in sc->sc_features
96 */
97 uint32_t
virtio_negotiate_features(struct virtio_softc * sc,uint32_t guest_features)98 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
99 {
100 uint32_t host_features;
101 uint32_t features;
102
103 host_features = ddi_get32(sc->sc_ioh,
104 /* LINTED E_BAD_PTR_CAST_ALIGN */
105 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_DEVICE_FEATURES));
106
107 dev_debug(sc->sc_dev, CE_NOTE, "host features: %x, guest features: %x",
108 host_features, guest_features);
109
110 features = host_features & guest_features;
111 ddi_put32(sc->sc_ioh,
112 /* LINTED E_BAD_PTR_CAST_ALIGN */
113 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_GUEST_FEATURES),
114 features);
115
116 sc->sc_features = features;
117
118 return (host_features);
119 }
120
121 size_t
virtio_show_features(uint32_t features,char * buf,size_t len)122 virtio_show_features(uint32_t features, char *buf, size_t len)
123 {
124 char *orig_buf = buf;
125 char *bufend = buf + len;
126
127 /* LINTED E_PTRDIFF_OVERFLOW */
128 buf += snprintf(buf, bufend - buf, "Generic ( ");
129 if (features & VIRTIO_F_RING_INDIRECT_DESC)
130 /* LINTED E_PTRDIFF_OVERFLOW */
131 buf += snprintf(buf, bufend - buf, "INDIRECT_DESC ");
132
133 /* LINTED E_PTRDIFF_OVERFLOW */
134 buf += snprintf(buf, bufend - buf, ") ");
135
136 /* LINTED E_PTRDIFF_OVERFLOW */
137 return (buf - orig_buf);
138 }
139
140 boolean_t
virtio_has_feature(struct virtio_softc * sc,uint32_t feature)141 virtio_has_feature(struct virtio_softc *sc, uint32_t feature)
142 {
143 return (sc->sc_features & feature);
144 }
145
146 /*
147 * Device configuration registers.
148 */
149 uint8_t
virtio_read_device_config_1(struct virtio_softc * sc,unsigned int index)150 virtio_read_device_config_1(struct virtio_softc *sc, unsigned int index)
151 {
152 ASSERT(sc->sc_config_offset);
153 return ddi_get8(sc->sc_ioh,
154 (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
155 }
156
157 uint16_t
virtio_read_device_config_2(struct virtio_softc * sc,unsigned int index)158 virtio_read_device_config_2(struct virtio_softc *sc, unsigned int index)
159 {
160 ASSERT(sc->sc_config_offset);
161 return ddi_get16(sc->sc_ioh,
162 /* LINTED E_BAD_PTR_CAST_ALIGN */
163 (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
164 }
165
166 uint32_t
virtio_read_device_config_4(struct virtio_softc * sc,unsigned int index)167 virtio_read_device_config_4(struct virtio_softc *sc, unsigned int index)
168 {
169 ASSERT(sc->sc_config_offset);
170 return ddi_get32(sc->sc_ioh,
171 /* LINTED E_BAD_PTR_CAST_ALIGN */
172 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
173 }
174
175 uint64_t
virtio_read_device_config_8(struct virtio_softc * sc,unsigned int index)176 virtio_read_device_config_8(struct virtio_softc *sc, unsigned int index)
177 {
178 uint64_t r;
179
180 ASSERT(sc->sc_config_offset);
181 r = ddi_get32(sc->sc_ioh,
182 /* LINTED E_BAD_PTR_CAST_ALIGN */
183 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
184 index + sizeof (uint32_t)));
185
186 r <<= 32;
187
188 r += ddi_get32(sc->sc_ioh,
189 /* LINTED E_BAD_PTR_CAST_ALIGN */
190 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
191 return (r);
192 }
193
194 void
virtio_write_device_config_1(struct virtio_softc * sc,unsigned int index,uint8_t value)195 virtio_write_device_config_1(struct virtio_softc *sc, unsigned int index,
196 uint8_t value)
197 {
198 ASSERT(sc->sc_config_offset);
199 ddi_put8(sc->sc_ioh,
200 (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
201 }
202
203 void
virtio_write_device_config_2(struct virtio_softc * sc,unsigned int index,uint16_t value)204 virtio_write_device_config_2(struct virtio_softc *sc, unsigned int index,
205 uint16_t value)
206 {
207 ASSERT(sc->sc_config_offset);
208 ddi_put16(sc->sc_ioh,
209 /* LINTED E_BAD_PTR_CAST_ALIGN */
210 (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
211 }
212
213 void
virtio_write_device_config_4(struct virtio_softc * sc,unsigned int index,uint32_t value)214 virtio_write_device_config_4(struct virtio_softc *sc, unsigned int index,
215 uint32_t value)
216 {
217 ASSERT(sc->sc_config_offset);
218 ddi_put32(sc->sc_ioh,
219 /* LINTED E_BAD_PTR_CAST_ALIGN */
220 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
221 }
222
223 void
virtio_write_device_config_8(struct virtio_softc * sc,unsigned int index,uint64_t value)224 virtio_write_device_config_8(struct virtio_softc *sc, unsigned int index,
225 uint64_t value)
226 {
227 ASSERT(sc->sc_config_offset);
228 ddi_put32(sc->sc_ioh,
229 /* LINTED E_BAD_PTR_CAST_ALIGN */
230 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index),
231 value & 0xFFFFFFFF);
232 ddi_put32(sc->sc_ioh,
233 /* LINTED E_BAD_PTR_CAST_ALIGN */
234 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
235 index + sizeof (uint32_t)), value >> 32);
236 }
237
238 /*
239 * Start/stop vq interrupt. No guarantee.
240 */
241 void
virtio_stop_vq_intr(struct virtqueue * vq)242 virtio_stop_vq_intr(struct virtqueue *vq)
243 {
244 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
245 }
246
247 void
virtio_start_vq_intr(struct virtqueue * vq)248 virtio_start_vq_intr(struct virtqueue *vq)
249 {
250 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
251 }
252
253 static ddi_dma_attr_t virtio_vq_dma_attr = {
254 DMA_ATTR_V0, /* Version number */
255 0, /* low address */
256 0x00000FFFFFFFFFFF, /* high address. Has to fit into 32 bits */
257 /* after page-shifting */
258 0xFFFFFFFF, /* counter register max */
259 VIRTIO_PAGE_SIZE, /* page alignment required */
260 0x3F, /* burst sizes: 1 - 32 */
261 0x1, /* minimum transfer size */
262 0xFFFFFFFF, /* max transfer size */
263 0xFFFFFFFF, /* address register max */
264 1, /* no scatter-gather */
265 1, /* device operates on bytes */
266 0, /* attr flag: set to 0 */
267 };
268
269 static ddi_dma_attr_t virtio_vq_indirect_dma_attr = {
270 DMA_ATTR_V0, /* Version number */
271 0, /* low address */
272 0xFFFFFFFFFFFFFFFF, /* high address */
273 0xFFFFFFFF, /* counter register max */
274 1, /* No specific alignment */
275 0x3F, /* burst sizes: 1 - 32 */
276 0x1, /* minimum transfer size */
277 0xFFFFFFFF, /* max transfer size */
278 0xFFFFFFFF, /* address register max */
279 1, /* no scatter-gather */
280 1, /* device operates on bytes */
281 0, /* attr flag: set to 0 */
282 };
283
284 /* Same for direct and indirect descriptors. */
285 static ddi_device_acc_attr_t virtio_vq_devattr = {
286 DDI_DEVICE_ATTR_V0,
287 DDI_NEVERSWAP_ACC,
288 DDI_STORECACHING_OK_ACC,
289 DDI_DEFAULT_ACC
290 };
291
292 static void
virtio_free_indirect(struct vq_entry * entry)293 virtio_free_indirect(struct vq_entry *entry)
294 {
295
296 (void) ddi_dma_unbind_handle(entry->qe_indirect_dma_handle);
297 ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
298 ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
299
300 entry->qe_indirect_descs = NULL;
301 }
302
303
304 static int
virtio_alloc_indirect(struct virtio_softc * sc,struct vq_entry * entry)305 virtio_alloc_indirect(struct virtio_softc *sc, struct vq_entry *entry)
306 {
307 int allocsize, num;
308 size_t len;
309 unsigned int ncookies;
310 int ret;
311
312 num = entry->qe_queue->vq_indirect_num;
313 ASSERT(num > 1);
314
315 allocsize = sizeof (struct vring_desc) * num;
316
317 ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_indirect_dma_attr,
318 DDI_DMA_SLEEP, NULL, &entry->qe_indirect_dma_handle);
319 if (ret != DDI_SUCCESS) {
320 dev_err(sc->sc_dev, CE_WARN,
321 "Failed to allocate dma handle for indirect descriptors, "
322 "entry %d, vq %d", entry->qe_index,
323 entry->qe_queue->vq_index);
324 goto out_alloc_handle;
325 }
326
327 ret = ddi_dma_mem_alloc(entry->qe_indirect_dma_handle, allocsize,
328 &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
329 (caddr_t *)&entry->qe_indirect_descs, &len,
330 &entry->qe_indirect_dma_acch);
331 if (ret != DDI_SUCCESS) {
332 dev_err(sc->sc_dev, CE_WARN,
333 "Failed to allocate dma memory for indirect descriptors, "
334 "entry %d, vq %d,", entry->qe_index,
335 entry->qe_queue->vq_index);
336 goto out_alloc;
337 }
338
339 (void) memset(entry->qe_indirect_descs, 0xff, allocsize);
340
341 ret = ddi_dma_addr_bind_handle(entry->qe_indirect_dma_handle, NULL,
342 (caddr_t)entry->qe_indirect_descs, len,
343 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
344 &entry->qe_indirect_dma_cookie, &ncookies);
345 if (ret != DDI_DMA_MAPPED) {
346 dev_err(sc->sc_dev, CE_WARN,
347 "Failed to bind dma memory for indirect descriptors, "
348 "entry %d, vq %d", entry->qe_index,
349 entry->qe_queue->vq_index);
350 goto out_bind;
351 }
352
353 /* We asked for a single segment */
354 ASSERT(ncookies == 1);
355
356 return (0);
357
358 out_bind:
359 ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
360 out_alloc:
361 ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
362 out_alloc_handle:
363
364 return (ret);
365 }
366
367 /*
368 * Initialize the vq structure.
369 */
370 static int
virtio_init_vq(struct virtio_softc * sc,struct virtqueue * vq)371 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq)
372 {
373 int ret;
374 uint16_t i;
375 int vq_size = vq->vq_num;
376 int indirect_num = vq->vq_indirect_num;
377
378 /* free slot management */
379 list_create(&vq->vq_freelist, sizeof (struct vq_entry),
380 offsetof(struct vq_entry, qe_list));
381
382 for (i = 0; i < vq_size; i++) {
383 struct vq_entry *entry = &vq->vq_entries[i];
384 list_insert_tail(&vq->vq_freelist, entry);
385 entry->qe_index = i;
386 entry->qe_desc = &vq->vq_descs[i];
387 entry->qe_queue = vq;
388
389 if (indirect_num) {
390 ret = virtio_alloc_indirect(sc, entry);
391 if (ret)
392 goto out_indirect;
393 }
394 }
395
396 mutex_init(&vq->vq_freelist_lock, "virtio-freelist", MUTEX_DRIVER,
397 DDI_INTR_PRI(sc->sc_intr_prio));
398 mutex_init(&vq->vq_avail_lock, "virtio-avail", MUTEX_DRIVER,
399 DDI_INTR_PRI(sc->sc_intr_prio));
400 mutex_init(&vq->vq_used_lock, "virtio-used", MUTEX_DRIVER,
401 DDI_INTR_PRI(sc->sc_intr_prio));
402
403 return (0);
404
405 out_indirect:
406 for (i = 0; i < vq_size; i++) {
407 struct vq_entry *entry = &vq->vq_entries[i];
408 if (entry->qe_indirect_descs)
409 virtio_free_indirect(entry);
410 }
411
412 return (ret);
413 }
414
415 /*
416 * Allocate/free a vq.
417 */
418 struct virtqueue *
virtio_alloc_vq(struct virtio_softc * sc,unsigned int index,unsigned int size,unsigned int indirect_num,const char * name)419 virtio_alloc_vq(struct virtio_softc *sc, unsigned int index, unsigned int size,
420 unsigned int indirect_num, const char *name)
421 {
422 int vq_size, allocsize1, allocsize2, allocsize = 0;
423 int ret;
424 unsigned int ncookies;
425 size_t len;
426 struct virtqueue *vq;
427
428 ddi_put16(sc->sc_ioh,
429 /* LINTED E_BAD_PTR_CAST_ALIGN */
430 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT), index);
431 vq_size = ddi_get16(sc->sc_ioh,
432 /* LINTED E_BAD_PTR_CAST_ALIGN */
433 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SIZE));
434 if (vq_size == 0) {
435 dev_err(sc->sc_dev, CE_WARN,
436 "virtqueue dest not exist, index %d for %s\n", index, name);
437 goto out;
438 }
439
440 vq = kmem_zalloc(sizeof (struct virtqueue), KM_SLEEP);
441
442 /* size 0 => use native vq size, good for receive queues. */
443 if (size)
444 vq_size = MIN(vq_size, size);
445
446 /* allocsize1: descriptor table + avail ring + pad */
447 allocsize1 = VIRTQUEUE_ALIGN(sizeof (struct vring_desc) * vq_size +
448 sizeof (struct vring_avail) + sizeof (uint16_t) * vq_size);
449 /* allocsize2: used ring + pad */
450 allocsize2 = VIRTQUEUE_ALIGN(sizeof (struct vring_used) +
451 sizeof (struct vring_used_elem) * vq_size);
452
453 allocsize = allocsize1 + allocsize2;
454
455 ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_dma_attr,
456 DDI_DMA_SLEEP, NULL, &vq->vq_dma_handle);
457 if (ret != DDI_SUCCESS) {
458 dev_err(sc->sc_dev, CE_WARN,
459 "Failed to allocate dma handle for vq %d", index);
460 goto out_alloc_handle;
461 }
462
463 ret = ddi_dma_mem_alloc(vq->vq_dma_handle, allocsize,
464 &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
465 (caddr_t *)&vq->vq_vaddr, &len, &vq->vq_dma_acch);
466 if (ret != DDI_SUCCESS) {
467 dev_err(sc->sc_dev, CE_WARN,
468 "Failed to allocate dma memory for vq %d", index);
469 goto out_alloc;
470 }
471
472 ret = ddi_dma_addr_bind_handle(vq->vq_dma_handle, NULL,
473 (caddr_t)vq->vq_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
474 DDI_DMA_SLEEP, NULL, &vq->vq_dma_cookie, &ncookies);
475 if (ret != DDI_DMA_MAPPED) {
476 dev_err(sc->sc_dev, CE_WARN,
477 "Failed to bind dma memory for vq %d", index);
478 goto out_bind;
479 }
480
481 /* We asked for a single segment */
482 ASSERT(ncookies == 1);
483 /* and page-ligned buffers. */
484 ASSERT(vq->vq_dma_cookie.dmac_laddress % VIRTIO_PAGE_SIZE == 0);
485
486 (void) memset(vq->vq_vaddr, 0, allocsize);
487
488 /* Make sure all zeros hit the buffer before we point the host to it */
489 membar_producer();
490
491 /* set the vq address */
492 ddi_put32(sc->sc_ioh,
493 /* LINTED E_BAD_PTR_CAST_ALIGN */
494 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS),
495 (vq->vq_dma_cookie.dmac_laddress / VIRTIO_PAGE_SIZE));
496
497 /* remember addresses and offsets for later use */
498 vq->vq_owner = sc;
499 vq->vq_num = vq_size;
500 vq->vq_index = index;
501 vq->vq_descs = vq->vq_vaddr;
502 vq->vq_availoffset = sizeof (struct vring_desc)*vq_size;
503 vq->vq_avail = (void *)(((char *)vq->vq_descs) + vq->vq_availoffset);
504 vq->vq_usedoffset = allocsize1;
505 vq->vq_used = (void *)(((char *)vq->vq_descs) + vq->vq_usedoffset);
506
507 ASSERT(indirect_num == 0 ||
508 virtio_has_feature(sc, VIRTIO_F_RING_INDIRECT_DESC));
509 vq->vq_indirect_num = indirect_num;
510
511 /* free slot management */
512 vq->vq_entries = kmem_zalloc(sizeof (struct vq_entry) * vq_size,
513 KM_SLEEP);
514
515 ret = virtio_init_vq(sc, vq);
516 if (ret)
517 goto out_init;
518
519 dev_debug(sc->sc_dev, CE_NOTE,
520 "Allocated %d entries for vq %d:%s (%d indirect descs)",
521 vq_size, index, name, indirect_num * vq_size);
522
523 return (vq);
524
525 out_init:
526 kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq_size);
527 (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
528 out_bind:
529 ddi_dma_mem_free(&vq->vq_dma_acch);
530 out_alloc:
531 ddi_dma_free_handle(&vq->vq_dma_handle);
532 out_alloc_handle:
533 kmem_free(vq, sizeof (struct virtqueue));
534 out:
535 return (NULL);
536 }
537
538 void
virtio_free_vq(struct virtqueue * vq)539 virtio_free_vq(struct virtqueue *vq)
540 {
541 struct virtio_softc *sc = vq->vq_owner;
542 int i;
543
544 /* tell device that there's no virtqueue any longer */
545 ddi_put16(sc->sc_ioh,
546 /* LINTED E_BAD_PTR_CAST_ALIGN */
547 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT),
548 vq->vq_index);
549 ddi_put32(sc->sc_ioh,
550 /* LINTED E_BAD_PTR_CAST_ALIGN */
551 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS), 0);
552
553 /* Free the indirect descriptors, if any. */
554 for (i = 0; i < vq->vq_num; i++) {
555 struct vq_entry *entry = &vq->vq_entries[i];
556 if (entry->qe_indirect_descs)
557 virtio_free_indirect(entry);
558 }
559
560 kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq->vq_num);
561
562 (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
563 ddi_dma_mem_free(&vq->vq_dma_acch);
564 ddi_dma_free_handle(&vq->vq_dma_handle);
565
566 mutex_destroy(&vq->vq_used_lock);
567 mutex_destroy(&vq->vq_avail_lock);
568 mutex_destroy(&vq->vq_freelist_lock);
569
570 kmem_free(vq, sizeof (struct virtqueue));
571 }
572
573 /*
574 * Free descriptor management.
575 */
576 struct vq_entry *
vq_alloc_entry(struct virtqueue * vq)577 vq_alloc_entry(struct virtqueue *vq)
578 {
579 struct vq_entry *qe;
580
581 mutex_enter(&vq->vq_freelist_lock);
582 if (list_is_empty(&vq->vq_freelist)) {
583 mutex_exit(&vq->vq_freelist_lock);
584 return (NULL);
585 }
586 qe = list_remove_head(&vq->vq_freelist);
587
588 ASSERT(vq->vq_used_entries >= 0);
589 vq->vq_used_entries++;
590
591 mutex_exit(&vq->vq_freelist_lock);
592
593 qe->qe_next = NULL;
594 qe->qe_indirect_next = 0;
595 (void) memset(qe->qe_desc, 0, sizeof (struct vring_desc));
596
597 return (qe);
598 }
599
600 void
vq_free_entry(struct virtqueue * vq,struct vq_entry * qe)601 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
602 {
603 mutex_enter(&vq->vq_freelist_lock);
604
605 list_insert_head(&vq->vq_freelist, qe);
606 vq->vq_used_entries--;
607 ASSERT(vq->vq_used_entries >= 0);
608 mutex_exit(&vq->vq_freelist_lock);
609 }
610
611 /*
612 * We (intentionally) don't have a global vq mutex, so you are
613 * responsible for external locking to avoid allocting/freeing any
614 * entries before using the returned value. Have fun.
615 */
616 uint_t
vq_num_used(struct virtqueue * vq)617 vq_num_used(struct virtqueue *vq)
618 {
619 /* vq->vq_freelist_lock would not help here. */
620 return (vq->vq_used_entries);
621 }
622
623 static inline void
virtio_ve_set_desc(struct vring_desc * desc,uint64_t paddr,uint32_t len,boolean_t write)624 virtio_ve_set_desc(struct vring_desc *desc, uint64_t paddr, uint32_t len,
625 boolean_t write)
626 {
627 desc->addr = paddr;
628 desc->len = len;
629 desc->next = 0;
630 desc->flags = 0;
631
632 /* 'write' - from the driver's point of view */
633 if (!write)
634 desc->flags = VRING_DESC_F_WRITE;
635 }
636
637 void
virtio_ve_set(struct vq_entry * qe,uint64_t paddr,uint32_t len,boolean_t write)638 virtio_ve_set(struct vq_entry *qe, uint64_t paddr, uint32_t len,
639 boolean_t write)
640 {
641 virtio_ve_set_desc(qe->qe_desc, paddr, len, write);
642 }
643
644 unsigned int
virtio_ve_indirect_available(struct vq_entry * qe)645 virtio_ve_indirect_available(struct vq_entry *qe)
646 {
647 return (qe->qe_queue->vq_indirect_num - (qe->qe_indirect_next - 1));
648 }
649
650 void
virtio_ve_add_indirect_buf(struct vq_entry * qe,uint64_t paddr,uint32_t len,boolean_t write)651 virtio_ve_add_indirect_buf(struct vq_entry *qe, uint64_t paddr, uint32_t len,
652 boolean_t write)
653 {
654 struct vring_desc *indirect_desc;
655
656 ASSERT(qe->qe_queue->vq_indirect_num);
657 ASSERT(qe->qe_indirect_next < qe->qe_queue->vq_indirect_num);
658
659 indirect_desc = &qe->qe_indirect_descs[qe->qe_indirect_next];
660 virtio_ve_set_desc(indirect_desc, paddr, len, write);
661 qe->qe_indirect_next++;
662 }
663
664 void
virtio_ve_add_cookie(struct vq_entry * qe,ddi_dma_handle_t dma_handle,ddi_dma_cookie_t dma_cookie,unsigned int ncookies,boolean_t write)665 virtio_ve_add_cookie(struct vq_entry *qe, ddi_dma_handle_t dma_handle,
666 ddi_dma_cookie_t dma_cookie, unsigned int ncookies, boolean_t write)
667 {
668 int i;
669
670 for (i = 0; i < ncookies; i++) {
671 virtio_ve_add_indirect_buf(qe, dma_cookie.dmac_laddress,
672 dma_cookie.dmac_size, write);
673 ddi_dma_nextcookie(dma_handle, &dma_cookie);
674 }
675 }
676
677 void
virtio_sync_vq(struct virtqueue * vq)678 virtio_sync_vq(struct virtqueue *vq)
679 {
680 struct virtio_softc *vsc = vq->vq_owner;
681
682 /* Make sure the avail ring update hit the buffer */
683 membar_producer();
684
685 vq->vq_avail->idx = vq->vq_avail_idx;
686
687 /* Make sure the avail idx update hits the buffer */
688 membar_producer();
689
690 /* Make sure we see the flags update */
691 membar_consumer();
692
693 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) {
694 ddi_put16(vsc->sc_ioh,
695 /* LINTED E_BAD_PTR_CAST_ALIGN */
696 (uint16_t *)(vsc->sc_io_addr +
697 VIRTIO_CONFIG_QUEUE_NOTIFY),
698 vq->vq_index);
699 }
700 }
701
702 void
virtio_push_chain(struct vq_entry * qe,boolean_t sync)703 virtio_push_chain(struct vq_entry *qe, boolean_t sync)
704 {
705 struct virtqueue *vq = qe->qe_queue;
706 struct vq_entry *head = qe;
707 struct vring_desc *desc;
708 int idx;
709
710 ASSERT(qe);
711
712 /*
713 * Bind the descs together, paddr and len should be already
714 * set with virtio_ve_set
715 */
716 do {
717 /* Bind the indirect descriptors */
718 if (qe->qe_indirect_next > 1) {
719 uint16_t i = 0;
720
721 /*
722 * Set the pointer/flags to the
723 * first indirect descriptor
724 */
725 virtio_ve_set_desc(qe->qe_desc,
726 qe->qe_indirect_dma_cookie.dmac_laddress,
727 sizeof (struct vring_desc) * qe->qe_indirect_next,
728 B_FALSE);
729 qe->qe_desc->flags |= VRING_DESC_F_INDIRECT;
730
731 /* For all but the last one, add the next index/flag */
732 do {
733 desc = &qe->qe_indirect_descs[i];
734 i++;
735
736 desc->flags |= VRING_DESC_F_NEXT;
737 desc->next = i;
738 } while (i < qe->qe_indirect_next - 1);
739
740 }
741
742 if (qe->qe_next) {
743 qe->qe_desc->flags |= VRING_DESC_F_NEXT;
744 qe->qe_desc->next = qe->qe_next->qe_index;
745 }
746
747 qe = qe->qe_next;
748 } while (qe);
749
750 mutex_enter(&vq->vq_avail_lock);
751 idx = vq->vq_avail_idx;
752 vq->vq_avail_idx++;
753
754 /* Make sure the bits hit the descriptor(s) */
755 membar_producer();
756 vq->vq_avail->ring[idx % vq->vq_num] = head->qe_index;
757
758 /* Notify the device, if needed. */
759 if (sync)
760 virtio_sync_vq(vq);
761
762 mutex_exit(&vq->vq_avail_lock);
763 }
764
765 /*
766 * Get a chain of descriptors from the used ring, if one is available.
767 */
768 struct vq_entry *
virtio_pull_chain(struct virtqueue * vq,uint32_t * len)769 virtio_pull_chain(struct virtqueue *vq, uint32_t *len)
770 {
771 struct vq_entry *head;
772 int slot;
773 int usedidx;
774
775 mutex_enter(&vq->vq_used_lock);
776
777 /* No used entries? Bye. */
778 if (vq->vq_used_idx == vq->vq_used->idx) {
779 mutex_exit(&vq->vq_used_lock);
780 return (NULL);
781 }
782
783 usedidx = vq->vq_used_idx;
784 vq->vq_used_idx++;
785 mutex_exit(&vq->vq_used_lock);
786
787 usedidx %= vq->vq_num;
788
789 /* Make sure we do the next step _after_ checking the idx. */
790 membar_consumer();
791
792 slot = vq->vq_used->ring[usedidx].id;
793 *len = vq->vq_used->ring[usedidx].len;
794
795 head = &vq->vq_entries[slot];
796
797 return (head);
798 }
799
800 void
virtio_free_chain(struct vq_entry * qe)801 virtio_free_chain(struct vq_entry *qe)
802 {
803 struct vq_entry *tmp;
804 struct virtqueue *vq = qe->qe_queue;
805
806 ASSERT(qe);
807
808 do {
809 ASSERT(qe->qe_queue == vq);
810 tmp = qe->qe_next;
811 vq_free_entry(vq, qe);
812 qe = tmp;
813 } while (tmp != NULL);
814 }
815
816 void
virtio_ventry_stick(struct vq_entry * first,struct vq_entry * second)817 virtio_ventry_stick(struct vq_entry *first, struct vq_entry *second)
818 {
819 first->qe_next = second;
820 }
821
822 static int
virtio_register_msi(struct virtio_softc * sc,struct virtio_int_handler * config_handler,struct virtio_int_handler vq_handlers[],int intr_types)823 virtio_register_msi(struct virtio_softc *sc,
824 struct virtio_int_handler *config_handler,
825 struct virtio_int_handler vq_handlers[], int intr_types)
826 {
827 int count, actual;
828 int int_type;
829 int i;
830 int handler_count;
831 int ret;
832
833 /* If both MSI and MSI-x are reported, prefer MSI-x. */
834 int_type = DDI_INTR_TYPE_MSI;
835 if (intr_types & DDI_INTR_TYPE_MSIX)
836 int_type = DDI_INTR_TYPE_MSIX;
837
838 /* Walk the handler table to get the number of handlers. */
839 for (handler_count = 0;
840 vq_handlers && vq_handlers[handler_count].vh_func;
841 handler_count++)
842 ;
843
844 /* +1 if there is a config change handler. */
845 if (config_handler != NULL)
846 handler_count++;
847
848 /* Number of MSIs supported by the device. */
849 ret = ddi_intr_get_nintrs(sc->sc_dev, int_type, &count);
850 if (ret != DDI_SUCCESS) {
851 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_nintrs failed");
852 return (ret);
853 }
854
855 /*
856 * Those who try to register more handlers then the device
857 * supports shall suffer.
858 */
859 ASSERT(handler_count <= count);
860
861 sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t) *
862 handler_count, KM_SLEEP);
863
864 ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable, int_type, 0,
865 handler_count, &actual, DDI_INTR_ALLOC_NORMAL);
866 if (ret != DDI_SUCCESS) {
867 dev_err(sc->sc_dev, CE_WARN, "Failed to allocate MSI: %d", ret);
868 goto out_msi_alloc;
869 }
870
871 if (actual != handler_count) {
872 dev_err(sc->sc_dev, CE_WARN,
873 "Not enough MSI available: need %d, available %d",
874 handler_count, actual);
875 goto out_msi_available;
876 }
877
878 sc->sc_intr_num = handler_count;
879 sc->sc_intr_config = B_FALSE;
880 if (config_handler != NULL) {
881 sc->sc_intr_config = B_TRUE;
882 }
883
884 /* Assume they are all same priority */
885 ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
886 if (ret != DDI_SUCCESS) {
887 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
888 goto out_msi_prio;
889 }
890
891 /* Add the vq handlers */
892 for (i = 0; vq_handlers[i].vh_func; i++) {
893 ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
894 vq_handlers[i].vh_func, sc, vq_handlers[i].vh_priv);
895 if (ret != DDI_SUCCESS) {
896 dev_err(sc->sc_dev, CE_WARN,
897 "ddi_intr_add_handler failed");
898 /* Remove the handlers that succeeded. */
899 while (--i >= 0) {
900 (void) ddi_intr_remove_handler(
901 sc->sc_intr_htable[i]);
902 }
903 goto out_add_handlers;
904 }
905 }
906
907 /* Don't forget the config handler */
908 if (config_handler != NULL) {
909 ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
910 config_handler->vh_func, sc, config_handler->vh_priv);
911 if (ret != DDI_SUCCESS) {
912 dev_err(sc->sc_dev, CE_WARN,
913 "ddi_intr_add_handler failed");
914 /* Remove the handlers that succeeded. */
915 while (--i >= 0) {
916 (void) ddi_intr_remove_handler(
917 sc->sc_intr_htable[i]);
918 }
919 goto out_add_handlers;
920 }
921 }
922
923 ret = ddi_intr_get_cap(sc->sc_intr_htable[0], &sc->sc_intr_cap);
924 if (ret == DDI_SUCCESS) {
925 sc->sc_int_type = int_type;
926 return (DDI_SUCCESS);
927 }
928
929 out_add_handlers:
930 out_msi_prio:
931 out_msi_available:
932 for (i = 0; i < actual; i++)
933 (void) ddi_intr_free(sc->sc_intr_htable[i]);
934 out_msi_alloc:
935 kmem_free(sc->sc_intr_htable,
936 sizeof (ddi_intr_handle_t) * handler_count);
937
938 return (ret);
939 }
940
941 struct virtio_handler_container {
942 int nhandlers;
943 struct virtio_int_handler config_handler;
944 struct virtio_int_handler vq_handlers[];
945 };
946
947 uint_t
virtio_intx_dispatch(caddr_t arg1,caddr_t arg2)948 virtio_intx_dispatch(caddr_t arg1, caddr_t arg2)
949 {
950 struct virtio_softc *sc = (void *)arg1;
951 struct virtio_handler_container *vhc = (void *)arg2;
952 uint8_t isr_status;
953 int i;
954
955 isr_status = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
956 VIRTIO_CONFIG_ISR_STATUS));
957
958 if (!isr_status)
959 return (DDI_INTR_UNCLAIMED);
960
961 if ((isr_status & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
962 vhc->config_handler.vh_func) {
963 vhc->config_handler.vh_func((void *)sc,
964 vhc->config_handler.vh_priv);
965 }
966
967 /* Notify all handlers */
968 for (i = 0; i < vhc->nhandlers; i++) {
969 vhc->vq_handlers[i].vh_func((void *)sc,
970 vhc->vq_handlers[i].vh_priv);
971 }
972
973 return (DDI_INTR_CLAIMED);
974 }
975
976 /*
977 * config_handler and vq_handlers may be allocated on stack.
978 * Take precautions not to loose them.
979 */
980 static int
virtio_register_intx(struct virtio_softc * sc,struct virtio_int_handler * config_handler,struct virtio_int_handler vq_handlers[])981 virtio_register_intx(struct virtio_softc *sc,
982 struct virtio_int_handler *config_handler,
983 struct virtio_int_handler vq_handlers[])
984 {
985 int vq_handler_count;
986 int config_handler_count = 0;
987 int actual;
988 struct virtio_handler_container *vhc;
989 int ret = DDI_FAILURE;
990
991 /* Walk the handler table to get the number of handlers. */
992 for (vq_handler_count = 0;
993 vq_handlers && vq_handlers[vq_handler_count].vh_func;
994 vq_handler_count++)
995 ;
996
997 if (config_handler != NULL)
998 config_handler_count = 1;
999
1000 vhc = kmem_zalloc(sizeof (struct virtio_handler_container) +
1001 sizeof (struct virtio_int_handler) * vq_handler_count, KM_SLEEP);
1002
1003 vhc->nhandlers = vq_handler_count;
1004 (void) memcpy(vhc->vq_handlers, vq_handlers,
1005 sizeof (struct virtio_int_handler) * vq_handler_count);
1006
1007 if (config_handler != NULL) {
1008 (void) memcpy(&vhc->config_handler, config_handler,
1009 sizeof (struct virtio_int_handler));
1010 }
1011
1012 /* Just a single entry for a single interrupt. */
1013 sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP);
1014
1015 ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable,
1016 DDI_INTR_TYPE_FIXED, 0, 1, &actual, DDI_INTR_ALLOC_NORMAL);
1017 if (ret != DDI_SUCCESS) {
1018 dev_err(sc->sc_dev, CE_WARN,
1019 "Failed to allocate a fixed interrupt: %d", ret);
1020 goto out_int_alloc;
1021 }
1022
1023 ASSERT(actual == 1);
1024 sc->sc_intr_num = 1;
1025
1026 ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
1027 if (ret != DDI_SUCCESS) {
1028 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
1029 goto out_prio;
1030 }
1031
1032 ret = ddi_intr_add_handler(sc->sc_intr_htable[0],
1033 virtio_intx_dispatch, sc, vhc);
1034 if (ret != DDI_SUCCESS) {
1035 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_add_handler failed");
1036 goto out_add_handlers;
1037 }
1038
1039 sc->sc_int_type = DDI_INTR_TYPE_FIXED;
1040
1041 return (DDI_SUCCESS);
1042
1043 out_add_handlers:
1044 out_prio:
1045 (void) ddi_intr_free(sc->sc_intr_htable[0]);
1046 out_int_alloc:
1047 kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t));
1048 kmem_free(vhc, sizeof (struct virtio_int_handler) *
1049 (vq_handler_count + config_handler_count));
1050 return (ret);
1051 }
1052
1053 /*
1054 * We find out if we support MSI during this, and the register layout
1055 * depends on the MSI (doh). Don't acces the device specific bits in
1056 * BAR 0 before calling it!
1057 */
1058 int
virtio_register_ints(struct virtio_softc * sc,struct virtio_int_handler * config_handler,struct virtio_int_handler vq_handlers[])1059 virtio_register_ints(struct virtio_softc *sc,
1060 struct virtio_int_handler *config_handler,
1061 struct virtio_int_handler vq_handlers[])
1062 {
1063 int ret;
1064 int intr_types;
1065
1066 /* Default offset until MSI-X is enabled, if ever. */
1067 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
1068
1069 /* Determine which types of interrupts are supported */
1070 ret = ddi_intr_get_supported_types(sc->sc_dev, &intr_types);
1071 if (ret != DDI_SUCCESS) {
1072 dev_err(sc->sc_dev, CE_WARN, "Can't get supported int types");
1073 goto out_inttype;
1074 }
1075
1076 /* If we have msi, let's use them. */
1077 if (intr_types & (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
1078 ret = virtio_register_msi(sc, config_handler,
1079 vq_handlers, intr_types);
1080 if (!ret)
1081 return (0);
1082 }
1083
1084 /* Fall back to old-fashioned interrupts. */
1085 if (intr_types & DDI_INTR_TYPE_FIXED) {
1086 dev_debug(sc->sc_dev, CE_WARN,
1087 "Using legacy interrupts");
1088
1089 return (virtio_register_intx(sc, config_handler, vq_handlers));
1090 }
1091
1092 dev_err(sc->sc_dev, CE_WARN,
1093 "MSI failed and fixed interrupts not supported. Giving up.");
1094 ret = DDI_FAILURE;
1095
1096 out_inttype:
1097 return (ret);
1098 }
1099
1100 static int
virtio_enable_msi(struct virtio_softc * sc)1101 virtio_enable_msi(struct virtio_softc *sc)
1102 {
1103 int ret, i;
1104 int vq_handler_count = sc->sc_intr_num;
1105
1106 /* Number of handlers, not counting the counfig. */
1107 if (sc->sc_intr_config)
1108 vq_handler_count--;
1109
1110 /* Enable the interrupts. Either the whole block, or one by one. */
1111 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1112 ret = ddi_intr_block_enable(sc->sc_intr_htable,
1113 sc->sc_intr_num);
1114 if (ret != DDI_SUCCESS) {
1115 dev_err(sc->sc_dev, CE_WARN,
1116 "Failed to enable MSI, falling back to INTx");
1117 goto out_enable;
1118 }
1119 } else {
1120 for (i = 0; i < sc->sc_intr_num; i++) {
1121 ret = ddi_intr_enable(sc->sc_intr_htable[i]);
1122 if (ret != DDI_SUCCESS) {
1123 dev_err(sc->sc_dev, CE_WARN,
1124 "Failed to enable MSI %d, "
1125 "falling back to INTx", i);
1126
1127 while (--i >= 0) {
1128 (void) ddi_intr_disable(
1129 sc->sc_intr_htable[i]);
1130 }
1131 goto out_enable;
1132 }
1133 }
1134 }
1135
1136 /* Bind the allocated MSI to the queues and config */
1137 for (i = 0; i < vq_handler_count; i++) {
1138 int check;
1139
1140 ddi_put16(sc->sc_ioh,
1141 /* LINTED E_BAD_PTR_CAST_ALIGN */
1142 (uint16_t *)(sc->sc_io_addr +
1143 VIRTIO_CONFIG_QUEUE_SELECT), i);
1144
1145 ddi_put16(sc->sc_ioh,
1146 /* LINTED E_BAD_PTR_CAST_ALIGN */
1147 (uint16_t *)(sc->sc_io_addr +
1148 VIRTIO_CONFIG_QUEUE_VECTOR), i);
1149
1150 check = ddi_get16(sc->sc_ioh,
1151 /* LINTED E_BAD_PTR_CAST_ALIGN */
1152 (uint16_t *)(sc->sc_io_addr +
1153 VIRTIO_CONFIG_QUEUE_VECTOR));
1154 if (check != i) {
1155 dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
1156 "for VQ %d, MSI %d. Check = %x", i, i, check);
1157 ret = ENODEV;
1158 goto out_bind;
1159 }
1160 }
1161
1162 if (sc->sc_intr_config) {
1163 int check;
1164
1165 ddi_put16(sc->sc_ioh,
1166 /* LINTED E_BAD_PTR_CAST_ALIGN */
1167 (uint16_t *)(sc->sc_io_addr +
1168 VIRTIO_CONFIG_CONFIG_VECTOR), i);
1169
1170 check = ddi_get16(sc->sc_ioh,
1171 /* LINTED E_BAD_PTR_CAST_ALIGN */
1172 (uint16_t *)(sc->sc_io_addr +
1173 VIRTIO_CONFIG_CONFIG_VECTOR));
1174 if (check != i) {
1175 dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
1176 "for Config updates, MSI %d", i);
1177 ret = ENODEV;
1178 goto out_bind;
1179 }
1180 }
1181
1182 /* Configuration offset depends on whether MSI-X is used. */
1183 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX)
1184 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSIX;
1185 else
1186 ASSERT(sc->sc_int_type == DDI_INTR_TYPE_MSI);
1187
1188 return (DDI_SUCCESS);
1189
1190 out_bind:
1191 /* Unbind the vqs */
1192 for (i = 0; i < vq_handler_count - 1; i++) {
1193 ddi_put16(sc->sc_ioh,
1194 /* LINTED E_BAD_PTR_CAST_ALIGN */
1195 (uint16_t *)(sc->sc_io_addr +
1196 VIRTIO_CONFIG_QUEUE_SELECT), i);
1197
1198 ddi_put16(sc->sc_ioh,
1199 /* LINTED E_BAD_PTR_CAST_ALIGN */
1200 (uint16_t *)(sc->sc_io_addr +
1201 VIRTIO_CONFIG_QUEUE_VECTOR),
1202 VIRTIO_MSI_NO_VECTOR);
1203 }
1204 /* And the config */
1205 /* LINTED E_BAD_PTR_CAST_ALIGN */
1206 ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
1207 VIRTIO_CONFIG_CONFIG_VECTOR), VIRTIO_MSI_NO_VECTOR);
1208
1209 /* Disable the interrupts. Either the whole block, or one by one. */
1210 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1211 ret = ddi_intr_block_disable(sc->sc_intr_htable,
1212 sc->sc_intr_num);
1213 if (ret != DDI_SUCCESS) {
1214 dev_err(sc->sc_dev, CE_WARN,
1215 "Failed to disable MSIs, won't be able to "
1216 "reuse next time");
1217 }
1218 } else {
1219 for (i = 0; i < sc->sc_intr_num; i++) {
1220 ret = ddi_intr_disable(sc->sc_intr_htable[i]);
1221 if (ret != DDI_SUCCESS) {
1222 dev_err(sc->sc_dev, CE_WARN,
1223 "Failed to disable interrupt %d, "
1224 "won't be able to reuse", i);
1225 }
1226 }
1227 }
1228
1229 ret = DDI_FAILURE;
1230
1231 out_enable:
1232 return (ret);
1233 }
1234
1235 static int
virtio_enable_intx(struct virtio_softc * sc)1236 virtio_enable_intx(struct virtio_softc *sc)
1237 {
1238 int ret;
1239
1240 ret = ddi_intr_enable(sc->sc_intr_htable[0]);
1241 if (ret != DDI_SUCCESS) {
1242 dev_err(sc->sc_dev, CE_WARN,
1243 "Failed to enable interrupt: %d", ret);
1244 }
1245
1246 return (ret);
1247 }
1248
1249 /*
1250 * We can't enable/disable individual handlers in the INTx case so do
1251 * the whole bunch even in the msi case.
1252 */
1253 int
virtio_enable_ints(struct virtio_softc * sc)1254 virtio_enable_ints(struct virtio_softc *sc)
1255 {
1256
1257 ASSERT(sc->sc_config_offset == VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX);
1258
1259 /* See if we are using MSI. */
1260 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
1261 sc->sc_int_type == DDI_INTR_TYPE_MSI)
1262 return (virtio_enable_msi(sc));
1263
1264 ASSERT(sc->sc_int_type == DDI_INTR_TYPE_FIXED);
1265 return (virtio_enable_intx(sc));
1266 }
1267
1268 void
virtio_release_ints(struct virtio_softc * sc)1269 virtio_release_ints(struct virtio_softc *sc)
1270 {
1271 int i;
1272 int ret;
1273
1274 /* We were running with MSI, unbind them. */
1275 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
1276 sc->sc_int_type == DDI_INTR_TYPE_MSI) {
1277 /* Unbind all vqs */
1278 for (i = 0; i < sc->sc_nvqs; i++) {
1279 ddi_put16(sc->sc_ioh,
1280 /* LINTED E_BAD_PTR_CAST_ALIGN */
1281 (uint16_t *)(sc->sc_io_addr +
1282 VIRTIO_CONFIG_QUEUE_SELECT), i);
1283
1284 ddi_put16(sc->sc_ioh,
1285 /* LINTED E_BAD_PTR_CAST_ALIGN */
1286 (uint16_t *)(sc->sc_io_addr +
1287 VIRTIO_CONFIG_QUEUE_VECTOR),
1288 VIRTIO_MSI_NO_VECTOR);
1289 }
1290 /* And the config */
1291 /* LINTED E_BAD_PTR_CAST_ALIGN */
1292 ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
1293 VIRTIO_CONFIG_CONFIG_VECTOR),
1294 VIRTIO_MSI_NO_VECTOR);
1295
1296 }
1297
1298 /* Disable the interrupts. Either the whole block, or one by one. */
1299 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1300 ret = ddi_intr_block_disable(sc->sc_intr_htable,
1301 sc->sc_intr_num);
1302 if (ret != DDI_SUCCESS) {
1303 dev_err(sc->sc_dev, CE_WARN,
1304 "Failed to disable MSIs, won't be able to "
1305 "reuse next time");
1306 }
1307 } else {
1308 for (i = 0; i < sc->sc_intr_num; i++) {
1309 ret = ddi_intr_disable(sc->sc_intr_htable[i]);
1310 if (ret != DDI_SUCCESS) {
1311 dev_err(sc->sc_dev, CE_WARN,
1312 "Failed to disable interrupt %d, "
1313 "won't be able to reuse", i);
1314 }
1315 }
1316 }
1317
1318
1319 for (i = 0; i < sc->sc_intr_num; i++) {
1320 (void) ddi_intr_remove_handler(sc->sc_intr_htable[i]);
1321 }
1322
1323 for (i = 0; i < sc->sc_intr_num; i++)
1324 (void) ddi_intr_free(sc->sc_intr_htable[i]);
1325
1326 kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t) *
1327 sc->sc_intr_num);
1328
1329 /* After disabling interrupts, the config offset is non-MSI-X. */
1330 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
1331 }
1332
1333 /*
1334 * Module linkage information for the kernel.
1335 */
1336 static struct modlmisc modlmisc = {
1337 &mod_miscops, /* Type of module */
1338 "VirtIO common library module",
1339 };
1340
1341 static struct modlinkage modlinkage = {
1342 MODREV_1,
1343 {
1344 (void *)&modlmisc,
1345 NULL
1346 }
1347 };
1348
1349 int
_init(void)1350 _init(void)
1351 {
1352 return (mod_install(&modlinkage));
1353 }
1354
1355 int
_fini(void)1356 _fini(void)
1357 {
1358 return (mod_remove(&modlinkage));
1359 }
1360
1361 int
_info(struct modinfo * modinfop)1362 _info(struct modinfo *modinfop)
1363 {
1364 return (mod_info(&modlinkage, modinfop));
1365 }
1366