Lines Matching +full:atomic +full:- +full:threshold +full:- +full:us
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
33 #include <machine/atomic.h>
58 * front of virtio-based device softc" constraint, let's use
76 vs->vs_vc = vc; in vi_softc_linkup()
77 vs->vs_pi = pi; in vi_softc_linkup()
78 pi->pi_arg = vs; in vi_softc_linkup()
80 vs->vs_queues = queues; in vi_softc_linkup()
81 for (i = 0; i < vc->vc_nvq; i++) { in vi_softc_linkup()
88 * Reset device (device-wide). This erases all queues, i.e.,
94 * If MSI-X is enabled, this also resets all the vectors to NO_VECTOR.
102 if (vs->vs_mtx) in vi_reset_dev()
103 assert(pthread_mutex_isowned_np(vs->vs_mtx)); in vi_reset_dev()
105 nvq = vs->vs_vc->vc_nvq; in vi_reset_dev()
106 for (vq = vs->vs_queues, i = 0; i < nvq; vq++, i++) { in vi_reset_dev()
107 vq->vq_flags = 0; in vi_reset_dev()
108 vq->vq_last_avail = 0; in vi_reset_dev()
109 vq->vq_next_used = 0; in vi_reset_dev()
110 vq->vq_save_used = 0; in vi_reset_dev()
111 vq->vq_pfn = 0; in vi_reset_dev()
112 vq->vq_msix_idx = VIRTIO_MSI_NO_VECTOR; in vi_reset_dev()
114 vs->vs_negotiated_caps = 0; in vi_reset_dev()
115 vs->vs_curq = 0; in vi_reset_dev()
116 /* vs->vs_status = 0; -- redundant */ in vi_reset_dev()
117 if (vs->vs_isr) in vi_reset_dev()
118 pci_lintr_deassert(vs->vs_pi); in vi_reset_dev()
119 vs->vs_isr = 0; in vi_reset_dev()
120 vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR; in vi_reset_dev()
132 * ??? should we use VIRTIO_PCI_CONFIG_OFF(0) if MSI-X is disabled? in vi_set_io_bar()
135 size = VIRTIO_PCI_CONFIG_OFF(1) + vs->vs_vc->vc_cfgsize; in vi_set_io_bar()
136 pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_IO, size); in vi_set_io_bar()
140 * Initialize MSI-X vector capabilities if we're to use MSI-X,
143 * We assume we want one MSI-X vector per queue, here, plus one
152 vs->vs_flags |= VIRTIO_USE_MSIX; in vi_intr_init()
156 nvec = vs->vs_vc->vc_nvq + 1; in vi_intr_init()
157 if (pci_emul_add_msixcap(vs->vs_pi, nvec, barnum)) in vi_intr_init()
160 vs->vs_flags &= ~VIRTIO_USE_MSIX; in vi_intr_init()
163 pci_emul_add_msicap(vs->vs_pi, 1); in vi_intr_init()
166 pci_lintr_request(vs->vs_pi); in vi_intr_init()
172 * Initialize the currently-selected virtio queue (vs->vs_curq).
173 * The guest just gave us a page frame number, from which we can
184 vq = &vs->vs_queues[vs->vs_curq]; in vi_vq_init()
185 vq->vq_pfn = pfn; in vi_vq_init()
187 size = vring_size_aligned(vq->vq_qsize); in vi_vq_init()
188 base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); in vi_vq_init()
191 vq->vq_desc = (struct vring_desc *)base; in vi_vq_init()
192 base += vq->vq_qsize * sizeof(struct vring_desc); in vi_vq_init()
195 vq->vq_avail = (struct vring_avail *)base; in vi_vq_init()
196 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); in vi_vq_init()
202 vq->vq_used = (struct vring_used *)base; in vi_vq_init()
205 vq->vq_flags = VQ_ALLOC; in vi_vq_init()
206 vq->vq_last_avail = 0; in vi_vq_init()
207 vq->vq_next_used = 0; in vi_vq_init()
208 vq->vq_save_used = 0; in vi_vq_init()
224 len = atomic_load_32(&vd->len); in _vq_record()
225 addr = atomic_load_64(&vd->addr); in _vq_record()
228 if ((vd->flags & VRING_DESC_F_WRITE) == 0) in _vq_record()
229 reqp->readable++; in _vq_record()
231 reqp->writable++; in _vq_record()
251 * at vs->vs_pi) so that it can find indirect descriptors.
257 * up to VQ_MAX_DESCRIPTORS, before giving up and returning -1.
264 * and returns -1. If no descriptors are ready now it simply returns 0.
282 vs = vq->vq_vs; in vq_getchain()
283 name = vs->vs_vc->vc_name; in vq_getchain()
288 * update vq->vq_avail->idx until all of the descriptors in vq_getchain()
292 * Compute (vq_avail->idx - last_avail) in integers mod 2**16. This is in vq_getchain()
294 * since the last time we updated vq->vq_last_avail. in vq_getchain()
299 idx = vq->vq_last_avail; in vq_getchain()
300 ndesc = (uint16_t)((u_int)vq->vq_avail->idx - idx); in vq_getchain()
303 if (ndesc > vq->vq_qsize) { in vq_getchain()
308 return (-1); in vq_getchain()
316 * check whether we're re-visiting a previously visited in vq_getchain()
319 ctx = vs->vs_pi->pi_vmctx; in vq_getchain()
320 req.idx = next = vq->vq_avail->ring[idx & (vq->vq_qsize - 1)]; in vq_getchain()
321 vq->vq_last_avail++; in vq_getchain()
322 for (i = 0; i < VQ_MAX_DESCRIPTORS; next = vdir->next) { in vq_getchain()
323 if (next >= vq->vq_qsize) { in vq_getchain()
328 return (-1); in vq_getchain()
330 vdir = &vq->vq_desc[next]; in vq_getchain()
331 if ((vdir->flags & VRING_DESC_F_INDIRECT) == 0) { in vq_getchain()
334 } else if ((vs->vs_vc->vc_hv_caps & in vq_getchain()
340 return (-1); in vq_getchain()
342 n_indir = vdir->len / 16; in vq_getchain()
343 if ((vdir->len & 0xf) || n_indir == 0) { in vq_getchain()
347 name, (u_int)vdir->len); in vq_getchain()
348 return (-1); in vq_getchain()
351 vdir->addr, vdir->len); in vq_getchain()
362 if (vp->flags & VRING_DESC_F_INDIRECT) { in vq_getchain()
367 return (-1); in vq_getchain()
372 if ((vp->flags & VRING_DESC_F_NEXT) == 0) in vq_getchain()
374 next = vp->next; in vq_getchain()
380 return (-1); in vq_getchain()
384 if ((vdir->flags & VRING_DESC_F_NEXT) == 0) in vq_getchain()
390 "%s: descriptor loop? count > %d - driver confused?", in vq_getchain()
392 return (-1); in vq_getchain()
409 vq->vq_last_avail -= n_chains; in vq_retchains()
421 * - mask is N-1 where N is a power of 2 so computes x % N in vq_relchain_prepare()
422 * - vuh points to the "used" data shared with guest in vq_relchain_prepare()
423 * - vue points to the "used" ring entry we want to update in vq_relchain_prepare()
425 mask = vq->vq_qsize - 1; in vq_relchain_prepare()
426 vuh = vq->vq_used; in vq_relchain_prepare()
428 vue = &vuh->ring[vq->vq_next_used++ & mask]; in vq_relchain_prepare()
429 vue->id = idx; in vq_relchain_prepare()
430 vue->len = iolen; in vq_relchain_prepare()
442 vq->vq_used->idx = vq->vq_next_used; in vq_relchain_publish()
471 * processing -- it's possible that descriptors became available after
483 * interrupt if we've crossed the event threshold. in vq_endchains()
490 vs = vq->vq_vs; in vq_endchains()
491 old_idx = vq->vq_save_used; in vq_endchains()
492 vq->vq_save_used = new_idx = vq->vq_used->idx; in vq_endchains()
501 (vs->vs_negotiated_caps & VIRTIO_F_NOTIFY_ON_EMPTY)) in vq_endchains()
503 else if (vs->vs_negotiated_caps & VIRTIO_RING_F_EVENT_IDX) { in vq_endchains()
509 intr = (uint16_t)(new_idx - event_idx - 1) < in vq_endchains()
510 (uint16_t)(new_idx - old_idx); in vq_endchains()
513 !(vq->vq_avail->flags & VRING_AVAIL_F_NO_INTERRUPT); in vq_endchains()
544 hi = sizeof(config_regs) / sizeof(*config_regs) - 1; in vi_find_cr()
548 if (cr->cr_offset == offset) in vi_find_cr()
550 if (cr->cr_offset < offset) in vi_find_cr()
553 hi = mid - 1; in vi_find_cr()
560 * If it's to the MSI-X info, do that.
567 struct virtio_softc *vs = pi->pi_arg; in vi_pci_read()
576 if (vs->vs_flags & VIRTIO_USE_MSIX) { in vi_pci_read()
586 if (vs->vs_mtx) in vi_pci_read()
587 pthread_mutex_lock(vs->vs_mtx); in vi_pci_read()
589 vc = vs->vs_vc; in vi_pci_read()
590 name = vc->vc_name; in vi_pci_read()
600 * Subtract off the standard size (including MSI-X in vi_pci_read()
604 newoff = offset - virtio_config_size; in vi_pci_read()
605 max = vc->vc_cfgsize ? vc->vc_cfgsize : 0x100000000; in vi_pci_read()
608 if (vc->vc_cfgread != NULL) in vi_pci_read()
609 error = (*vc->vc_cfgread)(DEV_SOFTC(vs), newoff, size, &value); in vi_pci_read()
618 if (cr == NULL || cr->cr_size != size) { in vi_pci_read()
623 name, cr->cr_name, size); in vi_pci_read()
634 value = vc->vc_hv_caps; in vi_pci_read()
637 value = vs->vs_negotiated_caps; in vi_pci_read()
640 if (vs->vs_curq < vc->vc_nvq) in vi_pci_read()
641 value = vs->vs_queues[vs->vs_curq].vq_pfn; in vi_pci_read()
644 value = vs->vs_curq < vc->vc_nvq ? in vi_pci_read()
645 vs->vs_queues[vs->vs_curq].vq_qsize : 0; in vi_pci_read()
648 value = vs->vs_curq; in vi_pci_read()
654 value = vs->vs_status; in vi_pci_read()
657 value = vs->vs_isr; in vi_pci_read()
658 vs->vs_isr = 0; /* a read clears this flag */ in vi_pci_read()
663 value = vs->vs_msix_cfg_idx; in vi_pci_read()
666 value = vs->vs_curq < vc->vc_nvq ? in vi_pci_read()
667 vs->vs_queues[vs->vs_curq].vq_msix_idx : in vi_pci_read()
672 if (vs->vs_mtx) in vi_pci_read()
673 pthread_mutex_unlock(vs->vs_mtx); in vi_pci_read()
679 * If it's to the MSI-X info, do that.
687 struct virtio_softc *vs = pi->pi_arg; in vi_pci_write()
696 if (vs->vs_flags & VIRTIO_USE_MSIX) { in vi_pci_write()
707 if (vs->vs_mtx) in vi_pci_write()
708 pthread_mutex_lock(vs->vs_mtx); in vi_pci_write()
710 vc = vs->vs_vc; in vi_pci_write()
711 name = vc->vc_name; in vi_pci_write()
720 * Subtract off the standard size (including MSI-X in vi_pci_write()
723 newoff = offset - virtio_config_size; in vi_pci_write()
724 max = vc->vc_cfgsize ? vc->vc_cfgsize : 0x100000000; in vi_pci_write()
727 if (vc->vc_cfgwrite != NULL) in vi_pci_write()
728 error = (*vc->vc_cfgwrite)(DEV_SOFTC(vs), newoff, size, value); in vi_pci_write()
737 if (cr == NULL || cr->cr_size != size || cr->cr_ro) { in vi_pci_write()
740 if (cr->cr_size != size) in vi_pci_write()
743 name, cr->cr_name, size); in vi_pci_write()
744 if (cr->cr_ro) in vi_pci_write()
746 "%s: write to read-only reg %s", in vi_pci_write()
747 name, cr->cr_name); in vi_pci_write()
758 vs->vs_negotiated_caps = value & vc->vc_hv_caps; in vi_pci_write()
759 if (vc->vc_apply_features) in vi_pci_write()
760 (*vc->vc_apply_features)(DEV_SOFTC(vs), in vi_pci_write()
761 vs->vs_negotiated_caps); in vi_pci_write()
764 if (vs->vs_curq >= vc->vc_nvq) in vi_pci_write()
774 vs->vs_curq = value; in vi_pci_write()
777 if (value >= (unsigned int)vc->vc_nvq) { in vi_pci_write()
782 vq = &vs->vs_queues[value]; in vi_pci_write()
783 if (vq->vq_notify) in vi_pci_write()
784 (*vq->vq_notify)(DEV_SOFTC(vs), vq); in vi_pci_write()
785 else if (vc->vc_qnotify) in vi_pci_write()
786 (*vc->vc_qnotify)(DEV_SOFTC(vs), vq); in vi_pci_write()
793 vs->vs_status = value; in vi_pci_write()
795 (*vc->vc_reset)(DEV_SOFTC(vs)); in vi_pci_write()
798 vs->vs_msix_cfg_idx = value; in vi_pci_write()
801 if (vs->vs_curq >= vc->vc_nvq) in vi_pci_write()
803 vq = &vs->vs_queues[vs->vs_curq]; in vi_pci_write()
804 vq->vq_msix_idx = value; in vi_pci_write()
812 name, cr->cr_name, vs->vs_curq, vc->vc_nvq); in vi_pci_write()
814 if (vs->vs_mtx) in vi_pci_write()
815 pthread_mutex_unlock(vs->vs_mtx); in vi_pci_write()
825 vs = pi->pi_arg; in vi_pci_pause()
826 vc = vs->vs_vc; in vi_pci_pause()
828 vc = vs->vs_vc; in vi_pci_pause()
829 assert(vc->vc_pause != NULL); in vi_pci_pause()
830 (*vc->vc_pause)(DEV_SOFTC(vs)); in vi_pci_pause()
841 vs = pi->pi_arg; in vi_pci_resume()
842 vc = vs->vs_vc; in vi_pci_resume()
844 vc = vs->vs_vc; in vi_pci_resume()
845 assert(vc->vc_resume != NULL); in vi_pci_resume()
846 (*vc->vc_resume)(DEV_SOFTC(vs)); in vi_pci_resume()
856 SNAPSHOT_VAR_OR_LEAVE(vs->vs_flags, meta, ret, done); in vi_pci_snapshot_softc()
857 SNAPSHOT_VAR_OR_LEAVE(vs->vs_negotiated_caps, meta, ret, done); in vi_pci_snapshot_softc()
858 SNAPSHOT_VAR_OR_LEAVE(vs->vs_curq, meta, ret, done); in vi_pci_snapshot_softc()
859 SNAPSHOT_VAR_OR_LEAVE(vs->vs_status, meta, ret, done); in vi_pci_snapshot_softc()
860 SNAPSHOT_VAR_OR_LEAVE(vs->vs_isr, meta, ret, done); in vi_pci_snapshot_softc()
861 SNAPSHOT_VAR_OR_LEAVE(vs->vs_msix_cfg_idx, meta, ret, done); in vi_pci_snapshot_softc()
872 SNAPSHOT_VAR_CMP_OR_LEAVE(vc->vc_nvq, meta, ret, done); in vi_pci_snapshot_consts()
873 SNAPSHOT_VAR_CMP_OR_LEAVE(vc->vc_cfgsize, meta, ret, done); in vi_pci_snapshot_consts()
874 SNAPSHOT_VAR_CMP_OR_LEAVE(vc->vc_hv_caps, meta, ret, done); in vi_pci_snapshot_consts()
890 ctx = vs->vs_pi->pi_vmctx; in vi_pci_snapshot_queues()
891 vc = vs->vs_vc; in vi_pci_snapshot_queues()
894 for (i = 0; i < vc->vc_nvq; i++) { in vi_pci_snapshot_queues()
895 vq = &vs->vs_queues[i]; in vi_pci_snapshot_queues()
897 SNAPSHOT_VAR_CMP_OR_LEAVE(vq->vq_qsize, meta, ret, done); in vi_pci_snapshot_queues()
898 SNAPSHOT_VAR_CMP_OR_LEAVE(vq->vq_num, meta, ret, done); in vi_pci_snapshot_queues()
900 SNAPSHOT_VAR_OR_LEAVE(vq->vq_flags, meta, ret, done); in vi_pci_snapshot_queues()
901 SNAPSHOT_VAR_OR_LEAVE(vq->vq_last_avail, meta, ret, done); in vi_pci_snapshot_queues()
902 SNAPSHOT_VAR_OR_LEAVE(vq->vq_next_used, meta, ret, done); in vi_pci_snapshot_queues()
903 SNAPSHOT_VAR_OR_LEAVE(vq->vq_save_used, meta, ret, done); in vi_pci_snapshot_queues()
904 SNAPSHOT_VAR_OR_LEAVE(vq->vq_msix_idx, meta, ret, done); in vi_pci_snapshot_queues()
906 SNAPSHOT_VAR_OR_LEAVE(vq->vq_pfn, meta, ret, done); in vi_pci_snapshot_queues()
911 addr_size = vq->vq_qsize * sizeof(struct vring_desc); in vi_pci_snapshot_queues()
912 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ctx, vq->vq_desc, addr_size, in vi_pci_snapshot_queues()
915 addr_size = (2 + vq->vq_qsize + 1) * sizeof(uint16_t); in vi_pci_snapshot_queues()
916 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ctx, vq->vq_avail, addr_size, in vi_pci_snapshot_queues()
919 addr_size = (2 + 2 * vq->vq_qsize + 1) * sizeof(uint16_t); in vi_pci_snapshot_queues()
920 SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ctx, vq->vq_used, addr_size, in vi_pci_snapshot_queues()
923 SNAPSHOT_BUF_OR_LEAVE(vq->vq_desc, in vi_pci_snapshot_queues()
924 vring_size_aligned(vq->vq_qsize), meta, ret, done); in vi_pci_snapshot_queues()
939 pi = meta->dev_data; in vi_pci_snapshot()
940 vs = pi->pi_arg; in vi_pci_snapshot()
941 vc = vs->vs_vc; in vi_pci_snapshot()
959 if (vc->vc_snapshot != NULL) { in vi_pci_snapshot()
960 ret = (*vc->vc_snapshot)(DEV_SOFTC(vs), meta); in vi_pci_snapshot()