61 #define	DEV_SOFTC(vs) ((void *)(vs))  argument
68 vi_softc_linkup(struct virtio_softc *vs, struct virtio_consts *vc,  in vi_softc_linkup()  argument
74 	/* vs and dev_softc addresses must match */  in vi_softc_linkup()
75 	assert((void *)vs == dev_softc);  in vi_softc_linkup()
76 	vs->vs_vc = vc;  in vi_softc_linkup()
77 	vs->vs_pi = pi;  in vi_softc_linkup()
78 	pi->pi_arg = vs;  in vi_softc_linkup()
80 	vs->vs_queues = queues;  in vi_softc_linkup()
82 		queues[i].vq_vs = vs;  in vi_softc_linkup()
97 vi_reset_dev(struct virtio_softc *vs)  in vi_reset_dev()  argument
102 	if (vs->vs_mtx)  in vi_reset_dev()
103 		assert(pthread_mutex_isowned_np(vs->vs_mtx));  in vi_reset_dev()
105 	nvq = vs->vs_vc->vc_nvq;  in vi_reset_dev()
106 	for (vq = vs->vs_queues, i = 0; i < nvq; vq++, i++) {  in vi_reset_dev()
114 	vs->vs_negotiated_caps = 0;  in vi_reset_dev()
115 	vs->vs_curq = 0;  in vi_reset_dev()
116 	/* vs->vs_status = 0; -- redundant */  in vi_reset_dev()
117 	if (vs->vs_isr)  in vi_reset_dev()
118 		pci_lintr_deassert(vs->vs_pi);  in vi_reset_dev()
119 	vs->vs_isr = 0;  in vi_reset_dev()
120 	vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR;  in vi_reset_dev()
127 vi_set_io_bar(struct virtio_softc *vs, int barnum)  in vi_set_io_bar()  argument
135 	size = VIRTIO_PCI_CONFIG_OFF(1) + vs->vs_vc->vc_cfgsize;  in vi_set_io_bar()
136 	pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_IO, size);  in vi_set_io_bar()
147 vi_intr_init(struct virtio_softc *vs, int barnum, int use_msix)  in vi_intr_init()  argument
152 		vs->vs_flags |= VIRTIO_USE_MSIX;  in vi_intr_init()
153 		VS_LOCK(vs);  in vi_intr_init()
154 		vi_reset_dev(vs); /* set all vectors to NO_VECTOR */  in vi_intr_init()
155 		VS_UNLOCK(vs);  in vi_intr_init()
156 		nvec = vs->vs_vc->vc_nvq + 1;  in vi_intr_init()
157 		if (pci_emul_add_msixcap(vs->vs_pi, nvec, barnum))  in vi_intr_init()
160 		vs->vs_flags &= ~VIRTIO_USE_MSIX;  in vi_intr_init()
163 	pci_emul_add_msicap(vs->vs_pi, 1);  in vi_intr_init()
166 	pci_lintr_request(vs->vs_pi);  in vi_intr_init()
172  * Initialize the currently-selected virtio queue (vs->vs_curq).
177 vi_vq_init(struct virtio_softc *vs, uint32_t pfn)  in vi_vq_init()  argument
184 	vq = &vs->vs_queues[vs->vs_curq];  in vi_vq_init()
188 	base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size);  in vi_vq_init()
251  * at vs->vs_pi) so that it can find indirect descriptors.
279 	struct virtio_softc *vs;  in vq_getchain()  local
282 	vs = vq->vq_vs;  in vq_getchain()
283 	name = vs->vs_vc->vc_name;  in vq_getchain()
319 	ctx = vs->vs_pi->pi_vmctx;  in vq_getchain()
334 		} else if ((vs->vs_vc->vc_hv_caps &  in vq_getchain()
477 	struct virtio_softc *vs;  in vq_endchains()  local
490 	vs = vq->vq_vs;  in vq_endchains()
501 	    (vs->vs_negotiated_caps & VIRTIO_F_NOTIFY_ON_EMPTY))  in vq_endchains()
503 	else if (vs->vs_negotiated_caps & VIRTIO_RING_F_EVENT_IDX) {  in vq_endchains()
516 		vq_interrupt(vs, vq);  in vq_endchains()
567 	struct virtio_softc *vs = pi->pi_arg;  in vi_pci_read()  local
576 	if (vs->vs_flags & VIRTIO_USE_MSIX) {  in vi_pci_read()
586 	if (vs->vs_mtx)  in vi_pci_read()
587 		pthread_mutex_lock(vs->vs_mtx);  in vi_pci_read()
589 	vc = vs->vs_vc;  in vi_pci_read()
609 			error = (*vc->vc_cfgread)(DEV_SOFTC(vs), newoff, size, &value);  in vi_pci_read()
637 		value = vs->vs_negotiated_caps;  in vi_pci_read()
640 		if (vs->vs_curq < vc->vc_nvq)  in vi_pci_read()
641 			value = vs->vs_queues[vs->vs_curq].vq_pfn;  in vi_pci_read()
644 		value = vs->vs_curq < vc->vc_nvq ?  in vi_pci_read()
645 		    vs->vs_queues[vs->vs_curq].vq_qsize : 0;  in vi_pci_read()
648 		value = vs->vs_curq;  in vi_pci_read()
654 		value = vs->vs_status;  in vi_pci_read()
657 		value = vs->vs_isr;  in vi_pci_read()
658 		vs->vs_isr = 0;		/* a read clears this flag */  in vi_pci_read()
663 		value = vs->vs_msix_cfg_idx;  in vi_pci_read()
666 		value = vs->vs_curq < vc->vc_nvq ?  in vi_pci_read()
667 		    vs->vs_queues[vs->vs_curq].vq_msix_idx :  in vi_pci_read()
672 	if (vs->vs_mtx)  in vi_pci_read()
673 		pthread_mutex_unlock(vs->vs_mtx);  in vi_pci_read()
687 	struct virtio_softc *vs = pi->pi_arg;  in vi_pci_write()  local
696 	if (vs->vs_flags & VIRTIO_USE_MSIX) {  in vi_pci_write()
707 	if (vs->vs_mtx)  in vi_pci_write()
708 		pthread_mutex_lock(vs->vs_mtx);  in vi_pci_write()
710 	vc = vs->vs_vc;  in vi_pci_write()
728 			error = (*vc->vc_cfgwrite)(DEV_SOFTC(vs), newoff, size, value);  in vi_pci_write()
758 		vs->vs_negotiated_caps = value & vc->vc_hv_caps;  in vi_pci_write()
760 			(*vc->vc_apply_features)(DEV_SOFTC(vs),  in vi_pci_write()
761 			    vs->vs_negotiated_caps);  in vi_pci_write()
764 		if (vs->vs_curq >= vc->vc_nvq)  in vi_pci_write()
766 		vi_vq_init(vs, value);  in vi_pci_write()
774 		vs->vs_curq = value;  in vi_pci_write()
782 		vq = &vs->vs_queues[value];  in vi_pci_write()
784 			(*vq->vq_notify)(DEV_SOFTC(vs), vq);  in vi_pci_write()
786 			(*vc->vc_qnotify)(DEV_SOFTC(vs), vq);  in vi_pci_write()
793 		vs->vs_status = value;  in vi_pci_write()
795 			(*vc->vc_reset)(DEV_SOFTC(vs));  in vi_pci_write()
798 		vs->vs_msix_cfg_idx = value;  in vi_pci_write()
801 		if (vs->vs_curq >= vc->vc_nvq)  in vi_pci_write()
803 		vq = &vs->vs_queues[vs->vs_curq];  in vi_pci_write()
812 	    name, cr->cr_name, vs->vs_curq, vc->vc_nvq);  in vi_pci_write()
814 	if (vs->vs_mtx)  in vi_pci_write()
815 		pthread_mutex_unlock(vs->vs_mtx);  in vi_pci_write()
822 	struct virtio_softc *vs;  in vi_pci_pause()  local
825 	vs = pi->pi_arg;  in vi_pci_pause()
826 	vc = vs->vs_vc;  in vi_pci_pause()
828 	vc = vs->vs_vc;  in vi_pci_pause()
830 	(*vc->vc_pause)(DEV_SOFTC(vs));  in vi_pci_pause()
838 	struct virtio_softc *vs;  in vi_pci_resume()  local
841 	vs = pi->pi_arg;  in vi_pci_resume()
842 	vc = vs->vs_vc;  in vi_pci_resume()
844 	vc = vs->vs_vc;  in vi_pci_resume()
846 	(*vc->vc_resume)(DEV_SOFTC(vs));  in vi_pci_resume()
852 vi_pci_snapshot_softc(struct virtio_softc *vs, struct vm_snapshot_meta *meta)  in vi_pci_snapshot_softc()  argument
856 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_flags, meta, ret, done);  in vi_pci_snapshot_softc()
857 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_negotiated_caps, meta, ret, done);  in vi_pci_snapshot_softc()
858 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_curq, meta, ret, done);  in vi_pci_snapshot_softc()
859 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_status, meta, ret, done);  in vi_pci_snapshot_softc()
860 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_isr, meta, ret, done);  in vi_pci_snapshot_softc()
861 	SNAPSHOT_VAR_OR_LEAVE(vs->vs_msix_cfg_idx, meta, ret, done);  in vi_pci_snapshot_softc()
881 vi_pci_snapshot_queues(struct virtio_softc *vs, struct vm_snapshot_meta *meta)  in vi_pci_snapshot_queues()  argument
890 	ctx = vs->vs_pi->pi_vmctx;  in vi_pci_snapshot_queues()
891 	vc = vs->vs_vc;  in vi_pci_snapshot_queues()
895 		vq = &vs->vs_queues[i];  in vi_pci_snapshot_queues()
936 	struct virtio_softc *vs;  in vi_pci_snapshot()  local
940 	vs = pi->pi_arg;  in vi_pci_snapshot()
941 	vc = vs->vs_vc;  in vi_pci_snapshot()
944 	ret = vi_pci_snapshot_softc(vs, meta);  in vi_pci_snapshot()
954 	ret = vi_pci_snapshot_queues(vs, meta);  in vi_pci_snapshot()
960 		ret = (*vc->vc_snapshot)(DEV_SOFTC(vs), meta);  in vi_pci_snapshot()