157 	vioblk_req_t *vbr;  in vioblk_req_alloc()  local
161 	if ((vbr = list_remove_head(&vib->vib_reqs)) == NULL) {  in vioblk_req_alloc()
166 	VERIFY0(vbr->vbr_status);  in vioblk_req_alloc()
167 	vbr->vbr_status |= VIOBLK_REQSTAT_ALLOCATED;  in vioblk_req_alloc()
169 	VERIFY3P(vbr->vbr_chain, !=, NULL);  in vioblk_req_alloc()
170 	VERIFY3P(vbr->vbr_xfer, ==, NULL);  in vioblk_req_alloc()
171 	VERIFY3S(vbr->vbr_error, ==, 0);  in vioblk_req_alloc()
173 	return (vbr);  in vioblk_req_alloc()
177 vioblk_req_free(vioblk_t *vib, vioblk_req_t *vbr)  in vioblk_req_free()  argument
185 	VERIFY(vbr->vbr_status & VIOBLK_REQSTAT_ALLOCATED);  in vioblk_req_free()
186 	vbr->vbr_status = 0;  in vioblk_req_free()
188 	vbr->vbr_xfer = NULL;  in vioblk_req_free()
189 	vbr->vbr_error = 0;  in vioblk_req_free()
190 	vbr->vbr_type = 0;  in vioblk_req_free()
191 	virtio_chain_clear(vbr->vbr_chain);  in vioblk_req_free()
193 	list_insert_head(&vib->vib_reqs, vbr);  in vioblk_req_free()
200 vioblk_complete(vioblk_t *vib, vioblk_req_t *vbr)  in vioblk_complete()  argument
204 	VERIFY(!(vbr->vbr_status & VIOBLK_REQSTAT_COMPLETE));  in vioblk_complete()
205 	vbr->vbr_status |= VIOBLK_REQSTAT_COMPLETE;  in vioblk_complete()
207 	if (vbr->vbr_type == VIRTIO_BLK_T_FLUSH) {  in vioblk_complete()
211 	if (vbr->vbr_xfer != NULL) {  in vioblk_complete()
216 		bd_xfer_done(vbr->vbr_xfer, vbr->vbr_error);  in vioblk_complete()
218 		vbr->vbr_xfer = NULL;  in vioblk_complete()
226 	vioblk_req_t *vbr = NULL;  in vioblk_common_start()  local
228 	if ((vbr = vioblk_req_alloc(vib)) == NULL) {  in vioblk_common_start()
232 	vbr->vbr_type = type;  in vioblk_common_start()
239 		vbr->vbr_status |= VIOBLK_REQSTAT_POLLED;  in vioblk_common_start()
246 	bcopy(&vbh, virtio_dma_va(vbr->vbr_dma, 0), sizeof (vbh));  in vioblk_common_start()
252 	if (virtio_chain_append(vbr->vbr_chain,  in vioblk_common_start()
253 	    virtio_dma_cookie_pa(vbr->vbr_dma, 0),  in vioblk_common_start()
256 		vioblk_req_free(vib, vbr);  in vioblk_common_start()
260 	return (vbr);  in vioblk_common_start()
264 vioblk_common_submit(vioblk_t *vib, vioblk_req_t *vbr)  in vioblk_common_submit()  argument
266 	virtio_chain_t *vic = vbr->vbr_chain;  in vioblk_common_submit()
276 	if (virtio_chain_append(vic, virtio_dma_cookie_pa(vbr->vbr_dma, 0) +  in vioblk_common_submit()
279 		vioblk_req_free(vib, vbr);  in vioblk_common_submit()
283 	virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORDEV);  in vioblk_common_submit()
286 	if (!(vbr->vbr_status & VIOBLK_REQSTAT_POLLED)) {  in vioblk_common_submit()
298 	while (!(vbr->vbr_status & VIOBLK_REQSTAT_POLL_COMPLETE)) {  in vioblk_common_submit()
316 	vioblk_complete(vib, vbr);  in vioblk_common_submit()
317 	r = vbr->vbr_error;  in vioblk_common_submit()
318 	vioblk_req_free(vib, vbr);  in vioblk_common_submit()
326 	vioblk_req_t *vbr;  in vioblk_internal()  local
333 	if ((vbr = vioblk_common_start(vib, type, sector, B_TRUE)) == NULL) {  in vioblk_internal()
343 		virtio_chain_t *vic = vbr->vbr_chain;  in vioblk_internal()
349 				vioblk_req_free(vib, vbr);  in vioblk_internal()
355 	return (vioblk_common_submit(vib, vbr));  in vioblk_internal()
400 	vioblk_req_t *vbr = NULL;  in vioblk_request()  local
416 	if ((vbr = vioblk_common_start(vib, type, xfer->x_blkno, polled)) ==  in vioblk_request()
420 	vbr->vbr_xfer = xfer;  in vioblk_request()
431 		virtio_chain_t *vic = vbr->vbr_chain;  in vioblk_request()
447 				vioblk_req_free(vib, vbr);  in vioblk_request()
459 		int r = vioblk_map_discard(vib, vbr->vbr_chain, xfer);  in vioblk_request()
461 			vioblk_req_free(vib, vbr);  in vioblk_request()
470 	return (vioblk_common_submit(vib, vbr));  in vioblk_request()
687 		vioblk_req_t *vbr = virtio_chain_data(vic);  in vioblk_poll()  local
690 		virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORCPU);  in vioblk_poll()
692 		bcopy(virtio_dma_va(vbr->vbr_dma,  in vioblk_poll()
697 			vbr->vbr_error = 0;  in vioblk_poll()
700 			vbr->vbr_error = EIO;  in vioblk_poll()
704 			vbr->vbr_error = ENOTTY;  in vioblk_poll()
708 			vbr->vbr_error = ENXIO;  in vioblk_poll()
715 		if (vbr->vbr_status & VIOBLK_REQSTAT_POLLED) {  in vioblk_poll()
720 			VERIFY(!(vbr->vbr_status &  in vioblk_poll()
722 			vbr->vbr_status |= VIOBLK_REQSTAT_POLL_COMPLETE;  in vioblk_poll()
727 		vioblk_complete(vib, vbr);  in vioblk_poll()
729 		vioblk_req_free(vib, vbr);  in vioblk_poll()
793 		struct vioblk_req *vbr = &vib->vib_reqs_mem[i];  in vioblk_free_reqs()  local
795 		VERIFY(list_link_active(&vbr->vbr_link));  in vioblk_free_reqs()
796 		list_remove(&vib->vib_reqs, vbr);  in vioblk_free_reqs()
798 		VERIFY0(vbr->vbr_status);  in vioblk_free_reqs()
800 		if (vbr->vbr_chain != NULL) {  in vioblk_free_reqs()
801 			virtio_chain_free(vbr->vbr_chain);  in vioblk_free_reqs()
802 			vbr->vbr_chain = NULL;  in vioblk_free_reqs()
804 		if (vbr->vbr_dma != NULL) {  in vioblk_free_reqs()
805 			virtio_dma_free(vbr->vbr_dma);  in vioblk_free_reqs()
806 			vbr->vbr_dma = NULL;  in vioblk_free_reqs()
832 	for (vioblk_req_t *vbr = list_head(&vib->vib_reqs); vbr != NULL;  in vioblk_alloc_reqs()  local
833 	    vbr = list_next(&vib->vib_reqs, vbr)) {  in vioblk_alloc_reqs()
834 		if ((vbr->vbr_dma = virtio_dma_alloc(vib->vib_virtio,  in vioblk_alloc_reqs()
840 		vbr->vbr_chain = virtio_chain_alloc(vib->vib_vq, KM_SLEEP);  in vioblk_alloc_reqs()
841 		if (vbr->vbr_chain == NULL) {  in vioblk_alloc_reqs()
844 		virtio_chain_data_set(vbr->vbr_chain, vbr);  in vioblk_alloc_reqs()