140 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr)  in virtblk_add_req()  argument
145 	sg_init_one(&out_hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));  in virtblk_add_req()
148 	if (vbr->sg_table.nents) {  in virtblk_add_req()
149 		if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))  in virtblk_add_req()
150 			sgs[num_out++] = vbr->sg_table.sgl;  in virtblk_add_req()
152 			sgs[num_out + num_in++] = vbr->sg_table.sgl;  in virtblk_add_req()
155 	sg_init_one(&in_hdr, &vbr->in_hdr.status, vbr->in_hdr_len);  in virtblk_add_req()
158 	return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);  in virtblk_add_req()
207 static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr)  in virtblk_unmap_data()  argument
210 		sg_free_table_chained(&vbr->sg_table,  in virtblk_unmap_data()
215 		struct virtblk_req *vbr)  in virtblk_map_data()  argument
222 	vbr->sg_table.sgl = vbr->sg;  in virtblk_map_data()
223 	err = sg_alloc_table_chained(&vbr->sg_table,  in virtblk_map_data()
225 				     vbr->sg_table.sgl,  in virtblk_map_data()
230 	return blk_rq_map_sg(hctx->queue, req, vbr->sg_table.sgl);  in virtblk_map_data()
241 				      struct virtblk_req *vbr)  in virtblk_setup_cmd()  argument
243 	size_t in_hdr_len = sizeof(vbr->in_hdr.status);  in virtblk_setup_cmd()
252 	vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req));  in virtblk_setup_cmd()
291 		in_hdr_len = sizeof(vbr->in_hdr.zone_append);  in virtblk_setup_cmd()
312 	vbr->in_hdr_len = in_hdr_len;  in virtblk_setup_cmd()
313 	vbr->out_hdr.type = cpu_to_virtio32(vdev, type);  in virtblk_setup_cmd()
314 	vbr->out_hdr.sector = cpu_to_virtio64(vdev, sector);  in virtblk_setup_cmd()
330 static inline u8 virtblk_vbr_status(struct virtblk_req *vbr)  in virtblk_vbr_status()  argument
332 	return *((u8 *)&vbr->in_hdr + vbr->in_hdr_len - 1);  in virtblk_vbr_status()
337 	struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);  in virtblk_request_done()  local
338 	blk_status_t status = virtblk_result(virtblk_vbr_status(vbr));  in virtblk_request_done()
341 	virtblk_unmap_data(req, vbr);  in virtblk_request_done()
346 						vbr->in_hdr.zone_append.sector);  in virtblk_request_done()
356 	struct virtblk_req *vbr;  in virtblk_done()  local
363 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
364 			struct request *req = blk_mq_rq_from_pdu(vbr);  in virtblk_done()
408 					struct virtblk_req *vbr)  in virtblk_prep_rq()  argument
413 	status = virtblk_setup_cmd(vblk->vdev, req, vbr);  in virtblk_prep_rq()
417 	num = virtblk_map_data(hctx, req, vbr);  in virtblk_prep_rq()
420 	vbr->sg_table.nents = num;  in virtblk_prep_rq()
432 	struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);  in virtio_queue_rq()  local
439 	status = virtblk_prep_rq(hctx, vblk, req, vbr);  in virtio_queue_rq()
444 	err = virtblk_add_req(vblk->vqs[qid].vq, vbr);  in virtio_queue_rq()
453 		virtblk_unmap_data(req, vbr);  in virtio_queue_rq()
469 	struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);  in virtblk_prep_rq_batch()  local
471 	return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK;  in virtblk_prep_rq_batch()
484 		struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);  in virtblk_add_req_batch()  local
487 		err = virtblk_add_req(vq->vq, vbr);  in virtblk_add_req_batch()
489 			virtblk_unmap_data(req, vbr);  in virtblk_add_req_batch()
563 	struct virtblk_req *vbr;  in virtblk_submit_zone_report()  local
570 	vbr = blk_mq_rq_to_pdu(req);  in virtblk_submit_zone_report()
571 	vbr->in_hdr_len = sizeof(vbr->in_hdr.status);  in virtblk_submit_zone_report()
572 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_ZONE_REPORT);  in virtblk_submit_zone_report()
573 	vbr->out_hdr.sector = cpu_to_virtio64(vblk->vdev, sector);  in virtblk_submit_zone_report()
580 	err = blk_status_to_errno(virtblk_result(vbr->in_hdr.status));  in virtblk_submit_zone_report()
809 	struct virtblk_req *vbr;  in virtblk_get_id()  local
816 	vbr = blk_mq_rq_to_pdu(req);  in virtblk_get_id()
817 	vbr->in_hdr_len = sizeof(vbr->in_hdr.status);  in virtblk_get_id()
818 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID);  in virtblk_get_id()
819 	vbr->out_hdr.sector = 0;  in virtblk_get_id()
826 	err = blk_status_to_errno(virtblk_result(vbr->in_hdr.status));  in virtblk_get_id()
1203 	struct virtblk_req *vbr;  in virtblk_poll()  local
1210 	while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {  in virtblk_poll()
1211 		struct request *req = blk_mq_rq_from_pdu(vbr);  in virtblk_poll()
1215 		    !blk_mq_add_to_batch(req, iob, virtblk_vbr_status(vbr),  in virtblk_poll()