69 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
80 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)  in asid_to_as()  argument
82 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];  in asid_to_as()
92 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)  in asid_to_iotlb()  argument
94 	struct vhost_vdpa_as *as = asid_to_as(v, asid);  in asid_to_iotlb()
102 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)  in vhost_vdpa_alloc_as()  argument
104 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];  in vhost_vdpa_alloc_as()
107 	if (asid_to_as(v, asid))  in vhost_vdpa_alloc_as()
110 	if (asid >= v->vdpa->nas)  in vhost_vdpa_alloc_as()
124 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,  in vhost_vdpa_find_alloc_as()  argument
127 	struct vhost_vdpa_as *as = asid_to_as(v, asid);  in vhost_vdpa_find_alloc_as()
132 	return vhost_vdpa_alloc_as(v, asid);  in vhost_vdpa_find_alloc_as()
135 static void vhost_vdpa_reset_map(struct vhost_vdpa *v, u32 asid)  in vhost_vdpa_reset_map()  argument
137 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_reset_map()
144 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)  in vhost_vdpa_remove_as()  argument
146 	struct vhost_vdpa_as *as = asid_to_as(v, asid);  in vhost_vdpa_remove_as()
152 	vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);  in vhost_vdpa_remove_as()
160 	vhost_vdpa_reset_map(v, asid);  in vhost_vdpa_remove_as()
170 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);  in handle_vq_kick()  local
171 	const struct vdpa_config_ops *ops = v->vdpa->config;  in handle_vq_kick()
173 	ops->kick_vq(v->vdpa, vq - v->vqs);  in handle_vq_kick()
189 	struct vhost_vdpa *v = private;  in vhost_vdpa_config_cb()  local
190 	struct eventfd_ctx *config_ctx = v->config_ctx;  in vhost_vdpa_config_cb()
198 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_setup_vq_irq()  argument
200 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_setup_vq_irq()
201 	const struct vdpa_config_ops *ops = v->vdpa->config;  in vhost_vdpa_setup_vq_irq()
202 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_setup_vq_irq()
218 		dev_info(&v->dev, "vq %u, irq bypass producer (eventfd %p) registration fails, ret =  %d\n",  in vhost_vdpa_setup_vq_irq()
222 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_unsetup_vq_irq()  argument
224 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_unsetup_vq_irq()
229 static int _compat_vdpa_reset(struct vhost_vdpa *v)  in _compat_vdpa_reset()  argument
231 	struct vdpa_device *vdpa = v->vdpa;  in _compat_vdpa_reset()
234 	v->suspended = false;  in _compat_vdpa_reset()
236 	if (v->vdev.vqs) {  in _compat_vdpa_reset()
237 		flags |= !vhost_backend_has_feature(v->vdev.vqs[0],  in _compat_vdpa_reset()
245 static int vhost_vdpa_reset(struct vhost_vdpa *v)  in vhost_vdpa_reset()  argument
247 	v->in_batch = 0;  in vhost_vdpa_reset()
248 	return _compat_vdpa_reset(v);  in vhost_vdpa_reset()
251 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)  in vhost_vdpa_bind_mm()  argument
253 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_bind_mm()
259 	return ops->bind_mm(vdpa, v->vdev.mm);  in vhost_vdpa_bind_mm()
262 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)  in vhost_vdpa_unbind_mm()  argument
264 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_unbind_mm()
273 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)  in vhost_vdpa_get_device_id()  argument
275 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_device_id()
287 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_get_status()  argument
289 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_status()
301 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_set_status()  argument
303 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_status()
306 	u32 nvqs = v->nvqs;  in vhost_vdpa_set_status()
324 			vhost_vdpa_unsetup_vq_irq(v, i);  in vhost_vdpa_set_status()
327 		ret = _compat_vdpa_reset(v);  in vhost_vdpa_set_status()
335 			vhost_vdpa_setup_vq_irq(v, i);  in vhost_vdpa_set_status()
340 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,  in vhost_vdpa_config_validate()  argument
343 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_config_validate()
355 static long vhost_vdpa_get_config(struct vhost_vdpa *v,  in vhost_vdpa_get_config()  argument
358 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_config()
365 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_get_config()
382 static long vhost_vdpa_set_config(struct vhost_vdpa *v,  in vhost_vdpa_set_config()  argument
385 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_config()
392 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_set_config()
405 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)  in vhost_vdpa_can_suspend()  argument
407 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_can_suspend()
413 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)  in vhost_vdpa_can_resume()  argument
415 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_can_resume()
421 static bool vhost_vdpa_has_desc_group(const struct vhost_vdpa *v)  in vhost_vdpa_has_desc_group()  argument
423 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_has_desc_group()
429 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_get_features()  argument
431 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_features()
443 static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v)  in vhost_vdpa_get_backend_features()  argument
445 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_backend_features()
454 static bool vhost_vdpa_has_persistent_map(const struct vhost_vdpa *v)  in vhost_vdpa_has_persistent_map()  argument
456 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_has_persistent_map()
460 	       vhost_vdpa_get_backend_features(v) & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);  in vhost_vdpa_has_persistent_map()
463 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_set_features()  argument
465 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_features()
467 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_set_features()
498 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)  in vhost_vdpa_get_vring_num()  argument
500 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_vring_num()
512 static void vhost_vdpa_config_put(struct vhost_vdpa *v)  in vhost_vdpa_config_put()  argument
514 	if (v->config_ctx) {  in vhost_vdpa_config_put()
515 		eventfd_ctx_put(v->config_ctx);  in vhost_vdpa_config_put()
516 		v->config_ctx = NULL;  in vhost_vdpa_config_put()
520 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_set_config_call()  argument
527 	cb.private = v;  in vhost_vdpa_set_config_call()
532 	swap(ctx, v->config_ctx);  in vhost_vdpa_set_config_call()
537 	if (IS_ERR(v->config_ctx)) {  in vhost_vdpa_set_config_call()
538 		long ret = PTR_ERR(v->config_ctx);  in vhost_vdpa_set_config_call()
540 		v->config_ctx = NULL;  in vhost_vdpa_set_config_call()
544 	v->vdpa->config->set_config_cb(v->vdpa, &cb);  in vhost_vdpa_set_config_call()
549 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_get_iova_range()  argument
552 		.first = v->range.first,  in vhost_vdpa_get_iova_range()
553 		.last = v->range.last,  in vhost_vdpa_get_iova_range()
561 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_get_config_size()  argument
563 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_config_size()
575 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_get_vqs_count()  argument
577 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_vqs_count()
590 static long vhost_vdpa_suspend(struct vhost_vdpa *v)  in vhost_vdpa_suspend()  argument
592 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_suspend()
604 		v->suspended = true;  in vhost_vdpa_suspend()
613 static long vhost_vdpa_resume(struct vhost_vdpa *v)  in vhost_vdpa_resume()  argument
615 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_resume()
627 		v->suspended = false;  in vhost_vdpa_resume()
632 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,  in vhost_vdpa_vring_ioctl()  argument
635 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_vring_ioctl()
648 	if (idx >= v->nvqs)  in vhost_vdpa_vring_ioctl()
651 	idx = array_index_nospec(idx, v->nvqs);  in vhost_vdpa_vring_ioctl()
652 	vq = &v->vqs[idx];  in vhost_vdpa_vring_ioctl()
671 		if (!vhost_vdpa_has_desc_group(v))  in vhost_vdpa_vring_ioctl()
699 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);  in vhost_vdpa_vring_ioctl()
716 				vhost_vdpa_unsetup_vq_irq(v, idx);  in vhost_vdpa_vring_ioctl()
721 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_vring_ioctl()
727 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)  in vhost_vdpa_vring_ioctl()
738 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)  in vhost_vdpa_vring_ioctl()
759 				vhost_vdpa_setup_vq_irq(v, idx);  in vhost_vdpa_vring_ioctl()
779 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_unlocked_ioctl()  local
780 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_unlocked_ioctl()
797 		     !vhost_vdpa_can_suspend(v))  in vhost_vdpa_unlocked_ioctl()
800 		     !vhost_vdpa_can_resume(v))  in vhost_vdpa_unlocked_ioctl()
806 		     !vhost_vdpa_has_desc_group(v))  in vhost_vdpa_unlocked_ioctl()
809 		     !vhost_vdpa_has_persistent_map(v))  in vhost_vdpa_unlocked_ioctl()
811 		vhost_set_backend_features(&v->vdev, features);  in vhost_vdpa_unlocked_ioctl()
819 		r = vhost_vdpa_get_device_id(v, argp);  in vhost_vdpa_unlocked_ioctl()
822 		r = vhost_vdpa_get_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
825 		r = vhost_vdpa_set_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
828 		r = vhost_vdpa_get_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
831 		r = vhost_vdpa_set_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
834 		r = vhost_vdpa_get_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
837 		r = vhost_vdpa_set_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
840 		r = vhost_vdpa_get_vring_num(v, argp);  in vhost_vdpa_unlocked_ioctl()
843 		if (copy_to_user(argp, &v->vdpa->ngroups,  in vhost_vdpa_unlocked_ioctl()
844 				 sizeof(v->vdpa->ngroups)))  in vhost_vdpa_unlocked_ioctl()
848 		if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))  in vhost_vdpa_unlocked_ioctl()
856 		r = vhost_vdpa_set_config_call(v, argp);  in vhost_vdpa_unlocked_ioctl()
860 		if (vhost_vdpa_can_suspend(v))  in vhost_vdpa_unlocked_ioctl()
862 		if (vhost_vdpa_can_resume(v))  in vhost_vdpa_unlocked_ioctl()
864 		if (vhost_vdpa_has_desc_group(v))  in vhost_vdpa_unlocked_ioctl()
866 		if (vhost_vdpa_has_persistent_map(v))  in vhost_vdpa_unlocked_ioctl()
868 		features |= vhost_vdpa_get_backend_features(v);  in vhost_vdpa_unlocked_ioctl()
873 		r = vhost_vdpa_get_iova_range(v, argp);  in vhost_vdpa_unlocked_ioctl()
876 		r = vhost_vdpa_get_config_size(v, argp);  in vhost_vdpa_unlocked_ioctl()
879 		r = vhost_vdpa_get_vqs_count(v, argp);  in vhost_vdpa_unlocked_ioctl()
882 		r = vhost_vdpa_suspend(v);  in vhost_vdpa_unlocked_ioctl()
885 		r = vhost_vdpa_resume(v);  in vhost_vdpa_unlocked_ioctl()
888 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
890 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
899 		r = vhost_vdpa_bind_mm(v);  in vhost_vdpa_unlocked_ioctl()
908 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,  in vhost_vdpa_general_unmap()  argument
911 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_general_unmap()
916 		iommu_unmap(v->domain, map->start, map->size);  in vhost_vdpa_general_unmap()
920 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,  in vhost_vdpa_pa_unmap()  argument
923 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_pa_unmap()
938 		vhost_vdpa_general_unmap(v, map, asid);  in vhost_vdpa_pa_unmap()
943 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,  in vhost_vdpa_va_unmap()  argument
953 		vhost_vdpa_general_unmap(v, map, asid);  in vhost_vdpa_va_unmap()
958 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,  in vhost_vdpa_iotlb_unmap()  argument
962 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_iotlb_unmap()
965 		return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);  in vhost_vdpa_iotlb_unmap()
967 	return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);  in vhost_vdpa_iotlb_unmap()
992 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,  in vhost_vdpa_map()  argument
995 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_map()
996 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_map()
1009 		if (!v->in_batch)  in vhost_vdpa_map()
1012 		r = iommu_map(v->domain, iova, pa, size,  in vhost_vdpa_map()
1027 static void vhost_vdpa_unmap(struct vhost_vdpa *v,  in vhost_vdpa_unmap()  argument
1031 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_unmap()
1035 	vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);  in vhost_vdpa_unmap()
1038 		if (!v->in_batch)  in vhost_vdpa_unmap()
1044 static int vhost_vdpa_va_map(struct vhost_vdpa *v,  in vhost_vdpa_va_map()  argument
1048 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_va_map()
1075 		ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,  in vhost_vdpa_va_map()
1088 		vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);  in vhost_vdpa_va_map()
1095 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,  in vhost_vdpa_pa_map()  argument
1099 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_pa_map()
1160 				ret = vhost_vdpa_map(v, iotlb, iova, csize,  in vhost_vdpa_pa_map()
1190 	ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),  in vhost_vdpa_pa_map()
1210 		vhost_vdpa_unmap(v, iotlb, start, size);  in vhost_vdpa_pa_map()
1220 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,  in vhost_vdpa_process_iotlb_update()  argument
1224 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_process_iotlb_update()
1226 	if (msg->iova < v->range.first || !msg->size ||  in vhost_vdpa_process_iotlb_update()
1228 	    msg->iova + msg->size - 1 > v->range.last)  in vhost_vdpa_process_iotlb_update()
1236 		return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,  in vhost_vdpa_process_iotlb_update()
1239 	return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,  in vhost_vdpa_process_iotlb_update()
1246 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);  in vhost_vdpa_process_iotlb_msg()  local
1247 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_process_iotlb_msg()
1261 		as = vhost_vdpa_find_alloc_as(v, asid);  in vhost_vdpa_process_iotlb_msg()
1263 			dev_err(&v->dev, "can't find and alloc asid %d\n",  in vhost_vdpa_process_iotlb_msg()
1270 		iotlb = asid_to_iotlb(v, asid);  in vhost_vdpa_process_iotlb_msg()
1272 	if ((v->in_batch && v->batch_asid != asid) || !iotlb) {  in vhost_vdpa_process_iotlb_msg()
1273 		if (v->in_batch && v->batch_asid != asid) {  in vhost_vdpa_process_iotlb_msg()
1274 			dev_info(&v->dev, "batch id %d asid %d\n",  in vhost_vdpa_process_iotlb_msg()
1275 				 v->batch_asid, asid);  in vhost_vdpa_process_iotlb_msg()
1278 			dev_err(&v->dev, "no iotlb for asid %d\n", asid);  in vhost_vdpa_process_iotlb_msg()
1285 		r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);  in vhost_vdpa_process_iotlb_msg()
1288 		vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);  in vhost_vdpa_process_iotlb_msg()
1291 		v->batch_asid = asid;  in vhost_vdpa_process_iotlb_msg()
1292 		v->in_batch = true;  in vhost_vdpa_process_iotlb_msg()
1295 		if (v->in_batch && ops->set_map)  in vhost_vdpa_process_iotlb_msg()
1297 		v->in_batch = false;  in vhost_vdpa_process_iotlb_msg()
1313 	struct vhost_vdpa *v = file->private_data;  in vhost_vdpa_chr_write_iter()  local
1314 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_chr_write_iter()
1319 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)  in vhost_vdpa_alloc_domain()  argument
1321 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_alloc_domain()
1332 		dev_warn_once(&v->dev,  in vhost_vdpa_alloc_domain()
1337 	v->domain = iommu_paging_domain_alloc(dma_dev);  in vhost_vdpa_alloc_domain()
1338 	if (IS_ERR(v->domain)) {  in vhost_vdpa_alloc_domain()
1339 		ret = PTR_ERR(v->domain);  in vhost_vdpa_alloc_domain()
1340 		v->domain = NULL;  in vhost_vdpa_alloc_domain()
1344 	ret = iommu_attach_device(v->domain, dma_dev);  in vhost_vdpa_alloc_domain()
1351 	iommu_domain_free(v->domain);  in vhost_vdpa_alloc_domain()
1352 	v->domain = NULL;  in vhost_vdpa_alloc_domain()
1356 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)  in vhost_vdpa_free_domain()  argument
1358 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_free_domain()
1362 	if (v->domain) {  in vhost_vdpa_free_domain()
1363 		iommu_detach_device(v->domain, dma_dev);  in vhost_vdpa_free_domain()
1364 		iommu_domain_free(v->domain);  in vhost_vdpa_free_domain()
1367 	v->domain = NULL;  in vhost_vdpa_free_domain()
1370 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)  in vhost_vdpa_set_iova_range()  argument
1372 	struct vdpa_iova_range *range = &v->range;  in vhost_vdpa_set_iova_range()
1373 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_iova_range()
1378 	} else if (v->domain && v->domain->geometry.force_aperture) {  in vhost_vdpa_set_iova_range()
1379 		range->first = v->domain->geometry.aperture_start;  in vhost_vdpa_set_iova_range()
1380 		range->last = v->domain->geometry.aperture_end;  in vhost_vdpa_set_iova_range()
1387 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)  in vhost_vdpa_cleanup()  argument
1392 	for (asid = 0; asid < v->vdpa->nas; asid++) {  in vhost_vdpa_cleanup()
1393 		as = asid_to_as(v, asid);  in vhost_vdpa_cleanup()
1395 			vhost_vdpa_remove_as(v, asid);  in vhost_vdpa_cleanup()
1398 	vhost_vdpa_free_domain(v);  in vhost_vdpa_cleanup()
1399 	vhost_dev_cleanup(&v->vdev);  in vhost_vdpa_cleanup()
1400 	kfree(v->vdev.vqs);  in vhost_vdpa_cleanup()
1401 	v->vdev.vqs = NULL;  in vhost_vdpa_cleanup()
1406 	struct vhost_vdpa *v;  in vhost_vdpa_open()  local
1412 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);  in vhost_vdpa_open()
1414 	opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_open()
1418 	nvqs = v->nvqs;  in vhost_vdpa_open()
1419 	r = vhost_vdpa_reset(v);  in vhost_vdpa_open()
1429 	dev = &v->vdev;  in vhost_vdpa_open()
1431 		vqs[i] = &v->vqs[i];  in vhost_vdpa_open()
1438 	r = vhost_vdpa_alloc_domain(v);  in vhost_vdpa_open()
1442 	vhost_vdpa_set_iova_range(v);  in vhost_vdpa_open()
1444 	filep->private_data = v;  in vhost_vdpa_open()
1449 	vhost_vdpa_cleanup(v);  in vhost_vdpa_open()
1451 	atomic_dec(&v->opened);  in vhost_vdpa_open()
1455 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)  in vhost_vdpa_clean_irq()  argument
1459 	for (i = 0; i < v->nvqs; i++)  in vhost_vdpa_clean_irq()
1460 		vhost_vdpa_unsetup_vq_irq(v, i);  in vhost_vdpa_clean_irq()
1465 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_release()  local
1466 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_release()
1470 	vhost_vdpa_clean_irq(v);  in vhost_vdpa_release()
1471 	vhost_vdpa_reset(v);  in vhost_vdpa_release()
1472 	vhost_dev_stop(&v->vdev);  in vhost_vdpa_release()
1473 	vhost_vdpa_unbind_mm(v);  in vhost_vdpa_release()
1474 	vhost_vdpa_config_put(v);  in vhost_vdpa_release()
1475 	vhost_vdpa_cleanup(v);  in vhost_vdpa_release()
1478 	atomic_dec(&v->opened);  in vhost_vdpa_release()
1479 	complete(&v->completion);  in vhost_vdpa_release()
1487 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;  in vhost_vdpa_fault()  local
1488 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_fault()
1505 	struct vhost_vdpa *v = vma->vm_file->private_data;  in vhost_vdpa_mmap()  local
1506 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_mmap()
1553 	struct vhost_vdpa *v =  in vhost_vdpa_release_dev()  local
1556 	ida_free(&vhost_vdpa_ida, v->minor);  in vhost_vdpa_release_dev()
1557 	kfree(v->vqs);  in vhost_vdpa_release_dev()
1558 	kfree(v);  in vhost_vdpa_release_dev()
1564 	struct vhost_vdpa *v;  in vhost_vdpa_probe()  local
1575 	v = kzalloc_obj(*v, GFP_KERNEL | __GFP_RETRY_MAYFAIL);  in vhost_vdpa_probe()
1576 	if (!v)  in vhost_vdpa_probe()
1582 		kfree(v);  in vhost_vdpa_probe()
1586 	atomic_set(&v->opened, 0);  in vhost_vdpa_probe()
1587 	v->minor = minor;  in vhost_vdpa_probe()
1588 	v->vdpa = vdpa;  in vhost_vdpa_probe()
1589 	v->nvqs = vdpa->nvqs;  in vhost_vdpa_probe()
1590 	v->virtio_id = ops->get_device_id(vdpa);  in vhost_vdpa_probe()
1592 	device_initialize(&v->dev);  in vhost_vdpa_probe()
1593 	v->dev.release = vhost_vdpa_release_dev;  in vhost_vdpa_probe()
1594 	v->dev.parent = &vdpa->dev;  in vhost_vdpa_probe()
1595 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);  in vhost_vdpa_probe()
1596 	v->vqs = kmalloc_objs(struct vhost_virtqueue, v->nvqs);  in vhost_vdpa_probe()
1597 	if (!v->vqs) {  in vhost_vdpa_probe()
1602 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);  in vhost_vdpa_probe()
1606 	cdev_init(&v->cdev, &vhost_vdpa_fops);  in vhost_vdpa_probe()
1607 	v->cdev.owner = THIS_MODULE;  in vhost_vdpa_probe()
1609 	r = cdev_device_add(&v->cdev, &v->dev);  in vhost_vdpa_probe()
1613 	init_completion(&v->completion);  in vhost_vdpa_probe()
1614 	vdpa_set_drvdata(vdpa, v);  in vhost_vdpa_probe()
1617 		INIT_HLIST_HEAD(&v->as[i]);  in vhost_vdpa_probe()
1622 	put_device(&v->dev);  in vhost_vdpa_probe()
1628 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);  in vhost_vdpa_remove()  local
1631 	cdev_device_del(&v->cdev, &v->dev);  in vhost_vdpa_remove()
1634 		opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_remove()
1637 		wait_for_completion(&v->completion);  in vhost_vdpa_remove()
1640 	put_device(&v->dev);  in vhost_vdpa_remove()