xref: /linux/drivers/vhost/vdpa.c (revision 9d19ca5d0e8b4a3f4b2eaa14e86a25f1c93ff35b) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/mm.h>
19 #include <linux/slab.h>
20 #include <linux/iommu.h>
21 #include <linux/uuid.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vhost.h>
25 
26 #include "vhost.h"
27 
28 enum {
29 	VHOST_VDPA_BACKEND_FEATURES =
30 	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31 	(1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
32 	(1ULL << VHOST_BACKEND_F_IOTLB_ASID),
33 };
34 
35 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
36 
37 static int max_iotlb_entries = 2048;
38 module_param(max_iotlb_entries, int, 0444);
39 MODULE_PARM_DESC(max_iotlb_entries,
40 		 "Maximum number of iotlb entries. (default: 2048)");
41 
42 #define VHOST_VDPA_IOTLB_BUCKETS 16
43 
44 struct vhost_vdpa_as {
45 	struct hlist_node hash_link;
46 	struct vhost_iotlb iotlb;
47 	u32 id;
48 };
49 
50 struct vhost_vdpa {
51 	struct vhost_dev vdev;
52 	struct iommu_domain *domain;
53 	struct vhost_virtqueue *vqs;
54 	struct completion completion;
55 	struct vdpa_device *vdpa;
56 	struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
57 	struct device dev;
58 	struct cdev cdev;
59 	atomic_t opened;
60 	u32 nvqs;
61 	int virtio_id;
62 	int minor;
63 	struct eventfd_ctx *config_ctx;
64 	int in_batch;
65 	struct vdpa_iova_range range;
66 	u32 batch_asid;
67 	bool suspended;
68 };
69 
70 static DEFINE_IDA(vhost_vdpa_ida);
71 
72 static dev_t vhost_vdpa_major;
73 
74 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
75 				   struct vhost_iotlb *iotlb, u64 start,
76 				   u64 last, u32 asid);
77 
78 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
79 {
80 	struct vhost_vdpa_as *as = container_of(iotlb, struct
81 						vhost_vdpa_as, iotlb);
82 	return as->id;
83 }
84 
85 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
86 {
87 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
88 	struct vhost_vdpa_as *as;
89 
90 	hlist_for_each_entry(as, head, hash_link)
91 		if (as->id == asid)
92 			return as;
93 
94 	return NULL;
95 }
96 
97 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
98 {
99 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
100 
101 	if (!as)
102 		return NULL;
103 
104 	return &as->iotlb;
105 }
106 
107 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
108 {
109 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
110 	struct vhost_vdpa_as *as;
111 
112 	if (asid_to_as(v, asid))
113 		return NULL;
114 
115 	if (asid >= v->vdpa->nas)
116 		return NULL;
117 	if (max_iotlb_entries <= 0)
118 		return NULL;
119 
120 	as = kmalloc_obj(*as);
121 	if (!as)
122 		return NULL;
123 
124 	vhost_iotlb_init(&as->iotlb, max_iotlb_entries, 0);
125 	as->id = asid;
126 	hlist_add_head(&as->hash_link, head);
127 
128 	return as;
129 }
130 
131 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
132 						      u32 asid)
133 {
134 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
135 
136 	if (as)
137 		return as;
138 
139 	return vhost_vdpa_alloc_as(v, asid);
140 }
141 
142 static void vhost_vdpa_reset_map(struct vhost_vdpa *v, u32 asid)
143 {
144 	struct vdpa_device *vdpa = v->vdpa;
145 	const struct vdpa_config_ops *ops = vdpa->config;
146 
147 	if (ops->reset_map)
148 		ops->reset_map(vdpa, asid);
149 }
150 
151 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
152 {
153 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
154 
155 	if (!as)
156 		return -EINVAL;
157 
158 	hlist_del(&as->hash_link);
159 	vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
160 	/*
161 	 * Devices with vendor specific IOMMU may need to restore
162 	 * iotlb to the initial or default state, which cannot be
163 	 * cleaned up in the all range unmap call above. Give them
164 	 * a chance to clean up or reset the map to the desired
165 	 * state.
166 	 */
167 	vhost_vdpa_reset_map(v, asid);
168 	kfree(as);
169 
170 	return 0;
171 }
172 
173 static void handle_vq_kick(struct vhost_work *work)
174 {
175 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
176 						  poll.work);
177 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
178 	const struct vdpa_config_ops *ops = v->vdpa->config;
179 
180 	ops->kick_vq(v->vdpa, vq - v->vqs);
181 }
182 
183 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
184 {
185 	struct vhost_virtqueue *vq = private;
186 	struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
187 
188 	if (call_ctx)
189 		eventfd_signal(call_ctx);
190 
191 	return IRQ_HANDLED;
192 }
193 
194 static irqreturn_t vhost_vdpa_config_cb(void *private)
195 {
196 	struct vhost_vdpa *v = private;
197 	struct eventfd_ctx *config_ctx = v->config_ctx;
198 
199 	if (config_ctx)
200 		eventfd_signal(config_ctx);
201 
202 	return IRQ_HANDLED;
203 }
204 
205 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
206 {
207 	struct vhost_virtqueue *vq = &v->vqs[qid];
208 	const struct vdpa_config_ops *ops = v->vdpa->config;
209 	struct vdpa_device *vdpa = v->vdpa;
210 	int ret, irq;
211 
212 	if (!ops->get_vq_irq)
213 		return;
214 
215 	irq = ops->get_vq_irq(vdpa, qid);
216 	if (irq < 0)
217 		return;
218 
219 	if (!vq->call_ctx.ctx)
220 		return;
221 
222 	ret = irq_bypass_register_producer(&vq->call_ctx.producer,
223 					   vq->call_ctx.ctx, irq);
224 	if (unlikely(ret))
225 		dev_info(&v->dev, "vq %u, irq bypass producer (eventfd %p) registration fails, ret =  %d\n",
226 			 qid, vq->call_ctx.ctx, ret);
227 }
228 
229 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
230 {
231 	struct vhost_virtqueue *vq = &v->vqs[qid];
232 
233 	irq_bypass_unregister_producer(&vq->call_ctx.producer);
234 }
235 
236 static int _compat_vdpa_reset(struct vhost_vdpa *v)
237 {
238 	struct vdpa_device *vdpa = v->vdpa;
239 	u32 flags = 0;
240 
241 	v->suspended = false;
242 
243 	if (v->vdev.vqs) {
244 		flags |= !vhost_backend_has_feature(v->vdev.vqs[0],
245 						    VHOST_BACKEND_F_IOTLB_PERSIST) ?
246 			 VDPA_RESET_F_CLEAN_MAP : 0;
247 	}
248 
249 	return vdpa_reset(vdpa, flags);
250 }
251 
252 static int vhost_vdpa_reset(struct vhost_vdpa *v)
253 {
254 	v->in_batch = 0;
255 	return _compat_vdpa_reset(v);
256 }
257 
258 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
259 {
260 	struct vdpa_device *vdpa = v->vdpa;
261 	const struct vdpa_config_ops *ops = vdpa->config;
262 
263 	if (!vdpa->use_va || !ops->bind_mm)
264 		return 0;
265 
266 	return ops->bind_mm(vdpa, v->vdev.mm);
267 }
268 
269 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
270 {
271 	struct vdpa_device *vdpa = v->vdpa;
272 	const struct vdpa_config_ops *ops = vdpa->config;
273 
274 	if (!vdpa->use_va || !ops->unbind_mm)
275 		return;
276 
277 	ops->unbind_mm(vdpa);
278 }
279 
280 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
281 {
282 	struct vdpa_device *vdpa = v->vdpa;
283 	const struct vdpa_config_ops *ops = vdpa->config;
284 	u32 device_id;
285 
286 	device_id = ops->get_device_id(vdpa);
287 
288 	if (copy_to_user(argp, &device_id, sizeof(device_id)))
289 		return -EFAULT;
290 
291 	return 0;
292 }
293 
294 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
295 {
296 	struct vdpa_device *vdpa = v->vdpa;
297 	const struct vdpa_config_ops *ops = vdpa->config;
298 	u8 status;
299 
300 	status = ops->get_status(vdpa);
301 
302 	if (copy_to_user(statusp, &status, sizeof(status)))
303 		return -EFAULT;
304 
305 	return 0;
306 }
307 
308 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
309 {
310 	struct vdpa_device *vdpa = v->vdpa;
311 	const struct vdpa_config_ops *ops = vdpa->config;
312 	u8 status, status_old;
313 	u32 nvqs = v->nvqs;
314 	int ret;
315 	u16 i;
316 
317 	if (copy_from_user(&status, statusp, sizeof(status)))
318 		return -EFAULT;
319 
320 	status_old = ops->get_status(vdpa);
321 
322 	/*
323 	 * Userspace shouldn't remove status bits unless reset the
324 	 * status to 0.
325 	 */
326 	if (status != 0 && (status_old & ~status) != 0)
327 		return -EINVAL;
328 
329 	if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
330 		for (i = 0; i < nvqs; i++)
331 			vhost_vdpa_unsetup_vq_irq(v, i);
332 
333 	if (status == 0) {
334 		ret = _compat_vdpa_reset(v);
335 		if (ret)
336 			return ret;
337 	} else
338 		vdpa_set_status(vdpa, status);
339 
340 	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
341 		for (i = 0; i < nvqs; i++)
342 			vhost_vdpa_setup_vq_irq(v, i);
343 
344 	return 0;
345 }
346 
347 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
348 				      struct vhost_vdpa_config *c)
349 {
350 	struct vdpa_device *vdpa = v->vdpa;
351 	size_t size = vdpa->config->get_config_size(vdpa);
352 
353 	if (c->len == 0 || c->off > size)
354 		return -EINVAL;
355 
356 	if (c->len > size - c->off)
357 		return -E2BIG;
358 
359 	return 0;
360 }
361 
362 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
363 				  struct vhost_vdpa_config __user *c)
364 {
365 	struct vdpa_device *vdpa = v->vdpa;
366 	struct vhost_vdpa_config config;
367 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
368 	u8 *buf;
369 
370 	if (copy_from_user(&config, c, size))
371 		return -EFAULT;
372 	if (vhost_vdpa_config_validate(v, &config))
373 		return -EINVAL;
374 	buf = kvzalloc(config.len, GFP_KERNEL);
375 	if (!buf)
376 		return -ENOMEM;
377 
378 	vdpa_get_config(vdpa, config.off, buf, config.len);
379 
380 	if (copy_to_user(c->buf, buf, config.len)) {
381 		kvfree(buf);
382 		return -EFAULT;
383 	}
384 
385 	kvfree(buf);
386 	return 0;
387 }
388 
389 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
390 				  struct vhost_vdpa_config __user *c)
391 {
392 	struct vdpa_device *vdpa = v->vdpa;
393 	struct vhost_vdpa_config config;
394 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
395 	u8 *buf;
396 
397 	if (copy_from_user(&config, c, size))
398 		return -EFAULT;
399 	if (vhost_vdpa_config_validate(v, &config))
400 		return -EINVAL;
401 
402 	buf = vmemdup_user(c->buf, config.len);
403 	if (IS_ERR(buf))
404 		return PTR_ERR(buf);
405 
406 	vdpa_set_config(vdpa, config.off, buf, config.len);
407 
408 	kvfree(buf);
409 	return 0;
410 }
411 
412 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
413 {
414 	struct vdpa_device *vdpa = v->vdpa;
415 	const struct vdpa_config_ops *ops = vdpa->config;
416 
417 	return ops->suspend;
418 }
419 
420 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
421 {
422 	struct vdpa_device *vdpa = v->vdpa;
423 	const struct vdpa_config_ops *ops = vdpa->config;
424 
425 	return ops->resume;
426 }
427 
428 static bool vhost_vdpa_has_desc_group(const struct vhost_vdpa *v)
429 {
430 	struct vdpa_device *vdpa = v->vdpa;
431 	const struct vdpa_config_ops *ops = vdpa->config;
432 
433 	return ops->get_vq_desc_group;
434 }
435 
436 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
437 {
438 	struct vdpa_device *vdpa = v->vdpa;
439 	const struct vdpa_config_ops *ops = vdpa->config;
440 	u64 features;
441 
442 	features = ops->get_device_features(vdpa);
443 
444 	if (copy_to_user(featurep, &features, sizeof(features)))
445 		return -EFAULT;
446 
447 	return 0;
448 }
449 
450 static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v)
451 {
452 	struct vdpa_device *vdpa = v->vdpa;
453 	const struct vdpa_config_ops *ops = vdpa->config;
454 
455 	if (!ops->get_backend_features)
456 		return 0;
457 	else
458 		return ops->get_backend_features(vdpa);
459 }
460 
461 static bool vhost_vdpa_has_persistent_map(const struct vhost_vdpa *v)
462 {
463 	struct vdpa_device *vdpa = v->vdpa;
464 	const struct vdpa_config_ops *ops = vdpa->config;
465 
466 	return (!ops->set_map && !ops->dma_map) || ops->reset_map ||
467 	       vhost_vdpa_get_backend_features(v) & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
468 }
469 
470 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
471 {
472 	struct vdpa_device *vdpa = v->vdpa;
473 	const struct vdpa_config_ops *ops = vdpa->config;
474 	struct vhost_dev *d = &v->vdev;
475 	u64 actual_features;
476 	u64 features;
477 	int i;
478 
479 	/*
480 	 * It's not allowed to change the features after they have
481 	 * been negotiated.
482 	 */
483 	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
484 		return -EBUSY;
485 
486 	if (copy_from_user(&features, featurep, sizeof(features)))
487 		return -EFAULT;
488 
489 	if (vdpa_set_features(vdpa, features))
490 		return -EINVAL;
491 
492 	/* let the vqs know what has been configured */
493 	actual_features = ops->get_driver_features(vdpa);
494 	for (i = 0; i < d->nvqs; ++i) {
495 		struct vhost_virtqueue *vq = d->vqs[i];
496 
497 		mutex_lock(&vq->mutex);
498 		vq->acked_features = actual_features;
499 		mutex_unlock(&vq->mutex);
500 	}
501 
502 	return 0;
503 }
504 
505 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
506 {
507 	struct vdpa_device *vdpa = v->vdpa;
508 	const struct vdpa_config_ops *ops = vdpa->config;
509 	u16 num;
510 
511 	num = ops->get_vq_num_max(vdpa);
512 
513 	if (copy_to_user(argp, &num, sizeof(num)))
514 		return -EFAULT;
515 
516 	return 0;
517 }
518 
519 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
520 {
521 	if (v->config_ctx) {
522 		eventfd_ctx_put(v->config_ctx);
523 		v->config_ctx = NULL;
524 	}
525 }
526 
527 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
528 {
529 	struct vdpa_callback cb;
530 	int fd;
531 	struct eventfd_ctx *ctx;
532 
533 	cb.callback = vhost_vdpa_config_cb;
534 	cb.private = v;
535 	if (copy_from_user(&fd, argp, sizeof(fd)))
536 		return  -EFAULT;
537 
538 	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
539 	swap(ctx, v->config_ctx);
540 
541 	if (!IS_ERR_OR_NULL(ctx))
542 		eventfd_ctx_put(ctx);
543 
544 	if (IS_ERR(v->config_ctx)) {
545 		long ret = PTR_ERR(v->config_ctx);
546 
547 		v->config_ctx = NULL;
548 		return ret;
549 	}
550 
551 	v->vdpa->config->set_config_cb(v->vdpa, &cb);
552 
553 	return 0;
554 }
555 
556 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
557 {
558 	struct vhost_vdpa_iova_range range = {
559 		.first = v->range.first,
560 		.last = v->range.last,
561 	};
562 
563 	if (copy_to_user(argp, &range, sizeof(range)))
564 		return -EFAULT;
565 	return 0;
566 }
567 
568 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
569 {
570 	struct vdpa_device *vdpa = v->vdpa;
571 	const struct vdpa_config_ops *ops = vdpa->config;
572 	u32 size;
573 
574 	size = ops->get_config_size(vdpa);
575 
576 	if (copy_to_user(argp, &size, sizeof(size)))
577 		return -EFAULT;
578 
579 	return 0;
580 }
581 
582 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
583 {
584 	struct vdpa_device *vdpa = v->vdpa;
585 
586 	if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
587 		return -EFAULT;
588 
589 	return 0;
590 }
591 
592 /* After a successful return of ioctl the device must not process more
593  * virtqueue descriptors. The device can answer to read or writes of config
594  * fields as if it were not suspended. In particular, writing to "queue_enable"
595  * with a value of 1 will not make the device start processing buffers.
596  */
597 static long vhost_vdpa_suspend(struct vhost_vdpa *v)
598 {
599 	struct vdpa_device *vdpa = v->vdpa;
600 	const struct vdpa_config_ops *ops = vdpa->config;
601 	int ret;
602 
603 	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
604 		return 0;
605 
606 	if (!ops->suspend)
607 		return -EOPNOTSUPP;
608 
609 	ret = ops->suspend(vdpa);
610 	if (!ret)
611 		v->suspended = true;
612 
613 	return ret;
614 }
615 
616 /* After a successful return of this ioctl the device resumes processing
617  * virtqueue descriptors. The device becomes fully operational the same way it
618  * was before it was suspended.
619  */
620 static long vhost_vdpa_resume(struct vhost_vdpa *v)
621 {
622 	struct vdpa_device *vdpa = v->vdpa;
623 	const struct vdpa_config_ops *ops = vdpa->config;
624 	int ret;
625 
626 	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
627 		return 0;
628 
629 	if (!ops->resume)
630 		return -EOPNOTSUPP;
631 
632 	ret = ops->resume(vdpa);
633 	if (!ret)
634 		v->suspended = false;
635 
636 	return ret;
637 }
638 
639 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
640 				   void __user *argp)
641 {
642 	struct vdpa_device *vdpa = v->vdpa;
643 	const struct vdpa_config_ops *ops = vdpa->config;
644 	struct vdpa_vq_state vq_state;
645 	struct vdpa_callback cb;
646 	struct vhost_virtqueue *vq;
647 	struct vhost_vring_state s;
648 	u32 idx;
649 	long r;
650 
651 	r = get_user(idx, (u32 __user *)argp);
652 	if (r < 0)
653 		return r;
654 
655 	if (idx >= v->nvqs)
656 		return -ENOBUFS;
657 
658 	idx = array_index_nospec(idx, v->nvqs);
659 	vq = &v->vqs[idx];
660 
661 	switch (cmd) {
662 	case VHOST_VDPA_SET_VRING_ENABLE:
663 		if (copy_from_user(&s, argp, sizeof(s)))
664 			return -EFAULT;
665 		ops->set_vq_ready(vdpa, idx, s.num);
666 		return 0;
667 	case VHOST_VDPA_GET_VRING_GROUP:
668 		if (!ops->get_vq_group)
669 			return -EOPNOTSUPP;
670 		s.index = idx;
671 		s.num = ops->get_vq_group(vdpa, idx);
672 		if (s.num >= vdpa->ngroups)
673 			return -EIO;
674 		else if (copy_to_user(argp, &s, sizeof(s)))
675 			return -EFAULT;
676 		return 0;
677 	case VHOST_VDPA_GET_VRING_DESC_GROUP:
678 		if (!vhost_vdpa_has_desc_group(v))
679 			return -EOPNOTSUPP;
680 		s.index = idx;
681 		s.num = ops->get_vq_desc_group(vdpa, idx);
682 		if (s.num >= vdpa->ngroups)
683 			return -EIO;
684 		else if (copy_to_user(argp, &s, sizeof(s)))
685 			return -EFAULT;
686 		return 0;
687 	case VHOST_VDPA_SET_GROUP_ASID:
688 		if (copy_from_user(&s, argp, sizeof(s)))
689 			return -EFAULT;
690 		if (idx >= vdpa->ngroups || s.num >= vdpa->nas)
691 			return -EINVAL;
692 		if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK)
693 			return -EBUSY;
694 		if (!ops->set_group_asid)
695 			return -EOPNOTSUPP;
696 		return ops->set_group_asid(vdpa, idx, s.num);
697 	case VHOST_VDPA_GET_VRING_SIZE:
698 		if (!ops->get_vq_size)
699 			return -EOPNOTSUPP;
700 		s.index = idx;
701 		s.num = ops->get_vq_size(vdpa, idx);
702 		if (copy_to_user(argp, &s, sizeof(s)))
703 			return -EFAULT;
704 		return 0;
705 	case VHOST_GET_VRING_BASE:
706 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);
707 		if (r)
708 			return r;
709 
710 		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
711 			vq->last_avail_idx = vq_state.packed.last_avail_idx |
712 					     (vq_state.packed.last_avail_counter << 15);
713 			vq->last_used_idx = vq_state.packed.last_used_idx |
714 					    (vq_state.packed.last_used_counter << 15);
715 		} else {
716 			vq->last_avail_idx = vq_state.split.avail_index;
717 		}
718 		break;
719 	case VHOST_SET_VRING_CALL:
720 		if (vq->call_ctx.ctx) {
721 			if (ops->get_status(vdpa) &
722 			    VIRTIO_CONFIG_S_DRIVER_OK)
723 				vhost_vdpa_unsetup_vq_irq(v, idx);
724 		}
725 		break;
726 	}
727 
728 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
729 	if (r)
730 		return r;
731 
732 	switch (cmd) {
733 	case VHOST_SET_VRING_ADDR:
734 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
735 			return -EINVAL;
736 
737 		if (ops->set_vq_address(vdpa, idx,
738 					(u64)(uintptr_t)vq->desc,
739 					(u64)(uintptr_t)vq->avail,
740 					(u64)(uintptr_t)vq->used))
741 			r = -EINVAL;
742 		break;
743 
744 	case VHOST_SET_VRING_BASE:
745 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
746 			return -EINVAL;
747 
748 		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
749 			vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff;
750 			vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000);
751 			vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff;
752 			vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000);
753 		} else {
754 			vq_state.split.avail_index = vq->last_avail_idx;
755 		}
756 		r = ops->set_vq_state(vdpa, idx, &vq_state);
757 		break;
758 
759 	case VHOST_SET_VRING_CALL:
760 		if (vq->call_ctx.ctx) {
761 			cb.callback = vhost_vdpa_virtqueue_cb;
762 			cb.private = vq;
763 			cb.trigger = vq->call_ctx.ctx;
764 			if (ops->get_status(vdpa) &
765 			    VIRTIO_CONFIG_S_DRIVER_OK)
766 				vhost_vdpa_setup_vq_irq(v, idx);
767 		} else {
768 			cb.callback = NULL;
769 			cb.private = NULL;
770 			cb.trigger = NULL;
771 		}
772 		ops->set_vq_cb(vdpa, idx, &cb);
773 		break;
774 
775 	case VHOST_SET_VRING_NUM:
776 		ops->set_vq_num(vdpa, idx, vq->num);
777 		break;
778 	}
779 
780 	return r;
781 }
782 
783 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
784 				      unsigned int cmd, unsigned long arg)
785 {
786 	struct vhost_vdpa *v = filep->private_data;
787 	struct vhost_dev *d = &v->vdev;
788 	void __user *argp = (void __user *)arg;
789 	u64 __user *featurep = argp;
790 	u64 features;
791 	long r = 0;
792 
793 	if (cmd == VHOST_SET_BACKEND_FEATURES) {
794 		if (copy_from_user(&features, featurep, sizeof(features)))
795 			return -EFAULT;
796 		if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
797 				 BIT_ULL(VHOST_BACKEND_F_DESC_ASID) |
798 				 BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST) |
799 				 BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
800 				 BIT_ULL(VHOST_BACKEND_F_RESUME) |
801 				 BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK)))
802 			return -EOPNOTSUPP;
803 		if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
804 		     !vhost_vdpa_can_suspend(v))
805 			return -EOPNOTSUPP;
806 		if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
807 		     !vhost_vdpa_can_resume(v))
808 			return -EOPNOTSUPP;
809 		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
810 		    !(features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID)))
811 			return -EINVAL;
812 		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
813 		     !vhost_vdpa_has_desc_group(v))
814 			return -EOPNOTSUPP;
815 		if ((features & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST)) &&
816 		     !vhost_vdpa_has_persistent_map(v))
817 			return -EOPNOTSUPP;
818 		vhost_set_backend_features(&v->vdev, features);
819 		return 0;
820 	}
821 
822 	mutex_lock(&d->mutex);
823 
824 	switch (cmd) {
825 	case VHOST_VDPA_GET_DEVICE_ID:
826 		r = vhost_vdpa_get_device_id(v, argp);
827 		break;
828 	case VHOST_VDPA_GET_STATUS:
829 		r = vhost_vdpa_get_status(v, argp);
830 		break;
831 	case VHOST_VDPA_SET_STATUS:
832 		r = vhost_vdpa_set_status(v, argp);
833 		break;
834 	case VHOST_VDPA_GET_CONFIG:
835 		r = vhost_vdpa_get_config(v, argp);
836 		break;
837 	case VHOST_VDPA_SET_CONFIG:
838 		r = vhost_vdpa_set_config(v, argp);
839 		break;
840 	case VHOST_GET_FEATURES:
841 		r = vhost_vdpa_get_features(v, argp);
842 		break;
843 	case VHOST_SET_FEATURES:
844 		r = vhost_vdpa_set_features(v, argp);
845 		break;
846 	case VHOST_VDPA_GET_VRING_NUM:
847 		r = vhost_vdpa_get_vring_num(v, argp);
848 		break;
849 	case VHOST_VDPA_GET_GROUP_NUM:
850 		if (copy_to_user(argp, &v->vdpa->ngroups,
851 				 sizeof(v->vdpa->ngroups)))
852 			r = -EFAULT;
853 		break;
854 	case VHOST_VDPA_GET_AS_NUM:
855 		if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
856 			r = -EFAULT;
857 		break;
858 	case VHOST_SET_LOG_BASE:
859 	case VHOST_SET_LOG_FD:
860 		r = -ENOIOCTLCMD;
861 		break;
862 	case VHOST_VDPA_SET_CONFIG_CALL:
863 		r = vhost_vdpa_set_config_call(v, argp);
864 		break;
865 	case VHOST_GET_BACKEND_FEATURES:
866 		features = VHOST_VDPA_BACKEND_FEATURES;
867 		if (vhost_vdpa_can_suspend(v))
868 			features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
869 		if (vhost_vdpa_can_resume(v))
870 			features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
871 		if (vhost_vdpa_has_desc_group(v))
872 			features |= BIT_ULL(VHOST_BACKEND_F_DESC_ASID);
873 		if (vhost_vdpa_has_persistent_map(v))
874 			features |= BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
875 		features |= vhost_vdpa_get_backend_features(v);
876 		if (copy_to_user(featurep, &features, sizeof(features)))
877 			r = -EFAULT;
878 		break;
879 	case VHOST_VDPA_GET_IOVA_RANGE:
880 		r = vhost_vdpa_get_iova_range(v, argp);
881 		break;
882 	case VHOST_VDPA_GET_CONFIG_SIZE:
883 		r = vhost_vdpa_get_config_size(v, argp);
884 		break;
885 	case VHOST_VDPA_GET_VQS_COUNT:
886 		r = vhost_vdpa_get_vqs_count(v, argp);
887 		break;
888 	case VHOST_VDPA_SUSPEND:
889 		r = vhost_vdpa_suspend(v);
890 		break;
891 	case VHOST_VDPA_RESUME:
892 		r = vhost_vdpa_resume(v);
893 		break;
894 	default:
895 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
896 		if (r == -ENOIOCTLCMD)
897 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
898 		break;
899 	}
900 
901 	if (r)
902 		goto out;
903 
904 	switch (cmd) {
905 	case VHOST_SET_OWNER:
906 		r = vhost_vdpa_bind_mm(v);
907 		if (r)
908 			vhost_dev_reset_owner(d, NULL);
909 		break;
910 	}
911 out:
912 	mutex_unlock(&d->mutex);
913 	return r;
914 }
915 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
916 				     struct vhost_iotlb_map *map, u32 asid)
917 {
918 	struct vdpa_device *vdpa = v->vdpa;
919 	const struct vdpa_config_ops *ops = vdpa->config;
920 	if (ops->dma_map) {
921 		ops->dma_unmap(vdpa, asid, map->start, map->size);
922 	} else if (ops->set_map == NULL) {
923 		iommu_unmap(v->domain, map->start, map->size);
924 	}
925 }
926 
927 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
928 				u64 start, u64 last, u32 asid)
929 {
930 	struct vhost_dev *dev = &v->vdev;
931 	struct vhost_iotlb_map *map;
932 	struct page *page;
933 	unsigned long pfn, pinned;
934 
935 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
936 		pinned = PFN_DOWN(map->size);
937 		for (pfn = PFN_DOWN(map->addr);
938 		     pinned > 0; pfn++, pinned--) {
939 			page = pfn_to_page(pfn);
940 			if (map->perm & VHOST_ACCESS_WO)
941 				set_page_dirty_lock(page);
942 			unpin_user_page(page);
943 		}
944 		atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
945 		vhost_vdpa_general_unmap(v, map, asid);
946 		vhost_iotlb_map_free(iotlb, map);
947 	}
948 }
949 
950 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
951 				u64 start, u64 last, u32 asid)
952 {
953 	struct vhost_iotlb_map *map;
954 	struct vdpa_map_file *map_file;
955 
956 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
957 		map_file = (struct vdpa_map_file *)map->opaque;
958 		fput(map_file->file);
959 		kfree(map_file);
960 		vhost_vdpa_general_unmap(v, map, asid);
961 		vhost_iotlb_map_free(iotlb, map);
962 	}
963 }
964 
965 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
966 				   struct vhost_iotlb *iotlb, u64 start,
967 				   u64 last, u32 asid)
968 {
969 	struct vdpa_device *vdpa = v->vdpa;
970 
971 	if (vdpa->use_va)
972 		return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
973 
974 	return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
975 }
976 
977 static int perm_to_iommu_flags(u32 perm)
978 {
979 	int flags = 0;
980 
981 	switch (perm) {
982 	case VHOST_ACCESS_WO:
983 		flags |= IOMMU_WRITE;
984 		break;
985 	case VHOST_ACCESS_RO:
986 		flags |= IOMMU_READ;
987 		break;
988 	case VHOST_ACCESS_RW:
989 		flags |= (IOMMU_WRITE | IOMMU_READ);
990 		break;
991 	default:
992 		WARN(1, "invalidate vhost IOTLB permission\n");
993 		break;
994 	}
995 
996 	return flags | IOMMU_CACHE;
997 }
998 
999 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
1000 			  u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
1001 {
1002 	struct vhost_dev *dev = &v->vdev;
1003 	struct vdpa_device *vdpa = v->vdpa;
1004 	const struct vdpa_config_ops *ops = vdpa->config;
1005 	u32 asid = iotlb_to_asid(iotlb);
1006 	int r = 0;
1007 
1008 	r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
1009 				      pa, perm, opaque);
1010 	if (r)
1011 		return r;
1012 
1013 	if (ops->dma_map) {
1014 		r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
1015 	} else if (ops->set_map) {
1016 		if (!v->in_batch)
1017 			r = ops->set_map(vdpa, asid, iotlb);
1018 	} else {
1019 		r = iommu_map(v->domain, iova, pa, size,
1020 			      perm_to_iommu_flags(perm),
1021 			      GFP_KERNEL_ACCOUNT);
1022 	}
1023 	if (r) {
1024 		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
1025 		return r;
1026 	}
1027 
1028 	if (!vdpa->use_va)
1029 		atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
1030 
1031 	return 0;
1032 }
1033 
1034 static void vhost_vdpa_unmap(struct vhost_vdpa *v,
1035 			     struct vhost_iotlb *iotlb,
1036 			     u64 iova, u64 size)
1037 {
1038 	struct vdpa_device *vdpa = v->vdpa;
1039 	const struct vdpa_config_ops *ops = vdpa->config;
1040 	u32 asid = iotlb_to_asid(iotlb);
1041 
1042 	vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
1043 
1044 	if (ops->set_map) {
1045 		if (!v->in_batch)
1046 			ops->set_map(vdpa, asid, iotlb);
1047 	}
1048 
1049 }
1050 
1051 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
1052 			     struct vhost_iotlb *iotlb,
1053 			     u64 iova, u64 size, u64 uaddr, u32 perm)
1054 {
1055 	struct vhost_dev *dev = &v->vdev;
1056 	u64 offset, map_size, map_iova = iova;
1057 	struct vdpa_map_file *map_file;
1058 	struct vm_area_struct *vma;
1059 	int ret = 0;
1060 
1061 	mmap_read_lock(dev->mm);
1062 
1063 	while (size) {
1064 		vma = find_vma(dev->mm, uaddr);
1065 		if (!vma) {
1066 			ret = -EINVAL;
1067 			break;
1068 		}
1069 		map_size = min(size, vma->vm_end - uaddr);
1070 		if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
1071 			!(vma->vm_flags & (VM_IO | VM_PFNMAP))))
1072 			goto next;
1073 
1074 		map_file = kzalloc_obj(*map_file);
1075 		if (!map_file) {
1076 			ret = -ENOMEM;
1077 			break;
1078 		}
1079 		offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
1080 		map_file->offset = offset;
1081 		map_file->file = get_file(vma->vm_file);
1082 		ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
1083 				     perm, map_file);
1084 		if (ret) {
1085 			fput(map_file->file);
1086 			kfree(map_file);
1087 			break;
1088 		}
1089 next:
1090 		size -= map_size;
1091 		uaddr += map_size;
1092 		map_iova += map_size;
1093 	}
1094 	if (ret)
1095 		vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
1096 
1097 	mmap_read_unlock(dev->mm);
1098 
1099 	return ret;
1100 }
1101 
1102 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
1103 			     struct vhost_iotlb *iotlb,
1104 			     u64 iova, u64 size, u64 uaddr, u32 perm)
1105 {
1106 	struct vhost_dev *dev = &v->vdev;
1107 	struct page **page_list;
1108 	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
1109 	unsigned int gup_flags = FOLL_LONGTERM;
1110 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
1111 	unsigned long lock_limit, sz2pin, nchunks, i;
1112 	unsigned long page_offset;
1113 	u64 start = iova;
1114 	long pinned;
1115 	int ret = 0;
1116 
1117 	/* Limit the use of memory for bookkeeping */
1118 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
1119 	if (!page_list)
1120 		return -ENOMEM;
1121 
1122 	if (perm & VHOST_ACCESS_WO)
1123 		gup_flags |= FOLL_WRITE;
1124 
1125 	page_offset = iova & ~PAGE_MASK;
1126 	if (size > ULONG_MAX - page_offset) {
1127 		ret = -EINVAL;
1128 		goto free;
1129 	}
1130 
1131 	npages = PFN_UP(size + page_offset);
1132 	if (!npages) {
1133 		ret = -EINVAL;
1134 		goto free;
1135 	}
1136 
1137 	mmap_read_lock(dev->mm);
1138 
1139 	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1140 	if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
1141 		ret = -ENOMEM;
1142 		goto unlock;
1143 	}
1144 
1145 	cur_base = uaddr & PAGE_MASK;
1146 	iova &= PAGE_MASK;
1147 	nchunks = 0;
1148 
1149 	while (npages) {
1150 		sz2pin = min_t(unsigned long, npages, list_size);
1151 		pinned = pin_user_pages(cur_base, sz2pin,
1152 					gup_flags, page_list);
1153 		if (sz2pin != pinned) {
1154 			if (pinned < 0) {
1155 				ret = pinned;
1156 			} else {
1157 				unpin_user_pages(page_list, pinned);
1158 				ret = -ENOMEM;
1159 			}
1160 			goto out;
1161 		}
1162 		nchunks++;
1163 
1164 		if (!last_pfn)
1165 			map_pfn = page_to_pfn(page_list[0]);
1166 
1167 		for (i = 0; i < pinned; i++) {
1168 			unsigned long this_pfn = page_to_pfn(page_list[i]);
1169 			u64 csize;
1170 
1171 			if (last_pfn && (this_pfn != last_pfn + 1)) {
1172 				/* Pin a contiguous chunk of memory */
1173 				csize = PFN_PHYS(last_pfn - map_pfn + 1);
1174 				ret = vhost_vdpa_map(v, iotlb, iova, csize,
1175 						     PFN_PHYS(map_pfn),
1176 						     perm, NULL);
1177 				if (ret) {
1178 					/*
1179 					 * Unpin the pages that are left unmapped
1180 					 * from this point on in the current
1181 					 * page_list. The remaining outstanding
1182 					 * ones which may stride across several
1183 					 * chunks will be covered in the common
1184 					 * error path subsequently.
1185 					 */
1186 					unpin_user_pages(&page_list[i],
1187 							 pinned - i);
1188 					goto out;
1189 				}
1190 
1191 				map_pfn = this_pfn;
1192 				iova += csize;
1193 				nchunks = 0;
1194 			}
1195 
1196 			last_pfn = this_pfn;
1197 		}
1198 
1199 		cur_base += PFN_PHYS(pinned);
1200 		npages -= pinned;
1201 	}
1202 
1203 	/* Pin the rest chunk */
1204 	ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1205 			     PFN_PHYS(map_pfn), perm, NULL);
1206 out:
1207 	if (ret) {
1208 		if (nchunks) {
1209 			unsigned long pfn;
1210 
1211 			/*
1212 			 * Unpin the outstanding pages which are yet to be
1213 			 * mapped but haven't due to vdpa_map() or
1214 			 * pin_user_pages() failure.
1215 			 *
1216 			 * Mapped pages are accounted in vdpa_map(), hence
1217 			 * the corresponding unpinning will be handled by
1218 			 * vdpa_unmap().
1219 			 */
1220 			WARN_ON(!last_pfn);
1221 			for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1222 				unpin_user_page(pfn_to_page(pfn));
1223 		}
1224 		vhost_vdpa_unmap(v, iotlb, start, size);
1225 	}
1226 unlock:
1227 	mmap_read_unlock(dev->mm);
1228 free:
1229 	free_page((unsigned long)page_list);
1230 	return ret;
1231 
1232 }
1233 
1234 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1235 					   struct vhost_iotlb *iotlb,
1236 					   struct vhost_iotlb_msg *msg)
1237 {
1238 	struct vdpa_device *vdpa = v->vdpa;
1239 
1240 	if (msg->iova < v->range.first || !msg->size ||
1241 	    msg->iova > U64_MAX - msg->size + 1 ||
1242 	    msg->iova + msg->size - 1 > v->range.last)
1243 		return -EINVAL;
1244 
1245 	if (vhost_iotlb_itree_first(iotlb, msg->iova,
1246 				    msg->iova + msg->size - 1))
1247 		return -EEXIST;
1248 
1249 	if (vdpa->use_va)
1250 		return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1251 					 msg->uaddr, msg->perm);
1252 
1253 	return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1254 				 msg->perm);
1255 }
1256 
1257 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1258 					struct vhost_iotlb_msg *msg)
1259 {
1260 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1261 	struct vdpa_device *vdpa = v->vdpa;
1262 	const struct vdpa_config_ops *ops = vdpa->config;
1263 	struct vhost_iotlb *iotlb = NULL;
1264 	struct vhost_vdpa_as *as = NULL;
1265 	int r = 0;
1266 
1267 	mutex_lock(&dev->mutex);
1268 
1269 	r = vhost_dev_check_owner(dev);
1270 	if (r)
1271 		goto unlock;
1272 
1273 	if (msg->type == VHOST_IOTLB_UPDATE ||
1274 	    msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1275 		as = vhost_vdpa_find_alloc_as(v, asid);
1276 		if (!as) {
1277 			dev_err(&v->dev, "can't find and alloc asid %d\n",
1278 				asid);
1279 			r = -EINVAL;
1280 			goto unlock;
1281 		}
1282 		iotlb = &as->iotlb;
1283 	} else
1284 		iotlb = asid_to_iotlb(v, asid);
1285 
1286 	if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1287 		if (v->in_batch && v->batch_asid != asid) {
1288 			dev_info(&v->dev, "batch id %d asid %d\n",
1289 				 v->batch_asid, asid);
1290 		}
1291 		if (!iotlb)
1292 			dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1293 		r = -EINVAL;
1294 		goto unlock;
1295 	}
1296 
1297 	switch (msg->type) {
1298 	case VHOST_IOTLB_UPDATE:
1299 		r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1300 		break;
1301 	case VHOST_IOTLB_INVALIDATE:
1302 		vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1303 		break;
1304 	case VHOST_IOTLB_BATCH_BEGIN:
1305 		v->batch_asid = asid;
1306 		v->in_batch = true;
1307 		break;
1308 	case VHOST_IOTLB_BATCH_END:
1309 		if (v->in_batch && ops->set_map)
1310 			ops->set_map(vdpa, asid, iotlb);
1311 		v->in_batch = false;
1312 		break;
1313 	default:
1314 		r = -EINVAL;
1315 		break;
1316 	}
1317 unlock:
1318 	mutex_unlock(&dev->mutex);
1319 
1320 	return r;
1321 }
1322 
1323 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1324 					 struct iov_iter *from)
1325 {
1326 	struct file *file = iocb->ki_filp;
1327 	struct vhost_vdpa *v = file->private_data;
1328 	struct vhost_dev *dev = &v->vdev;
1329 
1330 	return vhost_chr_write_iter(dev, from);
1331 }
1332 
1333 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1334 {
1335 	struct vdpa_device *vdpa = v->vdpa;
1336 	const struct vdpa_config_ops *ops = vdpa->config;
1337 	union virtio_map map = vdpa_get_map(vdpa);
1338 	struct device *dma_dev = map.dma_dev;
1339 	int ret;
1340 
1341 	/* Device want to do DMA by itself */
1342 	if (ops->set_map || ops->dma_map)
1343 		return 0;
1344 
1345 	if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1346 		dev_warn_once(&v->dev,
1347 			      "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1348 		return -ENOTSUPP;
1349 	}
1350 
1351 	v->domain = iommu_paging_domain_alloc(dma_dev);
1352 	if (IS_ERR(v->domain)) {
1353 		ret = PTR_ERR(v->domain);
1354 		v->domain = NULL;
1355 		return ret;
1356 	}
1357 
1358 	ret = iommu_attach_device(v->domain, dma_dev);
1359 	if (ret)
1360 		goto err_attach;
1361 
1362 	return 0;
1363 
1364 err_attach:
1365 	iommu_domain_free(v->domain);
1366 	v->domain = NULL;
1367 	return ret;
1368 }
1369 
1370 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1371 {
1372 	struct vdpa_device *vdpa = v->vdpa;
1373 	union virtio_map map = vdpa_get_map(vdpa);
1374 	struct device *dma_dev = map.dma_dev;
1375 
1376 	if (v->domain) {
1377 		iommu_detach_device(v->domain, dma_dev);
1378 		iommu_domain_free(v->domain);
1379 	}
1380 
1381 	v->domain = NULL;
1382 }
1383 
1384 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1385 {
1386 	struct vdpa_iova_range *range = &v->range;
1387 	struct vdpa_device *vdpa = v->vdpa;
1388 	const struct vdpa_config_ops *ops = vdpa->config;
1389 
1390 	if (ops->get_iova_range) {
1391 		*range = ops->get_iova_range(vdpa);
1392 	} else if (v->domain && v->domain->geometry.force_aperture) {
1393 		range->first = v->domain->geometry.aperture_start;
1394 		range->last = v->domain->geometry.aperture_end;
1395 	} else {
1396 		range->first = 0;
1397 		range->last = ULLONG_MAX;
1398 	}
1399 }
1400 
1401 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1402 {
1403 	struct vhost_vdpa_as *as;
1404 	u32 asid;
1405 
1406 	for (asid = 0; asid < v->vdpa->nas; asid++) {
1407 		as = asid_to_as(v, asid);
1408 		if (as)
1409 			vhost_vdpa_remove_as(v, asid);
1410 	}
1411 
1412 	vhost_vdpa_free_domain(v);
1413 	vhost_dev_cleanup(&v->vdev);
1414 	kfree(v->vdev.vqs);
1415 	v->vdev.vqs = NULL;
1416 }
1417 
1418 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1419 {
1420 	struct vhost_vdpa *v;
1421 	struct vhost_dev *dev;
1422 	struct vhost_virtqueue **vqs;
1423 	int r, opened;
1424 	u32 i, nvqs;
1425 
1426 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1427 
1428 	opened = atomic_cmpxchg(&v->opened, 0, 1);
1429 	if (opened)
1430 		return -EBUSY;
1431 
1432 	nvqs = v->nvqs;
1433 	r = vhost_vdpa_reset(v);
1434 	if (r)
1435 		goto err;
1436 
1437 	vqs = kmalloc_objs(*vqs, nvqs);
1438 	if (!vqs) {
1439 		r = -ENOMEM;
1440 		goto err;
1441 	}
1442 
1443 	dev = &v->vdev;
1444 	for (i = 0; i < nvqs; i++) {
1445 		vqs[i] = &v->vqs[i];
1446 		vqs[i]->handle_kick = handle_vq_kick;
1447 		vqs[i]->call_ctx.ctx = NULL;
1448 	}
1449 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1450 		       vhost_vdpa_process_iotlb_msg);
1451 
1452 	r = vhost_vdpa_alloc_domain(v);
1453 	if (r)
1454 		goto err_alloc_domain;
1455 
1456 	vhost_vdpa_set_iova_range(v);
1457 
1458 	filep->private_data = v;
1459 
1460 	return 0;
1461 
1462 err_alloc_domain:
1463 	vhost_vdpa_cleanup(v);
1464 err:
1465 	atomic_dec(&v->opened);
1466 	return r;
1467 }
1468 
1469 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1470 {
1471 	u32 i;
1472 
1473 	for (i = 0; i < v->nvqs; i++)
1474 		vhost_vdpa_unsetup_vq_irq(v, i);
1475 }
1476 
1477 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1478 {
1479 	struct vhost_vdpa *v = filep->private_data;
1480 	struct vhost_dev *d = &v->vdev;
1481 
1482 	mutex_lock(&d->mutex);
1483 	filep->private_data = NULL;
1484 	vhost_vdpa_clean_irq(v);
1485 	vhost_vdpa_reset(v);
1486 	vhost_dev_stop(&v->vdev);
1487 	vhost_vdpa_unbind_mm(v);
1488 	vhost_vdpa_config_put(v);
1489 	vhost_vdpa_cleanup(v);
1490 	mutex_unlock(&d->mutex);
1491 
1492 	atomic_dec(&v->opened);
1493 	complete(&v->completion);
1494 
1495 	return 0;
1496 }
1497 
1498 #ifdef CONFIG_MMU
1499 static int
1500 vhost_vdpa_get_vq_notification(struct vhost_vdpa *v, unsigned long index,
1501 			       struct vdpa_notification_area *notify)
1502 {
1503 	struct vdpa_device *vdpa = v->vdpa;
1504 	const struct vdpa_config_ops *ops = vdpa->config;
1505 
1506 	if (index > 65535 || index >= v->nvqs)
1507 		return -EINVAL;
1508 
1509 	index = array_index_nospec(index, v->nvqs);
1510 
1511 	*notify = ops->get_vq_notification(vdpa, index);
1512 
1513 	return 0;
1514 }
1515 
1516 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1517 {
1518 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1519 	struct vdpa_notification_area notify;
1520 	struct vm_area_struct *vma = vmf->vma;
1521 	unsigned long index = vma->vm_pgoff;
1522 
1523 	if (vhost_vdpa_get_vq_notification(v, index, &notify))
1524 		return VM_FAULT_SIGBUS;
1525 
1526 	return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr));
1527 }
1528 
1529 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1530 	.fault = vhost_vdpa_fault,
1531 };
1532 
1533 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1534 {
1535 	struct vhost_vdpa *v = vma->vm_file->private_data;
1536 	struct vdpa_device *vdpa = v->vdpa;
1537 	const struct vdpa_config_ops *ops = vdpa->config;
1538 	struct vdpa_notification_area notify;
1539 	unsigned long index = vma->vm_pgoff;
1540 
1541 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1542 		return -EINVAL;
1543 	if ((vma->vm_flags & VM_SHARED) == 0)
1544 		return -EINVAL;
1545 	if (vma->vm_flags & VM_READ)
1546 		return -EINVAL;
1547 	if (!ops->get_vq_notification)
1548 		return -ENOTSUPP;
1549 
1550 	/* To be safe and easily modelled by userspace, We only
1551 	 * support the doorbell which sits on the page boundary and
1552 	 * does not share the page with other registers.
1553 	 */
1554 	if (vhost_vdpa_get_vq_notification(v, index, &notify))
1555 		return -EINVAL;
1556 	if (notify.addr & (PAGE_SIZE - 1))
1557 		return -EINVAL;
1558 	if (vma->vm_end - vma->vm_start != notify.size)
1559 		return -ENOTSUPP;
1560 
1561 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1562 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1563 	vma->vm_ops = &vhost_vdpa_vm_ops;
1564 	return 0;
1565 }
1566 #endif /* CONFIG_MMU */
1567 
1568 static const struct file_operations vhost_vdpa_fops = {
1569 	.owner		= THIS_MODULE,
1570 	.open		= vhost_vdpa_open,
1571 	.release	= vhost_vdpa_release,
1572 	.write_iter	= vhost_vdpa_chr_write_iter,
1573 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
1574 #ifdef CONFIG_MMU
1575 	.mmap		= vhost_vdpa_mmap,
1576 #endif /* CONFIG_MMU */
1577 	.compat_ioctl	= compat_ptr_ioctl,
1578 };
1579 
1580 static void vhost_vdpa_release_dev(struct device *device)
1581 {
1582 	struct vhost_vdpa *v =
1583 	       container_of(device, struct vhost_vdpa, dev);
1584 
1585 	ida_free(&vhost_vdpa_ida, v->minor);
1586 	kfree(v->vqs);
1587 	kfree(v);
1588 }
1589 
1590 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1591 {
1592 	const struct vdpa_config_ops *ops = vdpa->config;
1593 	struct vhost_vdpa *v;
1594 	int minor;
1595 	int i, r;
1596 
1597 	/* We can't support platform IOMMU device with more than 1
1598 	 * group or as
1599 	 */
1600 	if (!ops->set_map && !ops->dma_map &&
1601 	    (vdpa->ngroups > 1 || vdpa->nas > 1))
1602 		return -EOPNOTSUPP;
1603 
1604 	v = kzalloc_obj(*v, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1605 	if (!v)
1606 		return -ENOMEM;
1607 
1608 	minor = ida_alloc_max(&vhost_vdpa_ida, VHOST_VDPA_DEV_MAX - 1,
1609 			      GFP_KERNEL);
1610 	if (minor < 0) {
1611 		kfree(v);
1612 		return minor;
1613 	}
1614 
1615 	atomic_set(&v->opened, 0);
1616 	v->minor = minor;
1617 	v->vdpa = vdpa;
1618 	v->nvqs = vdpa->nvqs;
1619 	v->virtio_id = ops->get_device_id(vdpa);
1620 
1621 	device_initialize(&v->dev);
1622 	v->dev.release = vhost_vdpa_release_dev;
1623 	v->dev.parent = &vdpa->dev;
1624 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1625 	v->vqs = kmalloc_objs(struct vhost_virtqueue, v->nvqs);
1626 	if (!v->vqs) {
1627 		r = -ENOMEM;
1628 		goto err;
1629 	}
1630 
1631 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1632 	if (r)
1633 		goto err;
1634 
1635 	cdev_init(&v->cdev, &vhost_vdpa_fops);
1636 	v->cdev.owner = THIS_MODULE;
1637 
1638 	r = cdev_device_add(&v->cdev, &v->dev);
1639 	if (r)
1640 		goto err;
1641 
1642 	init_completion(&v->completion);
1643 	vdpa_set_drvdata(vdpa, v);
1644 
1645 	for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1646 		INIT_HLIST_HEAD(&v->as[i]);
1647 
1648 	return 0;
1649 
1650 err:
1651 	put_device(&v->dev);
1652 	return r;
1653 }
1654 
1655 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1656 {
1657 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1658 	int opened;
1659 
1660 	cdev_device_del(&v->cdev, &v->dev);
1661 
1662 	do {
1663 		opened = atomic_cmpxchg(&v->opened, 0, 1);
1664 		if (!opened)
1665 			break;
1666 		wait_for_completion(&v->completion);
1667 	} while (1);
1668 
1669 	put_device(&v->dev);
1670 }
1671 
1672 static struct vdpa_driver vhost_vdpa_driver = {
1673 	.driver = {
1674 		.name	= "vhost_vdpa",
1675 	},
1676 	.probe	= vhost_vdpa_probe,
1677 	.remove	= vhost_vdpa_remove,
1678 };
1679 
1680 static int __init vhost_vdpa_init(void)
1681 {
1682 	int r;
1683 
1684 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1685 				"vhost-vdpa");
1686 	if (r)
1687 		goto err_alloc_chrdev;
1688 
1689 	r = vdpa_register_driver(&vhost_vdpa_driver);
1690 	if (r)
1691 		goto err_vdpa_register_driver;
1692 
1693 	return 0;
1694 
1695 err_vdpa_register_driver:
1696 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1697 err_alloc_chrdev:
1698 	return r;
1699 }
1700 module_init(vhost_vdpa_init);
1701 
1702 static void __exit vhost_vdpa_exit(void)
1703 {
1704 	vdpa_unregister_driver(&vhost_vdpa_driver);
1705 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1706 }
1707 module_exit(vhost_vdpa_exit);
1708 
1709 MODULE_VERSION("0.0.1");
1710 MODULE_LICENSE("GPL v2");
1711 MODULE_AUTHOR("Intel Corporation");
1712 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");
1713