xref: /linux/drivers/vhost/vdpa.c (revision 55d0969c451159cff86949b38c39171cab962069) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/mm.h>
19 #include <linux/slab.h>
20 #include <linux/iommu.h>
21 #include <linux/uuid.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vhost.h>
25 
26 #include "vhost.h"
27 
28 enum {
29 	VHOST_VDPA_BACKEND_FEATURES =
30 	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31 	(1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
32 	(1ULL << VHOST_BACKEND_F_IOTLB_ASID),
33 };
34 
35 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
36 
37 #define VHOST_VDPA_IOTLB_BUCKETS 16
38 
39 struct vhost_vdpa_as {
40 	struct hlist_node hash_link;
41 	struct vhost_iotlb iotlb;
42 	u32 id;
43 };
44 
45 struct vhost_vdpa {
46 	struct vhost_dev vdev;
47 	struct iommu_domain *domain;
48 	struct vhost_virtqueue *vqs;
49 	struct completion completion;
50 	struct vdpa_device *vdpa;
51 	struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
52 	struct device dev;
53 	struct cdev cdev;
54 	atomic_t opened;
55 	u32 nvqs;
56 	int virtio_id;
57 	int minor;
58 	struct eventfd_ctx *config_ctx;
59 	int in_batch;
60 	struct vdpa_iova_range range;
61 	u32 batch_asid;
62 	bool suspended;
63 };
64 
65 static DEFINE_IDA(vhost_vdpa_ida);
66 
67 static dev_t vhost_vdpa_major;
68 
69 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
70 				   struct vhost_iotlb *iotlb, u64 start,
71 				   u64 last, u32 asid);
72 
73 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
74 {
75 	struct vhost_vdpa_as *as = container_of(iotlb, struct
76 						vhost_vdpa_as, iotlb);
77 	return as->id;
78 }
79 
80 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
81 {
82 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
83 	struct vhost_vdpa_as *as;
84 
85 	hlist_for_each_entry(as, head, hash_link)
86 		if (as->id == asid)
87 			return as;
88 
89 	return NULL;
90 }
91 
92 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
93 {
94 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
95 
96 	if (!as)
97 		return NULL;
98 
99 	return &as->iotlb;
100 }
101 
102 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
103 {
104 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
105 	struct vhost_vdpa_as *as;
106 
107 	if (asid_to_as(v, asid))
108 		return NULL;
109 
110 	if (asid >= v->vdpa->nas)
111 		return NULL;
112 
113 	as = kmalloc(sizeof(*as), GFP_KERNEL);
114 	if (!as)
115 		return NULL;
116 
117 	vhost_iotlb_init(&as->iotlb, 0, 0);
118 	as->id = asid;
119 	hlist_add_head(&as->hash_link, head);
120 
121 	return as;
122 }
123 
124 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
125 						      u32 asid)
126 {
127 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
128 
129 	if (as)
130 		return as;
131 
132 	return vhost_vdpa_alloc_as(v, asid);
133 }
134 
135 static void vhost_vdpa_reset_map(struct vhost_vdpa *v, u32 asid)
136 {
137 	struct vdpa_device *vdpa = v->vdpa;
138 	const struct vdpa_config_ops *ops = vdpa->config;
139 
140 	if (ops->reset_map)
141 		ops->reset_map(vdpa, asid);
142 }
143 
144 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
145 {
146 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
147 
148 	if (!as)
149 		return -EINVAL;
150 
151 	hlist_del(&as->hash_link);
152 	vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
153 	/*
154 	 * Devices with vendor specific IOMMU may need to restore
155 	 * iotlb to the initial or default state, which cannot be
156 	 * cleaned up in the all range unmap call above. Give them
157 	 * a chance to clean up or reset the map to the desired
158 	 * state.
159 	 */
160 	vhost_vdpa_reset_map(v, asid);
161 	kfree(as);
162 
163 	return 0;
164 }
165 
166 static void handle_vq_kick(struct vhost_work *work)
167 {
168 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
169 						  poll.work);
170 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
171 	const struct vdpa_config_ops *ops = v->vdpa->config;
172 
173 	ops->kick_vq(v->vdpa, vq - v->vqs);
174 }
175 
176 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
177 {
178 	struct vhost_virtqueue *vq = private;
179 	struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
180 
181 	if (call_ctx)
182 		eventfd_signal(call_ctx);
183 
184 	return IRQ_HANDLED;
185 }
186 
187 static irqreturn_t vhost_vdpa_config_cb(void *private)
188 {
189 	struct vhost_vdpa *v = private;
190 	struct eventfd_ctx *config_ctx = v->config_ctx;
191 
192 	if (config_ctx)
193 		eventfd_signal(config_ctx);
194 
195 	return IRQ_HANDLED;
196 }
197 
198 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
199 {
200 	struct vhost_virtqueue *vq = &v->vqs[qid];
201 	const struct vdpa_config_ops *ops = v->vdpa->config;
202 	struct vdpa_device *vdpa = v->vdpa;
203 	int ret, irq;
204 
205 	if (!ops->get_vq_irq)
206 		return;
207 
208 	irq = ops->get_vq_irq(vdpa, qid);
209 	if (irq < 0)
210 		return;
211 
212 	if (!vq->call_ctx.ctx)
213 		return;
214 
215 	vq->call_ctx.producer.irq = irq;
216 	ret = irq_bypass_register_producer(&vq->call_ctx.producer);
217 	if (unlikely(ret))
218 		dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",
219 			 qid, vq->call_ctx.producer.token, ret);
220 }
221 
222 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
223 {
224 	struct vhost_virtqueue *vq = &v->vqs[qid];
225 
226 	irq_bypass_unregister_producer(&vq->call_ctx.producer);
227 }
228 
229 static int _compat_vdpa_reset(struct vhost_vdpa *v)
230 {
231 	struct vdpa_device *vdpa = v->vdpa;
232 	u32 flags = 0;
233 
234 	v->suspended = false;
235 
236 	if (v->vdev.vqs) {
237 		flags |= !vhost_backend_has_feature(v->vdev.vqs[0],
238 						    VHOST_BACKEND_F_IOTLB_PERSIST) ?
239 			 VDPA_RESET_F_CLEAN_MAP : 0;
240 	}
241 
242 	return vdpa_reset(vdpa, flags);
243 }
244 
245 static int vhost_vdpa_reset(struct vhost_vdpa *v)
246 {
247 	v->in_batch = 0;
248 	return _compat_vdpa_reset(v);
249 }
250 
251 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
252 {
253 	struct vdpa_device *vdpa = v->vdpa;
254 	const struct vdpa_config_ops *ops = vdpa->config;
255 
256 	if (!vdpa->use_va || !ops->bind_mm)
257 		return 0;
258 
259 	return ops->bind_mm(vdpa, v->vdev.mm);
260 }
261 
262 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
263 {
264 	struct vdpa_device *vdpa = v->vdpa;
265 	const struct vdpa_config_ops *ops = vdpa->config;
266 
267 	if (!vdpa->use_va || !ops->unbind_mm)
268 		return;
269 
270 	ops->unbind_mm(vdpa);
271 }
272 
273 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
274 {
275 	struct vdpa_device *vdpa = v->vdpa;
276 	const struct vdpa_config_ops *ops = vdpa->config;
277 	u32 device_id;
278 
279 	device_id = ops->get_device_id(vdpa);
280 
281 	if (copy_to_user(argp, &device_id, sizeof(device_id)))
282 		return -EFAULT;
283 
284 	return 0;
285 }
286 
287 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
288 {
289 	struct vdpa_device *vdpa = v->vdpa;
290 	const struct vdpa_config_ops *ops = vdpa->config;
291 	u8 status;
292 
293 	status = ops->get_status(vdpa);
294 
295 	if (copy_to_user(statusp, &status, sizeof(status)))
296 		return -EFAULT;
297 
298 	return 0;
299 }
300 
301 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
302 {
303 	struct vdpa_device *vdpa = v->vdpa;
304 	const struct vdpa_config_ops *ops = vdpa->config;
305 	u8 status, status_old;
306 	u32 nvqs = v->nvqs;
307 	int ret;
308 	u16 i;
309 
310 	if (copy_from_user(&status, statusp, sizeof(status)))
311 		return -EFAULT;
312 
313 	status_old = ops->get_status(vdpa);
314 
315 	/*
316 	 * Userspace shouldn't remove status bits unless reset the
317 	 * status to 0.
318 	 */
319 	if (status != 0 && (status_old & ~status) != 0)
320 		return -EINVAL;
321 
322 	if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
323 		for (i = 0; i < nvqs; i++)
324 			vhost_vdpa_unsetup_vq_irq(v, i);
325 
326 	if (status == 0) {
327 		ret = _compat_vdpa_reset(v);
328 		if (ret)
329 			return ret;
330 	} else
331 		vdpa_set_status(vdpa, status);
332 
333 	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
334 		for (i = 0; i < nvqs; i++)
335 			vhost_vdpa_setup_vq_irq(v, i);
336 
337 	return 0;
338 }
339 
340 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
341 				      struct vhost_vdpa_config *c)
342 {
343 	struct vdpa_device *vdpa = v->vdpa;
344 	size_t size = vdpa->config->get_config_size(vdpa);
345 
346 	if (c->len == 0 || c->off > size)
347 		return -EINVAL;
348 
349 	if (c->len > size - c->off)
350 		return -E2BIG;
351 
352 	return 0;
353 }
354 
355 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
356 				  struct vhost_vdpa_config __user *c)
357 {
358 	struct vdpa_device *vdpa = v->vdpa;
359 	struct vhost_vdpa_config config;
360 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
361 	u8 *buf;
362 
363 	if (copy_from_user(&config, c, size))
364 		return -EFAULT;
365 	if (vhost_vdpa_config_validate(v, &config))
366 		return -EINVAL;
367 	buf = kvzalloc(config.len, GFP_KERNEL);
368 	if (!buf)
369 		return -ENOMEM;
370 
371 	vdpa_get_config(vdpa, config.off, buf, config.len);
372 
373 	if (copy_to_user(c->buf, buf, config.len)) {
374 		kvfree(buf);
375 		return -EFAULT;
376 	}
377 
378 	kvfree(buf);
379 	return 0;
380 }
381 
382 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
383 				  struct vhost_vdpa_config __user *c)
384 {
385 	struct vdpa_device *vdpa = v->vdpa;
386 	struct vhost_vdpa_config config;
387 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
388 	u8 *buf;
389 
390 	if (copy_from_user(&config, c, size))
391 		return -EFAULT;
392 	if (vhost_vdpa_config_validate(v, &config))
393 		return -EINVAL;
394 
395 	buf = vmemdup_user(c->buf, config.len);
396 	if (IS_ERR(buf))
397 		return PTR_ERR(buf);
398 
399 	vdpa_set_config(vdpa, config.off, buf, config.len);
400 
401 	kvfree(buf);
402 	return 0;
403 }
404 
405 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
406 {
407 	struct vdpa_device *vdpa = v->vdpa;
408 	const struct vdpa_config_ops *ops = vdpa->config;
409 
410 	return ops->suspend;
411 }
412 
413 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
414 {
415 	struct vdpa_device *vdpa = v->vdpa;
416 	const struct vdpa_config_ops *ops = vdpa->config;
417 
418 	return ops->resume;
419 }
420 
421 static bool vhost_vdpa_has_desc_group(const struct vhost_vdpa *v)
422 {
423 	struct vdpa_device *vdpa = v->vdpa;
424 	const struct vdpa_config_ops *ops = vdpa->config;
425 
426 	return ops->get_vq_desc_group;
427 }
428 
429 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
430 {
431 	struct vdpa_device *vdpa = v->vdpa;
432 	const struct vdpa_config_ops *ops = vdpa->config;
433 	u64 features;
434 
435 	features = ops->get_device_features(vdpa);
436 
437 	if (copy_to_user(featurep, &features, sizeof(features)))
438 		return -EFAULT;
439 
440 	return 0;
441 }
442 
443 static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v)
444 {
445 	struct vdpa_device *vdpa = v->vdpa;
446 	const struct vdpa_config_ops *ops = vdpa->config;
447 
448 	if (!ops->get_backend_features)
449 		return 0;
450 	else
451 		return ops->get_backend_features(vdpa);
452 }
453 
454 static bool vhost_vdpa_has_persistent_map(const struct vhost_vdpa *v)
455 {
456 	struct vdpa_device *vdpa = v->vdpa;
457 	const struct vdpa_config_ops *ops = vdpa->config;
458 
459 	return (!ops->set_map && !ops->dma_map) || ops->reset_map ||
460 	       vhost_vdpa_get_backend_features(v) & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
461 }
462 
463 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
464 {
465 	struct vdpa_device *vdpa = v->vdpa;
466 	const struct vdpa_config_ops *ops = vdpa->config;
467 	struct vhost_dev *d = &v->vdev;
468 	u64 actual_features;
469 	u64 features;
470 	int i;
471 
472 	/*
473 	 * It's not allowed to change the features after they have
474 	 * been negotiated.
475 	 */
476 	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
477 		return -EBUSY;
478 
479 	if (copy_from_user(&features, featurep, sizeof(features)))
480 		return -EFAULT;
481 
482 	if (vdpa_set_features(vdpa, features))
483 		return -EINVAL;
484 
485 	/* let the vqs know what has been configured */
486 	actual_features = ops->get_driver_features(vdpa);
487 	for (i = 0; i < d->nvqs; ++i) {
488 		struct vhost_virtqueue *vq = d->vqs[i];
489 
490 		mutex_lock(&vq->mutex);
491 		vq->acked_features = actual_features;
492 		mutex_unlock(&vq->mutex);
493 	}
494 
495 	return 0;
496 }
497 
498 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
499 {
500 	struct vdpa_device *vdpa = v->vdpa;
501 	const struct vdpa_config_ops *ops = vdpa->config;
502 	u16 num;
503 
504 	num = ops->get_vq_num_max(vdpa);
505 
506 	if (copy_to_user(argp, &num, sizeof(num)))
507 		return -EFAULT;
508 
509 	return 0;
510 }
511 
512 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
513 {
514 	if (v->config_ctx) {
515 		eventfd_ctx_put(v->config_ctx);
516 		v->config_ctx = NULL;
517 	}
518 }
519 
520 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
521 {
522 	struct vdpa_callback cb;
523 	int fd;
524 	struct eventfd_ctx *ctx;
525 
526 	cb.callback = vhost_vdpa_config_cb;
527 	cb.private = v;
528 	if (copy_from_user(&fd, argp, sizeof(fd)))
529 		return  -EFAULT;
530 
531 	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
532 	swap(ctx, v->config_ctx);
533 
534 	if (!IS_ERR_OR_NULL(ctx))
535 		eventfd_ctx_put(ctx);
536 
537 	if (IS_ERR(v->config_ctx)) {
538 		long ret = PTR_ERR(v->config_ctx);
539 
540 		v->config_ctx = NULL;
541 		return ret;
542 	}
543 
544 	v->vdpa->config->set_config_cb(v->vdpa, &cb);
545 
546 	return 0;
547 }
548 
549 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
550 {
551 	struct vhost_vdpa_iova_range range = {
552 		.first = v->range.first,
553 		.last = v->range.last,
554 	};
555 
556 	if (copy_to_user(argp, &range, sizeof(range)))
557 		return -EFAULT;
558 	return 0;
559 }
560 
561 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
562 {
563 	struct vdpa_device *vdpa = v->vdpa;
564 	const struct vdpa_config_ops *ops = vdpa->config;
565 	u32 size;
566 
567 	size = ops->get_config_size(vdpa);
568 
569 	if (copy_to_user(argp, &size, sizeof(size)))
570 		return -EFAULT;
571 
572 	return 0;
573 }
574 
575 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
576 {
577 	struct vdpa_device *vdpa = v->vdpa;
578 
579 	if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
580 		return -EFAULT;
581 
582 	return 0;
583 }
584 
585 /* After a successful return of ioctl the device must not process more
586  * virtqueue descriptors. The device can answer to read or writes of config
587  * fields as if it were not suspended. In particular, writing to "queue_enable"
588  * with a value of 1 will not make the device start processing buffers.
589  */
590 static long vhost_vdpa_suspend(struct vhost_vdpa *v)
591 {
592 	struct vdpa_device *vdpa = v->vdpa;
593 	const struct vdpa_config_ops *ops = vdpa->config;
594 	int ret;
595 
596 	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
597 		return 0;
598 
599 	if (!ops->suspend)
600 		return -EOPNOTSUPP;
601 
602 	ret = ops->suspend(vdpa);
603 	if (!ret)
604 		v->suspended = true;
605 
606 	return ret;
607 }
608 
609 /* After a successful return of this ioctl the device resumes processing
610  * virtqueue descriptors. The device becomes fully operational the same way it
611  * was before it was suspended.
612  */
613 static long vhost_vdpa_resume(struct vhost_vdpa *v)
614 {
615 	struct vdpa_device *vdpa = v->vdpa;
616 	const struct vdpa_config_ops *ops = vdpa->config;
617 	int ret;
618 
619 	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
620 		return 0;
621 
622 	if (!ops->resume)
623 		return -EOPNOTSUPP;
624 
625 	ret = ops->resume(vdpa);
626 	if (!ret)
627 		v->suspended = false;
628 
629 	return ret;
630 }
631 
632 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
633 				   void __user *argp)
634 {
635 	struct vdpa_device *vdpa = v->vdpa;
636 	const struct vdpa_config_ops *ops = vdpa->config;
637 	struct vdpa_vq_state vq_state;
638 	struct vdpa_callback cb;
639 	struct vhost_virtqueue *vq;
640 	struct vhost_vring_state s;
641 	u32 idx;
642 	long r;
643 
644 	r = get_user(idx, (u32 __user *)argp);
645 	if (r < 0)
646 		return r;
647 
648 	if (idx >= v->nvqs)
649 		return -ENOBUFS;
650 
651 	idx = array_index_nospec(idx, v->nvqs);
652 	vq = &v->vqs[idx];
653 
654 	switch (cmd) {
655 	case VHOST_VDPA_SET_VRING_ENABLE:
656 		if (copy_from_user(&s, argp, sizeof(s)))
657 			return -EFAULT;
658 		ops->set_vq_ready(vdpa, idx, s.num);
659 		return 0;
660 	case VHOST_VDPA_GET_VRING_GROUP:
661 		if (!ops->get_vq_group)
662 			return -EOPNOTSUPP;
663 		s.index = idx;
664 		s.num = ops->get_vq_group(vdpa, idx);
665 		if (s.num >= vdpa->ngroups)
666 			return -EIO;
667 		else if (copy_to_user(argp, &s, sizeof(s)))
668 			return -EFAULT;
669 		return 0;
670 	case VHOST_VDPA_GET_VRING_DESC_GROUP:
671 		if (!vhost_vdpa_has_desc_group(v))
672 			return -EOPNOTSUPP;
673 		s.index = idx;
674 		s.num = ops->get_vq_desc_group(vdpa, idx);
675 		if (s.num >= vdpa->ngroups)
676 			return -EIO;
677 		else if (copy_to_user(argp, &s, sizeof(s)))
678 			return -EFAULT;
679 		return 0;
680 	case VHOST_VDPA_SET_GROUP_ASID:
681 		if (copy_from_user(&s, argp, sizeof(s)))
682 			return -EFAULT;
683 		if (s.num >= vdpa->nas)
684 			return -EINVAL;
685 		if (!ops->set_group_asid)
686 			return -EOPNOTSUPP;
687 		return ops->set_group_asid(vdpa, idx, s.num);
688 	case VHOST_VDPA_GET_VRING_SIZE:
689 		if (!ops->get_vq_size)
690 			return -EOPNOTSUPP;
691 		s.index = idx;
692 		s.num = ops->get_vq_size(vdpa, idx);
693 		if (copy_to_user(argp, &s, sizeof(s)))
694 			return -EFAULT;
695 		return 0;
696 	case VHOST_GET_VRING_BASE:
697 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);
698 		if (r)
699 			return r;
700 
701 		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
702 			vq->last_avail_idx = vq_state.packed.last_avail_idx |
703 					     (vq_state.packed.last_avail_counter << 15);
704 			vq->last_used_idx = vq_state.packed.last_used_idx |
705 					    (vq_state.packed.last_used_counter << 15);
706 		} else {
707 			vq->last_avail_idx = vq_state.split.avail_index;
708 		}
709 		break;
710 	case VHOST_SET_VRING_CALL:
711 		if (vq->call_ctx.ctx) {
712 			if (ops->get_status(vdpa) &
713 			    VIRTIO_CONFIG_S_DRIVER_OK)
714 				vhost_vdpa_unsetup_vq_irq(v, idx);
715 			vq->call_ctx.producer.token = NULL;
716 		}
717 		break;
718 	}
719 
720 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
721 	if (r)
722 		return r;
723 
724 	switch (cmd) {
725 	case VHOST_SET_VRING_ADDR:
726 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
727 			return -EINVAL;
728 
729 		if (ops->set_vq_address(vdpa, idx,
730 					(u64)(uintptr_t)vq->desc,
731 					(u64)(uintptr_t)vq->avail,
732 					(u64)(uintptr_t)vq->used))
733 			r = -EINVAL;
734 		break;
735 
736 	case VHOST_SET_VRING_BASE:
737 		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
738 			return -EINVAL;
739 
740 		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
741 			vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff;
742 			vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000);
743 			vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff;
744 			vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000);
745 		} else {
746 			vq_state.split.avail_index = vq->last_avail_idx;
747 		}
748 		r = ops->set_vq_state(vdpa, idx, &vq_state);
749 		break;
750 
751 	case VHOST_SET_VRING_CALL:
752 		if (vq->call_ctx.ctx) {
753 			cb.callback = vhost_vdpa_virtqueue_cb;
754 			cb.private = vq;
755 			cb.trigger = vq->call_ctx.ctx;
756 			vq->call_ctx.producer.token = vq->call_ctx.ctx;
757 			if (ops->get_status(vdpa) &
758 			    VIRTIO_CONFIG_S_DRIVER_OK)
759 				vhost_vdpa_setup_vq_irq(v, idx);
760 		} else {
761 			cb.callback = NULL;
762 			cb.private = NULL;
763 			cb.trigger = NULL;
764 		}
765 		ops->set_vq_cb(vdpa, idx, &cb);
766 		break;
767 
768 	case VHOST_SET_VRING_NUM:
769 		ops->set_vq_num(vdpa, idx, vq->num);
770 		break;
771 	}
772 
773 	return r;
774 }
775 
776 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
777 				      unsigned int cmd, unsigned long arg)
778 {
779 	struct vhost_vdpa *v = filep->private_data;
780 	struct vhost_dev *d = &v->vdev;
781 	void __user *argp = (void __user *)arg;
782 	u64 __user *featurep = argp;
783 	u64 features;
784 	long r = 0;
785 
786 	if (cmd == VHOST_SET_BACKEND_FEATURES) {
787 		if (copy_from_user(&features, featurep, sizeof(features)))
788 			return -EFAULT;
789 		if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
790 				 BIT_ULL(VHOST_BACKEND_F_DESC_ASID) |
791 				 BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST) |
792 				 BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
793 				 BIT_ULL(VHOST_BACKEND_F_RESUME) |
794 				 BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK)))
795 			return -EOPNOTSUPP;
796 		if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
797 		     !vhost_vdpa_can_suspend(v))
798 			return -EOPNOTSUPP;
799 		if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
800 		     !vhost_vdpa_can_resume(v))
801 			return -EOPNOTSUPP;
802 		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
803 		    !(features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID)))
804 			return -EINVAL;
805 		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
806 		     !vhost_vdpa_has_desc_group(v))
807 			return -EOPNOTSUPP;
808 		if ((features & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST)) &&
809 		     !vhost_vdpa_has_persistent_map(v))
810 			return -EOPNOTSUPP;
811 		vhost_set_backend_features(&v->vdev, features);
812 		return 0;
813 	}
814 
815 	mutex_lock(&d->mutex);
816 
817 	switch (cmd) {
818 	case VHOST_VDPA_GET_DEVICE_ID:
819 		r = vhost_vdpa_get_device_id(v, argp);
820 		break;
821 	case VHOST_VDPA_GET_STATUS:
822 		r = vhost_vdpa_get_status(v, argp);
823 		break;
824 	case VHOST_VDPA_SET_STATUS:
825 		r = vhost_vdpa_set_status(v, argp);
826 		break;
827 	case VHOST_VDPA_GET_CONFIG:
828 		r = vhost_vdpa_get_config(v, argp);
829 		break;
830 	case VHOST_VDPA_SET_CONFIG:
831 		r = vhost_vdpa_set_config(v, argp);
832 		break;
833 	case VHOST_GET_FEATURES:
834 		r = vhost_vdpa_get_features(v, argp);
835 		break;
836 	case VHOST_SET_FEATURES:
837 		r = vhost_vdpa_set_features(v, argp);
838 		break;
839 	case VHOST_VDPA_GET_VRING_NUM:
840 		r = vhost_vdpa_get_vring_num(v, argp);
841 		break;
842 	case VHOST_VDPA_GET_GROUP_NUM:
843 		if (copy_to_user(argp, &v->vdpa->ngroups,
844 				 sizeof(v->vdpa->ngroups)))
845 			r = -EFAULT;
846 		break;
847 	case VHOST_VDPA_GET_AS_NUM:
848 		if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
849 			r = -EFAULT;
850 		break;
851 	case VHOST_SET_LOG_BASE:
852 	case VHOST_SET_LOG_FD:
853 		r = -ENOIOCTLCMD;
854 		break;
855 	case VHOST_VDPA_SET_CONFIG_CALL:
856 		r = vhost_vdpa_set_config_call(v, argp);
857 		break;
858 	case VHOST_GET_BACKEND_FEATURES:
859 		features = VHOST_VDPA_BACKEND_FEATURES;
860 		if (vhost_vdpa_can_suspend(v))
861 			features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
862 		if (vhost_vdpa_can_resume(v))
863 			features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
864 		if (vhost_vdpa_has_desc_group(v))
865 			features |= BIT_ULL(VHOST_BACKEND_F_DESC_ASID);
866 		if (vhost_vdpa_has_persistent_map(v))
867 			features |= BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
868 		features |= vhost_vdpa_get_backend_features(v);
869 		if (copy_to_user(featurep, &features, sizeof(features)))
870 			r = -EFAULT;
871 		break;
872 	case VHOST_VDPA_GET_IOVA_RANGE:
873 		r = vhost_vdpa_get_iova_range(v, argp);
874 		break;
875 	case VHOST_VDPA_GET_CONFIG_SIZE:
876 		r = vhost_vdpa_get_config_size(v, argp);
877 		break;
878 	case VHOST_VDPA_GET_VQS_COUNT:
879 		r = vhost_vdpa_get_vqs_count(v, argp);
880 		break;
881 	case VHOST_VDPA_SUSPEND:
882 		r = vhost_vdpa_suspend(v);
883 		break;
884 	case VHOST_VDPA_RESUME:
885 		r = vhost_vdpa_resume(v);
886 		break;
887 	default:
888 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
889 		if (r == -ENOIOCTLCMD)
890 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
891 		break;
892 	}
893 
894 	if (r)
895 		goto out;
896 
897 	switch (cmd) {
898 	case VHOST_SET_OWNER:
899 		r = vhost_vdpa_bind_mm(v);
900 		if (r)
901 			vhost_dev_reset_owner(d, NULL);
902 		break;
903 	}
904 out:
905 	mutex_unlock(&d->mutex);
906 	return r;
907 }
908 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
909 				     struct vhost_iotlb_map *map, u32 asid)
910 {
911 	struct vdpa_device *vdpa = v->vdpa;
912 	const struct vdpa_config_ops *ops = vdpa->config;
913 	if (ops->dma_map) {
914 		ops->dma_unmap(vdpa, asid, map->start, map->size);
915 	} else if (ops->set_map == NULL) {
916 		iommu_unmap(v->domain, map->start, map->size);
917 	}
918 }
919 
920 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
921 				u64 start, u64 last, u32 asid)
922 {
923 	struct vhost_dev *dev = &v->vdev;
924 	struct vhost_iotlb_map *map;
925 	struct page *page;
926 	unsigned long pfn, pinned;
927 
928 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
929 		pinned = PFN_DOWN(map->size);
930 		for (pfn = PFN_DOWN(map->addr);
931 		     pinned > 0; pfn++, pinned--) {
932 			page = pfn_to_page(pfn);
933 			if (map->perm & VHOST_ACCESS_WO)
934 				set_page_dirty_lock(page);
935 			unpin_user_page(page);
936 		}
937 		atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
938 		vhost_vdpa_general_unmap(v, map, asid);
939 		vhost_iotlb_map_free(iotlb, map);
940 	}
941 }
942 
943 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
944 				u64 start, u64 last, u32 asid)
945 {
946 	struct vhost_iotlb_map *map;
947 	struct vdpa_map_file *map_file;
948 
949 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
950 		map_file = (struct vdpa_map_file *)map->opaque;
951 		fput(map_file->file);
952 		kfree(map_file);
953 		vhost_vdpa_general_unmap(v, map, asid);
954 		vhost_iotlb_map_free(iotlb, map);
955 	}
956 }
957 
958 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
959 				   struct vhost_iotlb *iotlb, u64 start,
960 				   u64 last, u32 asid)
961 {
962 	struct vdpa_device *vdpa = v->vdpa;
963 
964 	if (vdpa->use_va)
965 		return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
966 
967 	return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
968 }
969 
970 static int perm_to_iommu_flags(u32 perm)
971 {
972 	int flags = 0;
973 
974 	switch (perm) {
975 	case VHOST_ACCESS_WO:
976 		flags |= IOMMU_WRITE;
977 		break;
978 	case VHOST_ACCESS_RO:
979 		flags |= IOMMU_READ;
980 		break;
981 	case VHOST_ACCESS_RW:
982 		flags |= (IOMMU_WRITE | IOMMU_READ);
983 		break;
984 	default:
985 		WARN(1, "invalidate vhost IOTLB permission\n");
986 		break;
987 	}
988 
989 	return flags | IOMMU_CACHE;
990 }
991 
992 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
993 			  u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
994 {
995 	struct vhost_dev *dev = &v->vdev;
996 	struct vdpa_device *vdpa = v->vdpa;
997 	const struct vdpa_config_ops *ops = vdpa->config;
998 	u32 asid = iotlb_to_asid(iotlb);
999 	int r = 0;
1000 
1001 	r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
1002 				      pa, perm, opaque);
1003 	if (r)
1004 		return r;
1005 
1006 	if (ops->dma_map) {
1007 		r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
1008 	} else if (ops->set_map) {
1009 		if (!v->in_batch)
1010 			r = ops->set_map(vdpa, asid, iotlb);
1011 	} else {
1012 		r = iommu_map(v->domain, iova, pa, size,
1013 			      perm_to_iommu_flags(perm),
1014 			      GFP_KERNEL_ACCOUNT);
1015 	}
1016 	if (r) {
1017 		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
1018 		return r;
1019 	}
1020 
1021 	if (!vdpa->use_va)
1022 		atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
1023 
1024 	return 0;
1025 }
1026 
1027 static void vhost_vdpa_unmap(struct vhost_vdpa *v,
1028 			     struct vhost_iotlb *iotlb,
1029 			     u64 iova, u64 size)
1030 {
1031 	struct vdpa_device *vdpa = v->vdpa;
1032 	const struct vdpa_config_ops *ops = vdpa->config;
1033 	u32 asid = iotlb_to_asid(iotlb);
1034 
1035 	vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
1036 
1037 	if (ops->set_map) {
1038 		if (!v->in_batch)
1039 			ops->set_map(vdpa, asid, iotlb);
1040 	}
1041 
1042 }
1043 
1044 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
1045 			     struct vhost_iotlb *iotlb,
1046 			     u64 iova, u64 size, u64 uaddr, u32 perm)
1047 {
1048 	struct vhost_dev *dev = &v->vdev;
1049 	u64 offset, map_size, map_iova = iova;
1050 	struct vdpa_map_file *map_file;
1051 	struct vm_area_struct *vma;
1052 	int ret = 0;
1053 
1054 	mmap_read_lock(dev->mm);
1055 
1056 	while (size) {
1057 		vma = find_vma(dev->mm, uaddr);
1058 		if (!vma) {
1059 			ret = -EINVAL;
1060 			break;
1061 		}
1062 		map_size = min(size, vma->vm_end - uaddr);
1063 		if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
1064 			!(vma->vm_flags & (VM_IO | VM_PFNMAP))))
1065 			goto next;
1066 
1067 		map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
1068 		if (!map_file) {
1069 			ret = -ENOMEM;
1070 			break;
1071 		}
1072 		offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
1073 		map_file->offset = offset;
1074 		map_file->file = get_file(vma->vm_file);
1075 		ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
1076 				     perm, map_file);
1077 		if (ret) {
1078 			fput(map_file->file);
1079 			kfree(map_file);
1080 			break;
1081 		}
1082 next:
1083 		size -= map_size;
1084 		uaddr += map_size;
1085 		map_iova += map_size;
1086 	}
1087 	if (ret)
1088 		vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
1089 
1090 	mmap_read_unlock(dev->mm);
1091 
1092 	return ret;
1093 }
1094 
1095 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
1096 			     struct vhost_iotlb *iotlb,
1097 			     u64 iova, u64 size, u64 uaddr, u32 perm)
1098 {
1099 	struct vhost_dev *dev = &v->vdev;
1100 	struct page **page_list;
1101 	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
1102 	unsigned int gup_flags = FOLL_LONGTERM;
1103 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
1104 	unsigned long lock_limit, sz2pin, nchunks, i;
1105 	u64 start = iova;
1106 	long pinned;
1107 	int ret = 0;
1108 
1109 	/* Limit the use of memory for bookkeeping */
1110 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
1111 	if (!page_list)
1112 		return -ENOMEM;
1113 
1114 	if (perm & VHOST_ACCESS_WO)
1115 		gup_flags |= FOLL_WRITE;
1116 
1117 	npages = PFN_UP(size + (iova & ~PAGE_MASK));
1118 	if (!npages) {
1119 		ret = -EINVAL;
1120 		goto free;
1121 	}
1122 
1123 	mmap_read_lock(dev->mm);
1124 
1125 	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1126 	if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
1127 		ret = -ENOMEM;
1128 		goto unlock;
1129 	}
1130 
1131 	cur_base = uaddr & PAGE_MASK;
1132 	iova &= PAGE_MASK;
1133 	nchunks = 0;
1134 
1135 	while (npages) {
1136 		sz2pin = min_t(unsigned long, npages, list_size);
1137 		pinned = pin_user_pages(cur_base, sz2pin,
1138 					gup_flags, page_list);
1139 		if (sz2pin != pinned) {
1140 			if (pinned < 0) {
1141 				ret = pinned;
1142 			} else {
1143 				unpin_user_pages(page_list, pinned);
1144 				ret = -ENOMEM;
1145 			}
1146 			goto out;
1147 		}
1148 		nchunks++;
1149 
1150 		if (!last_pfn)
1151 			map_pfn = page_to_pfn(page_list[0]);
1152 
1153 		for (i = 0; i < pinned; i++) {
1154 			unsigned long this_pfn = page_to_pfn(page_list[i]);
1155 			u64 csize;
1156 
1157 			if (last_pfn && (this_pfn != last_pfn + 1)) {
1158 				/* Pin a contiguous chunk of memory */
1159 				csize = PFN_PHYS(last_pfn - map_pfn + 1);
1160 				ret = vhost_vdpa_map(v, iotlb, iova, csize,
1161 						     PFN_PHYS(map_pfn),
1162 						     perm, NULL);
1163 				if (ret) {
1164 					/*
1165 					 * Unpin the pages that are left unmapped
1166 					 * from this point on in the current
1167 					 * page_list. The remaining outstanding
1168 					 * ones which may stride across several
1169 					 * chunks will be covered in the common
1170 					 * error path subsequently.
1171 					 */
1172 					unpin_user_pages(&page_list[i],
1173 							 pinned - i);
1174 					goto out;
1175 				}
1176 
1177 				map_pfn = this_pfn;
1178 				iova += csize;
1179 				nchunks = 0;
1180 			}
1181 
1182 			last_pfn = this_pfn;
1183 		}
1184 
1185 		cur_base += PFN_PHYS(pinned);
1186 		npages -= pinned;
1187 	}
1188 
1189 	/* Pin the rest chunk */
1190 	ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1191 			     PFN_PHYS(map_pfn), perm, NULL);
1192 out:
1193 	if (ret) {
1194 		if (nchunks) {
1195 			unsigned long pfn;
1196 
1197 			/*
1198 			 * Unpin the outstanding pages which are yet to be
1199 			 * mapped but haven't due to vdpa_map() or
1200 			 * pin_user_pages() failure.
1201 			 *
1202 			 * Mapped pages are accounted in vdpa_map(), hence
1203 			 * the corresponding unpinning will be handled by
1204 			 * vdpa_unmap().
1205 			 */
1206 			WARN_ON(!last_pfn);
1207 			for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1208 				unpin_user_page(pfn_to_page(pfn));
1209 		}
1210 		vhost_vdpa_unmap(v, iotlb, start, size);
1211 	}
1212 unlock:
1213 	mmap_read_unlock(dev->mm);
1214 free:
1215 	free_page((unsigned long)page_list);
1216 	return ret;
1217 
1218 }
1219 
1220 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1221 					   struct vhost_iotlb *iotlb,
1222 					   struct vhost_iotlb_msg *msg)
1223 {
1224 	struct vdpa_device *vdpa = v->vdpa;
1225 
1226 	if (msg->iova < v->range.first || !msg->size ||
1227 	    msg->iova > U64_MAX - msg->size + 1 ||
1228 	    msg->iova + msg->size - 1 > v->range.last)
1229 		return -EINVAL;
1230 
1231 	if (vhost_iotlb_itree_first(iotlb, msg->iova,
1232 				    msg->iova + msg->size - 1))
1233 		return -EEXIST;
1234 
1235 	if (vdpa->use_va)
1236 		return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1237 					 msg->uaddr, msg->perm);
1238 
1239 	return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1240 				 msg->perm);
1241 }
1242 
1243 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1244 					struct vhost_iotlb_msg *msg)
1245 {
1246 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1247 	struct vdpa_device *vdpa = v->vdpa;
1248 	const struct vdpa_config_ops *ops = vdpa->config;
1249 	struct vhost_iotlb *iotlb = NULL;
1250 	struct vhost_vdpa_as *as = NULL;
1251 	int r = 0;
1252 
1253 	mutex_lock(&dev->mutex);
1254 
1255 	r = vhost_dev_check_owner(dev);
1256 	if (r)
1257 		goto unlock;
1258 
1259 	if (msg->type == VHOST_IOTLB_UPDATE ||
1260 	    msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1261 		as = vhost_vdpa_find_alloc_as(v, asid);
1262 		if (!as) {
1263 			dev_err(&v->dev, "can't find and alloc asid %d\n",
1264 				asid);
1265 			r = -EINVAL;
1266 			goto unlock;
1267 		}
1268 		iotlb = &as->iotlb;
1269 	} else
1270 		iotlb = asid_to_iotlb(v, asid);
1271 
1272 	if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1273 		if (v->in_batch && v->batch_asid != asid) {
1274 			dev_info(&v->dev, "batch id %d asid %d\n",
1275 				 v->batch_asid, asid);
1276 		}
1277 		if (!iotlb)
1278 			dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1279 		r = -EINVAL;
1280 		goto unlock;
1281 	}
1282 
1283 	switch (msg->type) {
1284 	case VHOST_IOTLB_UPDATE:
1285 		r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1286 		break;
1287 	case VHOST_IOTLB_INVALIDATE:
1288 		vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1289 		break;
1290 	case VHOST_IOTLB_BATCH_BEGIN:
1291 		v->batch_asid = asid;
1292 		v->in_batch = true;
1293 		break;
1294 	case VHOST_IOTLB_BATCH_END:
1295 		if (v->in_batch && ops->set_map)
1296 			ops->set_map(vdpa, asid, iotlb);
1297 		v->in_batch = false;
1298 		break;
1299 	default:
1300 		r = -EINVAL;
1301 		break;
1302 	}
1303 unlock:
1304 	mutex_unlock(&dev->mutex);
1305 
1306 	return r;
1307 }
1308 
1309 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1310 					 struct iov_iter *from)
1311 {
1312 	struct file *file = iocb->ki_filp;
1313 	struct vhost_vdpa *v = file->private_data;
1314 	struct vhost_dev *dev = &v->vdev;
1315 
1316 	return vhost_chr_write_iter(dev, from);
1317 }
1318 
1319 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1320 {
1321 	struct vdpa_device *vdpa = v->vdpa;
1322 	const struct vdpa_config_ops *ops = vdpa->config;
1323 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1324 	int ret;
1325 
1326 	/* Device want to do DMA by itself */
1327 	if (ops->set_map || ops->dma_map)
1328 		return 0;
1329 
1330 	if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1331 		dev_warn_once(&v->dev,
1332 			      "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1333 		return -ENOTSUPP;
1334 	}
1335 
1336 	v->domain = iommu_paging_domain_alloc(dma_dev);
1337 	if (IS_ERR(v->domain)) {
1338 		ret = PTR_ERR(v->domain);
1339 		v->domain = NULL;
1340 		return ret;
1341 	}
1342 
1343 	ret = iommu_attach_device(v->domain, dma_dev);
1344 	if (ret)
1345 		goto err_attach;
1346 
1347 	return 0;
1348 
1349 err_attach:
1350 	iommu_domain_free(v->domain);
1351 	v->domain = NULL;
1352 	return ret;
1353 }
1354 
1355 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1356 {
1357 	struct vdpa_device *vdpa = v->vdpa;
1358 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1359 
1360 	if (v->domain) {
1361 		iommu_detach_device(v->domain, dma_dev);
1362 		iommu_domain_free(v->domain);
1363 	}
1364 
1365 	v->domain = NULL;
1366 }
1367 
1368 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1369 {
1370 	struct vdpa_iova_range *range = &v->range;
1371 	struct vdpa_device *vdpa = v->vdpa;
1372 	const struct vdpa_config_ops *ops = vdpa->config;
1373 
1374 	if (ops->get_iova_range) {
1375 		*range = ops->get_iova_range(vdpa);
1376 	} else if (v->domain && v->domain->geometry.force_aperture) {
1377 		range->first = v->domain->geometry.aperture_start;
1378 		range->last = v->domain->geometry.aperture_end;
1379 	} else {
1380 		range->first = 0;
1381 		range->last = ULLONG_MAX;
1382 	}
1383 }
1384 
1385 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1386 {
1387 	struct vhost_vdpa_as *as;
1388 	u32 asid;
1389 
1390 	for (asid = 0; asid < v->vdpa->nas; asid++) {
1391 		as = asid_to_as(v, asid);
1392 		if (as)
1393 			vhost_vdpa_remove_as(v, asid);
1394 	}
1395 
1396 	vhost_vdpa_free_domain(v);
1397 	vhost_dev_cleanup(&v->vdev);
1398 	kfree(v->vdev.vqs);
1399 	v->vdev.vqs = NULL;
1400 }
1401 
1402 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1403 {
1404 	struct vhost_vdpa *v;
1405 	struct vhost_dev *dev;
1406 	struct vhost_virtqueue **vqs;
1407 	int r, opened;
1408 	u32 i, nvqs;
1409 
1410 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1411 
1412 	opened = atomic_cmpxchg(&v->opened, 0, 1);
1413 	if (opened)
1414 		return -EBUSY;
1415 
1416 	nvqs = v->nvqs;
1417 	r = vhost_vdpa_reset(v);
1418 	if (r)
1419 		goto err;
1420 
1421 	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1422 	if (!vqs) {
1423 		r = -ENOMEM;
1424 		goto err;
1425 	}
1426 
1427 	dev = &v->vdev;
1428 	for (i = 0; i < nvqs; i++) {
1429 		vqs[i] = &v->vqs[i];
1430 		vqs[i]->handle_kick = handle_vq_kick;
1431 		vqs[i]->call_ctx.ctx = NULL;
1432 	}
1433 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1434 		       vhost_vdpa_process_iotlb_msg);
1435 
1436 	r = vhost_vdpa_alloc_domain(v);
1437 	if (r)
1438 		goto err_alloc_domain;
1439 
1440 	vhost_vdpa_set_iova_range(v);
1441 
1442 	filep->private_data = v;
1443 
1444 	return 0;
1445 
1446 err_alloc_domain:
1447 	vhost_vdpa_cleanup(v);
1448 err:
1449 	atomic_dec(&v->opened);
1450 	return r;
1451 }
1452 
1453 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1454 {
1455 	u32 i;
1456 
1457 	for (i = 0; i < v->nvqs; i++)
1458 		vhost_vdpa_unsetup_vq_irq(v, i);
1459 }
1460 
1461 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1462 {
1463 	struct vhost_vdpa *v = filep->private_data;
1464 	struct vhost_dev *d = &v->vdev;
1465 
1466 	mutex_lock(&d->mutex);
1467 	filep->private_data = NULL;
1468 	vhost_vdpa_clean_irq(v);
1469 	vhost_vdpa_reset(v);
1470 	vhost_dev_stop(&v->vdev);
1471 	vhost_vdpa_unbind_mm(v);
1472 	vhost_vdpa_config_put(v);
1473 	vhost_vdpa_cleanup(v);
1474 	mutex_unlock(&d->mutex);
1475 
1476 	atomic_dec(&v->opened);
1477 	complete(&v->completion);
1478 
1479 	return 0;
1480 }
1481 
1482 #ifdef CONFIG_MMU
1483 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1484 {
1485 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1486 	struct vdpa_device *vdpa = v->vdpa;
1487 	const struct vdpa_config_ops *ops = vdpa->config;
1488 	struct vdpa_notification_area notify;
1489 	struct vm_area_struct *vma = vmf->vma;
1490 	u16 index = vma->vm_pgoff;
1491 
1492 	notify = ops->get_vq_notification(vdpa, index);
1493 
1494 	return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr));
1495 }
1496 
1497 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1498 	.fault = vhost_vdpa_fault,
1499 };
1500 
1501 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1502 {
1503 	struct vhost_vdpa *v = vma->vm_file->private_data;
1504 	struct vdpa_device *vdpa = v->vdpa;
1505 	const struct vdpa_config_ops *ops = vdpa->config;
1506 	struct vdpa_notification_area notify;
1507 	unsigned long index = vma->vm_pgoff;
1508 
1509 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1510 		return -EINVAL;
1511 	if ((vma->vm_flags & VM_SHARED) == 0)
1512 		return -EINVAL;
1513 	if (vma->vm_flags & VM_READ)
1514 		return -EINVAL;
1515 	if (index > 65535)
1516 		return -EINVAL;
1517 	if (!ops->get_vq_notification)
1518 		return -ENOTSUPP;
1519 
1520 	/* To be safe and easily modelled by userspace, We only
1521 	 * support the doorbell which sits on the page boundary and
1522 	 * does not share the page with other registers.
1523 	 */
1524 	notify = ops->get_vq_notification(vdpa, index);
1525 	if (notify.addr & (PAGE_SIZE - 1))
1526 		return -EINVAL;
1527 	if (vma->vm_end - vma->vm_start != notify.size)
1528 		return -ENOTSUPP;
1529 
1530 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1531 	vma->vm_ops = &vhost_vdpa_vm_ops;
1532 	return 0;
1533 }
1534 #endif /* CONFIG_MMU */
1535 
1536 static const struct file_operations vhost_vdpa_fops = {
1537 	.owner		= THIS_MODULE,
1538 	.open		= vhost_vdpa_open,
1539 	.release	= vhost_vdpa_release,
1540 	.write_iter	= vhost_vdpa_chr_write_iter,
1541 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
1542 #ifdef CONFIG_MMU
1543 	.mmap		= vhost_vdpa_mmap,
1544 #endif /* CONFIG_MMU */
1545 	.compat_ioctl	= compat_ptr_ioctl,
1546 };
1547 
1548 static void vhost_vdpa_release_dev(struct device *device)
1549 {
1550 	struct vhost_vdpa *v =
1551 	       container_of(device, struct vhost_vdpa, dev);
1552 
1553 	ida_free(&vhost_vdpa_ida, v->minor);
1554 	kfree(v->vqs);
1555 	kfree(v);
1556 }
1557 
1558 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1559 {
1560 	const struct vdpa_config_ops *ops = vdpa->config;
1561 	struct vhost_vdpa *v;
1562 	int minor;
1563 	int i, r;
1564 
1565 	/* We can't support platform IOMMU device with more than 1
1566 	 * group or as
1567 	 */
1568 	if (!ops->set_map && !ops->dma_map &&
1569 	    (vdpa->ngroups > 1 || vdpa->nas > 1))
1570 		return -EOPNOTSUPP;
1571 
1572 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1573 	if (!v)
1574 		return -ENOMEM;
1575 
1576 	minor = ida_alloc_max(&vhost_vdpa_ida, VHOST_VDPA_DEV_MAX - 1,
1577 			      GFP_KERNEL);
1578 	if (minor < 0) {
1579 		kfree(v);
1580 		return minor;
1581 	}
1582 
1583 	atomic_set(&v->opened, 0);
1584 	v->minor = minor;
1585 	v->vdpa = vdpa;
1586 	v->nvqs = vdpa->nvqs;
1587 	v->virtio_id = ops->get_device_id(vdpa);
1588 
1589 	device_initialize(&v->dev);
1590 	v->dev.release = vhost_vdpa_release_dev;
1591 	v->dev.parent = &vdpa->dev;
1592 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1593 	v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1594 			       GFP_KERNEL);
1595 	if (!v->vqs) {
1596 		r = -ENOMEM;
1597 		goto err;
1598 	}
1599 
1600 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1601 	if (r)
1602 		goto err;
1603 
1604 	cdev_init(&v->cdev, &vhost_vdpa_fops);
1605 	v->cdev.owner = THIS_MODULE;
1606 
1607 	r = cdev_device_add(&v->cdev, &v->dev);
1608 	if (r)
1609 		goto err;
1610 
1611 	init_completion(&v->completion);
1612 	vdpa_set_drvdata(vdpa, v);
1613 
1614 	for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1615 		INIT_HLIST_HEAD(&v->as[i]);
1616 
1617 	return 0;
1618 
1619 err:
1620 	put_device(&v->dev);
1621 	return r;
1622 }
1623 
1624 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1625 {
1626 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1627 	int opened;
1628 
1629 	cdev_device_del(&v->cdev, &v->dev);
1630 
1631 	do {
1632 		opened = atomic_cmpxchg(&v->opened, 0, 1);
1633 		if (!opened)
1634 			break;
1635 		wait_for_completion(&v->completion);
1636 	} while (1);
1637 
1638 	put_device(&v->dev);
1639 }
1640 
1641 static struct vdpa_driver vhost_vdpa_driver = {
1642 	.driver = {
1643 		.name	= "vhost_vdpa",
1644 	},
1645 	.probe	= vhost_vdpa_probe,
1646 	.remove	= vhost_vdpa_remove,
1647 };
1648 
1649 static int __init vhost_vdpa_init(void)
1650 {
1651 	int r;
1652 
1653 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1654 				"vhost-vdpa");
1655 	if (r)
1656 		goto err_alloc_chrdev;
1657 
1658 	r = vdpa_register_driver(&vhost_vdpa_driver);
1659 	if (r)
1660 		goto err_vdpa_register_driver;
1661 
1662 	return 0;
1663 
1664 err_vdpa_register_driver:
1665 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1666 err_alloc_chrdev:
1667 	return r;
1668 }
1669 module_init(vhost_vdpa_init);
1670 
1671 static void __exit vhost_vdpa_exit(void)
1672 {
1673 	vdpa_unregister_driver(&vhost_vdpa_driver);
1674 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1675 }
1676 module_exit(vhost_vdpa_exit);
1677 
1678 MODULE_VERSION("0.0.1");
1679 MODULE_LICENSE("GPL v2");
1680 MODULE_AUTHOR("Intel Corporation");
1681 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");
1682