xref: /linux/drivers/vhost/vdpa.c (revision d8e473182ab9e85708067be81d20424045d939fa) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/mm.h>
19 #include <linux/slab.h>
20 #include <linux/iommu.h>
21 #include <linux/uuid.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vhost.h>
25 
26 #include "vhost.h"
27 
28 enum {
29 	VHOST_VDPA_BACKEND_FEATURES =
30 	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31 	(1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
32 	(1ULL << VHOST_BACKEND_F_IOTLB_ASID),
33 };
34 
35 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
36 
37 #define VHOST_VDPA_IOTLB_BUCKETS 16
38 
39 struct vhost_vdpa_as {
40 	struct hlist_node hash_link;
41 	struct vhost_iotlb iotlb;
42 	u32 id;
43 };
44 
45 struct vhost_vdpa {
46 	struct vhost_dev vdev;
47 	struct iommu_domain *domain;
48 	struct vhost_virtqueue *vqs;
49 	struct completion completion;
50 	struct vdpa_device *vdpa;
51 	struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
52 	struct device dev;
53 	struct cdev cdev;
54 	atomic_t opened;
55 	u32 nvqs;
56 	int virtio_id;
57 	int minor;
58 	struct eventfd_ctx *config_ctx;
59 	int in_batch;
60 	struct vdpa_iova_range range;
61 	u32 batch_asid;
62 };
63 
64 static DEFINE_IDA(vhost_vdpa_ida);
65 
66 static dev_t vhost_vdpa_major;
67 
68 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
69 				   struct vhost_iotlb *iotlb, u64 start,
70 				   u64 last, u32 asid);
71 
72 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
73 {
74 	struct vhost_vdpa_as *as = container_of(iotlb, struct
75 						vhost_vdpa_as, iotlb);
76 	return as->id;
77 }
78 
79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
80 {
81 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
82 	struct vhost_vdpa_as *as;
83 
84 	hlist_for_each_entry(as, head, hash_link)
85 		if (as->id == asid)
86 			return as;
87 
88 	return NULL;
89 }
90 
91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
92 {
93 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
94 
95 	if (!as)
96 		return NULL;
97 
98 	return &as->iotlb;
99 }
100 
101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
102 {
103 	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
104 	struct vhost_vdpa_as *as;
105 
106 	if (asid_to_as(v, asid))
107 		return NULL;
108 
109 	if (asid >= v->vdpa->nas)
110 		return NULL;
111 
112 	as = kmalloc(sizeof(*as), GFP_KERNEL);
113 	if (!as)
114 		return NULL;
115 
116 	vhost_iotlb_init(&as->iotlb, 0, 0);
117 	as->id = asid;
118 	hlist_add_head(&as->hash_link, head);
119 
120 	return as;
121 }
122 
123 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
124 						      u32 asid)
125 {
126 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
127 
128 	if (as)
129 		return as;
130 
131 	return vhost_vdpa_alloc_as(v, asid);
132 }
133 
134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
135 {
136 	struct vhost_vdpa_as *as = asid_to_as(v, asid);
137 
138 	if (!as)
139 		return -EINVAL;
140 
141 	hlist_del(&as->hash_link);
142 	vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
143 	kfree(as);
144 
145 	return 0;
146 }
147 
148 static void handle_vq_kick(struct vhost_work *work)
149 {
150 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
151 						  poll.work);
152 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
153 	const struct vdpa_config_ops *ops = v->vdpa->config;
154 
155 	ops->kick_vq(v->vdpa, vq - v->vqs);
156 }
157 
158 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
159 {
160 	struct vhost_virtqueue *vq = private;
161 	struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
162 
163 	if (call_ctx)
164 		eventfd_signal(call_ctx, 1);
165 
166 	return IRQ_HANDLED;
167 }
168 
169 static irqreturn_t vhost_vdpa_config_cb(void *private)
170 {
171 	struct vhost_vdpa *v = private;
172 	struct eventfd_ctx *config_ctx = v->config_ctx;
173 
174 	if (config_ctx)
175 		eventfd_signal(config_ctx, 1);
176 
177 	return IRQ_HANDLED;
178 }
179 
180 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
181 {
182 	struct vhost_virtqueue *vq = &v->vqs[qid];
183 	const struct vdpa_config_ops *ops = v->vdpa->config;
184 	struct vdpa_device *vdpa = v->vdpa;
185 	int ret, irq;
186 
187 	if (!ops->get_vq_irq)
188 		return;
189 
190 	irq = ops->get_vq_irq(vdpa, qid);
191 	if (irq < 0)
192 		return;
193 
194 	irq_bypass_unregister_producer(&vq->call_ctx.producer);
195 	if (!vq->call_ctx.ctx)
196 		return;
197 
198 	vq->call_ctx.producer.token = vq->call_ctx.ctx;
199 	vq->call_ctx.producer.irq = irq;
200 	ret = irq_bypass_register_producer(&vq->call_ctx.producer);
201 	if (unlikely(ret))
202 		dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",
203 			 qid, vq->call_ctx.producer.token, ret);
204 }
205 
206 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
207 {
208 	struct vhost_virtqueue *vq = &v->vqs[qid];
209 
210 	irq_bypass_unregister_producer(&vq->call_ctx.producer);
211 }
212 
213 static int vhost_vdpa_reset(struct vhost_vdpa *v)
214 {
215 	struct vdpa_device *vdpa = v->vdpa;
216 
217 	v->in_batch = 0;
218 
219 	return vdpa_reset(vdpa);
220 }
221 
222 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
223 {
224 	struct vdpa_device *vdpa = v->vdpa;
225 	const struct vdpa_config_ops *ops = vdpa->config;
226 	u32 device_id;
227 
228 	device_id = ops->get_device_id(vdpa);
229 
230 	if (copy_to_user(argp, &device_id, sizeof(device_id)))
231 		return -EFAULT;
232 
233 	return 0;
234 }
235 
236 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
237 {
238 	struct vdpa_device *vdpa = v->vdpa;
239 	const struct vdpa_config_ops *ops = vdpa->config;
240 	u8 status;
241 
242 	status = ops->get_status(vdpa);
243 
244 	if (copy_to_user(statusp, &status, sizeof(status)))
245 		return -EFAULT;
246 
247 	return 0;
248 }
249 
250 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
251 {
252 	struct vdpa_device *vdpa = v->vdpa;
253 	const struct vdpa_config_ops *ops = vdpa->config;
254 	u8 status, status_old;
255 	u32 nvqs = v->nvqs;
256 	int ret;
257 	u16 i;
258 
259 	if (copy_from_user(&status, statusp, sizeof(status)))
260 		return -EFAULT;
261 
262 	status_old = ops->get_status(vdpa);
263 
264 	/*
265 	 * Userspace shouldn't remove status bits unless reset the
266 	 * status to 0.
267 	 */
268 	if (status != 0 && (status_old & ~status) != 0)
269 		return -EINVAL;
270 
271 	if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
272 		for (i = 0; i < nvqs; i++)
273 			vhost_vdpa_unsetup_vq_irq(v, i);
274 
275 	if (status == 0) {
276 		ret = vdpa_reset(vdpa);
277 		if (ret)
278 			return ret;
279 	} else
280 		vdpa_set_status(vdpa, status);
281 
282 	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
283 		for (i = 0; i < nvqs; i++)
284 			vhost_vdpa_setup_vq_irq(v, i);
285 
286 	return 0;
287 }
288 
289 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
290 				      struct vhost_vdpa_config *c)
291 {
292 	struct vdpa_device *vdpa = v->vdpa;
293 	size_t size = vdpa->config->get_config_size(vdpa);
294 
295 	if (c->len == 0 || c->off > size)
296 		return -EINVAL;
297 
298 	if (c->len > size - c->off)
299 		return -E2BIG;
300 
301 	return 0;
302 }
303 
304 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
305 				  struct vhost_vdpa_config __user *c)
306 {
307 	struct vdpa_device *vdpa = v->vdpa;
308 	struct vhost_vdpa_config config;
309 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
310 	u8 *buf;
311 
312 	if (copy_from_user(&config, c, size))
313 		return -EFAULT;
314 	if (vhost_vdpa_config_validate(v, &config))
315 		return -EINVAL;
316 	buf = kvzalloc(config.len, GFP_KERNEL);
317 	if (!buf)
318 		return -ENOMEM;
319 
320 	vdpa_get_config(vdpa, config.off, buf, config.len);
321 
322 	if (copy_to_user(c->buf, buf, config.len)) {
323 		kvfree(buf);
324 		return -EFAULT;
325 	}
326 
327 	kvfree(buf);
328 	return 0;
329 }
330 
331 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
332 				  struct vhost_vdpa_config __user *c)
333 {
334 	struct vdpa_device *vdpa = v->vdpa;
335 	struct vhost_vdpa_config config;
336 	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
337 	u8 *buf;
338 
339 	if (copy_from_user(&config, c, size))
340 		return -EFAULT;
341 	if (vhost_vdpa_config_validate(v, &config))
342 		return -EINVAL;
343 
344 	buf = vmemdup_user(c->buf, config.len);
345 	if (IS_ERR(buf))
346 		return PTR_ERR(buf);
347 
348 	vdpa_set_config(vdpa, config.off, buf, config.len);
349 
350 	kvfree(buf);
351 	return 0;
352 }
353 
354 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
355 {
356 	struct vdpa_device *vdpa = v->vdpa;
357 	const struct vdpa_config_ops *ops = vdpa->config;
358 
359 	return ops->suspend;
360 }
361 
362 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
363 {
364 	struct vdpa_device *vdpa = v->vdpa;
365 	const struct vdpa_config_ops *ops = vdpa->config;
366 	u64 features;
367 
368 	features = ops->get_device_features(vdpa);
369 
370 	if (copy_to_user(featurep, &features, sizeof(features)))
371 		return -EFAULT;
372 
373 	return 0;
374 }
375 
376 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
377 {
378 	struct vdpa_device *vdpa = v->vdpa;
379 	const struct vdpa_config_ops *ops = vdpa->config;
380 	u64 features;
381 
382 	/*
383 	 * It's not allowed to change the features after they have
384 	 * been negotiated.
385 	 */
386 	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
387 		return -EBUSY;
388 
389 	if (copy_from_user(&features, featurep, sizeof(features)))
390 		return -EFAULT;
391 
392 	if (vdpa_set_features(vdpa, features))
393 		return -EINVAL;
394 
395 	return 0;
396 }
397 
398 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
399 {
400 	struct vdpa_device *vdpa = v->vdpa;
401 	const struct vdpa_config_ops *ops = vdpa->config;
402 	u16 num;
403 
404 	num = ops->get_vq_num_max(vdpa);
405 
406 	if (copy_to_user(argp, &num, sizeof(num)))
407 		return -EFAULT;
408 
409 	return 0;
410 }
411 
412 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
413 {
414 	if (v->config_ctx) {
415 		eventfd_ctx_put(v->config_ctx);
416 		v->config_ctx = NULL;
417 	}
418 }
419 
420 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
421 {
422 	struct vdpa_callback cb;
423 	int fd;
424 	struct eventfd_ctx *ctx;
425 
426 	cb.callback = vhost_vdpa_config_cb;
427 	cb.private = v;
428 	if (copy_from_user(&fd, argp, sizeof(fd)))
429 		return  -EFAULT;
430 
431 	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
432 	swap(ctx, v->config_ctx);
433 
434 	if (!IS_ERR_OR_NULL(ctx))
435 		eventfd_ctx_put(ctx);
436 
437 	if (IS_ERR(v->config_ctx)) {
438 		long ret = PTR_ERR(v->config_ctx);
439 
440 		v->config_ctx = NULL;
441 		return ret;
442 	}
443 
444 	v->vdpa->config->set_config_cb(v->vdpa, &cb);
445 
446 	return 0;
447 }
448 
449 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
450 {
451 	struct vhost_vdpa_iova_range range = {
452 		.first = v->range.first,
453 		.last = v->range.last,
454 	};
455 
456 	if (copy_to_user(argp, &range, sizeof(range)))
457 		return -EFAULT;
458 	return 0;
459 }
460 
461 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
462 {
463 	struct vdpa_device *vdpa = v->vdpa;
464 	const struct vdpa_config_ops *ops = vdpa->config;
465 	u32 size;
466 
467 	size = ops->get_config_size(vdpa);
468 
469 	if (copy_to_user(argp, &size, sizeof(size)))
470 		return -EFAULT;
471 
472 	return 0;
473 }
474 
475 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
476 {
477 	struct vdpa_device *vdpa = v->vdpa;
478 
479 	if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
480 		return -EFAULT;
481 
482 	return 0;
483 }
484 
485 /* After a successful return of ioctl the device must not process more
486  * virtqueue descriptors. The device can answer to read or writes of config
487  * fields as if it were not suspended. In particular, writing to "queue_enable"
488  * with a value of 1 will not make the device start processing buffers.
489  */
490 static long vhost_vdpa_suspend(struct vhost_vdpa *v)
491 {
492 	struct vdpa_device *vdpa = v->vdpa;
493 	const struct vdpa_config_ops *ops = vdpa->config;
494 
495 	if (!ops->suspend)
496 		return -EOPNOTSUPP;
497 
498 	return ops->suspend(vdpa);
499 }
500 
501 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
502 				   void __user *argp)
503 {
504 	struct vdpa_device *vdpa = v->vdpa;
505 	const struct vdpa_config_ops *ops = vdpa->config;
506 	struct vdpa_vq_state vq_state;
507 	struct vdpa_callback cb;
508 	struct vhost_virtqueue *vq;
509 	struct vhost_vring_state s;
510 	u32 idx;
511 	long r;
512 
513 	r = get_user(idx, (u32 __user *)argp);
514 	if (r < 0)
515 		return r;
516 
517 	if (idx >= v->nvqs)
518 		return -ENOBUFS;
519 
520 	idx = array_index_nospec(idx, v->nvqs);
521 	vq = &v->vqs[idx];
522 
523 	switch (cmd) {
524 	case VHOST_VDPA_SET_VRING_ENABLE:
525 		if (copy_from_user(&s, argp, sizeof(s)))
526 			return -EFAULT;
527 		ops->set_vq_ready(vdpa, idx, s.num);
528 		return 0;
529 	case VHOST_VDPA_GET_VRING_GROUP:
530 		if (!ops->get_vq_group)
531 			return -EOPNOTSUPP;
532 		s.index = idx;
533 		s.num = ops->get_vq_group(vdpa, idx);
534 		if (s.num >= vdpa->ngroups)
535 			return -EIO;
536 		else if (copy_to_user(argp, &s, sizeof(s)))
537 			return -EFAULT;
538 		return 0;
539 	case VHOST_VDPA_SET_GROUP_ASID:
540 		if (copy_from_user(&s, argp, sizeof(s)))
541 			return -EFAULT;
542 		if (s.num >= vdpa->nas)
543 			return -EINVAL;
544 		if (!ops->set_group_asid)
545 			return -EOPNOTSUPP;
546 		return ops->set_group_asid(vdpa, idx, s.num);
547 	case VHOST_GET_VRING_BASE:
548 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);
549 		if (r)
550 			return r;
551 
552 		vq->last_avail_idx = vq_state.split.avail_index;
553 		break;
554 	}
555 
556 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
557 	if (r)
558 		return r;
559 
560 	switch (cmd) {
561 	case VHOST_SET_VRING_ADDR:
562 		if (ops->set_vq_address(vdpa, idx,
563 					(u64)(uintptr_t)vq->desc,
564 					(u64)(uintptr_t)vq->avail,
565 					(u64)(uintptr_t)vq->used))
566 			r = -EINVAL;
567 		break;
568 
569 	case VHOST_SET_VRING_BASE:
570 		vq_state.split.avail_index = vq->last_avail_idx;
571 		if (ops->set_vq_state(vdpa, idx, &vq_state))
572 			r = -EINVAL;
573 		break;
574 
575 	case VHOST_SET_VRING_CALL:
576 		if (vq->call_ctx.ctx) {
577 			cb.callback = vhost_vdpa_virtqueue_cb;
578 			cb.private = vq;
579 		} else {
580 			cb.callback = NULL;
581 			cb.private = NULL;
582 		}
583 		ops->set_vq_cb(vdpa, idx, &cb);
584 		vhost_vdpa_setup_vq_irq(v, idx);
585 		break;
586 
587 	case VHOST_SET_VRING_NUM:
588 		ops->set_vq_num(vdpa, idx, vq->num);
589 		break;
590 	}
591 
592 	return r;
593 }
594 
595 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
596 				      unsigned int cmd, unsigned long arg)
597 {
598 	struct vhost_vdpa *v = filep->private_data;
599 	struct vhost_dev *d = &v->vdev;
600 	void __user *argp = (void __user *)arg;
601 	u64 __user *featurep = argp;
602 	u64 features;
603 	long r = 0;
604 
605 	if (cmd == VHOST_SET_BACKEND_FEATURES) {
606 		if (copy_from_user(&features, featurep, sizeof(features)))
607 			return -EFAULT;
608 		if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
609 				 BIT_ULL(VHOST_BACKEND_F_SUSPEND)))
610 			return -EOPNOTSUPP;
611 		if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
612 		     !vhost_vdpa_can_suspend(v))
613 			return -EOPNOTSUPP;
614 		vhost_set_backend_features(&v->vdev, features);
615 		return 0;
616 	}
617 
618 	mutex_lock(&d->mutex);
619 
620 	switch (cmd) {
621 	case VHOST_VDPA_GET_DEVICE_ID:
622 		r = vhost_vdpa_get_device_id(v, argp);
623 		break;
624 	case VHOST_VDPA_GET_STATUS:
625 		r = vhost_vdpa_get_status(v, argp);
626 		break;
627 	case VHOST_VDPA_SET_STATUS:
628 		r = vhost_vdpa_set_status(v, argp);
629 		break;
630 	case VHOST_VDPA_GET_CONFIG:
631 		r = vhost_vdpa_get_config(v, argp);
632 		break;
633 	case VHOST_VDPA_SET_CONFIG:
634 		r = vhost_vdpa_set_config(v, argp);
635 		break;
636 	case VHOST_GET_FEATURES:
637 		r = vhost_vdpa_get_features(v, argp);
638 		break;
639 	case VHOST_SET_FEATURES:
640 		r = vhost_vdpa_set_features(v, argp);
641 		break;
642 	case VHOST_VDPA_GET_VRING_NUM:
643 		r = vhost_vdpa_get_vring_num(v, argp);
644 		break;
645 	case VHOST_VDPA_GET_GROUP_NUM:
646 		if (copy_to_user(argp, &v->vdpa->ngroups,
647 				 sizeof(v->vdpa->ngroups)))
648 			r = -EFAULT;
649 		break;
650 	case VHOST_VDPA_GET_AS_NUM:
651 		if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
652 			r = -EFAULT;
653 		break;
654 	case VHOST_SET_LOG_BASE:
655 	case VHOST_SET_LOG_FD:
656 		r = -ENOIOCTLCMD;
657 		break;
658 	case VHOST_VDPA_SET_CONFIG_CALL:
659 		r = vhost_vdpa_set_config_call(v, argp);
660 		break;
661 	case VHOST_GET_BACKEND_FEATURES:
662 		features = VHOST_VDPA_BACKEND_FEATURES;
663 		if (vhost_vdpa_can_suspend(v))
664 			features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
665 		if (copy_to_user(featurep, &features, sizeof(features)))
666 			r = -EFAULT;
667 		break;
668 	case VHOST_VDPA_GET_IOVA_RANGE:
669 		r = vhost_vdpa_get_iova_range(v, argp);
670 		break;
671 	case VHOST_VDPA_GET_CONFIG_SIZE:
672 		r = vhost_vdpa_get_config_size(v, argp);
673 		break;
674 	case VHOST_VDPA_GET_VQS_COUNT:
675 		r = vhost_vdpa_get_vqs_count(v, argp);
676 		break;
677 	case VHOST_VDPA_SUSPEND:
678 		r = vhost_vdpa_suspend(v);
679 		break;
680 	default:
681 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
682 		if (r == -ENOIOCTLCMD)
683 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
684 		break;
685 	}
686 
687 	mutex_unlock(&d->mutex);
688 	return r;
689 }
690 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
691 				     struct vhost_iotlb_map *map, u32 asid)
692 {
693 	struct vdpa_device *vdpa = v->vdpa;
694 	const struct vdpa_config_ops *ops = vdpa->config;
695 	if (ops->dma_map) {
696 		ops->dma_unmap(vdpa, asid, map->start, map->size);
697 	} else if (ops->set_map == NULL) {
698 		iommu_unmap(v->domain, map->start, map->size);
699 	}
700 }
701 
702 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
703 				u64 start, u64 last, u32 asid)
704 {
705 	struct vhost_dev *dev = &v->vdev;
706 	struct vhost_iotlb_map *map;
707 	struct page *page;
708 	unsigned long pfn, pinned;
709 
710 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
711 		pinned = PFN_DOWN(map->size);
712 		for (pfn = PFN_DOWN(map->addr);
713 		     pinned > 0; pfn++, pinned--) {
714 			page = pfn_to_page(pfn);
715 			if (map->perm & VHOST_ACCESS_WO)
716 				set_page_dirty_lock(page);
717 			unpin_user_page(page);
718 		}
719 		atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
720 		vhost_vdpa_general_unmap(v, map, asid);
721 		vhost_iotlb_map_free(iotlb, map);
722 	}
723 }
724 
725 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
726 				u64 start, u64 last, u32 asid)
727 {
728 	struct vhost_iotlb_map *map;
729 	struct vdpa_map_file *map_file;
730 
731 	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
732 		map_file = (struct vdpa_map_file *)map->opaque;
733 		fput(map_file->file);
734 		kfree(map_file);
735 		vhost_vdpa_general_unmap(v, map, asid);
736 		vhost_iotlb_map_free(iotlb, map);
737 	}
738 }
739 
740 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
741 				   struct vhost_iotlb *iotlb, u64 start,
742 				   u64 last, u32 asid)
743 {
744 	struct vdpa_device *vdpa = v->vdpa;
745 
746 	if (vdpa->use_va)
747 		return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
748 
749 	return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
750 }
751 
752 static int perm_to_iommu_flags(u32 perm)
753 {
754 	int flags = 0;
755 
756 	switch (perm) {
757 	case VHOST_ACCESS_WO:
758 		flags |= IOMMU_WRITE;
759 		break;
760 	case VHOST_ACCESS_RO:
761 		flags |= IOMMU_READ;
762 		break;
763 	case VHOST_ACCESS_RW:
764 		flags |= (IOMMU_WRITE | IOMMU_READ);
765 		break;
766 	default:
767 		WARN(1, "invalidate vhost IOTLB permission\n");
768 		break;
769 	}
770 
771 	return flags | IOMMU_CACHE;
772 }
773 
774 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
775 			  u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
776 {
777 	struct vhost_dev *dev = &v->vdev;
778 	struct vdpa_device *vdpa = v->vdpa;
779 	const struct vdpa_config_ops *ops = vdpa->config;
780 	u32 asid = iotlb_to_asid(iotlb);
781 	int r = 0;
782 
783 	r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
784 				      pa, perm, opaque);
785 	if (r)
786 		return r;
787 
788 	if (ops->dma_map) {
789 		r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
790 	} else if (ops->set_map) {
791 		if (!v->in_batch)
792 			r = ops->set_map(vdpa, asid, iotlb);
793 	} else {
794 		r = iommu_map(v->domain, iova, pa, size,
795 			      perm_to_iommu_flags(perm), GFP_KERNEL);
796 	}
797 	if (r) {
798 		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
799 		return r;
800 	}
801 
802 	if (!vdpa->use_va)
803 		atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
804 
805 	return 0;
806 }
807 
808 static void vhost_vdpa_unmap(struct vhost_vdpa *v,
809 			     struct vhost_iotlb *iotlb,
810 			     u64 iova, u64 size)
811 {
812 	struct vdpa_device *vdpa = v->vdpa;
813 	const struct vdpa_config_ops *ops = vdpa->config;
814 	u32 asid = iotlb_to_asid(iotlb);
815 
816 	vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
817 
818 	if (ops->set_map) {
819 		if (!v->in_batch)
820 			ops->set_map(vdpa, asid, iotlb);
821 	}
822 	/* If we are in the middle of batch processing, delay the free
823 	 * of AS until BATCH_END.
824 	 */
825 	if (!v->in_batch && !iotlb->nmaps)
826 		vhost_vdpa_remove_as(v, asid);
827 }
828 
829 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
830 			     struct vhost_iotlb *iotlb,
831 			     u64 iova, u64 size, u64 uaddr, u32 perm)
832 {
833 	struct vhost_dev *dev = &v->vdev;
834 	u64 offset, map_size, map_iova = iova;
835 	struct vdpa_map_file *map_file;
836 	struct vm_area_struct *vma;
837 	int ret = 0;
838 
839 	mmap_read_lock(dev->mm);
840 
841 	while (size) {
842 		vma = find_vma(dev->mm, uaddr);
843 		if (!vma) {
844 			ret = -EINVAL;
845 			break;
846 		}
847 		map_size = min(size, vma->vm_end - uaddr);
848 		if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
849 			!(vma->vm_flags & (VM_IO | VM_PFNMAP))))
850 			goto next;
851 
852 		map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
853 		if (!map_file) {
854 			ret = -ENOMEM;
855 			break;
856 		}
857 		offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
858 		map_file->offset = offset;
859 		map_file->file = get_file(vma->vm_file);
860 		ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
861 				     perm, map_file);
862 		if (ret) {
863 			fput(map_file->file);
864 			kfree(map_file);
865 			break;
866 		}
867 next:
868 		size -= map_size;
869 		uaddr += map_size;
870 		map_iova += map_size;
871 	}
872 	if (ret)
873 		vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
874 
875 	mmap_read_unlock(dev->mm);
876 
877 	return ret;
878 }
879 
880 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
881 			     struct vhost_iotlb *iotlb,
882 			     u64 iova, u64 size, u64 uaddr, u32 perm)
883 {
884 	struct vhost_dev *dev = &v->vdev;
885 	struct page **page_list;
886 	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
887 	unsigned int gup_flags = FOLL_LONGTERM;
888 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
889 	unsigned long lock_limit, sz2pin, nchunks, i;
890 	u64 start = iova;
891 	long pinned;
892 	int ret = 0;
893 
894 	/* Limit the use of memory for bookkeeping */
895 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
896 	if (!page_list)
897 		return -ENOMEM;
898 
899 	if (perm & VHOST_ACCESS_WO)
900 		gup_flags |= FOLL_WRITE;
901 
902 	npages = PFN_UP(size + (iova & ~PAGE_MASK));
903 	if (!npages) {
904 		ret = -EINVAL;
905 		goto free;
906 	}
907 
908 	mmap_read_lock(dev->mm);
909 
910 	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
911 	if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
912 		ret = -ENOMEM;
913 		goto unlock;
914 	}
915 
916 	cur_base = uaddr & PAGE_MASK;
917 	iova &= PAGE_MASK;
918 	nchunks = 0;
919 
920 	while (npages) {
921 		sz2pin = min_t(unsigned long, npages, list_size);
922 		pinned = pin_user_pages(cur_base, sz2pin,
923 					gup_flags, page_list, NULL);
924 		if (sz2pin != pinned) {
925 			if (pinned < 0) {
926 				ret = pinned;
927 			} else {
928 				unpin_user_pages(page_list, pinned);
929 				ret = -ENOMEM;
930 			}
931 			goto out;
932 		}
933 		nchunks++;
934 
935 		if (!last_pfn)
936 			map_pfn = page_to_pfn(page_list[0]);
937 
938 		for (i = 0; i < pinned; i++) {
939 			unsigned long this_pfn = page_to_pfn(page_list[i]);
940 			u64 csize;
941 
942 			if (last_pfn && (this_pfn != last_pfn + 1)) {
943 				/* Pin a contiguous chunk of memory */
944 				csize = PFN_PHYS(last_pfn - map_pfn + 1);
945 				ret = vhost_vdpa_map(v, iotlb, iova, csize,
946 						     PFN_PHYS(map_pfn),
947 						     perm, NULL);
948 				if (ret) {
949 					/*
950 					 * Unpin the pages that are left unmapped
951 					 * from this point on in the current
952 					 * page_list. The remaining outstanding
953 					 * ones which may stride across several
954 					 * chunks will be covered in the common
955 					 * error path subsequently.
956 					 */
957 					unpin_user_pages(&page_list[i],
958 							 pinned - i);
959 					goto out;
960 				}
961 
962 				map_pfn = this_pfn;
963 				iova += csize;
964 				nchunks = 0;
965 			}
966 
967 			last_pfn = this_pfn;
968 		}
969 
970 		cur_base += PFN_PHYS(pinned);
971 		npages -= pinned;
972 	}
973 
974 	/* Pin the rest chunk */
975 	ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
976 			     PFN_PHYS(map_pfn), perm, NULL);
977 out:
978 	if (ret) {
979 		if (nchunks) {
980 			unsigned long pfn;
981 
982 			/*
983 			 * Unpin the outstanding pages which are yet to be
984 			 * mapped but haven't due to vdpa_map() or
985 			 * pin_user_pages() failure.
986 			 *
987 			 * Mapped pages are accounted in vdpa_map(), hence
988 			 * the corresponding unpinning will be handled by
989 			 * vdpa_unmap().
990 			 */
991 			WARN_ON(!last_pfn);
992 			for (pfn = map_pfn; pfn <= last_pfn; pfn++)
993 				unpin_user_page(pfn_to_page(pfn));
994 		}
995 		vhost_vdpa_unmap(v, iotlb, start, size);
996 	}
997 unlock:
998 	mmap_read_unlock(dev->mm);
999 free:
1000 	free_page((unsigned long)page_list);
1001 	return ret;
1002 
1003 }
1004 
1005 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1006 					   struct vhost_iotlb *iotlb,
1007 					   struct vhost_iotlb_msg *msg)
1008 {
1009 	struct vdpa_device *vdpa = v->vdpa;
1010 
1011 	if (msg->iova < v->range.first || !msg->size ||
1012 	    msg->iova > U64_MAX - msg->size + 1 ||
1013 	    msg->iova + msg->size - 1 > v->range.last)
1014 		return -EINVAL;
1015 
1016 	if (vhost_iotlb_itree_first(iotlb, msg->iova,
1017 				    msg->iova + msg->size - 1))
1018 		return -EEXIST;
1019 
1020 	if (vdpa->use_va)
1021 		return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1022 					 msg->uaddr, msg->perm);
1023 
1024 	return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1025 				 msg->perm);
1026 }
1027 
1028 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1029 					struct vhost_iotlb_msg *msg)
1030 {
1031 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1032 	struct vdpa_device *vdpa = v->vdpa;
1033 	const struct vdpa_config_ops *ops = vdpa->config;
1034 	struct vhost_iotlb *iotlb = NULL;
1035 	struct vhost_vdpa_as *as = NULL;
1036 	int r = 0;
1037 
1038 	mutex_lock(&dev->mutex);
1039 
1040 	r = vhost_dev_check_owner(dev);
1041 	if (r)
1042 		goto unlock;
1043 
1044 	if (msg->type == VHOST_IOTLB_UPDATE ||
1045 	    msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1046 		as = vhost_vdpa_find_alloc_as(v, asid);
1047 		if (!as) {
1048 			dev_err(&v->dev, "can't find and alloc asid %d\n",
1049 				asid);
1050 			r = -EINVAL;
1051 			goto unlock;
1052 		}
1053 		iotlb = &as->iotlb;
1054 	} else
1055 		iotlb = asid_to_iotlb(v, asid);
1056 
1057 	if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1058 		if (v->in_batch && v->batch_asid != asid) {
1059 			dev_info(&v->dev, "batch id %d asid %d\n",
1060 				 v->batch_asid, asid);
1061 		}
1062 		if (!iotlb)
1063 			dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1064 		r = -EINVAL;
1065 		goto unlock;
1066 	}
1067 
1068 	switch (msg->type) {
1069 	case VHOST_IOTLB_UPDATE:
1070 		r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1071 		break;
1072 	case VHOST_IOTLB_INVALIDATE:
1073 		vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1074 		break;
1075 	case VHOST_IOTLB_BATCH_BEGIN:
1076 		v->batch_asid = asid;
1077 		v->in_batch = true;
1078 		break;
1079 	case VHOST_IOTLB_BATCH_END:
1080 		if (v->in_batch && ops->set_map)
1081 			ops->set_map(vdpa, asid, iotlb);
1082 		v->in_batch = false;
1083 		if (!iotlb->nmaps)
1084 			vhost_vdpa_remove_as(v, asid);
1085 		break;
1086 	default:
1087 		r = -EINVAL;
1088 		break;
1089 	}
1090 unlock:
1091 	mutex_unlock(&dev->mutex);
1092 
1093 	return r;
1094 }
1095 
1096 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1097 					 struct iov_iter *from)
1098 {
1099 	struct file *file = iocb->ki_filp;
1100 	struct vhost_vdpa *v = file->private_data;
1101 	struct vhost_dev *dev = &v->vdev;
1102 
1103 	return vhost_chr_write_iter(dev, from);
1104 }
1105 
1106 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1107 {
1108 	struct vdpa_device *vdpa = v->vdpa;
1109 	const struct vdpa_config_ops *ops = vdpa->config;
1110 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1111 	struct bus_type *bus;
1112 	int ret;
1113 
1114 	/* Device want to do DMA by itself */
1115 	if (ops->set_map || ops->dma_map)
1116 		return 0;
1117 
1118 	bus = dma_dev->bus;
1119 	if (!bus)
1120 		return -EFAULT;
1121 
1122 	if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY))
1123 		return -ENOTSUPP;
1124 
1125 	v->domain = iommu_domain_alloc(bus);
1126 	if (!v->domain)
1127 		return -EIO;
1128 
1129 	ret = iommu_attach_device(v->domain, dma_dev);
1130 	if (ret)
1131 		goto err_attach;
1132 
1133 	return 0;
1134 
1135 err_attach:
1136 	iommu_domain_free(v->domain);
1137 	return ret;
1138 }
1139 
1140 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1141 {
1142 	struct vdpa_device *vdpa = v->vdpa;
1143 	struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1144 
1145 	if (v->domain) {
1146 		iommu_detach_device(v->domain, dma_dev);
1147 		iommu_domain_free(v->domain);
1148 	}
1149 
1150 	v->domain = NULL;
1151 }
1152 
1153 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1154 {
1155 	struct vdpa_iova_range *range = &v->range;
1156 	struct vdpa_device *vdpa = v->vdpa;
1157 	const struct vdpa_config_ops *ops = vdpa->config;
1158 
1159 	if (ops->get_iova_range) {
1160 		*range = ops->get_iova_range(vdpa);
1161 	} else if (v->domain && v->domain->geometry.force_aperture) {
1162 		range->first = v->domain->geometry.aperture_start;
1163 		range->last = v->domain->geometry.aperture_end;
1164 	} else {
1165 		range->first = 0;
1166 		range->last = ULLONG_MAX;
1167 	}
1168 }
1169 
1170 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1171 {
1172 	struct vhost_vdpa_as *as;
1173 	u32 asid;
1174 
1175 	for (asid = 0; asid < v->vdpa->nas; asid++) {
1176 		as = asid_to_as(v, asid);
1177 		if (as)
1178 			vhost_vdpa_remove_as(v, asid);
1179 	}
1180 
1181 	vhost_dev_cleanup(&v->vdev);
1182 	kfree(v->vdev.vqs);
1183 }
1184 
1185 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1186 {
1187 	struct vhost_vdpa *v;
1188 	struct vhost_dev *dev;
1189 	struct vhost_virtqueue **vqs;
1190 	int r, opened;
1191 	u32 i, nvqs;
1192 
1193 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1194 
1195 	opened = atomic_cmpxchg(&v->opened, 0, 1);
1196 	if (opened)
1197 		return -EBUSY;
1198 
1199 	nvqs = v->nvqs;
1200 	r = vhost_vdpa_reset(v);
1201 	if (r)
1202 		goto err;
1203 
1204 	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1205 	if (!vqs) {
1206 		r = -ENOMEM;
1207 		goto err;
1208 	}
1209 
1210 	dev = &v->vdev;
1211 	for (i = 0; i < nvqs; i++) {
1212 		vqs[i] = &v->vqs[i];
1213 		vqs[i]->handle_kick = handle_vq_kick;
1214 	}
1215 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1216 		       vhost_vdpa_process_iotlb_msg);
1217 
1218 	r = vhost_vdpa_alloc_domain(v);
1219 	if (r)
1220 		goto err_alloc_domain;
1221 
1222 	vhost_vdpa_set_iova_range(v);
1223 
1224 	filep->private_data = v;
1225 
1226 	return 0;
1227 
1228 err_alloc_domain:
1229 	vhost_vdpa_cleanup(v);
1230 err:
1231 	atomic_dec(&v->opened);
1232 	return r;
1233 }
1234 
1235 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1236 {
1237 	u32 i;
1238 
1239 	for (i = 0; i < v->nvqs; i++)
1240 		vhost_vdpa_unsetup_vq_irq(v, i);
1241 }
1242 
1243 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1244 {
1245 	struct vhost_vdpa *v = filep->private_data;
1246 	struct vhost_dev *d = &v->vdev;
1247 
1248 	mutex_lock(&d->mutex);
1249 	filep->private_data = NULL;
1250 	vhost_vdpa_clean_irq(v);
1251 	vhost_vdpa_reset(v);
1252 	vhost_dev_stop(&v->vdev);
1253 	vhost_vdpa_free_domain(v);
1254 	vhost_vdpa_config_put(v);
1255 	vhost_vdpa_cleanup(v);
1256 	mutex_unlock(&d->mutex);
1257 
1258 	atomic_dec(&v->opened);
1259 	complete(&v->completion);
1260 
1261 	return 0;
1262 }
1263 
1264 #ifdef CONFIG_MMU
1265 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1266 {
1267 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1268 	struct vdpa_device *vdpa = v->vdpa;
1269 	const struct vdpa_config_ops *ops = vdpa->config;
1270 	struct vdpa_notification_area notify;
1271 	struct vm_area_struct *vma = vmf->vma;
1272 	u16 index = vma->vm_pgoff;
1273 
1274 	notify = ops->get_vq_notification(vdpa, index);
1275 
1276 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1277 	if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
1278 			    PFN_DOWN(notify.addr), PAGE_SIZE,
1279 			    vma->vm_page_prot))
1280 		return VM_FAULT_SIGBUS;
1281 
1282 	return VM_FAULT_NOPAGE;
1283 }
1284 
1285 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1286 	.fault = vhost_vdpa_fault,
1287 };
1288 
1289 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1290 {
1291 	struct vhost_vdpa *v = vma->vm_file->private_data;
1292 	struct vdpa_device *vdpa = v->vdpa;
1293 	const struct vdpa_config_ops *ops = vdpa->config;
1294 	struct vdpa_notification_area notify;
1295 	unsigned long index = vma->vm_pgoff;
1296 
1297 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1298 		return -EINVAL;
1299 	if ((vma->vm_flags & VM_SHARED) == 0)
1300 		return -EINVAL;
1301 	if (vma->vm_flags & VM_READ)
1302 		return -EINVAL;
1303 	if (index > 65535)
1304 		return -EINVAL;
1305 	if (!ops->get_vq_notification)
1306 		return -ENOTSUPP;
1307 
1308 	/* To be safe and easily modelled by userspace, We only
1309 	 * support the doorbell which sits on the page boundary and
1310 	 * does not share the page with other registers.
1311 	 */
1312 	notify = ops->get_vq_notification(vdpa, index);
1313 	if (notify.addr & (PAGE_SIZE - 1))
1314 		return -EINVAL;
1315 	if (vma->vm_end - vma->vm_start != notify.size)
1316 		return -ENOTSUPP;
1317 
1318 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1319 	vma->vm_ops = &vhost_vdpa_vm_ops;
1320 	return 0;
1321 }
1322 #endif /* CONFIG_MMU */
1323 
1324 static const struct file_operations vhost_vdpa_fops = {
1325 	.owner		= THIS_MODULE,
1326 	.open		= vhost_vdpa_open,
1327 	.release	= vhost_vdpa_release,
1328 	.write_iter	= vhost_vdpa_chr_write_iter,
1329 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
1330 #ifdef CONFIG_MMU
1331 	.mmap		= vhost_vdpa_mmap,
1332 #endif /* CONFIG_MMU */
1333 	.compat_ioctl	= compat_ptr_ioctl,
1334 };
1335 
1336 static void vhost_vdpa_release_dev(struct device *device)
1337 {
1338 	struct vhost_vdpa *v =
1339 	       container_of(device, struct vhost_vdpa, dev);
1340 
1341 	ida_simple_remove(&vhost_vdpa_ida, v->minor);
1342 	kfree(v->vqs);
1343 	kfree(v);
1344 }
1345 
1346 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1347 {
1348 	const struct vdpa_config_ops *ops = vdpa->config;
1349 	struct vhost_vdpa *v;
1350 	int minor;
1351 	int i, r;
1352 
1353 	/* We can't support platform IOMMU device with more than 1
1354 	 * group or as
1355 	 */
1356 	if (!ops->set_map && !ops->dma_map &&
1357 	    (vdpa->ngroups > 1 || vdpa->nas > 1))
1358 		return -EOPNOTSUPP;
1359 
1360 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1361 	if (!v)
1362 		return -ENOMEM;
1363 
1364 	minor = ida_simple_get(&vhost_vdpa_ida, 0,
1365 			       VHOST_VDPA_DEV_MAX, GFP_KERNEL);
1366 	if (minor < 0) {
1367 		kfree(v);
1368 		return minor;
1369 	}
1370 
1371 	atomic_set(&v->opened, 0);
1372 	v->minor = minor;
1373 	v->vdpa = vdpa;
1374 	v->nvqs = vdpa->nvqs;
1375 	v->virtio_id = ops->get_device_id(vdpa);
1376 
1377 	device_initialize(&v->dev);
1378 	v->dev.release = vhost_vdpa_release_dev;
1379 	v->dev.parent = &vdpa->dev;
1380 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1381 	v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1382 			       GFP_KERNEL);
1383 	if (!v->vqs) {
1384 		r = -ENOMEM;
1385 		goto err;
1386 	}
1387 
1388 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1389 	if (r)
1390 		goto err;
1391 
1392 	cdev_init(&v->cdev, &vhost_vdpa_fops);
1393 	v->cdev.owner = THIS_MODULE;
1394 
1395 	r = cdev_device_add(&v->cdev, &v->dev);
1396 	if (r)
1397 		goto err;
1398 
1399 	init_completion(&v->completion);
1400 	vdpa_set_drvdata(vdpa, v);
1401 
1402 	for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1403 		INIT_HLIST_HEAD(&v->as[i]);
1404 
1405 	return 0;
1406 
1407 err:
1408 	put_device(&v->dev);
1409 	ida_simple_remove(&vhost_vdpa_ida, v->minor);
1410 	return r;
1411 }
1412 
1413 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1414 {
1415 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1416 	int opened;
1417 
1418 	cdev_device_del(&v->cdev, &v->dev);
1419 
1420 	do {
1421 		opened = atomic_cmpxchg(&v->opened, 0, 1);
1422 		if (!opened)
1423 			break;
1424 		wait_for_completion(&v->completion);
1425 	} while (1);
1426 
1427 	put_device(&v->dev);
1428 }
1429 
1430 static struct vdpa_driver vhost_vdpa_driver = {
1431 	.driver = {
1432 		.name	= "vhost_vdpa",
1433 	},
1434 	.probe	= vhost_vdpa_probe,
1435 	.remove	= vhost_vdpa_remove,
1436 };
1437 
1438 static int __init vhost_vdpa_init(void)
1439 {
1440 	int r;
1441 
1442 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1443 				"vhost-vdpa");
1444 	if (r)
1445 		goto err_alloc_chrdev;
1446 
1447 	r = vdpa_register_driver(&vhost_vdpa_driver);
1448 	if (r)
1449 		goto err_vdpa_register_driver;
1450 
1451 	return 0;
1452 
1453 err_vdpa_register_driver:
1454 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1455 err_alloc_chrdev:
1456 	return r;
1457 }
1458 module_init(vhost_vdpa_init);
1459 
1460 static void __exit vhost_vdpa_exit(void)
1461 {
1462 	vdpa_unregister_driver(&vhost_vdpa_driver);
1463 	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1464 }
1465 module_exit(vhost_vdpa_exit);
1466 
1467 MODULE_VERSION("0.0.1");
1468 MODULE_LICENSE("GPL v2");
1469 MODULE_AUTHOR("Intel Corporation");
1470 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");
1471