xref: /linux/drivers/vdpa/vdpa_user/iova_domain.c (revision 3ea5eb68b9d624935108b5e696859304edfac202) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * MMU-based software IOTLB.
4  *
5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6  *
7  * Author: Xie Yongji <xieyongji@bytedance.com>
8  *
9  */
10 
11 #include <linux/slab.h>
12 #include <linux/file.h>
13 #include <linux/anon_inodes.h>
14 #include <linux/highmem.h>
15 #include <linux/vmalloc.h>
16 #include <linux/vdpa.h>
17 
18 #include "iova_domain.h"
19 
20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,
21 				 u64 start, u64 last,
22 				 u64 addr, unsigned int perm,
23 				 struct file *file, u64 offset)
24 {
25 	struct vdpa_map_file *map_file;
26 	int ret;
27 
28 	map_file = kmalloc(sizeof(*map_file), GFP_ATOMIC);
29 	if (!map_file)
30 		return -ENOMEM;
31 
32 	map_file->file = get_file(file);
33 	map_file->offset = offset;
34 
35 	ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,
36 					addr, perm, map_file);
37 	if (ret) {
38 		fput(map_file->file);
39 		kfree(map_file);
40 		return ret;
41 	}
42 	return 0;
43 }
44 
45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,
46 				  u64 start, u64 last)
47 {
48 	struct vdpa_map_file *map_file;
49 	struct vhost_iotlb_map *map;
50 
51 	while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {
52 		map_file = (struct vdpa_map_file *)map->opaque;
53 		fput(map_file->file);
54 		kfree(map_file);
55 		vhost_iotlb_map_free(domain->iotlb, map);
56 	}
57 }
58 
59 int vduse_domain_set_map(struct vduse_iova_domain *domain,
60 			 struct vhost_iotlb *iotlb)
61 {
62 	struct vdpa_map_file *map_file;
63 	struct vhost_iotlb_map *map;
64 	u64 start = 0ULL, last = ULLONG_MAX;
65 	int ret;
66 
67 	spin_lock(&domain->iotlb_lock);
68 	vduse_iotlb_del_range(domain, start, last);
69 
70 	for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
71 	     map = vhost_iotlb_itree_next(map, start, last)) {
72 		map_file = (struct vdpa_map_file *)map->opaque;
73 		ret = vduse_iotlb_add_range(domain, map->start, map->last,
74 					    map->addr, map->perm,
75 					    map_file->file,
76 					    map_file->offset);
77 		if (ret)
78 			goto err;
79 	}
80 	spin_unlock(&domain->iotlb_lock);
81 
82 	return 0;
83 err:
84 	vduse_iotlb_del_range(domain, start, last);
85 	spin_unlock(&domain->iotlb_lock);
86 	return ret;
87 }
88 
89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,
90 			    struct vhost_iotlb *iotlb)
91 {
92 	struct vhost_iotlb_map *map;
93 	u64 start = 0ULL, last = ULLONG_MAX;
94 
95 	spin_lock(&domain->iotlb_lock);
96 	for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
97 	     map = vhost_iotlb_itree_next(map, start, last)) {
98 		vduse_iotlb_del_range(domain, map->start, map->last);
99 	}
100 	spin_unlock(&domain->iotlb_lock);
101 }
102 
103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
104 					 u64 iova, u64 size, u64 paddr)
105 {
106 	struct vduse_bounce_map *map;
107 	u64 last = iova + size - 1;
108 
109 	while (iova <= last) {
110 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
111 		if (!map->bounce_page) {
112 			map->bounce_page = alloc_page(GFP_ATOMIC);
113 			if (!map->bounce_page)
114 				return -ENOMEM;
115 		}
116 		map->orig_phys = paddr;
117 		paddr += PAGE_SIZE;
118 		iova += PAGE_SIZE;
119 	}
120 	return 0;
121 }
122 
123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
124 					   u64 iova, u64 size)
125 {
126 	struct vduse_bounce_map *map;
127 	u64 last = iova + size - 1;
128 
129 	while (iova <= last) {
130 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
131 		map->orig_phys = INVALID_PHYS_ADDR;
132 		iova += PAGE_SIZE;
133 	}
134 }
135 
136 static void do_bounce(phys_addr_t orig, void *addr, size_t size,
137 		      enum dma_data_direction dir)
138 {
139 	unsigned long pfn = PFN_DOWN(orig);
140 	unsigned int offset = offset_in_page(orig);
141 	struct page *page;
142 	unsigned int sz = 0;
143 
144 	while (size) {
145 		sz = min_t(size_t, PAGE_SIZE - offset, size);
146 
147 		page = pfn_to_page(pfn);
148 		if (dir == DMA_TO_DEVICE)
149 			memcpy_from_page(addr, page, offset, sz);
150 		else
151 			memcpy_to_page(page, offset, addr, sz);
152 
153 		size -= sz;
154 		pfn++;
155 		addr += sz;
156 		offset = 0;
157 	}
158 }
159 
160 static void vduse_domain_bounce(struct vduse_iova_domain *domain,
161 				dma_addr_t iova, size_t size,
162 				enum dma_data_direction dir)
163 {
164 	struct vduse_bounce_map *map;
165 	struct page *page;
166 	unsigned int offset;
167 	void *addr;
168 	size_t sz;
169 
170 	if (iova >= domain->bounce_size)
171 		return;
172 
173 	while (size) {
174 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
175 		offset = offset_in_page(iova);
176 		sz = min_t(size_t, PAGE_SIZE - offset, size);
177 
178 		if (WARN_ON(!map->bounce_page ||
179 			    map->orig_phys == INVALID_PHYS_ADDR))
180 			return;
181 
182 		page = domain->user_bounce_pages ?
183 		       map->user_bounce_page : map->bounce_page;
184 
185 		addr = kmap_local_page(page);
186 		do_bounce(map->orig_phys + offset, addr + offset, sz, dir);
187 		kunmap_local(addr);
188 		size -= sz;
189 		iova += sz;
190 	}
191 }
192 
193 static struct page *
194 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)
195 {
196 	u64 start = iova & PAGE_MASK;
197 	u64 last = start + PAGE_SIZE - 1;
198 	struct vhost_iotlb_map *map;
199 	struct page *page = NULL;
200 
201 	spin_lock(&domain->iotlb_lock);
202 	map = vhost_iotlb_itree_first(domain->iotlb, start, last);
203 	if (!map)
204 		goto out;
205 
206 	page = pfn_to_page((map->addr + iova - map->start) >> PAGE_SHIFT);
207 	get_page(page);
208 out:
209 	spin_unlock(&domain->iotlb_lock);
210 
211 	return page;
212 }
213 
214 static struct page *
215 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
216 {
217 	struct vduse_bounce_map *map;
218 	struct page *page = NULL;
219 
220 	read_lock(&domain->bounce_lock);
221 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];
222 	if (domain->user_bounce_pages || !map->bounce_page)
223 		goto out;
224 
225 	page = map->bounce_page;
226 	get_page(page);
227 out:
228 	read_unlock(&domain->bounce_lock);
229 
230 	return page;
231 }
232 
233 static void
234 vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
235 {
236 	struct vduse_bounce_map *map;
237 	unsigned long pfn, bounce_pfns;
238 
239 	bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
240 
241 	for (pfn = 0; pfn < bounce_pfns; pfn++) {
242 		map = &domain->bounce_maps[pfn];
243 		if (WARN_ON(map->orig_phys != INVALID_PHYS_ADDR))
244 			continue;
245 
246 		if (!map->bounce_page)
247 			continue;
248 
249 		__free_page(map->bounce_page);
250 		map->bounce_page = NULL;
251 	}
252 }
253 
254 int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
255 				       struct page **pages, int count)
256 {
257 	struct vduse_bounce_map *map;
258 	int i, ret;
259 
260 	/* Now we don't support partial mapping */
261 	if (count != (domain->bounce_size >> PAGE_SHIFT))
262 		return -EINVAL;
263 
264 	write_lock(&domain->bounce_lock);
265 	ret = -EEXIST;
266 	if (domain->user_bounce_pages)
267 		goto out;
268 
269 	for (i = 0; i < count; i++) {
270 		map = &domain->bounce_maps[i];
271 		if (map->bounce_page) {
272 			/* Copy kernel page to user page if it's in use */
273 			if (map->orig_phys != INVALID_PHYS_ADDR)
274 				memcpy_to_page(pages[i], 0,
275 					       page_address(map->bounce_page),
276 					       PAGE_SIZE);
277 		}
278 		map->user_bounce_page = pages[i];
279 		get_page(pages[i]);
280 	}
281 	domain->user_bounce_pages = true;
282 	ret = 0;
283 out:
284 	write_unlock(&domain->bounce_lock);
285 
286 	return ret;
287 }
288 
289 void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
290 {
291 	struct vduse_bounce_map *map;
292 	unsigned long i, count;
293 
294 	write_lock(&domain->bounce_lock);
295 	if (!domain->user_bounce_pages)
296 		goto out;
297 
298 	count = domain->bounce_size >> PAGE_SHIFT;
299 	for (i = 0; i < count; i++) {
300 		struct page *page = NULL;
301 
302 		map = &domain->bounce_maps[i];
303 		if (WARN_ON(!map->user_bounce_page))
304 			continue;
305 
306 		/* Copy user page to kernel page if it's in use */
307 		if (map->orig_phys != INVALID_PHYS_ADDR) {
308 			page = map->bounce_page;
309 			memcpy_from_page(page_address(page),
310 					 map->user_bounce_page, 0, PAGE_SIZE);
311 		}
312 		put_page(map->user_bounce_page);
313 		map->user_bounce_page = NULL;
314 	}
315 	domain->user_bounce_pages = false;
316 out:
317 	write_unlock(&domain->bounce_lock);
318 }
319 
320 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
321 {
322 	if (!domain->bounce_map)
323 		return;
324 
325 	spin_lock(&domain->iotlb_lock);
326 	if (!domain->bounce_map)
327 		goto unlock;
328 
329 	vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);
330 	domain->bounce_map = 0;
331 unlock:
332 	spin_unlock(&domain->iotlb_lock);
333 }
334 
335 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
336 {
337 	int ret = 0;
338 
339 	if (domain->bounce_map)
340 		return 0;
341 
342 	spin_lock(&domain->iotlb_lock);
343 	if (domain->bounce_map)
344 		goto unlock;
345 
346 	ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,
347 				    0, VHOST_MAP_RW, domain->file, 0);
348 	if (ret)
349 		goto unlock;
350 
351 	domain->bounce_map = 1;
352 unlock:
353 	spin_unlock(&domain->iotlb_lock);
354 	return ret;
355 }
356 
357 static dma_addr_t
358 vduse_domain_alloc_iova(struct iova_domain *iovad,
359 			unsigned long size, unsigned long limit)
360 {
361 	unsigned long shift = iova_shift(iovad);
362 	unsigned long iova_len = iova_align(iovad, size) >> shift;
363 	unsigned long iova_pfn;
364 
365 	iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
366 
367 	return (dma_addr_t)iova_pfn << shift;
368 }
369 
370 static void vduse_domain_free_iova(struct iova_domain *iovad,
371 				   dma_addr_t iova, size_t size)
372 {
373 	unsigned long shift = iova_shift(iovad);
374 	unsigned long iova_len = iova_align(iovad, size) >> shift;
375 
376 	free_iova_fast(iovad, iova >> shift, iova_len);
377 }
378 
379 void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain,
380 				      dma_addr_t dma_addr, size_t size,
381 				      enum dma_data_direction dir)
382 {
383 	read_lock(&domain->bounce_lock);
384 	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
385 		vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE);
386 	read_unlock(&domain->bounce_lock);
387 }
388 
389 void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain,
390 				      dma_addr_t dma_addr, size_t size,
391 				      enum dma_data_direction dir)
392 {
393 	read_lock(&domain->bounce_lock);
394 	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
395 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
396 	read_unlock(&domain->bounce_lock);
397 }
398 
399 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
400 				 struct page *page, unsigned long offset,
401 				 size_t size, enum dma_data_direction dir,
402 				 unsigned long attrs)
403 {
404 	struct iova_domain *iovad = &domain->stream_iovad;
405 	unsigned long limit = domain->bounce_size - 1;
406 	phys_addr_t pa = page_to_phys(page) + offset;
407 	dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
408 
409 	if (!iova)
410 		return DMA_MAPPING_ERROR;
411 
412 	if (vduse_domain_init_bounce_map(domain))
413 		goto err;
414 
415 	read_lock(&domain->bounce_lock);
416 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
417 		goto err_unlock;
418 
419 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
420 	    (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
421 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
422 
423 	read_unlock(&domain->bounce_lock);
424 
425 	return iova;
426 err_unlock:
427 	read_unlock(&domain->bounce_lock);
428 err:
429 	vduse_domain_free_iova(iovad, iova, size);
430 	return DMA_MAPPING_ERROR;
431 }
432 
433 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
434 			     dma_addr_t dma_addr, size_t size,
435 			     enum dma_data_direction dir, unsigned long attrs)
436 {
437 	struct iova_domain *iovad = &domain->stream_iovad;
438 	read_lock(&domain->bounce_lock);
439 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
440 	    (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
441 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
442 
443 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
444 	read_unlock(&domain->bounce_lock);
445 	vduse_domain_free_iova(iovad, dma_addr, size);
446 }
447 
448 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
449 				  size_t size, dma_addr_t *dma_addr,
450 				  gfp_t flag, unsigned long attrs)
451 {
452 	struct iova_domain *iovad = &domain->consistent_iovad;
453 	unsigned long limit = domain->iova_limit;
454 	dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
455 	void *orig = alloc_pages_exact(size, flag);
456 
457 	if (!iova || !orig)
458 		goto err;
459 
460 	spin_lock(&domain->iotlb_lock);
461 	if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,
462 				  virt_to_phys(orig), VHOST_MAP_RW,
463 				  domain->file, (u64)iova)) {
464 		spin_unlock(&domain->iotlb_lock);
465 		goto err;
466 	}
467 	spin_unlock(&domain->iotlb_lock);
468 
469 	*dma_addr = iova;
470 
471 	return orig;
472 err:
473 	*dma_addr = DMA_MAPPING_ERROR;
474 	if (orig)
475 		free_pages_exact(orig, size);
476 	if (iova)
477 		vduse_domain_free_iova(iovad, iova, size);
478 
479 	return NULL;
480 }
481 
482 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
483 				void *vaddr, dma_addr_t dma_addr,
484 				unsigned long attrs)
485 {
486 	struct iova_domain *iovad = &domain->consistent_iovad;
487 	struct vhost_iotlb_map *map;
488 	struct vdpa_map_file *map_file;
489 	phys_addr_t pa;
490 
491 	spin_lock(&domain->iotlb_lock);
492 	map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,
493 				      (u64)dma_addr + size - 1);
494 	if (WARN_ON(!map)) {
495 		spin_unlock(&domain->iotlb_lock);
496 		return;
497 	}
498 	map_file = (struct vdpa_map_file *)map->opaque;
499 	fput(map_file->file);
500 	kfree(map_file);
501 	pa = map->addr;
502 	vhost_iotlb_map_free(domain->iotlb, map);
503 	spin_unlock(&domain->iotlb_lock);
504 
505 	vduse_domain_free_iova(iovad, dma_addr, size);
506 	free_pages_exact(phys_to_virt(pa), size);
507 }
508 
509 static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf)
510 {
511 	struct vduse_iova_domain *domain = vmf->vma->vm_private_data;
512 	unsigned long iova = vmf->pgoff << PAGE_SHIFT;
513 	struct page *page;
514 
515 	if (!domain)
516 		return VM_FAULT_SIGBUS;
517 
518 	if (iova < domain->bounce_size)
519 		page = vduse_domain_get_bounce_page(domain, iova);
520 	else
521 		page = vduse_domain_get_coherent_page(domain, iova);
522 
523 	if (!page)
524 		return VM_FAULT_SIGBUS;
525 
526 	vmf->page = page;
527 
528 	return 0;
529 }
530 
531 static const struct vm_operations_struct vduse_domain_mmap_ops = {
532 	.fault = vduse_domain_mmap_fault,
533 };
534 
535 static int vduse_domain_mmap(struct file *file, struct vm_area_struct *vma)
536 {
537 	struct vduse_iova_domain *domain = file->private_data;
538 
539 	vm_flags_set(vma, VM_DONTDUMP | VM_DONTEXPAND);
540 	vma->vm_private_data = domain;
541 	vma->vm_ops = &vduse_domain_mmap_ops;
542 
543 	return 0;
544 }
545 
546 static int vduse_domain_release(struct inode *inode, struct file *file)
547 {
548 	struct vduse_iova_domain *domain = file->private_data;
549 
550 	spin_lock(&domain->iotlb_lock);
551 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
552 	vduse_domain_remove_user_bounce_pages(domain);
553 	vduse_domain_free_kernel_bounce_pages(domain);
554 	spin_unlock(&domain->iotlb_lock);
555 	put_iova_domain(&domain->stream_iovad);
556 	put_iova_domain(&domain->consistent_iovad);
557 	vhost_iotlb_free(domain->iotlb);
558 	vfree(domain->bounce_maps);
559 	kfree(domain);
560 
561 	return 0;
562 }
563 
564 static const struct file_operations vduse_domain_fops = {
565 	.owner = THIS_MODULE,
566 	.mmap = vduse_domain_mmap,
567 	.release = vduse_domain_release,
568 };
569 
570 void vduse_domain_destroy(struct vduse_iova_domain *domain)
571 {
572 	fput(domain->file);
573 }
574 
575 struct vduse_iova_domain *
576 vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
577 {
578 	struct vduse_iova_domain *domain;
579 	struct file *file;
580 	struct vduse_bounce_map *map;
581 	unsigned long pfn, bounce_pfns;
582 	int ret;
583 
584 	bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
585 	if (iova_limit <= bounce_size)
586 		return NULL;
587 
588 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
589 	if (!domain)
590 		return NULL;
591 
592 	domain->iotlb = vhost_iotlb_alloc(0, 0);
593 	if (!domain->iotlb)
594 		goto err_iotlb;
595 
596 	domain->iova_limit = iova_limit;
597 	domain->bounce_size = PAGE_ALIGN(bounce_size);
598 	domain->bounce_maps = vzalloc(bounce_pfns *
599 				sizeof(struct vduse_bounce_map));
600 	if (!domain->bounce_maps)
601 		goto err_map;
602 
603 	for (pfn = 0; pfn < bounce_pfns; pfn++) {
604 		map = &domain->bounce_maps[pfn];
605 		map->orig_phys = INVALID_PHYS_ADDR;
606 	}
607 	file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops,
608 				domain, O_RDWR);
609 	if (IS_ERR(file))
610 		goto err_file;
611 
612 	domain->file = file;
613 	rwlock_init(&domain->bounce_lock);
614 	spin_lock_init(&domain->iotlb_lock);
615 	init_iova_domain(&domain->stream_iovad,
616 			PAGE_SIZE, IOVA_START_PFN);
617 	ret = iova_domain_init_rcaches(&domain->stream_iovad);
618 	if (ret)
619 		goto err_iovad_stream;
620 	init_iova_domain(&domain->consistent_iovad,
621 			PAGE_SIZE, bounce_pfns);
622 	ret = iova_domain_init_rcaches(&domain->consistent_iovad);
623 	if (ret)
624 		goto err_iovad_consistent;
625 
626 	return domain;
627 err_iovad_consistent:
628 	put_iova_domain(&domain->stream_iovad);
629 err_iovad_stream:
630 	fput(file);
631 err_file:
632 	vfree(domain->bounce_maps);
633 err_map:
634 	vhost_iotlb_free(domain->iotlb);
635 err_iotlb:
636 	kfree(domain);
637 	return NULL;
638 }
639 
640 int vduse_domain_init(void)
641 {
642 	return iova_cache_get();
643 }
644 
645 void vduse_domain_exit(void)
646 {
647 	iova_cache_put();
648 }
649