18c773d53SXie Yongji // SPDX-License-Identifier: GPL-2.0-only 28c773d53SXie Yongji /* 38c773d53SXie Yongji * MMU-based software IOTLB. 48c773d53SXie Yongji * 58c773d53SXie Yongji * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 68c773d53SXie Yongji * 78c773d53SXie Yongji * Author: Xie Yongji <xieyongji@bytedance.com> 88c773d53SXie Yongji * 98c773d53SXie Yongji */ 108c773d53SXie Yongji 118c773d53SXie Yongji #include <linux/slab.h> 128c773d53SXie Yongji #include <linux/file.h> 138c773d53SXie Yongji #include <linux/anon_inodes.h> 148c773d53SXie Yongji #include <linux/highmem.h> 158c773d53SXie Yongji #include <linux/vmalloc.h> 168c773d53SXie Yongji #include <linux/vdpa.h> 178c773d53SXie Yongji 188c773d53SXie Yongji #include "iova_domain.h" 198c773d53SXie Yongji 208c773d53SXie Yongji static int vduse_iotlb_add_range(struct vduse_iova_domain *domain, 218c773d53SXie Yongji u64 start, u64 last, 228c773d53SXie Yongji u64 addr, unsigned int perm, 238c773d53SXie Yongji struct file *file, u64 offset) 248c773d53SXie Yongji { 258c773d53SXie Yongji struct vdpa_map_file *map_file; 268c773d53SXie Yongji int ret; 278c773d53SXie Yongji 288c773d53SXie Yongji map_file = kmalloc(sizeof(*map_file), GFP_ATOMIC); 298c773d53SXie Yongji if (!map_file) 308c773d53SXie Yongji return -ENOMEM; 318c773d53SXie Yongji 328c773d53SXie Yongji map_file->file = get_file(file); 338c773d53SXie Yongji map_file->offset = offset; 348c773d53SXie Yongji 358c773d53SXie Yongji ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last, 368c773d53SXie Yongji addr, perm, map_file); 378c773d53SXie Yongji if (ret) { 388c773d53SXie Yongji fput(map_file->file); 398c773d53SXie Yongji kfree(map_file); 408c773d53SXie Yongji return ret; 418c773d53SXie Yongji } 428c773d53SXie Yongji return 0; 438c773d53SXie Yongji } 448c773d53SXie Yongji 458c773d53SXie Yongji static void vduse_iotlb_del_range(struct vduse_iova_domain *domain, 468c773d53SXie Yongji u64 start, u64 last) 478c773d53SXie Yongji { 488c773d53SXie Yongji struct vdpa_map_file *map_file; 498c773d53SXie Yongji struct vhost_iotlb_map *map; 508c773d53SXie Yongji 518c773d53SXie Yongji while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) { 528c773d53SXie Yongji map_file = (struct vdpa_map_file *)map->opaque; 538c773d53SXie Yongji fput(map_file->file); 548c773d53SXie Yongji kfree(map_file); 558c773d53SXie Yongji vhost_iotlb_map_free(domain->iotlb, map); 568c773d53SXie Yongji } 578c773d53SXie Yongji } 588c773d53SXie Yongji 598c773d53SXie Yongji int vduse_domain_set_map(struct vduse_iova_domain *domain, 608c773d53SXie Yongji struct vhost_iotlb *iotlb) 618c773d53SXie Yongji { 628c773d53SXie Yongji struct vdpa_map_file *map_file; 638c773d53SXie Yongji struct vhost_iotlb_map *map; 648c773d53SXie Yongji u64 start = 0ULL, last = ULLONG_MAX; 658c773d53SXie Yongji int ret; 668c773d53SXie Yongji 678c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 688c773d53SXie Yongji vduse_iotlb_del_range(domain, start, last); 698c773d53SXie Yongji 708c773d53SXie Yongji for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 718c773d53SXie Yongji map = vhost_iotlb_itree_next(map, start, last)) { 728c773d53SXie Yongji map_file = (struct vdpa_map_file *)map->opaque; 738c773d53SXie Yongji ret = vduse_iotlb_add_range(domain, map->start, map->last, 748c773d53SXie Yongji map->addr, map->perm, 758c773d53SXie Yongji map_file->file, 768c773d53SXie Yongji map_file->offset); 778c773d53SXie Yongji if (ret) 788c773d53SXie Yongji goto err; 798c773d53SXie Yongji } 808c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 818c773d53SXie Yongji 828c773d53SXie Yongji return 0; 838c773d53SXie Yongji err: 848c773d53SXie Yongji vduse_iotlb_del_range(domain, start, last); 858c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 868c773d53SXie Yongji return ret; 878c773d53SXie Yongji } 888c773d53SXie Yongji 898c773d53SXie Yongji void vduse_domain_clear_map(struct vduse_iova_domain *domain, 908c773d53SXie Yongji struct vhost_iotlb *iotlb) 918c773d53SXie Yongji { 928c773d53SXie Yongji struct vhost_iotlb_map *map; 938c773d53SXie Yongji u64 start = 0ULL, last = ULLONG_MAX; 948c773d53SXie Yongji 958c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 968c773d53SXie Yongji for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 978c773d53SXie Yongji map = vhost_iotlb_itree_next(map, start, last)) { 988c773d53SXie Yongji vduse_iotlb_del_range(domain, map->start, map->last); 998c773d53SXie Yongji } 1008c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 1018c773d53SXie Yongji } 1028c773d53SXie Yongji 1038c773d53SXie Yongji static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain, 1048c773d53SXie Yongji u64 iova, u64 size, u64 paddr) 1058c773d53SXie Yongji { 1068c773d53SXie Yongji struct vduse_bounce_map *map; 1078c773d53SXie Yongji u64 last = iova + size - 1; 1088c773d53SXie Yongji 1098c773d53SXie Yongji while (iova <= last) { 1108c773d53SXie Yongji map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 1118c773d53SXie Yongji if (!map->bounce_page) { 1128c773d53SXie Yongji map->bounce_page = alloc_page(GFP_ATOMIC); 1138c773d53SXie Yongji if (!map->bounce_page) 1148c773d53SXie Yongji return -ENOMEM; 1158c773d53SXie Yongji } 1168c773d53SXie Yongji map->orig_phys = paddr; 1178c773d53SXie Yongji paddr += PAGE_SIZE; 1188c773d53SXie Yongji iova += PAGE_SIZE; 1198c773d53SXie Yongji } 1208c773d53SXie Yongji return 0; 1218c773d53SXie Yongji } 1228c773d53SXie Yongji 1238c773d53SXie Yongji static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain, 1248c773d53SXie Yongji u64 iova, u64 size) 1258c773d53SXie Yongji { 1268c773d53SXie Yongji struct vduse_bounce_map *map; 1278c773d53SXie Yongji u64 last = iova + size - 1; 1288c773d53SXie Yongji 1298c773d53SXie Yongji while (iova <= last) { 1308c773d53SXie Yongji map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 1318c773d53SXie Yongji map->orig_phys = INVALID_PHYS_ADDR; 1328c773d53SXie Yongji iova += PAGE_SIZE; 1338c773d53SXie Yongji } 1348c773d53SXie Yongji } 1358c773d53SXie Yongji 1368c773d53SXie Yongji static void do_bounce(phys_addr_t orig, void *addr, size_t size, 1378c773d53SXie Yongji enum dma_data_direction dir) 1388c773d53SXie Yongji { 1398c773d53SXie Yongji unsigned long pfn = PFN_DOWN(orig); 1408c773d53SXie Yongji unsigned int offset = offset_in_page(orig); 14182eb46f9SXie Yongji struct page *page; 1428c773d53SXie Yongji unsigned int sz = 0; 1438c773d53SXie Yongji 1448c773d53SXie Yongji while (size) { 1458c773d53SXie Yongji sz = min_t(size_t, PAGE_SIZE - offset, size); 1468c773d53SXie Yongji 14782eb46f9SXie Yongji page = pfn_to_page(pfn); 1488c773d53SXie Yongji if (dir == DMA_TO_DEVICE) 14982eb46f9SXie Yongji memcpy_from_page(addr, page, offset, sz); 1508c773d53SXie Yongji else 15182eb46f9SXie Yongji memcpy_to_page(page, offset, addr, sz); 1528c773d53SXie Yongji 1538c773d53SXie Yongji size -= sz; 1548c773d53SXie Yongji pfn++; 1558c773d53SXie Yongji addr += sz; 1568c773d53SXie Yongji offset = 0; 1578c773d53SXie Yongji } 1588c773d53SXie Yongji } 1598c773d53SXie Yongji 1608c773d53SXie Yongji static void vduse_domain_bounce(struct vduse_iova_domain *domain, 1618c773d53SXie Yongji dma_addr_t iova, size_t size, 1628c773d53SXie Yongji enum dma_data_direction dir) 1638c773d53SXie Yongji { 1648c773d53SXie Yongji struct vduse_bounce_map *map; 165*955abe0aSJason Wang struct page *page; 1668c773d53SXie Yongji unsigned int offset; 1678c773d53SXie Yongji void *addr; 1688c773d53SXie Yongji size_t sz; 1698c773d53SXie Yongji 1708c773d53SXie Yongji if (iova >= domain->bounce_size) 1718c773d53SXie Yongji return; 1728c773d53SXie Yongji 1738c773d53SXie Yongji while (size) { 1748c773d53SXie Yongji map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 1758c773d53SXie Yongji offset = offset_in_page(iova); 1768c773d53SXie Yongji sz = min_t(size_t, PAGE_SIZE - offset, size); 1778c773d53SXie Yongji 1788c773d53SXie Yongji if (WARN_ON(!map->bounce_page || 1798c773d53SXie Yongji map->orig_phys == INVALID_PHYS_ADDR)) 1808c773d53SXie Yongji return; 1818c773d53SXie Yongji 182*955abe0aSJason Wang page = domain->user_bounce_pages ? 183*955abe0aSJason Wang map->user_bounce_page : map->bounce_page; 184*955abe0aSJason Wang 185*955abe0aSJason Wang addr = kmap_local_page(page); 1866c77ed22SXie Yongji do_bounce(map->orig_phys + offset, addr + offset, sz, dir); 1876c77ed22SXie Yongji kunmap_local(addr); 1888c773d53SXie Yongji size -= sz; 1898c773d53SXie Yongji iova += sz; 1908c773d53SXie Yongji } 1918c773d53SXie Yongji } 1928c773d53SXie Yongji 1938c773d53SXie Yongji static struct page * 1948c773d53SXie Yongji vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova) 1958c773d53SXie Yongji { 1968c773d53SXie Yongji u64 start = iova & PAGE_MASK; 1978c773d53SXie Yongji u64 last = start + PAGE_SIZE - 1; 1988c773d53SXie Yongji struct vhost_iotlb_map *map; 1998c773d53SXie Yongji struct page *page = NULL; 2008c773d53SXie Yongji 2018c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 2028c773d53SXie Yongji map = vhost_iotlb_itree_first(domain->iotlb, start, last); 2038c773d53SXie Yongji if (!map) 2048c773d53SXie Yongji goto out; 2058c773d53SXie Yongji 2068c773d53SXie Yongji page = pfn_to_page((map->addr + iova - map->start) >> PAGE_SHIFT); 2078c773d53SXie Yongji get_page(page); 2088c773d53SXie Yongji out: 2098c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 2108c773d53SXie Yongji 2118c773d53SXie Yongji return page; 2128c773d53SXie Yongji } 2138c773d53SXie Yongji 2148c773d53SXie Yongji static struct page * 2158c773d53SXie Yongji vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova) 2168c773d53SXie Yongji { 2178c773d53SXie Yongji struct vduse_bounce_map *map; 2186c77ed22SXie Yongji struct page *page = NULL; 2198c773d53SXie Yongji 2206c77ed22SXie Yongji read_lock(&domain->bounce_lock); 2218c773d53SXie Yongji map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 2226c77ed22SXie Yongji if (domain->user_bounce_pages || !map->bounce_page) 2236c77ed22SXie Yongji goto out; 2248c773d53SXie Yongji 2258c773d53SXie Yongji page = map->bounce_page; 2268c773d53SXie Yongji get_page(page); 2276c77ed22SXie Yongji out: 2286c77ed22SXie Yongji read_unlock(&domain->bounce_lock); 2298c773d53SXie Yongji 2308c773d53SXie Yongji return page; 2318c773d53SXie Yongji } 2328c773d53SXie Yongji 2338c773d53SXie Yongji static void 2346c77ed22SXie Yongji vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain) 2358c773d53SXie Yongji { 2368c773d53SXie Yongji struct vduse_bounce_map *map; 2378c773d53SXie Yongji unsigned long pfn, bounce_pfns; 2388c773d53SXie Yongji 2398c773d53SXie Yongji bounce_pfns = domain->bounce_size >> PAGE_SHIFT; 2408c773d53SXie Yongji 2418c773d53SXie Yongji for (pfn = 0; pfn < bounce_pfns; pfn++) { 2428c773d53SXie Yongji map = &domain->bounce_maps[pfn]; 2438c773d53SXie Yongji if (WARN_ON(map->orig_phys != INVALID_PHYS_ADDR)) 2448c773d53SXie Yongji continue; 2458c773d53SXie Yongji 2468c773d53SXie Yongji if (!map->bounce_page) 2478c773d53SXie Yongji continue; 2488c773d53SXie Yongji 2498c773d53SXie Yongji __free_page(map->bounce_page); 2508c773d53SXie Yongji map->bounce_page = NULL; 2518c773d53SXie Yongji } 2528c773d53SXie Yongji } 2538c773d53SXie Yongji 2546c77ed22SXie Yongji int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain, 2556c77ed22SXie Yongji struct page **pages, int count) 2566c77ed22SXie Yongji { 2576c77ed22SXie Yongji struct vduse_bounce_map *map; 2586c77ed22SXie Yongji int i, ret; 2596c77ed22SXie Yongji 2606c77ed22SXie Yongji /* Now we don't support partial mapping */ 2616c77ed22SXie Yongji if (count != (domain->bounce_size >> PAGE_SHIFT)) 2626c77ed22SXie Yongji return -EINVAL; 2636c77ed22SXie Yongji 2646c77ed22SXie Yongji write_lock(&domain->bounce_lock); 2656c77ed22SXie Yongji ret = -EEXIST; 2666c77ed22SXie Yongji if (domain->user_bounce_pages) 2676c77ed22SXie Yongji goto out; 2686c77ed22SXie Yongji 2696c77ed22SXie Yongji for (i = 0; i < count; i++) { 2706c77ed22SXie Yongji map = &domain->bounce_maps[i]; 2716c77ed22SXie Yongji if (map->bounce_page) { 2726c77ed22SXie Yongji /* Copy kernel page to user page if it's in use */ 2736c77ed22SXie Yongji if (map->orig_phys != INVALID_PHYS_ADDR) 2746c77ed22SXie Yongji memcpy_to_page(pages[i], 0, 2756c77ed22SXie Yongji page_address(map->bounce_page), 2766c77ed22SXie Yongji PAGE_SIZE); 2776c77ed22SXie Yongji } 278*955abe0aSJason Wang map->user_bounce_page = pages[i]; 2796c77ed22SXie Yongji get_page(pages[i]); 2806c77ed22SXie Yongji } 2816c77ed22SXie Yongji domain->user_bounce_pages = true; 2826c77ed22SXie Yongji ret = 0; 2836c77ed22SXie Yongji out: 2846c77ed22SXie Yongji write_unlock(&domain->bounce_lock); 2856c77ed22SXie Yongji 2866c77ed22SXie Yongji return ret; 2876c77ed22SXie Yongji } 2886c77ed22SXie Yongji 2896c77ed22SXie Yongji void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain) 2906c77ed22SXie Yongji { 2916c77ed22SXie Yongji struct vduse_bounce_map *map; 2926c77ed22SXie Yongji unsigned long i, count; 2936c77ed22SXie Yongji 2946c77ed22SXie Yongji write_lock(&domain->bounce_lock); 2956c77ed22SXie Yongji if (!domain->user_bounce_pages) 2966c77ed22SXie Yongji goto out; 2976c77ed22SXie Yongji 2986c77ed22SXie Yongji count = domain->bounce_size >> PAGE_SHIFT; 2996c77ed22SXie Yongji for (i = 0; i < count; i++) { 3006c77ed22SXie Yongji struct page *page = NULL; 3016c77ed22SXie Yongji 3026c77ed22SXie Yongji map = &domain->bounce_maps[i]; 303*955abe0aSJason Wang if (WARN_ON(!map->user_bounce_page)) 3046c77ed22SXie Yongji continue; 3056c77ed22SXie Yongji 3066c77ed22SXie Yongji /* Copy user page to kernel page if it's in use */ 3076c77ed22SXie Yongji if (map->orig_phys != INVALID_PHYS_ADDR) { 308*955abe0aSJason Wang page = map->bounce_page; 3096c77ed22SXie Yongji memcpy_from_page(page_address(page), 310*955abe0aSJason Wang map->user_bounce_page, 0, PAGE_SIZE); 3116c77ed22SXie Yongji } 312*955abe0aSJason Wang put_page(map->user_bounce_page); 313*955abe0aSJason Wang map->user_bounce_page = NULL; 3146c77ed22SXie Yongji } 3156c77ed22SXie Yongji domain->user_bounce_pages = false; 3166c77ed22SXie Yongji out: 3176c77ed22SXie Yongji write_unlock(&domain->bounce_lock); 3186c77ed22SXie Yongji } 3196c77ed22SXie Yongji 3208c773d53SXie Yongji void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain) 3218c773d53SXie Yongji { 3228c773d53SXie Yongji if (!domain->bounce_map) 3238c773d53SXie Yongji return; 3248c773d53SXie Yongji 3258c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 3268c773d53SXie Yongji if (!domain->bounce_map) 3278c773d53SXie Yongji goto unlock; 3288c773d53SXie Yongji 3298c773d53SXie Yongji vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1); 3308c773d53SXie Yongji domain->bounce_map = 0; 3318c773d53SXie Yongji unlock: 3328c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 3338c773d53SXie Yongji } 3348c773d53SXie Yongji 3358c773d53SXie Yongji static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain) 3368c773d53SXie Yongji { 3378c773d53SXie Yongji int ret = 0; 3388c773d53SXie Yongji 3398c773d53SXie Yongji if (domain->bounce_map) 3408c773d53SXie Yongji return 0; 3418c773d53SXie Yongji 3428c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 3438c773d53SXie Yongji if (domain->bounce_map) 3448c773d53SXie Yongji goto unlock; 3458c773d53SXie Yongji 3468c773d53SXie Yongji ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1, 3478c773d53SXie Yongji 0, VHOST_MAP_RW, domain->file, 0); 3488c773d53SXie Yongji if (ret) 3498c773d53SXie Yongji goto unlock; 3508c773d53SXie Yongji 3518c773d53SXie Yongji domain->bounce_map = 1; 3528c773d53SXie Yongji unlock: 3538c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 3548c773d53SXie Yongji return ret; 3558c773d53SXie Yongji } 3568c773d53SXie Yongji 3578c773d53SXie Yongji static dma_addr_t 3588c773d53SXie Yongji vduse_domain_alloc_iova(struct iova_domain *iovad, 3598c773d53SXie Yongji unsigned long size, unsigned long limit) 3608c773d53SXie Yongji { 3618c773d53SXie Yongji unsigned long shift = iova_shift(iovad); 3628c773d53SXie Yongji unsigned long iova_len = iova_align(iovad, size) >> shift; 3638c773d53SXie Yongji unsigned long iova_pfn; 3648c773d53SXie Yongji 3658c773d53SXie Yongji iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true); 3668c773d53SXie Yongji 367b9d102daSXie Yongji return (dma_addr_t)iova_pfn << shift; 3688c773d53SXie Yongji } 3698c773d53SXie Yongji 3708c773d53SXie Yongji static void vduse_domain_free_iova(struct iova_domain *iovad, 3718c773d53SXie Yongji dma_addr_t iova, size_t size) 3728c773d53SXie Yongji { 3738c773d53SXie Yongji unsigned long shift = iova_shift(iovad); 3748c773d53SXie Yongji unsigned long iova_len = iova_align(iovad, size) >> shift; 3758c773d53SXie Yongji 3768c773d53SXie Yongji free_iova_fast(iovad, iova >> shift, iova_len); 3778c773d53SXie Yongji } 3788c773d53SXie Yongji 379d7b4e328SMaxime Coquelin void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain, 380d7b4e328SMaxime Coquelin dma_addr_t dma_addr, size_t size, 381d7b4e328SMaxime Coquelin enum dma_data_direction dir) 382d7b4e328SMaxime Coquelin { 383d7b4e328SMaxime Coquelin read_lock(&domain->bounce_lock); 384d7b4e328SMaxime Coquelin if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) 385d7b4e328SMaxime Coquelin vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE); 386d7b4e328SMaxime Coquelin read_unlock(&domain->bounce_lock); 387d7b4e328SMaxime Coquelin } 388d7b4e328SMaxime Coquelin 389d7b4e328SMaxime Coquelin void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain, 390d7b4e328SMaxime Coquelin dma_addr_t dma_addr, size_t size, 391d7b4e328SMaxime Coquelin enum dma_data_direction dir) 392d7b4e328SMaxime Coquelin { 393d7b4e328SMaxime Coquelin read_lock(&domain->bounce_lock); 394d7b4e328SMaxime Coquelin if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) 395d7b4e328SMaxime Coquelin vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); 396d7b4e328SMaxime Coquelin read_unlock(&domain->bounce_lock); 397d7b4e328SMaxime Coquelin } 398d7b4e328SMaxime Coquelin 3998c773d53SXie Yongji dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, 4008c773d53SXie Yongji struct page *page, unsigned long offset, 4018c773d53SXie Yongji size_t size, enum dma_data_direction dir, 4028c773d53SXie Yongji unsigned long attrs) 4038c773d53SXie Yongji { 4048c773d53SXie Yongji struct iova_domain *iovad = &domain->stream_iovad; 4058c773d53SXie Yongji unsigned long limit = domain->bounce_size - 1; 4068c773d53SXie Yongji phys_addr_t pa = page_to_phys(page) + offset; 4078c773d53SXie Yongji dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); 4088c773d53SXie Yongji 4098c773d53SXie Yongji if (!iova) 4108c773d53SXie Yongji return DMA_MAPPING_ERROR; 4118c773d53SXie Yongji 4128c773d53SXie Yongji if (vduse_domain_init_bounce_map(domain)) 4138c773d53SXie Yongji goto err; 4148c773d53SXie Yongji 4156c77ed22SXie Yongji read_lock(&domain->bounce_lock); 4168c773d53SXie Yongji if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa)) 4176c77ed22SXie Yongji goto err_unlock; 4188c773d53SXie Yongji 419d7b4e328SMaxime Coquelin if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 420d7b4e328SMaxime Coquelin (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) 4218c773d53SXie Yongji vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE); 4228c773d53SXie Yongji 4236c77ed22SXie Yongji read_unlock(&domain->bounce_lock); 4246c77ed22SXie Yongji 4258c773d53SXie Yongji return iova; 4266c77ed22SXie Yongji err_unlock: 4276c77ed22SXie Yongji read_unlock(&domain->bounce_lock); 4288c773d53SXie Yongji err: 4298c773d53SXie Yongji vduse_domain_free_iova(iovad, iova, size); 4308c773d53SXie Yongji return DMA_MAPPING_ERROR; 4318c773d53SXie Yongji } 4328c773d53SXie Yongji 4338c773d53SXie Yongji void vduse_domain_unmap_page(struct vduse_iova_domain *domain, 4348c773d53SXie Yongji dma_addr_t dma_addr, size_t size, 4358c773d53SXie Yongji enum dma_data_direction dir, unsigned long attrs) 4368c773d53SXie Yongji { 4378c773d53SXie Yongji struct iova_domain *iovad = &domain->stream_iovad; 4386c77ed22SXie Yongji read_lock(&domain->bounce_lock); 439d7b4e328SMaxime Coquelin if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 440d7b4e328SMaxime Coquelin (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) 4418c773d53SXie Yongji vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); 4428c773d53SXie Yongji 4438c773d53SXie Yongji vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size); 4446c77ed22SXie Yongji read_unlock(&domain->bounce_lock); 4458c773d53SXie Yongji vduse_domain_free_iova(iovad, dma_addr, size); 4468c773d53SXie Yongji } 4478c773d53SXie Yongji 4488c773d53SXie Yongji void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, 4498c773d53SXie Yongji size_t size, dma_addr_t *dma_addr, 4508c773d53SXie Yongji gfp_t flag, unsigned long attrs) 4518c773d53SXie Yongji { 4528c773d53SXie Yongji struct iova_domain *iovad = &domain->consistent_iovad; 4538c773d53SXie Yongji unsigned long limit = domain->iova_limit; 4548c773d53SXie Yongji dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); 4558c773d53SXie Yongji void *orig = alloc_pages_exact(size, flag); 4568c773d53SXie Yongji 4578c773d53SXie Yongji if (!iova || !orig) 4588c773d53SXie Yongji goto err; 4598c773d53SXie Yongji 4608c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 4618c773d53SXie Yongji if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1, 4628c773d53SXie Yongji virt_to_phys(orig), VHOST_MAP_RW, 4638c773d53SXie Yongji domain->file, (u64)iova)) { 4648c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 4658c773d53SXie Yongji goto err; 4668c773d53SXie Yongji } 4678c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 4688c773d53SXie Yongji 4698c773d53SXie Yongji *dma_addr = iova; 4708c773d53SXie Yongji 4718c773d53SXie Yongji return orig; 4728c773d53SXie Yongji err: 4738c773d53SXie Yongji *dma_addr = DMA_MAPPING_ERROR; 4748c773d53SXie Yongji if (orig) 4758c773d53SXie Yongji free_pages_exact(orig, size); 4768c773d53SXie Yongji if (iova) 4778c773d53SXie Yongji vduse_domain_free_iova(iovad, iova, size); 4788c773d53SXie Yongji 4798c773d53SXie Yongji return NULL; 4808c773d53SXie Yongji } 4818c773d53SXie Yongji 4828c773d53SXie Yongji void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, 4838c773d53SXie Yongji void *vaddr, dma_addr_t dma_addr, 4848c773d53SXie Yongji unsigned long attrs) 4858c773d53SXie Yongji { 4868c773d53SXie Yongji struct iova_domain *iovad = &domain->consistent_iovad; 4878c773d53SXie Yongji struct vhost_iotlb_map *map; 4888c773d53SXie Yongji struct vdpa_map_file *map_file; 4898c773d53SXie Yongji phys_addr_t pa; 4908c773d53SXie Yongji 4918c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 4928c773d53SXie Yongji map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr, 4938c773d53SXie Yongji (u64)dma_addr + size - 1); 4948c773d53SXie Yongji if (WARN_ON(!map)) { 4958c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 4968c773d53SXie Yongji return; 4978c773d53SXie Yongji } 4988c773d53SXie Yongji map_file = (struct vdpa_map_file *)map->opaque; 4998c773d53SXie Yongji fput(map_file->file); 5008c773d53SXie Yongji kfree(map_file); 5018c773d53SXie Yongji pa = map->addr; 5028c773d53SXie Yongji vhost_iotlb_map_free(domain->iotlb, map); 5038c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 5048c773d53SXie Yongji 5058c773d53SXie Yongji vduse_domain_free_iova(iovad, dma_addr, size); 5068c773d53SXie Yongji free_pages_exact(phys_to_virt(pa), size); 5078c773d53SXie Yongji } 5088c773d53SXie Yongji 5098c773d53SXie Yongji static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf) 5108c773d53SXie Yongji { 5118c773d53SXie Yongji struct vduse_iova_domain *domain = vmf->vma->vm_private_data; 5128c773d53SXie Yongji unsigned long iova = vmf->pgoff << PAGE_SHIFT; 5138c773d53SXie Yongji struct page *page; 5148c773d53SXie Yongji 5158c773d53SXie Yongji if (!domain) 5168c773d53SXie Yongji return VM_FAULT_SIGBUS; 5178c773d53SXie Yongji 5188c773d53SXie Yongji if (iova < domain->bounce_size) 5198c773d53SXie Yongji page = vduse_domain_get_bounce_page(domain, iova); 5208c773d53SXie Yongji else 5218c773d53SXie Yongji page = vduse_domain_get_coherent_page(domain, iova); 5228c773d53SXie Yongji 5238c773d53SXie Yongji if (!page) 5248c773d53SXie Yongji return VM_FAULT_SIGBUS; 5258c773d53SXie Yongji 5268c773d53SXie Yongji vmf->page = page; 5278c773d53SXie Yongji 5288c773d53SXie Yongji return 0; 5298c773d53SXie Yongji } 5308c773d53SXie Yongji 5318c773d53SXie Yongji static const struct vm_operations_struct vduse_domain_mmap_ops = { 5328c773d53SXie Yongji .fault = vduse_domain_mmap_fault, 5338c773d53SXie Yongji }; 5348c773d53SXie Yongji 5358c773d53SXie Yongji static int vduse_domain_mmap(struct file *file, struct vm_area_struct *vma) 5368c773d53SXie Yongji { 5378c773d53SXie Yongji struct vduse_iova_domain *domain = file->private_data; 5388c773d53SXie Yongji 5391c71222eSSuren Baghdasaryan vm_flags_set(vma, VM_DONTDUMP | VM_DONTEXPAND); 5408c773d53SXie Yongji vma->vm_private_data = domain; 5418c773d53SXie Yongji vma->vm_ops = &vduse_domain_mmap_ops; 5428c773d53SXie Yongji 5438c773d53SXie Yongji return 0; 5448c773d53SXie Yongji } 5458c773d53SXie Yongji 5468c773d53SXie Yongji static int vduse_domain_release(struct inode *inode, struct file *file) 5478c773d53SXie Yongji { 5488c773d53SXie Yongji struct vduse_iova_domain *domain = file->private_data; 5498c773d53SXie Yongji 5508c773d53SXie Yongji spin_lock(&domain->iotlb_lock); 5518c773d53SXie Yongji vduse_iotlb_del_range(domain, 0, ULLONG_MAX); 5526c77ed22SXie Yongji vduse_domain_remove_user_bounce_pages(domain); 5536c77ed22SXie Yongji vduse_domain_free_kernel_bounce_pages(domain); 5548c773d53SXie Yongji spin_unlock(&domain->iotlb_lock); 5558c773d53SXie Yongji put_iova_domain(&domain->stream_iovad); 5568c773d53SXie Yongji put_iova_domain(&domain->consistent_iovad); 5578c773d53SXie Yongji vhost_iotlb_free(domain->iotlb); 5588c773d53SXie Yongji vfree(domain->bounce_maps); 5598c773d53SXie Yongji kfree(domain); 5608c773d53SXie Yongji 5618c773d53SXie Yongji return 0; 5628c773d53SXie Yongji } 5638c773d53SXie Yongji 5648c773d53SXie Yongji static const struct file_operations vduse_domain_fops = { 5658c773d53SXie Yongji .owner = THIS_MODULE, 5668c773d53SXie Yongji .mmap = vduse_domain_mmap, 5678c773d53SXie Yongji .release = vduse_domain_release, 5688c773d53SXie Yongji }; 5698c773d53SXie Yongji 5708c773d53SXie Yongji void vduse_domain_destroy(struct vduse_iova_domain *domain) 5718c773d53SXie Yongji { 5728c773d53SXie Yongji fput(domain->file); 5738c773d53SXie Yongji } 5748c773d53SXie Yongji 5758c773d53SXie Yongji struct vduse_iova_domain * 5768c773d53SXie Yongji vduse_domain_create(unsigned long iova_limit, size_t bounce_size) 5778c773d53SXie Yongji { 5788c773d53SXie Yongji struct vduse_iova_domain *domain; 5798c773d53SXie Yongji struct file *file; 5808c773d53SXie Yongji struct vduse_bounce_map *map; 5818c773d53SXie Yongji unsigned long pfn, bounce_pfns; 58232e92d9fSJohn Garry int ret; 5838c773d53SXie Yongji 5848c773d53SXie Yongji bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT; 5858c773d53SXie Yongji if (iova_limit <= bounce_size) 5868c773d53SXie Yongji return NULL; 5878c773d53SXie Yongji 5888c773d53SXie Yongji domain = kzalloc(sizeof(*domain), GFP_KERNEL); 5898c773d53SXie Yongji if (!domain) 5908c773d53SXie Yongji return NULL; 5918c773d53SXie Yongji 5928c773d53SXie Yongji domain->iotlb = vhost_iotlb_alloc(0, 0); 5938c773d53SXie Yongji if (!domain->iotlb) 5948c773d53SXie Yongji goto err_iotlb; 5958c773d53SXie Yongji 5968c773d53SXie Yongji domain->iova_limit = iova_limit; 5978c773d53SXie Yongji domain->bounce_size = PAGE_ALIGN(bounce_size); 5988c773d53SXie Yongji domain->bounce_maps = vzalloc(bounce_pfns * 5998c773d53SXie Yongji sizeof(struct vduse_bounce_map)); 6008c773d53SXie Yongji if (!domain->bounce_maps) 6018c773d53SXie Yongji goto err_map; 6028c773d53SXie Yongji 6038c773d53SXie Yongji for (pfn = 0; pfn < bounce_pfns; pfn++) { 6048c773d53SXie Yongji map = &domain->bounce_maps[pfn]; 6058c773d53SXie Yongji map->orig_phys = INVALID_PHYS_ADDR; 6068c773d53SXie Yongji } 6078c773d53SXie Yongji file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops, 6088c773d53SXie Yongji domain, O_RDWR); 6098c773d53SXie Yongji if (IS_ERR(file)) 6108c773d53SXie Yongji goto err_file; 6118c773d53SXie Yongji 6128c773d53SXie Yongji domain->file = file; 6136c77ed22SXie Yongji rwlock_init(&domain->bounce_lock); 6148c773d53SXie Yongji spin_lock_init(&domain->iotlb_lock); 6158c773d53SXie Yongji init_iova_domain(&domain->stream_iovad, 6168c773d53SXie Yongji PAGE_SIZE, IOVA_START_PFN); 61732e92d9fSJohn Garry ret = iova_domain_init_rcaches(&domain->stream_iovad); 61832e92d9fSJohn Garry if (ret) 61932e92d9fSJohn Garry goto err_iovad_stream; 6208c773d53SXie Yongji init_iova_domain(&domain->consistent_iovad, 6218c773d53SXie Yongji PAGE_SIZE, bounce_pfns); 62232e92d9fSJohn Garry ret = iova_domain_init_rcaches(&domain->consistent_iovad); 62332e92d9fSJohn Garry if (ret) 62432e92d9fSJohn Garry goto err_iovad_consistent; 6258c773d53SXie Yongji 6268c773d53SXie Yongji return domain; 62732e92d9fSJohn Garry err_iovad_consistent: 62832e92d9fSJohn Garry put_iova_domain(&domain->stream_iovad); 62932e92d9fSJohn Garry err_iovad_stream: 63032e92d9fSJohn Garry fput(file); 6318c773d53SXie Yongji err_file: 6328c773d53SXie Yongji vfree(domain->bounce_maps); 6338c773d53SXie Yongji err_map: 6348c773d53SXie Yongji vhost_iotlb_free(domain->iotlb); 6358c773d53SXie Yongji err_iotlb: 6368c773d53SXie Yongji kfree(domain); 6378c773d53SXie Yongji return NULL; 6388c773d53SXie Yongji } 6398c773d53SXie Yongji 6408c773d53SXie Yongji int vduse_domain_init(void) 6418c773d53SXie Yongji { 6428c773d53SXie Yongji return iova_cache_get(); 6438c773d53SXie Yongji } 6448c773d53SXie Yongji 6458c773d53SXie Yongji void vduse_domain_exit(void) 6468c773d53SXie Yongji { 6478c773d53SXie Yongji iova_cache_put(); 6488c773d53SXie Yongji } 649