188f537d5SShuo Liu // SPDX-License-Identifier: GPL-2.0 288f537d5SShuo Liu /* 388f537d5SShuo Liu * ACRN: Memory mapping management 488f537d5SShuo Liu * 588f537d5SShuo Liu * Copyright (C) 2020 Intel Corporation. All rights reserved. 688f537d5SShuo Liu * 788f537d5SShuo Liu * Authors: 888f537d5SShuo Liu * Fei Li <lei1.li@intel.com> 988f537d5SShuo Liu * Shuo Liu <shuo.a.liu@intel.com> 1088f537d5SShuo Liu */ 1188f537d5SShuo Liu 1288f537d5SShuo Liu #include <linux/io.h> 1388f537d5SShuo Liu #include <linux/mm.h> 1488f537d5SShuo Liu #include <linux/slab.h> 1588f537d5SShuo Liu 1688f537d5SShuo Liu #include "acrn_drv.h" 1788f537d5SShuo Liu 1888f537d5SShuo Liu static int modify_region(struct acrn_vm *vm, struct vm_memory_region_op *region) 1988f537d5SShuo Liu { 2088f537d5SShuo Liu struct vm_memory_region_batch *regions; 2188f537d5SShuo Liu int ret; 2288f537d5SShuo Liu 2388f537d5SShuo Liu regions = kzalloc(sizeof(*regions), GFP_KERNEL); 2488f537d5SShuo Liu if (!regions) 2588f537d5SShuo Liu return -ENOMEM; 2688f537d5SShuo Liu 2788f537d5SShuo Liu regions->vmid = vm->vmid; 2888f537d5SShuo Liu regions->regions_num = 1; 2988f537d5SShuo Liu regions->regions_gpa = virt_to_phys(region); 3088f537d5SShuo Liu 3188f537d5SShuo Liu ret = hcall_set_memory_regions(virt_to_phys(regions)); 3288f537d5SShuo Liu if (ret < 0) 3388f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 3488f537d5SShuo Liu "Failed to set memory region for VM[%u]!\n", vm->vmid); 3588f537d5SShuo Liu 3688f537d5SShuo Liu kfree(regions); 3788f537d5SShuo Liu return ret; 3888f537d5SShuo Liu } 3988f537d5SShuo Liu 4088f537d5SShuo Liu /** 4188f537d5SShuo Liu * acrn_mm_region_add() - Set up the EPT mapping of a memory region. 4288f537d5SShuo Liu * @vm: User VM. 4388f537d5SShuo Liu * @user_gpa: A GPA of User VM. 4488f537d5SShuo Liu * @service_gpa: A GPA of Service VM. 4588f537d5SShuo Liu * @size: Size of the region. 4688f537d5SShuo Liu * @mem_type: Combination of ACRN_MEM_TYPE_*. 4788f537d5SShuo Liu * @mem_access_right: Combination of ACRN_MEM_ACCESS_*. 4888f537d5SShuo Liu * 4988f537d5SShuo Liu * Return: 0 on success, <0 on error. 5088f537d5SShuo Liu */ 5188f537d5SShuo Liu int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, 5288f537d5SShuo Liu u64 size, u32 mem_type, u32 mem_access_right) 5388f537d5SShuo Liu { 5488f537d5SShuo Liu struct vm_memory_region_op *region; 5588f537d5SShuo Liu int ret = 0; 5688f537d5SShuo Liu 5788f537d5SShuo Liu region = kzalloc(sizeof(*region), GFP_KERNEL); 5888f537d5SShuo Liu if (!region) 5988f537d5SShuo Liu return -ENOMEM; 6088f537d5SShuo Liu 6188f537d5SShuo Liu region->type = ACRN_MEM_REGION_ADD; 6288f537d5SShuo Liu region->user_vm_pa = user_gpa; 6388f537d5SShuo Liu region->service_vm_pa = service_gpa; 6488f537d5SShuo Liu region->size = size; 6588f537d5SShuo Liu region->attr = ((mem_type & ACRN_MEM_TYPE_MASK) | 6688f537d5SShuo Liu (mem_access_right & ACRN_MEM_ACCESS_RIGHT_MASK)); 6788f537d5SShuo Liu ret = modify_region(vm, region); 6888f537d5SShuo Liu 6988f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 7088f537d5SShuo Liu "%s: user-GPA[%pK] service-GPA[%pK] size[0x%llx].\n", 7188f537d5SShuo Liu __func__, (void *)user_gpa, (void *)service_gpa, size); 7288f537d5SShuo Liu kfree(region); 7388f537d5SShuo Liu return ret; 7488f537d5SShuo Liu } 7588f537d5SShuo Liu 7688f537d5SShuo Liu /** 7788f537d5SShuo Liu * acrn_mm_region_del() - Del the EPT mapping of a memory region. 7888f537d5SShuo Liu * @vm: User VM. 7988f537d5SShuo Liu * @user_gpa: A GPA of the User VM. 8088f537d5SShuo Liu * @size: Size of the region. 8188f537d5SShuo Liu * 8288f537d5SShuo Liu * Return: 0 on success, <0 for error. 8388f537d5SShuo Liu */ 8488f537d5SShuo Liu int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size) 8588f537d5SShuo Liu { 8688f537d5SShuo Liu struct vm_memory_region_op *region; 8788f537d5SShuo Liu int ret = 0; 8888f537d5SShuo Liu 8988f537d5SShuo Liu region = kzalloc(sizeof(*region), GFP_KERNEL); 9088f537d5SShuo Liu if (!region) 9188f537d5SShuo Liu return -ENOMEM; 9288f537d5SShuo Liu 9388f537d5SShuo Liu region->type = ACRN_MEM_REGION_DEL; 9488f537d5SShuo Liu region->user_vm_pa = user_gpa; 9588f537d5SShuo Liu region->service_vm_pa = 0UL; 9688f537d5SShuo Liu region->size = size; 9788f537d5SShuo Liu region->attr = 0U; 9888f537d5SShuo Liu 9988f537d5SShuo Liu ret = modify_region(vm, region); 10088f537d5SShuo Liu 10188f537d5SShuo Liu dev_dbg(acrn_dev.this_device, "%s: user-GPA[%pK] size[0x%llx].\n", 10288f537d5SShuo Liu __func__, (void *)user_gpa, size); 10388f537d5SShuo Liu kfree(region); 10488f537d5SShuo Liu return ret; 10588f537d5SShuo Liu } 10688f537d5SShuo Liu 10788f537d5SShuo Liu int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 10888f537d5SShuo Liu { 10988f537d5SShuo Liu int ret; 11088f537d5SShuo Liu 11188f537d5SShuo Liu if (memmap->type == ACRN_MEMMAP_RAM) 11288f537d5SShuo Liu return acrn_vm_ram_map(vm, memmap); 11388f537d5SShuo Liu 11488f537d5SShuo Liu if (memmap->type != ACRN_MEMMAP_MMIO) { 11588f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 11688f537d5SShuo Liu "Invalid memmap type: %u\n", memmap->type); 11788f537d5SShuo Liu return -EINVAL; 11888f537d5SShuo Liu } 11988f537d5SShuo Liu 12088f537d5SShuo Liu ret = acrn_mm_region_add(vm, memmap->user_vm_pa, 12188f537d5SShuo Liu memmap->service_vm_pa, memmap->len, 12288f537d5SShuo Liu ACRN_MEM_TYPE_UC, memmap->attr); 12388f537d5SShuo Liu if (ret < 0) 12488f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 12588f537d5SShuo Liu "Add memory region failed, VM[%u]!\n", vm->vmid); 12688f537d5SShuo Liu 12788f537d5SShuo Liu return ret; 12888f537d5SShuo Liu } 12988f537d5SShuo Liu 13088f537d5SShuo Liu int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 13188f537d5SShuo Liu { 13288f537d5SShuo Liu int ret; 13388f537d5SShuo Liu 13488f537d5SShuo Liu if (memmap->type != ACRN_MEMMAP_MMIO) { 13588f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 13688f537d5SShuo Liu "Invalid memmap type: %u\n", memmap->type); 13788f537d5SShuo Liu return -EINVAL; 13888f537d5SShuo Liu } 13988f537d5SShuo Liu 14088f537d5SShuo Liu ret = acrn_mm_region_del(vm, memmap->user_vm_pa, memmap->len); 14188f537d5SShuo Liu if (ret < 0) 14288f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 14388f537d5SShuo Liu "Del memory region failed, VM[%u]!\n", vm->vmid); 14488f537d5SShuo Liu 14588f537d5SShuo Liu return ret; 14688f537d5SShuo Liu } 14788f537d5SShuo Liu 14888f537d5SShuo Liu /** 14988f537d5SShuo Liu * acrn_vm_ram_map() - Create a RAM EPT mapping of User VM. 15088f537d5SShuo Liu * @vm: The User VM pointer 15188f537d5SShuo Liu * @memmap: Info of the EPT mapping 15288f537d5SShuo Liu * 15388f537d5SShuo Liu * Return: 0 on success, <0 for error. 15488f537d5SShuo Liu */ 15588f537d5SShuo Liu int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 15688f537d5SShuo Liu { 15788f537d5SShuo Liu struct vm_memory_region_batch *regions_info; 15888f537d5SShuo Liu int nr_pages, i = 0, order, nr_regions = 0; 15988f537d5SShuo Liu struct vm_memory_mapping *region_mapping; 16088f537d5SShuo Liu struct vm_memory_region_op *vm_region; 16188f537d5SShuo Liu struct page **pages = NULL, *page; 16288f537d5SShuo Liu void *remap_vaddr; 16388f537d5SShuo Liu int ret, pinned; 16488f537d5SShuo Liu u64 user_vm_pa; 165*8a6e85f7SYonghua Huang unsigned long pfn; 166*8a6e85f7SYonghua Huang struct vm_area_struct *vma; 16788f537d5SShuo Liu 16888f537d5SShuo Liu if (!vm || !memmap) 16988f537d5SShuo Liu return -EINVAL; 17088f537d5SShuo Liu 171*8a6e85f7SYonghua Huang mmap_read_lock(current->mm); 172*8a6e85f7SYonghua Huang vma = vma_lookup(current->mm, memmap->vma_base); 173*8a6e85f7SYonghua Huang if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) { 174*8a6e85f7SYonghua Huang if ((memmap->vma_base + memmap->len) > vma->vm_end) { 175*8a6e85f7SYonghua Huang mmap_read_unlock(current->mm); 176*8a6e85f7SYonghua Huang return -EINVAL; 177*8a6e85f7SYonghua Huang } 178*8a6e85f7SYonghua Huang 179*8a6e85f7SYonghua Huang ret = follow_pfn(vma, memmap->vma_base, &pfn); 180*8a6e85f7SYonghua Huang mmap_read_unlock(current->mm); 181*8a6e85f7SYonghua Huang if (ret < 0) { 182*8a6e85f7SYonghua Huang dev_dbg(acrn_dev.this_device, 183*8a6e85f7SYonghua Huang "Failed to lookup PFN at VMA:%pK.\n", (void *)memmap->vma_base); 184*8a6e85f7SYonghua Huang return ret; 185*8a6e85f7SYonghua Huang } 186*8a6e85f7SYonghua Huang 187*8a6e85f7SYonghua Huang return acrn_mm_region_add(vm, memmap->user_vm_pa, 188*8a6e85f7SYonghua Huang PFN_PHYS(pfn), memmap->len, 189*8a6e85f7SYonghua Huang ACRN_MEM_TYPE_WB, memmap->attr); 190*8a6e85f7SYonghua Huang } 191*8a6e85f7SYonghua Huang mmap_read_unlock(current->mm); 192*8a6e85f7SYonghua Huang 19388f537d5SShuo Liu /* Get the page number of the map region */ 19488f537d5SShuo Liu nr_pages = memmap->len >> PAGE_SHIFT; 19588f537d5SShuo Liu pages = vzalloc(nr_pages * sizeof(struct page *)); 19688f537d5SShuo Liu if (!pages) 19788f537d5SShuo Liu return -ENOMEM; 19888f537d5SShuo Liu 19988f537d5SShuo Liu /* Lock the pages of user memory map region */ 20088f537d5SShuo Liu pinned = pin_user_pages_fast(memmap->vma_base, 20188f537d5SShuo Liu nr_pages, FOLL_WRITE | FOLL_LONGTERM, 20288f537d5SShuo Liu pages); 20388f537d5SShuo Liu if (pinned < 0) { 20488f537d5SShuo Liu ret = pinned; 20588f537d5SShuo Liu goto free_pages; 20688f537d5SShuo Liu } else if (pinned != nr_pages) { 20788f537d5SShuo Liu ret = -EFAULT; 20888f537d5SShuo Liu goto put_pages; 20988f537d5SShuo Liu } 21088f537d5SShuo Liu 21188f537d5SShuo Liu /* Create a kernel map for the map region */ 21288f537d5SShuo Liu remap_vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); 21388f537d5SShuo Liu if (!remap_vaddr) { 21488f537d5SShuo Liu ret = -ENOMEM; 21588f537d5SShuo Liu goto put_pages; 21688f537d5SShuo Liu } 21788f537d5SShuo Liu 21888f537d5SShuo Liu /* Record Service VM va <-> User VM pa mapping */ 21988f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 22088f537d5SShuo Liu region_mapping = &vm->regions_mapping[vm->regions_mapping_count]; 22188f537d5SShuo Liu if (vm->regions_mapping_count < ACRN_MEM_MAPPING_MAX) { 22288f537d5SShuo Liu region_mapping->pages = pages; 22388f537d5SShuo Liu region_mapping->npages = nr_pages; 22488f537d5SShuo Liu region_mapping->size = memmap->len; 22588f537d5SShuo Liu region_mapping->service_vm_va = remap_vaddr; 22688f537d5SShuo Liu region_mapping->user_vm_pa = memmap->user_vm_pa; 22788f537d5SShuo Liu vm->regions_mapping_count++; 22888f537d5SShuo Liu } else { 22988f537d5SShuo Liu dev_warn(acrn_dev.this_device, 23088f537d5SShuo Liu "Run out of memory mapping slots!\n"); 23188f537d5SShuo Liu ret = -ENOMEM; 23288f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 23388f537d5SShuo Liu goto unmap_no_count; 23488f537d5SShuo Liu } 23588f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 23688f537d5SShuo Liu 23788f537d5SShuo Liu /* Calculate count of vm_memory_region_op */ 23888f537d5SShuo Liu while (i < nr_pages) { 23988f537d5SShuo Liu page = pages[i]; 24088f537d5SShuo Liu VM_BUG_ON_PAGE(PageTail(page), page); 24188f537d5SShuo Liu order = compound_order(page); 24288f537d5SShuo Liu nr_regions++; 24388f537d5SShuo Liu i += 1 << order; 24488f537d5SShuo Liu } 24588f537d5SShuo Liu 24688f537d5SShuo Liu /* Prepare the vm_memory_region_batch */ 24788f537d5SShuo Liu regions_info = kzalloc(sizeof(*regions_info) + 24888f537d5SShuo Liu sizeof(*vm_region) * nr_regions, 24988f537d5SShuo Liu GFP_KERNEL); 25088f537d5SShuo Liu if (!regions_info) { 25188f537d5SShuo Liu ret = -ENOMEM; 25288f537d5SShuo Liu goto unmap_kernel_map; 25388f537d5SShuo Liu } 25488f537d5SShuo Liu 25588f537d5SShuo Liu /* Fill each vm_memory_region_op */ 25688f537d5SShuo Liu vm_region = (struct vm_memory_region_op *)(regions_info + 1); 25788f537d5SShuo Liu regions_info->vmid = vm->vmid; 25888f537d5SShuo Liu regions_info->regions_num = nr_regions; 25988f537d5SShuo Liu regions_info->regions_gpa = virt_to_phys(vm_region); 26088f537d5SShuo Liu user_vm_pa = memmap->user_vm_pa; 26188f537d5SShuo Liu i = 0; 26288f537d5SShuo Liu while (i < nr_pages) { 26388f537d5SShuo Liu u32 region_size; 26488f537d5SShuo Liu 26588f537d5SShuo Liu page = pages[i]; 26688f537d5SShuo Liu VM_BUG_ON_PAGE(PageTail(page), page); 26788f537d5SShuo Liu order = compound_order(page); 26888f537d5SShuo Liu region_size = PAGE_SIZE << order; 26988f537d5SShuo Liu vm_region->type = ACRN_MEM_REGION_ADD; 27088f537d5SShuo Liu vm_region->user_vm_pa = user_vm_pa; 27188f537d5SShuo Liu vm_region->service_vm_pa = page_to_phys(page); 27288f537d5SShuo Liu vm_region->size = region_size; 27388f537d5SShuo Liu vm_region->attr = (ACRN_MEM_TYPE_WB & ACRN_MEM_TYPE_MASK) | 27488f537d5SShuo Liu (memmap->attr & ACRN_MEM_ACCESS_RIGHT_MASK); 27588f537d5SShuo Liu 27688f537d5SShuo Liu vm_region++; 27788f537d5SShuo Liu user_vm_pa += region_size; 27888f537d5SShuo Liu i += 1 << order; 27988f537d5SShuo Liu } 28088f537d5SShuo Liu 28188f537d5SShuo Liu /* Inform the ACRN Hypervisor to set up EPT mappings */ 28288f537d5SShuo Liu ret = hcall_set_memory_regions(virt_to_phys(regions_info)); 28388f537d5SShuo Liu if (ret < 0) { 28488f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 28588f537d5SShuo Liu "Failed to set regions, VM[%u]!\n", vm->vmid); 28688f537d5SShuo Liu goto unset_region; 28788f537d5SShuo Liu } 28888f537d5SShuo Liu kfree(regions_info); 28988f537d5SShuo Liu 29088f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 29188f537d5SShuo Liu "%s: VM[%u] service-GVA[%pK] user-GPA[%pK] size[0x%llx]\n", 29288f537d5SShuo Liu __func__, vm->vmid, 29388f537d5SShuo Liu remap_vaddr, (void *)memmap->user_vm_pa, memmap->len); 29488f537d5SShuo Liu return ret; 29588f537d5SShuo Liu 29688f537d5SShuo Liu unset_region: 29788f537d5SShuo Liu kfree(regions_info); 29888f537d5SShuo Liu unmap_kernel_map: 29988f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 30088f537d5SShuo Liu vm->regions_mapping_count--; 30188f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 30288f537d5SShuo Liu unmap_no_count: 30388f537d5SShuo Liu vunmap(remap_vaddr); 30488f537d5SShuo Liu put_pages: 30588f537d5SShuo Liu for (i = 0; i < pinned; i++) 30688f537d5SShuo Liu unpin_user_page(pages[i]); 30788f537d5SShuo Liu free_pages: 30888f537d5SShuo Liu vfree(pages); 30988f537d5SShuo Liu return ret; 31088f537d5SShuo Liu } 31188f537d5SShuo Liu 31288f537d5SShuo Liu /** 31388f537d5SShuo Liu * acrn_vm_all_ram_unmap() - Destroy a RAM EPT mapping of User VM. 31488f537d5SShuo Liu * @vm: The User VM 31588f537d5SShuo Liu */ 31688f537d5SShuo Liu void acrn_vm_all_ram_unmap(struct acrn_vm *vm) 31788f537d5SShuo Liu { 31888f537d5SShuo Liu struct vm_memory_mapping *region_mapping; 31988f537d5SShuo Liu int i, j; 32088f537d5SShuo Liu 32188f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 32288f537d5SShuo Liu for (i = 0; i < vm->regions_mapping_count; i++) { 32388f537d5SShuo Liu region_mapping = &vm->regions_mapping[i]; 32488f537d5SShuo Liu vunmap(region_mapping->service_vm_va); 32588f537d5SShuo Liu for (j = 0; j < region_mapping->npages; j++) 32688f537d5SShuo Liu unpin_user_page(region_mapping->pages[j]); 32788f537d5SShuo Liu vfree(region_mapping->pages); 32888f537d5SShuo Liu } 32988f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 33088f537d5SShuo Liu } 331