1*88f537d5SShuo Liu // SPDX-License-Identifier: GPL-2.0 2*88f537d5SShuo Liu /* 3*88f537d5SShuo Liu * ACRN: Memory mapping management 4*88f537d5SShuo Liu * 5*88f537d5SShuo Liu * Copyright (C) 2020 Intel Corporation. All rights reserved. 6*88f537d5SShuo Liu * 7*88f537d5SShuo Liu * Authors: 8*88f537d5SShuo Liu * Fei Li <lei1.li@intel.com> 9*88f537d5SShuo Liu * Shuo Liu <shuo.a.liu@intel.com> 10*88f537d5SShuo Liu */ 11*88f537d5SShuo Liu 12*88f537d5SShuo Liu #include <linux/io.h> 13*88f537d5SShuo Liu #include <linux/mm.h> 14*88f537d5SShuo Liu #include <linux/slab.h> 15*88f537d5SShuo Liu 16*88f537d5SShuo Liu #include "acrn_drv.h" 17*88f537d5SShuo Liu 18*88f537d5SShuo Liu static int modify_region(struct acrn_vm *vm, struct vm_memory_region_op *region) 19*88f537d5SShuo Liu { 20*88f537d5SShuo Liu struct vm_memory_region_batch *regions; 21*88f537d5SShuo Liu int ret; 22*88f537d5SShuo Liu 23*88f537d5SShuo Liu regions = kzalloc(sizeof(*regions), GFP_KERNEL); 24*88f537d5SShuo Liu if (!regions) 25*88f537d5SShuo Liu return -ENOMEM; 26*88f537d5SShuo Liu 27*88f537d5SShuo Liu regions->vmid = vm->vmid; 28*88f537d5SShuo Liu regions->regions_num = 1; 29*88f537d5SShuo Liu regions->regions_gpa = virt_to_phys(region); 30*88f537d5SShuo Liu 31*88f537d5SShuo Liu ret = hcall_set_memory_regions(virt_to_phys(regions)); 32*88f537d5SShuo Liu if (ret < 0) 33*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 34*88f537d5SShuo Liu "Failed to set memory region for VM[%u]!\n", vm->vmid); 35*88f537d5SShuo Liu 36*88f537d5SShuo Liu kfree(regions); 37*88f537d5SShuo Liu return ret; 38*88f537d5SShuo Liu } 39*88f537d5SShuo Liu 40*88f537d5SShuo Liu /** 41*88f537d5SShuo Liu * acrn_mm_region_add() - Set up the EPT mapping of a memory region. 42*88f537d5SShuo Liu * @vm: User VM. 43*88f537d5SShuo Liu * @user_gpa: A GPA of User VM. 44*88f537d5SShuo Liu * @service_gpa: A GPA of Service VM. 45*88f537d5SShuo Liu * @size: Size of the region. 46*88f537d5SShuo Liu * @mem_type: Combination of ACRN_MEM_TYPE_*. 47*88f537d5SShuo Liu * @mem_access_right: Combination of ACRN_MEM_ACCESS_*. 48*88f537d5SShuo Liu * 49*88f537d5SShuo Liu * Return: 0 on success, <0 on error. 50*88f537d5SShuo Liu */ 51*88f537d5SShuo Liu int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, 52*88f537d5SShuo Liu u64 size, u32 mem_type, u32 mem_access_right) 53*88f537d5SShuo Liu { 54*88f537d5SShuo Liu struct vm_memory_region_op *region; 55*88f537d5SShuo Liu int ret = 0; 56*88f537d5SShuo Liu 57*88f537d5SShuo Liu region = kzalloc(sizeof(*region), GFP_KERNEL); 58*88f537d5SShuo Liu if (!region) 59*88f537d5SShuo Liu return -ENOMEM; 60*88f537d5SShuo Liu 61*88f537d5SShuo Liu region->type = ACRN_MEM_REGION_ADD; 62*88f537d5SShuo Liu region->user_vm_pa = user_gpa; 63*88f537d5SShuo Liu region->service_vm_pa = service_gpa; 64*88f537d5SShuo Liu region->size = size; 65*88f537d5SShuo Liu region->attr = ((mem_type & ACRN_MEM_TYPE_MASK) | 66*88f537d5SShuo Liu (mem_access_right & ACRN_MEM_ACCESS_RIGHT_MASK)); 67*88f537d5SShuo Liu ret = modify_region(vm, region); 68*88f537d5SShuo Liu 69*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 70*88f537d5SShuo Liu "%s: user-GPA[%pK] service-GPA[%pK] size[0x%llx].\n", 71*88f537d5SShuo Liu __func__, (void *)user_gpa, (void *)service_gpa, size); 72*88f537d5SShuo Liu kfree(region); 73*88f537d5SShuo Liu return ret; 74*88f537d5SShuo Liu } 75*88f537d5SShuo Liu 76*88f537d5SShuo Liu /** 77*88f537d5SShuo Liu * acrn_mm_region_del() - Del the EPT mapping of a memory region. 78*88f537d5SShuo Liu * @vm: User VM. 79*88f537d5SShuo Liu * @user_gpa: A GPA of the User VM. 80*88f537d5SShuo Liu * @size: Size of the region. 81*88f537d5SShuo Liu * 82*88f537d5SShuo Liu * Return: 0 on success, <0 for error. 83*88f537d5SShuo Liu */ 84*88f537d5SShuo Liu int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size) 85*88f537d5SShuo Liu { 86*88f537d5SShuo Liu struct vm_memory_region_op *region; 87*88f537d5SShuo Liu int ret = 0; 88*88f537d5SShuo Liu 89*88f537d5SShuo Liu region = kzalloc(sizeof(*region), GFP_KERNEL); 90*88f537d5SShuo Liu if (!region) 91*88f537d5SShuo Liu return -ENOMEM; 92*88f537d5SShuo Liu 93*88f537d5SShuo Liu region->type = ACRN_MEM_REGION_DEL; 94*88f537d5SShuo Liu region->user_vm_pa = user_gpa; 95*88f537d5SShuo Liu region->service_vm_pa = 0UL; 96*88f537d5SShuo Liu region->size = size; 97*88f537d5SShuo Liu region->attr = 0U; 98*88f537d5SShuo Liu 99*88f537d5SShuo Liu ret = modify_region(vm, region); 100*88f537d5SShuo Liu 101*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, "%s: user-GPA[%pK] size[0x%llx].\n", 102*88f537d5SShuo Liu __func__, (void *)user_gpa, size); 103*88f537d5SShuo Liu kfree(region); 104*88f537d5SShuo Liu return ret; 105*88f537d5SShuo Liu } 106*88f537d5SShuo Liu 107*88f537d5SShuo Liu int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 108*88f537d5SShuo Liu { 109*88f537d5SShuo Liu int ret; 110*88f537d5SShuo Liu 111*88f537d5SShuo Liu if (memmap->type == ACRN_MEMMAP_RAM) 112*88f537d5SShuo Liu return acrn_vm_ram_map(vm, memmap); 113*88f537d5SShuo Liu 114*88f537d5SShuo Liu if (memmap->type != ACRN_MEMMAP_MMIO) { 115*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 116*88f537d5SShuo Liu "Invalid memmap type: %u\n", memmap->type); 117*88f537d5SShuo Liu return -EINVAL; 118*88f537d5SShuo Liu } 119*88f537d5SShuo Liu 120*88f537d5SShuo Liu ret = acrn_mm_region_add(vm, memmap->user_vm_pa, 121*88f537d5SShuo Liu memmap->service_vm_pa, memmap->len, 122*88f537d5SShuo Liu ACRN_MEM_TYPE_UC, memmap->attr); 123*88f537d5SShuo Liu if (ret < 0) 124*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 125*88f537d5SShuo Liu "Add memory region failed, VM[%u]!\n", vm->vmid); 126*88f537d5SShuo Liu 127*88f537d5SShuo Liu return ret; 128*88f537d5SShuo Liu } 129*88f537d5SShuo Liu 130*88f537d5SShuo Liu int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 131*88f537d5SShuo Liu { 132*88f537d5SShuo Liu int ret; 133*88f537d5SShuo Liu 134*88f537d5SShuo Liu if (memmap->type != ACRN_MEMMAP_MMIO) { 135*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 136*88f537d5SShuo Liu "Invalid memmap type: %u\n", memmap->type); 137*88f537d5SShuo Liu return -EINVAL; 138*88f537d5SShuo Liu } 139*88f537d5SShuo Liu 140*88f537d5SShuo Liu ret = acrn_mm_region_del(vm, memmap->user_vm_pa, memmap->len); 141*88f537d5SShuo Liu if (ret < 0) 142*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 143*88f537d5SShuo Liu "Del memory region failed, VM[%u]!\n", vm->vmid); 144*88f537d5SShuo Liu 145*88f537d5SShuo Liu return ret; 146*88f537d5SShuo Liu } 147*88f537d5SShuo Liu 148*88f537d5SShuo Liu /** 149*88f537d5SShuo Liu * acrn_vm_ram_map() - Create a RAM EPT mapping of User VM. 150*88f537d5SShuo Liu * @vm: The User VM pointer 151*88f537d5SShuo Liu * @memmap: Info of the EPT mapping 152*88f537d5SShuo Liu * 153*88f537d5SShuo Liu * Return: 0 on success, <0 for error. 154*88f537d5SShuo Liu */ 155*88f537d5SShuo Liu int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) 156*88f537d5SShuo Liu { 157*88f537d5SShuo Liu struct vm_memory_region_batch *regions_info; 158*88f537d5SShuo Liu int nr_pages, i = 0, order, nr_regions = 0; 159*88f537d5SShuo Liu struct vm_memory_mapping *region_mapping; 160*88f537d5SShuo Liu struct vm_memory_region_op *vm_region; 161*88f537d5SShuo Liu struct page **pages = NULL, *page; 162*88f537d5SShuo Liu void *remap_vaddr; 163*88f537d5SShuo Liu int ret, pinned; 164*88f537d5SShuo Liu u64 user_vm_pa; 165*88f537d5SShuo Liu 166*88f537d5SShuo Liu if (!vm || !memmap) 167*88f537d5SShuo Liu return -EINVAL; 168*88f537d5SShuo Liu 169*88f537d5SShuo Liu /* Get the page number of the map region */ 170*88f537d5SShuo Liu nr_pages = memmap->len >> PAGE_SHIFT; 171*88f537d5SShuo Liu pages = vzalloc(nr_pages * sizeof(struct page *)); 172*88f537d5SShuo Liu if (!pages) 173*88f537d5SShuo Liu return -ENOMEM; 174*88f537d5SShuo Liu 175*88f537d5SShuo Liu /* Lock the pages of user memory map region */ 176*88f537d5SShuo Liu pinned = pin_user_pages_fast(memmap->vma_base, 177*88f537d5SShuo Liu nr_pages, FOLL_WRITE | FOLL_LONGTERM, 178*88f537d5SShuo Liu pages); 179*88f537d5SShuo Liu if (pinned < 0) { 180*88f537d5SShuo Liu ret = pinned; 181*88f537d5SShuo Liu goto free_pages; 182*88f537d5SShuo Liu } else if (pinned != nr_pages) { 183*88f537d5SShuo Liu ret = -EFAULT; 184*88f537d5SShuo Liu goto put_pages; 185*88f537d5SShuo Liu } 186*88f537d5SShuo Liu 187*88f537d5SShuo Liu /* Create a kernel map for the map region */ 188*88f537d5SShuo Liu remap_vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); 189*88f537d5SShuo Liu if (!remap_vaddr) { 190*88f537d5SShuo Liu ret = -ENOMEM; 191*88f537d5SShuo Liu goto put_pages; 192*88f537d5SShuo Liu } 193*88f537d5SShuo Liu 194*88f537d5SShuo Liu /* Record Service VM va <-> User VM pa mapping */ 195*88f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 196*88f537d5SShuo Liu region_mapping = &vm->regions_mapping[vm->regions_mapping_count]; 197*88f537d5SShuo Liu if (vm->regions_mapping_count < ACRN_MEM_MAPPING_MAX) { 198*88f537d5SShuo Liu region_mapping->pages = pages; 199*88f537d5SShuo Liu region_mapping->npages = nr_pages; 200*88f537d5SShuo Liu region_mapping->size = memmap->len; 201*88f537d5SShuo Liu region_mapping->service_vm_va = remap_vaddr; 202*88f537d5SShuo Liu region_mapping->user_vm_pa = memmap->user_vm_pa; 203*88f537d5SShuo Liu vm->regions_mapping_count++; 204*88f537d5SShuo Liu } else { 205*88f537d5SShuo Liu dev_warn(acrn_dev.this_device, 206*88f537d5SShuo Liu "Run out of memory mapping slots!\n"); 207*88f537d5SShuo Liu ret = -ENOMEM; 208*88f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 209*88f537d5SShuo Liu goto unmap_no_count; 210*88f537d5SShuo Liu } 211*88f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 212*88f537d5SShuo Liu 213*88f537d5SShuo Liu /* Calculate count of vm_memory_region_op */ 214*88f537d5SShuo Liu while (i < nr_pages) { 215*88f537d5SShuo Liu page = pages[i]; 216*88f537d5SShuo Liu VM_BUG_ON_PAGE(PageTail(page), page); 217*88f537d5SShuo Liu order = compound_order(page); 218*88f537d5SShuo Liu nr_regions++; 219*88f537d5SShuo Liu i += 1 << order; 220*88f537d5SShuo Liu } 221*88f537d5SShuo Liu 222*88f537d5SShuo Liu /* Prepare the vm_memory_region_batch */ 223*88f537d5SShuo Liu regions_info = kzalloc(sizeof(*regions_info) + 224*88f537d5SShuo Liu sizeof(*vm_region) * nr_regions, 225*88f537d5SShuo Liu GFP_KERNEL); 226*88f537d5SShuo Liu if (!regions_info) { 227*88f537d5SShuo Liu ret = -ENOMEM; 228*88f537d5SShuo Liu goto unmap_kernel_map; 229*88f537d5SShuo Liu } 230*88f537d5SShuo Liu 231*88f537d5SShuo Liu /* Fill each vm_memory_region_op */ 232*88f537d5SShuo Liu vm_region = (struct vm_memory_region_op *)(regions_info + 1); 233*88f537d5SShuo Liu regions_info->vmid = vm->vmid; 234*88f537d5SShuo Liu regions_info->regions_num = nr_regions; 235*88f537d5SShuo Liu regions_info->regions_gpa = virt_to_phys(vm_region); 236*88f537d5SShuo Liu user_vm_pa = memmap->user_vm_pa; 237*88f537d5SShuo Liu i = 0; 238*88f537d5SShuo Liu while (i < nr_pages) { 239*88f537d5SShuo Liu u32 region_size; 240*88f537d5SShuo Liu 241*88f537d5SShuo Liu page = pages[i]; 242*88f537d5SShuo Liu VM_BUG_ON_PAGE(PageTail(page), page); 243*88f537d5SShuo Liu order = compound_order(page); 244*88f537d5SShuo Liu region_size = PAGE_SIZE << order; 245*88f537d5SShuo Liu vm_region->type = ACRN_MEM_REGION_ADD; 246*88f537d5SShuo Liu vm_region->user_vm_pa = user_vm_pa; 247*88f537d5SShuo Liu vm_region->service_vm_pa = page_to_phys(page); 248*88f537d5SShuo Liu vm_region->size = region_size; 249*88f537d5SShuo Liu vm_region->attr = (ACRN_MEM_TYPE_WB & ACRN_MEM_TYPE_MASK) | 250*88f537d5SShuo Liu (memmap->attr & ACRN_MEM_ACCESS_RIGHT_MASK); 251*88f537d5SShuo Liu 252*88f537d5SShuo Liu vm_region++; 253*88f537d5SShuo Liu user_vm_pa += region_size; 254*88f537d5SShuo Liu i += 1 << order; 255*88f537d5SShuo Liu } 256*88f537d5SShuo Liu 257*88f537d5SShuo Liu /* Inform the ACRN Hypervisor to set up EPT mappings */ 258*88f537d5SShuo Liu ret = hcall_set_memory_regions(virt_to_phys(regions_info)); 259*88f537d5SShuo Liu if (ret < 0) { 260*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 261*88f537d5SShuo Liu "Failed to set regions, VM[%u]!\n", vm->vmid); 262*88f537d5SShuo Liu goto unset_region; 263*88f537d5SShuo Liu } 264*88f537d5SShuo Liu kfree(regions_info); 265*88f537d5SShuo Liu 266*88f537d5SShuo Liu dev_dbg(acrn_dev.this_device, 267*88f537d5SShuo Liu "%s: VM[%u] service-GVA[%pK] user-GPA[%pK] size[0x%llx]\n", 268*88f537d5SShuo Liu __func__, vm->vmid, 269*88f537d5SShuo Liu remap_vaddr, (void *)memmap->user_vm_pa, memmap->len); 270*88f537d5SShuo Liu return ret; 271*88f537d5SShuo Liu 272*88f537d5SShuo Liu unset_region: 273*88f537d5SShuo Liu kfree(regions_info); 274*88f537d5SShuo Liu unmap_kernel_map: 275*88f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 276*88f537d5SShuo Liu vm->regions_mapping_count--; 277*88f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 278*88f537d5SShuo Liu unmap_no_count: 279*88f537d5SShuo Liu vunmap(remap_vaddr); 280*88f537d5SShuo Liu put_pages: 281*88f537d5SShuo Liu for (i = 0; i < pinned; i++) 282*88f537d5SShuo Liu unpin_user_page(pages[i]); 283*88f537d5SShuo Liu free_pages: 284*88f537d5SShuo Liu vfree(pages); 285*88f537d5SShuo Liu return ret; 286*88f537d5SShuo Liu } 287*88f537d5SShuo Liu 288*88f537d5SShuo Liu /** 289*88f537d5SShuo Liu * acrn_vm_all_ram_unmap() - Destroy a RAM EPT mapping of User VM. 290*88f537d5SShuo Liu * @vm: The User VM 291*88f537d5SShuo Liu */ 292*88f537d5SShuo Liu void acrn_vm_all_ram_unmap(struct acrn_vm *vm) 293*88f537d5SShuo Liu { 294*88f537d5SShuo Liu struct vm_memory_mapping *region_mapping; 295*88f537d5SShuo Liu int i, j; 296*88f537d5SShuo Liu 297*88f537d5SShuo Liu mutex_lock(&vm->regions_mapping_lock); 298*88f537d5SShuo Liu for (i = 0; i < vm->regions_mapping_count; i++) { 299*88f537d5SShuo Liu region_mapping = &vm->regions_mapping[i]; 300*88f537d5SShuo Liu vunmap(region_mapping->service_vm_va); 301*88f537d5SShuo Liu for (j = 0; j < region_mapping->npages; j++) 302*88f537d5SShuo Liu unpin_user_page(region_mapping->pages[j]); 303*88f537d5SShuo Liu vfree(region_mapping->pages); 304*88f537d5SShuo Liu } 305*88f537d5SShuo Liu mutex_unlock(&vm->regions_mapping_lock); 306*88f537d5SShuo Liu } 307