1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020 - Google LLC 4 * Author: Quentin Perret <qperret@google.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <linux/memblock.h> 9 #include <linux/mutex.h> 10 #include <linux/sort.h> 11 12 #include <asm/kvm_pkvm.h> 13 14 #include "hyp_constants.h" 15 16 static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); 17 static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr); 18 19 phys_addr_t hyp_mem_base; 20 phys_addr_t hyp_mem_size; 21 22 static int cmp_hyp_memblock(const void *p1, const void *p2) 23 { 24 const struct memblock_region *r1 = p1; 25 const struct memblock_region *r2 = p2; 26 27 return r1->base < r2->base ? -1 : (r1->base > r2->base); 28 } 29 30 static void __init sort_memblock_regions(void) 31 { 32 sort(hyp_memory, 33 *hyp_memblock_nr_ptr, 34 sizeof(struct memblock_region), 35 cmp_hyp_memblock, 36 NULL); 37 } 38 39 static int __init register_memblock_regions(void) 40 { 41 struct memblock_region *reg; 42 43 for_each_mem_region(reg) { 44 if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS) 45 return -ENOMEM; 46 47 hyp_memory[*hyp_memblock_nr_ptr] = *reg; 48 (*hyp_memblock_nr_ptr)++; 49 } 50 sort_memblock_regions(); 51 52 return 0; 53 } 54 55 void __init kvm_hyp_reserve(void) 56 { 57 u64 hyp_mem_pages = 0; 58 int ret; 59 60 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode()) 61 return; 62 63 if (kvm_get_mode() != KVM_MODE_PROTECTED) 64 return; 65 66 ret = register_memblock_regions(); 67 if (ret) { 68 *hyp_memblock_nr_ptr = 0; 69 kvm_err("Failed to register hyp memblocks: %d\n", ret); 70 return; 71 } 72 73 hyp_mem_pages += hyp_s1_pgtable_pages(); 74 hyp_mem_pages += host_s2_pgtable_pages(); 75 hyp_mem_pages += hyp_vm_table_pages(); 76 hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE); 77 78 /* 79 * Try to allocate a PMD-aligned region to reduce TLB pressure once 80 * this is unmapped from the host stage-2, and fallback to PAGE_SIZE. 81 */ 82 hyp_mem_size = hyp_mem_pages << PAGE_SHIFT; 83 hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE), 84 PMD_SIZE); 85 if (!hyp_mem_base) 86 hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE); 87 else 88 hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE); 89 90 if (!hyp_mem_base) { 91 kvm_err("Failed to reserve hyp memory\n"); 92 return; 93 } 94 95 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20, 96 hyp_mem_base); 97 } 98 99 /* 100 * Allocates and donates memory for hypervisor VM structs at EL2. 101 * 102 * Allocates space for the VM state, which includes the hyp vm as well as 103 * the hyp vcpus. 104 * 105 * Stores an opaque handler in the kvm struct for future reference. 106 * 107 * Return 0 on success, negative error code on failure. 108 */ 109 static int __pkvm_create_hyp_vm(struct kvm *host_kvm) 110 { 111 size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz; 112 struct kvm_vcpu *host_vcpu; 113 pkvm_handle_t handle; 114 void *pgd, *hyp_vm; 115 unsigned long idx; 116 int ret; 117 118 if (host_kvm->created_vcpus < 1) 119 return -EINVAL; 120 121 pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.vtcr); 122 123 /* 124 * The PGD pages will be reclaimed using a hyp_memcache which implies 125 * page granularity. So, use alloc_pages_exact() to get individual 126 * refcounts. 127 */ 128 pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT); 129 if (!pgd) 130 return -ENOMEM; 131 132 /* Allocate memory to donate to hyp for vm and vcpu pointers. */ 133 hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, 134 size_mul(sizeof(void *), 135 host_kvm->created_vcpus))); 136 hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); 137 if (!hyp_vm) { 138 ret = -ENOMEM; 139 goto free_pgd; 140 } 141 142 /* Donate the VM memory to hyp and let hyp initialize it. */ 143 ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd); 144 if (ret < 0) 145 goto free_vm; 146 147 handle = ret; 148 149 host_kvm->arch.pkvm.handle = handle; 150 151 /* Donate memory for the vcpus at hyp and initialize it. */ 152 hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE); 153 kvm_for_each_vcpu(idx, host_vcpu, host_kvm) { 154 void *hyp_vcpu; 155 156 /* Indexing of the vcpus to be sequential starting at 0. */ 157 if (WARN_ON(host_vcpu->vcpu_idx != idx)) { 158 ret = -EINVAL; 159 goto destroy_vm; 160 } 161 162 hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); 163 if (!hyp_vcpu) { 164 ret = -ENOMEM; 165 goto destroy_vm; 166 } 167 168 ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu, 169 hyp_vcpu); 170 if (ret) { 171 free_pages_exact(hyp_vcpu, hyp_vcpu_sz); 172 goto destroy_vm; 173 } 174 } 175 176 return 0; 177 178 destroy_vm: 179 pkvm_destroy_hyp_vm(host_kvm); 180 return ret; 181 free_vm: 182 free_pages_exact(hyp_vm, hyp_vm_sz); 183 free_pgd: 184 free_pages_exact(pgd, pgd_sz); 185 return ret; 186 } 187 188 int pkvm_create_hyp_vm(struct kvm *host_kvm) 189 { 190 int ret = 0; 191 192 mutex_lock(&host_kvm->lock); 193 if (!host_kvm->arch.pkvm.handle) 194 ret = __pkvm_create_hyp_vm(host_kvm); 195 mutex_unlock(&host_kvm->lock); 196 197 return ret; 198 } 199 200 void pkvm_destroy_hyp_vm(struct kvm *host_kvm) 201 { 202 if (host_kvm->arch.pkvm.handle) { 203 WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm, 204 host_kvm->arch.pkvm.handle)); 205 } 206 207 host_kvm->arch.pkvm.handle = 0; 208 free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc); 209 } 210 211 int pkvm_init_host_vm(struct kvm *host_kvm) 212 { 213 mutex_init(&host_kvm->lock); 214 return 0; 215 } 216