1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Google LLC 4 * Author: Quentin Perret <qperret@google.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <asm/kvm_hyp.h> 9 #include <asm/kvm_mmu.h> 10 #include <asm/kvm_pgtable.h> 11 #include <asm/kvm_pkvm.h> 12 13 #include <nvhe/alloc.h> 14 #include <nvhe/early_alloc.h> 15 #include <nvhe/ffa.h> 16 #include <nvhe/gfp.h> 17 #include <nvhe/memory.h> 18 #include <nvhe/mem_protect.h> 19 #include <nvhe/mm.h> 20 #include <nvhe/pkvm.h> 21 #include <nvhe/trap_handler.h> 22 23 unsigned long hyp_nr_cpus; 24 25 #define hyp_percpu_size ((unsigned long)__per_cpu_end - \ 26 (unsigned long)__per_cpu_start) 27 28 static void *vmemmap_base; 29 static void *vm_table_base; 30 static void *hyp_pgt_base; 31 static void *host_s2_pgt_base; 32 static void *selftest_base; 33 static void *ffa_proxy_pages; 34 static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops; 35 static struct hyp_pool hpool; 36 37 static int divide_memory_pool(void *virt, unsigned long size) 38 { 39 unsigned long nr_pages; 40 41 hyp_early_alloc_init(virt, size); 42 43 nr_pages = pkvm_selftest_pages(); 44 selftest_base = hyp_early_alloc_contig(nr_pages); 45 if (nr_pages && !selftest_base) 46 return -ENOMEM; 47 48 nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page)); 49 vmemmap_base = hyp_early_alloc_contig(nr_pages); 50 if (!vmemmap_base) 51 return -ENOMEM; 52 53 nr_pages = hyp_vm_table_pages(); 54 vm_table_base = hyp_early_alloc_contig(nr_pages); 55 if (!vm_table_base) 56 return -ENOMEM; 57 58 nr_pages = hyp_s1_pgtable_pages(); 59 hyp_pgt_base = hyp_early_alloc_contig(nr_pages); 60 if (!hyp_pgt_base) 61 return -ENOMEM; 62 63 nr_pages = host_s2_pgtable_pages(); 64 host_s2_pgt_base = hyp_early_alloc_contig(nr_pages); 65 if (!host_s2_pgt_base) 66 return -ENOMEM; 67 68 nr_pages = hyp_ffa_proxy_pages(); 69 ffa_proxy_pages = hyp_early_alloc_contig(nr_pages); 70 if (!ffa_proxy_pages) 71 return -ENOMEM; 72 73 return 0; 74 } 75 76 static int pkvm_create_host_sve_mappings(void) 77 { 78 void *start, *end; 79 int ret, i; 80 81 if (!system_supports_sve()) 82 return 0; 83 84 for (i = 0; i < hyp_nr_cpus; i++) { 85 struct kvm_host_data *host_data = per_cpu_ptr(&kvm_host_data, i); 86 struct arm64_sve_state *sve_regs = host_data->sve_regs; 87 88 start = kern_hyp_va(sve_regs); 89 end = start + PAGE_ALIGN(pkvm_host_sve_state_size()); 90 ret = pkvm_create_mappings(start, end, PAGE_HYP); 91 if (ret) 92 return ret; 93 } 94 95 return 0; 96 } 97 98 static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, 99 unsigned long *per_cpu_base, 100 u32 hyp_va_bits) 101 { 102 void *start, *end, *virt = hyp_phys_to_virt(phys); 103 unsigned long pgt_size = hyp_s1_pgtable_pages() << PAGE_SHIFT; 104 int ret, i; 105 106 /* Recreate the hyp page-table using the early page allocator */ 107 hyp_early_alloc_init(hyp_pgt_base, pgt_size); 108 ret = kvm_pgtable_hyp_init(&pkvm_pgtable, hyp_va_bits, 109 &hyp_early_alloc_mm_ops); 110 if (ret) 111 return ret; 112 113 ret = hyp_create_idmap(hyp_va_bits); 114 if (ret) 115 return ret; 116 117 ret = hyp_map_vectors(); 118 if (ret) 119 return ret; 120 121 ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base)); 122 if (ret) 123 return ret; 124 125 ret = pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_EXEC); 126 if (ret) 127 return ret; 128 129 ret = pkvm_create_mappings(__hyp_data_start, __hyp_data_end, PAGE_HYP); 130 if (ret) 131 return ret; 132 133 ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO); 134 if (ret) 135 return ret; 136 137 ret = pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP); 138 if (ret) 139 return ret; 140 141 ret = pkvm_create_mappings(virt, virt + size, PAGE_HYP); 142 if (ret) 143 return ret; 144 145 for (i = 0; i < hyp_nr_cpus; i++) { 146 struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); 147 148 start = (void *)kern_hyp_va(per_cpu_base[i]); 149 end = start + PAGE_ALIGN(hyp_percpu_size); 150 ret = pkvm_create_mappings(start, end, PAGE_HYP); 151 if (ret) 152 return ret; 153 154 ret = pkvm_create_stack(params->stack_pa, ¶ms->stack_hyp_va); 155 if (ret) 156 return ret; 157 } 158 159 return pkvm_create_host_sve_mappings(); 160 } 161 162 static void update_nvhe_init_params(void) 163 { 164 struct kvm_nvhe_init_params *params; 165 unsigned long i; 166 167 for (i = 0; i < hyp_nr_cpus; i++) { 168 params = per_cpu_ptr(&kvm_init_params, i); 169 params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd); 170 dcache_clean_inval_poc((unsigned long)params, 171 (unsigned long)params + sizeof(*params)); 172 } 173 } 174 175 static void *hyp_zalloc_hyp_page(void *arg) 176 { 177 return hyp_alloc_pages(&hpool, 0); 178 } 179 180 static void hpool_get_page(void *addr) 181 { 182 hyp_get_page(&hpool, addr); 183 } 184 185 static void hpool_put_page(void *addr) 186 { 187 hyp_put_page(&hpool, addr); 188 } 189 190 static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, 191 enum kvm_pgtable_walk_flags visit) 192 { 193 enum pkvm_page_state state; 194 struct hyp_page *page; 195 phys_addr_t phys; 196 enum kvm_pgtable_prot prot; 197 198 if (!kvm_pte_valid(ctx->old)) 199 return 0; 200 201 if (ctx->level != KVM_PGTABLE_LAST_LEVEL) 202 return -EINVAL; 203 204 phys = kvm_pte_to_phys(ctx->old); 205 if (!addr_is_memory(phys)) 206 return -EINVAL; 207 208 page = hyp_phys_to_page(phys); 209 210 /* 211 * Adjust the host stage-2 mappings to match the ownership attributes 212 * configured in the hypervisor stage-1, and make sure to propagate them 213 * to the hyp_vmemmap state. 214 */ 215 prot = kvm_pgtable_hyp_pte_prot(ctx->old); 216 state = pkvm_getstate(prot); 217 switch (state) { 218 case PKVM_PAGE_OWNED: 219 set_hyp_state(page, PKVM_PAGE_OWNED); 220 /* hyp text is RO in the host stage-2 to be inspected on panic. */ 221 if (addr_is_hyp_text(phys)) { 222 set_host_state(page, PKVM_NOPAGE); 223 return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R); 224 } else { 225 return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP); 226 } 227 case PKVM_PAGE_SHARED_OWNED: 228 set_hyp_state(page, PKVM_PAGE_SHARED_OWNED); 229 set_host_state(page, PKVM_PAGE_SHARED_BORROWED); 230 break; 231 case PKVM_PAGE_SHARED_BORROWED: 232 set_hyp_state(page, PKVM_PAGE_SHARED_BORROWED); 233 set_host_state(page, PKVM_PAGE_SHARED_OWNED); 234 break; 235 default: 236 return -EINVAL; 237 } 238 239 return 0; 240 } 241 242 static int fix_hyp_pgtable_refcnt_walker(const struct kvm_pgtable_visit_ctx *ctx, 243 enum kvm_pgtable_walk_flags visit) 244 { 245 /* 246 * Fix-up the refcount for the page-table pages as the early allocator 247 * was unable to access the hyp_vmemmap and so the buddy allocator has 248 * initialised the refcount to '1'. 249 */ 250 if (kvm_pte_valid(ctx->old)) 251 ctx->mm_ops->get_page(ctx->ptep); 252 253 return 0; 254 } 255 256 static int fix_host_ownership(void) 257 { 258 struct kvm_pgtable_walker walker = { 259 .cb = fix_host_ownership_walker, 260 .flags = KVM_PGTABLE_WALK_LEAF, 261 }; 262 int i, ret; 263 264 for (i = 0; i < hyp_memblock_nr; i++) { 265 struct memblock_region *reg = &hyp_memory[i]; 266 u64 start = (u64)hyp_phys_to_virt(reg->base); 267 268 ret = kvm_pgtable_walk(&pkvm_pgtable, start, reg->size, &walker); 269 if (ret) 270 return ret; 271 } 272 273 /* The stacks sit in the private VA range, not the linear map. */ 274 for (i = 0; i < hyp_nr_cpus; i++) { 275 struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); 276 u64 start = params->stack_hyp_va - NVHE_STACK_SIZE; 277 278 ret = kvm_pgtable_walk(&pkvm_pgtable, start, NVHE_STACK_SIZE, &walker); 279 if (ret) 280 return ret; 281 } 282 283 return 0; 284 } 285 286 static int fix_hyp_pgtable_refcnt(void) 287 { 288 struct kvm_pgtable_walker walker = { 289 .cb = fix_hyp_pgtable_refcnt_walker, 290 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, 291 .arg = pkvm_pgtable.mm_ops, 292 }; 293 294 return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits), 295 &walker); 296 } 297 298 void __noreturn __pkvm_init_finalise(void) 299 { 300 struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); 301 unsigned long nr_pages, reserved_pages, pfn; 302 int ret; 303 304 /* Now that the vmemmap is backed, install the full-fledged allocator */ 305 pfn = hyp_virt_to_pfn(hyp_pgt_base); 306 nr_pages = hyp_s1_pgtable_pages(); 307 reserved_pages = hyp_early_alloc_nr_used_pages(); 308 ret = hyp_pool_init(&hpool, pfn, nr_pages, reserved_pages); 309 if (ret) 310 goto out; 311 312 ret = kvm_host_prepare_stage2(host_s2_pgt_base); 313 if (ret) 314 goto out; 315 316 pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) { 317 .zalloc_page = hyp_zalloc_hyp_page, 318 .phys_to_virt = hyp_phys_to_virt, 319 .virt_to_phys = hyp_virt_to_phys, 320 .get_page = hpool_get_page, 321 .put_page = hpool_put_page, 322 .page_count = hyp_page_count, 323 }; 324 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops; 325 326 ret = fix_hyp_pgtable_refcnt(); 327 if (ret) 328 goto out; 329 330 ret = hyp_create_fixmap(); 331 if (ret) 332 goto out; 333 334 ret = fix_host_ownership(); 335 if (ret) 336 goto out; 337 338 ret = pkvm_check_host_ownership(); 339 if (ret) 340 goto out; 341 342 ret = hyp_ffa_init(ffa_proxy_pages); 343 if (ret) 344 goto out; 345 346 pkvm_hyp_vm_table_init(vm_table_base); 347 348 pkvm_ownership_selftest(selftest_base); 349 out: 350 /* 351 * We tail-called to here from handle___pkvm_init() and will not return, 352 * so make sure to propagate the return value to the host. 353 */ 354 cpu_reg(host_ctxt, 1) = ret; 355 356 __host_enter(host_ctxt); 357 } 358 359 int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_base, u32 hyp_va_bits) 360 { 361 struct kvm_nvhe_init_params *params; 362 void *virt = hyp_phys_to_virt(phys); 363 typeof(__pkvm_init_switch_pgd) *fn; 364 int ret; 365 366 BUG_ON(kvm_check_pvm_sysreg_table()); 367 368 if (!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(size)) 369 return -EINVAL; 370 371 hyp_spin_lock_init(&pkvm_pgd_lock); 372 373 ret = divide_memory_pool(virt, size); 374 if (ret) 375 return ret; 376 377 ret = recreate_hyp_mappings(phys, size, per_cpu_base, hyp_va_bits); 378 if (ret) 379 return ret; 380 381 /* Arbitrary size, unmapped at boot and only consume VA-space */ 382 ret = hyp_alloc_init(SZ_128M); 383 if (ret) 384 return ret; 385 386 update_nvhe_init_params(); 387 388 /* Jump in the idmap page to switch to the new page-tables */ 389 params = this_cpu_ptr(&kvm_init_params); 390 fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd); 391 fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise); 392 393 unreachable(); 394 } 395