1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Google LLC 4 * Author: Quentin Perret <qperret@google.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <asm/kvm_hyp.h> 9 #include <asm/kvm_mmu.h> 10 #include <asm/kvm_pgtable.h> 11 #include <asm/kvm_pkvm.h> 12 #include <asm/spectre.h> 13 14 #include <nvhe/early_alloc.h> 15 #include <nvhe/gfp.h> 16 #include <nvhe/memory.h> 17 #include <nvhe/mem_protect.h> 18 #include <nvhe/mm.h> 19 #include <nvhe/spinlock.h> 20 21 struct kvm_pgtable pkvm_pgtable; 22 hyp_spinlock_t pkvm_pgd_lock; 23 24 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; 25 unsigned int hyp_memblock_nr; 26 27 static u64 __io_map_base; 28 static u64 __io_map_next; 29 30 struct hyp_fixmap_slot { 31 u64 addr; 32 kvm_pte_t *ptep; 33 }; 34 static DEFINE_PER_CPU(struct hyp_fixmap_slot, fixmap_slots); 35 36 static int __pkvm_create_mappings(unsigned long start, unsigned long size, 37 unsigned long phys, enum kvm_pgtable_prot prot) 38 { 39 int err; 40 41 hyp_spin_lock(&pkvm_pgd_lock); 42 err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, size, phys, prot); 43 hyp_spin_unlock(&pkvm_pgd_lock); 44 45 return err; 46 } 47 48 static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) 49 { 50 unsigned long cur; 51 52 hyp_assert_lock_held(&pkvm_pgd_lock); 53 54 if (!start || start < __io_map_next) 55 return -EINVAL; 56 57 /* The allocated size is always a multiple of PAGE_SIZE */ 58 cur = start + PAGE_ALIGN(size); 59 60 /* Are we overflowing on the vmemmap ? */ 61 if (cur > __hyp_vmemmap) 62 return -ENOMEM; 63 64 __io_map_next = cur; 65 66 return 0; 67 } 68 69 /** 70 * pkvm_alloc_private_va_range - Allocates a private VA range. 71 * @size: The size of the VA range to reserve. 72 * @haddr: The hypervisor virtual start address of the allocation. 73 * 74 * The private virtual address (VA) range is allocated above __io_map_next 75 * and aligned based on the order of @size. 76 * 77 * Return: 0 on success or negative error code on failure. 78 */ 79 int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) 80 { 81 unsigned long addr; 82 int ret; 83 84 hyp_spin_lock(&pkvm_pgd_lock); 85 addr = __io_map_next; 86 ret = __pkvm_alloc_private_va_range(addr, size); 87 hyp_spin_unlock(&pkvm_pgd_lock); 88 89 *haddr = addr; 90 91 return ret; 92 } 93 94 int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, 95 enum kvm_pgtable_prot prot, 96 unsigned long *haddr) 97 { 98 unsigned long addr; 99 int err; 100 101 size = PAGE_ALIGN(size + offset_in_page(phys)); 102 err = pkvm_alloc_private_va_range(size, &addr); 103 if (err) 104 return err; 105 106 err = __pkvm_create_mappings(addr, size, phys, prot); 107 if (err) 108 return err; 109 110 *haddr = addr + offset_in_page(phys); 111 return err; 112 } 113 114 int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot) 115 { 116 unsigned long start = (unsigned long)from; 117 unsigned long end = (unsigned long)to; 118 unsigned long virt_addr; 119 phys_addr_t phys; 120 121 hyp_assert_lock_held(&pkvm_pgd_lock); 122 123 start = start & PAGE_MASK; 124 end = PAGE_ALIGN(end); 125 126 for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) { 127 int err; 128 129 phys = hyp_virt_to_phys((void *)virt_addr); 130 err = kvm_pgtable_hyp_map(&pkvm_pgtable, virt_addr, PAGE_SIZE, 131 phys, prot); 132 if (err) 133 return err; 134 } 135 136 return 0; 137 } 138 139 int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) 140 { 141 int ret; 142 143 hyp_spin_lock(&pkvm_pgd_lock); 144 ret = pkvm_create_mappings_locked(from, to, prot); 145 hyp_spin_unlock(&pkvm_pgd_lock); 146 147 return ret; 148 } 149 150 int hyp_back_vmemmap(phys_addr_t back) 151 { 152 unsigned long i, start, size, end = 0; 153 int ret; 154 155 for (i = 0; i < hyp_memblock_nr; i++) { 156 start = hyp_memory[i].base; 157 start = ALIGN_DOWN((u64)hyp_phys_to_page(start), PAGE_SIZE); 158 /* 159 * The beginning of the hyp_vmemmap region for the current 160 * memblock may already be backed by the page backing the end 161 * the previous region, so avoid mapping it twice. 162 */ 163 start = max(start, end); 164 165 end = hyp_memory[i].base + hyp_memory[i].size; 166 end = PAGE_ALIGN((u64)hyp_phys_to_page(end)); 167 if (start >= end) 168 continue; 169 170 size = end - start; 171 ret = __pkvm_create_mappings(start, size, back, PAGE_HYP); 172 if (ret) 173 return ret; 174 175 memset(hyp_phys_to_virt(back), 0, size); 176 back += size; 177 } 178 179 return 0; 180 } 181 182 static void *__hyp_bp_vect_base; 183 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot) 184 { 185 void *vector; 186 187 switch (slot) { 188 case HYP_VECTOR_DIRECT: { 189 vector = __kvm_hyp_vector; 190 break; 191 } 192 case HYP_VECTOR_SPECTRE_DIRECT: { 193 vector = __bp_harden_hyp_vecs; 194 break; 195 } 196 case HYP_VECTOR_INDIRECT: 197 case HYP_VECTOR_SPECTRE_INDIRECT: { 198 vector = (void *)__hyp_bp_vect_base; 199 break; 200 } 201 default: 202 return -EINVAL; 203 } 204 205 vector = __kvm_vector_slot2addr(vector, slot); 206 *this_cpu_ptr(&kvm_hyp_vector) = (unsigned long)vector; 207 208 return 0; 209 } 210 211 int hyp_map_vectors(void) 212 { 213 phys_addr_t phys; 214 unsigned long bp_base; 215 int ret; 216 217 if (!kvm_system_needs_idmapped_vectors()) { 218 __hyp_bp_vect_base = __bp_harden_hyp_vecs; 219 return 0; 220 } 221 222 phys = __hyp_pa(__bp_harden_hyp_vecs); 223 ret = __pkvm_create_private_mapping(phys, __BP_HARDEN_HYP_VECS_SZ, 224 PAGE_HYP_EXEC, &bp_base); 225 if (ret) 226 return ret; 227 228 __hyp_bp_vect_base = (void *)bp_base; 229 230 return 0; 231 } 232 233 static void *fixmap_map_slot(struct hyp_fixmap_slot *slot, phys_addr_t phys) 234 { 235 kvm_pte_t pte, *ptep = slot->ptep; 236 237 pte = *ptep; 238 pte &= ~kvm_phys_to_pte(KVM_PHYS_INVALID); 239 pte |= kvm_phys_to_pte(phys) | KVM_PTE_VALID; 240 WRITE_ONCE(*ptep, pte); 241 dsb(ishst); 242 243 return (void *)slot->addr; 244 } 245 246 void *hyp_fixmap_map(phys_addr_t phys) 247 { 248 return fixmap_map_slot(this_cpu_ptr(&fixmap_slots), phys) + offset_in_page(phys); 249 } 250 251 static void fixmap_clear_slot(struct hyp_fixmap_slot *slot) 252 { 253 kvm_pte_t *ptep = slot->ptep; 254 u64 addr = slot->addr; 255 u32 level; 256 257 if (FIELD_GET(KVM_PTE_TYPE, *ptep) == KVM_PTE_TYPE_PAGE) 258 level = KVM_PGTABLE_LAST_LEVEL; 259 else 260 level = KVM_PGTABLE_LAST_LEVEL - 1; /* create_fixblock() guarantees PMD level */ 261 262 WRITE_ONCE(*ptep, *ptep & ~KVM_PTE_VALID); 263 264 /* 265 * Irritatingly, the architecture requires that we use inner-shareable 266 * broadcast TLB invalidation here in case another CPU speculates 267 * through our fixmap and decides to create an "amalagamation of the 268 * values held in the TLB" due to the apparent lack of a 269 * break-before-make sequence. 270 * 271 * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#mf10dfbaf1eaef9274c581b81c53758918c1d0f03 272 */ 273 dsb(ishst); 274 __tlbi_level(vale2is, addr, level); 275 __tlbi_sync_s1ish_hyp(); 276 isb(); 277 } 278 279 void hyp_fixmap_unmap(void) 280 { 281 fixmap_clear_slot(this_cpu_ptr(&fixmap_slots)); 282 } 283 284 static int __create_fixmap_slot_cb(const struct kvm_pgtable_visit_ctx *ctx, 285 enum kvm_pgtable_walk_flags visit) 286 { 287 struct hyp_fixmap_slot *slot = (struct hyp_fixmap_slot *)ctx->arg; 288 289 if (!kvm_pte_valid(ctx->old) || (ctx->end - ctx->start) != kvm_granule_size(ctx->level)) 290 return -EINVAL; 291 292 slot->addr = ctx->addr; 293 slot->ptep = ctx->ptep; 294 295 /* 296 * Clear the PTE, but keep the page-table page refcount elevated to 297 * prevent it from ever being freed. This lets us manipulate the PTEs 298 * by hand safely without ever needing to allocate memory. 299 */ 300 fixmap_clear_slot(slot); 301 302 return 0; 303 } 304 305 static int create_fixmap_slot(u64 addr, u64 cpu) 306 { 307 struct kvm_pgtable_walker walker = { 308 .cb = __create_fixmap_slot_cb, 309 .flags = KVM_PGTABLE_WALK_LEAF, 310 .arg = per_cpu_ptr(&fixmap_slots, cpu), 311 }; 312 313 return kvm_pgtable_walk(&pkvm_pgtable, addr, PAGE_SIZE, &walker); 314 } 315 316 #if PAGE_SHIFT < 16 317 #define HAS_FIXBLOCK 318 static struct hyp_fixmap_slot hyp_fixblock_slot; 319 static DEFINE_HYP_SPINLOCK(hyp_fixblock_lock); 320 #endif 321 322 static int create_fixblock(void) 323 { 324 #ifdef HAS_FIXBLOCK 325 struct kvm_pgtable_walker walker = { 326 .cb = __create_fixmap_slot_cb, 327 .flags = KVM_PGTABLE_WALK_LEAF, 328 .arg = &hyp_fixblock_slot, 329 }; 330 unsigned long addr; 331 phys_addr_t phys; 332 int ret, i; 333 334 /* Find a RAM phys address, PMD aligned */ 335 for (i = 0; i < hyp_memblock_nr; i++) { 336 phys = ALIGN(hyp_memory[i].base, PMD_SIZE); 337 if (phys + PMD_SIZE < (hyp_memory[i].base + hyp_memory[i].size)) 338 break; 339 } 340 341 if (i >= hyp_memblock_nr) 342 return -EINVAL; 343 344 hyp_spin_lock(&pkvm_pgd_lock); 345 addr = ALIGN(__io_map_next, PMD_SIZE); 346 ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE); 347 if (ret) 348 goto unlock; 349 350 ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PMD_SIZE, phys, PAGE_HYP); 351 if (ret) 352 goto unlock; 353 354 ret = kvm_pgtable_walk(&pkvm_pgtable, addr, PMD_SIZE, &walker); 355 356 unlock: 357 hyp_spin_unlock(&pkvm_pgd_lock); 358 359 return ret; 360 #else 361 return 0; 362 #endif 363 } 364 365 void *hyp_fixblock_map(phys_addr_t phys, size_t *size) 366 { 367 #ifdef HAS_FIXBLOCK 368 *size = PMD_SIZE; 369 hyp_spin_lock(&hyp_fixblock_lock); 370 return fixmap_map_slot(&hyp_fixblock_slot, phys) + offset_in_page(phys); 371 #else 372 *size = PAGE_SIZE; 373 return hyp_fixmap_map(phys); 374 #endif 375 } 376 377 void hyp_fixblock_unmap(void) 378 { 379 #ifdef HAS_FIXBLOCK 380 fixmap_clear_slot(&hyp_fixblock_slot); 381 hyp_spin_unlock(&hyp_fixblock_lock); 382 #else 383 hyp_fixmap_unmap(); 384 #endif 385 } 386 387 int hyp_create_fixmap(void) 388 { 389 unsigned long addr, i; 390 int ret; 391 392 for (i = 0; i < hyp_nr_cpus; i++) { 393 ret = pkvm_alloc_private_va_range(PAGE_SIZE, &addr); 394 if (ret) 395 return ret; 396 397 ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PAGE_SIZE, 398 __hyp_pa(__hyp_bss_start), PAGE_HYP); 399 if (ret) 400 return ret; 401 402 ret = create_fixmap_slot(addr, i); 403 if (ret) 404 return ret; 405 } 406 407 return create_fixblock(); 408 } 409 410 int hyp_create_idmap(u32 hyp_va_bits) 411 { 412 unsigned long start, end; 413 414 start = hyp_virt_to_phys((void *)__hyp_idmap_text_start); 415 start = ALIGN_DOWN(start, PAGE_SIZE); 416 417 end = hyp_virt_to_phys((void *)__hyp_idmap_text_end); 418 end = ALIGN(end, PAGE_SIZE); 419 420 /* 421 * One half of the VA space is reserved to linearly map portions of 422 * memory -- see va_layout.c for more details. The other half of the VA 423 * space contains the trampoline page, and needs some care. Split that 424 * second half in two and find the quarter of VA space not conflicting 425 * with the idmap to place the IOs and the vmemmap. IOs use the lower 426 * half of the quarter and the vmemmap the upper half. 427 */ 428 __io_map_base = start & BIT(hyp_va_bits - 2); 429 __io_map_base ^= BIT(hyp_va_bits - 2); 430 __io_map_next = __io_map_base; 431 __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3); 432 433 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); 434 } 435 436 int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) 437 { 438 unsigned long addr, prev_next; 439 size_t size; 440 int ret; 441 442 hyp_spin_lock(&pkvm_pgd_lock); 443 444 prev_next = __io_map_next; 445 /* 446 * Efficient stack verification using the NVHE_STACK_SHIFT bit implies 447 * an alignment of our allocation on the order of the size. 448 */ 449 size = NVHE_STACK_SIZE * 2; 450 addr = ALIGN(__io_map_next, size); 451 452 ret = __pkvm_alloc_private_va_range(addr, size); 453 if (!ret) { 454 /* 455 * Since the stack grows downwards, map the stack to the page 456 * at the higher address and leave the lower guard page 457 * unbacked. 458 * 459 * Any valid stack address now has the NVHE_STACK_SHIFT bit as 1 460 * and addresses corresponding to the guard page have the 461 * NVHE_STACK_SHIFT bit as 0 - this is used for overflow detection. 462 */ 463 ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE, 464 NVHE_STACK_SIZE, phys, PAGE_HYP); 465 if (ret) 466 __io_map_next = prev_next; 467 } 468 hyp_spin_unlock(&pkvm_pgd_lock); 469 470 *haddr = addr + size; 471 472 return ret; 473 } 474 475 static int check_page_ownership(phys_addr_t phys) 476 { 477 kvm_pte_t pte; 478 bool host_ok; 479 int ret; 480 481 if (addr_is_memory(phys)) { 482 struct hyp_page *page = hyp_phys_to_page(phys); 483 484 if (get_hyp_state(page) != PKVM_PAGE_OWNED || 485 get_host_state(page) != PKVM_NOPAGE) 486 return -EPERM; 487 } 488 489 ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); 490 if (ret) 491 return ret; 492 493 /* Hyp text may stay host-readable, see fix_host_ownership_walker(). */ 494 if (kvm_pte_valid(pte) && addr_is_hyp_text(phys)) 495 host_ok = !(kvm_pgtable_stage2_pte_prot(pte) & KVM_PGTABLE_PROT_W); 496 else 497 host_ok = host_stage2_pte_is_hyp_owned(pte); 498 499 return host_ok ? 0 : -EPERM; 500 } 501 502 static int check_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, 503 enum kvm_pgtable_walk_flags visit) 504 { 505 phys_addr_t phys, end; 506 int ret; 507 508 if (!kvm_pte_valid(ctx->old)) 509 return 0; 510 511 phys = kvm_pte_to_phys(ctx->old); 512 end = phys + kvm_granule_size(ctx->level); 513 for (; phys < end; phys += PAGE_SIZE) { 514 ret = check_page_ownership(phys); 515 if (ret) 516 return ret; 517 } 518 519 return 0; 520 } 521 522 int pkvm_check_host_ownership(void) 523 { 524 struct kvm_pgtable_walker walker = { 525 .cb = check_host_ownership_walker, 526 .flags = KVM_PGTABLE_WALK_LEAF, 527 }; 528 529 /* The private range and the vmemmap share one quarter of the VA space. */ 530 return kvm_pgtable_walk(&pkvm_pgtable, __io_map_base, 531 BIT(pkvm_pgtable.ia_bits - 2), &walker); 532 } 533 534 static void *admit_host_page(void *arg) 535 { 536 struct kvm_hyp_memcache *host_mc = arg; 537 538 if (!host_mc->nr_pages) 539 return NULL; 540 541 /* 542 * The host still owns the pages in its memcache, so we need to go 543 * through a full host-to-hyp donation cycle to change it. Fortunately, 544 * __pkvm_host_donate_hyp() takes care of races for us, so if it 545 * succeeds we're good to go. 546 */ 547 if (__pkvm_host_donate_hyp(hyp_phys_to_pfn(host_mc->head), 1)) 548 return NULL; 549 550 return pop_hyp_memcache(host_mc, hyp_phys_to_virt); 551 } 552 553 /* Refill our local memcache by popping pages from the one provided by the host. */ 554 int refill_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages, 555 struct kvm_hyp_memcache *host_mc) 556 { 557 struct kvm_hyp_memcache tmp = *host_mc; 558 int ret; 559 560 ret = __topup_hyp_memcache(mc, min_pages, admit_host_page, 561 hyp_virt_to_phys, &tmp); 562 *host_mc = tmp; 563 564 return ret; 565 } 566