1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/fixmap.h> 12 #include <asm/kvm_arm.h> 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_mmu.h> 15 #include <asm/kvm_nested.h> 16 #include <asm/sysreg.h> 17 18 #include "sys_regs.h" 19 #include "vgic/vgic.h" 20 21 struct vncr_tlb { 22 /* The guest's VNCR_EL2 */ 23 u64 gva; 24 struct s1_walk_info wi; 25 struct s1_walk_result wr; 26 27 u64 hpa; 28 bool hpa_writable; 29 30 /* -1 when not mapped on a CPU */ 31 atomic_t cpu; 32 33 /* 34 * true if the TLB is valid. Can only be changed with the 35 * mmu_lock held. 36 */ 37 bool valid; 38 }; 39 40 /* 41 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 42 * memory usage and potential number of different sets of S2 PTs in 43 * the guests. Running out of S2 MMUs only affects performance (we 44 * will invalidate them more often). 45 */ 46 #define S2_MMU_PER_VCPU 2 47 48 void kvm_init_nested(struct kvm *kvm) 49 { 50 kvm->arch.nested_mmus = NULL; 51 kvm->arch.nested_mmus_size = 0; 52 atomic_set(&kvm->arch.vncr_tlb_count, 0); 53 } 54 55 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 56 { 57 /* 58 * We only initialise the IPA range on the canonical MMU, which 59 * defines the contract between KVM and userspace on where the 60 * "hardware" is in the IPA space. This affects the validity of MMIO 61 * exits forwarded to userspace, for example. 62 * 63 * For nested S2s, we use the PARange as exposed to the guest, as it 64 * is allowed to use it at will to expose whatever memory map it 65 * wants to its own guests as it would be on real HW. 66 */ 67 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 68 } 69 70 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 71 { 72 struct kvm *kvm = vcpu->kvm; 73 struct kvm_s2_mmu *tmp; 74 int num_mmus, ret = 0; 75 76 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && 77 !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 78 return -EINVAL; 79 80 if (!vcpu->arch.ctxt.vncr_array) 81 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT | 82 __GFP_ZERO); 83 84 if (!vcpu->arch.ctxt.vncr_array) 85 return -ENOMEM; 86 87 /* 88 * Let's treat memory allocation failures as benign: If we fail to 89 * allocate anything, return an error and keep the allocated array 90 * alive. Userspace may try to recover by initializing the vcpu 91 * again, and there is no reason to affect the whole VM for this. 92 */ 93 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 94 95 if (num_mmus > kvm->arch.nested_mmus_size) { 96 tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT); 97 if (!tmp) 98 return -ENOMEM; 99 100 write_lock(&kvm->mmu_lock); 101 102 if (kvm->arch.nested_mmus_size) { 103 memcpy(tmp, kvm->arch.nested_mmus, 104 size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); 105 106 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 107 tmp[i].pgt->mmu = &tmp[i]; 108 } 109 110 swap(kvm->arch.nested_mmus, tmp); 111 112 write_unlock(&kvm->mmu_lock); 113 114 kvfree(tmp); 115 } 116 117 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 118 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); 119 120 if (ret) { 121 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 122 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); 123 124 free_page((unsigned long)vcpu->arch.ctxt.vncr_array); 125 vcpu->arch.ctxt.vncr_array = NULL; 126 127 return ret; 128 } 129 130 kvm->arch.nested_mmus_size = num_mmus; 131 132 return 0; 133 } 134 135 struct s2_walk_info { 136 u64 baddr; 137 unsigned int max_oa_bits; 138 unsigned int pgshift; 139 unsigned int sl; 140 unsigned int t0sz; 141 bool be; 142 bool ha; 143 }; 144 145 static u32 compute_fsc(int level, u32 fsc) 146 { 147 return fsc | (level & 0x3); 148 } 149 150 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 151 { 152 u32 esr; 153 154 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 155 esr |= compute_fsc(level, fsc); 156 return esr; 157 } 158 159 static int get_ia_size(struct s2_walk_info *wi) 160 { 161 return 64 - wi->t0sz; 162 } 163 164 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi, 165 int level, int input_size, int stride) 166 { 167 int start_size, pa_max; 168 169 pa_max = kvm_get_pa_bits(vcpu->kvm); 170 171 /* Check translation limits */ 172 switch (BIT(wi->pgshift)) { 173 case SZ_64K: 174 if (level == 0 || (level == 1 && pa_max <= 42)) 175 return -EFAULT; 176 break; 177 case SZ_16K: 178 if (level == 0 || (level == 1 && pa_max <= 40)) 179 return -EFAULT; 180 break; 181 case SZ_4K: 182 if (level < 0 || (level == 0 && pa_max <= 42)) 183 return -EFAULT; 184 break; 185 } 186 187 /* Check input size limits */ 188 if (input_size > pa_max) 189 return -EFAULT; 190 191 /* Check number of entries in starting level table */ 192 start_size = input_size - ((3 - level) * stride + wi->pgshift); 193 if (start_size < 1 || start_size > stride + 4) 194 return -EFAULT; 195 196 return 0; 197 } 198 199 /* Check if output is within boundaries */ 200 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 201 { 202 unsigned int output_size = wi->max_oa_bits; 203 204 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 205 return -1; 206 207 return 0; 208 } 209 210 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc, 211 struct s2_walk_info *wi) 212 { 213 u64 val; 214 int r; 215 216 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val)); 217 if (r) 218 return r; 219 220 /* 221 * Handle reversedescriptors if endianness differs between the 222 * host and the guest hypervisor. 223 */ 224 if (wi->be) 225 *desc = be64_to_cpu((__force __be64)val); 226 else 227 *desc = le64_to_cpu((__force __le64)val); 228 229 return 0; 230 } 231 232 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new, 233 struct s2_walk_info *wi) 234 { 235 if (wi->be) { 236 old = (__force u64)cpu_to_be64(old); 237 new = (__force u64)cpu_to_be64(new); 238 } else { 239 old = (__force u64)cpu_to_le64(old); 240 new = (__force u64)cpu_to_le64(new); 241 } 242 243 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new); 244 } 245 246 /* 247 * This is essentially a C-version of the pseudo code from the ARM ARM 248 * AArch64.TranslationTableWalk function. I strongly recommend looking at 249 * that pseudocode in trying to understand this. 250 * 251 * Must be called with the kvm->srcu read lock held 252 */ 253 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa, 254 struct s2_walk_info *wi, struct kvm_s2_trans *out) 255 { 256 int first_block_level, level, stride, input_size, base_lower_bound; 257 phys_addr_t base_addr; 258 unsigned int addr_top, addr_bottom; 259 u64 desc, new_desc; /* page table entry */ 260 int ret; 261 phys_addr_t paddr; 262 263 switch (BIT(wi->pgshift)) { 264 default: 265 case SZ_64K: 266 case SZ_16K: 267 level = 3 - wi->sl; 268 first_block_level = 2; 269 break; 270 case SZ_4K: 271 level = 2 - wi->sl; 272 first_block_level = 1; 273 break; 274 } 275 276 stride = wi->pgshift - 3; 277 input_size = get_ia_size(wi); 278 if (input_size > 48 || input_size < 25) 279 return -EFAULT; 280 281 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride); 282 if (WARN_ON(ret)) { 283 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT); 284 return ret; 285 } 286 287 base_lower_bound = 3 + input_size - ((3 - level) * stride + 288 wi->pgshift); 289 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 290 291 if (check_output_size(wi, base_addr)) { 292 /* R_BFHQH */ 293 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ); 294 return 1; 295 } 296 297 addr_top = input_size - 1; 298 299 while (1) { 300 phys_addr_t index; 301 302 addr_bottom = (3 - level) * stride + wi->pgshift; 303 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 304 >> (addr_bottom - 3); 305 306 paddr = base_addr | index; 307 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi); 308 if (ret < 0) { 309 out->esr = ESR_ELx_FSC_SEA_TTW(level); 310 return ret; 311 } 312 313 new_desc = desc; 314 315 /* Check for valid descriptor at this point */ 316 if (!(desc & KVM_PTE_VALID)) { 317 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 318 out->desc = desc; 319 return 1; 320 } 321 322 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) { 323 if (level < 3) 324 break; 325 326 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 327 out->desc = desc; 328 return 1; 329 } 330 331 /* We're at the final level */ 332 if (level == 3) 333 break; 334 335 if (check_output_size(wi, desc)) { 336 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 337 out->desc = desc; 338 return 1; 339 } 340 341 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 342 343 level += 1; 344 addr_top = addr_bottom - 1; 345 } 346 347 if (level < first_block_level) { 348 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 349 out->desc = desc; 350 return 1; 351 } 352 353 if (check_output_size(wi, desc)) { 354 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 355 out->desc = desc; 356 return 1; 357 } 358 359 if (wi->ha) 360 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF; 361 362 if (new_desc != desc) { 363 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi); 364 if (ret == -EAGAIN) 365 return ret; 366 if (ret) { 367 out->esr = ESR_ELx_FSC_SEA_TTW(level); 368 out->desc = desc; 369 return 1; 370 } 371 372 desc = new_desc; 373 } 374 375 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) { 376 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 377 out->desc = desc; 378 return 1; 379 } 380 381 addr_bottom += contiguous_bit_shift(desc, wi, level); 382 383 /* Calculate and return the result */ 384 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 385 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 386 out->output = paddr; 387 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 388 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R; 389 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; 390 out->level = level; 391 out->desc = desc; 392 return 0; 393 } 394 395 #define _has_tgran_2(__r, __sz) \ 396 ({ \ 397 u64 _s1, _s2, _mmfr0 = __r; \ 398 \ 399 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 400 TGRAN##__sz##_2, _mmfr0); \ 401 \ 402 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 403 TGRAN##__sz, _mmfr0); \ 404 \ 405 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \ 406 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \ 407 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \ 408 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \ 409 }) 410 411 static bool has_tgran_2(u64 mmfr0, unsigned int shift) 412 { 413 switch (shift) { 414 case 12: 415 return _has_tgran_2(mmfr0, 4); 416 case 14: 417 return _has_tgran_2(mmfr0, 16); 418 case 16: 419 return _has_tgran_2(mmfr0, 64); 420 default: 421 BUG(); 422 } 423 } 424 425 static unsigned int fallback_tgran2_shift(u64 mmfr0) 426 { 427 if (has_tgran_2(mmfr0, PAGE_SHIFT)) 428 return PAGE_SHIFT; 429 else if (has_tgran_2(mmfr0, 12)) 430 return 12; 431 else if (has_tgran_2(mmfr0, 14)) 432 return 14; 433 else if (has_tgran_2(mmfr0, 16)) 434 return 16; 435 else 436 return PAGE_SHIFT; 437 } 438 439 static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr) 440 { 441 u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr); 442 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 443 unsigned int shift; 444 445 switch (tg0) { 446 case VTCR_EL2_TG0_4K: 447 shift = 12; 448 break; 449 case VTCR_EL2_TG0_16K: 450 shift = 14; 451 break; 452 case VTCR_EL2_TG0_64K: 453 /* IMPDEF: treat any other value as 64k, subject to fallback */ 454 default: 455 shift = 16; 456 } 457 458 /* 459 * If TGx is programmed to an unimplemented value (not advertised in 460 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is 461 * written, as per the architecture. Choose an available one while 462 * prioritizing PAGE_SIZE. 463 */ 464 if (!has_tgran_2(mmfr0, shift)) 465 return fallback_tgran2_shift(mmfr0); 466 467 return shift; 468 } 469 470 static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr) 471 { 472 return BIT(vtcr_to_tg0_pgshift(kvm, vtcr)); 473 } 474 475 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi) 476 { 477 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 478 479 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 480 wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK; 481 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr); 482 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 483 /* Global limit for now, should eventually be per-VM */ 484 wi->max_oa_bits = min(get_kvm_ipa_limit(), 485 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false)); 486 wi->ha = vtcr & VTCR_EL2_HA; 487 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 488 } 489 490 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 491 struct kvm_s2_trans *result) 492 { 493 struct s2_walk_info wi; 494 int ret; 495 496 result->esr = 0; 497 498 if (!vcpu_has_nv(vcpu)) 499 return 0; 500 501 setup_s2_walk(vcpu, &wi); 502 503 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result); 504 if (ret) 505 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 506 507 return ret; 508 } 509 510 static unsigned int __ttl_to_size(u8 ttl) 511 { 512 int level = ttl & 3; 513 int gran = (ttl >> 2) & 3; 514 unsigned int max_size = 0; 515 516 switch (gran) { 517 case TLBI_TTL_TG_4K: 518 switch (level) { 519 case 0: 520 break; 521 case 1: 522 max_size = SZ_1G; 523 break; 524 case 2: 525 max_size = SZ_2M; 526 break; 527 case 3: 528 max_size = SZ_4K; 529 break; 530 } 531 break; 532 case TLBI_TTL_TG_16K: 533 switch (level) { 534 case 0: 535 case 1: 536 break; 537 case 2: 538 max_size = SZ_32M; 539 break; 540 case 3: 541 max_size = SZ_16K; 542 break; 543 } 544 break; 545 case TLBI_TTL_TG_64K: 546 switch (level) { 547 case 0: 548 case 1: 549 /* No 52bit IPA support */ 550 break; 551 case 2: 552 max_size = SZ_512M; 553 break; 554 case 3: 555 max_size = SZ_64K; 556 break; 557 } 558 break; 559 default: /* No size information */ 560 break; 561 } 562 563 return max_size; 564 } 565 566 static unsigned int ttl_to_size(u8 ttl) 567 { 568 return __ttl_to_size(ttl) ?: SZ_1G; 569 } 570 571 static u8 pgshift_level_to_ttl(u16 shift, s8 level) 572 { 573 u8 ttl; 574 575 /* 576 * If we don't have a proper level, fallback to the maximum 577 * size. 578 */ 579 if (level < 0) 580 return 0; 581 582 switch(shift) { 583 case 12: 584 ttl = TLBI_TTL_TG_4K; 585 break; 586 case 14: 587 ttl = TLBI_TTL_TG_16K; 588 break; 589 case 16: 590 ttl = TLBI_TTL_TG_64K; 591 break; 592 default: 593 BUG(); 594 } 595 596 ttl <<= 2; 597 ttl |= level & 3; 598 599 return ttl; 600 } 601 602 /* 603 * Compute the equivalent of the TTL field by parsing the shadow PT. The 604 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 605 * retrieved from first entry carrying the level as a tag. 606 */ 607 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 608 { 609 size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr); 610 u64 tmp, sz = 0; 611 kvm_pte_t pte; 612 u8 ttl, level; 613 614 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 615 616 switch (tg0_size) { 617 case SZ_4K: 618 ttl = (TLBI_TTL_TG_4K << 2); 619 break; 620 case SZ_16K: 621 ttl = (TLBI_TTL_TG_16K << 2); 622 break; 623 case SZ_64K: 624 default: /* IMPDEF: treat any other value as 64k */ 625 ttl = (TLBI_TTL_TG_64K << 2); 626 break; 627 } 628 629 tmp = addr; 630 631 again: 632 /* Iteratively compute the block sizes for a particular granule size */ 633 switch (tg0_size) { 634 case SZ_4K: 635 if (sz < SZ_4K) sz = SZ_4K; 636 else if (sz < SZ_2M) sz = SZ_2M; 637 else if (sz < SZ_1G) sz = SZ_1G; 638 else sz = 0; 639 break; 640 case SZ_16K: 641 if (sz < SZ_16K) sz = SZ_16K; 642 else if (sz < SZ_32M) sz = SZ_32M; 643 else sz = 0; 644 break; 645 case SZ_64K: 646 default: /* IMPDEF: treat any other value as 64k */ 647 if (sz < SZ_64K) sz = SZ_64K; 648 else if (sz < SZ_512M) sz = SZ_512M; 649 else sz = 0; 650 break; 651 } 652 653 if (sz == 0) 654 return 0; 655 656 tmp &= ~(sz - 1); 657 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 658 goto again; 659 if (!(pte & PTE_VALID)) 660 goto again; 661 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 662 if (!level) 663 goto again; 664 665 ttl |= level; 666 667 /* 668 * We now have found some level information in the shadow S2. Check 669 * that the resulting range is actually including the original IPA. 670 */ 671 sz = ttl_to_size(ttl); 672 if (addr < (tmp + sz)) 673 return ttl; 674 675 return 0; 676 } 677 678 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 679 { 680 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 681 unsigned long max_size; 682 u8 ttl; 683 684 ttl = FIELD_GET(TLBI_TTL_MASK, val); 685 686 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 687 /* No TTL, check the shadow S2 for a hint */ 688 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 689 ttl = get_guest_mapping_ttl(mmu, addr); 690 } 691 692 /* 693 * Don't use the default 1GB fallback, as we can adapt to the 694 * max mapping size we allow at S2. 695 */ 696 max_size = __ttl_to_size(ttl); 697 698 if (!max_size) { 699 /* Compute the maximum extent of the invalidation */ 700 switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) { 701 case SZ_4K: 702 max_size = SZ_1G; 703 break; 704 case SZ_16K: 705 max_size = SZ_32M; 706 break; 707 case SZ_64K: 708 default: /* IMPDEF: treat any other value as 64k */ 709 /* 710 * No, we do not support 52bit IPA in nested yet. Once 711 * we do, this should be 4TB. 712 */ 713 max_size = SZ_512M; 714 break; 715 } 716 } 717 718 WARN_ON(!max_size); 719 return max_size; 720 } 721 722 /* 723 * We can have multiple *different* MMU contexts with the same VMID: 724 * 725 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 726 * 727 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 728 * VBBTR_EL2.BADDR address 729 * 730 * - A combination of the above... 731 * 732 * We can always identify which MMU context to pick at run-time. However, 733 * TLB invalidation involving a VMID must take action on all the TLBs using 734 * this particular VMID. This translates into applying the same invalidation 735 * operation to all the contexts that are using this VMID. Moar phun! 736 */ 737 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 738 const union tlbi_info *info, 739 void (*tlbi_callback)(struct kvm_s2_mmu *, 740 const union tlbi_info *)) 741 { 742 write_lock(&kvm->mmu_lock); 743 744 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 745 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 746 747 if (!kvm_s2_mmu_valid(mmu)) 748 continue; 749 750 if (vmid == get_vmid(mmu->tlb_vttbr)) 751 tlbi_callback(mmu, info); 752 } 753 754 write_unlock(&kvm->mmu_lock); 755 } 756 757 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 758 { 759 struct kvm *kvm = vcpu->kvm; 760 bool nested_stage2_enabled; 761 u64 vttbr, vtcr, hcr; 762 763 lockdep_assert_held_write(&kvm->mmu_lock); 764 765 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 766 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 767 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 768 769 nested_stage2_enabled = hcr & HCR_VM; 770 771 /* Don't consider the CnP bit for the vttbr match */ 772 vttbr &= ~VTTBR_CNP_BIT; 773 774 /* 775 * Two possibilities when looking up a S2 MMU context: 776 * 777 * - either S2 is enabled in the guest, and we need a context that is 778 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 779 * which makes it safe from a TLB conflict perspective (a broken 780 * guest won't be able to generate them), 781 * 782 * - or S2 is disabled, and we need a context that is S2-disabled 783 * and matches the VMID only, as all TLBs are tagged by VMID even 784 * if S2 translation is disabled. 785 */ 786 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 787 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 788 789 if (!kvm_s2_mmu_valid(mmu)) 790 continue; 791 792 if (nested_stage2_enabled && 793 mmu->nested_stage2_enabled && 794 vttbr == mmu->tlb_vttbr && 795 vtcr == mmu->tlb_vtcr) 796 return mmu; 797 798 if (!nested_stage2_enabled && 799 !mmu->nested_stage2_enabled && 800 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 801 return mmu; 802 } 803 return NULL; 804 } 805 806 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 807 { 808 struct kvm *kvm = vcpu->kvm; 809 struct kvm_s2_mmu *s2_mmu; 810 int i; 811 812 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 813 814 s2_mmu = lookup_s2_mmu(vcpu); 815 if (s2_mmu) 816 goto out; 817 818 /* 819 * Make sure we don't always search from the same point, or we 820 * will always reuse a potentially active context, leaving 821 * free contexts unused. 822 */ 823 for (i = kvm->arch.nested_mmus_next; 824 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 825 i++) { 826 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 827 828 if (atomic_read(&s2_mmu->refcnt) == 0) 829 break; 830 } 831 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 832 833 /* Set the scene for the next search */ 834 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 835 836 /* Make sure we don't forget to do the laundry */ 837 if (kvm_s2_mmu_valid(s2_mmu)) { 838 kvm_nested_s2_ptdump_remove_debugfs(s2_mmu); 839 s2_mmu->pending_unmap = true; 840 } 841 842 /* 843 * The virtual VMID (modulo CnP) will be used as a key when matching 844 * an existing kvm_s2_mmu. 845 * 846 * We cache VTCR at allocation time, once and for all. It'd be great 847 * if the guest didn't screw that one up, as this is not very 848 * forgiving... 849 */ 850 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 851 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 852 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 853 854 kvm_nested_s2_ptdump_create_debugfs(s2_mmu); 855 856 out: 857 atomic_inc(&s2_mmu->refcnt); 858 859 /* 860 * Set the vCPU request to perform an unmap, even if the pending unmap 861 * originates from another vCPU. This guarantees that the MMU has been 862 * completely unmapped before any vCPU actually uses it, and allows 863 * multiple vCPUs to lend a hand with completing the unmap. 864 */ 865 if (s2_mmu->pending_unmap) 866 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 867 868 return s2_mmu; 869 } 870 871 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 872 { 873 /* CnP being set denotes an invalid entry */ 874 mmu->tlb_vttbr = VTTBR_CNP_BIT; 875 mmu->nested_stage2_enabled = false; 876 atomic_set(&mmu->refcnt, 0); 877 } 878 879 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 880 { 881 /* 882 * If the vCPU kept its reference on the MMU after the last put, 883 * keep rolling with it. 884 */ 885 if (is_hyp_ctxt(vcpu)) { 886 if (!vcpu->arch.hw_mmu) 887 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 888 } else { 889 if (!vcpu->arch.hw_mmu) { 890 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 891 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 892 } 893 894 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) 895 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 896 } 897 } 898 899 /* 900 * Unmapping an L1 VNCR can happen concurrently without the mmu lock being 901 * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures 902 * that only one CPU sets it to -1 while getting a valid CPU number back. 903 */ 904 static int unmap_l1_vncr(struct vncr_tlb *vt) 905 { 906 int cpu = atomic_xchg_relaxed(&vt->cpu, -1); 907 908 if (cpu != -1) 909 clear_fixmap(vncr_fixmap(cpu)); 910 911 return cpu; 912 } 913 914 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) 915 { 916 if (!host_data_test_flag(L1_VNCR_MAPPED)) 917 return; 918 919 BUG_ON(is_hyp_ctxt(vcpu)); 920 921 /* 922 * Unconditionally unmap the local VNCR if we have lost the race 923 * against a concurrent TLBI. Otherwise we could end-up running 924 * another vcpu with VNCR still mapped if the TLBI thread is 925 * preempted between the exchange and the clear_fixmap(). 926 * 927 * Note that we do not care about the TLBI nuking the fixmap behind 928 * the back of an running vcpu. This will only generate a fault and 929 * possibly a retranslation. 930 */ 931 if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) 932 clear_fixmap(vncr_fixmap(smp_processor_id())); 933 host_data_clear_flag(L1_VNCR_MAPPED); 934 } 935 936 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 937 { 938 /* Unconditionally drop the VNCR mapping if we have one */ 939 this_cpu_reset_vncr_fixmap(vcpu); 940 941 /* 942 * Keep a reference on the associated stage-2 MMU if the vCPU is 943 * scheduling out and not in WFI emulation, suggesting it is likely to 944 * reuse the MMU sometime soon. 945 */ 946 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 947 return; 948 949 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 950 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 951 952 vcpu->arch.hw_mmu = NULL; 953 } 954 955 /* 956 * Returns non-zero if permission fault is handled by injecting it to the next 957 * level hypervisor. 958 */ 959 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 960 { 961 bool forward_fault = false; 962 963 trans->esr = 0; 964 965 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 966 return 0; 967 968 if (kvm_vcpu_trap_is_iabt(vcpu)) { 969 if (vcpu_mode_priv(vcpu)) 970 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans); 971 else 972 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans); 973 } else { 974 bool write_fault = kvm_is_write_fault(vcpu); 975 976 forward_fault = ((write_fault && !trans->writable) || 977 (!write_fault && !trans->readable)); 978 } 979 980 if (forward_fault) 981 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 982 983 return forward_fault; 984 } 985 986 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 987 { 988 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 989 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 990 991 return kvm_inject_nested_sync(vcpu, esr_el2); 992 } 993 994 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) 995 { 996 enum vcpu_sysreg ttbr_elx; 997 u64 tcr; 998 u16 asid; 999 1000 switch (regime) { 1001 case TR_EL10: 1002 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); 1003 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1; 1004 break; 1005 case TR_EL20: 1006 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); 1007 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2; 1008 break; 1009 default: 1010 BUG(); 1011 } 1012 1013 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx)); 1014 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || 1015 !(tcr & TCR_ASID16)) 1016 asid &= GENMASK(7, 0); 1017 1018 return asid; 1019 } 1020 1021 static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt) 1022 { 1023 BUG_ON(!vt->valid); 1024 vt->valid = false; 1025 unmap_l1_vncr(vt); 1026 atomic_dec(&kvm->arch.vncr_tlb_count); 1027 } 1028 1029 static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, 1030 u64 scope_start, u64 scope_size) 1031 { 1032 u64 tlb_size, tlb_start, tlb_end, scope_end; 1033 1034 tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); 1035 1036 tlb_start = addr & ~(tlb_size - 1); 1037 tlb_end = tlb_start + tlb_size - 1; 1038 scope_end = scope_start + scope_size - 1; 1039 1040 return !(tlb_end < scope_start || tlb_start > scope_end); 1041 } 1042 1043 /* 1044 * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and 1045 * either can race against a vcpu not being onlined yet (no pseudo-TLB 1046 * allocated). Similarly, the TLB might be invalid. Skip those, as they 1047 * obviously don't participate in the invalidation at this stage. 1048 */ 1049 #define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \ 1050 kvm_for_each_vcpu(idx, vcpup, kvm) \ 1051 if (((tlbp) = vcpup->arch.vncr_tlb) && \ 1052 (tlbp)->valid) 1053 1054 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) 1055 { 1056 struct kvm_vcpu *vcpu; 1057 struct vncr_tlb *vt; 1058 unsigned long i; 1059 1060 lockdep_assert_held_write(&kvm->mmu_lock); 1061 1062 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1063 return; 1064 1065 /* 1066 * Note that invalidating the VNCR on the back of an MMU notifier 1067 * doesn't require messing with the invalidation counter for a 1068 * parallel walk. The notifier itself will have bumped the counter, 1069 * making sure we rewalk. 1070 */ 1071 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) 1072 if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) 1073 invalidate_vncr(kvm, vt); 1074 } 1075 1076 struct s1e2_tlbi_scope { 1077 enum { 1078 TLBI_ALL, 1079 TLBI_VA, 1080 TLBI_VAA, 1081 TLBI_ASID, 1082 } type; 1083 1084 u16 asid; 1085 u64 va; 1086 u64 size; 1087 }; 1088 1089 static void invalidate_vncr_va(struct kvm *kvm, 1090 struct s1e2_tlbi_scope *scope) 1091 { 1092 struct kvm_vcpu *vcpu; 1093 struct vncr_tlb *vt; 1094 unsigned long i; 1095 1096 lockdep_assert_held_write(&kvm->mmu_lock); 1097 1098 /* 1099 * We might be performing a parallel S1 walk, so bump up the 1100 * invalidation counter even in the absence of an actual VNCR TLB 1101 * invalidation, as this could indicate that the guest has gone 1102 * through a BBM sequence. 1103 */ 1104 kvm->mmu_invalidate_seq++; 1105 smp_wmb(); 1106 1107 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { 1108 switch (scope->type) { 1109 case TLBI_ALL: 1110 break; 1111 1112 case TLBI_VA: 1113 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) 1114 continue; 1115 if (vt->wr.nG && vt->wr.asid != scope->asid) 1116 continue; 1117 break; 1118 1119 case TLBI_VAA: 1120 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) 1121 continue; 1122 break; 1123 1124 case TLBI_ASID: 1125 if (!vt->wr.nG || vt->wr.asid != scope->asid) 1126 continue; 1127 break; 1128 } 1129 1130 invalidate_vncr(kvm, vt); 1131 } 1132 } 1133 1134 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48) 1135 1136 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, 1137 struct s1e2_tlbi_scope *scope) 1138 { 1139 switch (inst) { 1140 case OP_TLBI_ALLE2: 1141 case OP_TLBI_ALLE2IS: 1142 case OP_TLBI_ALLE2OS: 1143 case OP_TLBI_VMALLE1: 1144 case OP_TLBI_VMALLE1IS: 1145 case OP_TLBI_VMALLE1OS: 1146 case OP_TLBI_ALLE2NXS: 1147 case OP_TLBI_ALLE2ISNXS: 1148 case OP_TLBI_ALLE2OSNXS: 1149 case OP_TLBI_VMALLE1NXS: 1150 case OP_TLBI_VMALLE1ISNXS: 1151 case OP_TLBI_VMALLE1OSNXS: 1152 scope->type = TLBI_ALL; 1153 break; 1154 case OP_TLBI_VAE2: 1155 case OP_TLBI_VAE2IS: 1156 case OP_TLBI_VAE2OS: 1157 case OP_TLBI_VAE1: 1158 case OP_TLBI_VAE1IS: 1159 case OP_TLBI_VAE1OS: 1160 case OP_TLBI_VAE2NXS: 1161 case OP_TLBI_VAE2ISNXS: 1162 case OP_TLBI_VAE2OSNXS: 1163 case OP_TLBI_VAE1NXS: 1164 case OP_TLBI_VAE1ISNXS: 1165 case OP_TLBI_VAE1OSNXS: 1166 case OP_TLBI_VALE2: 1167 case OP_TLBI_VALE2IS: 1168 case OP_TLBI_VALE2OS: 1169 case OP_TLBI_VALE1: 1170 case OP_TLBI_VALE1IS: 1171 case OP_TLBI_VALE1OS: 1172 case OP_TLBI_VALE2NXS: 1173 case OP_TLBI_VALE2ISNXS: 1174 case OP_TLBI_VALE2OSNXS: 1175 case OP_TLBI_VALE1NXS: 1176 case OP_TLBI_VALE1ISNXS: 1177 case OP_TLBI_VALE1OSNXS: 1178 scope->type = TLBI_VA; 1179 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1180 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1181 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1182 break; 1183 case OP_TLBI_ASIDE1: 1184 case OP_TLBI_ASIDE1IS: 1185 case OP_TLBI_ASIDE1OS: 1186 case OP_TLBI_ASIDE1NXS: 1187 case OP_TLBI_ASIDE1ISNXS: 1188 case OP_TLBI_ASIDE1OSNXS: 1189 scope->type = TLBI_ASID; 1190 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1191 break; 1192 case OP_TLBI_VAAE1: 1193 case OP_TLBI_VAAE1IS: 1194 case OP_TLBI_VAAE1OS: 1195 case OP_TLBI_VAAE1NXS: 1196 case OP_TLBI_VAAE1ISNXS: 1197 case OP_TLBI_VAAE1OSNXS: 1198 case OP_TLBI_VAALE1: 1199 case OP_TLBI_VAALE1IS: 1200 case OP_TLBI_VAALE1OS: 1201 case OP_TLBI_VAALE1NXS: 1202 case OP_TLBI_VAALE1ISNXS: 1203 case OP_TLBI_VAALE1OSNXS: 1204 scope->type = TLBI_VAA; 1205 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1206 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1207 break; 1208 case OP_TLBI_RVAE2: 1209 case OP_TLBI_RVAE2IS: 1210 case OP_TLBI_RVAE2OS: 1211 case OP_TLBI_RVAE1: 1212 case OP_TLBI_RVAE1IS: 1213 case OP_TLBI_RVAE1OS: 1214 case OP_TLBI_RVAE2NXS: 1215 case OP_TLBI_RVAE2ISNXS: 1216 case OP_TLBI_RVAE2OSNXS: 1217 case OP_TLBI_RVAE1NXS: 1218 case OP_TLBI_RVAE1ISNXS: 1219 case OP_TLBI_RVAE1OSNXS: 1220 case OP_TLBI_RVALE2: 1221 case OP_TLBI_RVALE2IS: 1222 case OP_TLBI_RVALE2OS: 1223 case OP_TLBI_RVALE1: 1224 case OP_TLBI_RVALE1IS: 1225 case OP_TLBI_RVALE1OS: 1226 case OP_TLBI_RVALE2NXS: 1227 case OP_TLBI_RVALE2ISNXS: 1228 case OP_TLBI_RVALE2OSNXS: 1229 case OP_TLBI_RVALE1NXS: 1230 case OP_TLBI_RVALE1ISNXS: 1231 case OP_TLBI_RVALE1OSNXS: 1232 scope->type = TLBI_VA; 1233 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid); 1234 break; 1235 case OP_TLBI_RVAAE1: 1236 case OP_TLBI_RVAAE1IS: 1237 case OP_TLBI_RVAAE1OS: 1238 case OP_TLBI_RVAAE1NXS: 1239 case OP_TLBI_RVAAE1ISNXS: 1240 case OP_TLBI_RVAAE1OSNXS: 1241 case OP_TLBI_RVAALE1: 1242 case OP_TLBI_RVAALE1IS: 1243 case OP_TLBI_RVAALE1OS: 1244 case OP_TLBI_RVAALE1NXS: 1245 case OP_TLBI_RVAALE1ISNXS: 1246 case OP_TLBI_RVAALE1OSNXS: 1247 scope->type = TLBI_VAA; 1248 scope->va = decode_range_tlbi(val, &scope->size, NULL); 1249 break; 1250 } 1251 } 1252 1253 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) 1254 { 1255 struct s1e2_tlbi_scope scope = {}; 1256 1257 compute_s1_tlbi_range(vcpu, inst, val, &scope); 1258 1259 guard(write_lock)(&vcpu->kvm->mmu_lock); 1260 invalidate_vncr_va(vcpu->kvm, &scope); 1261 } 1262 1263 void kvm_nested_s2_wp(struct kvm *kvm) 1264 { 1265 int i; 1266 1267 lockdep_assert_held_write(&kvm->mmu_lock); 1268 1269 if (!kvm->arch.nested_mmus_size) 1270 return; 1271 1272 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1273 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1274 1275 if (kvm_s2_mmu_valid(mmu)) 1276 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 1277 } 1278 1279 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); 1280 } 1281 1282 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 1283 { 1284 int i; 1285 1286 lockdep_assert_held_write(&kvm->mmu_lock); 1287 1288 if (!kvm->arch.nested_mmus_size) 1289 return; 1290 1291 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1292 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1293 1294 if (kvm_s2_mmu_valid(mmu)) 1295 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 1296 } 1297 1298 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); 1299 } 1300 1301 void kvm_nested_s2_flush(struct kvm *kvm) 1302 { 1303 int i; 1304 1305 lockdep_assert_held_write(&kvm->mmu_lock); 1306 1307 if (!kvm->arch.nested_mmus_size) 1308 return; 1309 1310 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1311 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1312 1313 if (kvm_s2_mmu_valid(mmu)) 1314 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 1315 } 1316 } 1317 1318 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1319 { 1320 int i; 1321 1322 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1323 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1324 1325 if (!WARN_ON(atomic_read(&mmu->refcnt))) 1326 kvm_free_stage2_pgd(mmu); 1327 } 1328 kvfree(kvm->arch.nested_mmus); 1329 kvm->arch.nested_mmus = NULL; 1330 kvm->arch.nested_mmus_size = 0; 1331 kvm_uninit_stage2_mmu(kvm); 1332 } 1333 1334 /* 1335 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter: 1336 * 1337 * - We introduce an internal representation of a vcpu-private TLB, 1338 * representing the mapping between the guest VA contained in VNCR_EL2, 1339 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at. 1340 * 1341 * - On translation fault from a nested VNCR access, we create such a TLB. 1342 * If there is no mapping to describe, the guest inherits the fault. 1343 * Crucially, no actual mapping is done at this stage. 1344 * 1345 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above 1346 * TLB exists, we map it in the fixmap for this CPU, and run with it. We 1347 * have to respect the permissions dictated by the guest, but not the 1348 * memory type (FWB is a must). 1349 * 1350 * - Note that we usually don't do a vcpu_load() on the back of a fault 1351 * (unless we are preempted), so the resolution of a translation fault 1352 * must go via a request that will map the VNCR page in the fixmap. 1353 * vcpu_load() might as well use the same mechanism. 1354 * 1355 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was 1356 * mapped, we unmap it. Yes it is that simple. The TLB still exists 1357 * though, and may be reused at a later load. 1358 * 1359 * - On permission fault, we simply forward the fault to the guest's EL2. 1360 * Get out of my way. 1361 * 1362 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that 1363 * intersects with the TLBI request, invalidate it, and unmap the page 1364 * from the fixmap. Because we need to look at all the vcpu-private TLBs, 1365 * this requires some wide-ranging locking to ensure that nothing races 1366 * against it. This requires some refcounting to avoid the search when 1367 * no such TLB is present (see below). 1368 * 1369 * - On MMU notifiers, we must invalidate our TLB in a similar way, but 1370 * looking at the IPA instead. The funny part is that there may not be a 1371 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST 1372 * instructions. 1373 * 1374 * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't 1375 * the number of *mapped* L1 VNCR pages, which is likely be a subset (and 1376 * by definition, a TLBI handled from L1 runs with the canonical VNCR 1377 * page, not the L1's). The innermost trap handling code checks this to 1378 * find out whether to return to the guest ASAP (no L1 TLBs) or to visit 1379 * this part of the world for some extra invalidation work. 1380 */ 1381 1382 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) 1383 { 1384 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1385 return 0; 1386 1387 if (!vcpu->arch.vncr_tlb) { 1388 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb, 1389 GFP_KERNEL_ACCOUNT); 1390 1391 /* 1392 * Taking the lock on assignment ensures that the TLB is 1393 * seen as initialised when following the pointer (release 1394 * semantics of the unlock), and avoids having acquires on 1395 * each user which already take the lock. 1396 */ 1397 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 1398 vcpu->arch.vncr_tlb = vt; 1399 } 1400 1401 if (!vcpu->arch.vncr_tlb) 1402 return -ENOMEM; 1403 1404 return 0; 1405 } 1406 1407 static u64 read_vncr_el2(struct kvm_vcpu *vcpu) 1408 { 1409 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); 1410 } 1411 1412 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) 1413 { 1414 struct kvm_memory_slot *memslot; 1415 bool write_fault, writable; 1416 unsigned long mmu_seq; 1417 struct vncr_tlb *vt; 1418 struct page *page; 1419 u64 va, pfn, gfn; 1420 int ret; 1421 1422 vt = vcpu->arch.vncr_tlb; 1423 1424 /* 1425 * If we're about to walk the EL2 S1 PTs, we must invalidate the 1426 * current TLB, as it could be sampled from another vcpu doing a 1427 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single 1428 * translation, so not much of a choice. 1429 * 1430 * We also prepare the next walk wilst we're at it. 1431 */ 1432 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1433 this_cpu_reset_vncr_fixmap(vcpu); 1434 if (vt->valid) 1435 invalidate_vncr(vcpu->kvm, vt); 1436 1437 vt->wi = (struct s1_walk_info) { 1438 .regime = TR_EL20, 1439 .as_el0 = false, 1440 .pan = false, 1441 }; 1442 vt->wr = (struct s1_walk_result){}; 1443 } 1444 1445 guard(srcu)(&vcpu->kvm->srcu); 1446 1447 va = read_vncr_el2(vcpu); 1448 1449 mmu_seq = vcpu->kvm->mmu_invalidate_seq; 1450 smp_rmb(); 1451 1452 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); 1453 if (ret) 1454 return ret; 1455 1456 write_fault = kvm_is_write_fault(vcpu); 1457 1458 gfn = vt->wr.pa >> PAGE_SHIFT; 1459 memslot = gfn_to_memslot(vcpu->kvm, gfn); 1460 if (!memslot) { 1461 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1462 return -EFAULT; 1463 } 1464 1465 *is_gmem = kvm_slot_has_gmem(memslot); 1466 if (!*is_gmem) { 1467 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, 1468 &writable, &page); 1469 if (is_error_noslot_pfn(pfn)) { 1470 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1471 return -EFAULT; 1472 } 1473 } else { 1474 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); 1475 if (ret) { 1476 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, 1477 write_fault, false, false); 1478 return ret; 1479 } 1480 1481 writable = !(memslot->flags & KVM_MEM_READONLY); 1482 } 1483 1484 /* 1485 * FIXME: This check is too restrictive as KVM allows cacheable memory 1486 * attributes for PFNMAP VMAs that have cacheable attributes in host 1487 * stage-1. 1488 */ 1489 if (!pfn_is_map_memory(pfn)) { 1490 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1491 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1492 return -EINVAL; 1493 } 1494 1495 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1496 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { 1497 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1498 return -EAGAIN; 1499 } 1500 1501 vt->gva = va; 1502 vt->hpa = pfn << PAGE_SHIFT; 1503 vt->hpa_writable = writable; 1504 vt->valid = true; 1505 atomic_set(&vt->cpu, -1); 1506 1507 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 1508 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); 1509 } 1510 1511 if (vt->wr.pw && vt->hpa_writable) 1512 mark_page_dirty(vcpu->kvm, gfn); 1513 1514 return 0; 1515 } 1516 1517 static void handle_vncr_perm(struct kvm_vcpu *vcpu) 1518 { 1519 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1520 u64 esr = kvm_vcpu_get_esr(vcpu); 1521 u64 fsc; 1522 1523 /* 1524 * Promote to an external abort if the stage-1 permits writes but the 1525 * HPA is read-only (e.g. RO memslot). 1526 */ 1527 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) 1528 fsc = ESR_ELx_FSC_EXTABT; 1529 /* 1530 * Otherwise, inject a permission fault using the guest's translation 1531 * level rather than the host's. 1532 */ 1533 else 1534 fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); 1535 1536 esr &= ~ESR_ELx_FSC; 1537 esr |= FIELD_PREP(ESR_ELx_FSC, fsc); 1538 1539 kvm_inject_nested_sync(vcpu, esr); 1540 } 1541 1542 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) 1543 { 1544 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1545 u64 esr = kvm_vcpu_get_esr(vcpu); 1546 bool is_gmem = false; 1547 bool perm; 1548 int ret; 1549 1550 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); 1551 1552 if (kvm_vcpu_abt_issea(vcpu)) 1553 return kvm_handle_guest_sea(vcpu); 1554 1555 if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { 1556 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); 1557 return -EIO; 1558 } 1559 1560 /* 1561 * Speculatively increment the TLB count to make sure concurrent 1562 * TLBIs will take the slow path, and will interact with the retry 1563 * mechanism. Drop it again on error. 1564 */ 1565 atomic_inc(&vcpu->kvm->arch.vncr_tlb_count); 1566 smp_mb__after_atomic(); 1567 1568 ret = kvm_translate_vncr(vcpu, &is_gmem); 1569 if (ret) { 1570 smp_mb__before_atomic(); 1571 atomic_dec(&vcpu->kvm->arch.vncr_tlb_count); 1572 } 1573 1574 switch (ret) { 1575 case -EAGAIN: 1576 /* Let's try again... */ 1577 return 1; 1578 case -ENOMEM: 1579 /* 1580 * For guest_memfd, this indicates that it failed to 1581 * create a folio to back the memory. Inform userspace. 1582 */ 1583 if (is_gmem) 1584 return 0; 1585 /* Otherwise, let's try again... */ 1586 break; 1587 case -EFAULT: 1588 case -EIO: 1589 case -EHWPOISON: 1590 if (is_gmem) 1591 return 0; 1592 fallthrough; 1593 case -EINVAL: 1594 case -ENOENT: 1595 case -EACCES: 1596 /* 1597 * Translation failed, inject the corresponding 1598 * exception back to EL2. 1599 */ 1600 esr &= ~ESR_ELx_FSC; 1601 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); 1602 1603 kvm_inject_nested_sync(vcpu, esr); 1604 break; 1605 case 0: 1606 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; 1607 if (!perm) 1608 handle_vncr_perm(vcpu); 1609 break; 1610 } 1611 1612 return 1; 1613 } 1614 1615 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) 1616 { 1617 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1618 pgprot_t prot; 1619 1620 guard(preempt)(); 1621 guard(read_lock)(&vcpu->kvm->mmu_lock); 1622 1623 /* 1624 * The request to map VNCR may have raced against some other 1625 * event, such as an interrupt, and may not be valid anymore. 1626 */ 1627 if (is_hyp_ctxt(vcpu)) 1628 return; 1629 1630 /* 1631 * Check that the pseudo-TLB is valid and that VNCR_EL2 still 1632 * contains the expected value. If it doesn't, we simply bail out 1633 * without a mapping -- a transformed MSR/MRS will generate the 1634 * fault and allows us to populate the pseudo-TLB. 1635 */ 1636 if (!vt->valid) 1637 return; 1638 1639 /* We cache the MMU state in the TLB. Check that it matches. */ 1640 if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) 1641 return; 1642 1643 if (read_vncr_el2(vcpu) != vt->gva) 1644 return; 1645 1646 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) 1647 return; 1648 1649 if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) 1650 prot = PAGE_KERNEL; 1651 else if (vt->wr.pr) 1652 prot = PAGE_KERNEL_RO; 1653 else 1654 prot = PAGE_NONE; 1655 1656 /* 1657 * We can't map write-only (or no permission at all) in the kernel, 1658 * but the guest can do it if using POE, so we'll have to turn a 1659 * translation fault into a permission fault at runtime. 1660 * FIXME: WO doesn't work at all, need POE support in the kernel. 1661 */ 1662 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { 1663 atomic_set(&vt->cpu, smp_processor_id()); 1664 __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); 1665 host_data_set_flag(L1_VNCR_MAPPED); 1666 } 1667 } 1668 1669 /* 1670 * Our emulated CPU doesn't support all the possible features. For the 1671 * sake of simplicity (and probably mental sanity), wipe out a number 1672 * of feature bits we don't intend to support for the time being. 1673 * This list should get updated as new features get added to the NV 1674 * support, and new extension to the architecture. 1675 */ 1676 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) 1677 { 1678 u64 orig_val = val; 1679 1680 switch (reg) { 1681 case SYS_ID_AA64ISAR1_EL1: 1682 /* Support everything but LS64 and Spec Invalidation */ 1683 val &= ~(ID_AA64ISAR1_EL1_LS64 | 1684 ID_AA64ISAR1_EL1_SPECRES); 1685 break; 1686 1687 case SYS_ID_AA64PFR0_EL1: 1688 /* No RME, AMU, MPAM, or S-EL2 */ 1689 val &= ~(ID_AA64PFR0_EL1_RME | 1690 ID_AA64PFR0_EL1_AMU | 1691 ID_AA64PFR0_EL1_MPAM | 1692 ID_AA64PFR0_EL1_SEL2 | 1693 ID_AA64PFR0_EL1_EL3 | 1694 ID_AA64PFR0_EL1_EL2 | 1695 ID_AA64PFR0_EL1_EL1 | 1696 ID_AA64PFR0_EL1_EL0); 1697 /* 64bit only at any EL */ 1698 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP); 1699 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP); 1700 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP); 1701 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP); 1702 break; 1703 1704 case SYS_ID_AA64PFR1_EL1: 1705 /* Only support BTI, SSBS, CSV2_frac */ 1706 val &= ~(ID_AA64PFR1_EL1_PFAR | 1707 ID_AA64PFR1_EL1_MTEX | 1708 ID_AA64PFR1_EL1_THE | 1709 ID_AA64PFR1_EL1_GCS | 1710 ID_AA64PFR1_EL1_MTE_frac | 1711 ID_AA64PFR1_EL1_NMI | 1712 ID_AA64PFR1_EL1_SME | 1713 ID_AA64PFR1_EL1_RES0 | 1714 ID_AA64PFR1_EL1_MPAM_frac | 1715 ID_AA64PFR1_EL1_MTE); 1716 break; 1717 1718 case SYS_ID_AA64PFR2_EL1: 1719 /* GICv5 is not yet supported for NV */ 1720 val &= ~ID_AA64PFR2_EL1_GCIE; 1721 break; 1722 1723 case SYS_ID_AA64MMFR0_EL1: 1724 /* Hide ExS, Secure Memory */ 1725 val &= ~(ID_AA64MMFR0_EL1_EXS | 1726 ID_AA64MMFR0_EL1_TGRAN4_2 | 1727 ID_AA64MMFR0_EL1_TGRAN16_2 | 1728 ID_AA64MMFR0_EL1_TGRAN64_2 | 1729 ID_AA64MMFR0_EL1_SNSMEM); 1730 1731 /* Hide CNTPOFF if present */ 1732 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP); 1733 1734 /* Disallow unsupported S2 page sizes */ 1735 switch (PAGE_SIZE) { 1736 case SZ_64K: 1737 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI); 1738 fallthrough; 1739 case SZ_16K: 1740 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI); 1741 fallthrough; 1742 case SZ_4K: 1743 /* Support everything */ 1744 break; 1745 } 1746 1747 /* 1748 * Since we can't support a guest S2 page size smaller 1749 * than the host's own page size (due to KVM only 1750 * populating its own S2 using the kernel's page 1751 * size), advertise the limitation using FEAT_GTG. 1752 */ 1753 switch (PAGE_SIZE) { 1754 case SZ_4K: 1755 if (_has_tgran_2(orig_val, 4)) 1756 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP); 1757 fallthrough; 1758 case SZ_16K: 1759 if (_has_tgran_2(orig_val, 16)) 1760 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP); 1761 fallthrough; 1762 case SZ_64K: 1763 if (_has_tgran_2(orig_val, 64)) 1764 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP); 1765 break; 1766 } 1767 1768 /* Cap PARange to 48bits */ 1769 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48); 1770 break; 1771 1772 case SYS_ID_AA64MMFR1_EL1: 1773 val &= ~(ID_AA64MMFR1_EL1_CMOW | 1774 ID_AA64MMFR1_EL1_nTLBPA | 1775 ID_AA64MMFR1_EL1_ETS); 1776 1777 /* FEAT_E2H0 implies no VHE */ 1778 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) 1779 val &= ~ID_AA64MMFR1_EL1_VH; 1780 1781 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF); 1782 break; 1783 1784 case SYS_ID_AA64MMFR2_EL1: 1785 val &= ~(ID_AA64MMFR2_EL1_BBM | 1786 ID_AA64MMFR2_EL1_TTL | 1787 GENMASK_ULL(47, 44) | 1788 ID_AA64MMFR2_EL1_ST | 1789 ID_AA64MMFR2_EL1_CCIDX | 1790 ID_AA64MMFR2_EL1_VARange); 1791 1792 /* Force TTL support */ 1793 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP); 1794 break; 1795 1796 case SYS_ID_AA64MMFR4_EL1: 1797 /* 1798 * You get EITHER 1799 * 1800 * - FEAT_VHE without FEAT_E2H0 1801 * - FEAT_NV limited to FEAT_NV2(p1)/NV3 1802 * - HCR_EL2.NV1 being RES0 1803 * 1804 * OR 1805 * 1806 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV 1807 * 1808 * Life is too short for anything else. 1809 */ 1810 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { 1811 val = 0; 1812 } else { 1813 val &= ID_AA64MMFR4_EL1_NV_frac; 1814 if (cpus_have_final_cap(ARM64_HAS_NV3)) 1815 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3); 1816 else if (cpus_have_final_cap(ARM64_HAS_NV2P1)) 1817 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1); 1818 else 1819 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); 1820 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); 1821 } 1822 break; 1823 1824 case SYS_ID_AA64DFR0_EL1: 1825 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 1826 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff | 1827 ID_AA64DFR0_EL1_BRBE | 1828 ID_AA64DFR0_EL1_MTPMU | 1829 ID_AA64DFR0_EL1_TraceBuffer | 1830 ID_AA64DFR0_EL1_TraceFilt | 1831 ID_AA64DFR0_EL1_PMSVer | 1832 ID_AA64DFR0_EL1_CTX_CMPs | 1833 ID_AA64DFR0_EL1_SEBEP | 1834 ID_AA64DFR0_EL1_PMSS | 1835 ID_AA64DFR0_EL1_TraceVer); 1836 1837 /* 1838 * FEAT_Debugv8p9 requires support for extended breakpoints / 1839 * watchpoints. 1840 */ 1841 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8); 1842 break; 1843 } 1844 1845 return val; 1846 } 1847 1848 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu, 1849 enum vcpu_sysreg sr, u64 v) 1850 { 1851 struct resx resx; 1852 1853 resx = kvm_get_sysreg_resx(vcpu->kvm, sr); 1854 v &= ~resx.res0; 1855 v |= resx.res1; 1856 1857 return v; 1858 } 1859 1860 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx) 1861 { 1862 BUILD_BUG_ON(!__builtin_constant_p(sr)); 1863 BUILD_BUG_ON(sr < __SANITISED_REG_START__); 1864 BUILD_BUG_ON(sr >= NR_SYS_REGS); 1865 1866 kvm_set_sysreg_resx(kvm, sr, resx); 1867 } 1868 1869 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) 1870 { 1871 struct kvm *kvm = vcpu->kvm; 1872 struct resx resx; 1873 1874 lockdep_assert_held(&kvm->arch.config_lock); 1875 1876 if (kvm->arch.sysreg_masks) 1877 goto out; 1878 1879 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks), 1880 GFP_KERNEL_ACCOUNT); 1881 if (!kvm->arch.sysreg_masks) 1882 return -ENOMEM; 1883 1884 /* VTTBR_EL2 */ 1885 resx = (typeof(resx)){}; 1886 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 1887 resx.res0 |= GENMASK(63, 56); 1888 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 1889 resx.res0 |= VTTBR_CNP_BIT; 1890 set_sysreg_masks(kvm, VTTBR_EL2, resx); 1891 1892 /* VTCR_EL2 */ 1893 resx = get_reg_fixed_bits(kvm, VTCR_EL2); 1894 set_sysreg_masks(kvm, VTCR_EL2, resx); 1895 1896 /* VMPIDR_EL2 */ 1897 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24); 1898 resx.res1 = BIT(31); 1899 set_sysreg_masks(kvm, VMPIDR_EL2, resx); 1900 1901 /* HCR_EL2 */ 1902 resx = get_reg_fixed_bits(kvm, HCR_EL2); 1903 set_sysreg_masks(kvm, HCR_EL2, resx); 1904 1905 /* NVHCR_EL2 */ 1906 resx = get_reg_fixed_bits(kvm, NVHCR_EL2); 1907 set_sysreg_masks(kvm, NVHCR_EL2, resx); 1908 1909 /* HCRX_EL2 */ 1910 resx = get_reg_fixed_bits(kvm, HCRX_EL2); 1911 set_sysreg_masks(kvm, HCRX_EL2, resx); 1912 1913 /* HFG[RW]TR_EL2 */ 1914 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2); 1915 set_sysreg_masks(kvm, HFGRTR_EL2, resx); 1916 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2); 1917 set_sysreg_masks(kvm, HFGWTR_EL2, resx); 1918 1919 /* HDFG[RW]TR_EL2 */ 1920 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2); 1921 set_sysreg_masks(kvm, HDFGRTR_EL2, resx); 1922 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2); 1923 set_sysreg_masks(kvm, HDFGWTR_EL2, resx); 1924 1925 /* HFGITR_EL2 */ 1926 resx = get_reg_fixed_bits(kvm, HFGITR_EL2); 1927 set_sysreg_masks(kvm, HFGITR_EL2, resx); 1928 1929 /* HAFGRTR_EL2 - not a lot to see here */ 1930 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2); 1931 set_sysreg_masks(kvm, HAFGRTR_EL2, resx); 1932 1933 /* HFG[RW]TR2_EL2 */ 1934 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2); 1935 set_sysreg_masks(kvm, HFGRTR2_EL2, resx); 1936 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2); 1937 set_sysreg_masks(kvm, HFGWTR2_EL2, resx); 1938 1939 /* HDFG[RW]TR2_EL2 */ 1940 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2); 1941 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx); 1942 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2); 1943 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx); 1944 1945 /* HFGITR2_EL2 */ 1946 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2); 1947 set_sysreg_masks(kvm, HFGITR2_EL2, resx); 1948 1949 /* TCR2_EL2 */ 1950 resx = get_reg_fixed_bits(kvm, TCR2_EL2); 1951 set_sysreg_masks(kvm, TCR2_EL2, resx); 1952 1953 /* SCTLR_EL1 */ 1954 resx = get_reg_fixed_bits(kvm, SCTLR_EL1); 1955 set_sysreg_masks(kvm, SCTLR_EL1, resx); 1956 1957 /* SCTLR_EL2 */ 1958 resx = get_reg_fixed_bits(kvm, SCTLR_EL2); 1959 set_sysreg_masks(kvm, SCTLR_EL2, resx); 1960 1961 /* SCTLR2_ELx */ 1962 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1); 1963 set_sysreg_masks(kvm, SCTLR2_EL1, resx); 1964 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2); 1965 set_sysreg_masks(kvm, SCTLR2_EL2, resx); 1966 1967 /* MDCR_EL2 */ 1968 resx = get_reg_fixed_bits(kvm, MDCR_EL2); 1969 set_sysreg_masks(kvm, MDCR_EL2, resx); 1970 1971 /* CNTHCTL_EL2 */ 1972 resx.res0 = GENMASK(63, 20); 1973 resx.res1 = 0; 1974 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP)) 1975 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK; 1976 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) { 1977 resx.res0 |= CNTHCTL_ECV; 1978 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP)) 1979 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT | 1980 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT); 1981 } 1982 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1983 resx.res0 |= GENMASK(11, 8); 1984 set_sysreg_masks(kvm, CNTHCTL_EL2, resx); 1985 1986 /* ICH_HCR_EL2 */ 1987 resx.res0 = ICH_HCR_EL2_RES0; 1988 resx.res1 = ICH_HCR_EL2_RES1; 1989 if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS)) 1990 resx.res0 |= ICH_HCR_EL2_TDIR; 1991 /* No GICv4 is presented to the guest */ 1992 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; 1993 set_sysreg_masks(kvm, ICH_HCR_EL2, resx); 1994 1995 /* VNCR_EL2 */ 1996 resx.res0 = VNCR_EL2_RES0; 1997 resx.res1 = VNCR_EL2_RES1; 1998 set_sysreg_masks(kvm, VNCR_EL2, resx); 1999 2000 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */ 2001 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4); 2002 resx.res1 = ZCR_ELx_RES1; 2003 set_sysreg_masks(kvm, ZCR_EL2, resx); 2004 2005 out: 2006 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) 2007 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0); 2008 2009 return 0; 2010 } 2011 2012 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 2013 { 2014 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 2015 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 2016 2017 write_lock(&vcpu->kvm->mmu_lock); 2018 if (mmu->pending_unmap) { 2019 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 2020 mmu->pending_unmap = false; 2021 } 2022 write_unlock(&vcpu->kvm->mmu_lock); 2023 } 2024 2025 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu)) 2026 kvm_map_l1_vncr(vcpu); 2027 2028 /* Must be last, as may switch context! */ 2029 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) 2030 kvm_inject_nested_irq(vcpu); 2031 } 2032 2033 /* 2034 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor 2035 * can write to HCR_EL2 behind our back, potentially changing the exception 2036 * routing / masking for even the host context. 2037 * 2038 * What follows is some slop to (1) react to exception routing / masking and (2) 2039 * preserve the pending SError state across translation regimes. 2040 */ 2041 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) 2042 { 2043 if (!vcpu_has_nv(vcpu)) 2044 return; 2045 2046 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 2047 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 2048 } 2049 2050 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) 2051 { 2052 unsigned long *hcr = vcpu_hcr(vcpu); 2053 2054 if (!vcpu_has_nv(vcpu)) 2055 return; 2056 2057 /* 2058 * We previously decided that an SError was deliverable to the guest. 2059 * Reap the pending state from HCR_EL2 and... 2060 */ 2061 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr))) 2062 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); 2063 2064 /* 2065 * Re-attempt SError injection in case the deliverability has changed, 2066 * which is necessary to faithfully emulate WFI the case of a pending 2067 * SError being a wakeup condition. 2068 */ 2069 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 2070 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 2071 } 2072 2073 /* 2074 * KVM unconditionally sets most of these traps anyway but use an allowlist 2075 * to document the guest hypervisor traps that may take precedence and guard 2076 * against future changes to the non-nested trap configuration. 2077 */ 2078 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \ 2079 MDCR_EL2_TDA | \ 2080 MDCR_EL2_TDRA | \ 2081 MDCR_EL2_TTRF | \ 2082 MDCR_EL2_TPMS | \ 2083 MDCR_EL2_TPM | \ 2084 MDCR_EL2_TPMCR | \ 2085 MDCR_EL2_TDCC | \ 2086 MDCR_EL2_TDOSA) 2087 2088 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) 2089 { 2090 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 2091 2092 if (is_nested_ctxt(vcpu)) 2093 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE); 2094 /* 2095 * In yet another example where FEAT_NV2 is fscking broken, accesses 2096 * to MDSCR_EL1 are redirected to the VNCR despite having an effect 2097 * at EL2. Use a big hammer to apply sanity. 2098 * 2099 * Unless of course we have FEAT_FGT, in which case we can precisely 2100 * trap MDSCR_EL1. 2101 */ 2102 else if (!cpus_have_final_cap(ARM64_HAS_FGT)) 2103 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; 2104 } 2105