1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/interval_tree.h> 9 #include <linux/kvm.h> 10 #include <linux/kvm_host.h> 11 12 #include <asm/fixmap.h> 13 #include <asm/kvm_arm.h> 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_mmu.h> 16 #include <asm/kvm_nested.h> 17 #include <asm/sysreg.h> 18 19 #include "sys_regs.h" 20 #include "vgic/vgic.h" 21 22 struct vncr_tlb { 23 /* The guest's VNCR_EL2 */ 24 u64 gva; 25 struct s1_walk_info wi; 26 struct s1_walk_result wr; 27 28 u64 hpa; 29 bool hpa_writable; 30 31 /* -1 when not mapped on a CPU */ 32 atomic_t cpu; 33 34 /* 35 * true if the TLB is valid. Can only be changed with the 36 * mmu_lock held. 37 */ 38 bool valid; 39 }; 40 41 /* 42 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 43 * memory usage and potential number of different sets of S2 PTs in 44 * the guests. Running out of S2 MMUs only affects performance (we 45 * will invalidate them more often). 46 */ 47 #define S2_MMU_PER_VCPU 2 48 #define S2_MMU_PER_VM (KVM_MAX_VCPUS * S2_MMU_PER_VCPU) 49 50 int kvm_init_nested(struct kvm *kvm) 51 { 52 kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, 53 S2_MMU_PER_VM, 54 GFP_KERNEL_ACCOUNT); 55 kvm->arch.nested_mmus_size = 0; 56 atomic_set(&kvm->arch.vncr_tlb_count, 0); 57 spin_lock_init(&kvm->arch.guest_s2_tracking_lock); 58 59 return kvm->arch.nested_mmus ? 0 : -ENOMEM; 60 } 61 62 void kvm_destroy_nested(struct kvm *kvm) 63 { 64 for (int i = 0; i < kvm->arch.nested_mmus_size; i+= S2_MMU_PER_VCPU) 65 kvfree(kvm->arch.nested_mmus[i]); 66 67 kvm->arch.nested_mmus_size = 0; 68 kvfree(kvm->arch.nested_mmus); 69 } 70 71 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 72 { 73 /* 74 * We only initialise the IPA range on the canonical MMU, which 75 * defines the contract between KVM and userspace on where the 76 * "hardware" is in the IPA space. This affects the validity of MMIO 77 * exits forwarded to userspace, for example. 78 * 79 * For nested S2s, we use the PARange as exposed to the guest, as it 80 * is allowed to use it at will to expose whatever memory map it 81 * wants to its own guests as it would be on real HW. 82 */ 83 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 84 } 85 86 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 87 { 88 struct kvm *kvm = vcpu->kvm; 89 int num_mmus; 90 91 lockdep_assert_held(&kvm->arch.config_lock); 92 93 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && 94 !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 95 return -EINVAL; 96 97 if (!vcpu->arch.ctxt.vncr_array) 98 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT | 99 __GFP_ZERO); 100 101 if (!vcpu->arch.ctxt.vncr_array) 102 return -ENOMEM; 103 104 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 105 106 if (num_mmus > kvm->arch.nested_mmus_size) { 107 struct kvm_s2_mmu *tmp; 108 int i, ret = 0; 109 110 tmp = kvzalloc_objs(*tmp, S2_MMU_PER_VCPU, GFP_KERNEL_ACCOUNT); 111 if (!tmp) 112 ret = -ENOMEM; 113 114 for (i = 0; !ret && i < S2_MMU_PER_VCPU; i++) { 115 ret = init_nested_s2_mmu(kvm, &tmp[i]); 116 if (ret) 117 break; 118 } 119 120 if (ret) { 121 while (--i >= 0) 122 kvm_free_stage2_pgd(&tmp[i]); 123 124 kvfree(tmp); 125 free_page((unsigned long)vcpu->arch.ctxt.vncr_array); 126 vcpu->arch.ctxt.vncr_array = NULL; 127 return ret; 128 } 129 130 for (i = 0; i < S2_MMU_PER_VCPU; i++) 131 kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size); 132 133 guard(write_lock)(&kvm->mmu_lock); 134 135 for (i = 0; i < S2_MMU_PER_VCPU; i++) { 136 tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size; 137 kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; 138 } 139 140 kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; 141 } 142 143 return 0; 144 } 145 146 struct s2_walk_info { 147 u64 baddr; 148 unsigned int max_oa_bits; 149 unsigned int pgshift; 150 unsigned int sl; 151 unsigned int t0sz; 152 bool be; 153 bool ha; 154 }; 155 156 static u32 compute_fsc(int level, u32 fsc) 157 { 158 return fsc | (level & 0x3); 159 } 160 161 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 162 { 163 u32 esr; 164 165 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 166 esr |= compute_fsc(level, fsc); 167 return esr; 168 } 169 170 static int get_ia_size(struct s2_walk_info *wi) 171 { 172 return 64 - wi->t0sz; 173 } 174 175 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi, 176 int level, int input_size, int stride) 177 { 178 int start_size, pa_max; 179 180 pa_max = kvm_get_pa_bits(vcpu->kvm); 181 182 /* Check translation limits */ 183 switch (BIT(wi->pgshift)) { 184 case SZ_64K: 185 if (level == 0 || (level == 1 && pa_max <= 42)) 186 return -EFAULT; 187 break; 188 case SZ_16K: 189 if (level == 0 || (level == 1 && pa_max <= 40)) 190 return -EFAULT; 191 break; 192 case SZ_4K: 193 if (level < 0 || (level == 0 && pa_max <= 42)) 194 return -EFAULT; 195 break; 196 } 197 198 /* Check input size limits */ 199 if (input_size > pa_max) 200 return -EFAULT; 201 202 /* Check number of entries in starting level table */ 203 start_size = input_size - ((3 - level) * stride + wi->pgshift); 204 if (start_size < 1 || start_size > stride + 4) 205 return -EFAULT; 206 207 return 0; 208 } 209 210 /* Check if output is within boundaries */ 211 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 212 { 213 unsigned int output_size = wi->max_oa_bits; 214 215 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 216 return -1; 217 218 return 0; 219 } 220 221 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc, 222 struct s2_walk_info *wi) 223 { 224 u64 val; 225 int r; 226 227 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val)); 228 if (r) 229 return r; 230 231 /* 232 * Handle reversedescriptors if endianness differs between the 233 * host and the guest hypervisor. 234 */ 235 if (wi->be) 236 *desc = be64_to_cpu((__force __be64)val); 237 else 238 *desc = le64_to_cpu((__force __le64)val); 239 240 return 0; 241 } 242 243 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new, 244 struct s2_walk_info *wi) 245 { 246 if (wi->be) { 247 old = (__force u64)cpu_to_be64(old); 248 new = (__force u64)cpu_to_be64(new); 249 } else { 250 old = (__force u64)cpu_to_le64(old); 251 new = (__force u64)cpu_to_le64(new); 252 } 253 254 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new); 255 } 256 257 /* 258 * This is essentially a C-version of the pseudo code from the ARM ARM 259 * AArch64.TranslationTableWalk function. I strongly recommend looking at 260 * that pseudocode in trying to understand this. 261 * 262 * Must be called with the kvm->srcu read lock held 263 */ 264 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa, 265 struct s2_walk_info *wi, struct kvm_s2_trans *out) 266 { 267 int first_block_level, level, stride, input_size, base_lower_bound; 268 phys_addr_t base_addr; 269 unsigned int addr_top, addr_bottom; 270 u64 desc, new_desc; /* page table entry */ 271 int ret; 272 phys_addr_t paddr; 273 274 switch (BIT(wi->pgshift)) { 275 default: 276 case SZ_64K: 277 case SZ_16K: 278 level = 3 - wi->sl; 279 first_block_level = 2; 280 break; 281 case SZ_4K: 282 level = 2 - wi->sl; 283 first_block_level = 1; 284 break; 285 } 286 287 stride = wi->pgshift - 3; 288 input_size = get_ia_size(wi); 289 if (input_size > 48 || input_size < 25) 290 return -EFAULT; 291 292 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride); 293 if (WARN_ON(ret)) { 294 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT); 295 return ret; 296 } 297 298 base_lower_bound = 3 + input_size - ((3 - level) * stride + 299 wi->pgshift); 300 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 301 302 if (check_output_size(wi, base_addr)) { 303 /* R_BFHQH */ 304 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ); 305 return 1; 306 } 307 308 addr_top = input_size - 1; 309 310 while (1) { 311 phys_addr_t index; 312 313 addr_bottom = (3 - level) * stride + wi->pgshift; 314 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 315 >> (addr_bottom - 3); 316 317 paddr = base_addr | index; 318 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi); 319 if (ret < 0) { 320 out->esr = ESR_ELx_FSC_SEA_TTW(level); 321 return ret; 322 } 323 324 new_desc = desc; 325 326 /* Check for valid descriptor at this point */ 327 if (!(desc & KVM_PTE_VALID)) { 328 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 329 out->desc = desc; 330 return 1; 331 } 332 333 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) { 334 if (level < 3) 335 break; 336 337 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 338 out->desc = desc; 339 return 1; 340 } 341 342 /* We're at the final level */ 343 if (level == 3) 344 break; 345 346 if (check_output_size(wi, desc)) { 347 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 348 out->desc = desc; 349 return 1; 350 } 351 352 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 353 354 level += 1; 355 addr_top = addr_bottom - 1; 356 } 357 358 if (level < first_block_level) { 359 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 360 out->desc = desc; 361 return 1; 362 } 363 364 if (check_output_size(wi, desc)) { 365 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 366 out->desc = desc; 367 return 1; 368 } 369 370 if (wi->ha) 371 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF; 372 373 if (new_desc != desc) { 374 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi); 375 if (ret == -EAGAIN) 376 return ret; 377 if (ret) { 378 out->esr = ESR_ELx_FSC_SEA_TTW(level); 379 out->desc = desc; 380 return 1; 381 } 382 383 desc = new_desc; 384 } 385 386 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) { 387 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 388 out->desc = desc; 389 return 1; 390 } 391 392 addr_bottom += contiguous_bit_shift(desc, wi, level); 393 394 /* Calculate and return the result */ 395 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 396 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 397 out->output = paddr; 398 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 399 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R; 400 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; 401 out->level = level; 402 out->desc = desc; 403 return 0; 404 } 405 406 #define _has_tgran_2(__r, __sz) \ 407 ({ \ 408 u64 _s1, _s2, _mmfr0 = __r; \ 409 \ 410 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 411 TGRAN##__sz##_2, _mmfr0); \ 412 \ 413 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 414 TGRAN##__sz, _mmfr0); \ 415 \ 416 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \ 417 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \ 418 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \ 419 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \ 420 }) 421 422 static bool has_tgran_2(u64 mmfr0, unsigned int shift) 423 { 424 switch (shift) { 425 case 12: 426 return _has_tgran_2(mmfr0, 4); 427 case 14: 428 return _has_tgran_2(mmfr0, 16); 429 case 16: 430 return _has_tgran_2(mmfr0, 64); 431 default: 432 BUG(); 433 } 434 } 435 436 static unsigned int fallback_tgran2_shift(u64 mmfr0) 437 { 438 if (has_tgran_2(mmfr0, PAGE_SHIFT)) 439 return PAGE_SHIFT; 440 else if (has_tgran_2(mmfr0, 12)) 441 return 12; 442 else if (has_tgran_2(mmfr0, 14)) 443 return 14; 444 else if (has_tgran_2(mmfr0, 16)) 445 return 16; 446 else 447 return PAGE_SHIFT; 448 } 449 450 static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr) 451 { 452 u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr); 453 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 454 unsigned int shift; 455 456 switch (tg0) { 457 case VTCR_EL2_TG0_4K: 458 shift = 12; 459 break; 460 case VTCR_EL2_TG0_16K: 461 shift = 14; 462 break; 463 case VTCR_EL2_TG0_64K: 464 /* IMPDEF: treat any other value as 64k, subject to fallback */ 465 default: 466 shift = 16; 467 } 468 469 /* 470 * If TGx is programmed to an unimplemented value (not advertised in 471 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is 472 * written, as per the architecture. Choose an available one while 473 * prioritizing PAGE_SIZE. 474 */ 475 if (!has_tgran_2(mmfr0, shift)) 476 return fallback_tgran2_shift(mmfr0); 477 478 return shift; 479 } 480 481 static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr) 482 { 483 return BIT(vtcr_to_tg0_pgshift(kvm, vtcr)); 484 } 485 486 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi) 487 { 488 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 489 490 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 491 wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK; 492 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr); 493 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 494 /* Global limit for now, should eventually be per-VM */ 495 wi->max_oa_bits = min(get_kvm_ipa_limit(), 496 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false)); 497 wi->ha = vtcr & VTCR_EL2_HA; 498 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 499 } 500 501 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 502 struct kvm_s2_trans *result) 503 { 504 struct s2_walk_info wi; 505 int ret; 506 507 result->esr = 0; 508 509 if (!vcpu_has_nv(vcpu)) 510 return 0; 511 512 setup_s2_walk(vcpu, &wi); 513 514 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result); 515 if (ret) 516 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 517 518 return ret; 519 } 520 521 static unsigned int __ttl_to_size(u8 ttl) 522 { 523 int level = ttl & 3; 524 int gran = (ttl >> 2) & 3; 525 unsigned int max_size = 0; 526 527 switch (gran) { 528 case TLBI_TTL_TG_4K: 529 switch (level) { 530 case 0: 531 break; 532 case 1: 533 max_size = SZ_1G; 534 break; 535 case 2: 536 max_size = SZ_2M; 537 break; 538 case 3: 539 max_size = SZ_4K; 540 break; 541 } 542 break; 543 case TLBI_TTL_TG_16K: 544 switch (level) { 545 case 0: 546 case 1: 547 break; 548 case 2: 549 max_size = SZ_32M; 550 break; 551 case 3: 552 max_size = SZ_16K; 553 break; 554 } 555 break; 556 case TLBI_TTL_TG_64K: 557 switch (level) { 558 case 0: 559 case 1: 560 /* No 52bit IPA support */ 561 break; 562 case 2: 563 max_size = SZ_512M; 564 break; 565 case 3: 566 max_size = SZ_64K; 567 break; 568 } 569 break; 570 default: /* No size information */ 571 break; 572 } 573 574 return max_size; 575 } 576 577 static unsigned int ttl_to_size(u8 ttl) 578 { 579 return __ttl_to_size(ttl) ?: SZ_1G; 580 } 581 582 static u8 pgshift_level_to_ttl(u16 shift, s8 level) 583 { 584 u8 ttl; 585 586 /* 587 * If we don't have a proper level, fallback to the maximum 588 * size. 589 */ 590 if (level < 0) 591 return 0; 592 593 switch(shift) { 594 case 12: 595 ttl = TLBI_TTL_TG_4K; 596 break; 597 case 14: 598 ttl = TLBI_TTL_TG_16K; 599 break; 600 case 16: 601 ttl = TLBI_TTL_TG_64K; 602 break; 603 default: 604 BUG(); 605 } 606 607 ttl <<= 2; 608 ttl |= level & 3; 609 610 return ttl; 611 } 612 613 /* 614 * Compute the equivalent of the TTL field by parsing the shadow PT. The 615 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 616 * retrieved from first entry carrying the level as a tag. 617 */ 618 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 619 { 620 size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr); 621 u64 tmp, sz = 0; 622 kvm_pte_t pte; 623 u8 ttl, level; 624 625 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 626 627 switch (tg0_size) { 628 case SZ_4K: 629 ttl = (TLBI_TTL_TG_4K << 2); 630 break; 631 case SZ_16K: 632 ttl = (TLBI_TTL_TG_16K << 2); 633 break; 634 case SZ_64K: 635 default: /* IMPDEF: treat any other value as 64k */ 636 ttl = (TLBI_TTL_TG_64K << 2); 637 break; 638 } 639 640 tmp = addr; 641 642 again: 643 /* Iteratively compute the block sizes for a particular granule size */ 644 switch (tg0_size) { 645 case SZ_4K: 646 if (sz < SZ_4K) sz = SZ_4K; 647 else if (sz < SZ_2M) sz = SZ_2M; 648 else if (sz < SZ_1G) sz = SZ_1G; 649 else sz = 0; 650 break; 651 case SZ_16K: 652 if (sz < SZ_16K) sz = SZ_16K; 653 else if (sz < SZ_32M) sz = SZ_32M; 654 else sz = 0; 655 break; 656 case SZ_64K: 657 default: /* IMPDEF: treat any other value as 64k */ 658 if (sz < SZ_64K) sz = SZ_64K; 659 else if (sz < SZ_512M) sz = SZ_512M; 660 else sz = 0; 661 break; 662 } 663 664 if (sz == 0) 665 return 0; 666 667 tmp &= ~(sz - 1); 668 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0)) 669 goto again; 670 if (!(pte & PTE_VALID)) 671 goto again; 672 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 673 if (!level) 674 goto again; 675 676 ttl |= level; 677 678 /* 679 * We now have found some level information in the shadow S2. Check 680 * that the resulting range is actually including the original IPA. 681 */ 682 sz = ttl_to_size(ttl); 683 if (addr < (tmp + sz)) 684 return ttl; 685 686 return 0; 687 } 688 689 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 690 { 691 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 692 unsigned long max_size; 693 u8 ttl; 694 695 ttl = FIELD_GET(TLBI_TTL_MASK, val); 696 697 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 698 /* No TTL, check the shadow S2 for a hint */ 699 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 700 ttl = get_guest_mapping_ttl(mmu, addr); 701 } 702 703 /* 704 * Don't use the default 1GB fallback, as we can adapt to the 705 * max mapping size we allow at S2. 706 */ 707 max_size = __ttl_to_size(ttl); 708 709 if (!max_size) { 710 /* Compute the maximum extent of the invalidation */ 711 switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) { 712 case SZ_4K: 713 max_size = SZ_1G; 714 break; 715 case SZ_16K: 716 max_size = SZ_32M; 717 break; 718 case SZ_64K: 719 default: /* IMPDEF: treat any other value as 64k */ 720 /* 721 * No, we do not support 52bit IPA in nested yet. Once 722 * we do, this should be 4TB. 723 */ 724 max_size = SZ_512M; 725 break; 726 } 727 } 728 729 WARN_ON(!max_size); 730 return max_size; 731 } 732 733 /* 734 * We can have multiple *different* MMU contexts with the same VMID: 735 * 736 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 737 * 738 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 739 * VBBTR_EL2.BADDR address 740 * 741 * - A combination of the above... 742 * 743 * We can always identify which MMU context to pick at run-time. However, 744 * TLB invalidation involving a VMID must take action on all the TLBs using 745 * this particular VMID. This translates into applying the same invalidation 746 * operation to all the contexts that are using this VMID. Moar phun! 747 */ 748 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 749 const union tlbi_info *info, 750 void (*tlbi_callback)(struct kvm_s2_mmu *, 751 const union tlbi_info *)) 752 { 753 write_lock(&kvm->mmu_lock); 754 755 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 756 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 757 758 if (!kvm_s2_mmu_valid(mmu)) 759 continue; 760 761 if (vmid == get_vmid(mmu->tlb_vttbr)) 762 tlbi_callback(mmu, info); 763 } 764 765 write_unlock(&kvm->mmu_lock); 766 } 767 768 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 769 { 770 struct kvm *kvm = vcpu->kvm; 771 bool nested_stage2_enabled; 772 u64 vttbr, vtcr, hcr; 773 774 lockdep_assert_held_write(&kvm->mmu_lock); 775 776 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 777 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 778 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 779 780 nested_stage2_enabled = hcr & HCR_VM; 781 782 /* Don't consider the CnP bit for the vttbr match */ 783 vttbr &= ~VTTBR_CNP_BIT; 784 785 /* 786 * Two possibilities when looking up a S2 MMU context: 787 * 788 * - either S2 is enabled in the guest, and we need a context that is 789 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 790 * which makes it safe from a TLB conflict perspective (a broken 791 * guest won't be able to generate them), 792 * 793 * - or S2 is disabled, and we need a context that is S2-disabled 794 * and matches the VMID only, as all TLBs are tagged by VMID even 795 * if S2 translation is disabled. 796 */ 797 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 798 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 799 800 if (!kvm_s2_mmu_valid(mmu)) 801 continue; 802 803 if (nested_stage2_enabled && 804 mmu->nested_stage2_enabled && 805 vttbr == mmu->tlb_vttbr && 806 vtcr == mmu->tlb_vtcr) 807 return mmu; 808 809 if (!nested_stage2_enabled && 810 !mmu->nested_stage2_enabled && 811 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 812 return mmu; 813 } 814 return NULL; 815 } 816 817 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 818 { 819 struct kvm *kvm = vcpu->kvm; 820 struct kvm_s2_mmu *s2_mmu; 821 int i; 822 823 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 824 825 s2_mmu = lookup_s2_mmu(vcpu); 826 if (s2_mmu) 827 goto out; 828 829 /* 830 * Make sure we don't always search from the same point, or we 831 * will always reuse a potentially active context, leaving 832 * free contexts unused. 833 */ 834 for (i = kvm->arch.nested_mmus_next; 835 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 836 i++) { 837 s2_mmu = kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 838 839 if (atomic_read(&s2_mmu->refcnt) == 0) 840 break; 841 } 842 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 843 844 /* Set the scene for the next search */ 845 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 846 847 /* Make sure we don't forget to do the laundry */ 848 if (kvm_s2_mmu_valid(s2_mmu)) 849 s2_mmu->pending_unmap = true; 850 851 /* 852 * The virtual VMID (modulo CnP) will be used as a key when matching 853 * an existing kvm_s2_mmu. 854 * 855 * We cache VTCR at allocation time, once and for all. It'd be great 856 * if the guest didn't screw that one up, as this is not very 857 * forgiving... 858 */ 859 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 860 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 861 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 862 863 out: 864 atomic_inc(&s2_mmu->refcnt); 865 866 /* 867 * Set the vCPU request to perform an unmap, even if the pending unmap 868 * originates from another vCPU. This guarantees that the MMU has been 869 * completely unmapped before any vCPU actually uses it, and allows 870 * multiple vCPUs to lend a hand with completing the unmap. 871 */ 872 if (s2_mmu->pending_unmap) 873 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 874 875 return s2_mmu; 876 } 877 878 #define S2_MMU_IDX_MASK GENMASK_ULL(11, 0) 879 880 static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping, 881 struct kvm_s2_mmu *mmu) 882 { 883 BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K); 884 mapping->nested.start &= ~S2_MMU_IDX_MASK; 885 mapping->nested.start |= mmu->s2_mmu_idx; 886 } 887 888 static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm, 889 struct kvm_guest_s2_mapping *mapping) 890 { 891 return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK]; 892 } 893 894 static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping) 895 { 896 return mapping->nested.start & ~S2_MMU_IDX_MASK; 897 } 898 899 void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, 900 gpa_t nested_ipa, size_t map_size, 901 struct kvm_guest_s2_mapping *mapping) 902 { 903 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 904 905 lockdep_assert_held_read(&kvm->mmu_lock); 906 907 canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size); 908 nested_ipa = ALIGN_DOWN(nested_ipa, map_size); 909 910 mapping->canonical.start = canonical_ipa; 911 mapping->canonical.last = canonical_ipa + map_size - 1; 912 913 mapping->nested.start = nested_ipa; 914 mapping->nested.last = nested_ipa + map_size - 1; 915 916 tag_s2_mapping_mmu(mapping, mmu); 917 918 guard(spinlock)(&kvm->arch.guest_s2_tracking_lock); 919 interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings); 920 interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); 921 } 922 923 void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, 924 size_t size) 925 { 926 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 927 struct interval_tree_node *node, *next; 928 struct kvm_guest_s2_mapping *mapping; 929 gpa_t nipa_end = nipa + size - 1; 930 931 /* 932 * See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock 933 * isn't taken here. 934 */ 935 lockdep_assert_held_write(&kvm->mmu_lock); 936 937 node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end); 938 while (node) { 939 unsigned long nested_start; 940 941 next = interval_tree_iter_next(node, nipa, nipa_end); 942 mapping = container_of(node, struct kvm_guest_s2_mapping, 943 nested); 944 /* 945 * Tracking must be conservative on removal, only remove 946 * mappings that are within the unmap range. 947 */ 948 nested_start = s2_mapping_to_nested_start(mapping); 949 if (nipa <= nested_start && nipa_end >= mapping->nested.last) { 950 interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); 951 interval_tree_remove(&mapping->canonical, 952 &kvm->arch.mmu.guest_s2_mappings); 953 kfree(mapping); 954 } 955 node = next; 956 } 957 } 958 959 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 960 { 961 /* CnP being set denotes an invalid entry */ 962 mmu->tlb_vttbr = VTTBR_CNP_BIT; 963 mmu->nested_stage2_enabled = false; 964 atomic_set(&mmu->refcnt, 0); 965 } 966 967 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 968 { 969 /* 970 * If the vCPU kept its reference on the MMU after the last put, 971 * keep rolling with it. 972 */ 973 if (is_hyp_ctxt(vcpu)) { 974 if (!vcpu->arch.hw_mmu) 975 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 976 } else { 977 if (!vcpu->arch.hw_mmu) { 978 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 979 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 980 } 981 982 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) 983 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 984 } 985 } 986 987 /* 988 * Unmapping an L1 VNCR can happen concurrently without the mmu lock being 989 * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures 990 * that only one CPU sets it to -1 while getting a valid CPU number back. 991 */ 992 static int unmap_l1_vncr(struct vncr_tlb *vt) 993 { 994 int cpu = atomic_xchg_relaxed(&vt->cpu, -1); 995 996 if (cpu != -1) 997 clear_fixmap(vncr_fixmap(cpu)); 998 999 return cpu; 1000 } 1001 1002 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) 1003 { 1004 if (!host_data_test_flag(L1_VNCR_MAPPED)) 1005 return; 1006 1007 BUG_ON(is_hyp_ctxt(vcpu)); 1008 1009 /* 1010 * Unconditionally unmap the local VNCR if we have lost the race 1011 * against a concurrent TLBI. Otherwise we could end-up running 1012 * another vcpu with VNCR still mapped if the TLBI thread is 1013 * preempted between the exchange and the clear_fixmap(). 1014 * 1015 * Note that we do not care about the TLBI nuking the fixmap behind 1016 * the back of an running vcpu. This will only generate a fault and 1017 * possibly a retranslation. 1018 */ 1019 if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) 1020 clear_fixmap(vncr_fixmap(smp_processor_id())); 1021 host_data_clear_flag(L1_VNCR_MAPPED); 1022 } 1023 1024 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 1025 { 1026 /* Unconditionally drop the VNCR mapping if we have one */ 1027 this_cpu_reset_vncr_fixmap(vcpu); 1028 1029 /* 1030 * Keep a reference on the associated stage-2 MMU if the vCPU is 1031 * scheduling out and not in WFI emulation, suggesting it is likely to 1032 * reuse the MMU sometime soon. 1033 */ 1034 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 1035 return; 1036 1037 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 1038 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 1039 1040 vcpu->arch.hw_mmu = NULL; 1041 } 1042 1043 /* 1044 * Returns non-zero if permission fault is handled by injecting it to the next 1045 * level hypervisor. 1046 */ 1047 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 1048 { 1049 bool forward_fault = false; 1050 1051 trans->esr = 0; 1052 1053 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 1054 return 0; 1055 1056 if (kvm_vcpu_trap_is_iabt(vcpu)) { 1057 if (vcpu_mode_priv(vcpu)) 1058 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans); 1059 else 1060 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans); 1061 } else { 1062 bool write_fault = kvm_is_write_fault(vcpu); 1063 1064 forward_fault = ((write_fault && !trans->writable) || 1065 (!write_fault && !trans->readable)); 1066 } 1067 1068 if (forward_fault) 1069 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 1070 1071 return forward_fault; 1072 } 1073 1074 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 1075 { 1076 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 1077 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 1078 1079 return kvm_inject_nested_sync(vcpu, esr_el2); 1080 } 1081 1082 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) 1083 { 1084 enum vcpu_sysreg ttbr_elx; 1085 u64 tcr; 1086 u16 asid; 1087 1088 switch (regime) { 1089 case TR_EL10: 1090 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); 1091 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1; 1092 break; 1093 case TR_EL20: 1094 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); 1095 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2; 1096 break; 1097 default: 1098 BUG(); 1099 } 1100 1101 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx)); 1102 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || 1103 !(tcr & TCR_ASID16)) 1104 asid &= GENMASK(7, 0); 1105 1106 return asid; 1107 } 1108 1109 static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt) 1110 { 1111 BUG_ON(!vt->valid); 1112 vt->valid = false; 1113 unmap_l1_vncr(vt); 1114 atomic_dec(&kvm->arch.vncr_tlb_count); 1115 } 1116 1117 static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, 1118 u64 scope_start, u64 scope_size) 1119 { 1120 u64 tlb_size, tlb_start, tlb_end, scope_end; 1121 1122 tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); 1123 1124 tlb_start = addr & ~(tlb_size - 1); 1125 tlb_end = tlb_start + tlb_size - 1; 1126 scope_end = scope_start + scope_size - 1; 1127 1128 return !(tlb_end < scope_start || tlb_start > scope_end); 1129 } 1130 1131 /* 1132 * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and 1133 * either can race against a vcpu not being onlined yet (no pseudo-TLB 1134 * allocated). Similarly, the TLB might be invalid. Skip those, as they 1135 * obviously don't participate in the invalidation at this stage. 1136 */ 1137 #define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \ 1138 kvm_for_each_vcpu(idx, vcpup, kvm) \ 1139 if (((tlbp) = vcpup->arch.vncr_tlb) && \ 1140 (tlbp)->valid) 1141 1142 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) 1143 { 1144 struct kvm_vcpu *vcpu; 1145 struct vncr_tlb *vt; 1146 unsigned long i; 1147 1148 lockdep_assert_held_write(&kvm->mmu_lock); 1149 1150 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1151 return; 1152 1153 /* 1154 * Note that invalidating the VNCR on the back of an MMU notifier 1155 * doesn't require messing with the invalidation counter for a 1156 * parallel walk. The notifier itself will have bumped the counter, 1157 * making sure we rewalk. 1158 */ 1159 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) 1160 if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) 1161 invalidate_vncr(kvm, vt); 1162 } 1163 1164 struct s1e2_tlbi_scope { 1165 enum { 1166 TLBI_ALL, 1167 TLBI_VA, 1168 TLBI_VAA, 1169 TLBI_ASID, 1170 } type; 1171 1172 u16 asid; 1173 u64 va; 1174 u64 size; 1175 }; 1176 1177 static void invalidate_vncr_va(struct kvm *kvm, 1178 struct s1e2_tlbi_scope *scope) 1179 { 1180 struct kvm_vcpu *vcpu; 1181 struct vncr_tlb *vt; 1182 unsigned long i; 1183 1184 lockdep_assert_held_write(&kvm->mmu_lock); 1185 1186 /* 1187 * We might be performing a parallel S1 walk, so bump up the 1188 * invalidation counter even in the absence of an actual VNCR TLB 1189 * invalidation, as this could indicate that the guest has gone 1190 * through a BBM sequence. 1191 */ 1192 kvm->mmu_invalidate_seq++; 1193 smp_wmb(); 1194 1195 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { 1196 switch (scope->type) { 1197 case TLBI_ALL: 1198 break; 1199 1200 case TLBI_VA: 1201 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) 1202 continue; 1203 if (vt->wr.nG && vt->wr.asid != scope->asid) 1204 continue; 1205 break; 1206 1207 case TLBI_VAA: 1208 if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) 1209 continue; 1210 break; 1211 1212 case TLBI_ASID: 1213 if (!vt->wr.nG || vt->wr.asid != scope->asid) 1214 continue; 1215 break; 1216 } 1217 1218 invalidate_vncr(kvm, vt); 1219 } 1220 } 1221 1222 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48) 1223 1224 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, 1225 struct s1e2_tlbi_scope *scope) 1226 { 1227 switch (inst) { 1228 case OP_TLBI_ALLE2: 1229 case OP_TLBI_ALLE2IS: 1230 case OP_TLBI_ALLE2OS: 1231 case OP_TLBI_VMALLE1: 1232 case OP_TLBI_VMALLE1IS: 1233 case OP_TLBI_VMALLE1OS: 1234 case OP_TLBI_ALLE2NXS: 1235 case OP_TLBI_ALLE2ISNXS: 1236 case OP_TLBI_ALLE2OSNXS: 1237 case OP_TLBI_VMALLE1NXS: 1238 case OP_TLBI_VMALLE1ISNXS: 1239 case OP_TLBI_VMALLE1OSNXS: 1240 scope->type = TLBI_ALL; 1241 break; 1242 case OP_TLBI_VAE2: 1243 case OP_TLBI_VAE2IS: 1244 case OP_TLBI_VAE2OS: 1245 case OP_TLBI_VAE1: 1246 case OP_TLBI_VAE1IS: 1247 case OP_TLBI_VAE1OS: 1248 case OP_TLBI_VAE2NXS: 1249 case OP_TLBI_VAE2ISNXS: 1250 case OP_TLBI_VAE2OSNXS: 1251 case OP_TLBI_VAE1NXS: 1252 case OP_TLBI_VAE1ISNXS: 1253 case OP_TLBI_VAE1OSNXS: 1254 case OP_TLBI_VALE2: 1255 case OP_TLBI_VALE2IS: 1256 case OP_TLBI_VALE2OS: 1257 case OP_TLBI_VALE1: 1258 case OP_TLBI_VALE1IS: 1259 case OP_TLBI_VALE1OS: 1260 case OP_TLBI_VALE2NXS: 1261 case OP_TLBI_VALE2ISNXS: 1262 case OP_TLBI_VALE2OSNXS: 1263 case OP_TLBI_VALE1NXS: 1264 case OP_TLBI_VALE1ISNXS: 1265 case OP_TLBI_VALE1OSNXS: 1266 scope->type = TLBI_VA; 1267 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1268 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1269 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1270 break; 1271 case OP_TLBI_ASIDE1: 1272 case OP_TLBI_ASIDE1IS: 1273 case OP_TLBI_ASIDE1OS: 1274 case OP_TLBI_ASIDE1NXS: 1275 case OP_TLBI_ASIDE1ISNXS: 1276 case OP_TLBI_ASIDE1OSNXS: 1277 scope->type = TLBI_ASID; 1278 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1279 break; 1280 case OP_TLBI_VAAE1: 1281 case OP_TLBI_VAAE1IS: 1282 case OP_TLBI_VAAE1OS: 1283 case OP_TLBI_VAAE1NXS: 1284 case OP_TLBI_VAAE1ISNXS: 1285 case OP_TLBI_VAAE1OSNXS: 1286 case OP_TLBI_VAALE1: 1287 case OP_TLBI_VAALE1IS: 1288 case OP_TLBI_VAALE1OS: 1289 case OP_TLBI_VAALE1NXS: 1290 case OP_TLBI_VAALE1ISNXS: 1291 case OP_TLBI_VAALE1OSNXS: 1292 scope->type = TLBI_VAA; 1293 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1294 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1295 break; 1296 case OP_TLBI_RVAE2: 1297 case OP_TLBI_RVAE2IS: 1298 case OP_TLBI_RVAE2OS: 1299 case OP_TLBI_RVAE1: 1300 case OP_TLBI_RVAE1IS: 1301 case OP_TLBI_RVAE1OS: 1302 case OP_TLBI_RVAE2NXS: 1303 case OP_TLBI_RVAE2ISNXS: 1304 case OP_TLBI_RVAE2OSNXS: 1305 case OP_TLBI_RVAE1NXS: 1306 case OP_TLBI_RVAE1ISNXS: 1307 case OP_TLBI_RVAE1OSNXS: 1308 case OP_TLBI_RVALE2: 1309 case OP_TLBI_RVALE2IS: 1310 case OP_TLBI_RVALE2OS: 1311 case OP_TLBI_RVALE1: 1312 case OP_TLBI_RVALE1IS: 1313 case OP_TLBI_RVALE1OS: 1314 case OP_TLBI_RVALE2NXS: 1315 case OP_TLBI_RVALE2ISNXS: 1316 case OP_TLBI_RVALE2OSNXS: 1317 case OP_TLBI_RVALE1NXS: 1318 case OP_TLBI_RVALE1ISNXS: 1319 case OP_TLBI_RVALE1OSNXS: 1320 scope->type = TLBI_VA; 1321 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid); 1322 break; 1323 case OP_TLBI_RVAAE1: 1324 case OP_TLBI_RVAAE1IS: 1325 case OP_TLBI_RVAAE1OS: 1326 case OP_TLBI_RVAAE1NXS: 1327 case OP_TLBI_RVAAE1ISNXS: 1328 case OP_TLBI_RVAAE1OSNXS: 1329 case OP_TLBI_RVAALE1: 1330 case OP_TLBI_RVAALE1IS: 1331 case OP_TLBI_RVAALE1OS: 1332 case OP_TLBI_RVAALE1NXS: 1333 case OP_TLBI_RVAALE1ISNXS: 1334 case OP_TLBI_RVAALE1OSNXS: 1335 scope->type = TLBI_VAA; 1336 scope->va = decode_range_tlbi(val, &scope->size, NULL); 1337 break; 1338 } 1339 } 1340 1341 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) 1342 { 1343 struct s1e2_tlbi_scope scope = {}; 1344 1345 compute_s1_tlbi_range(vcpu, inst, val, &scope); 1346 1347 guard(write_lock)(&vcpu->kvm->mmu_lock); 1348 invalidate_vncr_va(vcpu->kvm, &scope); 1349 } 1350 1351 static void kvm_invalidate_vncr_ipa_all(struct kvm *kvm) 1352 { 1353 struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; 1354 1355 lockdep_assert_held_write(&kvm->mmu_lock); 1356 1357 /* if the mmu lock was dropped, pgt teardown may have raced. */ 1358 if (pgt) 1359 kvm_invalidate_vncr_ipa(kvm, 0, BIT(pgt->ia_bits)); 1360 } 1361 1362 void kvm_nested_s2_wp(struct kvm *kvm) 1363 { 1364 int i; 1365 1366 lockdep_assert_held_write(&kvm->mmu_lock); 1367 1368 if (!kvm->arch.nested_mmus_size) 1369 return; 1370 1371 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1372 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 1373 1374 if (kvm_s2_mmu_valid(mmu)) 1375 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 1376 } 1377 1378 kvm_invalidate_vncr_ipa_all(kvm); 1379 } 1380 1381 void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size, 1382 bool may_block) 1383 { 1384 gpa_t cipa_end = cipa + unmap_size - 1; 1385 struct kvm_guest_s2_mapping *mapping; 1386 struct interval_tree_node *node; 1387 size_t mapping_size; 1388 1389 /* 1390 * Guest s2 tracking interval trees are only accessed while holding the 1391 * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the 1392 * mmu_lock is held for write. This saves us from having to manually 1393 * lock/unlock guest_s2_tracking_lock below around 1394 * cond_resched_rwlock_write(). 1395 */ 1396 lockdep_assert_held_write(&kvm->mmu_lock); 1397 1398 if (!kvm->arch.nested_mmus_size) 1399 return; 1400 1401 while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings, 1402 cipa, cipa_end))) { 1403 unsigned long nested_start; 1404 struct kvm_s2_mmu *mmu; 1405 1406 mapping = container_of(node, struct kvm_guest_s2_mapping, 1407 canonical); 1408 nested_start = s2_mapping_to_nested_start(mapping); 1409 mmu = s2_mapping_to_mmu(kvm, mapping); 1410 1411 /* We could race against MMU teardown, which frees mmu->pgt. */ 1412 if (mmu->pgt) { 1413 mapping_size = mapping->nested.last - nested_start + 1; 1414 1415 if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start, 1416 mapping_size))) 1417 return; 1418 1419 interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); 1420 } 1421 interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings); 1422 kfree(mapping); 1423 1424 if (may_block) 1425 cond_resched_rwlock_write(&kvm->mmu_lock); 1426 } 1427 1428 kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size); 1429 } 1430 1431 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 1432 { 1433 int i; 1434 1435 lockdep_assert_held_write(&kvm->mmu_lock); 1436 1437 if (!kvm->arch.nested_mmus_size) 1438 return; 1439 1440 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1441 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 1442 1443 if (kvm_s2_mmu_valid(mmu)) 1444 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 1445 } 1446 1447 kvm_invalidate_vncr_ipa_all(kvm); 1448 } 1449 1450 void kvm_nested_s2_flush(struct kvm *kvm) 1451 { 1452 int i; 1453 1454 lockdep_assert_held_write(&kvm->mmu_lock); 1455 1456 if (!kvm->arch.nested_mmus_size) 1457 return; 1458 1459 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1460 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 1461 1462 if (kvm_s2_mmu_valid(mmu)) 1463 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 1464 } 1465 } 1466 1467 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1468 { 1469 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 1470 struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; 1471 1472 if (!WARN_ON(atomic_read(&mmu->refcnt))) 1473 kvm_free_stage2_pgd(mmu); 1474 } 1475 kvm_uninit_stage2_mmu(kvm); 1476 } 1477 1478 /* 1479 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter: 1480 * 1481 * - We introduce an internal representation of a vcpu-private TLB, 1482 * representing the mapping between the guest VA contained in VNCR_EL2, 1483 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at. 1484 * 1485 * - On translation fault from a nested VNCR access, we create such a TLB. 1486 * If there is no mapping to describe, the guest inherits the fault. 1487 * Crucially, no actual mapping is done at this stage. 1488 * 1489 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above 1490 * TLB exists, we map it in the fixmap for this CPU, and run with it. We 1491 * have to respect the permissions dictated by the guest, but not the 1492 * memory type (FWB is a must). 1493 * 1494 * - Note that we usually don't do a vcpu_load() on the back of a fault 1495 * (unless we are preempted), so the resolution of a translation fault 1496 * must go via a request that will map the VNCR page in the fixmap. 1497 * vcpu_load() might as well use the same mechanism. 1498 * 1499 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was 1500 * mapped, we unmap it. Yes it is that simple. The TLB still exists 1501 * though, and may be reused at a later load. 1502 * 1503 * - On permission fault, we simply forward the fault to the guest's EL2. 1504 * Get out of my way. 1505 * 1506 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that 1507 * intersects with the TLBI request, invalidate it, and unmap the page 1508 * from the fixmap. Because we need to look at all the vcpu-private TLBs, 1509 * this requires some wide-ranging locking to ensure that nothing races 1510 * against it. This requires some refcounting to avoid the search when 1511 * no such TLB is present (see below). 1512 * 1513 * - On MMU notifiers, we must invalidate our TLB in a similar way, but 1514 * looking at the IPA instead. The funny part is that there may not be a 1515 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST 1516 * instructions. 1517 * 1518 * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't 1519 * the number of *mapped* L1 VNCR pages, which is likely be a subset (and 1520 * by definition, a TLBI handled from L1 runs with the canonical VNCR 1521 * page, not the L1's). The innermost trap handling code checks this to 1522 * find out whether to return to the guest ASAP (no L1 TLBs) or to visit 1523 * this part of the world for some extra invalidation work. 1524 */ 1525 1526 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) 1527 { 1528 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1529 return 0; 1530 1531 if (!vcpu->arch.vncr_tlb) { 1532 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb, 1533 GFP_KERNEL_ACCOUNT); 1534 1535 /* 1536 * Taking the lock on assignment ensures that the TLB is 1537 * seen as initialised when following the pointer (release 1538 * semantics of the unlock), and avoids having acquires on 1539 * each user which already take the lock. 1540 */ 1541 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 1542 vcpu->arch.vncr_tlb = vt; 1543 } 1544 1545 if (!vcpu->arch.vncr_tlb) 1546 return -ENOMEM; 1547 1548 return 0; 1549 } 1550 1551 static u64 read_vncr_el2(struct kvm_vcpu *vcpu) 1552 { 1553 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); 1554 } 1555 1556 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) 1557 { 1558 struct kvm_memory_slot *memslot; 1559 bool write_fault, writable; 1560 unsigned long mmu_seq; 1561 struct vncr_tlb *vt; 1562 struct page *page = NULL; 1563 u64 va, pfn, gfn; 1564 int ret; 1565 1566 vt = vcpu->arch.vncr_tlb; 1567 1568 /* 1569 * If we're about to walk the EL2 S1 PTs, we must invalidate the 1570 * current TLB, as it could be sampled from another vcpu doing a 1571 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single 1572 * translation, so not much of a choice. 1573 * 1574 * We also prepare the next walk wilst we're at it. 1575 */ 1576 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1577 this_cpu_reset_vncr_fixmap(vcpu); 1578 if (vt->valid) 1579 invalidate_vncr(vcpu->kvm, vt); 1580 1581 vt->wi = (struct s1_walk_info) { 1582 .regime = TR_EL20, 1583 .as_el0 = false, 1584 .pan = false, 1585 }; 1586 vt->wr = (struct s1_walk_result){}; 1587 } 1588 1589 guard(srcu)(&vcpu->kvm->srcu); 1590 1591 va = read_vncr_el2(vcpu); 1592 1593 mmu_seq = vcpu->kvm->mmu_invalidate_seq; 1594 smp_rmb(); 1595 1596 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); 1597 if (ret) 1598 return ret; 1599 1600 write_fault = kvm_is_write_fault(vcpu); 1601 1602 gfn = vt->wr.pa >> PAGE_SHIFT; 1603 memslot = gfn_to_memslot(vcpu->kvm, gfn); 1604 if (!memslot) { 1605 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1606 return -EFAULT; 1607 } 1608 1609 *is_gmem = kvm_slot_has_gmem(memslot); 1610 if (!*is_gmem) { 1611 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, 1612 &writable, &page); 1613 if (is_error_noslot_pfn(pfn)) { 1614 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1615 return -EFAULT; 1616 } 1617 } else { 1618 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL); 1619 if (ret) { 1620 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, 1621 write_fault, false, false); 1622 return ret; 1623 } 1624 1625 writable = !(memslot->flags & KVM_MEM_READONLY); 1626 } 1627 1628 /* 1629 * FIXME: This check is too restrictive as KVM allows cacheable memory 1630 * attributes for PFNMAP VMAs that have cacheable attributes in host 1631 * stage-1. 1632 */ 1633 if (!pfn_is_map_memory(pfn)) { 1634 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1635 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1636 return -EINVAL; 1637 } 1638 1639 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1640 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { 1641 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1642 return -EAGAIN; 1643 } 1644 1645 vt->gva = va; 1646 vt->hpa = pfn << PAGE_SHIFT; 1647 vt->hpa_writable = writable; 1648 vt->valid = true; 1649 atomic_set(&vt->cpu, -1); 1650 1651 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 1652 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); 1653 } 1654 1655 if (vt->wr.pw && vt->hpa_writable) 1656 mark_page_dirty(vcpu->kvm, gfn); 1657 1658 return 0; 1659 } 1660 1661 static void handle_vncr_perm(struct kvm_vcpu *vcpu) 1662 { 1663 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1664 u64 esr = kvm_vcpu_get_esr(vcpu); 1665 u64 fsc; 1666 1667 /* 1668 * Promote to an external abort if the stage-1 permits writes but the 1669 * HPA is read-only (e.g. RO memslot). 1670 */ 1671 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) 1672 fsc = ESR_ELx_FSC_EXTABT; 1673 /* 1674 * Otherwise, inject a permission fault using the guest's translation 1675 * level rather than the host's. 1676 */ 1677 else 1678 fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); 1679 1680 esr &= ~ESR_ELx_FSC; 1681 esr |= FIELD_PREP(ESR_ELx_FSC, fsc); 1682 1683 kvm_inject_nested_sync(vcpu, esr); 1684 } 1685 1686 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) 1687 { 1688 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1689 u64 esr = kvm_vcpu_get_esr(vcpu); 1690 bool is_gmem = false; 1691 bool perm; 1692 int ret; 1693 1694 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); 1695 1696 if (kvm_vcpu_abt_issea(vcpu)) 1697 return kvm_handle_guest_sea(vcpu); 1698 1699 if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { 1700 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); 1701 return -EIO; 1702 } 1703 1704 /* 1705 * Speculatively increment the TLB count to make sure concurrent 1706 * TLBIs will take the slow path, and will interact with the retry 1707 * mechanism. Drop it again on error. 1708 */ 1709 atomic_inc(&vcpu->kvm->arch.vncr_tlb_count); 1710 smp_mb__after_atomic(); 1711 1712 ret = kvm_translate_vncr(vcpu, &is_gmem); 1713 if (ret) { 1714 smp_mb__before_atomic(); 1715 atomic_dec(&vcpu->kvm->arch.vncr_tlb_count); 1716 } 1717 1718 switch (ret) { 1719 case -EAGAIN: 1720 /* Let's try again... */ 1721 return 1; 1722 case -ENOMEM: 1723 /* 1724 * For guest_memfd, this indicates that it failed to 1725 * create a folio to back the memory. Inform userspace. 1726 */ 1727 if (is_gmem) 1728 return 0; 1729 /* Otherwise, let's try again... */ 1730 break; 1731 case -EFAULT: 1732 case -EIO: 1733 case -EHWPOISON: 1734 if (is_gmem) 1735 return 0; 1736 fallthrough; 1737 case -EINVAL: 1738 case -ENOENT: 1739 case -EACCES: 1740 /* 1741 * Translation failed, inject the corresponding 1742 * exception back to EL2. 1743 */ 1744 esr &= ~ESR_ELx_FSC; 1745 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); 1746 1747 kvm_inject_nested_sync(vcpu, esr); 1748 break; 1749 case 0: 1750 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; 1751 if (!perm) 1752 handle_vncr_perm(vcpu); 1753 break; 1754 } 1755 1756 return 1; 1757 } 1758 1759 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) 1760 { 1761 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1762 pgprot_t prot; 1763 1764 guard(preempt)(); 1765 guard(read_lock)(&vcpu->kvm->mmu_lock); 1766 1767 /* 1768 * The request to map VNCR may have raced against some other 1769 * event, such as an interrupt, and may not be valid anymore. 1770 */ 1771 if (is_hyp_ctxt(vcpu)) 1772 return; 1773 1774 /* 1775 * Check that the pseudo-TLB is valid and that VNCR_EL2 still 1776 * contains the expected value. If it doesn't, we simply bail out 1777 * without a mapping -- a transformed MSR/MRS will generate the 1778 * fault and allows us to populate the pseudo-TLB. 1779 */ 1780 if (!vt->valid) 1781 return; 1782 1783 /* We cache the MMU state in the TLB. Check that it matches. */ 1784 if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) 1785 return; 1786 1787 if (read_vncr_el2(vcpu) != vt->gva) 1788 return; 1789 1790 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) 1791 return; 1792 1793 if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) 1794 prot = PAGE_KERNEL; 1795 else if (vt->wr.pr) 1796 prot = PAGE_KERNEL_RO; 1797 else 1798 prot = PAGE_NONE; 1799 1800 /* 1801 * We can't map write-only (or no permission at all) in the kernel, 1802 * but the guest can do it if using POE, so we'll have to turn a 1803 * translation fault into a permission fault at runtime. 1804 * FIXME: WO doesn't work at all, need POE support in the kernel. 1805 */ 1806 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { 1807 atomic_set(&vt->cpu, smp_processor_id()); 1808 __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); 1809 host_data_set_flag(L1_VNCR_MAPPED); 1810 } 1811 } 1812 1813 /* 1814 * Our emulated CPU doesn't support all the possible features. For the 1815 * sake of simplicity (and probably mental sanity), wipe out a number 1816 * of feature bits we don't intend to support for the time being. 1817 * This list should get updated as new features get added to the NV 1818 * support, and new extension to the architecture. 1819 */ 1820 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) 1821 { 1822 u64 orig_val = val; 1823 1824 switch (reg) { 1825 case SYS_ID_AA64ISAR1_EL1: 1826 /* Support everything but LS64 and Spec Invalidation */ 1827 val &= ~(ID_AA64ISAR1_EL1_LS64 | 1828 ID_AA64ISAR1_EL1_SPECRES); 1829 break; 1830 1831 case SYS_ID_AA64PFR0_EL1: 1832 /* No RME, AMU, MPAM, or S-EL2 */ 1833 val &= ~(ID_AA64PFR0_EL1_RME | 1834 ID_AA64PFR0_EL1_AMU | 1835 ID_AA64PFR0_EL1_MPAM | 1836 ID_AA64PFR0_EL1_SEL2 | 1837 ID_AA64PFR0_EL1_EL3 | 1838 ID_AA64PFR0_EL1_EL2 | 1839 ID_AA64PFR0_EL1_EL1 | 1840 ID_AA64PFR0_EL1_EL0); 1841 /* 64bit only at any EL */ 1842 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP); 1843 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP); 1844 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP); 1845 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP); 1846 break; 1847 1848 case SYS_ID_AA64PFR1_EL1: 1849 /* Only support BTI, SSBS, CSV2_frac */ 1850 val &= ~(ID_AA64PFR1_EL1_PFAR | 1851 ID_AA64PFR1_EL1_MTEX | 1852 ID_AA64PFR1_EL1_THE | 1853 ID_AA64PFR1_EL1_GCS | 1854 ID_AA64PFR1_EL1_MTE_frac | 1855 ID_AA64PFR1_EL1_NMI | 1856 ID_AA64PFR1_EL1_SME | 1857 ID_AA64PFR1_EL1_RES0 | 1858 ID_AA64PFR1_EL1_MPAM_frac | 1859 ID_AA64PFR1_EL1_MTE); 1860 break; 1861 1862 case SYS_ID_AA64PFR2_EL1: 1863 /* GICv5 is not yet supported for NV */ 1864 val &= ~ID_AA64PFR2_EL1_GCIE; 1865 break; 1866 1867 case SYS_ID_AA64MMFR0_EL1: 1868 /* Hide ExS, Secure Memory */ 1869 val &= ~(ID_AA64MMFR0_EL1_EXS | 1870 ID_AA64MMFR0_EL1_TGRAN4_2 | 1871 ID_AA64MMFR0_EL1_TGRAN16_2 | 1872 ID_AA64MMFR0_EL1_TGRAN64_2 | 1873 ID_AA64MMFR0_EL1_SNSMEM); 1874 1875 /* Hide CNTPOFF if present */ 1876 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP); 1877 1878 /* Disallow unsupported S2 page sizes */ 1879 switch (PAGE_SIZE) { 1880 case SZ_64K: 1881 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI); 1882 fallthrough; 1883 case SZ_16K: 1884 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI); 1885 fallthrough; 1886 case SZ_4K: 1887 /* Support everything */ 1888 break; 1889 } 1890 1891 /* 1892 * Since we can't support a guest S2 page size smaller 1893 * than the host's own page size (due to KVM only 1894 * populating its own S2 using the kernel's page 1895 * size), advertise the limitation using FEAT_GTG. 1896 */ 1897 switch (PAGE_SIZE) { 1898 case SZ_4K: 1899 if (_has_tgran_2(orig_val, 4)) 1900 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP); 1901 fallthrough; 1902 case SZ_16K: 1903 if (_has_tgran_2(orig_val, 16)) 1904 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP); 1905 fallthrough; 1906 case SZ_64K: 1907 if (_has_tgran_2(orig_val, 64)) 1908 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP); 1909 break; 1910 } 1911 1912 /* Cap PARange to 48bits */ 1913 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48); 1914 break; 1915 1916 case SYS_ID_AA64MMFR1_EL1: 1917 val &= ~(ID_AA64MMFR1_EL1_CMOW | 1918 ID_AA64MMFR1_EL1_nTLBPA | 1919 ID_AA64MMFR1_EL1_ETS); 1920 1921 /* FEAT_E2H0 implies no VHE */ 1922 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) 1923 val &= ~ID_AA64MMFR1_EL1_VH; 1924 1925 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF); 1926 break; 1927 1928 case SYS_ID_AA64MMFR2_EL1: 1929 val &= ~(ID_AA64MMFR2_EL1_BBM | 1930 ID_AA64MMFR2_EL1_TTL | 1931 GENMASK_ULL(47, 44) | 1932 ID_AA64MMFR2_EL1_ST | 1933 ID_AA64MMFR2_EL1_CCIDX | 1934 ID_AA64MMFR2_EL1_VARange); 1935 1936 /* Force TTL support */ 1937 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP); 1938 break; 1939 1940 case SYS_ID_AA64MMFR4_EL1: 1941 /* 1942 * You get EITHER 1943 * 1944 * - FEAT_VHE without FEAT_E2H0 1945 * - FEAT_NV limited to FEAT_NV2(p1)/NV3 1946 * - HCR_EL2.NV1 being RES0 1947 * 1948 * OR 1949 * 1950 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV 1951 * 1952 * Life is too short for anything else. 1953 */ 1954 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { 1955 val = 0; 1956 } else { 1957 val &= ID_AA64MMFR4_EL1_NV_frac; 1958 if (cpus_have_final_cap(ARM64_HAS_NV3)) 1959 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3); 1960 else if (cpus_have_final_cap(ARM64_HAS_NV2P1)) 1961 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1); 1962 else 1963 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); 1964 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); 1965 } 1966 break; 1967 1968 case SYS_ID_AA64DFR0_EL1: 1969 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 1970 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff | 1971 ID_AA64DFR0_EL1_BRBE | 1972 ID_AA64DFR0_EL1_MTPMU | 1973 ID_AA64DFR0_EL1_TraceBuffer | 1974 ID_AA64DFR0_EL1_TraceFilt | 1975 ID_AA64DFR0_EL1_PMSVer | 1976 ID_AA64DFR0_EL1_CTX_CMPs | 1977 ID_AA64DFR0_EL1_SEBEP | 1978 ID_AA64DFR0_EL1_PMSS | 1979 ID_AA64DFR0_EL1_TraceVer); 1980 1981 /* 1982 * FEAT_Debugv8p9 requires support for extended breakpoints / 1983 * watchpoints. 1984 */ 1985 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8); 1986 break; 1987 } 1988 1989 return val; 1990 } 1991 1992 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu, 1993 enum vcpu_sysreg sr, u64 v) 1994 { 1995 struct resx resx; 1996 1997 resx = kvm_get_sysreg_resx(vcpu->kvm, sr); 1998 v &= ~resx.res0; 1999 v |= resx.res1; 2000 2001 return v; 2002 } 2003 2004 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx) 2005 { 2006 BUILD_BUG_ON(!__builtin_constant_p(sr)); 2007 BUILD_BUG_ON(sr < __SANITISED_REG_START__); 2008 BUILD_BUG_ON(sr >= NR_SYS_REGS); 2009 2010 kvm_set_sysreg_resx(kvm, sr, resx); 2011 } 2012 2013 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) 2014 { 2015 struct kvm *kvm = vcpu->kvm; 2016 struct resx resx; 2017 2018 lockdep_assert_held(&kvm->arch.config_lock); 2019 2020 if (kvm->arch.sysreg_masks) 2021 goto out; 2022 2023 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks), 2024 GFP_KERNEL_ACCOUNT); 2025 if (!kvm->arch.sysreg_masks) 2026 return -ENOMEM; 2027 2028 /* VTTBR_EL2 */ 2029 resx = (typeof(resx)){}; 2030 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 2031 resx.res0 |= GENMASK(63, 56); 2032 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 2033 resx.res0 |= VTTBR_CNP_BIT; 2034 set_sysreg_masks(kvm, VTTBR_EL2, resx); 2035 2036 /* VTCR_EL2 */ 2037 resx = get_reg_fixed_bits(kvm, VTCR_EL2); 2038 set_sysreg_masks(kvm, VTCR_EL2, resx); 2039 2040 /* VMPIDR_EL2 */ 2041 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24); 2042 resx.res1 = BIT(31); 2043 set_sysreg_masks(kvm, VMPIDR_EL2, resx); 2044 2045 /* HCR_EL2 */ 2046 resx = get_reg_fixed_bits(kvm, HCR_EL2); 2047 set_sysreg_masks(kvm, HCR_EL2, resx); 2048 2049 /* NVHCR_EL2 */ 2050 resx = get_reg_fixed_bits(kvm, NVHCR_EL2); 2051 set_sysreg_masks(kvm, NVHCR_EL2, resx); 2052 2053 /* HCRX_EL2 */ 2054 resx = get_reg_fixed_bits(kvm, HCRX_EL2); 2055 set_sysreg_masks(kvm, HCRX_EL2, resx); 2056 2057 /* HFG[RW]TR_EL2 */ 2058 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2); 2059 set_sysreg_masks(kvm, HFGRTR_EL2, resx); 2060 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2); 2061 set_sysreg_masks(kvm, HFGWTR_EL2, resx); 2062 2063 /* HDFG[RW]TR_EL2 */ 2064 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2); 2065 set_sysreg_masks(kvm, HDFGRTR_EL2, resx); 2066 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2); 2067 set_sysreg_masks(kvm, HDFGWTR_EL2, resx); 2068 2069 /* HFGITR_EL2 */ 2070 resx = get_reg_fixed_bits(kvm, HFGITR_EL2); 2071 set_sysreg_masks(kvm, HFGITR_EL2, resx); 2072 2073 /* HAFGRTR_EL2 - not a lot to see here */ 2074 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2); 2075 set_sysreg_masks(kvm, HAFGRTR_EL2, resx); 2076 2077 /* HFG[RW]TR2_EL2 */ 2078 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2); 2079 set_sysreg_masks(kvm, HFGRTR2_EL2, resx); 2080 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2); 2081 set_sysreg_masks(kvm, HFGWTR2_EL2, resx); 2082 2083 /* HDFG[RW]TR2_EL2 */ 2084 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2); 2085 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx); 2086 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2); 2087 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx); 2088 2089 /* HFGITR2_EL2 */ 2090 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2); 2091 set_sysreg_masks(kvm, HFGITR2_EL2, resx); 2092 2093 /* TCR2_EL2 */ 2094 resx = get_reg_fixed_bits(kvm, TCR2_EL2); 2095 set_sysreg_masks(kvm, TCR2_EL2, resx); 2096 2097 /* SCTLR_EL1 */ 2098 resx = get_reg_fixed_bits(kvm, SCTLR_EL1); 2099 set_sysreg_masks(kvm, SCTLR_EL1, resx); 2100 2101 /* SCTLR_EL2 */ 2102 resx = get_reg_fixed_bits(kvm, SCTLR_EL2); 2103 set_sysreg_masks(kvm, SCTLR_EL2, resx); 2104 2105 /* SCTLR2_ELx */ 2106 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1); 2107 set_sysreg_masks(kvm, SCTLR2_EL1, resx); 2108 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2); 2109 set_sysreg_masks(kvm, SCTLR2_EL2, resx); 2110 2111 /* MDCR_EL2 */ 2112 resx = get_reg_fixed_bits(kvm, MDCR_EL2); 2113 set_sysreg_masks(kvm, MDCR_EL2, resx); 2114 2115 /* CNTHCTL_EL2 */ 2116 resx.res0 = GENMASK(63, 20); 2117 resx.res1 = 0; 2118 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP)) 2119 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK; 2120 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) { 2121 resx.res0 |= CNTHCTL_ECV; 2122 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP)) 2123 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT | 2124 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT); 2125 } 2126 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 2127 resx.res0 |= GENMASK(11, 8); 2128 set_sysreg_masks(kvm, CNTHCTL_EL2, resx); 2129 2130 /* ICH_HCR_EL2 */ 2131 resx.res0 = ICH_HCR_EL2_RES0; 2132 resx.res1 = ICH_HCR_EL2_RES1; 2133 if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS)) 2134 resx.res0 |= ICH_HCR_EL2_TDIR; 2135 /* No GICv4 is presented to the guest */ 2136 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; 2137 set_sysreg_masks(kvm, ICH_HCR_EL2, resx); 2138 2139 /* VNCR_EL2 */ 2140 resx.res0 = VNCR_EL2_RES0; 2141 resx.res1 = VNCR_EL2_RES1; 2142 set_sysreg_masks(kvm, VNCR_EL2, resx); 2143 2144 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */ 2145 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4); 2146 resx.res1 = ZCR_ELx_RES1; 2147 set_sysreg_masks(kvm, ZCR_EL2, resx); 2148 2149 out: 2150 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) 2151 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0); 2152 2153 return 0; 2154 } 2155 2156 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 2157 { 2158 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 2159 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 2160 2161 write_lock(&vcpu->kvm->mmu_lock); 2162 if (mmu->pending_unmap) { 2163 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 2164 mmu->pending_unmap = false; 2165 } 2166 write_unlock(&vcpu->kvm->mmu_lock); 2167 } 2168 2169 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu)) 2170 kvm_map_l1_vncr(vcpu); 2171 2172 /* Must be last, as may switch context! */ 2173 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) 2174 kvm_inject_nested_irq(vcpu); 2175 } 2176 2177 /* 2178 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor 2179 * can write to HCR_EL2 behind our back, potentially changing the exception 2180 * routing / masking for even the host context. 2181 * 2182 * What follows is some slop to (1) react to exception routing / masking and (2) 2183 * preserve the pending SError state across translation regimes. 2184 */ 2185 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) 2186 { 2187 if (!vcpu_has_nv(vcpu)) 2188 return; 2189 2190 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 2191 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 2192 } 2193 2194 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) 2195 { 2196 unsigned long *hcr = vcpu_hcr(vcpu); 2197 2198 if (!vcpu_has_nv(vcpu)) 2199 return; 2200 2201 /* 2202 * We previously decided that an SError was deliverable to the guest. 2203 * Reap the pending state from HCR_EL2 and... 2204 */ 2205 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr))) 2206 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); 2207 2208 /* 2209 * Re-attempt SError injection in case the deliverability has changed, 2210 * which is necessary to faithfully emulate WFI the case of a pending 2211 * SError being a wakeup condition. 2212 */ 2213 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 2214 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 2215 } 2216 2217 /* 2218 * KVM unconditionally sets most of these traps anyway but use an allowlist 2219 * to document the guest hypervisor traps that may take precedence and guard 2220 * against future changes to the non-nested trap configuration. 2221 */ 2222 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \ 2223 MDCR_EL2_TDA | \ 2224 MDCR_EL2_TDRA | \ 2225 MDCR_EL2_TTRF | \ 2226 MDCR_EL2_TPMS | \ 2227 MDCR_EL2_TPM | \ 2228 MDCR_EL2_TPMCR | \ 2229 MDCR_EL2_TDCC | \ 2230 MDCR_EL2_TDOSA) 2231 2232 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) 2233 { 2234 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 2235 2236 if (is_nested_ctxt(vcpu)) 2237 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE); 2238 /* 2239 * In yet another example where FEAT_NV2 is fscking broken, accesses 2240 * to MDSCR_EL1 are redirected to the VNCR despite having an effect 2241 * at EL2. Use a big hammer to apply sanity. 2242 * 2243 * Unless of course we have FEAT_FGT, in which case we can precisely 2244 * trap MDSCR_EL1. 2245 */ 2246 else if (!cpus_have_final_cap(ARM64_HAS_FGT)) 2247 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; 2248 } 2249