1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/fixmap.h> 12 #include <asm/kvm_arm.h> 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_mmu.h> 15 #include <asm/kvm_nested.h> 16 #include <asm/sysreg.h> 17 18 #include "sys_regs.h" 19 20 struct vncr_tlb { 21 /* The guest's VNCR_EL2 */ 22 u64 gva; 23 struct s1_walk_info wi; 24 struct s1_walk_result wr; 25 26 u64 hpa; 27 bool hpa_writable; 28 29 /* -1 when not mapped on a CPU */ 30 int cpu; 31 32 /* 33 * true if the TLB is valid. Can only be changed with the 34 * mmu_lock held. 35 */ 36 bool valid; 37 }; 38 39 /* 40 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 41 * memory usage and potential number of different sets of S2 PTs in 42 * the guests. Running out of S2 MMUs only affects performance (we 43 * will invalidate them more often). 44 */ 45 #define S2_MMU_PER_VCPU 2 46 47 void kvm_init_nested(struct kvm *kvm) 48 { 49 kvm->arch.nested_mmus = NULL; 50 kvm->arch.nested_mmus_size = 0; 51 atomic_set(&kvm->arch.vncr_map_count, 0); 52 } 53 54 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 55 { 56 /* 57 * We only initialise the IPA range on the canonical MMU, which 58 * defines the contract between KVM and userspace on where the 59 * "hardware" is in the IPA space. This affects the validity of MMIO 60 * exits forwarded to userspace, for example. 61 * 62 * For nested S2s, we use the PARange as exposed to the guest, as it 63 * is allowed to use it at will to expose whatever memory map it 64 * wants to its own guests as it would be on real HW. 65 */ 66 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 67 } 68 69 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 70 { 71 struct kvm *kvm = vcpu->kvm; 72 struct kvm_s2_mmu *tmp; 73 int num_mmus, ret = 0; 74 75 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && 76 !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 77 return -EINVAL; 78 79 if (!vcpu->arch.ctxt.vncr_array) 80 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT | 81 __GFP_ZERO); 82 83 if (!vcpu->arch.ctxt.vncr_array) 84 return -ENOMEM; 85 86 /* 87 * Let's treat memory allocation failures as benign: If we fail to 88 * allocate anything, return an error and keep the allocated array 89 * alive. Userspace may try to recover by initializing the vcpu 90 * again, and there is no reason to affect the whole VM for this. 91 */ 92 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 93 94 if (num_mmus > kvm->arch.nested_mmus_size) { 95 tmp = kvcalloc(num_mmus, sizeof(*tmp), GFP_KERNEL_ACCOUNT); 96 if (!tmp) 97 return -ENOMEM; 98 99 write_lock(&kvm->mmu_lock); 100 101 if (kvm->arch.nested_mmus_size) { 102 memcpy(tmp, kvm->arch.nested_mmus, 103 size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); 104 105 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 106 tmp[i].pgt->mmu = &tmp[i]; 107 } 108 109 swap(kvm->arch.nested_mmus, tmp); 110 111 write_unlock(&kvm->mmu_lock); 112 113 kvfree(tmp); 114 } 115 116 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 117 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); 118 119 if (ret) { 120 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 121 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); 122 123 free_page((unsigned long)vcpu->arch.ctxt.vncr_array); 124 vcpu->arch.ctxt.vncr_array = NULL; 125 126 return ret; 127 } 128 129 kvm->arch.nested_mmus_size = num_mmus; 130 131 return 0; 132 } 133 134 struct s2_walk_info { 135 u64 baddr; 136 unsigned int max_oa_bits; 137 unsigned int pgshift; 138 unsigned int sl; 139 unsigned int t0sz; 140 bool be; 141 bool ha; 142 }; 143 144 static u32 compute_fsc(int level, u32 fsc) 145 { 146 return fsc | (level & 0x3); 147 } 148 149 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 150 { 151 u32 esr; 152 153 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 154 esr |= compute_fsc(level, fsc); 155 return esr; 156 } 157 158 static int get_ia_size(struct s2_walk_info *wi) 159 { 160 return 64 - wi->t0sz; 161 } 162 163 static int check_base_s2_limits(struct kvm_vcpu *vcpu, struct s2_walk_info *wi, 164 int level, int input_size, int stride) 165 { 166 int start_size, pa_max; 167 168 pa_max = kvm_get_pa_bits(vcpu->kvm); 169 170 /* Check translation limits */ 171 switch (BIT(wi->pgshift)) { 172 case SZ_64K: 173 if (level == 0 || (level == 1 && pa_max <= 42)) 174 return -EFAULT; 175 break; 176 case SZ_16K: 177 if (level == 0 || (level == 1 && pa_max <= 40)) 178 return -EFAULT; 179 break; 180 case SZ_4K: 181 if (level < 0 || (level == 0 && pa_max <= 42)) 182 return -EFAULT; 183 break; 184 } 185 186 /* Check input size limits */ 187 if (input_size > pa_max) 188 return -EFAULT; 189 190 /* Check number of entries in starting level table */ 191 start_size = input_size - ((3 - level) * stride + wi->pgshift); 192 if (start_size < 1 || start_size > stride + 4) 193 return -EFAULT; 194 195 return 0; 196 } 197 198 /* Check if output is within boundaries */ 199 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 200 { 201 unsigned int output_size = wi->max_oa_bits; 202 203 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 204 return -1; 205 206 return 0; 207 } 208 209 static int read_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 *desc, 210 struct s2_walk_info *wi) 211 { 212 u64 val; 213 int r; 214 215 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val)); 216 if (r) 217 return r; 218 219 /* 220 * Handle reversedescriptors if endianness differs between the 221 * host and the guest hypervisor. 222 */ 223 if (wi->be) 224 *desc = be64_to_cpu((__force __be64)val); 225 else 226 *desc = le64_to_cpu((__force __le64)val); 227 228 return 0; 229 } 230 231 static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new, 232 struct s2_walk_info *wi) 233 { 234 if (wi->be) { 235 old = (__force u64)cpu_to_be64(old); 236 new = (__force u64)cpu_to_be64(new); 237 } else { 238 old = (__force u64)cpu_to_le64(old); 239 new = (__force u64)cpu_to_le64(new); 240 } 241 242 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new); 243 } 244 245 /* 246 * This is essentially a C-version of the pseudo code from the ARM ARM 247 * AArch64.TranslationTableWalk function. I strongly recommend looking at 248 * that pseudocode in trying to understand this. 249 * 250 * Must be called with the kvm->srcu read lock held 251 */ 252 static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa, 253 struct s2_walk_info *wi, struct kvm_s2_trans *out) 254 { 255 int first_block_level, level, stride, input_size, base_lower_bound; 256 phys_addr_t base_addr; 257 unsigned int addr_top, addr_bottom; 258 u64 desc, new_desc; /* page table entry */ 259 int ret; 260 phys_addr_t paddr; 261 262 switch (BIT(wi->pgshift)) { 263 default: 264 case SZ_64K: 265 case SZ_16K: 266 level = 3 - wi->sl; 267 first_block_level = 2; 268 break; 269 case SZ_4K: 270 level = 2 - wi->sl; 271 first_block_level = 1; 272 break; 273 } 274 275 stride = wi->pgshift - 3; 276 input_size = get_ia_size(wi); 277 if (input_size > 48 || input_size < 25) 278 return -EFAULT; 279 280 ret = check_base_s2_limits(vcpu, wi, level, input_size, stride); 281 if (WARN_ON(ret)) { 282 out->esr = compute_fsc(0, ESR_ELx_FSC_FAULT); 283 return ret; 284 } 285 286 base_lower_bound = 3 + input_size - ((3 - level) * stride + 287 wi->pgshift); 288 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 289 290 if (check_output_size(wi, base_addr)) { 291 /* R_BFHQH */ 292 out->esr = compute_fsc(0, ESR_ELx_FSC_ADDRSZ); 293 return 1; 294 } 295 296 addr_top = input_size - 1; 297 298 while (1) { 299 phys_addr_t index; 300 301 addr_bottom = (3 - level) * stride + wi->pgshift; 302 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 303 >> (addr_bottom - 3); 304 305 paddr = base_addr | index; 306 ret = read_guest_s2_desc(vcpu, paddr, &desc, wi); 307 if (ret < 0) { 308 out->esr = ESR_ELx_FSC_SEA_TTW(level); 309 return ret; 310 } 311 312 new_desc = desc; 313 314 /* Check for valid descriptor at this point */ 315 if (!(desc & KVM_PTE_VALID)) { 316 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 317 out->desc = desc; 318 return 1; 319 } 320 321 if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) { 322 if (level < 3) 323 break; 324 325 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 326 out->desc = desc; 327 return 1; 328 } 329 330 /* We're at the final level */ 331 if (level == 3) 332 break; 333 334 if (check_output_size(wi, desc)) { 335 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 336 out->desc = desc; 337 return 1; 338 } 339 340 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 341 342 level += 1; 343 addr_top = addr_bottom - 1; 344 } 345 346 if (level < first_block_level) { 347 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 348 out->desc = desc; 349 return 1; 350 } 351 352 if (check_output_size(wi, desc)) { 353 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 354 out->desc = desc; 355 return 1; 356 } 357 358 if (wi->ha) 359 new_desc |= KVM_PTE_LEAF_ATTR_LO_S2_AF; 360 361 if (new_desc != desc) { 362 ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi); 363 if (ret == -EAGAIN) 364 return ret; 365 if (ret) { 366 out->esr = ESR_ELx_FSC_SEA_TTW(level); 367 out->desc = desc; 368 return 1; 369 } 370 371 desc = new_desc; 372 } 373 374 if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) { 375 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 376 out->desc = desc; 377 return 1; 378 } 379 380 addr_bottom += contiguous_bit_shift(desc, wi, level); 381 382 /* Calculate and return the result */ 383 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 384 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 385 out->output = paddr; 386 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 387 out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R; 388 out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; 389 out->level = level; 390 out->desc = desc; 391 return 0; 392 } 393 394 #define _has_tgran_2(__r, __sz) \ 395 ({ \ 396 u64 _s1, _s2, _mmfr0 = __r; \ 397 \ 398 _s2 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 399 TGRAN##__sz##_2, _mmfr0); \ 400 \ 401 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 402 TGRAN##__sz, _mmfr0); \ 403 \ 404 ((_s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_NI && \ 405 _s2 != ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz) || \ 406 (_s2 == ID_AA64MMFR0_EL1_TGRAN##__sz##_2_TGRAN##__sz && \ 407 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI)); \ 408 }) 409 410 static bool has_tgran_2(u64 mmfr0, unsigned int shift) 411 { 412 switch (shift) { 413 case 12: 414 return _has_tgran_2(mmfr0, 4); 415 case 14: 416 return _has_tgran_2(mmfr0, 16); 417 case 16: 418 return _has_tgran_2(mmfr0, 64); 419 default: 420 BUG(); 421 } 422 } 423 424 static unsigned int fallback_tgran2_shift(u64 mmfr0) 425 { 426 if (has_tgran_2(mmfr0, PAGE_SHIFT)) 427 return PAGE_SHIFT; 428 else if (has_tgran_2(mmfr0, 12)) 429 return 12; 430 else if (has_tgran_2(mmfr0, 14)) 431 return 14; 432 else if (has_tgran_2(mmfr0, 16)) 433 return 16; 434 else 435 return PAGE_SHIFT; 436 } 437 438 static unsigned int vtcr_to_tg0_pgshift(struct kvm *kvm, u64 vtcr) 439 { 440 u64 tg0 = FIELD_GET(VTCR_EL2_TG0_MASK, vtcr); 441 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 442 unsigned int shift; 443 444 switch (tg0) { 445 case VTCR_EL2_TG0_4K: 446 shift = 12; 447 break; 448 case VTCR_EL2_TG0_16K: 449 shift = 14; 450 break; 451 case VTCR_EL2_TG0_64K: 452 /* IMPDEF: treat any other value as 64k, subject to fallback */ 453 default: 454 shift = 16; 455 } 456 457 /* 458 * If TGx is programmed to an unimplemented value (not advertised in 459 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is 460 * written, as per the architecture. Choose an available one while 461 * prioritizing PAGE_SIZE. 462 */ 463 if (!has_tgran_2(mmfr0, shift)) 464 return fallback_tgran2_shift(mmfr0); 465 466 return shift; 467 } 468 469 static size_t vtcr_to_tg0_pgsize(struct kvm *kvm, u64 vtcr) 470 { 471 return BIT(vtcr_to_tg0_pgshift(kvm, vtcr)); 472 } 473 474 static void setup_s2_walk(struct kvm_vcpu *vcpu, struct s2_walk_info *wi) 475 { 476 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 477 478 wi->baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 479 wi->t0sz = vtcr & VTCR_EL2_T0SZ_MASK; 480 wi->pgshift = vtcr_to_tg0_pgshift(vcpu->kvm, vtcr); 481 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 482 /* Global limit for now, should eventually be per-VM */ 483 wi->max_oa_bits = min(get_kvm_ipa_limit(), 484 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false)); 485 wi->ha = vtcr & VTCR_EL2_HA; 486 wi->be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 487 } 488 489 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 490 struct kvm_s2_trans *result) 491 { 492 struct s2_walk_info wi; 493 int ret; 494 495 result->esr = 0; 496 497 if (!vcpu_has_nv(vcpu)) 498 return 0; 499 500 setup_s2_walk(vcpu, &wi); 501 502 ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result); 503 if (ret) 504 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 505 506 return ret; 507 } 508 509 static unsigned int ttl_to_size(u8 ttl) 510 { 511 int level = ttl & 3; 512 int gran = (ttl >> 2) & 3; 513 unsigned int max_size = 0; 514 515 switch (gran) { 516 case TLBI_TTL_TG_4K: 517 switch (level) { 518 case 0: 519 break; 520 case 1: 521 max_size = SZ_1G; 522 break; 523 case 2: 524 max_size = SZ_2M; 525 break; 526 case 3: 527 max_size = SZ_4K; 528 break; 529 } 530 break; 531 case TLBI_TTL_TG_16K: 532 switch (level) { 533 case 0: 534 case 1: 535 break; 536 case 2: 537 max_size = SZ_32M; 538 break; 539 case 3: 540 max_size = SZ_16K; 541 break; 542 } 543 break; 544 case TLBI_TTL_TG_64K: 545 switch (level) { 546 case 0: 547 case 1: 548 /* No 52bit IPA support */ 549 break; 550 case 2: 551 max_size = SZ_512M; 552 break; 553 case 3: 554 max_size = SZ_64K; 555 break; 556 } 557 break; 558 default: /* No size information */ 559 break; 560 } 561 562 return max_size; 563 } 564 565 static u8 pgshift_level_to_ttl(u16 shift, u8 level) 566 { 567 u8 ttl; 568 569 switch(shift) { 570 case 12: 571 ttl = TLBI_TTL_TG_4K; 572 break; 573 case 14: 574 ttl = TLBI_TTL_TG_16K; 575 break; 576 case 16: 577 ttl = TLBI_TTL_TG_64K; 578 break; 579 default: 580 BUG(); 581 } 582 583 ttl <<= 2; 584 ttl |= level & 3; 585 586 return ttl; 587 } 588 589 /* 590 * Compute the equivalent of the TTL field by parsing the shadow PT. The 591 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 592 * retrieved from first entry carrying the level as a tag. 593 */ 594 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 595 { 596 size_t tg0_size = vtcr_to_tg0_pgsize(kvm_s2_mmu_to_kvm(mmu), mmu->tlb_vtcr); 597 u64 tmp, sz = 0; 598 kvm_pte_t pte; 599 u8 ttl, level; 600 601 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 602 603 switch (tg0_size) { 604 case SZ_4K: 605 ttl = (TLBI_TTL_TG_4K << 2); 606 break; 607 case SZ_16K: 608 ttl = (TLBI_TTL_TG_16K << 2); 609 break; 610 case SZ_64K: 611 default: /* IMPDEF: treat any other value as 64k */ 612 ttl = (TLBI_TTL_TG_64K << 2); 613 break; 614 } 615 616 tmp = addr; 617 618 again: 619 /* Iteratively compute the block sizes for a particular granule size */ 620 switch (tg0_size) { 621 case SZ_4K: 622 if (sz < SZ_4K) sz = SZ_4K; 623 else if (sz < SZ_2M) sz = SZ_2M; 624 else if (sz < SZ_1G) sz = SZ_1G; 625 else sz = 0; 626 break; 627 case SZ_16K: 628 if (sz < SZ_16K) sz = SZ_16K; 629 else if (sz < SZ_32M) sz = SZ_32M; 630 else sz = 0; 631 break; 632 case SZ_64K: 633 default: /* IMPDEF: treat any other value as 64k */ 634 if (sz < SZ_64K) sz = SZ_64K; 635 else if (sz < SZ_512M) sz = SZ_512M; 636 else sz = 0; 637 break; 638 } 639 640 if (sz == 0) 641 return 0; 642 643 tmp &= ~(sz - 1); 644 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 645 goto again; 646 if (!(pte & PTE_VALID)) 647 goto again; 648 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 649 if (!level) 650 goto again; 651 652 ttl |= level; 653 654 /* 655 * We now have found some level information in the shadow S2. Check 656 * that the resulting range is actually including the original IPA. 657 */ 658 sz = ttl_to_size(ttl); 659 if (addr < (tmp + sz)) 660 return ttl; 661 662 return 0; 663 } 664 665 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 666 { 667 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 668 unsigned long max_size; 669 u8 ttl; 670 671 ttl = FIELD_GET(TLBI_TTL_MASK, val); 672 673 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 674 /* No TTL, check the shadow S2 for a hint */ 675 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 676 ttl = get_guest_mapping_ttl(mmu, addr); 677 } 678 679 max_size = ttl_to_size(ttl); 680 681 if (!max_size) { 682 /* Compute the maximum extent of the invalidation */ 683 switch (vtcr_to_tg0_pgsize(kvm, mmu->tlb_vtcr)) { 684 case SZ_4K: 685 max_size = SZ_1G; 686 break; 687 case SZ_16K: 688 max_size = SZ_32M; 689 break; 690 case SZ_64K: 691 default: /* IMPDEF: treat any other value as 64k */ 692 /* 693 * No, we do not support 52bit IPA in nested yet. Once 694 * we do, this should be 4TB. 695 */ 696 max_size = SZ_512M; 697 break; 698 } 699 } 700 701 WARN_ON(!max_size); 702 return max_size; 703 } 704 705 /* 706 * We can have multiple *different* MMU contexts with the same VMID: 707 * 708 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 709 * 710 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 711 * VBBTR_EL2.BADDR address 712 * 713 * - A combination of the above... 714 * 715 * We can always identify which MMU context to pick at run-time. However, 716 * TLB invalidation involving a VMID must take action on all the TLBs using 717 * this particular VMID. This translates into applying the same invalidation 718 * operation to all the contexts that are using this VMID. Moar phun! 719 */ 720 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 721 const union tlbi_info *info, 722 void (*tlbi_callback)(struct kvm_s2_mmu *, 723 const union tlbi_info *)) 724 { 725 write_lock(&kvm->mmu_lock); 726 727 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 728 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 729 730 if (!kvm_s2_mmu_valid(mmu)) 731 continue; 732 733 if (vmid == get_vmid(mmu->tlb_vttbr)) 734 tlbi_callback(mmu, info); 735 } 736 737 write_unlock(&kvm->mmu_lock); 738 } 739 740 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 741 { 742 struct kvm *kvm = vcpu->kvm; 743 bool nested_stage2_enabled; 744 u64 vttbr, vtcr, hcr; 745 746 lockdep_assert_held_write(&kvm->mmu_lock); 747 748 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 749 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 750 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 751 752 nested_stage2_enabled = hcr & HCR_VM; 753 754 /* Don't consider the CnP bit for the vttbr match */ 755 vttbr &= ~VTTBR_CNP_BIT; 756 757 /* 758 * Two possibilities when looking up a S2 MMU context: 759 * 760 * - either S2 is enabled in the guest, and we need a context that is 761 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 762 * which makes it safe from a TLB conflict perspective (a broken 763 * guest won't be able to generate them), 764 * 765 * - or S2 is disabled, and we need a context that is S2-disabled 766 * and matches the VMID only, as all TLBs are tagged by VMID even 767 * if S2 translation is disabled. 768 */ 769 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 770 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 771 772 if (!kvm_s2_mmu_valid(mmu)) 773 continue; 774 775 if (nested_stage2_enabled && 776 mmu->nested_stage2_enabled && 777 vttbr == mmu->tlb_vttbr && 778 vtcr == mmu->tlb_vtcr) 779 return mmu; 780 781 if (!nested_stage2_enabled && 782 !mmu->nested_stage2_enabled && 783 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 784 return mmu; 785 } 786 return NULL; 787 } 788 789 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 790 { 791 struct kvm *kvm = vcpu->kvm; 792 struct kvm_s2_mmu *s2_mmu; 793 int i; 794 795 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 796 797 s2_mmu = lookup_s2_mmu(vcpu); 798 if (s2_mmu) 799 goto out; 800 801 /* 802 * Make sure we don't always search from the same point, or we 803 * will always reuse a potentially active context, leaving 804 * free contexts unused. 805 */ 806 for (i = kvm->arch.nested_mmus_next; 807 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 808 i++) { 809 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 810 811 if (atomic_read(&s2_mmu->refcnt) == 0) 812 break; 813 } 814 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 815 816 /* Set the scene for the next search */ 817 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 818 819 /* Make sure we don't forget to do the laundry */ 820 if (kvm_s2_mmu_valid(s2_mmu)) { 821 kvm_nested_s2_ptdump_remove_debugfs(s2_mmu); 822 s2_mmu->pending_unmap = true; 823 } 824 825 /* 826 * The virtual VMID (modulo CnP) will be used as a key when matching 827 * an existing kvm_s2_mmu. 828 * 829 * We cache VTCR at allocation time, once and for all. It'd be great 830 * if the guest didn't screw that one up, as this is not very 831 * forgiving... 832 */ 833 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 834 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 835 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 836 837 kvm_nested_s2_ptdump_create_debugfs(s2_mmu); 838 839 out: 840 atomic_inc(&s2_mmu->refcnt); 841 842 /* 843 * Set the vCPU request to perform an unmap, even if the pending unmap 844 * originates from another vCPU. This guarantees that the MMU has been 845 * completely unmapped before any vCPU actually uses it, and allows 846 * multiple vCPUs to lend a hand with completing the unmap. 847 */ 848 if (s2_mmu->pending_unmap) 849 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 850 851 return s2_mmu; 852 } 853 854 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 855 { 856 /* CnP being set denotes an invalid entry */ 857 mmu->tlb_vttbr = VTTBR_CNP_BIT; 858 mmu->nested_stage2_enabled = false; 859 atomic_set(&mmu->refcnt, 0); 860 } 861 862 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 863 { 864 /* 865 * If the vCPU kept its reference on the MMU after the last put, 866 * keep rolling with it. 867 */ 868 if (is_hyp_ctxt(vcpu)) { 869 if (!vcpu->arch.hw_mmu) 870 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 871 } else { 872 if (!vcpu->arch.hw_mmu) { 873 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 874 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 875 } 876 877 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) 878 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 879 } 880 } 881 882 static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) 883 { 884 if (!host_data_test_flag(L1_VNCR_MAPPED)) 885 return; 886 887 BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); 888 BUG_ON(is_hyp_ctxt(vcpu)); 889 890 clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); 891 vcpu->arch.vncr_tlb->cpu = -1; 892 host_data_clear_flag(L1_VNCR_MAPPED); 893 atomic_dec(&vcpu->kvm->arch.vncr_map_count); 894 } 895 896 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 897 { 898 /* Unconditionally drop the VNCR mapping if we have one */ 899 this_cpu_reset_vncr_fixmap(vcpu); 900 901 /* 902 * Keep a reference on the associated stage-2 MMU if the vCPU is 903 * scheduling out and not in WFI emulation, suggesting it is likely to 904 * reuse the MMU sometime soon. 905 */ 906 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 907 return; 908 909 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 910 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 911 912 vcpu->arch.hw_mmu = NULL; 913 } 914 915 /* 916 * Returns non-zero if permission fault is handled by injecting it to the next 917 * level hypervisor. 918 */ 919 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 920 { 921 bool forward_fault = false; 922 923 trans->esr = 0; 924 925 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 926 return 0; 927 928 if (kvm_vcpu_trap_is_iabt(vcpu)) { 929 if (vcpu_mode_priv(vcpu)) 930 forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans); 931 else 932 forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans); 933 } else { 934 bool write_fault = kvm_is_write_fault(vcpu); 935 936 forward_fault = ((write_fault && !trans->writable) || 937 (!write_fault && !trans->readable)); 938 } 939 940 if (forward_fault) 941 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 942 943 return forward_fault; 944 } 945 946 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 947 { 948 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 949 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 950 951 return kvm_inject_nested_sync(vcpu, esr_el2); 952 } 953 954 u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) 955 { 956 enum vcpu_sysreg ttbr_elx; 957 u64 tcr; 958 u16 asid; 959 960 switch (regime) { 961 case TR_EL10: 962 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); 963 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL1 : TTBR0_EL1; 964 break; 965 case TR_EL20: 966 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); 967 ttbr_elx = (tcr & TCR_A1) ? TTBR1_EL2 : TTBR0_EL2; 968 break; 969 default: 970 BUG(); 971 } 972 973 asid = FIELD_GET(TTBRx_EL1_ASID, vcpu_read_sys_reg(vcpu, ttbr_elx)); 974 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || 975 !(tcr & TCR_ASID16)) 976 asid &= GENMASK(7, 0); 977 978 return asid; 979 } 980 981 static void invalidate_vncr(struct vncr_tlb *vt) 982 { 983 vt->valid = false; 984 if (vt->cpu != -1) 985 clear_fixmap(vncr_fixmap(vt->cpu)); 986 } 987 988 /* 989 * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and 990 * either can race against a vcpu not being onlined yet (no pseudo-TLB 991 * allocated). Similarly, the TLB might be invalid. Skip those, as they 992 * obviously don't participate in the invalidation at this stage. 993 */ 994 #define kvm_for_each_vncr_tlb(idx, vcpup, tlbp, kvm) \ 995 kvm_for_each_vcpu(idx, vcpup, kvm) \ 996 if (((tlbp) = vcpup->arch.vncr_tlb) && \ 997 (tlbp)->valid) 998 999 static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) 1000 { 1001 struct kvm_vcpu *vcpu; 1002 struct vncr_tlb *vt; 1003 unsigned long i; 1004 1005 lockdep_assert_held_write(&kvm->mmu_lock); 1006 1007 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1008 return; 1009 1010 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { 1011 u64 ipa_start, ipa_end, ipa_size; 1012 1013 ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, 1014 vt->wr.level)); 1015 ipa_start = vt->wr.pa & ~(ipa_size - 1); 1016 ipa_end = ipa_start + ipa_size; 1017 1018 if (ipa_end <= start || ipa_start >= end) 1019 continue; 1020 1021 invalidate_vncr(vt); 1022 } 1023 } 1024 1025 struct s1e2_tlbi_scope { 1026 enum { 1027 TLBI_ALL, 1028 TLBI_VA, 1029 TLBI_VAA, 1030 TLBI_ASID, 1031 } type; 1032 1033 u16 asid; 1034 u64 va; 1035 u64 size; 1036 }; 1037 1038 static void invalidate_vncr_va(struct kvm *kvm, 1039 struct s1e2_tlbi_scope *scope) 1040 { 1041 struct kvm_vcpu *vcpu; 1042 struct vncr_tlb *vt; 1043 unsigned long i; 1044 1045 lockdep_assert_held_write(&kvm->mmu_lock); 1046 1047 kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { 1048 u64 va_start, va_end, va_size; 1049 1050 va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, 1051 vt->wr.level)); 1052 va_start = vt->gva & ~(va_size - 1); 1053 va_end = va_start + va_size; 1054 1055 switch (scope->type) { 1056 case TLBI_ALL: 1057 break; 1058 1059 case TLBI_VA: 1060 if (va_end <= scope->va || 1061 va_start >= (scope->va + scope->size)) 1062 continue; 1063 if (vt->wr.nG && vt->wr.asid != scope->asid) 1064 continue; 1065 break; 1066 1067 case TLBI_VAA: 1068 if (va_end <= scope->va || 1069 va_start >= (scope->va + scope->size)) 1070 continue; 1071 break; 1072 1073 case TLBI_ASID: 1074 if (!vt->wr.nG || vt->wr.asid != scope->asid) 1075 continue; 1076 break; 1077 } 1078 1079 invalidate_vncr(vt); 1080 } 1081 } 1082 1083 #define tlbi_va_s1_to_va(v) (u64)sign_extend64((v) << 12, 48) 1084 1085 static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, 1086 struct s1e2_tlbi_scope *scope) 1087 { 1088 switch (inst) { 1089 case OP_TLBI_ALLE2: 1090 case OP_TLBI_ALLE2IS: 1091 case OP_TLBI_ALLE2OS: 1092 case OP_TLBI_VMALLE1: 1093 case OP_TLBI_VMALLE1IS: 1094 case OP_TLBI_VMALLE1OS: 1095 case OP_TLBI_ALLE2NXS: 1096 case OP_TLBI_ALLE2ISNXS: 1097 case OP_TLBI_ALLE2OSNXS: 1098 case OP_TLBI_VMALLE1NXS: 1099 case OP_TLBI_VMALLE1ISNXS: 1100 case OP_TLBI_VMALLE1OSNXS: 1101 scope->type = TLBI_ALL; 1102 break; 1103 case OP_TLBI_VAE2: 1104 case OP_TLBI_VAE2IS: 1105 case OP_TLBI_VAE2OS: 1106 case OP_TLBI_VAE1: 1107 case OP_TLBI_VAE1IS: 1108 case OP_TLBI_VAE1OS: 1109 case OP_TLBI_VAE2NXS: 1110 case OP_TLBI_VAE2ISNXS: 1111 case OP_TLBI_VAE2OSNXS: 1112 case OP_TLBI_VAE1NXS: 1113 case OP_TLBI_VAE1ISNXS: 1114 case OP_TLBI_VAE1OSNXS: 1115 case OP_TLBI_VALE2: 1116 case OP_TLBI_VALE2IS: 1117 case OP_TLBI_VALE2OS: 1118 case OP_TLBI_VALE1: 1119 case OP_TLBI_VALE1IS: 1120 case OP_TLBI_VALE1OS: 1121 case OP_TLBI_VALE2NXS: 1122 case OP_TLBI_VALE2ISNXS: 1123 case OP_TLBI_VALE2OSNXS: 1124 case OP_TLBI_VALE1NXS: 1125 case OP_TLBI_VALE1ISNXS: 1126 case OP_TLBI_VALE1OSNXS: 1127 scope->type = TLBI_VA; 1128 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1129 if (!scope->size) 1130 scope->size = SZ_1G; 1131 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1132 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1133 break; 1134 case OP_TLBI_ASIDE1: 1135 case OP_TLBI_ASIDE1IS: 1136 case OP_TLBI_ASIDE1OS: 1137 case OP_TLBI_ASIDE1NXS: 1138 case OP_TLBI_ASIDE1ISNXS: 1139 case OP_TLBI_ASIDE1OSNXS: 1140 scope->type = TLBI_ASID; 1141 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); 1142 break; 1143 case OP_TLBI_VAAE1: 1144 case OP_TLBI_VAAE1IS: 1145 case OP_TLBI_VAAE1OS: 1146 case OP_TLBI_VAAE1NXS: 1147 case OP_TLBI_VAAE1ISNXS: 1148 case OP_TLBI_VAAE1OSNXS: 1149 case OP_TLBI_VAALE1: 1150 case OP_TLBI_VAALE1IS: 1151 case OP_TLBI_VAALE1OS: 1152 case OP_TLBI_VAALE1NXS: 1153 case OP_TLBI_VAALE1ISNXS: 1154 case OP_TLBI_VAALE1OSNXS: 1155 scope->type = TLBI_VAA; 1156 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); 1157 if (!scope->size) 1158 scope->size = SZ_1G; 1159 scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); 1160 break; 1161 case OP_TLBI_RVAE2: 1162 case OP_TLBI_RVAE2IS: 1163 case OP_TLBI_RVAE2OS: 1164 case OP_TLBI_RVAE1: 1165 case OP_TLBI_RVAE1IS: 1166 case OP_TLBI_RVAE1OS: 1167 case OP_TLBI_RVAE2NXS: 1168 case OP_TLBI_RVAE2ISNXS: 1169 case OP_TLBI_RVAE2OSNXS: 1170 case OP_TLBI_RVAE1NXS: 1171 case OP_TLBI_RVAE1ISNXS: 1172 case OP_TLBI_RVAE1OSNXS: 1173 case OP_TLBI_RVALE2: 1174 case OP_TLBI_RVALE2IS: 1175 case OP_TLBI_RVALE2OS: 1176 case OP_TLBI_RVALE1: 1177 case OP_TLBI_RVALE1IS: 1178 case OP_TLBI_RVALE1OS: 1179 case OP_TLBI_RVALE2NXS: 1180 case OP_TLBI_RVALE2ISNXS: 1181 case OP_TLBI_RVALE2OSNXS: 1182 case OP_TLBI_RVALE1NXS: 1183 case OP_TLBI_RVALE1ISNXS: 1184 case OP_TLBI_RVALE1OSNXS: 1185 scope->type = TLBI_VA; 1186 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid); 1187 break; 1188 case OP_TLBI_RVAAE1: 1189 case OP_TLBI_RVAAE1IS: 1190 case OP_TLBI_RVAAE1OS: 1191 case OP_TLBI_RVAAE1NXS: 1192 case OP_TLBI_RVAAE1ISNXS: 1193 case OP_TLBI_RVAAE1OSNXS: 1194 case OP_TLBI_RVAALE1: 1195 case OP_TLBI_RVAALE1IS: 1196 case OP_TLBI_RVAALE1OS: 1197 case OP_TLBI_RVAALE1NXS: 1198 case OP_TLBI_RVAALE1ISNXS: 1199 case OP_TLBI_RVAALE1OSNXS: 1200 scope->type = TLBI_VAA; 1201 scope->va = decode_range_tlbi(val, &scope->size, NULL); 1202 break; 1203 } 1204 } 1205 1206 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) 1207 { 1208 struct s1e2_tlbi_scope scope = {}; 1209 1210 compute_s1_tlbi_range(vcpu, inst, val, &scope); 1211 1212 guard(write_lock)(&vcpu->kvm->mmu_lock); 1213 invalidate_vncr_va(vcpu->kvm, &scope); 1214 } 1215 1216 void kvm_nested_s2_wp(struct kvm *kvm) 1217 { 1218 int i; 1219 1220 lockdep_assert_held_write(&kvm->mmu_lock); 1221 1222 if (!kvm->arch.nested_mmus_size) 1223 return; 1224 1225 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1226 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1227 1228 if (kvm_s2_mmu_valid(mmu)) 1229 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 1230 } 1231 1232 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); 1233 } 1234 1235 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 1236 { 1237 int i; 1238 1239 lockdep_assert_held_write(&kvm->mmu_lock); 1240 1241 if (!kvm->arch.nested_mmus_size) 1242 return; 1243 1244 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1245 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1246 1247 if (kvm_s2_mmu_valid(mmu)) 1248 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 1249 } 1250 1251 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); 1252 } 1253 1254 void kvm_nested_s2_flush(struct kvm *kvm) 1255 { 1256 int i; 1257 1258 lockdep_assert_held_write(&kvm->mmu_lock); 1259 1260 if (!kvm->arch.nested_mmus_size) 1261 return; 1262 1263 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1264 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1265 1266 if (kvm_s2_mmu_valid(mmu)) 1267 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 1268 } 1269 } 1270 1271 void kvm_arch_flush_shadow_all(struct kvm *kvm) 1272 { 1273 int i; 1274 1275 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 1276 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 1277 1278 if (!WARN_ON(atomic_read(&mmu->refcnt))) 1279 kvm_free_stage2_pgd(mmu); 1280 } 1281 kvfree(kvm->arch.nested_mmus); 1282 kvm->arch.nested_mmus = NULL; 1283 kvm->arch.nested_mmus_size = 0; 1284 kvm_uninit_stage2_mmu(kvm); 1285 } 1286 1287 /* 1288 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter: 1289 * 1290 * - We introduce an internal representation of a vcpu-private TLB, 1291 * representing the mapping between the guest VA contained in VNCR_EL2, 1292 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at. 1293 * 1294 * - On translation fault from a nested VNCR access, we create such a TLB. 1295 * If there is no mapping to describe, the guest inherits the fault. 1296 * Crucially, no actual mapping is done at this stage. 1297 * 1298 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above 1299 * TLB exists, we map it in the fixmap for this CPU, and run with it. We 1300 * have to respect the permissions dictated by the guest, but not the 1301 * memory type (FWB is a must). 1302 * 1303 * - Note that we usually don't do a vcpu_load() on the back of a fault 1304 * (unless we are preempted), so the resolution of a translation fault 1305 * must go via a request that will map the VNCR page in the fixmap. 1306 * vcpu_load() might as well use the same mechanism. 1307 * 1308 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was 1309 * mapped, we unmap it. Yes it is that simple. The TLB still exists 1310 * though, and may be reused at a later load. 1311 * 1312 * - On permission fault, we simply forward the fault to the guest's EL2. 1313 * Get out of my way. 1314 * 1315 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that 1316 * intersects with the TLBI request, invalidate it, and unmap the page 1317 * from the fixmap. Because we need to look at all the vcpu-private TLBs, 1318 * this requires some wide-ranging locking to ensure that nothing races 1319 * against it. This may require some refcounting to avoid the search when 1320 * no such TLB is present. 1321 * 1322 * - On MMU notifiers, we must invalidate our TLB in a similar way, but 1323 * looking at the IPA instead. The funny part is that there may not be a 1324 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST 1325 * instructions. 1326 */ 1327 1328 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) 1329 { 1330 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1331 return 0; 1332 1333 if (!vcpu->arch.vncr_tlb) { 1334 struct vncr_tlb *vt = kzalloc_obj(*vcpu->arch.vncr_tlb, 1335 GFP_KERNEL_ACCOUNT); 1336 1337 /* 1338 * Taking the lock on assignment ensures that the TLB is 1339 * seen as initialised when following the pointer (release 1340 * semantics of the unlock), and avoids having acquires on 1341 * each user which already take the lock. 1342 */ 1343 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) 1344 vcpu->arch.vncr_tlb = vt; 1345 } 1346 1347 if (!vcpu->arch.vncr_tlb) 1348 return -ENOMEM; 1349 1350 return 0; 1351 } 1352 1353 static u64 read_vncr_el2(struct kvm_vcpu *vcpu) 1354 { 1355 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); 1356 } 1357 1358 static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) 1359 { 1360 struct kvm_memory_slot *memslot; 1361 bool write_fault, writable; 1362 unsigned long mmu_seq; 1363 struct vncr_tlb *vt; 1364 struct page *page; 1365 u64 va, pfn, gfn; 1366 int ret; 1367 1368 vt = vcpu->arch.vncr_tlb; 1369 1370 /* 1371 * If we're about to walk the EL2 S1 PTs, we must invalidate the 1372 * current TLB, as it could be sampled from another vcpu doing a 1373 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single 1374 * translation, so not much of a choice. 1375 * 1376 * We also prepare the next walk wilst we're at it. 1377 */ 1378 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1379 this_cpu_reset_vncr_fixmap(vcpu); 1380 vt->valid = false; 1381 1382 vt->wi = (struct s1_walk_info) { 1383 .regime = TR_EL20, 1384 .as_el0 = false, 1385 .pan = false, 1386 }; 1387 vt->wr = (struct s1_walk_result){}; 1388 } 1389 1390 guard(srcu)(&vcpu->kvm->srcu); 1391 1392 va = read_vncr_el2(vcpu); 1393 1394 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); 1395 if (ret) 1396 return ret; 1397 1398 write_fault = kvm_is_write_fault(vcpu); 1399 1400 mmu_seq = vcpu->kvm->mmu_invalidate_seq; 1401 smp_rmb(); 1402 1403 gfn = vt->wr.pa >> PAGE_SHIFT; 1404 memslot = gfn_to_memslot(vcpu->kvm, gfn); 1405 if (!memslot) { 1406 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1407 return -EFAULT; 1408 } 1409 1410 *is_gmem = kvm_slot_has_gmem(memslot); 1411 if (!*is_gmem) { 1412 pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, 1413 &writable, &page); 1414 if (is_error_noslot_pfn(pfn)) { 1415 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1416 return -EFAULT; 1417 } 1418 } else { 1419 ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); 1420 if (ret) { 1421 kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, 1422 write_fault, false, false); 1423 return ret; 1424 } 1425 1426 writable = !(memslot->flags & KVM_MEM_READONLY); 1427 } 1428 1429 /* 1430 * FIXME: This check is too restrictive as KVM allows cacheable memory 1431 * attributes for PFNMAP VMAs that have cacheable attributes in host 1432 * stage-1. 1433 */ 1434 if (!pfn_is_map_memory(pfn)) { 1435 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1436 fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); 1437 return -EINVAL; 1438 } 1439 1440 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { 1441 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) { 1442 kvm_release_faultin_page(vcpu->kvm, page, true, false); 1443 return -EAGAIN; 1444 } 1445 1446 vt->gva = va; 1447 vt->hpa = pfn << PAGE_SHIFT; 1448 vt->hpa_writable = writable; 1449 vt->valid = true; 1450 vt->cpu = -1; 1451 1452 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); 1453 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); 1454 } 1455 1456 if (vt->wr.pw && vt->hpa_writable) 1457 mark_page_dirty(vcpu->kvm, gfn); 1458 1459 return 0; 1460 } 1461 1462 static void handle_vncr_perm(struct kvm_vcpu *vcpu) 1463 { 1464 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1465 u64 esr = kvm_vcpu_get_esr(vcpu); 1466 u64 fsc; 1467 1468 /* 1469 * Promote to an external abort if the stage-1 permits writes but the 1470 * HPA is read-only (e.g. RO memslot). 1471 */ 1472 if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) 1473 fsc = ESR_ELx_FSC_EXTABT; 1474 /* 1475 * Otherwise, inject a permission fault using the guest's translation 1476 * level rather than the host's. 1477 */ 1478 else 1479 fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); 1480 1481 esr &= ~ESR_ELx_FSC; 1482 esr |= FIELD_PREP(ESR_ELx_FSC, fsc); 1483 1484 kvm_inject_nested_sync(vcpu, esr); 1485 } 1486 1487 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) 1488 { 1489 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1490 u64 esr = kvm_vcpu_get_esr(vcpu); 1491 bool is_gmem = false; 1492 bool perm; 1493 int ret; 1494 1495 WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); 1496 1497 if (kvm_vcpu_abt_issea(vcpu)) 1498 return kvm_handle_guest_sea(vcpu); 1499 1500 if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { 1501 KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); 1502 return -EIO; 1503 } 1504 1505 ret = kvm_translate_vncr(vcpu, &is_gmem); 1506 switch (ret) { 1507 case -EAGAIN: 1508 /* Let's try again... */ 1509 return 1; 1510 case -ENOMEM: 1511 /* 1512 * For guest_memfd, this indicates that it failed to 1513 * create a folio to back the memory. Inform userspace. 1514 */ 1515 if (is_gmem) 1516 return 0; 1517 /* Otherwise, let's try again... */ 1518 break; 1519 case -EFAULT: 1520 case -EIO: 1521 case -EHWPOISON: 1522 if (is_gmem) 1523 return 0; 1524 fallthrough; 1525 case -EINVAL: 1526 case -ENOENT: 1527 case -EACCES: 1528 /* 1529 * Translation failed, inject the corresponding 1530 * exception back to EL2. 1531 */ 1532 esr &= ~ESR_ELx_FSC; 1533 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); 1534 1535 kvm_inject_nested_sync(vcpu, esr); 1536 break; 1537 case 0: 1538 perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; 1539 if (!perm) 1540 handle_vncr_perm(vcpu); 1541 break; 1542 } 1543 1544 return 1; 1545 } 1546 1547 static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) 1548 { 1549 struct vncr_tlb *vt = vcpu->arch.vncr_tlb; 1550 pgprot_t prot; 1551 1552 guard(preempt)(); 1553 guard(read_lock)(&vcpu->kvm->mmu_lock); 1554 1555 /* 1556 * The request to map VNCR may have raced against some other 1557 * event, such as an interrupt, and may not be valid anymore. 1558 */ 1559 if (is_hyp_ctxt(vcpu)) 1560 return; 1561 1562 /* 1563 * Check that the pseudo-TLB is valid and that VNCR_EL2 still 1564 * contains the expected value. If it doesn't, we simply bail out 1565 * without a mapping -- a transformed MSR/MRS will generate the 1566 * fault and allows us to populate the pseudo-TLB. 1567 */ 1568 if (!vt->valid) 1569 return; 1570 1571 if (read_vncr_el2(vcpu) != vt->gva) 1572 return; 1573 1574 if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) 1575 return; 1576 1577 vt->cpu = smp_processor_id(); 1578 1579 if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) 1580 prot = PAGE_KERNEL; 1581 else if (vt->wr.pr) 1582 prot = PAGE_KERNEL_RO; 1583 else 1584 prot = PAGE_NONE; 1585 1586 /* 1587 * We can't map write-only (or no permission at all) in the kernel, 1588 * but the guest can do it if using POE, so we'll have to turn a 1589 * translation fault into a permission fault at runtime. 1590 * FIXME: WO doesn't work at all, need POE support in the kernel. 1591 */ 1592 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { 1593 __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); 1594 host_data_set_flag(L1_VNCR_MAPPED); 1595 atomic_inc(&vcpu->kvm->arch.vncr_map_count); 1596 } 1597 } 1598 1599 /* 1600 * Our emulated CPU doesn't support all the possible features. For the 1601 * sake of simplicity (and probably mental sanity), wipe out a number 1602 * of feature bits we don't intend to support for the time being. 1603 * This list should get updated as new features get added to the NV 1604 * support, and new extension to the architecture. 1605 */ 1606 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) 1607 { 1608 u64 orig_val = val; 1609 1610 switch (reg) { 1611 case SYS_ID_AA64ISAR1_EL1: 1612 /* Support everything but LS64 and Spec Invalidation */ 1613 val &= ~(ID_AA64ISAR1_EL1_LS64 | 1614 ID_AA64ISAR1_EL1_SPECRES); 1615 break; 1616 1617 case SYS_ID_AA64PFR0_EL1: 1618 /* No RME, AMU, MPAM, or S-EL2 */ 1619 val &= ~(ID_AA64PFR0_EL1_RME | 1620 ID_AA64PFR0_EL1_AMU | 1621 ID_AA64PFR0_EL1_MPAM | 1622 ID_AA64PFR0_EL1_SEL2 | 1623 ID_AA64PFR0_EL1_EL3 | 1624 ID_AA64PFR0_EL1_EL2 | 1625 ID_AA64PFR0_EL1_EL1 | 1626 ID_AA64PFR0_EL1_EL0); 1627 /* 64bit only at any EL */ 1628 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP); 1629 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP); 1630 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP); 1631 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP); 1632 break; 1633 1634 case SYS_ID_AA64PFR1_EL1: 1635 /* Only support BTI, SSBS, CSV2_frac */ 1636 val &= ~(ID_AA64PFR1_EL1_PFAR | 1637 ID_AA64PFR1_EL1_MTEX | 1638 ID_AA64PFR1_EL1_THE | 1639 ID_AA64PFR1_EL1_GCS | 1640 ID_AA64PFR1_EL1_MTE_frac | 1641 ID_AA64PFR1_EL1_NMI | 1642 ID_AA64PFR1_EL1_SME | 1643 ID_AA64PFR1_EL1_RES0 | 1644 ID_AA64PFR1_EL1_MPAM_frac | 1645 ID_AA64PFR1_EL1_MTE); 1646 break; 1647 1648 case SYS_ID_AA64PFR2_EL1: 1649 /* GICv5 is not yet supported for NV */ 1650 val &= ~ID_AA64PFR2_EL1_GCIE; 1651 break; 1652 1653 case SYS_ID_AA64MMFR0_EL1: 1654 /* Hide ExS, Secure Memory */ 1655 val &= ~(ID_AA64MMFR0_EL1_EXS | 1656 ID_AA64MMFR0_EL1_TGRAN4_2 | 1657 ID_AA64MMFR0_EL1_TGRAN16_2 | 1658 ID_AA64MMFR0_EL1_TGRAN64_2 | 1659 ID_AA64MMFR0_EL1_SNSMEM); 1660 1661 /* Hide CNTPOFF if present */ 1662 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP); 1663 1664 /* Disallow unsupported S2 page sizes */ 1665 switch (PAGE_SIZE) { 1666 case SZ_64K: 1667 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI); 1668 fallthrough; 1669 case SZ_16K: 1670 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI); 1671 fallthrough; 1672 case SZ_4K: 1673 /* Support everything */ 1674 break; 1675 } 1676 1677 /* 1678 * Since we can't support a guest S2 page size smaller 1679 * than the host's own page size (due to KVM only 1680 * populating its own S2 using the kernel's page 1681 * size), advertise the limitation using FEAT_GTG. 1682 */ 1683 switch (PAGE_SIZE) { 1684 case SZ_4K: 1685 if (_has_tgran_2(orig_val, 4)) 1686 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP); 1687 fallthrough; 1688 case SZ_16K: 1689 if (_has_tgran_2(orig_val, 16)) 1690 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP); 1691 fallthrough; 1692 case SZ_64K: 1693 if (_has_tgran_2(orig_val, 64)) 1694 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP); 1695 break; 1696 } 1697 1698 /* Cap PARange to 48bits */ 1699 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48); 1700 break; 1701 1702 case SYS_ID_AA64MMFR1_EL1: 1703 val &= ~(ID_AA64MMFR1_EL1_CMOW | 1704 ID_AA64MMFR1_EL1_nTLBPA | 1705 ID_AA64MMFR1_EL1_ETS); 1706 1707 /* FEAT_E2H0 implies no VHE */ 1708 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) 1709 val &= ~ID_AA64MMFR1_EL1_VH; 1710 1711 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF); 1712 break; 1713 1714 case SYS_ID_AA64MMFR2_EL1: 1715 val &= ~(ID_AA64MMFR2_EL1_BBM | 1716 ID_AA64MMFR2_EL1_TTL | 1717 GENMASK_ULL(47, 44) | 1718 ID_AA64MMFR2_EL1_ST | 1719 ID_AA64MMFR2_EL1_CCIDX | 1720 ID_AA64MMFR2_EL1_VARange); 1721 1722 /* Force TTL support */ 1723 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP); 1724 break; 1725 1726 case SYS_ID_AA64MMFR4_EL1: 1727 /* 1728 * You get EITHER 1729 * 1730 * - FEAT_VHE without FEAT_E2H0 1731 * - FEAT_NV limited to FEAT_NV2 1732 * - HCR_EL2.NV1 being RES0 1733 * 1734 * OR 1735 * 1736 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV 1737 * 1738 * Life is too short for anything else. 1739 */ 1740 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { 1741 val = 0; 1742 } else { 1743 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); 1744 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); 1745 } 1746 break; 1747 1748 case SYS_ID_AA64DFR0_EL1: 1749 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 1750 val &= ~(ID_AA64DFR0_EL1_ExtTrcBuff | 1751 ID_AA64DFR0_EL1_BRBE | 1752 ID_AA64DFR0_EL1_MTPMU | 1753 ID_AA64DFR0_EL1_TraceBuffer | 1754 ID_AA64DFR0_EL1_TraceFilt | 1755 ID_AA64DFR0_EL1_PMSVer | 1756 ID_AA64DFR0_EL1_CTX_CMPs | 1757 ID_AA64DFR0_EL1_SEBEP | 1758 ID_AA64DFR0_EL1_PMSS | 1759 ID_AA64DFR0_EL1_TraceVer); 1760 1761 /* 1762 * FEAT_Debugv8p9 requires support for extended breakpoints / 1763 * watchpoints. 1764 */ 1765 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8); 1766 break; 1767 } 1768 1769 return val; 1770 } 1771 1772 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu, 1773 enum vcpu_sysreg sr, u64 v) 1774 { 1775 struct resx resx; 1776 1777 resx = kvm_get_sysreg_resx(vcpu->kvm, sr); 1778 v &= ~resx.res0; 1779 v |= resx.res1; 1780 1781 return v; 1782 } 1783 1784 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, struct resx resx) 1785 { 1786 BUILD_BUG_ON(!__builtin_constant_p(sr)); 1787 BUILD_BUG_ON(sr < __SANITISED_REG_START__); 1788 BUILD_BUG_ON(sr >= NR_SYS_REGS); 1789 1790 kvm_set_sysreg_resx(kvm, sr, resx); 1791 } 1792 1793 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) 1794 { 1795 struct kvm *kvm = vcpu->kvm; 1796 struct resx resx; 1797 1798 lockdep_assert_held(&kvm->arch.config_lock); 1799 1800 if (kvm->arch.sysreg_masks) 1801 goto out; 1802 1803 kvm->arch.sysreg_masks = kzalloc_obj(*(kvm->arch.sysreg_masks), 1804 GFP_KERNEL_ACCOUNT); 1805 if (!kvm->arch.sysreg_masks) 1806 return -ENOMEM; 1807 1808 /* VTTBR_EL2 */ 1809 resx = (typeof(resx)){}; 1810 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 1811 resx.res0 |= GENMASK(63, 56); 1812 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 1813 resx.res0 |= VTTBR_CNP_BIT; 1814 set_sysreg_masks(kvm, VTTBR_EL2, resx); 1815 1816 /* VTCR_EL2 */ 1817 resx = get_reg_fixed_bits(kvm, VTCR_EL2); 1818 set_sysreg_masks(kvm, VTCR_EL2, resx); 1819 1820 /* VMPIDR_EL2 */ 1821 resx.res0 = GENMASK(63, 40) | GENMASK(30, 24); 1822 resx.res1 = BIT(31); 1823 set_sysreg_masks(kvm, VMPIDR_EL2, resx); 1824 1825 /* HCR_EL2 */ 1826 resx = get_reg_fixed_bits(kvm, HCR_EL2); 1827 set_sysreg_masks(kvm, HCR_EL2, resx); 1828 1829 /* HCRX_EL2 */ 1830 resx = get_reg_fixed_bits(kvm, HCRX_EL2); 1831 set_sysreg_masks(kvm, HCRX_EL2, resx); 1832 1833 /* HFG[RW]TR_EL2 */ 1834 resx = get_reg_fixed_bits(kvm, HFGRTR_EL2); 1835 set_sysreg_masks(kvm, HFGRTR_EL2, resx); 1836 resx = get_reg_fixed_bits(kvm, HFGWTR_EL2); 1837 set_sysreg_masks(kvm, HFGWTR_EL2, resx); 1838 1839 /* HDFG[RW]TR_EL2 */ 1840 resx = get_reg_fixed_bits(kvm, HDFGRTR_EL2); 1841 set_sysreg_masks(kvm, HDFGRTR_EL2, resx); 1842 resx = get_reg_fixed_bits(kvm, HDFGWTR_EL2); 1843 set_sysreg_masks(kvm, HDFGWTR_EL2, resx); 1844 1845 /* HFGITR_EL2 */ 1846 resx = get_reg_fixed_bits(kvm, HFGITR_EL2); 1847 set_sysreg_masks(kvm, HFGITR_EL2, resx); 1848 1849 /* HAFGRTR_EL2 - not a lot to see here */ 1850 resx = get_reg_fixed_bits(kvm, HAFGRTR_EL2); 1851 set_sysreg_masks(kvm, HAFGRTR_EL2, resx); 1852 1853 /* HFG[RW]TR2_EL2 */ 1854 resx = get_reg_fixed_bits(kvm, HFGRTR2_EL2); 1855 set_sysreg_masks(kvm, HFGRTR2_EL2, resx); 1856 resx = get_reg_fixed_bits(kvm, HFGWTR2_EL2); 1857 set_sysreg_masks(kvm, HFGWTR2_EL2, resx); 1858 1859 /* HDFG[RW]TR2_EL2 */ 1860 resx = get_reg_fixed_bits(kvm, HDFGRTR2_EL2); 1861 set_sysreg_masks(kvm, HDFGRTR2_EL2, resx); 1862 resx = get_reg_fixed_bits(kvm, HDFGWTR2_EL2); 1863 set_sysreg_masks(kvm, HDFGWTR2_EL2, resx); 1864 1865 /* HFGITR2_EL2 */ 1866 resx = get_reg_fixed_bits(kvm, HFGITR2_EL2); 1867 set_sysreg_masks(kvm, HFGITR2_EL2, resx); 1868 1869 /* TCR2_EL2 */ 1870 resx = get_reg_fixed_bits(kvm, TCR2_EL2); 1871 set_sysreg_masks(kvm, TCR2_EL2, resx); 1872 1873 /* SCTLR_EL1 */ 1874 resx = get_reg_fixed_bits(kvm, SCTLR_EL1); 1875 set_sysreg_masks(kvm, SCTLR_EL1, resx); 1876 1877 /* SCTLR_EL2 */ 1878 resx = get_reg_fixed_bits(kvm, SCTLR_EL2); 1879 set_sysreg_masks(kvm, SCTLR_EL2, resx); 1880 1881 /* SCTLR2_ELx */ 1882 resx = get_reg_fixed_bits(kvm, SCTLR2_EL1); 1883 set_sysreg_masks(kvm, SCTLR2_EL1, resx); 1884 resx = get_reg_fixed_bits(kvm, SCTLR2_EL2); 1885 set_sysreg_masks(kvm, SCTLR2_EL2, resx); 1886 1887 /* MDCR_EL2 */ 1888 resx = get_reg_fixed_bits(kvm, MDCR_EL2); 1889 set_sysreg_masks(kvm, MDCR_EL2, resx); 1890 1891 /* CNTHCTL_EL2 */ 1892 resx.res0 = GENMASK(63, 20); 1893 resx.res1 = 0; 1894 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP)) 1895 resx.res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK; 1896 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) { 1897 resx.res0 |= CNTHCTL_ECV; 1898 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP)) 1899 resx.res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT | 1900 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT); 1901 } 1902 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1903 resx.res0 |= GENMASK(11, 8); 1904 set_sysreg_masks(kvm, CNTHCTL_EL2, resx); 1905 1906 /* ICH_HCR_EL2 */ 1907 resx.res0 = ICH_HCR_EL2_RES0; 1908 resx.res1 = ICH_HCR_EL2_RES1; 1909 if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS)) 1910 resx.res0 |= ICH_HCR_EL2_TDIR; 1911 /* No GICv4 is presented to the guest */ 1912 resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; 1913 set_sysreg_masks(kvm, ICH_HCR_EL2, resx); 1914 1915 /* VNCR_EL2 */ 1916 resx.res0 = VNCR_EL2_RES0; 1917 resx.res1 = VNCR_EL2_RES1; 1918 set_sysreg_masks(kvm, VNCR_EL2, resx); 1919 1920 /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */ 1921 resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4); 1922 resx.res1 = ZCR_ELx_RES1; 1923 set_sysreg_masks(kvm, ZCR_EL2, resx); 1924 1925 out: 1926 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) 1927 __vcpu_rmw_sys_reg(vcpu, sr, |=, 0); 1928 1929 return 0; 1930 } 1931 1932 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 1933 { 1934 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 1935 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 1936 1937 write_lock(&vcpu->kvm->mmu_lock); 1938 if (mmu->pending_unmap) { 1939 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 1940 mmu->pending_unmap = false; 1941 } 1942 write_unlock(&vcpu->kvm->mmu_lock); 1943 } 1944 1945 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu)) 1946 kvm_map_l1_vncr(vcpu); 1947 1948 /* Must be last, as may switch context! */ 1949 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) 1950 kvm_inject_nested_irq(vcpu); 1951 } 1952 1953 /* 1954 * One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor 1955 * can write to HCR_EL2 behind our back, potentially changing the exception 1956 * routing / masking for even the host context. 1957 * 1958 * What follows is some slop to (1) react to exception routing / masking and (2) 1959 * preserve the pending SError state across translation regimes. 1960 */ 1961 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) 1962 { 1963 if (!vcpu_has_nv(vcpu)) 1964 return; 1965 1966 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 1967 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 1968 } 1969 1970 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) 1971 { 1972 unsigned long *hcr = vcpu_hcr(vcpu); 1973 1974 if (!vcpu_has_nv(vcpu)) 1975 return; 1976 1977 /* 1978 * We previously decided that an SError was deliverable to the guest. 1979 * Reap the pending state from HCR_EL2 and... 1980 */ 1981 if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr))) 1982 vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); 1983 1984 /* 1985 * Re-attempt SError injection in case the deliverability has changed, 1986 * which is necessary to faithfully emulate WFI the case of a pending 1987 * SError being a wakeup condition. 1988 */ 1989 if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) 1990 kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); 1991 } 1992 1993 /* 1994 * KVM unconditionally sets most of these traps anyway but use an allowlist 1995 * to document the guest hypervisor traps that may take precedence and guard 1996 * against future changes to the non-nested trap configuration. 1997 */ 1998 #define NV_MDCR_GUEST_INCLUDE (MDCR_EL2_TDE | \ 1999 MDCR_EL2_TDA | \ 2000 MDCR_EL2_TDRA | \ 2001 MDCR_EL2_TTRF | \ 2002 MDCR_EL2_TPMS | \ 2003 MDCR_EL2_TPM | \ 2004 MDCR_EL2_TPMCR | \ 2005 MDCR_EL2_TDCC | \ 2006 MDCR_EL2_TDOSA) 2007 2008 void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) 2009 { 2010 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2); 2011 2012 if (is_nested_ctxt(vcpu)) 2013 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE); 2014 /* 2015 * In yet another example where FEAT_NV2 is fscking broken, accesses 2016 * to MDSCR_EL1 are redirected to the VNCR despite having an effect 2017 * at EL2. Use a big hammer to apply sanity. 2018 * 2019 * Unless of course we have FEAT_FGT, in which case we can precisely 2020 * trap MDSCR_EL1. 2021 */ 2022 else if (!cpus_have_final_cap(ARM64_HAS_FGT)) 2023 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; 2024 } 2025