1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/kvm_arm.h> 12 #include <asm/kvm_emulate.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/kvm_nested.h> 15 #include <asm/sysreg.h> 16 17 #include "sys_regs.h" 18 19 /* Protection against the sysreg repainting madness... */ 20 #define NV_FTR(r, f) ID_AA64##r##_EL1_##f 21 22 /* 23 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 24 * memory usage and potential number of different sets of S2 PTs in 25 * the guests. Running out of S2 MMUs only affects performance (we 26 * will invalidate them more often). 27 */ 28 #define S2_MMU_PER_VCPU 2 29 30 void kvm_init_nested(struct kvm *kvm) 31 { 32 kvm->arch.nested_mmus = NULL; 33 kvm->arch.nested_mmus_size = 0; 34 } 35 36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 37 { 38 /* 39 * We only initialise the IPA range on the canonical MMU, which 40 * defines the contract between KVM and userspace on where the 41 * "hardware" is in the IPA space. This affects the validity of MMIO 42 * exits forwarded to userspace, for example. 43 * 44 * For nested S2s, we use the PARange as exposed to the guest, as it 45 * is allowed to use it at will to expose whatever memory map it 46 * wants to its own guests as it would be on real HW. 47 */ 48 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 49 } 50 51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 52 { 53 struct kvm *kvm = vcpu->kvm; 54 struct kvm_s2_mmu *tmp; 55 int num_mmus, ret = 0; 56 57 /* 58 * Let's treat memory allocation failures as benign: If we fail to 59 * allocate anything, return an error and keep the allocated array 60 * alive. Userspace may try to recover by intializing the vcpu 61 * again, and there is no reason to affect the whole VM for this. 62 */ 63 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 64 tmp = kvrealloc(kvm->arch.nested_mmus, 65 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus), 66 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 67 if (!tmp) 68 return -ENOMEM; 69 70 swap(kvm->arch.nested_mmus, tmp); 71 72 /* 73 * If we went through a realocation, adjust the MMU back-pointers in 74 * the previously initialised kvm_pgtable structures. 75 */ 76 if (kvm->arch.nested_mmus != tmp) 77 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 78 kvm->arch.nested_mmus[i].pgt->mmu = &kvm->arch.nested_mmus[i]; 79 80 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 81 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); 82 83 if (ret) { 84 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 85 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); 86 87 return ret; 88 } 89 90 kvm->arch.nested_mmus_size = num_mmus; 91 92 return 0; 93 } 94 95 struct s2_walk_info { 96 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data); 97 void *data; 98 u64 baddr; 99 unsigned int max_oa_bits; 100 unsigned int pgshift; 101 unsigned int sl; 102 unsigned int t0sz; 103 bool be; 104 }; 105 106 static u32 compute_fsc(int level, u32 fsc) 107 { 108 return fsc | (level & 0x3); 109 } 110 111 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 112 { 113 u32 esr; 114 115 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 116 esr |= compute_fsc(level, fsc); 117 return esr; 118 } 119 120 static int get_ia_size(struct s2_walk_info *wi) 121 { 122 return 64 - wi->t0sz; 123 } 124 125 static int check_base_s2_limits(struct s2_walk_info *wi, 126 int level, int input_size, int stride) 127 { 128 int start_size, ia_size; 129 130 ia_size = get_ia_size(wi); 131 132 /* Check translation limits */ 133 switch (BIT(wi->pgshift)) { 134 case SZ_64K: 135 if (level == 0 || (level == 1 && ia_size <= 42)) 136 return -EFAULT; 137 break; 138 case SZ_16K: 139 if (level == 0 || (level == 1 && ia_size <= 40)) 140 return -EFAULT; 141 break; 142 case SZ_4K: 143 if (level < 0 || (level == 0 && ia_size <= 42)) 144 return -EFAULT; 145 break; 146 } 147 148 /* Check input size limits */ 149 if (input_size > ia_size) 150 return -EFAULT; 151 152 /* Check number of entries in starting level table */ 153 start_size = input_size - ((3 - level) * stride + wi->pgshift); 154 if (start_size < 1 || start_size > stride + 4) 155 return -EFAULT; 156 157 return 0; 158 } 159 160 /* Check if output is within boundaries */ 161 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 162 { 163 unsigned int output_size = wi->max_oa_bits; 164 165 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 166 return -1; 167 168 return 0; 169 } 170 171 /* 172 * This is essentially a C-version of the pseudo code from the ARM ARM 173 * AArch64.TranslationTableWalk function. I strongly recommend looking at 174 * that pseudocode in trying to understand this. 175 * 176 * Must be called with the kvm->srcu read lock held 177 */ 178 static int walk_nested_s2_pgd(phys_addr_t ipa, 179 struct s2_walk_info *wi, struct kvm_s2_trans *out) 180 { 181 int first_block_level, level, stride, input_size, base_lower_bound; 182 phys_addr_t base_addr; 183 unsigned int addr_top, addr_bottom; 184 u64 desc; /* page table entry */ 185 int ret; 186 phys_addr_t paddr; 187 188 switch (BIT(wi->pgshift)) { 189 default: 190 case SZ_64K: 191 case SZ_16K: 192 level = 3 - wi->sl; 193 first_block_level = 2; 194 break; 195 case SZ_4K: 196 level = 2 - wi->sl; 197 first_block_level = 1; 198 break; 199 } 200 201 stride = wi->pgshift - 3; 202 input_size = get_ia_size(wi); 203 if (input_size > 48 || input_size < 25) 204 return -EFAULT; 205 206 ret = check_base_s2_limits(wi, level, input_size, stride); 207 if (WARN_ON(ret)) 208 return ret; 209 210 base_lower_bound = 3 + input_size - ((3 - level) * stride + 211 wi->pgshift); 212 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 213 214 if (check_output_size(wi, base_addr)) { 215 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 216 return 1; 217 } 218 219 addr_top = input_size - 1; 220 221 while (1) { 222 phys_addr_t index; 223 224 addr_bottom = (3 - level) * stride + wi->pgshift; 225 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 226 >> (addr_bottom - 3); 227 228 paddr = base_addr | index; 229 ret = wi->read_desc(paddr, &desc, wi->data); 230 if (ret < 0) 231 return ret; 232 233 /* 234 * Handle reversedescriptors if endianness differs between the 235 * host and the guest hypervisor. 236 */ 237 if (wi->be) 238 desc = be64_to_cpu((__force __be64)desc); 239 else 240 desc = le64_to_cpu((__force __le64)desc); 241 242 /* Check for valid descriptor at this point */ 243 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) { 244 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 245 out->desc = desc; 246 return 1; 247 } 248 249 /* We're at the final level or block translation level */ 250 if ((desc & 3) == 1 || level == 3) 251 break; 252 253 if (check_output_size(wi, desc)) { 254 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 255 out->desc = desc; 256 return 1; 257 } 258 259 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 260 261 level += 1; 262 addr_top = addr_bottom - 1; 263 } 264 265 if (level < first_block_level) { 266 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 267 out->desc = desc; 268 return 1; 269 } 270 271 if (check_output_size(wi, desc)) { 272 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 273 out->desc = desc; 274 return 1; 275 } 276 277 if (!(desc & BIT(10))) { 278 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 279 out->desc = desc; 280 return 1; 281 } 282 283 addr_bottom += contiguous_bit_shift(desc, wi, level); 284 285 /* Calculate and return the result */ 286 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 287 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 288 out->output = paddr; 289 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 290 out->readable = desc & (0b01 << 6); 291 out->writable = desc & (0b10 << 6); 292 out->level = level; 293 out->desc = desc; 294 return 0; 295 } 296 297 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data) 298 { 299 struct kvm_vcpu *vcpu = data; 300 301 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc)); 302 } 303 304 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi) 305 { 306 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK; 307 308 switch (vtcr & VTCR_EL2_TG0_MASK) { 309 case VTCR_EL2_TG0_4K: 310 wi->pgshift = 12; break; 311 case VTCR_EL2_TG0_16K: 312 wi->pgshift = 14; break; 313 case VTCR_EL2_TG0_64K: 314 default: /* IMPDEF: treat any other value as 64k */ 315 wi->pgshift = 16; break; 316 } 317 318 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 319 /* Global limit for now, should eventually be per-VM */ 320 wi->max_oa_bits = min(get_kvm_ipa_limit(), 321 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr))); 322 } 323 324 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 325 struct kvm_s2_trans *result) 326 { 327 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 328 struct s2_walk_info wi; 329 int ret; 330 331 result->esr = 0; 332 333 if (!vcpu_has_nv(vcpu)) 334 return 0; 335 336 wi.read_desc = read_guest_s2_desc; 337 wi.data = vcpu; 338 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 339 340 vtcr_to_walk_info(vtcr, &wi); 341 342 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 343 344 ret = walk_nested_s2_pgd(gipa, &wi, result); 345 if (ret) 346 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 347 348 return ret; 349 } 350 351 static unsigned int ttl_to_size(u8 ttl) 352 { 353 int level = ttl & 3; 354 int gran = (ttl >> 2) & 3; 355 unsigned int max_size = 0; 356 357 switch (gran) { 358 case TLBI_TTL_TG_4K: 359 switch (level) { 360 case 0: 361 break; 362 case 1: 363 max_size = SZ_1G; 364 break; 365 case 2: 366 max_size = SZ_2M; 367 break; 368 case 3: 369 max_size = SZ_4K; 370 break; 371 } 372 break; 373 case TLBI_TTL_TG_16K: 374 switch (level) { 375 case 0: 376 case 1: 377 break; 378 case 2: 379 max_size = SZ_32M; 380 break; 381 case 3: 382 max_size = SZ_16K; 383 break; 384 } 385 break; 386 case TLBI_TTL_TG_64K: 387 switch (level) { 388 case 0: 389 case 1: 390 /* No 52bit IPA support */ 391 break; 392 case 2: 393 max_size = SZ_512M; 394 break; 395 case 3: 396 max_size = SZ_64K; 397 break; 398 } 399 break; 400 default: /* No size information */ 401 break; 402 } 403 404 return max_size; 405 } 406 407 /* 408 * Compute the equivalent of the TTL field by parsing the shadow PT. The 409 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 410 * retrieved from first entry carrying the level as a tag. 411 */ 412 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 413 { 414 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr; 415 kvm_pte_t pte; 416 u8 ttl, level; 417 418 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 419 420 switch (vtcr & VTCR_EL2_TG0_MASK) { 421 case VTCR_EL2_TG0_4K: 422 ttl = (TLBI_TTL_TG_4K << 2); 423 break; 424 case VTCR_EL2_TG0_16K: 425 ttl = (TLBI_TTL_TG_16K << 2); 426 break; 427 case VTCR_EL2_TG0_64K: 428 default: /* IMPDEF: treat any other value as 64k */ 429 ttl = (TLBI_TTL_TG_64K << 2); 430 break; 431 } 432 433 tmp = addr; 434 435 again: 436 /* Iteratively compute the block sizes for a particular granule size */ 437 switch (vtcr & VTCR_EL2_TG0_MASK) { 438 case VTCR_EL2_TG0_4K: 439 if (sz < SZ_4K) sz = SZ_4K; 440 else if (sz < SZ_2M) sz = SZ_2M; 441 else if (sz < SZ_1G) sz = SZ_1G; 442 else sz = 0; 443 break; 444 case VTCR_EL2_TG0_16K: 445 if (sz < SZ_16K) sz = SZ_16K; 446 else if (sz < SZ_32M) sz = SZ_32M; 447 else sz = 0; 448 break; 449 case VTCR_EL2_TG0_64K: 450 default: /* IMPDEF: treat any other value as 64k */ 451 if (sz < SZ_64K) sz = SZ_64K; 452 else if (sz < SZ_512M) sz = SZ_512M; 453 else sz = 0; 454 break; 455 } 456 457 if (sz == 0) 458 return 0; 459 460 tmp &= ~(sz - 1); 461 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 462 goto again; 463 if (!(pte & PTE_VALID)) 464 goto again; 465 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 466 if (!level) 467 goto again; 468 469 ttl |= level; 470 471 /* 472 * We now have found some level information in the shadow S2. Check 473 * that the resulting range is actually including the original IPA. 474 */ 475 sz = ttl_to_size(ttl); 476 if (addr < (tmp + sz)) 477 return ttl; 478 479 return 0; 480 } 481 482 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 483 { 484 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 485 unsigned long max_size; 486 u8 ttl; 487 488 ttl = FIELD_GET(TLBI_TTL_MASK, val); 489 490 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 491 /* No TTL, check the shadow S2 for a hint */ 492 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 493 ttl = get_guest_mapping_ttl(mmu, addr); 494 } 495 496 max_size = ttl_to_size(ttl); 497 498 if (!max_size) { 499 /* Compute the maximum extent of the invalidation */ 500 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) { 501 case VTCR_EL2_TG0_4K: 502 max_size = SZ_1G; 503 break; 504 case VTCR_EL2_TG0_16K: 505 max_size = SZ_32M; 506 break; 507 case VTCR_EL2_TG0_64K: 508 default: /* IMPDEF: treat any other value as 64k */ 509 /* 510 * No, we do not support 52bit IPA in nested yet. Once 511 * we do, this should be 4TB. 512 */ 513 max_size = SZ_512M; 514 break; 515 } 516 } 517 518 WARN_ON(!max_size); 519 return max_size; 520 } 521 522 /* 523 * We can have multiple *different* MMU contexts with the same VMID: 524 * 525 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 526 * 527 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 528 * VBBTR_EL2.BADDR address 529 * 530 * - A combination of the above... 531 * 532 * We can always identify which MMU context to pick at run-time. However, 533 * TLB invalidation involving a VMID must take action on all the TLBs using 534 * this particular VMID. This translates into applying the same invalidation 535 * operation to all the contexts that are using this VMID. Moar phun! 536 */ 537 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 538 const union tlbi_info *info, 539 void (*tlbi_callback)(struct kvm_s2_mmu *, 540 const union tlbi_info *)) 541 { 542 write_lock(&kvm->mmu_lock); 543 544 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 545 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 546 547 if (!kvm_s2_mmu_valid(mmu)) 548 continue; 549 550 if (vmid == get_vmid(mmu->tlb_vttbr)) 551 tlbi_callback(mmu, info); 552 } 553 554 write_unlock(&kvm->mmu_lock); 555 } 556 557 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 558 { 559 struct kvm *kvm = vcpu->kvm; 560 bool nested_stage2_enabled; 561 u64 vttbr, vtcr, hcr; 562 563 lockdep_assert_held_write(&kvm->mmu_lock); 564 565 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 566 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 567 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 568 569 nested_stage2_enabled = hcr & HCR_VM; 570 571 /* Don't consider the CnP bit for the vttbr match */ 572 vttbr &= ~VTTBR_CNP_BIT; 573 574 /* 575 * Two possibilities when looking up a S2 MMU context: 576 * 577 * - either S2 is enabled in the guest, and we need a context that is 578 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 579 * which makes it safe from a TLB conflict perspective (a broken 580 * guest won't be able to generate them), 581 * 582 * - or S2 is disabled, and we need a context that is S2-disabled 583 * and matches the VMID only, as all TLBs are tagged by VMID even 584 * if S2 translation is disabled. 585 */ 586 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 587 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 588 589 if (!kvm_s2_mmu_valid(mmu)) 590 continue; 591 592 if (nested_stage2_enabled && 593 mmu->nested_stage2_enabled && 594 vttbr == mmu->tlb_vttbr && 595 vtcr == mmu->tlb_vtcr) 596 return mmu; 597 598 if (!nested_stage2_enabled && 599 !mmu->nested_stage2_enabled && 600 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 601 return mmu; 602 } 603 return NULL; 604 } 605 606 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 607 { 608 struct kvm *kvm = vcpu->kvm; 609 struct kvm_s2_mmu *s2_mmu; 610 int i; 611 612 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 613 614 s2_mmu = lookup_s2_mmu(vcpu); 615 if (s2_mmu) 616 goto out; 617 618 /* 619 * Make sure we don't always search from the same point, or we 620 * will always reuse a potentially active context, leaving 621 * free contexts unused. 622 */ 623 for (i = kvm->arch.nested_mmus_next; 624 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 625 i++) { 626 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 627 628 if (atomic_read(&s2_mmu->refcnt) == 0) 629 break; 630 } 631 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 632 633 /* Set the scene for the next search */ 634 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 635 636 /* Make sure we don't forget to do the laundry */ 637 if (kvm_s2_mmu_valid(s2_mmu)) 638 s2_mmu->pending_unmap = true; 639 640 /* 641 * The virtual VMID (modulo CnP) will be used as a key when matching 642 * an existing kvm_s2_mmu. 643 * 644 * We cache VTCR at allocation time, once and for all. It'd be great 645 * if the guest didn't screw that one up, as this is not very 646 * forgiving... 647 */ 648 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 649 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 650 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 651 652 out: 653 atomic_inc(&s2_mmu->refcnt); 654 655 /* 656 * Set the vCPU request to perform an unmap, even if the pending unmap 657 * originates from another vCPU. This guarantees that the MMU has been 658 * completely unmapped before any vCPU actually uses it, and allows 659 * multiple vCPUs to lend a hand with completing the unmap. 660 */ 661 if (s2_mmu->pending_unmap) 662 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 663 664 return s2_mmu; 665 } 666 667 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 668 { 669 /* CnP being set denotes an invalid entry */ 670 mmu->tlb_vttbr = VTTBR_CNP_BIT; 671 mmu->nested_stage2_enabled = false; 672 atomic_set(&mmu->refcnt, 0); 673 } 674 675 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 676 { 677 /* 678 * The vCPU kept its reference on the MMU after the last put, keep 679 * rolling with it. 680 */ 681 if (vcpu->arch.hw_mmu) 682 return; 683 684 if (is_hyp_ctxt(vcpu)) { 685 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 686 } else { 687 write_lock(&vcpu->kvm->mmu_lock); 688 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 689 write_unlock(&vcpu->kvm->mmu_lock); 690 } 691 } 692 693 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 694 { 695 /* 696 * Keep a reference on the associated stage-2 MMU if the vCPU is 697 * scheduling out and not in WFI emulation, suggesting it is likely to 698 * reuse the MMU sometime soon. 699 */ 700 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 701 return; 702 703 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 704 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 705 706 vcpu->arch.hw_mmu = NULL; 707 } 708 709 /* 710 * Returns non-zero if permission fault is handled by injecting it to the next 711 * level hypervisor. 712 */ 713 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 714 { 715 bool forward_fault = false; 716 717 trans->esr = 0; 718 719 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 720 return 0; 721 722 if (kvm_vcpu_trap_is_iabt(vcpu)) { 723 forward_fault = !kvm_s2_trans_executable(trans); 724 } else { 725 bool write_fault = kvm_is_write_fault(vcpu); 726 727 forward_fault = ((write_fault && !trans->writable) || 728 (!write_fault && !trans->readable)); 729 } 730 731 if (forward_fault) 732 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 733 734 return forward_fault; 735 } 736 737 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 738 { 739 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 740 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 741 742 return kvm_inject_nested_sync(vcpu, esr_el2); 743 } 744 745 void kvm_nested_s2_wp(struct kvm *kvm) 746 { 747 int i; 748 749 lockdep_assert_held_write(&kvm->mmu_lock); 750 751 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 752 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 753 754 if (kvm_s2_mmu_valid(mmu)) 755 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 756 } 757 } 758 759 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 760 { 761 int i; 762 763 lockdep_assert_held_write(&kvm->mmu_lock); 764 765 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 766 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 767 768 if (kvm_s2_mmu_valid(mmu)) 769 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 770 } 771 } 772 773 void kvm_nested_s2_flush(struct kvm *kvm) 774 { 775 int i; 776 777 lockdep_assert_held_write(&kvm->mmu_lock); 778 779 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 780 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 781 782 if (kvm_s2_mmu_valid(mmu)) 783 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 784 } 785 } 786 787 void kvm_arch_flush_shadow_all(struct kvm *kvm) 788 { 789 int i; 790 791 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 792 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 793 794 if (!WARN_ON(atomic_read(&mmu->refcnt))) 795 kvm_free_stage2_pgd(mmu); 796 } 797 kvfree(kvm->arch.nested_mmus); 798 kvm->arch.nested_mmus = NULL; 799 kvm->arch.nested_mmus_size = 0; 800 kvm_uninit_stage2_mmu(kvm); 801 } 802 803 /* 804 * Our emulated CPU doesn't support all the possible features. For the 805 * sake of simplicity (and probably mental sanity), wipe out a number 806 * of feature bits we don't intend to support for the time being. 807 * This list should get updated as new features get added to the NV 808 * support, and new extension to the architecture. 809 */ 810 static void limit_nv_id_regs(struct kvm *kvm) 811 { 812 u64 val, tmp; 813 814 /* Support everything but TME */ 815 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1); 816 val &= ~NV_FTR(ISAR0, TME); 817 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR0_EL1, val); 818 819 /* Support everything but Spec Invalidation and LS64 */ 820 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1); 821 val &= ~(NV_FTR(ISAR1, LS64) | 822 NV_FTR(ISAR1, SPECRES)); 823 kvm_set_vm_id_reg(kvm, SYS_ID_AA64ISAR1_EL1, val); 824 825 /* No AMU, MPAM, S-EL2, or RAS */ 826 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1); 827 val &= ~(GENMASK_ULL(55, 52) | 828 NV_FTR(PFR0, AMU) | 829 NV_FTR(PFR0, MPAM) | 830 NV_FTR(PFR0, SEL2) | 831 NV_FTR(PFR0, RAS) | 832 NV_FTR(PFR0, EL3) | 833 NV_FTR(PFR0, EL2) | 834 NV_FTR(PFR0, EL1) | 835 NV_FTR(PFR0, EL0)); 836 /* 64bit only at any EL */ 837 val |= FIELD_PREP(NV_FTR(PFR0, EL0), 0b0001); 838 val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001); 839 val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001); 840 val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001); 841 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val); 842 843 /* Only support BTI, SSBS, CSV2_frac */ 844 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1); 845 val &= (NV_FTR(PFR1, BT) | 846 NV_FTR(PFR1, SSBS) | 847 NV_FTR(PFR1, CSV2_frac)); 848 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR1_EL1, val); 849 850 /* Hide ECV, ExS, Secure Memory */ 851 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 852 val &= ~(NV_FTR(MMFR0, ECV) | 853 NV_FTR(MMFR0, EXS) | 854 NV_FTR(MMFR0, TGRAN4_2) | 855 NV_FTR(MMFR0, TGRAN16_2) | 856 NV_FTR(MMFR0, TGRAN64_2) | 857 NV_FTR(MMFR0, SNSMEM)); 858 859 /* Disallow unsupported S2 page sizes */ 860 switch (PAGE_SIZE) { 861 case SZ_64K: 862 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001); 863 fallthrough; 864 case SZ_16K: 865 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001); 866 fallthrough; 867 case SZ_4K: 868 /* Support everything */ 869 break; 870 } 871 /* 872 * Since we can't support a guest S2 page size smaller than 873 * the host's own page size (due to KVM only populating its 874 * own S2 using the kernel's page size), advertise the 875 * limitation using FEAT_GTG. 876 */ 877 switch (PAGE_SIZE) { 878 case SZ_4K: 879 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010); 880 fallthrough; 881 case SZ_16K: 882 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010); 883 fallthrough; 884 case SZ_64K: 885 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010); 886 break; 887 } 888 /* Cap PARange to 48bits */ 889 tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val); 890 if (tmp > 0b0101) { 891 val &= ~NV_FTR(MMFR0, PARANGE); 892 val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101); 893 } 894 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1, val); 895 896 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1); 897 val &= (NV_FTR(MMFR1, HCX) | 898 NV_FTR(MMFR1, PAN) | 899 NV_FTR(MMFR1, LO) | 900 NV_FTR(MMFR1, HPDS) | 901 NV_FTR(MMFR1, VH) | 902 NV_FTR(MMFR1, VMIDBits)); 903 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR1_EL1, val); 904 905 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1); 906 val &= ~(NV_FTR(MMFR2, BBM) | 907 NV_FTR(MMFR2, TTL) | 908 GENMASK_ULL(47, 44) | 909 NV_FTR(MMFR2, ST) | 910 NV_FTR(MMFR2, CCIDX) | 911 NV_FTR(MMFR2, VARange)); 912 913 /* Force TTL support */ 914 val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001); 915 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR2_EL1, val); 916 917 val = 0; 918 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 919 val |= FIELD_PREP(NV_FTR(MMFR4, E2H0), 920 ID_AA64MMFR4_EL1_E2H0_NI_NV1); 921 kvm_set_vm_id_reg(kvm, SYS_ID_AA64MMFR4_EL1, val); 922 923 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 924 val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1); 925 val &= (NV_FTR(DFR0, PMUVer) | 926 NV_FTR(DFR0, WRPs) | 927 NV_FTR(DFR0, BRPs) | 928 NV_FTR(DFR0, DebugVer) | 929 NV_FTR(DFR0, HPMN0)); 930 931 /* Cap Debug to ARMv8.1 */ 932 tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val); 933 if (tmp > 0b0111) { 934 val &= ~NV_FTR(DFR0, DebugVer); 935 val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111); 936 } 937 kvm_set_vm_id_reg(kvm, SYS_ID_AA64DFR0_EL1, val); 938 } 939 940 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu, 941 enum vcpu_sysreg sr, u64 v) 942 { 943 struct kvm_sysreg_masks *masks; 944 945 masks = vcpu->kvm->arch.sysreg_masks; 946 947 if (masks) { 948 sr -= __SANITISED_REG_START__; 949 950 v &= ~masks->mask[sr].res0; 951 v |= masks->mask[sr].res1; 952 } 953 954 return v; 955 } 956 957 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1) 958 { 959 int i = sr - __SANITISED_REG_START__; 960 961 BUILD_BUG_ON(!__builtin_constant_p(sr)); 962 BUILD_BUG_ON(sr < __SANITISED_REG_START__); 963 BUILD_BUG_ON(sr >= NR_SYS_REGS); 964 965 kvm->arch.sysreg_masks->mask[i].res0 = res0; 966 kvm->arch.sysreg_masks->mask[i].res1 = res1; 967 } 968 969 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) 970 { 971 struct kvm *kvm = vcpu->kvm; 972 u64 res0, res1; 973 974 lockdep_assert_held(&kvm->arch.config_lock); 975 976 if (kvm->arch.sysreg_masks) 977 goto out; 978 979 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)), 980 GFP_KERNEL_ACCOUNT); 981 if (!kvm->arch.sysreg_masks) 982 return -ENOMEM; 983 984 limit_nv_id_regs(kvm); 985 986 /* VTTBR_EL2 */ 987 res0 = res1 = 0; 988 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 989 res0 |= GENMASK(63, 56); 990 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 991 res0 |= VTTBR_CNP_BIT; 992 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1); 993 994 /* VTCR_EL2 */ 995 res0 = GENMASK(63, 32) | GENMASK(30, 20); 996 res1 = BIT(31); 997 set_sysreg_masks(kvm, VTCR_EL2, res0, res1); 998 999 /* VMPIDR_EL2 */ 1000 res0 = GENMASK(63, 40) | GENMASK(30, 24); 1001 res1 = BIT(31); 1002 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1); 1003 1004 /* HCR_EL2 */ 1005 res0 = BIT(48); 1006 res1 = HCR_RW; 1007 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP)) 1008 res0 |= GENMASK(63, 59); 1009 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2)) 1010 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA); 1011 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS)) 1012 res0 |= (HCR_TTLBIS | HCR_TTLBOS); 1013 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1014 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1015 res0 |= HCR_ENSCXT; 1016 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP)) 1017 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4); 1018 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1019 res0 |= HCR_AMVOFFEN; 1020 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1)) 1021 res0 |= HCR_FIEN; 1022 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP)) 1023 res0 |= HCR_FWB; 1024 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2)) 1025 res0 |= HCR_NV2; 1026 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP)) 1027 res0 |= (HCR_AT | HCR_NV1 | HCR_NV); 1028 if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1029 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1030 res0 |= (HCR_API | HCR_APK); 1031 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP)) 1032 res0 |= BIT(39); 1033 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1034 res0 |= (HCR_TEA | HCR_TERR); 1035 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1036 res0 |= HCR_TLOR; 1037 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP)) 1038 res1 |= HCR_E2H; 1039 set_sysreg_masks(kvm, HCR_EL2, res0, res1); 1040 1041 /* HCRX_EL2 */ 1042 res0 = HCRX_EL2_RES0; 1043 res1 = HCRX_EL2_RES1; 1044 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP)) 1045 res0 |= HCRX_EL2_PACMEn; 1046 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP)) 1047 res0 |= HCRX_EL2_EnFPM; 1048 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1049 res0 |= HCRX_EL2_GCSEn; 1050 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP)) 1051 res0 |= HCRX_EL2_EnIDCP128; 1052 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC)) 1053 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR); 1054 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP)) 1055 res0 |= HCRX_EL2_TMEA; 1056 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1057 res0 |= HCRX_EL2_D128En; 1058 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1059 res0 |= HCRX_EL2_PTTWI; 1060 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP)) 1061 res0 |= HCRX_EL2_SCTLR2En; 1062 if (!kvm_has_tcr2(kvm)) 1063 res0 |= HCRX_EL2_TCR2En; 1064 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 1065 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 1066 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP)) 1067 res0 |= HCRX_EL2_CMOW; 1068 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP)) 1069 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT); 1070 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) || 1071 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)) 1072 res0 |= HCRX_EL2_SMPME; 1073 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)) 1074 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS); 1075 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 1076 res0 |= HCRX_EL2_EnASR; 1077 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 1078 res0 |= HCRX_EL2_EnALS; 1079 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1080 res0 |= HCRX_EL2_EnAS0; 1081 set_sysreg_masks(kvm, HCRX_EL2, res0, res1); 1082 1083 /* HFG[RW]TR_EL2 */ 1084 res0 = res1 = 0; 1085 if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1086 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1087 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey | 1088 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey | 1089 HFGxTR_EL2_APIBKey); 1090 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1091 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 | 1092 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 | 1093 HFGxTR_EL2_LORSA_EL1); 1094 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1095 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1096 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0); 1097 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP)) 1098 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1; 1099 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1100 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 | 1101 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 | 1102 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 | 1103 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 | 1104 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1); 1105 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1106 res0 |= HFGxTR_EL2_nACCDATA_EL1; 1107 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1108 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1); 1109 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 1110 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0); 1111 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1112 res0 |= HFGxTR_EL2_nRCWMASK_EL1; 1113 if (!kvm_has_s1pie(kvm)) 1114 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1); 1115 if (!kvm_has_s1poe(kvm)) 1116 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1); 1117 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 1118 res0 |= HFGxTR_EL2_nS2POR_EL1; 1119 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1120 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1); 1121 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1); 1122 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1); 1123 1124 /* HDFG[RW]TR_EL2 */ 1125 res0 = res1 = 0; 1126 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 1127 res0 |= HDFGRTR_EL2_OSDLR_EL1; 1128 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1129 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 | 1130 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 | 1131 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN | 1132 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 | 1133 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 | 1134 HDFGRTR_EL2_PMCEIDn_EL0); 1135 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1136 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 | 1137 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 | 1138 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 | 1139 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 | 1140 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 | 1141 HDFGRTR_EL2_PMBIDR_EL1); 1142 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1143 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS | 1144 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM | 1145 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID | 1146 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR | 1147 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR | 1148 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR | 1149 HDFGRTR_EL2_TRCVICTLR); 1150 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1151 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 | 1152 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 | 1153 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 | 1154 HDFGRTR_EL2_TRBTRG_EL1); 1155 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1156 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL | 1157 HDFGRTR_EL2_nBRBDATA); 1158 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1159 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1; 1160 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1); 1161 1162 /* Reuse the bits from the read-side and add the write-specific stuff */ 1163 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1164 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0); 1165 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1166 res0 |= HDFGWTR_EL2_TRCOSLAR; 1167 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1168 res0 |= HDFGWTR_EL2_TRFCR_EL1; 1169 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1); 1170 1171 /* HFGITR_EL2 */ 1172 res0 = HFGITR_EL2_RES0; 1173 res1 = HFGITR_EL2_RES1; 1174 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2)) 1175 res0 |= HFGITR_EL2_DCCVADP; 1176 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2)) 1177 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP); 1178 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 1179 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1180 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS | 1181 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS | 1182 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS | 1183 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS); 1184 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 1185 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 | 1186 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 | 1187 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS | 1188 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS | 1189 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1190 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS); 1191 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP)) 1192 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX | 1193 HFGITR_EL2_CPPRCTX); 1194 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1195 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL); 1196 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1197 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 | 1198 HFGITR_EL2_nGCSEPP); 1199 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX)) 1200 res0 |= HFGITR_EL2_COSPRCTX; 1201 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP)) 1202 res0 |= HFGITR_EL2_ATS1E1A; 1203 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1); 1204 1205 /* HAFGRTR_EL2 - not a lot to see here */ 1206 res0 = HAFGRTR_EL2_RES0; 1207 res1 = HAFGRTR_EL2_RES1; 1208 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1209 res0 |= ~(res0 | res1); 1210 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1); 1211 1212 /* TCR2_EL2 */ 1213 res0 = TCR2_EL2_RES0; 1214 res1 = TCR2_EL2_RES1; 1215 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1216 res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128); 1217 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP)) 1218 res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0; 1219 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT)) 1220 res0 |= TCR2_EL2_HAFT; 1221 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1222 res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH; 1223 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1224 res0 |= TCR2_EL2_AIE; 1225 if (!kvm_has_s1poe(kvm)) 1226 res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE; 1227 if (!kvm_has_s1pie(kvm)) 1228 res0 |= TCR2_EL2_PIE; 1229 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1230 res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 | 1231 TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1); 1232 set_sysreg_masks(kvm, TCR2_EL2, res0, res1); 1233 1234 /* SCTLR_EL1 */ 1235 res0 = SCTLR_EL1_RES0; 1236 res1 = SCTLR_EL1_RES1; 1237 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) 1238 res0 |= SCTLR_EL1_EPAN; 1239 set_sysreg_masks(kvm, SCTLR_EL1, res0, res1); 1240 1241 /* MDCR_EL2 */ 1242 res0 = MDCR_EL2_RES0; 1243 res1 = MDCR_EL2_RES1; 1244 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1245 res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR | 1246 MDCR_EL2_TPM | MDCR_EL2_HPME); 1247 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1248 res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS; 1249 if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP)) 1250 res0 |= MDCR_EL2_EnSPM; 1251 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1)) 1252 res0 |= MDCR_EL2_HPMD; 1253 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1254 res0 |= MDCR_EL2_TTRF; 1255 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)) 1256 res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP; 1257 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1258 res0 |= MDCR_EL2_E2TB; 1259 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP)) 1260 res0 |= MDCR_EL2_TDCC; 1261 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) || 1262 kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP)) 1263 res0 |= MDCR_EL2_MTPME; 1264 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7)) 1265 res0 |= MDCR_EL2_HPMFZO; 1266 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP)) 1267 res0 |= MDCR_EL2_PMSSE; 1268 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1269 res0 |= MDCR_EL2_HPMFZS; 1270 if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP)) 1271 res0 |= MDCR_EL2_PMEE; 1272 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9)) 1273 res0 |= MDCR_EL2_EBWE; 1274 if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP)) 1275 res0 |= MDCR_EL2_EnSTEPOP; 1276 set_sysreg_masks(kvm, MDCR_EL2, res0, res1); 1277 1278 /* CNTHCTL_EL2 */ 1279 res0 = GENMASK(63, 20); 1280 res1 = 0; 1281 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP)) 1282 res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK; 1283 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) { 1284 res0 |= CNTHCTL_ECV; 1285 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP)) 1286 res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT | 1287 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT); 1288 } 1289 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1290 res0 |= GENMASK(11, 8); 1291 set_sysreg_masks(kvm, CNTHCTL_EL2, res0, res1); 1292 1293 out: 1294 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) 1295 (void)__vcpu_sys_reg(vcpu, sr); 1296 1297 return 0; 1298 } 1299 1300 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 1301 { 1302 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 1303 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 1304 1305 write_lock(&vcpu->kvm->mmu_lock); 1306 if (mmu->pending_unmap) { 1307 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 1308 mmu->pending_unmap = false; 1309 } 1310 write_unlock(&vcpu->kvm->mmu_lock); 1311 } 1312 } 1313