1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/kvm_arm.h> 12 #include <asm/kvm_emulate.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/kvm_nested.h> 15 #include <asm/sysreg.h> 16 17 #include "sys_regs.h" 18 19 /* Protection against the sysreg repainting madness... */ 20 #define NV_FTR(r, f) ID_AA64##r##_EL1_##f 21 22 /* 23 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 24 * memory usage and potential number of different sets of S2 PTs in 25 * the guests. Running out of S2 MMUs only affects performance (we 26 * will invalidate them more often). 27 */ 28 #define S2_MMU_PER_VCPU 2 29 30 void kvm_init_nested(struct kvm *kvm) 31 { 32 kvm->arch.nested_mmus = NULL; 33 kvm->arch.nested_mmus_size = 0; 34 } 35 36 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 37 { 38 /* 39 * We only initialise the IPA range on the canonical MMU, which 40 * defines the contract between KVM and userspace on where the 41 * "hardware" is in the IPA space. This affects the validity of MMIO 42 * exits forwarded to userspace, for example. 43 * 44 * For nested S2s, we use the PARange as exposed to the guest, as it 45 * is allowed to use it at will to expose whatever memory map it 46 * wants to its own guests as it would be on real HW. 47 */ 48 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 49 } 50 51 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 52 { 53 struct kvm *kvm = vcpu->kvm; 54 struct kvm_s2_mmu *tmp; 55 int num_mmus, ret = 0; 56 57 /* 58 * Let's treat memory allocation failures as benign: If we fail to 59 * allocate anything, return an error and keep the allocated array 60 * alive. Userspace may try to recover by intializing the vcpu 61 * again, and there is no reason to affect the whole VM for this. 62 */ 63 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 64 tmp = kvrealloc(kvm->arch.nested_mmus, 65 size_mul(sizeof(*kvm->arch.nested_mmus), kvm->arch.nested_mmus_size), 66 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus), 67 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 68 if (!tmp) 69 return -ENOMEM; 70 71 /* 72 * If we went through a realocation, adjust the MMU back-pointers in 73 * the previously initialised kvm_pgtable structures. 74 */ 75 if (kvm->arch.nested_mmus != tmp) 76 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 77 tmp[i].pgt->mmu = &tmp[i]; 78 79 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 80 ret = init_nested_s2_mmu(kvm, &tmp[i]); 81 82 if (ret) { 83 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 84 kvm_free_stage2_pgd(&tmp[i]); 85 86 return ret; 87 } 88 89 kvm->arch.nested_mmus_size = num_mmus; 90 kvm->arch.nested_mmus = tmp; 91 92 return 0; 93 } 94 95 struct s2_walk_info { 96 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data); 97 void *data; 98 u64 baddr; 99 unsigned int max_oa_bits; 100 unsigned int pgshift; 101 unsigned int sl; 102 unsigned int t0sz; 103 bool be; 104 }; 105 106 static unsigned int ps_to_output_size(unsigned int ps) 107 { 108 switch (ps) { 109 case 0: return 32; 110 case 1: return 36; 111 case 2: return 40; 112 case 3: return 42; 113 case 4: return 44; 114 case 5: 115 default: 116 return 48; 117 } 118 } 119 120 static u32 compute_fsc(int level, u32 fsc) 121 { 122 return fsc | (level & 0x3); 123 } 124 125 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 126 { 127 u32 esr; 128 129 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 130 esr |= compute_fsc(level, fsc); 131 return esr; 132 } 133 134 static int get_ia_size(struct s2_walk_info *wi) 135 { 136 return 64 - wi->t0sz; 137 } 138 139 static int check_base_s2_limits(struct s2_walk_info *wi, 140 int level, int input_size, int stride) 141 { 142 int start_size, ia_size; 143 144 ia_size = get_ia_size(wi); 145 146 /* Check translation limits */ 147 switch (BIT(wi->pgshift)) { 148 case SZ_64K: 149 if (level == 0 || (level == 1 && ia_size <= 42)) 150 return -EFAULT; 151 break; 152 case SZ_16K: 153 if (level == 0 || (level == 1 && ia_size <= 40)) 154 return -EFAULT; 155 break; 156 case SZ_4K: 157 if (level < 0 || (level == 0 && ia_size <= 42)) 158 return -EFAULT; 159 break; 160 } 161 162 /* Check input size limits */ 163 if (input_size > ia_size) 164 return -EFAULT; 165 166 /* Check number of entries in starting level table */ 167 start_size = input_size - ((3 - level) * stride + wi->pgshift); 168 if (start_size < 1 || start_size > stride + 4) 169 return -EFAULT; 170 171 return 0; 172 } 173 174 /* Check if output is within boundaries */ 175 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 176 { 177 unsigned int output_size = wi->max_oa_bits; 178 179 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 180 return -1; 181 182 return 0; 183 } 184 185 /* 186 * This is essentially a C-version of the pseudo code from the ARM ARM 187 * AArch64.TranslationTableWalk function. I strongly recommend looking at 188 * that pseudocode in trying to understand this. 189 * 190 * Must be called with the kvm->srcu read lock held 191 */ 192 static int walk_nested_s2_pgd(phys_addr_t ipa, 193 struct s2_walk_info *wi, struct kvm_s2_trans *out) 194 { 195 int first_block_level, level, stride, input_size, base_lower_bound; 196 phys_addr_t base_addr; 197 unsigned int addr_top, addr_bottom; 198 u64 desc; /* page table entry */ 199 int ret; 200 phys_addr_t paddr; 201 202 switch (BIT(wi->pgshift)) { 203 default: 204 case SZ_64K: 205 case SZ_16K: 206 level = 3 - wi->sl; 207 first_block_level = 2; 208 break; 209 case SZ_4K: 210 level = 2 - wi->sl; 211 first_block_level = 1; 212 break; 213 } 214 215 stride = wi->pgshift - 3; 216 input_size = get_ia_size(wi); 217 if (input_size > 48 || input_size < 25) 218 return -EFAULT; 219 220 ret = check_base_s2_limits(wi, level, input_size, stride); 221 if (WARN_ON(ret)) 222 return ret; 223 224 base_lower_bound = 3 + input_size - ((3 - level) * stride + 225 wi->pgshift); 226 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 227 228 if (check_output_size(wi, base_addr)) { 229 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 230 return 1; 231 } 232 233 addr_top = input_size - 1; 234 235 while (1) { 236 phys_addr_t index; 237 238 addr_bottom = (3 - level) * stride + wi->pgshift; 239 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 240 >> (addr_bottom - 3); 241 242 paddr = base_addr | index; 243 ret = wi->read_desc(paddr, &desc, wi->data); 244 if (ret < 0) 245 return ret; 246 247 /* 248 * Handle reversedescriptors if endianness differs between the 249 * host and the guest hypervisor. 250 */ 251 if (wi->be) 252 desc = be64_to_cpu((__force __be64)desc); 253 else 254 desc = le64_to_cpu((__force __le64)desc); 255 256 /* Check for valid descriptor at this point */ 257 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) { 258 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 259 out->upper_attr = desc; 260 return 1; 261 } 262 263 /* We're at the final level or block translation level */ 264 if ((desc & 3) == 1 || level == 3) 265 break; 266 267 if (check_output_size(wi, desc)) { 268 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 269 out->upper_attr = desc; 270 return 1; 271 } 272 273 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 274 275 level += 1; 276 addr_top = addr_bottom - 1; 277 } 278 279 if (level < first_block_level) { 280 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 281 out->upper_attr = desc; 282 return 1; 283 } 284 285 /* 286 * We don't use the contiguous bit in the stage-2 ptes, so skip check 287 * for misprogramming of the contiguous bit. 288 */ 289 290 if (check_output_size(wi, desc)) { 291 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 292 out->upper_attr = desc; 293 return 1; 294 } 295 296 if (!(desc & BIT(10))) { 297 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 298 out->upper_attr = desc; 299 return 1; 300 } 301 302 /* Calculate and return the result */ 303 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 304 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 305 out->output = paddr; 306 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 307 out->readable = desc & (0b01 << 6); 308 out->writable = desc & (0b10 << 6); 309 out->level = level; 310 out->upper_attr = desc & GENMASK_ULL(63, 52); 311 return 0; 312 } 313 314 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data) 315 { 316 struct kvm_vcpu *vcpu = data; 317 318 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc)); 319 } 320 321 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi) 322 { 323 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK; 324 325 switch (vtcr & VTCR_EL2_TG0_MASK) { 326 case VTCR_EL2_TG0_4K: 327 wi->pgshift = 12; break; 328 case VTCR_EL2_TG0_16K: 329 wi->pgshift = 14; break; 330 case VTCR_EL2_TG0_64K: 331 default: /* IMPDEF: treat any other value as 64k */ 332 wi->pgshift = 16; break; 333 } 334 335 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 336 /* Global limit for now, should eventually be per-VM */ 337 wi->max_oa_bits = min(get_kvm_ipa_limit(), 338 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr))); 339 } 340 341 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 342 struct kvm_s2_trans *result) 343 { 344 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 345 struct s2_walk_info wi; 346 int ret; 347 348 result->esr = 0; 349 350 if (!vcpu_has_nv(vcpu)) 351 return 0; 352 353 wi.read_desc = read_guest_s2_desc; 354 wi.data = vcpu; 355 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 356 357 vtcr_to_walk_info(vtcr, &wi); 358 359 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 360 361 ret = walk_nested_s2_pgd(gipa, &wi, result); 362 if (ret) 363 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 364 365 return ret; 366 } 367 368 static unsigned int ttl_to_size(u8 ttl) 369 { 370 int level = ttl & 3; 371 int gran = (ttl >> 2) & 3; 372 unsigned int max_size = 0; 373 374 switch (gran) { 375 case TLBI_TTL_TG_4K: 376 switch (level) { 377 case 0: 378 break; 379 case 1: 380 max_size = SZ_1G; 381 break; 382 case 2: 383 max_size = SZ_2M; 384 break; 385 case 3: 386 max_size = SZ_4K; 387 break; 388 } 389 break; 390 case TLBI_TTL_TG_16K: 391 switch (level) { 392 case 0: 393 case 1: 394 break; 395 case 2: 396 max_size = SZ_32M; 397 break; 398 case 3: 399 max_size = SZ_16K; 400 break; 401 } 402 break; 403 case TLBI_TTL_TG_64K: 404 switch (level) { 405 case 0: 406 case 1: 407 /* No 52bit IPA support */ 408 break; 409 case 2: 410 max_size = SZ_512M; 411 break; 412 case 3: 413 max_size = SZ_64K; 414 break; 415 } 416 break; 417 default: /* No size information */ 418 break; 419 } 420 421 return max_size; 422 } 423 424 /* 425 * Compute the equivalent of the TTL field by parsing the shadow PT. The 426 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 427 * retrieved from first entry carrying the level as a tag. 428 */ 429 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 430 { 431 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr; 432 kvm_pte_t pte; 433 u8 ttl, level; 434 435 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 436 437 switch (vtcr & VTCR_EL2_TG0_MASK) { 438 case VTCR_EL2_TG0_4K: 439 ttl = (TLBI_TTL_TG_4K << 2); 440 break; 441 case VTCR_EL2_TG0_16K: 442 ttl = (TLBI_TTL_TG_16K << 2); 443 break; 444 case VTCR_EL2_TG0_64K: 445 default: /* IMPDEF: treat any other value as 64k */ 446 ttl = (TLBI_TTL_TG_64K << 2); 447 break; 448 } 449 450 tmp = addr; 451 452 again: 453 /* Iteratively compute the block sizes for a particular granule size */ 454 switch (vtcr & VTCR_EL2_TG0_MASK) { 455 case VTCR_EL2_TG0_4K: 456 if (sz < SZ_4K) sz = SZ_4K; 457 else if (sz < SZ_2M) sz = SZ_2M; 458 else if (sz < SZ_1G) sz = SZ_1G; 459 else sz = 0; 460 break; 461 case VTCR_EL2_TG0_16K: 462 if (sz < SZ_16K) sz = SZ_16K; 463 else if (sz < SZ_32M) sz = SZ_32M; 464 else sz = 0; 465 break; 466 case VTCR_EL2_TG0_64K: 467 default: /* IMPDEF: treat any other value as 64k */ 468 if (sz < SZ_64K) sz = SZ_64K; 469 else if (sz < SZ_512M) sz = SZ_512M; 470 else sz = 0; 471 break; 472 } 473 474 if (sz == 0) 475 return 0; 476 477 tmp &= ~(sz - 1); 478 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 479 goto again; 480 if (!(pte & PTE_VALID)) 481 goto again; 482 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 483 if (!level) 484 goto again; 485 486 ttl |= level; 487 488 /* 489 * We now have found some level information in the shadow S2. Check 490 * that the resulting range is actually including the original IPA. 491 */ 492 sz = ttl_to_size(ttl); 493 if (addr < (tmp + sz)) 494 return ttl; 495 496 return 0; 497 } 498 499 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 500 { 501 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 502 unsigned long max_size; 503 u8 ttl; 504 505 ttl = FIELD_GET(TLBI_TTL_MASK, val); 506 507 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 508 /* No TTL, check the shadow S2 for a hint */ 509 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 510 ttl = get_guest_mapping_ttl(mmu, addr); 511 } 512 513 max_size = ttl_to_size(ttl); 514 515 if (!max_size) { 516 /* Compute the maximum extent of the invalidation */ 517 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) { 518 case VTCR_EL2_TG0_4K: 519 max_size = SZ_1G; 520 break; 521 case VTCR_EL2_TG0_16K: 522 max_size = SZ_32M; 523 break; 524 case VTCR_EL2_TG0_64K: 525 default: /* IMPDEF: treat any other value as 64k */ 526 /* 527 * No, we do not support 52bit IPA in nested yet. Once 528 * we do, this should be 4TB. 529 */ 530 max_size = SZ_512M; 531 break; 532 } 533 } 534 535 WARN_ON(!max_size); 536 return max_size; 537 } 538 539 /* 540 * We can have multiple *different* MMU contexts with the same VMID: 541 * 542 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 543 * 544 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 545 * VBBTR_EL2.BADDR address 546 * 547 * - A combination of the above... 548 * 549 * We can always identify which MMU context to pick at run-time. However, 550 * TLB invalidation involving a VMID must take action on all the TLBs using 551 * this particular VMID. This translates into applying the same invalidation 552 * operation to all the contexts that are using this VMID. Moar phun! 553 */ 554 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 555 const union tlbi_info *info, 556 void (*tlbi_callback)(struct kvm_s2_mmu *, 557 const union tlbi_info *)) 558 { 559 write_lock(&kvm->mmu_lock); 560 561 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 562 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 563 564 if (!kvm_s2_mmu_valid(mmu)) 565 continue; 566 567 if (vmid == get_vmid(mmu->tlb_vttbr)) 568 tlbi_callback(mmu, info); 569 } 570 571 write_unlock(&kvm->mmu_lock); 572 } 573 574 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 575 { 576 struct kvm *kvm = vcpu->kvm; 577 bool nested_stage2_enabled; 578 u64 vttbr, vtcr, hcr; 579 580 lockdep_assert_held_write(&kvm->mmu_lock); 581 582 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 583 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 584 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 585 586 nested_stage2_enabled = hcr & HCR_VM; 587 588 /* Don't consider the CnP bit for the vttbr match */ 589 vttbr &= ~VTTBR_CNP_BIT; 590 591 /* 592 * Two possibilities when looking up a S2 MMU context: 593 * 594 * - either S2 is enabled in the guest, and we need a context that is 595 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 596 * which makes it safe from a TLB conflict perspective (a broken 597 * guest won't be able to generate them), 598 * 599 * - or S2 is disabled, and we need a context that is S2-disabled 600 * and matches the VMID only, as all TLBs are tagged by VMID even 601 * if S2 translation is disabled. 602 */ 603 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 604 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 605 606 if (!kvm_s2_mmu_valid(mmu)) 607 continue; 608 609 if (nested_stage2_enabled && 610 mmu->nested_stage2_enabled && 611 vttbr == mmu->tlb_vttbr && 612 vtcr == mmu->tlb_vtcr) 613 return mmu; 614 615 if (!nested_stage2_enabled && 616 !mmu->nested_stage2_enabled && 617 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 618 return mmu; 619 } 620 return NULL; 621 } 622 623 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 624 { 625 struct kvm *kvm = vcpu->kvm; 626 struct kvm_s2_mmu *s2_mmu; 627 int i; 628 629 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 630 631 s2_mmu = lookup_s2_mmu(vcpu); 632 if (s2_mmu) 633 goto out; 634 635 /* 636 * Make sure we don't always search from the same point, or we 637 * will always reuse a potentially active context, leaving 638 * free contexts unused. 639 */ 640 for (i = kvm->arch.nested_mmus_next; 641 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 642 i++) { 643 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 644 645 if (atomic_read(&s2_mmu->refcnt) == 0) 646 break; 647 } 648 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 649 650 /* Set the scene for the next search */ 651 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 652 653 /* Clear the old state */ 654 if (kvm_s2_mmu_valid(s2_mmu)) 655 kvm_stage2_unmap_range(s2_mmu, 0, kvm_phys_size(s2_mmu)); 656 657 /* 658 * The virtual VMID (modulo CnP) will be used as a key when matching 659 * an existing kvm_s2_mmu. 660 * 661 * We cache VTCR at allocation time, once and for all. It'd be great 662 * if the guest didn't screw that one up, as this is not very 663 * forgiving... 664 */ 665 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 666 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 667 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 668 669 out: 670 atomic_inc(&s2_mmu->refcnt); 671 return s2_mmu; 672 } 673 674 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 675 { 676 /* CnP being set denotes an invalid entry */ 677 mmu->tlb_vttbr = VTTBR_CNP_BIT; 678 mmu->nested_stage2_enabled = false; 679 atomic_set(&mmu->refcnt, 0); 680 } 681 682 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 683 { 684 if (is_hyp_ctxt(vcpu)) { 685 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 686 } else { 687 write_lock(&vcpu->kvm->mmu_lock); 688 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 689 write_unlock(&vcpu->kvm->mmu_lock); 690 } 691 } 692 693 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 694 { 695 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) { 696 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 697 vcpu->arch.hw_mmu = NULL; 698 } 699 } 700 701 /* 702 * Returns non-zero if permission fault is handled by injecting it to the next 703 * level hypervisor. 704 */ 705 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 706 { 707 bool forward_fault = false; 708 709 trans->esr = 0; 710 711 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 712 return 0; 713 714 if (kvm_vcpu_trap_is_iabt(vcpu)) { 715 forward_fault = !kvm_s2_trans_executable(trans); 716 } else { 717 bool write_fault = kvm_is_write_fault(vcpu); 718 719 forward_fault = ((write_fault && !trans->writable) || 720 (!write_fault && !trans->readable)); 721 } 722 723 if (forward_fault) 724 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 725 726 return forward_fault; 727 } 728 729 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 730 { 731 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 732 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 733 734 return kvm_inject_nested_sync(vcpu, esr_el2); 735 } 736 737 void kvm_nested_s2_wp(struct kvm *kvm) 738 { 739 int i; 740 741 lockdep_assert_held_write(&kvm->mmu_lock); 742 743 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 744 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 745 746 if (kvm_s2_mmu_valid(mmu)) 747 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 748 } 749 } 750 751 void kvm_nested_s2_unmap(struct kvm *kvm) 752 { 753 int i; 754 755 lockdep_assert_held_write(&kvm->mmu_lock); 756 757 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 758 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 759 760 if (kvm_s2_mmu_valid(mmu)) 761 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu)); 762 } 763 } 764 765 void kvm_nested_s2_flush(struct kvm *kvm) 766 { 767 int i; 768 769 lockdep_assert_held_write(&kvm->mmu_lock); 770 771 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 772 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 773 774 if (kvm_s2_mmu_valid(mmu)) 775 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 776 } 777 } 778 779 void kvm_arch_flush_shadow_all(struct kvm *kvm) 780 { 781 int i; 782 783 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 784 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 785 786 if (!WARN_ON(atomic_read(&mmu->refcnt))) 787 kvm_free_stage2_pgd(mmu); 788 } 789 kfree(kvm->arch.nested_mmus); 790 kvm->arch.nested_mmus = NULL; 791 kvm->arch.nested_mmus_size = 0; 792 kvm_uninit_stage2_mmu(kvm); 793 } 794 795 /* 796 * Our emulated CPU doesn't support all the possible features. For the 797 * sake of simplicity (and probably mental sanity), wipe out a number 798 * of feature bits we don't intend to support for the time being. 799 * This list should get updated as new features get added to the NV 800 * support, and new extension to the architecture. 801 */ 802 static u64 limit_nv_id_reg(u32 id, u64 val) 803 { 804 u64 tmp; 805 806 switch (id) { 807 case SYS_ID_AA64ISAR0_EL1: 808 /* Support everything but TME and Range TLBIs */ 809 tmp = FIELD_GET(NV_FTR(ISAR0, TLB), val); 810 tmp = min(tmp, ID_AA64ISAR0_EL1_TLB_OS); 811 val &= ~(NV_FTR(ISAR0, TLB) | 812 NV_FTR(ISAR0, TME)); 813 val |= FIELD_PREP(NV_FTR(ISAR0, TLB), tmp); 814 break; 815 816 case SYS_ID_AA64ISAR1_EL1: 817 /* Support everything but Spec Invalidation */ 818 val &= ~(GENMASK_ULL(63, 56) | 819 NV_FTR(ISAR1, SPECRES)); 820 break; 821 822 case SYS_ID_AA64PFR0_EL1: 823 /* No AMU, MPAM, S-EL2, RAS or SVE */ 824 val &= ~(GENMASK_ULL(55, 52) | 825 NV_FTR(PFR0, AMU) | 826 NV_FTR(PFR0, MPAM) | 827 NV_FTR(PFR0, SEL2) | 828 NV_FTR(PFR0, RAS) | 829 NV_FTR(PFR0, SVE) | 830 NV_FTR(PFR0, EL3) | 831 NV_FTR(PFR0, EL2) | 832 NV_FTR(PFR0, EL1)); 833 /* 64bit EL1/EL2/EL3 only */ 834 val |= FIELD_PREP(NV_FTR(PFR0, EL1), 0b0001); 835 val |= FIELD_PREP(NV_FTR(PFR0, EL2), 0b0001); 836 val |= FIELD_PREP(NV_FTR(PFR0, EL3), 0b0001); 837 break; 838 839 case SYS_ID_AA64PFR1_EL1: 840 /* Only support BTI, SSBS, CSV2_frac */ 841 val &= (NV_FTR(PFR1, BT) | 842 NV_FTR(PFR1, SSBS) | 843 NV_FTR(PFR1, CSV2_frac)); 844 break; 845 846 case SYS_ID_AA64MMFR0_EL1: 847 /* Hide ECV, ExS, Secure Memory */ 848 val &= ~(NV_FTR(MMFR0, ECV) | 849 NV_FTR(MMFR0, EXS) | 850 NV_FTR(MMFR0, TGRAN4_2) | 851 NV_FTR(MMFR0, TGRAN16_2) | 852 NV_FTR(MMFR0, TGRAN64_2) | 853 NV_FTR(MMFR0, SNSMEM)); 854 855 /* Disallow unsupported S2 page sizes */ 856 switch (PAGE_SIZE) { 857 case SZ_64K: 858 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0001); 859 fallthrough; 860 case SZ_16K: 861 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0001); 862 fallthrough; 863 case SZ_4K: 864 /* Support everything */ 865 break; 866 } 867 /* 868 * Since we can't support a guest S2 page size smaller than 869 * the host's own page size (due to KVM only populating its 870 * own S2 using the kernel's page size), advertise the 871 * limitation using FEAT_GTG. 872 */ 873 switch (PAGE_SIZE) { 874 case SZ_4K: 875 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN4_2), 0b0010); 876 fallthrough; 877 case SZ_16K: 878 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN16_2), 0b0010); 879 fallthrough; 880 case SZ_64K: 881 val |= FIELD_PREP(NV_FTR(MMFR0, TGRAN64_2), 0b0010); 882 break; 883 } 884 /* Cap PARange to 48bits */ 885 tmp = FIELD_GET(NV_FTR(MMFR0, PARANGE), val); 886 if (tmp > 0b0101) { 887 val &= ~NV_FTR(MMFR0, PARANGE); 888 val |= FIELD_PREP(NV_FTR(MMFR0, PARANGE), 0b0101); 889 } 890 break; 891 892 case SYS_ID_AA64MMFR1_EL1: 893 val &= (NV_FTR(MMFR1, HCX) | 894 NV_FTR(MMFR1, PAN) | 895 NV_FTR(MMFR1, LO) | 896 NV_FTR(MMFR1, HPDS) | 897 NV_FTR(MMFR1, VH) | 898 NV_FTR(MMFR1, VMIDBits)); 899 break; 900 901 case SYS_ID_AA64MMFR2_EL1: 902 val &= ~(NV_FTR(MMFR2, BBM) | 903 NV_FTR(MMFR2, TTL) | 904 GENMASK_ULL(47, 44) | 905 NV_FTR(MMFR2, ST) | 906 NV_FTR(MMFR2, CCIDX) | 907 NV_FTR(MMFR2, VARange)); 908 909 /* Force TTL support */ 910 val |= FIELD_PREP(NV_FTR(MMFR2, TTL), 0b0001); 911 break; 912 913 case SYS_ID_AA64MMFR4_EL1: 914 val = 0; 915 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 916 val |= FIELD_PREP(NV_FTR(MMFR4, E2H0), 917 ID_AA64MMFR4_EL1_E2H0_NI_NV1); 918 break; 919 920 case SYS_ID_AA64DFR0_EL1: 921 /* Only limited support for PMU, Debug, BPs and WPs */ 922 val &= (NV_FTR(DFR0, PMUVer) | 923 NV_FTR(DFR0, WRPs) | 924 NV_FTR(DFR0, BRPs) | 925 NV_FTR(DFR0, DebugVer)); 926 927 /* Cap Debug to ARMv8.1 */ 928 tmp = FIELD_GET(NV_FTR(DFR0, DebugVer), val); 929 if (tmp > 0b0111) { 930 val &= ~NV_FTR(DFR0, DebugVer); 931 val |= FIELD_PREP(NV_FTR(DFR0, DebugVer), 0b0111); 932 } 933 break; 934 935 default: 936 /* Unknown register, just wipe it clean */ 937 val = 0; 938 break; 939 } 940 941 return val; 942 } 943 944 u64 kvm_vcpu_sanitise_vncr_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg sr) 945 { 946 u64 v = ctxt_sys_reg(&vcpu->arch.ctxt, sr); 947 struct kvm_sysreg_masks *masks; 948 949 masks = vcpu->kvm->arch.sysreg_masks; 950 951 if (masks) { 952 sr -= __VNCR_START__; 953 954 v &= ~masks->mask[sr].res0; 955 v |= masks->mask[sr].res1; 956 } 957 958 return v; 959 } 960 961 static void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1) 962 { 963 int i = sr - __VNCR_START__; 964 965 kvm->arch.sysreg_masks->mask[i].res0 = res0; 966 kvm->arch.sysreg_masks->mask[i].res1 = res1; 967 } 968 969 int kvm_init_nv_sysregs(struct kvm *kvm) 970 { 971 u64 res0, res1; 972 int ret = 0; 973 974 mutex_lock(&kvm->arch.config_lock); 975 976 if (kvm->arch.sysreg_masks) 977 goto out; 978 979 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)), 980 GFP_KERNEL); 981 if (!kvm->arch.sysreg_masks) { 982 ret = -ENOMEM; 983 goto out; 984 } 985 986 for (int i = 0; i < KVM_ARM_ID_REG_NUM; i++) 987 kvm->arch.id_regs[i] = limit_nv_id_reg(IDX_IDREG(i), 988 kvm->arch.id_regs[i]); 989 990 /* VTTBR_EL2 */ 991 res0 = res1 = 0; 992 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 993 res0 |= GENMASK(63, 56); 994 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 995 res0 |= VTTBR_CNP_BIT; 996 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1); 997 998 /* VTCR_EL2 */ 999 res0 = GENMASK(63, 32) | GENMASK(30, 20); 1000 res1 = BIT(31); 1001 set_sysreg_masks(kvm, VTCR_EL2, res0, res1); 1002 1003 /* VMPIDR_EL2 */ 1004 res0 = GENMASK(63, 40) | GENMASK(30, 24); 1005 res1 = BIT(31); 1006 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1); 1007 1008 /* HCR_EL2 */ 1009 res0 = BIT(48); 1010 res1 = HCR_RW; 1011 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP)) 1012 res0 |= GENMASK(63, 59); 1013 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2)) 1014 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA); 1015 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS)) 1016 res0 |= (HCR_TTLBIS | HCR_TTLBOS); 1017 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1018 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1019 res0 |= HCR_ENSCXT; 1020 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP)) 1021 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4); 1022 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1023 res0 |= HCR_AMVOFFEN; 1024 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1)) 1025 res0 |= HCR_FIEN; 1026 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP)) 1027 res0 |= HCR_FWB; 1028 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, NV2)) 1029 res0 |= HCR_NV2; 1030 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, NV, IMP)) 1031 res0 |= (HCR_AT | HCR_NV1 | HCR_NV); 1032 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1033 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1034 res0 |= (HCR_API | HCR_APK); 1035 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP)) 1036 res0 |= BIT(39); 1037 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1038 res0 |= (HCR_TEA | HCR_TERR); 1039 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1040 res0 |= HCR_TLOR; 1041 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP)) 1042 res1 |= HCR_E2H; 1043 set_sysreg_masks(kvm, HCR_EL2, res0, res1); 1044 1045 /* HCRX_EL2 */ 1046 res0 = HCRX_EL2_RES0; 1047 res1 = HCRX_EL2_RES1; 1048 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP)) 1049 res0 |= HCRX_EL2_PACMEn; 1050 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP)) 1051 res0 |= HCRX_EL2_EnFPM; 1052 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1053 res0 |= HCRX_EL2_GCSEn; 1054 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP)) 1055 res0 |= HCRX_EL2_EnIDCP128; 1056 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC)) 1057 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR); 1058 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP)) 1059 res0 |= HCRX_EL2_TMEA; 1060 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1061 res0 |= HCRX_EL2_D128En; 1062 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1063 res0 |= HCRX_EL2_PTTWI; 1064 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP)) 1065 res0 |= HCRX_EL2_SCTLR2En; 1066 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, TCRX, IMP)) 1067 res0 |= HCRX_EL2_TCR2En; 1068 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 1069 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 1070 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP)) 1071 res0 |= HCRX_EL2_CMOW; 1072 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP)) 1073 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT); 1074 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) || 1075 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)) 1076 res0 |= HCRX_EL2_SMPME; 1077 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)) 1078 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS); 1079 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 1080 res0 |= HCRX_EL2_EnASR; 1081 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 1082 res0 |= HCRX_EL2_EnALS; 1083 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1084 res0 |= HCRX_EL2_EnAS0; 1085 set_sysreg_masks(kvm, HCRX_EL2, res0, res1); 1086 1087 /* HFG[RW]TR_EL2 */ 1088 res0 = res1 = 0; 1089 if (!(__vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1090 __vcpu_has_feature(&kvm->arch, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1091 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey | 1092 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey | 1093 HFGxTR_EL2_APIBKey); 1094 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1095 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 | 1096 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 | 1097 HFGxTR_EL2_LORSA_EL1); 1098 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1099 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1100 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0); 1101 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP)) 1102 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1; 1103 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1104 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 | 1105 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 | 1106 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 | 1107 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 | 1108 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1); 1109 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1110 res0 |= HFGxTR_EL2_nACCDATA_EL1; 1111 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1112 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1); 1113 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 1114 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0); 1115 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1116 res0 |= HFGxTR_EL2_nRCWMASK_EL1; 1117 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1PIE, IMP)) 1118 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1); 1119 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1POE, IMP)) 1120 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1); 1121 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 1122 res0 |= HFGxTR_EL2_nS2POR_EL1; 1123 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1124 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1); 1125 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1); 1126 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1); 1127 1128 /* HDFG[RW]TR_EL2 */ 1129 res0 = res1 = 0; 1130 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 1131 res0 |= HDFGRTR_EL2_OSDLR_EL1; 1132 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1133 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 | 1134 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 | 1135 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN | 1136 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 | 1137 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 | 1138 HDFGRTR_EL2_PMCEIDn_EL0); 1139 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1140 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 | 1141 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 | 1142 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 | 1143 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 | 1144 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 | 1145 HDFGRTR_EL2_PMBIDR_EL1); 1146 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1147 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS | 1148 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM | 1149 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID | 1150 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR | 1151 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR | 1152 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR | 1153 HDFGRTR_EL2_TRCVICTLR); 1154 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1155 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 | 1156 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 | 1157 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 | 1158 HDFGRTR_EL2_TRBTRG_EL1); 1159 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1160 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL | 1161 HDFGRTR_EL2_nBRBDATA); 1162 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1163 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1; 1164 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1); 1165 1166 /* Reuse the bits from the read-side and add the write-specific stuff */ 1167 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1168 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0); 1169 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1170 res0 |= HDFGWTR_EL2_TRCOSLAR; 1171 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1172 res0 |= HDFGWTR_EL2_TRFCR_EL1; 1173 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1); 1174 1175 /* HFGITR_EL2 */ 1176 res0 = HFGITR_EL2_RES0; 1177 res1 = HFGITR_EL2_RES1; 1178 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2)) 1179 res0 |= HFGITR_EL2_DCCVADP; 1180 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2)) 1181 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP); 1182 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 1183 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1184 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS | 1185 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS | 1186 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS | 1187 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS); 1188 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 1189 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 | 1190 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 | 1191 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS | 1192 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS | 1193 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1194 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS); 1195 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP)) 1196 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX | 1197 HFGITR_EL2_CPPRCTX); 1198 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1199 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL); 1200 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1201 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 | 1202 HFGITR_EL2_nGCSEPP); 1203 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX)) 1204 res0 |= HFGITR_EL2_COSPRCTX; 1205 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP)) 1206 res0 |= HFGITR_EL2_ATS1E1A; 1207 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1); 1208 1209 /* HAFGRTR_EL2 - not a lot to see here */ 1210 res0 = HAFGRTR_EL2_RES0; 1211 res1 = HAFGRTR_EL2_RES1; 1212 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1213 res0 |= ~(res0 | res1); 1214 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1); 1215 out: 1216 mutex_unlock(&kvm->arch.config_lock); 1217 1218 return ret; 1219 } 1220