1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Linaro Ltd 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/kvm_host.h> 8 9 #include <asm/esr.h> 10 #include <asm/kvm_hyp.h> 11 #include <asm/kvm_mmu.h> 12 #include <asm/lsui.h> 13 14 #define S1_MMU_DISABLED (-127) 15 16 static int get_ia_size(struct s1_walk_info *wi) 17 { 18 return 64 - wi->txsz; 19 } 20 21 /* Return true if the IPA is out of the OA range */ 22 static bool check_output_size(u64 ipa, struct s1_walk_info *wi) 23 { 24 if (wi->pa52bit) 25 return wi->max_oa_bits < 52 && (ipa & GENMASK_ULL(51, wi->max_oa_bits)); 26 return wi->max_oa_bits < 48 && (ipa & GENMASK_ULL(47, wi->max_oa_bits)); 27 } 28 29 static bool has_52bit_pa(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, u64 tcr) 30 { 31 switch (BIT(wi->pgshift)) { 32 case SZ_64K: 33 default: /* IMPDEF: treat any other value as 64k */ 34 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52)) 35 return false; 36 return ((wi->regime == TR_EL2 ? 37 FIELD_GET(TCR_EL2_PS_MASK, tcr) : 38 FIELD_GET(TCR_IPS_MASK, tcr)) == 0b0110); 39 case SZ_16K: 40 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)) 41 return false; 42 break; 43 case SZ_4K: 44 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)) 45 return false; 46 break; 47 } 48 49 return (tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS)); 50 } 51 52 static u64 desc_to_oa(struct s1_walk_info *wi, u64 desc) 53 { 54 u64 addr; 55 56 if (!wi->pa52bit) 57 return desc & GENMASK_ULL(47, wi->pgshift); 58 59 switch (BIT(wi->pgshift)) { 60 case SZ_4K: 61 case SZ_16K: 62 addr = desc & GENMASK_ULL(49, wi->pgshift); 63 addr |= FIELD_GET(KVM_PTE_ADDR_51_50_LPA2, desc) << 50; 64 break; 65 case SZ_64K: 66 default: /* IMPDEF: treat any other value as 64k */ 67 addr = desc & GENMASK_ULL(47, wi->pgshift); 68 addr |= FIELD_GET(KVM_PTE_ADDR_51_48, desc) << 48; 69 break; 70 } 71 72 return addr; 73 } 74 75 /* Return the translation regime that applies to an AT instruction */ 76 static enum trans_regime compute_translation_regime(struct kvm_vcpu *vcpu, u32 op) 77 { 78 /* 79 * We only get here from guest EL2, so the translation 80 * regime AT applies to is solely defined by {E2H,TGE}. 81 */ 82 switch (op) { 83 case OP_AT_S1E2R: 84 case OP_AT_S1E2W: 85 case OP_AT_S1E2A: 86 return vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2; 87 default: 88 return (vcpu_el2_e2h_is_set(vcpu) && 89 vcpu_el2_tge_is_set(vcpu)) ? TR_EL20 : TR_EL10; 90 } 91 } 92 93 static u64 effective_tcr2(struct kvm_vcpu *vcpu, enum trans_regime regime) 94 { 95 if (regime == TR_EL10) { 96 if (vcpu_has_nv(vcpu) && 97 !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En)) 98 return 0; 99 100 return vcpu_read_sys_reg(vcpu, TCR2_EL1); 101 } 102 103 return vcpu_read_sys_reg(vcpu, TCR2_EL2); 104 } 105 106 static bool s1pie_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 107 { 108 if (!kvm_has_s1pie(vcpu->kvm)) 109 return false; 110 111 /* Abuse TCR2_EL1_PIE and use it for EL2 as well */ 112 return effective_tcr2(vcpu, regime) & TCR2_EL1_PIE; 113 } 114 115 static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) 116 { 117 u64 val; 118 119 if (!kvm_has_s1poe(vcpu->kvm)) { 120 wi->poe = wi->e0poe = false; 121 return; 122 } 123 124 val = effective_tcr2(vcpu, wi->regime); 125 126 /* Abuse TCR2_EL1_* for EL2 */ 127 wi->poe = val & TCR2_EL1_POE; 128 wi->e0poe = (wi->regime != TR_EL2) && (val & TCR2_EL1_E0POE); 129 } 130 131 #define _has_tgran(__r, __sz) \ 132 ({ \ 133 u64 _s1, _mmfr0 = __r; \ 134 \ 135 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 136 TGRAN##__sz, _mmfr0); \ 137 \ 138 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI; \ 139 }) 140 141 static bool has_tgran(u64 mmfr0, unsigned int shift) 142 { 143 switch (shift) { 144 case 12: 145 return _has_tgran(mmfr0, 4); 146 case 14: 147 return _has_tgran(mmfr0, 16); 148 case 16: 149 return _has_tgran(mmfr0, 64); 150 default: 151 BUG(); 152 } 153 } 154 155 static unsigned int tcr_to_tg0_pgshift(u64 tcr) 156 { 157 u64 tg0 = tcr & TCR_TG0_MASK; 158 159 switch (tg0) { 160 case TCR_TG0_4K: 161 return 12; 162 case TCR_TG0_16K: 163 return 14; 164 case TCR_TG0_64K: 165 default: /* IMPDEF: treat any other value as 64k */ 166 return 16; 167 } 168 } 169 170 static unsigned int tcr_to_tg1_pgshift(u64 tcr) 171 { 172 u64 tg1 = tcr & TCR_TG1_MASK; 173 174 switch (tg1) { 175 case TCR_TG1_4K: 176 return 12; 177 case TCR_TG1_16K: 178 return 14; 179 case TCR_TG1_64K: 180 default: /* IMPDEF: treat any other value as 64k */ 181 return 16; 182 } 183 } 184 185 static unsigned int fallback_tgran_shift(u64 mmfr0) 186 { 187 if (has_tgran(mmfr0, PAGE_SHIFT)) 188 return PAGE_SHIFT; 189 else if (has_tgran(mmfr0, 12)) 190 return 12; 191 else if (has_tgran(mmfr0, 14)) 192 return 14; 193 else if (has_tgran(mmfr0, 16)) 194 return 16; 195 else /* Should be unreacheable */ 196 return PAGE_SHIFT; 197 } 198 199 static unsigned int tcr_tg_pgshift(struct kvm *kvm, u64 tcr, bool upper_range) 200 { 201 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 202 unsigned int shift; 203 204 /* Someone was silly enough to encode TG0/TG1 differently */ 205 if (upper_range) 206 shift = tcr_to_tg1_pgshift(tcr); 207 else 208 shift = tcr_to_tg0_pgshift(tcr); 209 210 /* 211 * If TGx is programmed to an unimplemented value (not advertised in 212 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is 213 * written, as per the architecture. Choose an available one while 214 * prioritizing PAGE_SIZE. 215 */ 216 if (!has_tgran(mmfr0, shift)) 217 return fallback_tgran_shift(mmfr0); 218 219 return shift; 220 } 221 222 static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 223 struct s1_walk_result *wr, u64 va) 224 { 225 u64 hcr, sctlr, tcr, ps, ia_bits, ttbr; 226 unsigned int stride, x; 227 bool va55, tbi, lva, upper_range; 228 229 va55 = va & BIT(55); 230 upper_range = va55 && wi->regime != TR_EL2; 231 232 if (vcpu_has_nv(vcpu)) { 233 hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 234 wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); 235 } else { 236 WARN_ON_ONCE(wi->regime != TR_EL10); 237 wi->s2 = false; 238 hcr = 0; 239 } 240 241 switch (wi->regime) { 242 case TR_EL10: 243 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 244 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); 245 ttbr = (va55 ? 246 vcpu_read_sys_reg(vcpu, TTBR1_EL1) : 247 vcpu_read_sys_reg(vcpu, TTBR0_EL1)); 248 break; 249 case TR_EL2: 250 case TR_EL20: 251 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); 252 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); 253 ttbr = (va55 ? 254 vcpu_read_sys_reg(vcpu, TTBR1_EL2) : 255 vcpu_read_sys_reg(vcpu, TTBR0_EL2)); 256 break; 257 default: 258 BUG(); 259 } 260 261 if (upper_range) 262 wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr); 263 else 264 wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr); 265 266 wi->pgshift = tcr_tg_pgshift(vcpu->kvm, tcr, upper_range); 267 wi->pa52bit = has_52bit_pa(vcpu, wi, tcr); 268 269 ia_bits = get_ia_size(wi); 270 271 /* AArch64.S1StartLevel() */ 272 stride = wi->pgshift - 3; 273 wi->sl = 3 - (((ia_bits - 1) - wi->pgshift) / stride); 274 275 if (wi->regime == TR_EL2 && va55) 276 goto addrsz; 277 278 tbi = (wi->regime == TR_EL2 ? 279 FIELD_GET(TCR_EL2_TBI, tcr) : 280 (va55 ? 281 FIELD_GET(TCR_TBI1, tcr) : 282 FIELD_GET(TCR_TBI0, tcr))); 283 284 if (!tbi && (u64)sign_extend64(va, 55) != va) 285 goto addrsz; 286 287 wi->sh = (wi->regime == TR_EL2 ? 288 FIELD_GET(TCR_EL2_SH0_MASK, tcr) : 289 (va55 ? 290 FIELD_GET(TCR_SH1_MASK, tcr) : 291 FIELD_GET(TCR_SH0_MASK, tcr))); 292 293 va = (u64)sign_extend64(va, 55); 294 295 /* Let's put the MMU disabled case aside immediately */ 296 switch (wi->regime) { 297 case TR_EL10: 298 /* 299 * If dealing with the EL1&0 translation regime, 3 things 300 * can disable the S1 translation: 301 * 302 * - HCR_EL2.DC = 1 303 * - HCR_EL2.{E2H,TGE} = {0,1} 304 * - SCTLR_EL1.M = 0 305 * 306 * The TGE part is interesting. If we have decided that this 307 * is EL1&0, then it means that either {E2H,TGE} == {1,0} or 308 * {0,x}, and we only need to test for TGE == 1. 309 */ 310 if (hcr & (HCR_DC | HCR_TGE)) { 311 wr->level = S1_MMU_DISABLED; 312 break; 313 } 314 fallthrough; 315 case TR_EL2: 316 case TR_EL20: 317 if (!(sctlr & SCTLR_ELx_M)) 318 wr->level = S1_MMU_DISABLED; 319 break; 320 } 321 322 if (wr->level == S1_MMU_DISABLED) { 323 if (va >= BIT(kvm_get_pa_bits(vcpu->kvm))) 324 goto addrsz; 325 326 wr->pa = va; 327 return 0; 328 } 329 330 wi->be = sctlr & SCTLR_ELx_EE; 331 332 wi->hpd = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HPDS, IMP); 333 wi->hpd &= (wi->regime == TR_EL2 ? 334 FIELD_GET(TCR_EL2_HPD, tcr) : 335 (va55 ? 336 FIELD_GET(TCR_HPD1, tcr) : 337 FIELD_GET(TCR_HPD0, tcr))); 338 /* R_JHSVW */ 339 wi->hpd |= s1pie_enabled(vcpu, wi->regime); 340 341 /* Do we have POE? */ 342 compute_s1poe(vcpu, wi); 343 344 /* R_BVXDG */ 345 wi->hpd |= (wi->poe || wi->e0poe); 346 347 /* R_PLCGL, R_YXNYW */ 348 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR2_EL1, ST, 48_47)) { 349 if (wi->txsz > 39) 350 goto transfault; 351 } else { 352 if (wi->txsz > 48 || (BIT(wi->pgshift) == SZ_64K && wi->txsz > 47)) 353 goto transfault; 354 } 355 356 /* R_GTJBY, R_SXWGM */ 357 switch (BIT(wi->pgshift)) { 358 case SZ_4K: 359 case SZ_16K: 360 lva = wi->pa52bit; 361 break; 362 case SZ_64K: 363 lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, VARange, 52); 364 break; 365 } 366 367 if ((lva && wi->txsz < 12) || (!lva && wi->txsz < 16)) 368 goto transfault; 369 370 /* R_YYVYV, I_THCZK */ 371 if ((!va55 && va > GENMASK(ia_bits - 1, 0)) || 372 (va55 && va < GENMASK(63, ia_bits))) 373 goto transfault; 374 375 /* I_ZFSYQ */ 376 if (wi->regime != TR_EL2 && 377 (tcr & (va55 ? TCR_EPD1_MASK : TCR_EPD0_MASK))) 378 goto transfault; 379 380 /* R_BNDVG and following statements */ 381 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && 382 wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) 383 goto transfault; 384 385 ps = (wi->regime == TR_EL2 ? 386 FIELD_GET(TCR_EL2_PS_MASK, tcr) : FIELD_GET(TCR_IPS_MASK, tcr)); 387 388 wi->max_oa_bits = min(get_kvm_ipa_limit(), ps_to_output_size(ps, wi->pa52bit)); 389 390 /* Compute minimal alignment */ 391 x = 3 + ia_bits - ((3 - wi->sl) * stride + wi->pgshift); 392 393 wi->baddr = ttbr & TTBRx_EL1_BADDR; 394 if (wi->pa52bit) { 395 /* 396 * Force the alignment on 64 bytes for top-level tables 397 * smaller than 8 entries, since TTBR.BADDR[5:2] are used to 398 * store bits [51:48] of the first level of lookup. 399 */ 400 x = max(x, 6); 401 402 wi->baddr |= FIELD_GET(GENMASK_ULL(5, 2), ttbr) << 48; 403 } 404 405 /* R_VPBBF */ 406 if (check_output_size(wi->baddr, wi)) 407 goto addrsz; 408 409 wi->baddr &= GENMASK_ULL(wi->max_oa_bits - 1, x); 410 411 wi->ha = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HAFDBS, AF); 412 wi->ha &= (wi->regime == TR_EL2 ? 413 FIELD_GET(TCR_EL2_HA, tcr) : 414 FIELD_GET(TCR_HA, tcr)); 415 416 return 0; 417 418 addrsz: 419 /* 420 * Address Size Fault level 0 to indicate it comes from TTBR. 421 * yes, this is an oddity. 422 */ 423 fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(0), false); 424 return -EFAULT; 425 426 transfault: 427 /* Translation Fault on start level */ 428 fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(wi->sl), false); 429 return -EFAULT; 430 } 431 432 static int kvm_read_s1_desc(struct kvm_vcpu *vcpu, u64 pa, u64 *desc, 433 struct s1_walk_info *wi) 434 { 435 u64 val; 436 int r; 437 438 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val)); 439 if (r) 440 return r; 441 442 if (wi->be) 443 *desc = be64_to_cpu((__force __be64)val); 444 else 445 *desc = le64_to_cpu((__force __le64)val); 446 447 return 0; 448 } 449 450 static int kvm_swap_s1_desc(struct kvm_vcpu *vcpu, u64 pa, u64 old, u64 new, 451 struct s1_walk_info *wi) 452 { 453 if (wi->be) { 454 old = (__force u64)cpu_to_be64(old); 455 new = (__force u64)cpu_to_be64(new); 456 } else { 457 old = (__force u64)cpu_to_le64(old); 458 new = (__force u64)cpu_to_le64(new); 459 } 460 461 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new); 462 } 463 464 static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 465 struct s1_walk_result *wr, u64 va) 466 { 467 u64 va_top, va_bottom, baddr, desc, new_desc, ipa; 468 struct kvm_s2_trans s2_trans = {}; 469 int level, stride, ret; 470 471 level = wi->sl; 472 stride = wi->pgshift - 3; 473 baddr = wi->baddr; 474 475 va_top = get_ia_size(wi) - 1; 476 477 while (1) { 478 u64 index; 479 480 va_bottom = (3 - level) * stride + wi->pgshift; 481 index = (va & GENMASK_ULL(va_top, va_bottom)) >> (va_bottom - 3); 482 483 ipa = baddr | index; 484 485 if (wi->s2) { 486 ret = kvm_walk_nested_s2(vcpu, ipa, &s2_trans); 487 if (ret == -EAGAIN) 488 return ret; 489 490 if (ret) { 491 fail_s1_walk(wr, 492 (s2_trans.esr & ~ESR_ELx_FSC_LEVEL) | level, 493 true); 494 return ret; 495 } 496 497 if (!kvm_s2_trans_readable(&s2_trans)) { 498 fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level), 499 true); 500 501 return -EPERM; 502 } 503 504 ipa = kvm_s2_trans_output(&s2_trans); 505 } 506 507 if (wi->filter) { 508 ret = wi->filter->fn(&(struct s1_walk_context) 509 { 510 .wi = wi, 511 .table_ipa = baddr, 512 .level = level, 513 }, wi->filter->priv); 514 if (ret) 515 return ret; 516 } 517 518 ret = kvm_read_s1_desc(vcpu, ipa, &desc, wi); 519 if (ret) { 520 fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); 521 return ret; 522 } 523 524 new_desc = desc; 525 526 /* Invalid descriptor */ 527 if (!(desc & BIT(0))) 528 goto transfault; 529 530 /* Block mapping, check validity down the line */ 531 if (!(desc & BIT(1))) 532 break; 533 534 /* Page mapping */ 535 if (level == 3) 536 break; 537 538 /* Table handling */ 539 if (!wi->hpd) { 540 wr->APTable |= FIELD_GET(S1_TABLE_AP, desc); 541 wr->UXNTable |= FIELD_GET(PMD_TABLE_UXN, desc); 542 wr->PXNTable |= FIELD_GET(PMD_TABLE_PXN, desc); 543 } 544 545 baddr = desc_to_oa(wi, desc); 546 547 /* Check for out-of-range OA */ 548 if (check_output_size(baddr, wi)) 549 goto addrsz; 550 551 /* Prepare for next round */ 552 va_top = va_bottom - 1; 553 level++; 554 } 555 556 /* Block mapping, check the validity of the level */ 557 if (!(desc & BIT(1))) { 558 bool valid_block = false; 559 bool lpa = kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52); 560 561 switch (BIT(wi->pgshift)) { 562 case SZ_4K: 563 valid_block = level == 1 || level == 2 || (wi->pa52bit && level == 0); 564 break; 565 case SZ_16K: 566 valid_block = level == 2 || (wi->pa52bit && level == 1); 567 break; 568 case SZ_64K: 569 valid_block = level == 2 || (lpa && level == 1); 570 break; 571 } 572 573 if (!valid_block) 574 goto transfault; 575 } 576 577 baddr = desc_to_oa(wi, desc); 578 if (check_output_size(baddr & GENMASK(52, va_bottom), wi)) 579 goto addrsz; 580 581 if (wi->ha) 582 new_desc |= PTE_AF; 583 584 if (new_desc != desc) { 585 if (wi->s2 && !kvm_s2_trans_writable(&s2_trans)) { 586 fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level), true); 587 return -EPERM; 588 } 589 590 ret = kvm_swap_s1_desc(vcpu, ipa, desc, new_desc, wi); 591 if (ret == -EAGAIN) 592 return ret; 593 if (ret) { 594 fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); 595 return ret; 596 } 597 598 desc = new_desc; 599 } 600 601 if (!(desc & PTE_AF)) { 602 fail_s1_walk(wr, ESR_ELx_FSC_ACCESS_L(level), false); 603 return -EACCES; 604 } 605 606 va_bottom += contiguous_bit_shift(desc, wi, level); 607 608 wr->failed = false; 609 wr->level = level; 610 wr->desc = desc; 611 wr->pa = baddr & GENMASK(52, va_bottom); 612 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); 613 614 wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); 615 if (wr->nG) 616 wr->asid = get_asid_by_regime(vcpu, wi->regime); 617 618 return 0; 619 620 addrsz: 621 fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(level), false); 622 return -EINVAL; 623 transfault: 624 fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(level), false); 625 return -ENOENT; 626 } 627 628 struct mmu_config { 629 u64 ttbr0; 630 u64 ttbr1; 631 u64 tcr; 632 u64 mair; 633 u64 tcr2; 634 u64 pir; 635 u64 pire0; 636 u64 por_el0; 637 u64 por_el1; 638 u64 sctlr; 639 u64 vttbr; 640 u64 vtcr; 641 }; 642 643 static void __mmu_config_save(struct mmu_config *config) 644 { 645 config->ttbr0 = read_sysreg_el1(SYS_TTBR0); 646 config->ttbr1 = read_sysreg_el1(SYS_TTBR1); 647 config->tcr = read_sysreg_el1(SYS_TCR); 648 config->mair = read_sysreg_el1(SYS_MAIR); 649 if (cpus_have_final_cap(ARM64_HAS_TCR2)) { 650 config->tcr2 = read_sysreg_el1(SYS_TCR2); 651 if (cpus_have_final_cap(ARM64_HAS_S1PIE)) { 652 config->pir = read_sysreg_el1(SYS_PIR); 653 config->pire0 = read_sysreg_el1(SYS_PIRE0); 654 } 655 if (system_supports_poe()) { 656 config->por_el1 = read_sysreg_el1(SYS_POR); 657 config->por_el0 = read_sysreg_s(SYS_POR_EL0); 658 } 659 } 660 config->sctlr = read_sysreg_el1(SYS_SCTLR); 661 config->vttbr = read_sysreg(vttbr_el2); 662 config->vtcr = read_sysreg(vtcr_el2); 663 } 664 665 static void __mmu_config_restore(struct mmu_config *config) 666 { 667 /* 668 * ARM errata 1165522 and 1530923 require TGE to be 1 before 669 * we update the guest state. 670 */ 671 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); 672 673 write_sysreg_el1(config->ttbr0, SYS_TTBR0); 674 write_sysreg_el1(config->ttbr1, SYS_TTBR1); 675 write_sysreg_el1(config->tcr, SYS_TCR); 676 write_sysreg_el1(config->mair, SYS_MAIR); 677 if (cpus_have_final_cap(ARM64_HAS_TCR2)) { 678 write_sysreg_el1(config->tcr2, SYS_TCR2); 679 if (cpus_have_final_cap(ARM64_HAS_S1PIE)) { 680 write_sysreg_el1(config->pir, SYS_PIR); 681 write_sysreg_el1(config->pire0, SYS_PIRE0); 682 } 683 if (system_supports_poe()) { 684 write_sysreg_el1(config->por_el1, SYS_POR); 685 write_sysreg_s(config->por_el0, SYS_POR_EL0); 686 } 687 } 688 write_sysreg_el1(config->sctlr, SYS_SCTLR); 689 write_sysreg(config->vttbr, vttbr_el2); 690 write_sysreg(config->vtcr, vtcr_el2); 691 } 692 693 static bool at_s1e1p_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 694 { 695 u64 host_pan; 696 bool fail; 697 698 host_pan = read_sysreg_s(SYS_PSTATE_PAN); 699 write_sysreg_s(*vcpu_cpsr(vcpu) & PSTATE_PAN, SYS_PSTATE_PAN); 700 701 switch (op) { 702 case OP_AT_S1E1RP: 703 fail = __kvm_at(OP_AT_S1E1RP, vaddr); 704 break; 705 case OP_AT_S1E1WP: 706 fail = __kvm_at(OP_AT_S1E1WP, vaddr); 707 break; 708 } 709 710 write_sysreg_s(host_pan, SYS_PSTATE_PAN); 711 712 return fail; 713 } 714 715 #define MEMATTR(ic, oc) (MEMATTR_##oc << 4 | MEMATTR_##ic) 716 #define MEMATTR_NC 0b0100 717 #define MEMATTR_Wt 0b1000 718 #define MEMATTR_Wb 0b1100 719 #define MEMATTR_WbRaWa 0b1111 720 721 #define MEMATTR_IS_DEVICE(m) (((m) & GENMASK(7, 4)) == 0) 722 723 static u8 s2_memattr_to_attr(u8 memattr) 724 { 725 memattr &= 0b1111; 726 727 switch (memattr) { 728 case 0b0000: 729 case 0b0001: 730 case 0b0010: 731 case 0b0011: 732 return memattr << 2; 733 case 0b0100: 734 return MEMATTR(Wb, Wb); 735 case 0b0101: 736 return MEMATTR(NC, NC); 737 case 0b0110: 738 return MEMATTR(Wt, NC); 739 case 0b0111: 740 return MEMATTR(Wb, NC); 741 case 0b1000: 742 /* Reserved, assume NC */ 743 return MEMATTR(NC, NC); 744 case 0b1001: 745 return MEMATTR(NC, Wt); 746 case 0b1010: 747 return MEMATTR(Wt, Wt); 748 case 0b1011: 749 return MEMATTR(Wb, Wt); 750 case 0b1100: 751 /* Reserved, assume NC */ 752 return MEMATTR(NC, NC); 753 case 0b1101: 754 return MEMATTR(NC, Wb); 755 case 0b1110: 756 return MEMATTR(Wt, Wb); 757 case 0b1111: 758 return MEMATTR(Wb, Wb); 759 default: 760 unreachable(); 761 } 762 } 763 764 static u8 combine_s1_s2_attr(u8 s1, u8 s2) 765 { 766 bool transient; 767 u8 final = 0; 768 769 /* Upgrade transient s1 to non-transient to simplify things */ 770 switch (s1) { 771 case 0b0001 ... 0b0011: /* Normal, Write-Through Transient */ 772 transient = true; 773 s1 = MEMATTR_Wt | (s1 & GENMASK(1,0)); 774 break; 775 case 0b0101 ... 0b0111: /* Normal, Write-Back Transient */ 776 transient = true; 777 s1 = MEMATTR_Wb | (s1 & GENMASK(1,0)); 778 break; 779 default: 780 transient = false; 781 } 782 783 /* S2CombineS1AttrHints() */ 784 if ((s1 & GENMASK(3, 2)) == MEMATTR_NC || 785 (s2 & GENMASK(3, 2)) == MEMATTR_NC) 786 final = MEMATTR_NC; 787 else if ((s1 & GENMASK(3, 2)) == MEMATTR_Wt || 788 (s2 & GENMASK(3, 2)) == MEMATTR_Wt) 789 final = MEMATTR_Wt; 790 else 791 final = MEMATTR_Wb; 792 793 if (final != MEMATTR_NC) { 794 /* Inherit RaWa hints form S1 */ 795 if (transient) { 796 switch (s1 & GENMASK(3, 2)) { 797 case MEMATTR_Wt: 798 final = 0; 799 break; 800 case MEMATTR_Wb: 801 final = MEMATTR_NC; 802 break; 803 } 804 } 805 806 final |= s1 & GENMASK(1, 0); 807 } 808 809 return final; 810 } 811 812 #define ATTR_NSH 0b00 813 #define ATTR_RSV 0b01 814 #define ATTR_OSH 0b10 815 #define ATTR_ISH 0b11 816 817 static u8 compute_final_sh(u8 attr, u8 sh) 818 { 819 /* Any form of device, as well as NC has SH[1:0]=0b10 */ 820 if (MEMATTR_IS_DEVICE(attr) || attr == MEMATTR(NC, NC)) 821 return ATTR_OSH; 822 823 if (sh == ATTR_RSV) /* Reserved, mapped to NSH */ 824 sh = ATTR_NSH; 825 826 return sh; 827 } 828 829 static u8 compute_s1_sh(struct s1_walk_info *wi, struct s1_walk_result *wr, 830 u8 attr) 831 { 832 u8 sh; 833 834 /* 835 * non-52bit and LPA have their basic shareability described in the 836 * descriptor. LPA2 gets it from the corresponding field in TCR, 837 * conveniently recorded in the walk info. 838 */ 839 if (!wi->pa52bit || BIT(wi->pgshift) == SZ_64K) 840 sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_SH, wr->desc); 841 else 842 sh = wi->sh; 843 844 return compute_final_sh(attr, sh); 845 } 846 847 static u8 combine_sh(u8 s1_sh, u8 s2_sh) 848 { 849 if (s1_sh == ATTR_OSH || s2_sh == ATTR_OSH) 850 return ATTR_OSH; 851 if (s1_sh == ATTR_ISH || s2_sh == ATTR_ISH) 852 return ATTR_ISH; 853 854 return ATTR_NSH; 855 } 856 857 static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, 858 struct kvm_s2_trans *tr) 859 { 860 u8 s1_parattr, s2_memattr, final_attr, s2_sh; 861 u64 par; 862 863 /* If S2 has failed to translate, report the damage */ 864 if (tr->esr) { 865 par = SYS_PAR_EL1_RES1; 866 par |= SYS_PAR_EL1_F; 867 par |= SYS_PAR_EL1_S; 868 par |= FIELD_PREP(SYS_PAR_EL1_FST, tr->esr); 869 return par; 870 } 871 872 s1_parattr = FIELD_GET(SYS_PAR_EL1_ATTR, s1_par); 873 s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc); 874 875 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_FWB) { 876 if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP)) 877 s2_memattr &= ~BIT(3); 878 879 /* Combination of R_VRJSW and R_RHWZM */ 880 switch (s2_memattr) { 881 case 0b0101: 882 if (MEMATTR_IS_DEVICE(s1_parattr)) 883 final_attr = s1_parattr; 884 else 885 final_attr = MEMATTR(NC, NC); 886 break; 887 case 0b0110: 888 case 0b1110: 889 final_attr = MEMATTR(WbRaWa, WbRaWa); 890 break; 891 case 0b0111: 892 case 0b1111: 893 /* Preserve S1 attribute */ 894 final_attr = s1_parattr; 895 break; 896 case 0b0100: 897 case 0b1100: 898 case 0b1101: 899 /* Reserved, do something non-silly */ 900 final_attr = s1_parattr; 901 break; 902 default: 903 /* 904 * MemAttr[2]=0, Device from S2. 905 * 906 * FWB does not influence the way that stage 1 907 * memory types and attributes are combined 908 * with stage 2 Device type and attributes. 909 */ 910 final_attr = min(s2_memattr_to_attr(s2_memattr), 911 s1_parattr); 912 } 913 } else { 914 /* Combination of R_HMNDG, R_TNHFM and R_GQFSF */ 915 u8 s2_parattr = s2_memattr_to_attr(s2_memattr); 916 917 if (MEMATTR_IS_DEVICE(s1_parattr) || 918 MEMATTR_IS_DEVICE(s2_parattr)) { 919 final_attr = min(s1_parattr, s2_parattr); 920 } else { 921 /* At this stage, this is memory vs memory */ 922 final_attr = combine_s1_s2_attr(s1_parattr & 0xf, 923 s2_parattr & 0xf); 924 final_attr |= combine_s1_s2_attr(s1_parattr >> 4, 925 s2_parattr >> 4) << 4; 926 } 927 } 928 929 if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_CD) && 930 !MEMATTR_IS_DEVICE(final_attr)) 931 final_attr = MEMATTR(NC, NC); 932 933 s2_sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S2_SH, tr->desc); 934 935 par = FIELD_PREP(SYS_PAR_EL1_ATTR, final_attr); 936 par |= tr->output & GENMASK(47, 12); 937 par |= FIELD_PREP(SYS_PAR_EL1_SH, 938 combine_sh(FIELD_GET(SYS_PAR_EL1_SH, s1_par), 939 compute_final_sh(final_attr, s2_sh))); 940 941 return par; 942 } 943 944 static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 945 struct s1_walk_result *wr) 946 { 947 u64 par; 948 949 if (wr->failed) { 950 par = SYS_PAR_EL1_RES1; 951 par |= SYS_PAR_EL1_F; 952 par |= FIELD_PREP(SYS_PAR_EL1_FST, wr->fst); 953 par |= wr->ptw ? SYS_PAR_EL1_PTW : 0; 954 par |= wr->s2 ? SYS_PAR_EL1_S : 0; 955 } else if (wr->level == S1_MMU_DISABLED) { 956 /* MMU off or HCR_EL2.DC == 1 */ 957 par = SYS_PAR_EL1_NSE; 958 par |= wr->pa & SYS_PAR_EL1_PA; 959 960 if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && 961 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) { 962 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, 963 MEMATTR(WbRaWa, WbRaWa)); 964 par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_NSH); 965 } else { 966 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, 0); /* nGnRnE */ 967 par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_OSH); 968 } 969 } else { 970 u64 mair, sctlr; 971 u8 sh; 972 973 par = SYS_PAR_EL1_NSE; 974 975 mair = (wi->regime == TR_EL10 ? 976 vcpu_read_sys_reg(vcpu, MAIR_EL1) : 977 vcpu_read_sys_reg(vcpu, MAIR_EL2)); 978 979 mair >>= FIELD_GET(PTE_ATTRINDX_MASK, wr->desc) * 8; 980 mair &= 0xff; 981 982 sctlr = (wi->regime == TR_EL10 ? 983 vcpu_read_sys_reg(vcpu, SCTLR_EL1) : 984 vcpu_read_sys_reg(vcpu, SCTLR_EL2)); 985 986 /* Force NC for memory if SCTLR_ELx.C is clear */ 987 if (!(sctlr & SCTLR_EL1_C) && !MEMATTR_IS_DEVICE(mair)) 988 mair = MEMATTR(NC, NC); 989 990 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, mair); 991 par |= wr->pa & SYS_PAR_EL1_PA; 992 993 sh = compute_s1_sh(wi, wr, mair); 994 par |= FIELD_PREP(SYS_PAR_EL1_SH, sh); 995 } 996 997 return par; 998 } 999 1000 static bool pan3_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 1001 { 1002 u64 sctlr; 1003 1004 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) 1005 return false; 1006 1007 if (s1pie_enabled(vcpu, regime)) 1008 return true; 1009 1010 if (regime == TR_EL10) 1011 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 1012 else 1013 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); 1014 1015 return sctlr & SCTLR_EL1_EPAN; 1016 } 1017 1018 static void compute_s1_direct_permissions(struct kvm_vcpu *vcpu, 1019 struct s1_walk_info *wi, 1020 struct s1_walk_result *wr) 1021 { 1022 bool wxn; 1023 1024 /* Non-hierarchical part of AArch64.S1DirectBasePermissions() */ 1025 if (wi->regime != TR_EL2) { 1026 switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr->desc)) { 1027 case 0b00: 1028 wr->pr = wr->pw = true; 1029 wr->ur = wr->uw = false; 1030 break; 1031 case 0b01: 1032 wr->pr = wr->pw = wr->ur = wr->uw = true; 1033 break; 1034 case 0b10: 1035 wr->pr = true; 1036 wr->pw = wr->ur = wr->uw = false; 1037 break; 1038 case 0b11: 1039 wr->pr = wr->ur = true; 1040 wr->pw = wr->uw = false; 1041 break; 1042 } 1043 1044 /* We don't use px for anything yet, but hey... */ 1045 wr->px = !((wr->desc & PTE_PXN) || wr->uw); 1046 wr->ux = !(wr->desc & PTE_UXN); 1047 } else { 1048 wr->ur = wr->uw = wr->ux = false; 1049 1050 if (!(wr->desc & PTE_RDONLY)) { 1051 wr->pr = wr->pw = true; 1052 } else { 1053 wr->pr = true; 1054 wr->pw = false; 1055 } 1056 1057 /* XN maps to UXN */ 1058 wr->px = !(wr->desc & PTE_UXN); 1059 } 1060 1061 switch (wi->regime) { 1062 case TR_EL2: 1063 case TR_EL20: 1064 wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_WXN); 1065 break; 1066 case TR_EL10: 1067 wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_WXN); 1068 break; 1069 } 1070 1071 wr->pwxn = wr->uwxn = wxn; 1072 wr->pov = wi->poe; 1073 wr->uov = wi->e0poe; 1074 } 1075 1076 static void compute_s1_hierarchical_permissions(struct kvm_vcpu *vcpu, 1077 struct s1_walk_info *wi, 1078 struct s1_walk_result *wr) 1079 { 1080 /* Hierarchical part of AArch64.S1DirectBasePermissions() */ 1081 if (wi->regime != TR_EL2) { 1082 switch (wr->APTable) { 1083 case 0b00: 1084 break; 1085 case 0b01: 1086 wr->ur = wr->uw = false; 1087 break; 1088 case 0b10: 1089 wr->pw = wr->uw = false; 1090 break; 1091 case 0b11: 1092 wr->pw = wr->ur = wr->uw = false; 1093 break; 1094 } 1095 1096 wr->px &= !wr->PXNTable; 1097 wr->ux &= !wr->UXNTable; 1098 } else { 1099 if (wr->APTable & BIT(1)) 1100 wr->pw = false; 1101 1102 /* XN maps to UXN */ 1103 wr->px &= !wr->UXNTable; 1104 } 1105 } 1106 1107 #define perm_idx(v, r, i) ((vcpu_read_sys_reg((v), (r)) >> ((i) * 4)) & 0xf) 1108 1109 #define set_priv_perms(wr, r, w, x) \ 1110 do { \ 1111 (wr)->pr = (r); \ 1112 (wr)->pw = (w); \ 1113 (wr)->px = (x); \ 1114 } while (0) 1115 1116 #define set_unpriv_perms(wr, r, w, x) \ 1117 do { \ 1118 (wr)->ur = (r); \ 1119 (wr)->uw = (w); \ 1120 (wr)->ux = (x); \ 1121 } while (0) 1122 1123 #define set_priv_wxn(wr, v) \ 1124 do { \ 1125 (wr)->pwxn = (v); \ 1126 } while (0) 1127 1128 #define set_unpriv_wxn(wr, v) \ 1129 do { \ 1130 (wr)->uwxn = (v); \ 1131 } while (0) 1132 1133 /* Similar to AArch64.S1IndirectBasePermissions(), without GCS */ 1134 #define set_perms(w, wr, ip) \ 1135 do { \ 1136 /* R_LLZDZ */ \ 1137 switch ((ip)) { \ 1138 case 0b0000: \ 1139 set_ ## w ## _perms((wr), false, false, false); \ 1140 break; \ 1141 case 0b0001: \ 1142 set_ ## w ## _perms((wr), true , false, false); \ 1143 break; \ 1144 case 0b0010: \ 1145 set_ ## w ## _perms((wr), false, false, true ); \ 1146 break; \ 1147 case 0b0011: \ 1148 set_ ## w ## _perms((wr), true , false, true ); \ 1149 break; \ 1150 case 0b0100: \ 1151 set_ ## w ## _perms((wr), false, false, false); \ 1152 break; \ 1153 case 0b0101: \ 1154 set_ ## w ## _perms((wr), true , true , false); \ 1155 break; \ 1156 case 0b0110: \ 1157 set_ ## w ## _perms((wr), true , true , true ); \ 1158 break; \ 1159 case 0b0111: \ 1160 set_ ## w ## _perms((wr), true , true , true ); \ 1161 break; \ 1162 case 0b1000: \ 1163 set_ ## w ## _perms((wr), true , false, false); \ 1164 break; \ 1165 case 0b1001: \ 1166 set_ ## w ## _perms((wr), true , false, false); \ 1167 break; \ 1168 case 0b1010: \ 1169 set_ ## w ## _perms((wr), true , false, true ); \ 1170 break; \ 1171 case 0b1011: \ 1172 set_ ## w ## _perms((wr), false, false, false); \ 1173 break; \ 1174 case 0b1100: \ 1175 set_ ## w ## _perms((wr), true , true , false); \ 1176 break; \ 1177 case 0b1101: \ 1178 set_ ## w ## _perms((wr), false, false, false); \ 1179 break; \ 1180 case 0b1110: \ 1181 set_ ## w ## _perms((wr), true , true , true ); \ 1182 break; \ 1183 case 0b1111: \ 1184 set_ ## w ## _perms((wr), false, false, false); \ 1185 break; \ 1186 } \ 1187 \ 1188 /* R_HJYGR */ \ 1189 set_ ## w ## _wxn((wr), ((ip) == 0b0110)); \ 1190 \ 1191 } while (0) 1192 1193 static void compute_s1_indirect_permissions(struct kvm_vcpu *vcpu, 1194 struct s1_walk_info *wi, 1195 struct s1_walk_result *wr) 1196 { 1197 u8 up, pp, idx; 1198 1199 idx = pte_pi_index(wr->desc); 1200 1201 switch (wi->regime) { 1202 case TR_EL10: 1203 pp = perm_idx(vcpu, PIR_EL1, idx); 1204 up = perm_idx(vcpu, PIRE0_EL1, idx); 1205 break; 1206 case TR_EL20: 1207 pp = perm_idx(vcpu, PIR_EL2, idx); 1208 up = perm_idx(vcpu, PIRE0_EL2, idx); 1209 break; 1210 case TR_EL2: 1211 pp = perm_idx(vcpu, PIR_EL2, idx); 1212 up = 0; 1213 break; 1214 } 1215 1216 set_perms(priv, wr, pp); 1217 1218 if (wi->regime != TR_EL2) 1219 set_perms(unpriv, wr, up); 1220 else 1221 set_unpriv_perms(wr, false, false, false); 1222 1223 wr->pov = wi->poe && !(pp & BIT(3)); 1224 wr->uov = wi->e0poe && !(up & BIT(3)); 1225 1226 /* R_VFPJF */ 1227 if (wr->px && wr->uw) { 1228 set_priv_perms(wr, false, false, false); 1229 set_unpriv_perms(wr, false, false, false); 1230 } 1231 } 1232 1233 static void compute_s1_overlay_permissions(struct kvm_vcpu *vcpu, 1234 struct s1_walk_info *wi, 1235 struct s1_walk_result *wr) 1236 { 1237 u8 idx, pov_perms, uov_perms; 1238 1239 idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc); 1240 1241 if (wr->pov) { 1242 switch (wi->regime) { 1243 case TR_EL10: 1244 pov_perms = perm_idx(vcpu, POR_EL1, idx); 1245 break; 1246 case TR_EL20: 1247 pov_perms = perm_idx(vcpu, POR_EL2, idx); 1248 break; 1249 case TR_EL2: 1250 pov_perms = perm_idx(vcpu, POR_EL2, idx); 1251 break; 1252 } 1253 1254 if (pov_perms & ~POE_RWX) 1255 pov_perms = POE_NONE; 1256 1257 /* R_QXXPC, S1PrivOverflow enabled */ 1258 if (wr->pwxn && (pov_perms & POE_X)) 1259 pov_perms &= ~POE_W; 1260 1261 wr->pr &= pov_perms & POE_R; 1262 wr->pw &= pov_perms & POE_W; 1263 wr->px &= pov_perms & POE_X; 1264 } 1265 1266 if (wr->uov) { 1267 switch (wi->regime) { 1268 case TR_EL10: 1269 uov_perms = perm_idx(vcpu, POR_EL0, idx); 1270 break; 1271 case TR_EL20: 1272 uov_perms = perm_idx(vcpu, POR_EL0, idx); 1273 break; 1274 case TR_EL2: 1275 uov_perms = 0; 1276 break; 1277 } 1278 1279 if (uov_perms & ~POE_RWX) 1280 uov_perms = POE_NONE; 1281 1282 /* R_NPBXC, S1UnprivOverlay enabled */ 1283 if (wr->uwxn && (uov_perms & POE_X)) 1284 uov_perms &= ~POE_W; 1285 1286 wr->ur &= uov_perms & POE_R; 1287 wr->uw &= uov_perms & POE_W; 1288 wr->ux &= uov_perms & POE_X; 1289 } 1290 } 1291 1292 static void compute_s1_permissions(struct kvm_vcpu *vcpu, 1293 struct s1_walk_info *wi, 1294 struct s1_walk_result *wr) 1295 { 1296 bool pan; 1297 1298 if (!s1pie_enabled(vcpu, wi->regime)) 1299 compute_s1_direct_permissions(vcpu, wi, wr); 1300 else 1301 compute_s1_indirect_permissions(vcpu, wi, wr); 1302 1303 if (!wi->hpd) 1304 compute_s1_hierarchical_permissions(vcpu, wi, wr); 1305 1306 compute_s1_overlay_permissions(vcpu, wi, wr); 1307 1308 /* R_QXXPC, S1PrivOverlay disabled */ 1309 if (!wr->pov) 1310 wr->px &= !(wr->pwxn && wr->pw); 1311 1312 /* R_NPBXC, S1UnprivOverlay disabled */ 1313 if (!wr->uov) 1314 wr->ux &= !(wr->uwxn && wr->uw); 1315 1316 pan = wi->pan && (wr->ur || wr->uw || 1317 (pan3_enabled(vcpu, wi->regime) && wr->ux)); 1318 wr->pw &= !pan; 1319 wr->pr &= !pan; 1320 } 1321 1322 static int handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr, u64 *par) 1323 { 1324 struct s1_walk_result wr = {}; 1325 struct s1_walk_info wi = {}; 1326 bool perm_fail = false; 1327 int ret, idx; 1328 1329 wi.regime = compute_translation_regime(vcpu, op); 1330 wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); 1331 wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && 1332 (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); 1333 1334 ret = setup_s1_walk(vcpu, &wi, &wr, vaddr); 1335 if (ret) 1336 goto compute_par; 1337 1338 if (wr.level == S1_MMU_DISABLED) 1339 goto compute_par; 1340 1341 idx = srcu_read_lock(&vcpu->kvm->srcu); 1342 1343 ret = walk_s1(vcpu, &wi, &wr, vaddr); 1344 1345 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1346 1347 /* 1348 * Race to update a descriptor -- restart the walk. 1349 */ 1350 if (ret == -EAGAIN) 1351 return ret; 1352 if (ret) 1353 goto compute_par; 1354 1355 compute_s1_permissions(vcpu, &wi, &wr); 1356 1357 switch (op) { 1358 case OP_AT_S1E1RP: 1359 case OP_AT_S1E1R: 1360 case OP_AT_S1E2R: 1361 perm_fail = !wr.pr; 1362 break; 1363 case OP_AT_S1E1WP: 1364 case OP_AT_S1E1W: 1365 case OP_AT_S1E2W: 1366 perm_fail = !wr.pw; 1367 break; 1368 case OP_AT_S1E0R: 1369 perm_fail = !wr.ur; 1370 break; 1371 case OP_AT_S1E0W: 1372 perm_fail = !wr.uw; 1373 break; 1374 case OP_AT_S1E1A: 1375 case OP_AT_S1E2A: 1376 break; 1377 default: 1378 BUG(); 1379 } 1380 1381 if (perm_fail) 1382 fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false); 1383 1384 compute_par: 1385 *par = compute_par_s1(vcpu, &wi, &wr); 1386 return 0; 1387 } 1388 1389 /* 1390 * Return the PAR_EL1 value as the result of a valid translation. 1391 * 1392 * If the translation is unsuccessful, the value may only contain 1393 * PAR_EL1.F, and cannot be taken at face value. It isn't an 1394 * indication of the translation having failed, only that the fast 1395 * path did not succeed, *unless* it indicates a S1 permission or 1396 * access fault. 1397 */ 1398 static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1399 { 1400 struct mmu_config config; 1401 struct kvm_s2_mmu *mmu; 1402 bool fail, mmu_cs; 1403 u64 par; 1404 1405 par = SYS_PAR_EL1_F; 1406 1407 /* 1408 * We've trapped, so everything is live on the CPU. As we will 1409 * be switching contexts behind everybody's back, disable 1410 * interrupts while holding the mmu lock. 1411 */ 1412 guard(write_lock_irqsave)(&vcpu->kvm->mmu_lock); 1413 1414 /* 1415 * If HCR_EL2.{E2H,TGE} == {1,1}, the MMU context is already 1416 * the right one (as we trapped from vEL2). If not, save the 1417 * full MMU context. 1418 * 1419 * We are also guaranteed to be in the correct context if 1420 * we're not in a nested VM. 1421 */ 1422 mmu_cs = (vcpu_has_nv(vcpu) && 1423 !(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))); 1424 if (!mmu_cs) 1425 goto skip_mmu_switch; 1426 1427 /* 1428 * Obtaining the S2 MMU for a L2 is horribly racy, and we may not 1429 * find it (recycled by another vcpu, for example). When this 1430 * happens, admit defeat immediately and use the SW (slow) path. 1431 */ 1432 mmu = lookup_s2_mmu(vcpu); 1433 if (!mmu) 1434 return par; 1435 1436 __mmu_config_save(&config); 1437 1438 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR0_EL1), SYS_TTBR0); 1439 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR1_EL1), SYS_TTBR1); 1440 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR_EL1), SYS_TCR); 1441 write_sysreg_el1(vcpu_read_sys_reg(vcpu, MAIR_EL1), SYS_MAIR); 1442 if (kvm_has_tcr2(vcpu->kvm)) { 1443 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR2_EL1), SYS_TCR2); 1444 if (kvm_has_s1pie(vcpu->kvm)) { 1445 write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIR_EL1), SYS_PIR); 1446 write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIRE0_EL1), SYS_PIRE0); 1447 } 1448 if (kvm_has_s1poe(vcpu->kvm)) { 1449 write_sysreg_el1(vcpu_read_sys_reg(vcpu, POR_EL1), SYS_POR); 1450 write_sysreg_s(vcpu_read_sys_reg(vcpu, POR_EL0), SYS_POR_EL0); 1451 } 1452 } 1453 write_sysreg_el1(vcpu_read_sys_reg(vcpu, SCTLR_EL1), SYS_SCTLR); 1454 __load_stage2(mmu); 1455 1456 skip_mmu_switch: 1457 /* Temporarily switch back to guest context */ 1458 write_sysreg_hcr(vcpu->arch.hcr_el2); 1459 isb(); 1460 1461 switch (op) { 1462 case OP_AT_S1E1RP: 1463 case OP_AT_S1E1WP: 1464 fail = at_s1e1p_fast(vcpu, op, vaddr); 1465 break; 1466 case OP_AT_S1E1R: 1467 fail = __kvm_at(OP_AT_S1E1R, vaddr); 1468 break; 1469 case OP_AT_S1E1W: 1470 fail = __kvm_at(OP_AT_S1E1W, vaddr); 1471 break; 1472 case OP_AT_S1E0R: 1473 fail = __kvm_at(OP_AT_S1E0R, vaddr); 1474 break; 1475 case OP_AT_S1E0W: 1476 fail = __kvm_at(OP_AT_S1E0W, vaddr); 1477 break; 1478 case OP_AT_S1E1A: 1479 fail = __kvm_at(OP_AT_S1E1A, vaddr); 1480 break; 1481 default: 1482 WARN_ON_ONCE(1); 1483 fail = true; 1484 break; 1485 } 1486 1487 if (!fail) 1488 par = read_sysreg_par(); 1489 1490 write_sysreg_hcr(HCR_HOST_VHE_FLAGS); 1491 1492 if (mmu_cs) 1493 __mmu_config_restore(&config); 1494 1495 return par; 1496 } 1497 1498 static bool par_check_s1_perm_fault(u64 par) 1499 { 1500 u8 fst = FIELD_GET(SYS_PAR_EL1_FST, par); 1501 1502 return ((fst & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM && 1503 !(par & SYS_PAR_EL1_S)); 1504 } 1505 1506 static bool par_check_s1_access_fault(u64 par) 1507 { 1508 u8 fst = FIELD_GET(SYS_PAR_EL1_FST, par); 1509 1510 return ((fst & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_ACCESS && 1511 !(par & SYS_PAR_EL1_S)); 1512 } 1513 1514 int __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1515 { 1516 u64 par = __kvm_at_s1e01_fast(vcpu, op, vaddr); 1517 int ret; 1518 1519 /* 1520 * If PAR_EL1 reports that AT failed on a S1 permission or access 1521 * fault, we know for sure that the PTW was able to walk the S1 1522 * tables and there's nothing else to do. 1523 * 1524 * If AT failed for any other reason, then we must walk the guest S1 1525 * to emulate the instruction. 1526 */ 1527 if ((par & SYS_PAR_EL1_F) && 1528 !par_check_s1_perm_fault(par) && 1529 !par_check_s1_access_fault(par)) { 1530 ret = handle_at_slow(vcpu, op, vaddr, &par); 1531 if (ret) 1532 return ret; 1533 } 1534 1535 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1536 return 0; 1537 } 1538 1539 int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1540 { 1541 u64 par; 1542 int ret; 1543 1544 /* 1545 * We've trapped, so everything is live on the CPU. As we will be 1546 * switching context behind everybody's back, disable interrupts... 1547 */ 1548 scoped_guard(write_lock_irqsave, &vcpu->kvm->mmu_lock) { 1549 u64 val, hcr; 1550 bool fail; 1551 1552 val = hcr = read_sysreg(hcr_el2); 1553 val &= ~HCR_TGE; 1554 val |= HCR_VM; 1555 1556 if (!vcpu_el2_e2h_is_set(vcpu)) 1557 val |= HCR_NV | HCR_NV1; 1558 1559 write_sysreg_hcr(val); 1560 isb(); 1561 1562 par = SYS_PAR_EL1_F; 1563 1564 switch (op) { 1565 case OP_AT_S1E2R: 1566 fail = __kvm_at(OP_AT_S1E1R, vaddr); 1567 break; 1568 case OP_AT_S1E2W: 1569 fail = __kvm_at(OP_AT_S1E1W, vaddr); 1570 break; 1571 case OP_AT_S1E2A: 1572 fail = __kvm_at(OP_AT_S1E1A, vaddr); 1573 break; 1574 default: 1575 WARN_ON_ONCE(1); 1576 fail = true; 1577 } 1578 1579 if (!fail) 1580 par = read_sysreg_par(); 1581 1582 write_sysreg_hcr(hcr); 1583 isb(); 1584 } 1585 1586 /* We failed the translation, let's replay it in slow motion */ 1587 if ((par & SYS_PAR_EL1_F) && !par_check_s1_perm_fault(par)) { 1588 ret = handle_at_slow(vcpu, op, vaddr, &par); 1589 if (ret) 1590 return ret; 1591 } 1592 1593 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1594 return 0; 1595 } 1596 1597 int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1598 { 1599 struct kvm_s2_trans out = {}; 1600 u64 ipa, par; 1601 bool write; 1602 int ret; 1603 1604 /* Do the stage-1 translation */ 1605 switch (op) { 1606 case OP_AT_S12E1R: 1607 op = OP_AT_S1E1R; 1608 write = false; 1609 break; 1610 case OP_AT_S12E1W: 1611 op = OP_AT_S1E1W; 1612 write = true; 1613 break; 1614 case OP_AT_S12E0R: 1615 op = OP_AT_S1E0R; 1616 write = false; 1617 break; 1618 case OP_AT_S12E0W: 1619 op = OP_AT_S1E0W; 1620 write = true; 1621 break; 1622 default: 1623 WARN_ON_ONCE(1); 1624 return 0; 1625 } 1626 1627 ret = __kvm_at_s1e01(vcpu, op, vaddr); 1628 if (ret) 1629 return ret; 1630 1631 par = vcpu_read_sys_reg(vcpu, PAR_EL1); 1632 if (par & SYS_PAR_EL1_F) 1633 return 0; 1634 1635 /* 1636 * If we only have a single stage of translation (EL2&0), exit 1637 * early. Same thing if {VM,DC}=={0,0}. 1638 */ 1639 if (compute_translation_regime(vcpu, op) == TR_EL20 || 1640 !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM | HCR_DC))) 1641 return 0; 1642 1643 /* Do the stage-2 translation */ 1644 ipa = (par & GENMASK_ULL(47, 12)) | (vaddr & GENMASK_ULL(11, 0)); 1645 out.esr = 0; 1646 scoped_guard(srcu, &vcpu->kvm->srcu) 1647 ret = kvm_walk_nested_s2(vcpu, ipa, &out); 1648 if (ret < 0) 1649 return ret; 1650 1651 /* Check the access permission */ 1652 if (!out.esr && 1653 ((!write && !out.readable) || (write && !out.writable))) 1654 out.esr = ESR_ELx_FSC_PERM_L(out.level & 0x3); 1655 1656 par = compute_par_s12(vcpu, par, &out); 1657 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1658 return 0; 1659 } 1660 1661 /* 1662 * Translate a VA for a given EL in a given translation regime, with 1663 * or without PAN. This requires wi->{regime, as_el0, pan} to be 1664 * set. The rest of the wi and wr should be 0-initialised. 1665 */ 1666 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 1667 struct s1_walk_result *wr, u64 va) 1668 { 1669 int ret; 1670 1671 ret = setup_s1_walk(vcpu, wi, wr, va); 1672 if (ret) 1673 return ret; 1674 1675 if (wr->level == S1_MMU_DISABLED) { 1676 wr->ur = wr->uw = wr->ux = true; 1677 wr->pr = wr->pw = wr->px = true; 1678 } else { 1679 ret = walk_s1(vcpu, wi, wr, va); 1680 if (ret) 1681 return ret; 1682 1683 compute_s1_permissions(vcpu, wi, wr); 1684 } 1685 1686 return 0; 1687 } 1688 1689 struct desc_match { 1690 u64 ipa; 1691 int level; 1692 }; 1693 1694 static int match_s1_desc(struct s1_walk_context *ctxt, void *priv) 1695 { 1696 struct desc_match *dm = priv; 1697 u64 ipa = dm->ipa; 1698 1699 /* Use S1 granule alignment */ 1700 ipa &= GENMASK(51, ctxt->wi->pgshift); 1701 1702 /* Not the IPA we're looking for? Continue. */ 1703 if (ipa != ctxt->table_ipa) 1704 return 0; 1705 1706 /* Note the level and interrupt the walk */ 1707 dm->level = ctxt->level; 1708 return -EINTR; 1709 } 1710 1711 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, int *level) 1712 { 1713 struct desc_match dm = { 1714 .ipa = ipa, 1715 }; 1716 struct s1_walk_info wi = { 1717 .filter = &(struct s1_walk_filter){ 1718 .fn = match_s1_desc, 1719 .priv = &dm, 1720 }, 1721 .as_el0 = false, 1722 .pan = false, 1723 }; 1724 struct s1_walk_result wr = {}; 1725 int ret; 1726 1727 if (is_hyp_ctxt(vcpu)) 1728 wi.regime = vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2; 1729 else 1730 wi.regime = TR_EL10; 1731 1732 ret = setup_s1_walk(vcpu, &wi, &wr, va); 1733 if (ret) 1734 return ret; 1735 1736 /* We really expect the S1 MMU to be on here... */ 1737 if (WARN_ON_ONCE(wr.level == S1_MMU_DISABLED)) { 1738 *level = 0; 1739 return 0; 1740 } 1741 1742 /* Walk the guest's PT, looking for a match along the way */ 1743 scoped_guard(srcu, &vcpu->kvm->srcu) 1744 ret = walk_s1(vcpu, &wi, &wr, va); 1745 switch (ret) { 1746 case -EINTR: 1747 /* We interrupted the walk on a match, return the level */ 1748 *level = dm.level; 1749 return 0; 1750 case 0: 1751 /* The walk completed, we failed to find the entry */ 1752 return -ENOENT; 1753 default: 1754 /* Any other error... */ 1755 return ret; 1756 } 1757 } 1758 1759 static int __lsui_swap_desc(u64 __user *ptep, u64 old, u64 new) 1760 { 1761 u64 tmp = old; 1762 int ret = 0; 1763 1764 /* 1765 * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), 1766 * as PAN toggling is not required. 1767 */ 1768 uaccess_ttbr0_enable(); 1769 1770 asm volatile(__LSUI_PREAMBLE 1771 "1: cast %[old], %[new], %[addr]\n" 1772 "2:\n" 1773 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) 1774 : [old] "+r" (old), [addr] "+Q" (*ptep), [ret] "+r" (ret) 1775 : [new] "r" (new) 1776 : "memory"); 1777 1778 uaccess_ttbr0_disable(); 1779 1780 if (ret) 1781 return ret; 1782 if (tmp != old) 1783 return -EAGAIN; 1784 1785 return ret; 1786 } 1787 1788 static int __lse_swap_desc(u64 __user *ptep, u64 old, u64 new) 1789 { 1790 u64 tmp = old; 1791 int ret = 0; 1792 1793 uaccess_enable_privileged(); 1794 1795 asm volatile(__LSE_PREAMBLE 1796 "1: cas %[old], %[new], %[addr]\n" 1797 "2:\n" 1798 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) 1799 : [old] "+r" (old), [addr] "+Q" (*ptep), [ret] "+r" (ret) 1800 : [new] "r" (new) 1801 : "memory"); 1802 1803 uaccess_disable_privileged(); 1804 1805 if (ret) 1806 return ret; 1807 if (tmp != old) 1808 return -EAGAIN; 1809 1810 return ret; 1811 } 1812 1813 static int __llsc_swap_desc(u64 __user *ptep, u64 old, u64 new) 1814 { 1815 int ret = 1; 1816 u64 tmp; 1817 1818 uaccess_enable_privileged(); 1819 1820 asm volatile("prfm pstl1strm, %[addr]\n" 1821 "1: ldxr %[tmp], %[addr]\n" 1822 "sub %[tmp], %[tmp], %[old]\n" 1823 "cbnz %[tmp], 3f\n" 1824 "2: stlxr %w[ret], %[new], %[addr]\n" 1825 "3:\n" 1826 _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) 1827 _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) 1828 : [ret] "+r" (ret), [addr] "+Q" (*ptep), [tmp] "=&r" (tmp) 1829 : [old] "r" (old), [new] "r" (new) 1830 : "memory"); 1831 1832 uaccess_disable_privileged(); 1833 1834 /* STLXR didn't update the descriptor, or the compare failed */ 1835 if (ret == 1) 1836 return -EAGAIN; 1837 1838 return ret; 1839 } 1840 1841 int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new) 1842 { 1843 struct kvm_memory_slot *slot; 1844 unsigned long hva; 1845 u64 __user *ptep; 1846 bool writable; 1847 int offset; 1848 gfn_t gfn; 1849 int r; 1850 1851 lockdep_assert(srcu_read_lock_held(&kvm->srcu)); 1852 1853 gfn = ipa >> PAGE_SHIFT; 1854 offset = offset_in_page(ipa); 1855 slot = gfn_to_memslot(kvm, gfn); 1856 hva = gfn_to_hva_memslot_prot(slot, gfn, &writable); 1857 if (kvm_is_error_hva(hva)) 1858 return -EINVAL; 1859 if (!writable) 1860 return -EPERM; 1861 1862 ptep = (void __user *)hva + offset; 1863 if (cpus_have_final_cap(ARM64_HAS_LSUI)) 1864 r = __lsui_swap_desc(ptep, old, new); 1865 else if (cpus_have_final_cap(ARM64_HAS_LSE_ATOMICS)) 1866 r = __lse_swap_desc(ptep, old, new); 1867 else 1868 r = __llsc_swap_desc(ptep, old, new); 1869 1870 if (r < 0) 1871 return r; 1872 1873 mark_page_dirty_in_slot(kvm, slot, gfn); 1874 return 0; 1875 } 1876