1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Linaro Ltd 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/kvm_host.h> 8 9 #include <asm/esr.h> 10 #include <asm/kvm_hyp.h> 11 #include <asm/kvm_mmu.h> 12 #include <asm/lsui.h> 13 14 static int get_ia_size(struct s1_walk_info *wi) 15 { 16 return 64 - wi->txsz; 17 } 18 19 /* Return true if the IPA is out of the OA range */ 20 static bool check_output_size(u64 ipa, struct s1_walk_info *wi) 21 { 22 if (wi->pa52bit) 23 return wi->max_oa_bits < 52 && (ipa & GENMASK_ULL(51, wi->max_oa_bits)); 24 return wi->max_oa_bits < 48 && (ipa & GENMASK_ULL(47, wi->max_oa_bits)); 25 } 26 27 static bool has_52bit_pa(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, u64 tcr) 28 { 29 switch (BIT(wi->pgshift)) { 30 case SZ_64K: 31 default: /* IMPDEF: treat any other value as 64k */ 32 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52)) 33 return false; 34 return ((wi->regime == TR_EL2 ? 35 FIELD_GET(TCR_EL2_PS_MASK, tcr) : 36 FIELD_GET(TCR_IPS_MASK, tcr)) == 0b0110); 37 case SZ_16K: 38 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)) 39 return false; 40 break; 41 case SZ_4K: 42 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)) 43 return false; 44 break; 45 } 46 47 return (tcr & (wi->regime == TR_EL2 ? TCR_EL2_DS : TCR_DS)); 48 } 49 50 static u64 desc_to_oa(struct s1_walk_info *wi, u64 desc) 51 { 52 u64 addr; 53 54 if (!wi->pa52bit) 55 return desc & GENMASK_ULL(47, wi->pgshift); 56 57 switch (BIT(wi->pgshift)) { 58 case SZ_4K: 59 case SZ_16K: 60 addr = desc & GENMASK_ULL(49, wi->pgshift); 61 addr |= FIELD_GET(KVM_PTE_ADDR_51_50_LPA2, desc) << 50; 62 break; 63 case SZ_64K: 64 default: /* IMPDEF: treat any other value as 64k */ 65 addr = desc & GENMASK_ULL(47, wi->pgshift); 66 addr |= FIELD_GET(KVM_PTE_ADDR_51_48, desc) << 48; 67 break; 68 } 69 70 return addr; 71 } 72 73 /* Return the translation regime that applies to an AT instruction */ 74 static enum trans_regime compute_translation_regime(struct kvm_vcpu *vcpu, u32 op) 75 { 76 /* 77 * We only get here from guest EL2, so the translation 78 * regime AT applies to is solely defined by {E2H,TGE}. 79 */ 80 switch (op) { 81 case OP_AT_S1E2R: 82 case OP_AT_S1E2W: 83 case OP_AT_S1E2A: 84 return vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2; 85 default: 86 return (vcpu_el2_e2h_is_set(vcpu) && 87 vcpu_el2_tge_is_set(vcpu)) ? TR_EL20 : TR_EL10; 88 } 89 } 90 91 static u64 effective_tcr2(struct kvm_vcpu *vcpu, enum trans_regime regime) 92 { 93 if (regime == TR_EL10) { 94 if (vcpu_has_nv(vcpu) && 95 !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TCR2En)) 96 return 0; 97 98 return vcpu_read_sys_reg(vcpu, TCR2_EL1); 99 } 100 101 return vcpu_read_sys_reg(vcpu, TCR2_EL2); 102 } 103 104 static bool s1pie_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 105 { 106 if (!kvm_has_s1pie(vcpu->kvm)) 107 return false; 108 109 /* Abuse TCR2_EL1_PIE and use it for EL2 as well */ 110 return effective_tcr2(vcpu, regime) & TCR2_EL1_PIE; 111 } 112 113 static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) 114 { 115 u64 val; 116 117 if (!kvm_has_s1poe(vcpu->kvm)) { 118 wi->poe = wi->e0poe = false; 119 return; 120 } 121 122 val = effective_tcr2(vcpu, wi->regime); 123 124 /* Abuse TCR2_EL1_* for EL2 */ 125 wi->poe = val & TCR2_EL1_POE; 126 wi->e0poe = (wi->regime != TR_EL2) && (val & TCR2_EL1_E0POE); 127 } 128 129 #define _has_tgran(__r, __sz) \ 130 ({ \ 131 u64 _s1, _mmfr0 = __r; \ 132 \ 133 _s1 = SYS_FIELD_GET(ID_AA64MMFR0_EL1, \ 134 TGRAN##__sz, _mmfr0); \ 135 \ 136 _s1 != ID_AA64MMFR0_EL1_TGRAN##__sz##_NI; \ 137 }) 138 139 static bool has_tgran(u64 mmfr0, unsigned int shift) 140 { 141 switch (shift) { 142 case 12: 143 return _has_tgran(mmfr0, 4); 144 case 14: 145 return _has_tgran(mmfr0, 16); 146 case 16: 147 return _has_tgran(mmfr0, 64); 148 default: 149 BUG(); 150 } 151 } 152 153 static unsigned int tcr_to_tg0_pgshift(u64 tcr) 154 { 155 u64 tg0 = tcr & TCR_TG0_MASK; 156 157 switch (tg0) { 158 case TCR_TG0_4K: 159 return 12; 160 case TCR_TG0_16K: 161 return 14; 162 case TCR_TG0_64K: 163 default: /* IMPDEF: treat any other value as 64k */ 164 return 16; 165 } 166 } 167 168 static unsigned int tcr_to_tg1_pgshift(u64 tcr) 169 { 170 u64 tg1 = tcr & TCR_TG1_MASK; 171 172 switch (tg1) { 173 case TCR_TG1_4K: 174 return 12; 175 case TCR_TG1_16K: 176 return 14; 177 case TCR_TG1_64K: 178 default: /* IMPDEF: treat any other value as 64k */ 179 return 16; 180 } 181 } 182 183 static unsigned int fallback_tgran_shift(u64 mmfr0) 184 { 185 if (has_tgran(mmfr0, PAGE_SHIFT)) 186 return PAGE_SHIFT; 187 else if (has_tgran(mmfr0, 12)) 188 return 12; 189 else if (has_tgran(mmfr0, 14)) 190 return 14; 191 else if (has_tgran(mmfr0, 16)) 192 return 16; 193 else /* Should be unreacheable */ 194 return PAGE_SHIFT; 195 } 196 197 static unsigned int tcr_tg_pgshift(struct kvm *kvm, u64 tcr, bool upper_range) 198 { 199 u64 mmfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64MMFR0_EL1); 200 unsigned int shift; 201 202 /* Someone was silly enough to encode TG0/TG1 differently */ 203 if (upper_range) 204 shift = tcr_to_tg1_pgshift(tcr); 205 else 206 shift = tcr_to_tg0_pgshift(tcr); 207 208 /* 209 * If TGx is programmed to an unimplemented value (not advertised in 210 * ID_AA64MMFR0_EL1), we should treat it as if an implemented value is 211 * written, as per the architecture. Choose an available one while 212 * prioritizing PAGE_SIZE. 213 */ 214 if (!has_tgran(mmfr0, shift)) 215 return fallback_tgran_shift(mmfr0); 216 217 return shift; 218 } 219 220 static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 221 struct s1_walk_result *wr, u64 va) 222 { 223 u64 hcr, sctlr, tcr, ps, ia_bits, ttbr; 224 unsigned int stride, x; 225 bool va55, tbi, lva, upper_range; 226 227 va55 = va & BIT(55); 228 upper_range = va55 && wi->regime != TR_EL2; 229 230 if (vcpu_has_nv(vcpu)) { 231 hcr = __vcpu_sys_reg(vcpu, HCR_EL2); 232 wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); 233 } else { 234 WARN_ON_ONCE(wi->regime != TR_EL10); 235 wi->s2 = false; 236 hcr = 0; 237 } 238 239 switch (wi->regime) { 240 case TR_EL10: 241 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 242 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); 243 ttbr = (va55 ? 244 vcpu_read_sys_reg(vcpu, TTBR1_EL1) : 245 vcpu_read_sys_reg(vcpu, TTBR0_EL1)); 246 break; 247 case TR_EL2: 248 case TR_EL20: 249 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); 250 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); 251 ttbr = (va55 ? 252 vcpu_read_sys_reg(vcpu, TTBR1_EL2) : 253 vcpu_read_sys_reg(vcpu, TTBR0_EL2)); 254 break; 255 default: 256 BUG(); 257 } 258 259 if (upper_range) 260 wi->txsz = FIELD_GET(TCR_T1SZ_MASK, tcr); 261 else 262 wi->txsz = FIELD_GET(TCR_T0SZ_MASK, tcr); 263 264 wi->pgshift = tcr_tg_pgshift(vcpu->kvm, tcr, upper_range); 265 wi->pa52bit = has_52bit_pa(vcpu, wi, tcr); 266 267 ia_bits = get_ia_size(wi); 268 269 /* AArch64.S1StartLevel() */ 270 stride = wi->pgshift - 3; 271 wi->sl = 3 - (((ia_bits - 1) - wi->pgshift) / stride); 272 273 if (wi->regime == TR_EL2 && va55) 274 goto addrsz; 275 276 tbi = (wi->regime == TR_EL2 ? 277 FIELD_GET(TCR_EL2_TBI, tcr) : 278 (va55 ? 279 FIELD_GET(TCR_TBI1, tcr) : 280 FIELD_GET(TCR_TBI0, tcr))); 281 282 if (!tbi && (u64)sign_extend64(va, 55) != va) 283 goto addrsz; 284 285 wi->sh = (wi->regime == TR_EL2 ? 286 FIELD_GET(TCR_EL2_SH0_MASK, tcr) : 287 (va55 ? 288 FIELD_GET(TCR_SH1_MASK, tcr) : 289 FIELD_GET(TCR_SH0_MASK, tcr))); 290 291 va = (u64)sign_extend64(va, 55); 292 293 /* Let's put the MMU disabled case aside immediately */ 294 switch (wi->regime) { 295 case TR_EL10: 296 /* 297 * If dealing with the EL1&0 translation regime, 3 things 298 * can disable the S1 translation: 299 * 300 * - HCR_EL2.DC = 1 301 * - HCR_EL2.{E2H,TGE} = {0,1} 302 * - SCTLR_EL1.M = 0 303 * 304 * The TGE part is interesting. If we have decided that this 305 * is EL1&0, then it means that either {E2H,TGE} == {1,0} or 306 * {0,x}, and we only need to test for TGE == 1. 307 */ 308 if (hcr & (HCR_DC | HCR_TGE)) { 309 wr->level = S1_MMU_DISABLED; 310 break; 311 } 312 fallthrough; 313 case TR_EL2: 314 case TR_EL20: 315 if (!(sctlr & SCTLR_ELx_M)) 316 wr->level = S1_MMU_DISABLED; 317 break; 318 } 319 320 if (wr->level == S1_MMU_DISABLED) { 321 if (va >= BIT(kvm_get_pa_bits(vcpu->kvm))) 322 goto addrsz; 323 324 wr->pa = va; 325 return 0; 326 } 327 328 wi->be = sctlr & SCTLR_ELx_EE; 329 330 wi->hpd = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HPDS, IMP); 331 wi->hpd &= (wi->regime == TR_EL2 ? 332 FIELD_GET(TCR_EL2_HPD, tcr) : 333 (va55 ? 334 FIELD_GET(TCR_HPD1, tcr) : 335 FIELD_GET(TCR_HPD0, tcr))); 336 /* R_JHSVW */ 337 wi->hpd |= s1pie_enabled(vcpu, wi->regime); 338 339 /* Do we have POE? */ 340 compute_s1poe(vcpu, wi); 341 342 /* R_BVXDG */ 343 wi->hpd |= (wi->poe || wi->e0poe); 344 345 /* R_PLCGL, R_YXNYW */ 346 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR2_EL1, ST, 48_47)) { 347 if (wi->txsz > 39) 348 goto transfault; 349 } else { 350 if (wi->txsz > 48 || (BIT(wi->pgshift) == SZ_64K && wi->txsz > 47)) 351 goto transfault; 352 } 353 354 /* R_GTJBY, R_SXWGM */ 355 switch (BIT(wi->pgshift)) { 356 case SZ_4K: 357 case SZ_16K: 358 lva = wi->pa52bit; 359 break; 360 case SZ_64K: 361 lva = kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, VARange, 52); 362 break; 363 } 364 365 if ((lva && wi->txsz < 12) || (!lva && wi->txsz < 16)) 366 goto transfault; 367 368 /* R_YYVYV, I_THCZK */ 369 if ((!va55 && va > GENMASK(ia_bits - 1, 0)) || 370 (va55 && va < GENMASK(63, ia_bits))) 371 goto transfault; 372 373 /* I_ZFSYQ */ 374 if (wi->regime != TR_EL2 && 375 (tcr & (va55 ? TCR_EPD1_MASK : TCR_EPD0_MASK))) 376 goto transfault; 377 378 /* R_BNDVG and following statements */ 379 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && 380 wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) 381 goto transfault; 382 383 ps = (wi->regime == TR_EL2 ? 384 FIELD_GET(TCR_EL2_PS_MASK, tcr) : FIELD_GET(TCR_IPS_MASK, tcr)); 385 386 wi->max_oa_bits = min(get_kvm_ipa_limit(), ps_to_output_size(ps, wi->pa52bit)); 387 388 /* Compute minimal alignment */ 389 x = 3 + ia_bits - ((3 - wi->sl) * stride + wi->pgshift); 390 391 wi->baddr = ttbr & TTBRx_EL1_BADDR; 392 if (wi->pa52bit) { 393 /* 394 * Force the alignment on 64 bytes for top-level tables 395 * smaller than 8 entries, since TTBR.BADDR[5:2] are used to 396 * store bits [51:48] of the first level of lookup. 397 */ 398 x = max(x, 6); 399 400 wi->baddr |= FIELD_GET(GENMASK_ULL(5, 2), ttbr) << 48; 401 } 402 403 /* R_VPBBF */ 404 if (check_output_size(wi->baddr, wi)) 405 goto addrsz; 406 407 wi->baddr &= GENMASK_ULL(wi->max_oa_bits - 1, x); 408 409 wi->ha = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HAFDBS, AF); 410 wi->ha &= (wi->regime == TR_EL2 ? 411 FIELD_GET(TCR_EL2_HA, tcr) : 412 FIELD_GET(TCR_HA, tcr)); 413 414 return 0; 415 416 addrsz: 417 /* 418 * Address Size Fault level 0 to indicate it comes from TTBR. 419 * yes, this is an oddity. 420 */ 421 fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(0), false); 422 return -EFAULT; 423 424 transfault: 425 /* Translation Fault on start level */ 426 fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(wi->sl), false); 427 return -EFAULT; 428 } 429 430 static int kvm_read_s1_desc(struct kvm_vcpu *vcpu, u64 pa, u64 *desc, 431 struct s1_walk_info *wi) 432 { 433 u64 val; 434 int r; 435 436 r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val)); 437 if (r) 438 return r; 439 440 if (wi->be) 441 *desc = be64_to_cpu((__force __be64)val); 442 else 443 *desc = le64_to_cpu((__force __le64)val); 444 445 return 0; 446 } 447 448 static int kvm_swap_s1_desc(struct kvm_vcpu *vcpu, u64 pa, u64 old, u64 new, 449 struct s1_walk_info *wi) 450 { 451 if (wi->be) { 452 old = (__force u64)cpu_to_be64(old); 453 new = (__force u64)cpu_to_be64(new); 454 } else { 455 old = (__force u64)cpu_to_le64(old); 456 new = (__force u64)cpu_to_le64(new); 457 } 458 459 return __kvm_at_swap_desc(vcpu->kvm, pa, old, new); 460 } 461 462 static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 463 struct s1_walk_result *wr, u64 va) 464 { 465 u64 va_top, va_bottom, baddr, desc, new_desc, ipa; 466 struct kvm_s2_trans s2_trans = {}; 467 int level, stride, ret; 468 469 level = wi->sl; 470 stride = wi->pgshift - 3; 471 baddr = wi->baddr; 472 473 va_top = get_ia_size(wi) - 1; 474 475 while (1) { 476 u64 index; 477 478 va_bottom = (3 - level) * stride + wi->pgshift; 479 index = (va & GENMASK_ULL(va_top, va_bottom)) >> (va_bottom - 3); 480 481 ipa = baddr | index; 482 483 if (wi->s2) { 484 ret = kvm_walk_nested_s2(vcpu, ipa, &s2_trans); 485 if (ret == -EAGAIN) 486 return ret; 487 488 if (ret) { 489 fail_s1_walk(wr, 490 (s2_trans.esr & ~ESR_ELx_FSC_LEVEL) | level, 491 true); 492 return ret; 493 } 494 495 if (!kvm_s2_trans_readable(&s2_trans)) { 496 fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level), 497 true); 498 499 return -EPERM; 500 } 501 502 ipa = kvm_s2_trans_output(&s2_trans); 503 } 504 505 if (wi->filter) { 506 ret = wi->filter->fn(&(struct s1_walk_context) 507 { 508 .wi = wi, 509 .table_ipa = baddr, 510 .level = level, 511 }, wi->filter->priv); 512 if (ret) 513 return ret; 514 } 515 516 ret = kvm_read_s1_desc(vcpu, ipa, &desc, wi); 517 if (ret) { 518 fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); 519 return ret; 520 } 521 522 new_desc = desc; 523 524 /* Invalid descriptor */ 525 if (!(desc & BIT(0))) 526 goto transfault; 527 528 /* Block mapping, check validity down the line */ 529 if (!(desc & BIT(1))) 530 break; 531 532 /* Page mapping */ 533 if (level == 3) 534 break; 535 536 /* Table handling */ 537 if (!wi->hpd) { 538 wr->APTable |= FIELD_GET(S1_TABLE_AP, desc); 539 wr->UXNTable |= FIELD_GET(PMD_TABLE_UXN, desc); 540 wr->PXNTable |= FIELD_GET(PMD_TABLE_PXN, desc); 541 } 542 543 baddr = desc_to_oa(wi, desc); 544 545 /* Check for out-of-range OA */ 546 if (check_output_size(baddr, wi)) 547 goto addrsz; 548 549 /* Prepare for next round */ 550 va_top = va_bottom - 1; 551 level++; 552 } 553 554 /* Block mapping, check the validity of the level */ 555 if (!(desc & BIT(1))) { 556 bool valid_block = false; 557 bool lpa = kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, PARANGE, 52); 558 559 switch (BIT(wi->pgshift)) { 560 case SZ_4K: 561 valid_block = level == 1 || level == 2 || (wi->pa52bit && level == 0); 562 break; 563 case SZ_16K: 564 valid_block = level == 2 || (wi->pa52bit && level == 1); 565 break; 566 case SZ_64K: 567 valid_block = level == 2 || (lpa && level == 1); 568 break; 569 } 570 571 if (!valid_block) 572 goto transfault; 573 } 574 575 baddr = desc_to_oa(wi, desc); 576 if (check_output_size(baddr & GENMASK(52, va_bottom), wi)) 577 goto addrsz; 578 579 if (wi->ha) 580 new_desc |= PTE_AF; 581 582 if (new_desc != desc) { 583 if (wi->s2 && !kvm_s2_trans_writable(&s2_trans)) { 584 fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level), true); 585 return -EPERM; 586 } 587 588 ret = kvm_swap_s1_desc(vcpu, ipa, desc, new_desc, wi); 589 if (ret == -EAGAIN) 590 return ret; 591 if (ret) { 592 fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false); 593 return ret; 594 } 595 596 desc = new_desc; 597 } 598 599 if (!(desc & PTE_AF)) { 600 fail_s1_walk(wr, ESR_ELx_FSC_ACCESS_L(level), false); 601 return -EACCES; 602 } 603 604 va_bottom += contiguous_bit_shift(desc, wi, level); 605 606 wr->failed = false; 607 wr->level = level; 608 wr->desc = desc; 609 wr->pa = baddr & GENMASK(52, va_bottom); 610 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); 611 612 wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); 613 if (wr->nG) 614 wr->asid = get_asid_by_regime(vcpu, wi->regime); 615 616 return 0; 617 618 addrsz: 619 fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(level), false); 620 return -EINVAL; 621 transfault: 622 fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(level), false); 623 return -ENOENT; 624 } 625 626 struct mmu_config { 627 u64 ttbr0; 628 u64 ttbr1; 629 u64 tcr; 630 u64 mair; 631 u64 tcr2; 632 u64 pir; 633 u64 pire0; 634 u64 por_el0; 635 u64 por_el1; 636 u64 sctlr; 637 u64 vttbr; 638 u64 vtcr; 639 }; 640 641 static void __mmu_config_save(struct mmu_config *config) 642 { 643 config->ttbr0 = read_sysreg_el1(SYS_TTBR0); 644 config->ttbr1 = read_sysreg_el1(SYS_TTBR1); 645 config->tcr = read_sysreg_el1(SYS_TCR); 646 config->mair = read_sysreg_el1(SYS_MAIR); 647 if (cpus_have_final_cap(ARM64_HAS_TCR2)) { 648 config->tcr2 = read_sysreg_el1(SYS_TCR2); 649 if (cpus_have_final_cap(ARM64_HAS_S1PIE)) { 650 config->pir = read_sysreg_el1(SYS_PIR); 651 config->pire0 = read_sysreg_el1(SYS_PIRE0); 652 } 653 if (system_supports_poe()) { 654 config->por_el1 = read_sysreg_el1(SYS_POR); 655 config->por_el0 = read_sysreg_s(SYS_POR_EL0); 656 } 657 } 658 config->sctlr = read_sysreg_el1(SYS_SCTLR); 659 config->vttbr = read_sysreg(vttbr_el2); 660 config->vtcr = read_sysreg(vtcr_el2); 661 } 662 663 static void __mmu_config_restore(struct mmu_config *config) 664 { 665 /* 666 * ARM errata 1165522 and 1530923 require TGE to be 1 before 667 * we update the guest state. 668 */ 669 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); 670 671 write_sysreg_el1(config->ttbr0, SYS_TTBR0); 672 write_sysreg_el1(config->ttbr1, SYS_TTBR1); 673 write_sysreg_el1(config->tcr, SYS_TCR); 674 write_sysreg_el1(config->mair, SYS_MAIR); 675 if (cpus_have_final_cap(ARM64_HAS_TCR2)) { 676 write_sysreg_el1(config->tcr2, SYS_TCR2); 677 if (cpus_have_final_cap(ARM64_HAS_S1PIE)) { 678 write_sysreg_el1(config->pir, SYS_PIR); 679 write_sysreg_el1(config->pire0, SYS_PIRE0); 680 } 681 if (system_supports_poe()) { 682 write_sysreg_el1(config->por_el1, SYS_POR); 683 write_sysreg_s(config->por_el0, SYS_POR_EL0); 684 } 685 } 686 write_sysreg_el1(config->sctlr, SYS_SCTLR); 687 write_sysreg(config->vttbr, vttbr_el2); 688 write_sysreg(config->vtcr, vtcr_el2); 689 } 690 691 static bool at_s1e1p_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 692 { 693 u64 host_pan; 694 bool fail; 695 696 host_pan = read_sysreg_s(SYS_PSTATE_PAN); 697 write_sysreg_s(*vcpu_cpsr(vcpu) & PSTATE_PAN, SYS_PSTATE_PAN); 698 699 switch (op) { 700 case OP_AT_S1E1RP: 701 fail = __kvm_at(OP_AT_S1E1RP, vaddr); 702 break; 703 case OP_AT_S1E1WP: 704 fail = __kvm_at(OP_AT_S1E1WP, vaddr); 705 break; 706 } 707 708 write_sysreg_s(host_pan, SYS_PSTATE_PAN); 709 710 return fail; 711 } 712 713 #define MEMATTR(ic, oc) (MEMATTR_##oc << 4 | MEMATTR_##ic) 714 #define MEMATTR_NC 0b0100 715 #define MEMATTR_Wt 0b1000 716 #define MEMATTR_Wb 0b1100 717 #define MEMATTR_WbRaWa 0b1111 718 719 #define MEMATTR_IS_DEVICE(m) (((m) & GENMASK(7, 4)) == 0) 720 721 static u8 s2_memattr_to_attr(u8 memattr) 722 { 723 memattr &= 0b1111; 724 725 switch (memattr) { 726 case 0b0000: 727 case 0b0001: 728 case 0b0010: 729 case 0b0011: 730 return memattr << 2; 731 case 0b0100: 732 return MEMATTR(Wb, Wb); 733 case 0b0101: 734 return MEMATTR(NC, NC); 735 case 0b0110: 736 return MEMATTR(Wt, NC); 737 case 0b0111: 738 return MEMATTR(Wb, NC); 739 case 0b1000: 740 /* Reserved, assume NC */ 741 return MEMATTR(NC, NC); 742 case 0b1001: 743 return MEMATTR(NC, Wt); 744 case 0b1010: 745 return MEMATTR(Wt, Wt); 746 case 0b1011: 747 return MEMATTR(Wb, Wt); 748 case 0b1100: 749 /* Reserved, assume NC */ 750 return MEMATTR(NC, NC); 751 case 0b1101: 752 return MEMATTR(NC, Wb); 753 case 0b1110: 754 return MEMATTR(Wt, Wb); 755 case 0b1111: 756 return MEMATTR(Wb, Wb); 757 default: 758 unreachable(); 759 } 760 } 761 762 static u8 combine_s1_s2_attr(u8 s1, u8 s2) 763 { 764 bool transient; 765 u8 final = 0; 766 767 /* Upgrade transient s1 to non-transient to simplify things */ 768 switch (s1) { 769 case 0b0001 ... 0b0011: /* Normal, Write-Through Transient */ 770 transient = true; 771 s1 = MEMATTR_Wt | (s1 & GENMASK(1,0)); 772 break; 773 case 0b0101 ... 0b0111: /* Normal, Write-Back Transient */ 774 transient = true; 775 s1 = MEMATTR_Wb | (s1 & GENMASK(1,0)); 776 break; 777 default: 778 transient = false; 779 } 780 781 /* S2CombineS1AttrHints() */ 782 if ((s1 & GENMASK(3, 2)) == MEMATTR_NC || 783 (s2 & GENMASK(3, 2)) == MEMATTR_NC) 784 final = MEMATTR_NC; 785 else if ((s1 & GENMASK(3, 2)) == MEMATTR_Wt || 786 (s2 & GENMASK(3, 2)) == MEMATTR_Wt) 787 final = MEMATTR_Wt; 788 else 789 final = MEMATTR_Wb; 790 791 if (final != MEMATTR_NC) { 792 /* Inherit RaWa hints form S1 */ 793 if (transient) { 794 switch (s1 & GENMASK(3, 2)) { 795 case MEMATTR_Wt: 796 final = 0; 797 break; 798 case MEMATTR_Wb: 799 final = MEMATTR_NC; 800 break; 801 } 802 } 803 804 final |= s1 & GENMASK(1, 0); 805 } 806 807 return final; 808 } 809 810 #define ATTR_NSH 0b00 811 #define ATTR_RSV 0b01 812 #define ATTR_OSH 0b10 813 #define ATTR_ISH 0b11 814 815 static u8 compute_final_sh(u8 attr, u8 sh) 816 { 817 /* Any form of device, as well as NC has SH[1:0]=0b10 */ 818 if (MEMATTR_IS_DEVICE(attr) || attr == MEMATTR(NC, NC)) 819 return ATTR_OSH; 820 821 if (sh == ATTR_RSV) /* Reserved, mapped to NSH */ 822 sh = ATTR_NSH; 823 824 return sh; 825 } 826 827 static u8 compute_s1_sh(struct s1_walk_info *wi, struct s1_walk_result *wr, 828 u8 attr) 829 { 830 u8 sh; 831 832 /* 833 * non-52bit and LPA have their basic shareability described in the 834 * descriptor. LPA2 gets it from the corresponding field in TCR, 835 * conveniently recorded in the walk info. 836 */ 837 if (!wi->pa52bit || BIT(wi->pgshift) == SZ_64K) 838 sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_SH, wr->desc); 839 else 840 sh = wi->sh; 841 842 return compute_final_sh(attr, sh); 843 } 844 845 static u8 combine_sh(u8 s1_sh, u8 s2_sh) 846 { 847 if (s1_sh == ATTR_OSH || s2_sh == ATTR_OSH) 848 return ATTR_OSH; 849 if (s1_sh == ATTR_ISH || s2_sh == ATTR_ISH) 850 return ATTR_ISH; 851 852 return ATTR_NSH; 853 } 854 855 static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, 856 struct kvm_s2_trans *tr) 857 { 858 u8 s1_parattr, s2_memattr, final_attr, s2_sh; 859 u64 par; 860 861 /* If S2 has failed to translate, report the damage */ 862 if (tr->esr) { 863 par = SYS_PAR_EL1_RES1; 864 par |= SYS_PAR_EL1_F; 865 par |= SYS_PAR_EL1_S; 866 par |= FIELD_PREP(SYS_PAR_EL1_FST, tr->esr); 867 return par; 868 } 869 870 s1_parattr = FIELD_GET(SYS_PAR_EL1_ATTR, s1_par); 871 s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc); 872 873 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_FWB) { 874 if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP)) 875 s2_memattr &= ~BIT(3); 876 877 /* Combination of R_VRJSW and R_RHWZM */ 878 switch (s2_memattr) { 879 case 0b0101: 880 if (MEMATTR_IS_DEVICE(s1_parattr)) 881 final_attr = s1_parattr; 882 else 883 final_attr = MEMATTR(NC, NC); 884 break; 885 case 0b0110: 886 case 0b1110: 887 final_attr = MEMATTR(WbRaWa, WbRaWa); 888 break; 889 case 0b0111: 890 case 0b1111: 891 /* Preserve S1 attribute */ 892 final_attr = s1_parattr; 893 break; 894 case 0b0100: 895 case 0b1100: 896 case 0b1101: 897 /* Reserved, do something non-silly */ 898 final_attr = s1_parattr; 899 break; 900 default: 901 /* 902 * MemAttr[2]=0, Device from S2. 903 * 904 * FWB does not influence the way that stage 1 905 * memory types and attributes are combined 906 * with stage 2 Device type and attributes. 907 */ 908 final_attr = min(s2_memattr_to_attr(s2_memattr), 909 s1_parattr); 910 } 911 } else { 912 /* Combination of R_HMNDG, R_TNHFM and R_GQFSF */ 913 u8 s2_parattr = s2_memattr_to_attr(s2_memattr); 914 915 if (MEMATTR_IS_DEVICE(s1_parattr) || 916 MEMATTR_IS_DEVICE(s2_parattr)) { 917 final_attr = min(s1_parattr, s2_parattr); 918 } else { 919 /* At this stage, this is memory vs memory */ 920 final_attr = combine_s1_s2_attr(s1_parattr & 0xf, 921 s2_parattr & 0xf); 922 final_attr |= combine_s1_s2_attr(s1_parattr >> 4, 923 s2_parattr >> 4) << 4; 924 } 925 } 926 927 if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_CD) && 928 !MEMATTR_IS_DEVICE(final_attr)) 929 final_attr = MEMATTR(NC, NC); 930 931 s2_sh = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S2_SH, tr->desc); 932 933 par = FIELD_PREP(SYS_PAR_EL1_ATTR, final_attr); 934 par |= tr->output & GENMASK(47, 12); 935 par |= FIELD_PREP(SYS_PAR_EL1_SH, 936 combine_sh(FIELD_GET(SYS_PAR_EL1_SH, s1_par), 937 compute_final_sh(final_attr, s2_sh))); 938 939 return par; 940 } 941 942 static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 943 struct s1_walk_result *wr) 944 { 945 u64 par; 946 947 if (wr->failed) { 948 par = SYS_PAR_EL1_RES1; 949 par |= SYS_PAR_EL1_F; 950 par |= FIELD_PREP(SYS_PAR_EL1_FST, wr->fst); 951 par |= wr->ptw ? SYS_PAR_EL1_PTW : 0; 952 par |= wr->s2 ? SYS_PAR_EL1_S : 0; 953 } else if (wr->level == S1_MMU_DISABLED) { 954 /* MMU off or HCR_EL2.DC == 1 */ 955 par = SYS_PAR_EL1_NSE; 956 par |= wr->pa & SYS_PAR_EL1_PA; 957 958 if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && 959 (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) { 960 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, 961 MEMATTR(WbRaWa, WbRaWa)); 962 par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_NSH); 963 } else { 964 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, 0); /* nGnRnE */ 965 par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_OSH); 966 } 967 } else { 968 u64 mair, sctlr; 969 u8 sh; 970 971 par = SYS_PAR_EL1_NSE; 972 973 mair = (wi->regime == TR_EL10 ? 974 vcpu_read_sys_reg(vcpu, MAIR_EL1) : 975 vcpu_read_sys_reg(vcpu, MAIR_EL2)); 976 977 mair >>= FIELD_GET(PTE_ATTRINDX_MASK, wr->desc) * 8; 978 mair &= 0xff; 979 980 sctlr = (wi->regime == TR_EL10 ? 981 vcpu_read_sys_reg(vcpu, SCTLR_EL1) : 982 vcpu_read_sys_reg(vcpu, SCTLR_EL2)); 983 984 /* Force NC for memory if SCTLR_ELx.C is clear */ 985 if (!(sctlr & SCTLR_EL1_C) && !MEMATTR_IS_DEVICE(mair)) 986 mair = MEMATTR(NC, NC); 987 988 par |= FIELD_PREP(SYS_PAR_EL1_ATTR, mair); 989 par |= wr->pa & SYS_PAR_EL1_PA; 990 991 sh = compute_s1_sh(wi, wr, mair); 992 par |= FIELD_PREP(SYS_PAR_EL1_SH, sh); 993 } 994 995 return par; 996 } 997 998 static bool pan3_enabled(struct kvm_vcpu *vcpu, enum trans_regime regime) 999 { 1000 u64 sctlr; 1001 1002 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) 1003 return false; 1004 1005 if (s1pie_enabled(vcpu, regime)) 1006 return true; 1007 1008 if (regime == TR_EL10) 1009 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); 1010 else 1011 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); 1012 1013 return sctlr & SCTLR_EL1_EPAN; 1014 } 1015 1016 static void compute_s1_direct_permissions(struct kvm_vcpu *vcpu, 1017 struct s1_walk_info *wi, 1018 struct s1_walk_result *wr) 1019 { 1020 bool wxn; 1021 1022 /* Non-hierarchical part of AArch64.S1DirectBasePermissions() */ 1023 if (wi->regime != TR_EL2) { 1024 switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr->desc)) { 1025 case 0b00: 1026 wr->pr = wr->pw = true; 1027 wr->ur = wr->uw = false; 1028 break; 1029 case 0b01: 1030 wr->pr = wr->pw = wr->ur = wr->uw = true; 1031 break; 1032 case 0b10: 1033 wr->pr = true; 1034 wr->pw = wr->ur = wr->uw = false; 1035 break; 1036 case 0b11: 1037 wr->pr = wr->ur = true; 1038 wr->pw = wr->uw = false; 1039 break; 1040 } 1041 1042 /* We don't use px for anything yet, but hey... */ 1043 wr->px = !((wr->desc & PTE_PXN) || wr->uw); 1044 wr->ux = !(wr->desc & PTE_UXN); 1045 } else { 1046 wr->ur = wr->uw = wr->ux = false; 1047 1048 if (!(wr->desc & PTE_RDONLY)) { 1049 wr->pr = wr->pw = true; 1050 } else { 1051 wr->pr = true; 1052 wr->pw = false; 1053 } 1054 1055 /* XN maps to UXN */ 1056 wr->px = !(wr->desc & PTE_UXN); 1057 } 1058 1059 switch (wi->regime) { 1060 case TR_EL2: 1061 case TR_EL20: 1062 wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_WXN); 1063 break; 1064 case TR_EL10: 1065 wxn = (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_WXN); 1066 break; 1067 } 1068 1069 wr->pwxn = wr->uwxn = wxn; 1070 wr->pov = wi->poe; 1071 wr->uov = wi->e0poe; 1072 } 1073 1074 static void compute_s1_hierarchical_permissions(struct kvm_vcpu *vcpu, 1075 struct s1_walk_info *wi, 1076 struct s1_walk_result *wr) 1077 { 1078 /* Hierarchical part of AArch64.S1DirectBasePermissions() */ 1079 if (wi->regime != TR_EL2) { 1080 switch (wr->APTable) { 1081 case 0b00: 1082 break; 1083 case 0b01: 1084 wr->ur = wr->uw = false; 1085 break; 1086 case 0b10: 1087 wr->pw = wr->uw = false; 1088 break; 1089 case 0b11: 1090 wr->pw = wr->ur = wr->uw = false; 1091 break; 1092 } 1093 1094 wr->px &= !wr->PXNTable; 1095 wr->ux &= !wr->UXNTable; 1096 } else { 1097 if (wr->APTable & BIT(1)) 1098 wr->pw = false; 1099 1100 /* XN maps to UXN */ 1101 wr->px &= !wr->UXNTable; 1102 } 1103 } 1104 1105 #define perm_idx(v, r, i) ((vcpu_read_sys_reg((v), (r)) >> ((i) * 4)) & 0xf) 1106 1107 #define set_priv_perms(wr, r, w, x) \ 1108 do { \ 1109 (wr)->pr = (r); \ 1110 (wr)->pw = (w); \ 1111 (wr)->px = (x); \ 1112 } while (0) 1113 1114 #define set_unpriv_perms(wr, r, w, x) \ 1115 do { \ 1116 (wr)->ur = (r); \ 1117 (wr)->uw = (w); \ 1118 (wr)->ux = (x); \ 1119 } while (0) 1120 1121 #define set_priv_wxn(wr, v) \ 1122 do { \ 1123 (wr)->pwxn = (v); \ 1124 } while (0) 1125 1126 #define set_unpriv_wxn(wr, v) \ 1127 do { \ 1128 (wr)->uwxn = (v); \ 1129 } while (0) 1130 1131 /* Similar to AArch64.S1IndirectBasePermissions(), without GCS */ 1132 #define set_perms(w, wr, ip) \ 1133 do { \ 1134 /* R_LLZDZ */ \ 1135 switch ((ip)) { \ 1136 case 0b0000: \ 1137 set_ ## w ## _perms((wr), false, false, false); \ 1138 break; \ 1139 case 0b0001: \ 1140 set_ ## w ## _perms((wr), true , false, false); \ 1141 break; \ 1142 case 0b0010: \ 1143 set_ ## w ## _perms((wr), false, false, true ); \ 1144 break; \ 1145 case 0b0011: \ 1146 set_ ## w ## _perms((wr), true , false, true ); \ 1147 break; \ 1148 case 0b0100: \ 1149 set_ ## w ## _perms((wr), false, false, false); \ 1150 break; \ 1151 case 0b0101: \ 1152 set_ ## w ## _perms((wr), true , true , false); \ 1153 break; \ 1154 case 0b0110: \ 1155 set_ ## w ## _perms((wr), true , true , true ); \ 1156 break; \ 1157 case 0b0111: \ 1158 set_ ## w ## _perms((wr), true , true , true ); \ 1159 break; \ 1160 case 0b1000: \ 1161 set_ ## w ## _perms((wr), true , false, false); \ 1162 break; \ 1163 case 0b1001: \ 1164 set_ ## w ## _perms((wr), true , false, false); \ 1165 break; \ 1166 case 0b1010: \ 1167 set_ ## w ## _perms((wr), true , false, true ); \ 1168 break; \ 1169 case 0b1011: \ 1170 set_ ## w ## _perms((wr), false, false, false); \ 1171 break; \ 1172 case 0b1100: \ 1173 set_ ## w ## _perms((wr), true , true , false); \ 1174 break; \ 1175 case 0b1101: \ 1176 set_ ## w ## _perms((wr), false, false, false); \ 1177 break; \ 1178 case 0b1110: \ 1179 set_ ## w ## _perms((wr), true , true , true ); \ 1180 break; \ 1181 case 0b1111: \ 1182 set_ ## w ## _perms((wr), false, false, false); \ 1183 break; \ 1184 } \ 1185 \ 1186 /* R_HJYGR */ \ 1187 set_ ## w ## _wxn((wr), ((ip) == 0b0110)); \ 1188 \ 1189 } while (0) 1190 1191 static void compute_s1_indirect_permissions(struct kvm_vcpu *vcpu, 1192 struct s1_walk_info *wi, 1193 struct s1_walk_result *wr) 1194 { 1195 u8 up, pp, idx; 1196 1197 idx = pte_pi_index(wr->desc); 1198 1199 switch (wi->regime) { 1200 case TR_EL10: 1201 pp = perm_idx(vcpu, PIR_EL1, idx); 1202 up = perm_idx(vcpu, PIRE0_EL1, idx); 1203 break; 1204 case TR_EL20: 1205 pp = perm_idx(vcpu, PIR_EL2, idx); 1206 up = perm_idx(vcpu, PIRE0_EL2, idx); 1207 break; 1208 case TR_EL2: 1209 pp = perm_idx(vcpu, PIR_EL2, idx); 1210 up = 0; 1211 break; 1212 } 1213 1214 set_perms(priv, wr, pp); 1215 1216 if (wi->regime != TR_EL2) 1217 set_perms(unpriv, wr, up); 1218 else 1219 set_unpriv_perms(wr, false, false, false); 1220 1221 wr->pov = wi->poe && !(pp & BIT(3)); 1222 wr->uov = wi->e0poe && !(up & BIT(3)); 1223 1224 /* R_VFPJF */ 1225 if (wr->px && wr->uw) { 1226 set_priv_perms(wr, false, false, false); 1227 set_unpriv_perms(wr, false, false, false); 1228 } 1229 } 1230 1231 static void compute_s1_overlay_permissions(struct kvm_vcpu *vcpu, 1232 struct s1_walk_info *wi, 1233 struct s1_walk_result *wr) 1234 { 1235 u8 idx, pov_perms, uov_perms; 1236 1237 idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc); 1238 1239 if (wr->pov) { 1240 switch (wi->regime) { 1241 case TR_EL10: 1242 pov_perms = perm_idx(vcpu, POR_EL1, idx); 1243 break; 1244 case TR_EL20: 1245 pov_perms = perm_idx(vcpu, POR_EL2, idx); 1246 break; 1247 case TR_EL2: 1248 pov_perms = perm_idx(vcpu, POR_EL2, idx); 1249 break; 1250 } 1251 1252 if (pov_perms & ~POE_RWX) 1253 pov_perms = POE_NONE; 1254 1255 /* R_QXXPC, S1PrivOverflow enabled */ 1256 if (wr->pwxn && (pov_perms & POE_X)) 1257 pov_perms &= ~POE_W; 1258 1259 wr->pr &= pov_perms & POE_R; 1260 wr->pw &= pov_perms & POE_W; 1261 wr->px &= pov_perms & POE_X; 1262 } 1263 1264 if (wr->uov) { 1265 switch (wi->regime) { 1266 case TR_EL10: 1267 uov_perms = perm_idx(vcpu, POR_EL0, idx); 1268 break; 1269 case TR_EL20: 1270 uov_perms = perm_idx(vcpu, POR_EL0, idx); 1271 break; 1272 case TR_EL2: 1273 uov_perms = 0; 1274 break; 1275 } 1276 1277 if (uov_perms & ~POE_RWX) 1278 uov_perms = POE_NONE; 1279 1280 /* R_NPBXC, S1UnprivOverlay enabled */ 1281 if (wr->uwxn && (uov_perms & POE_X)) 1282 uov_perms &= ~POE_W; 1283 1284 wr->ur &= uov_perms & POE_R; 1285 wr->uw &= uov_perms & POE_W; 1286 wr->ux &= uov_perms & POE_X; 1287 } 1288 } 1289 1290 static void compute_s1_permissions(struct kvm_vcpu *vcpu, 1291 struct s1_walk_info *wi, 1292 struct s1_walk_result *wr) 1293 { 1294 bool pan; 1295 1296 if (!s1pie_enabled(vcpu, wi->regime)) 1297 compute_s1_direct_permissions(vcpu, wi, wr); 1298 else 1299 compute_s1_indirect_permissions(vcpu, wi, wr); 1300 1301 if (!wi->hpd) 1302 compute_s1_hierarchical_permissions(vcpu, wi, wr); 1303 1304 compute_s1_overlay_permissions(vcpu, wi, wr); 1305 1306 /* R_QXXPC, S1PrivOverlay disabled */ 1307 if (!wr->pov) 1308 wr->px &= !(wr->pwxn && wr->pw); 1309 1310 /* R_NPBXC, S1UnprivOverlay disabled */ 1311 if (!wr->uov) 1312 wr->ux &= !(wr->uwxn && wr->uw); 1313 1314 pan = wi->pan && (wr->ur || wr->uw || 1315 (pan3_enabled(vcpu, wi->regime) && wr->ux)); 1316 wr->pw &= !pan; 1317 wr->pr &= !pan; 1318 } 1319 1320 static int handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr, u64 *par) 1321 { 1322 struct s1_walk_result wr = {}; 1323 struct s1_walk_info wi = {}; 1324 bool perm_fail = false; 1325 int ret, idx; 1326 1327 wi.regime = compute_translation_regime(vcpu, op); 1328 wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); 1329 wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && 1330 (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); 1331 1332 ret = setup_s1_walk(vcpu, &wi, &wr, vaddr); 1333 if (ret) 1334 goto compute_par; 1335 1336 if (wr.level == S1_MMU_DISABLED) 1337 goto compute_par; 1338 1339 idx = srcu_read_lock(&vcpu->kvm->srcu); 1340 1341 ret = walk_s1(vcpu, &wi, &wr, vaddr); 1342 1343 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1344 1345 /* 1346 * Race to update a descriptor -- restart the walk. 1347 */ 1348 if (ret == -EAGAIN) 1349 return ret; 1350 if (ret) 1351 goto compute_par; 1352 1353 compute_s1_permissions(vcpu, &wi, &wr); 1354 1355 switch (op) { 1356 case OP_AT_S1E1RP: 1357 case OP_AT_S1E1R: 1358 case OP_AT_S1E2R: 1359 perm_fail = !wr.pr; 1360 break; 1361 case OP_AT_S1E1WP: 1362 case OP_AT_S1E1W: 1363 case OP_AT_S1E2W: 1364 perm_fail = !wr.pw; 1365 break; 1366 case OP_AT_S1E0R: 1367 perm_fail = !wr.ur; 1368 break; 1369 case OP_AT_S1E0W: 1370 perm_fail = !wr.uw; 1371 break; 1372 case OP_AT_S1E1A: 1373 case OP_AT_S1E2A: 1374 break; 1375 default: 1376 BUG(); 1377 } 1378 1379 if (perm_fail) 1380 fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false); 1381 1382 compute_par: 1383 *par = compute_par_s1(vcpu, &wi, &wr); 1384 return 0; 1385 } 1386 1387 /* 1388 * Return the PAR_EL1 value as the result of a valid translation. 1389 * 1390 * If the translation is unsuccessful, the value may only contain 1391 * PAR_EL1.F, and cannot be taken at face value. It isn't an 1392 * indication of the translation having failed, only that the fast 1393 * path did not succeed, *unless* it indicates a S1 permission or 1394 * access fault. 1395 */ 1396 static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1397 { 1398 struct mmu_config config; 1399 struct kvm_s2_mmu *mmu; 1400 bool fail, mmu_cs; 1401 u64 par; 1402 1403 par = SYS_PAR_EL1_F; 1404 1405 /* 1406 * We've trapped, so everything is live on the CPU. As we will 1407 * be switching contexts behind everybody's back, disable 1408 * interrupts while holding the mmu lock. 1409 */ 1410 guard(write_lock_irqsave)(&vcpu->kvm->mmu_lock); 1411 1412 /* 1413 * If HCR_EL2.{E2H,TGE} == {1,1}, the MMU context is already 1414 * the right one (as we trapped from vEL2). If not, save the 1415 * full MMU context. 1416 * 1417 * We are also guaranteed to be in the correct context if 1418 * we're not in a nested VM. 1419 */ 1420 mmu_cs = (vcpu_has_nv(vcpu) && 1421 !(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))); 1422 if (!mmu_cs) 1423 goto skip_mmu_switch; 1424 1425 /* 1426 * Obtaining the S2 MMU for a L2 is horribly racy, and we may not 1427 * find it (recycled by another vcpu, for example). When this 1428 * happens, admit defeat immediately and use the SW (slow) path. 1429 */ 1430 mmu = lookup_s2_mmu(vcpu); 1431 if (!mmu) 1432 return par; 1433 1434 __mmu_config_save(&config); 1435 1436 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR0_EL1), SYS_TTBR0); 1437 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR1_EL1), SYS_TTBR1); 1438 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR_EL1), SYS_TCR); 1439 write_sysreg_el1(vcpu_read_sys_reg(vcpu, MAIR_EL1), SYS_MAIR); 1440 if (kvm_has_tcr2(vcpu->kvm)) { 1441 write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR2_EL1), SYS_TCR2); 1442 if (kvm_has_s1pie(vcpu->kvm)) { 1443 write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIR_EL1), SYS_PIR); 1444 write_sysreg_el1(vcpu_read_sys_reg(vcpu, PIRE0_EL1), SYS_PIRE0); 1445 } 1446 if (kvm_has_s1poe(vcpu->kvm)) { 1447 write_sysreg_el1(vcpu_read_sys_reg(vcpu, POR_EL1), SYS_POR); 1448 write_sysreg_s(vcpu_read_sys_reg(vcpu, POR_EL0), SYS_POR_EL0); 1449 } 1450 } 1451 write_sysreg_el1(vcpu_read_sys_reg(vcpu, SCTLR_EL1), SYS_SCTLR); 1452 __load_stage2(mmu); 1453 1454 skip_mmu_switch: 1455 /* Temporarily switch back to guest context */ 1456 write_sysreg_hcr(vcpu->arch.hcr_el2); 1457 isb(); 1458 1459 switch (op) { 1460 case OP_AT_S1E1RP: 1461 case OP_AT_S1E1WP: 1462 fail = at_s1e1p_fast(vcpu, op, vaddr); 1463 break; 1464 case OP_AT_S1E1R: 1465 fail = __kvm_at(OP_AT_S1E1R, vaddr); 1466 break; 1467 case OP_AT_S1E1W: 1468 fail = __kvm_at(OP_AT_S1E1W, vaddr); 1469 break; 1470 case OP_AT_S1E0R: 1471 fail = __kvm_at(OP_AT_S1E0R, vaddr); 1472 break; 1473 case OP_AT_S1E0W: 1474 fail = __kvm_at(OP_AT_S1E0W, vaddr); 1475 break; 1476 case OP_AT_S1E1A: 1477 fail = __kvm_at(OP_AT_S1E1A, vaddr); 1478 break; 1479 default: 1480 WARN_ON_ONCE(1); 1481 fail = true; 1482 break; 1483 } 1484 1485 if (!fail) 1486 par = read_sysreg_par(); 1487 1488 write_sysreg_hcr(HCR_HOST_VHE_FLAGS); 1489 1490 if (mmu_cs) 1491 __mmu_config_restore(&config); 1492 1493 return par; 1494 } 1495 1496 static bool par_check_s1_perm_fault(u64 par) 1497 { 1498 u8 fst = FIELD_GET(SYS_PAR_EL1_FST, par); 1499 1500 return ((fst & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM && 1501 !(par & SYS_PAR_EL1_S)); 1502 } 1503 1504 static bool par_check_s1_access_fault(u64 par) 1505 { 1506 u8 fst = FIELD_GET(SYS_PAR_EL1_FST, par); 1507 1508 return ((fst & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_ACCESS && 1509 !(par & SYS_PAR_EL1_S)); 1510 } 1511 1512 int __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1513 { 1514 u64 par = __kvm_at_s1e01_fast(vcpu, op, vaddr); 1515 int ret; 1516 1517 /* 1518 * If PAR_EL1 reports that AT failed on a S1 permission or access 1519 * fault, we know for sure that the PTW was able to walk the S1 1520 * tables and there's nothing else to do. 1521 * 1522 * If AT failed for any other reason, then we must walk the guest S1 1523 * to emulate the instruction. 1524 */ 1525 if ((par & SYS_PAR_EL1_F) && 1526 !par_check_s1_perm_fault(par) && 1527 !par_check_s1_access_fault(par)) { 1528 ret = handle_at_slow(vcpu, op, vaddr, &par); 1529 if (ret) 1530 return ret; 1531 } 1532 1533 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1534 return 0; 1535 } 1536 1537 int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1538 { 1539 u64 par; 1540 int ret; 1541 1542 /* 1543 * We've trapped, so everything is live on the CPU. As we will be 1544 * switching context behind everybody's back, disable interrupts... 1545 */ 1546 scoped_guard(write_lock_irqsave, &vcpu->kvm->mmu_lock) { 1547 u64 val, hcr; 1548 bool fail; 1549 1550 val = hcr = read_sysreg(hcr_el2); 1551 val &= ~HCR_TGE; 1552 val |= HCR_VM; 1553 1554 if (!vcpu_el2_e2h_is_set(vcpu)) 1555 val |= HCR_NV | HCR_NV1; 1556 1557 write_sysreg_hcr(val); 1558 isb(); 1559 1560 par = SYS_PAR_EL1_F; 1561 1562 switch (op) { 1563 case OP_AT_S1E2R: 1564 fail = __kvm_at(OP_AT_S1E1R, vaddr); 1565 break; 1566 case OP_AT_S1E2W: 1567 fail = __kvm_at(OP_AT_S1E1W, vaddr); 1568 break; 1569 case OP_AT_S1E2A: 1570 fail = __kvm_at(OP_AT_S1E1A, vaddr); 1571 break; 1572 default: 1573 WARN_ON_ONCE(1); 1574 fail = true; 1575 } 1576 1577 if (!fail) 1578 par = read_sysreg_par(); 1579 1580 write_sysreg_hcr(hcr); 1581 isb(); 1582 } 1583 1584 /* We failed the translation, let's replay it in slow motion */ 1585 if ((par & SYS_PAR_EL1_F) && !par_check_s1_perm_fault(par)) { 1586 ret = handle_at_slow(vcpu, op, vaddr, &par); 1587 if (ret) 1588 return ret; 1589 } 1590 1591 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1592 return 0; 1593 } 1594 1595 int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) 1596 { 1597 struct kvm_s2_trans out = {}; 1598 u64 ipa, par; 1599 bool write; 1600 int ret; 1601 1602 /* Do the stage-1 translation */ 1603 switch (op) { 1604 case OP_AT_S12E1R: 1605 op = OP_AT_S1E1R; 1606 write = false; 1607 break; 1608 case OP_AT_S12E1W: 1609 op = OP_AT_S1E1W; 1610 write = true; 1611 break; 1612 case OP_AT_S12E0R: 1613 op = OP_AT_S1E0R; 1614 write = false; 1615 break; 1616 case OP_AT_S12E0W: 1617 op = OP_AT_S1E0W; 1618 write = true; 1619 break; 1620 default: 1621 WARN_ON_ONCE(1); 1622 return 0; 1623 } 1624 1625 ret = __kvm_at_s1e01(vcpu, op, vaddr); 1626 if (ret) 1627 return ret; 1628 1629 par = vcpu_read_sys_reg(vcpu, PAR_EL1); 1630 if (par & SYS_PAR_EL1_F) 1631 return 0; 1632 1633 /* 1634 * If we only have a single stage of translation (EL2&0), exit 1635 * early. Same thing if {VM,DC}=={0,0}. 1636 */ 1637 if (compute_translation_regime(vcpu, op) == TR_EL20 || 1638 !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM | HCR_DC))) 1639 return 0; 1640 1641 /* Do the stage-2 translation */ 1642 ipa = (par & GENMASK_ULL(47, 12)) | (vaddr & GENMASK_ULL(11, 0)); 1643 out.esr = 0; 1644 scoped_guard(srcu, &vcpu->kvm->srcu) 1645 ret = kvm_walk_nested_s2(vcpu, ipa, &out); 1646 if (ret < 0) 1647 return ret; 1648 1649 /* Check the access permission */ 1650 if (!out.esr && 1651 ((!write && !out.readable) || (write && !out.writable))) 1652 out.esr = ESR_ELx_FSC_PERM_L(out.level & 0x3); 1653 1654 par = compute_par_s12(vcpu, par, &out); 1655 vcpu_write_sys_reg(vcpu, par, PAR_EL1); 1656 return 0; 1657 } 1658 1659 /* 1660 * Translate a VA for a given EL in a given translation regime, with 1661 * or without PAN. This requires wi->{regime, as_el0, pan} to be 1662 * set. The rest of the wi and wr should be 0-initialised. 1663 */ 1664 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 1665 struct s1_walk_result *wr, u64 va) 1666 { 1667 int ret; 1668 1669 ret = setup_s1_walk(vcpu, wi, wr, va); 1670 if (ret) 1671 return ret; 1672 1673 if (wr->level == S1_MMU_DISABLED) { 1674 wr->ur = wr->uw = wr->ux = true; 1675 wr->pr = wr->pw = wr->px = true; 1676 } else { 1677 ret = walk_s1(vcpu, wi, wr, va); 1678 if (ret) 1679 return ret; 1680 1681 compute_s1_permissions(vcpu, wi, wr); 1682 } 1683 1684 return 0; 1685 } 1686 1687 struct desc_match { 1688 u64 ipa; 1689 int level; 1690 }; 1691 1692 static int match_s1_desc(struct s1_walk_context *ctxt, void *priv) 1693 { 1694 struct desc_match *dm = priv; 1695 u64 ipa = dm->ipa; 1696 1697 /* Use S1 granule alignment */ 1698 ipa &= GENMASK(51, ctxt->wi->pgshift); 1699 1700 /* Not the IPA we're looking for? Continue. */ 1701 if (ipa != ctxt->table_ipa) 1702 return 0; 1703 1704 /* Note the level and interrupt the walk */ 1705 dm->level = ctxt->level; 1706 return -EINTR; 1707 } 1708 1709 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, int *level) 1710 { 1711 struct desc_match dm = { 1712 .ipa = ipa, 1713 }; 1714 struct s1_walk_info wi = { 1715 .filter = &(struct s1_walk_filter){ 1716 .fn = match_s1_desc, 1717 .priv = &dm, 1718 }, 1719 .as_el0 = false, 1720 .pan = false, 1721 }; 1722 struct s1_walk_result wr = {}; 1723 int ret; 1724 1725 if (is_hyp_ctxt(vcpu)) 1726 wi.regime = vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2; 1727 else 1728 wi.regime = TR_EL10; 1729 1730 ret = setup_s1_walk(vcpu, &wi, &wr, va); 1731 if (ret) 1732 return ret; 1733 1734 /* We really expect the S1 MMU to be on here... */ 1735 if (WARN_ON_ONCE(wr.level == S1_MMU_DISABLED)) { 1736 *level = 0; 1737 return 0; 1738 } 1739 1740 /* Walk the guest's PT, looking for a match along the way */ 1741 scoped_guard(srcu, &vcpu->kvm->srcu) 1742 ret = walk_s1(vcpu, &wi, &wr, va); 1743 switch (ret) { 1744 case -EINTR: 1745 /* We interrupted the walk on a match, return the level */ 1746 *level = dm.level; 1747 return 0; 1748 case 0: 1749 /* The walk completed, we failed to find the entry */ 1750 return -ENOENT; 1751 default: 1752 /* Any other error... */ 1753 return ret; 1754 } 1755 } 1756 1757 static int __lsui_swap_desc(u64 __user *ptep, u64 old, u64 new) 1758 { 1759 u64 tmp = old; 1760 int ret = 0; 1761 1762 /* 1763 * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), 1764 * as PAN toggling is not required. 1765 */ 1766 uaccess_ttbr0_enable(); 1767 1768 asm volatile(__LSUI_PREAMBLE 1769 "1: cast %[old], %[new], %[addr]\n" 1770 "2:\n" 1771 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) 1772 : [old] "+r" (old), [addr] "+Q" (*ptep), [ret] "+r" (ret) 1773 : [new] "r" (new) 1774 : "memory"); 1775 1776 uaccess_ttbr0_disable(); 1777 1778 if (ret) 1779 return ret; 1780 if (tmp != old) 1781 return -EAGAIN; 1782 1783 return ret; 1784 } 1785 1786 static int __lse_swap_desc(u64 __user *ptep, u64 old, u64 new) 1787 { 1788 u64 tmp = old; 1789 int ret = 0; 1790 1791 uaccess_enable_privileged(); 1792 1793 asm volatile(__LSE_PREAMBLE 1794 "1: cas %[old], %[new], %[addr]\n" 1795 "2:\n" 1796 _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) 1797 : [old] "+r" (old), [addr] "+Q" (*ptep), [ret] "+r" (ret) 1798 : [new] "r" (new) 1799 : "memory"); 1800 1801 uaccess_disable_privileged(); 1802 1803 if (ret) 1804 return ret; 1805 if (tmp != old) 1806 return -EAGAIN; 1807 1808 return ret; 1809 } 1810 1811 static int __llsc_swap_desc(u64 __user *ptep, u64 old, u64 new) 1812 { 1813 int ret = 1; 1814 u64 tmp; 1815 1816 uaccess_enable_privileged(); 1817 1818 asm volatile("prfm pstl1strm, %[addr]\n" 1819 "1: ldxr %[tmp], %[addr]\n" 1820 "sub %[tmp], %[tmp], %[old]\n" 1821 "cbnz %[tmp], 3f\n" 1822 "2: stlxr %w[ret], %[new], %[addr]\n" 1823 "3:\n" 1824 _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) 1825 _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) 1826 : [ret] "+r" (ret), [addr] "+Q" (*ptep), [tmp] "=&r" (tmp) 1827 : [old] "r" (old), [new] "r" (new) 1828 : "memory"); 1829 1830 uaccess_disable_privileged(); 1831 1832 /* STLXR didn't update the descriptor, or the compare failed */ 1833 if (ret == 1) 1834 return -EAGAIN; 1835 1836 return ret; 1837 } 1838 1839 int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new) 1840 { 1841 struct kvm_memory_slot *slot; 1842 unsigned long hva; 1843 u64 __user *ptep; 1844 bool writable; 1845 int offset; 1846 gfn_t gfn; 1847 int r; 1848 1849 lockdep_assert(srcu_read_lock_held(&kvm->srcu)); 1850 1851 gfn = ipa >> PAGE_SHIFT; 1852 offset = offset_in_page(ipa); 1853 slot = gfn_to_memslot(kvm, gfn); 1854 hva = gfn_to_hva_memslot_prot(slot, gfn, &writable); 1855 if (kvm_is_error_hva(hva)) 1856 return -EINVAL; 1857 if (!writable) 1858 return -EPERM; 1859 1860 ptep = (void __user *)hva + offset; 1861 if (cpus_have_final_cap(ARM64_HAS_LSUI)) 1862 r = __lsui_swap_desc(ptep, old, new); 1863 else if (cpus_have_final_cap(ARM64_HAS_LSE_ATOMICS)) 1864 r = __lse_swap_desc(ptep, old, new); 1865 else 1866 r = __llsc_swap_desc(ptep, old, new); 1867 1868 if (r < 0) 1869 return r; 1870 1871 mark_page_dirty_in_slot(kvm, slot, gfn); 1872 return 0; 1873 } 1874