1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2. 4 * No bombay mix was harmed in the writing of this file. 5 * 6 * Copyright (C) 2020 Google LLC 7 * Author: Will Deacon <will@kernel.org> 8 */ 9 10 #include <linux/bitfield.h> 11 #include <asm/kvm_pgtable.h> 12 #include <asm/stage2_pgtable.h> 13 14 15 #define KVM_PTE_TYPE BIT(1) 16 #define KVM_PTE_TYPE_BLOCK 0 17 #define KVM_PTE_TYPE_PAGE 1 18 #define KVM_PTE_TYPE_TABLE 1 19 20 #define KVM_PTE_LEAF_ATTR_LO GENMASK(11, 2) 21 22 #define KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX GENMASK(4, 2) 23 #define KVM_PTE_LEAF_ATTR_LO_S1_AP GENMASK(7, 6) 24 #define KVM_PTE_LEAF_ATTR_LO_S1_AP_RO \ 25 ({ cpus_have_final_cap(ARM64_KVM_HVHE) ? 2 : 3; }) 26 #define KVM_PTE_LEAF_ATTR_LO_S1_AP_RW \ 27 ({ cpus_have_final_cap(ARM64_KVM_HVHE) ? 0 : 1; }) 28 #define KVM_PTE_LEAF_ATTR_LO_S1_SH GENMASK(9, 8) 29 #define KVM_PTE_LEAF_ATTR_LO_S1_SH_IS 3 30 #define KVM_PTE_LEAF_ATTR_LO_S1_AF BIT(10) 31 32 #define KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR GENMASK(5, 2) 33 #define KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R BIT(6) 34 #define KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W BIT(7) 35 #define KVM_PTE_LEAF_ATTR_LO_S2_SH GENMASK(9, 8) 36 #define KVM_PTE_LEAF_ATTR_LO_S2_SH_IS 3 37 #define KVM_PTE_LEAF_ATTR_LO_S2_AF BIT(10) 38 39 #define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 50) 40 41 #define KVM_PTE_LEAF_ATTR_HI_SW GENMASK(58, 55) 42 43 #define KVM_PTE_LEAF_ATTR_HI_S1_XN BIT(54) 44 45 #define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54) 46 47 #define KVM_PTE_LEAF_ATTR_HI_S1_GP BIT(50) 48 49 #define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | \ 50 KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \ 51 KVM_PTE_LEAF_ATTR_HI_S2_XN) 52 53 #define KVM_INVALID_PTE_OWNER_MASK GENMASK(9, 2) 54 #define KVM_MAX_OWNER_ID 1 55 56 /* 57 * Used to indicate a pte for which a 'break-before-make' sequence is in 58 * progress. 59 */ 60 #define KVM_INVALID_PTE_LOCKED BIT(10) 61 62 struct kvm_pgtable_walk_data { 63 struct kvm_pgtable_walker *walker; 64 65 const u64 start; 66 u64 addr; 67 const u64 end; 68 }; 69 70 static bool kvm_pgtable_walk_skip_bbm_tlbi(const struct kvm_pgtable_visit_ctx *ctx) 71 { 72 return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM_TLBI); 73 } 74 75 static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) 76 { 77 return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO); 78 } 79 80 static bool kvm_phys_is_valid(u64 phys) 81 { 82 u64 parange_max = kvm_get_parange_max(); 83 u8 shift = id_aa64mmfr0_parange_to_phys_shift(parange_max); 84 85 return phys < BIT(shift); 86 } 87 88 static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys) 89 { 90 u64 granule = kvm_granule_size(ctx->level); 91 92 if (!kvm_level_supports_block_mapping(ctx->level)) 93 return false; 94 95 if (granule > (ctx->end - ctx->addr)) 96 return false; 97 98 if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule)) 99 return false; 100 101 return IS_ALIGNED(ctx->addr, granule); 102 } 103 104 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, s8 level) 105 { 106 u64 shift = kvm_granule_shift(level); 107 u64 mask = BIT(PAGE_SHIFT - 3) - 1; 108 109 return (data->addr >> shift) & mask; 110 } 111 112 static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) 113 { 114 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */ 115 u64 mask = BIT(pgt->ia_bits) - 1; 116 117 return (addr & mask) >> shift; 118 } 119 120 static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level) 121 { 122 struct kvm_pgtable pgt = { 123 .ia_bits = ia_bits, 124 .start_level = start_level, 125 }; 126 127 return kvm_pgd_page_idx(&pgt, -1ULL) + 1; 128 } 129 130 static bool kvm_pte_table(kvm_pte_t pte, s8 level) 131 { 132 if (level == KVM_PGTABLE_LAST_LEVEL) 133 return false; 134 135 if (!kvm_pte_valid(pte)) 136 return false; 137 138 return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE; 139 } 140 141 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops) 142 { 143 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte)); 144 } 145 146 static void kvm_clear_pte(kvm_pte_t *ptep) 147 { 148 WRITE_ONCE(*ptep, 0); 149 } 150 151 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops) 152 { 153 kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp)); 154 155 pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE); 156 pte |= KVM_PTE_VALID; 157 return pte; 158 } 159 160 static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, s8 level) 161 { 162 kvm_pte_t pte = kvm_phys_to_pte(pa); 163 u64 type = (level == KVM_PGTABLE_LAST_LEVEL) ? KVM_PTE_TYPE_PAGE : 164 KVM_PTE_TYPE_BLOCK; 165 166 pte |= attr & (KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI); 167 pte |= FIELD_PREP(KVM_PTE_TYPE, type); 168 pte |= KVM_PTE_VALID; 169 170 return pte; 171 } 172 173 static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id) 174 { 175 return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id); 176 } 177 178 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, 179 const struct kvm_pgtable_visit_ctx *ctx, 180 enum kvm_pgtable_walk_flags visit) 181 { 182 struct kvm_pgtable_walker *walker = data->walker; 183 184 /* Ensure the appropriate lock is held (e.g. RCU lock for stage-2 MMU) */ 185 WARN_ON_ONCE(kvm_pgtable_walk_shared(ctx) && !kvm_pgtable_walk_lock_held()); 186 return walker->cb(ctx, visit); 187 } 188 189 static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker, 190 int r) 191 { 192 /* 193 * Visitor callbacks return EAGAIN when the conditions that led to a 194 * fault are no longer reflected in the page tables due to a race to 195 * update a PTE. In the context of a fault handler this is interpreted 196 * as a signal to retry guest execution. 197 * 198 * Ignore the return code altogether for walkers outside a fault handler 199 * (e.g. write protecting a range of memory) and chug along with the 200 * page table walk. 201 */ 202 if (r == -EAGAIN) 203 return !(walker->flags & KVM_PGTABLE_WALK_HANDLE_FAULT); 204 205 return !r; 206 } 207 208 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, 209 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level); 210 211 static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, 212 struct kvm_pgtable_mm_ops *mm_ops, 213 kvm_pteref_t pteref, s8 level) 214 { 215 enum kvm_pgtable_walk_flags flags = data->walker->flags; 216 kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref); 217 struct kvm_pgtable_visit_ctx ctx = { 218 .ptep = ptep, 219 .old = READ_ONCE(*ptep), 220 .arg = data->walker->arg, 221 .mm_ops = mm_ops, 222 .start = data->start, 223 .addr = data->addr, 224 .end = data->end, 225 .level = level, 226 .flags = flags, 227 }; 228 int ret = 0; 229 bool reload = false; 230 kvm_pteref_t childp; 231 bool table = kvm_pte_table(ctx.old, level); 232 233 if (table && (ctx.flags & KVM_PGTABLE_WALK_TABLE_PRE)) { 234 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_PRE); 235 reload = true; 236 } 237 238 if (!table && (ctx.flags & KVM_PGTABLE_WALK_LEAF)) { 239 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_LEAF); 240 reload = true; 241 } 242 243 /* 244 * Reload the page table after invoking the walker callback for leaf 245 * entries or after pre-order traversal, to allow the walker to descend 246 * into a newly installed or replaced table. 247 */ 248 if (reload) { 249 ctx.old = READ_ONCE(*ptep); 250 table = kvm_pte_table(ctx.old, level); 251 } 252 253 if (!kvm_pgtable_walk_continue(data->walker, ret)) 254 goto out; 255 256 if (!table) { 257 data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level)); 258 data->addr += kvm_granule_size(level); 259 goto out; 260 } 261 262 childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops); 263 ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1); 264 if (!kvm_pgtable_walk_continue(data->walker, ret)) 265 goto out; 266 267 if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST) 268 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST); 269 270 out: 271 if (kvm_pgtable_walk_continue(data->walker, ret)) 272 return 0; 273 274 return ret; 275 } 276 277 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, 278 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level) 279 { 280 u32 idx; 281 int ret = 0; 282 283 if (WARN_ON_ONCE(level < KVM_PGTABLE_FIRST_LEVEL || 284 level > KVM_PGTABLE_LAST_LEVEL)) 285 return -EINVAL; 286 287 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { 288 kvm_pteref_t pteref = &pgtable[idx]; 289 290 if (data->addr >= data->end) 291 break; 292 293 ret = __kvm_pgtable_visit(data, mm_ops, pteref, level); 294 if (ret) 295 break; 296 } 297 298 return ret; 299 } 300 301 static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data) 302 { 303 u32 idx; 304 int ret = 0; 305 u64 limit = BIT(pgt->ia_bits); 306 307 if (data->addr > limit || data->end > limit) 308 return -ERANGE; 309 310 if (!pgt->pgd) 311 return -EINVAL; 312 313 for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) { 314 kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE]; 315 316 ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level); 317 if (ret) 318 break; 319 } 320 321 return ret; 322 } 323 324 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, 325 struct kvm_pgtable_walker *walker) 326 { 327 struct kvm_pgtable_walk_data walk_data = { 328 .start = ALIGN_DOWN(addr, PAGE_SIZE), 329 .addr = ALIGN_DOWN(addr, PAGE_SIZE), 330 .end = PAGE_ALIGN(walk_data.addr + size), 331 .walker = walker, 332 }; 333 int r; 334 335 r = kvm_pgtable_walk_begin(walker); 336 if (r) 337 return r; 338 339 r = _kvm_pgtable_walk(pgt, &walk_data); 340 kvm_pgtable_walk_end(walker); 341 342 return r; 343 } 344 345 struct leaf_walk_data { 346 kvm_pte_t pte; 347 s8 level; 348 }; 349 350 static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, 351 enum kvm_pgtable_walk_flags visit) 352 { 353 struct leaf_walk_data *data = ctx->arg; 354 355 data->pte = ctx->old; 356 data->level = ctx->level; 357 358 return 0; 359 } 360 361 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, 362 kvm_pte_t *ptep, s8 *level) 363 { 364 struct leaf_walk_data data; 365 struct kvm_pgtable_walker walker = { 366 .cb = leaf_walker, 367 .flags = KVM_PGTABLE_WALK_LEAF, 368 .arg = &data, 369 }; 370 int ret; 371 372 ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE), 373 PAGE_SIZE, &walker); 374 if (!ret) { 375 if (ptep) 376 *ptep = data.pte; 377 if (level) 378 *level = data.level; 379 } 380 381 return ret; 382 } 383 384 struct hyp_map_data { 385 const u64 phys; 386 kvm_pte_t attr; 387 }; 388 389 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep) 390 { 391 bool device = prot & KVM_PGTABLE_PROT_DEVICE; 392 u32 mtype = device ? MT_DEVICE_nGnRE : MT_NORMAL; 393 kvm_pte_t attr = FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX, mtype); 394 u32 sh = KVM_PTE_LEAF_ATTR_LO_S1_SH_IS; 395 u32 ap = (prot & KVM_PGTABLE_PROT_W) ? KVM_PTE_LEAF_ATTR_LO_S1_AP_RW : 396 KVM_PTE_LEAF_ATTR_LO_S1_AP_RO; 397 398 if (!(prot & KVM_PGTABLE_PROT_R)) 399 return -EINVAL; 400 401 if (prot & KVM_PGTABLE_PROT_X) { 402 if (prot & KVM_PGTABLE_PROT_W) 403 return -EINVAL; 404 405 if (device) 406 return -EINVAL; 407 408 if (system_supports_bti_kernel()) 409 attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP; 410 } else { 411 attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN; 412 } 413 414 attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap); 415 if (!kvm_lpa2_is_enabled()) 416 attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh); 417 attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF; 418 attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW; 419 *ptep = attr; 420 421 return 0; 422 } 423 424 enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte) 425 { 426 enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW; 427 u32 ap; 428 429 if (!kvm_pte_valid(pte)) 430 return prot; 431 432 if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_XN)) 433 prot |= KVM_PGTABLE_PROT_X; 434 435 ap = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_AP, pte); 436 if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RO) 437 prot |= KVM_PGTABLE_PROT_R; 438 else if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RW) 439 prot |= KVM_PGTABLE_PROT_RW; 440 441 return prot; 442 } 443 444 static bool hyp_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, 445 struct hyp_map_data *data) 446 { 447 u64 phys = data->phys + (ctx->addr - ctx->start); 448 kvm_pte_t new; 449 450 if (!kvm_block_mapping_supported(ctx, phys)) 451 return false; 452 453 new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level); 454 if (ctx->old == new) 455 return true; 456 if (!kvm_pte_valid(ctx->old)) 457 ctx->mm_ops->get_page(ctx->ptep); 458 else if (WARN_ON((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW)) 459 return false; 460 461 smp_store_release(ctx->ptep, new); 462 return true; 463 } 464 465 static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx, 466 enum kvm_pgtable_walk_flags visit) 467 { 468 kvm_pte_t *childp, new; 469 struct hyp_map_data *data = ctx->arg; 470 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 471 472 if (hyp_map_walker_try_leaf(ctx, data)) 473 return 0; 474 475 if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL)) 476 return -EINVAL; 477 478 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL); 479 if (!childp) 480 return -ENOMEM; 481 482 new = kvm_init_table_pte(childp, mm_ops); 483 mm_ops->get_page(ctx->ptep); 484 smp_store_release(ctx->ptep, new); 485 486 return 0; 487 } 488 489 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, 490 enum kvm_pgtable_prot prot) 491 { 492 int ret; 493 struct hyp_map_data map_data = { 494 .phys = ALIGN_DOWN(phys, PAGE_SIZE), 495 }; 496 struct kvm_pgtable_walker walker = { 497 .cb = hyp_map_walker, 498 .flags = KVM_PGTABLE_WALK_LEAF, 499 .arg = &map_data, 500 }; 501 502 ret = hyp_set_prot_attr(prot, &map_data.attr); 503 if (ret) 504 return ret; 505 506 ret = kvm_pgtable_walk(pgt, addr, size, &walker); 507 dsb(ishst); 508 isb(); 509 return ret; 510 } 511 512 static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx, 513 enum kvm_pgtable_walk_flags visit) 514 { 515 kvm_pte_t *childp = NULL; 516 u64 granule = kvm_granule_size(ctx->level); 517 u64 *unmapped = ctx->arg; 518 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 519 520 if (!kvm_pte_valid(ctx->old)) 521 return -EINVAL; 522 523 if (kvm_pte_table(ctx->old, ctx->level)) { 524 childp = kvm_pte_follow(ctx->old, mm_ops); 525 526 if (mm_ops->page_count(childp) != 1) 527 return 0; 528 529 kvm_clear_pte(ctx->ptep); 530 dsb(ishst); 531 __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), ctx->level); 532 } else { 533 if (ctx->end - ctx->addr < granule) 534 return -EINVAL; 535 536 kvm_clear_pte(ctx->ptep); 537 dsb(ishst); 538 __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level); 539 *unmapped += granule; 540 } 541 542 dsb(ish); 543 isb(); 544 mm_ops->put_page(ctx->ptep); 545 546 if (childp) 547 mm_ops->put_page(childp); 548 549 return 0; 550 } 551 552 u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) 553 { 554 u64 unmapped = 0; 555 struct kvm_pgtable_walker walker = { 556 .cb = hyp_unmap_walker, 557 .arg = &unmapped, 558 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, 559 }; 560 561 if (!pgt->mm_ops->page_count) 562 return 0; 563 564 kvm_pgtable_walk(pgt, addr, size, &walker); 565 return unmapped; 566 } 567 568 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, 569 struct kvm_pgtable_mm_ops *mm_ops) 570 { 571 s8 start_level = KVM_PGTABLE_LAST_LEVEL + 1 - 572 ARM64_HW_PGTABLE_LEVELS(va_bits); 573 574 if (start_level < KVM_PGTABLE_FIRST_LEVEL || 575 start_level > KVM_PGTABLE_LAST_LEVEL) 576 return -EINVAL; 577 578 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL); 579 if (!pgt->pgd) 580 return -ENOMEM; 581 582 pgt->ia_bits = va_bits; 583 pgt->start_level = start_level; 584 pgt->mm_ops = mm_ops; 585 pgt->mmu = NULL; 586 pgt->force_pte_cb = NULL; 587 588 return 0; 589 } 590 591 static int hyp_free_walker(const struct kvm_pgtable_visit_ctx *ctx, 592 enum kvm_pgtable_walk_flags visit) 593 { 594 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 595 596 if (!kvm_pte_valid(ctx->old)) 597 return 0; 598 599 mm_ops->put_page(ctx->ptep); 600 601 if (kvm_pte_table(ctx->old, ctx->level)) 602 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops)); 603 604 return 0; 605 } 606 607 void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) 608 { 609 struct kvm_pgtable_walker walker = { 610 .cb = hyp_free_walker, 611 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, 612 }; 613 614 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); 615 pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd)); 616 pgt->pgd = NULL; 617 } 618 619 struct stage2_map_data { 620 const u64 phys; 621 kvm_pte_t attr; 622 u8 owner_id; 623 624 kvm_pte_t *anchor; 625 kvm_pte_t *childp; 626 627 struct kvm_s2_mmu *mmu; 628 void *memcache; 629 630 /* Force mappings to page granularity */ 631 bool force_pte; 632 }; 633 634 u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) 635 { 636 u64 vtcr = VTCR_EL2_FLAGS; 637 s8 lvls; 638 639 vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT; 640 vtcr |= VTCR_EL2_T0SZ(phys_shift); 641 /* 642 * Use a minimum 2 level page table to prevent splitting 643 * host PMD huge pages at stage2. 644 */ 645 lvls = stage2_pgtable_levels(phys_shift); 646 if (lvls < 2) 647 lvls = 2; 648 649 /* 650 * When LPA2 is enabled, the HW supports an extra level of translation 651 * (for 5 in total) when using 4K pages. It also introduces VTCR_EL2.SL2 652 * to as an addition to SL0 to enable encoding this extra start level. 653 * However, since we always use concatenated pages for the first level 654 * lookup, we will never need this extra level and therefore do not need 655 * to touch SL2. 656 */ 657 vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls); 658 659 #ifdef CONFIG_ARM64_HW_AFDBM 660 /* 661 * Enable the Hardware Access Flag management, unconditionally 662 * on all CPUs. In systems that have asymmetric support for the feature 663 * this allows KVM to leverage hardware support on the subset of cores 664 * that implement the feature. 665 * 666 * The architecture requires VTCR_EL2.HA to be RES0 (thus ignored by 667 * hardware) on implementations that do not advertise support for the 668 * feature. As such, setting HA unconditionally is safe, unless you 669 * happen to be running on a design that has unadvertised support for 670 * HAFDBS. Here be dragons. 671 */ 672 if (!cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38)) 673 vtcr |= VTCR_EL2_HA; 674 #endif /* CONFIG_ARM64_HW_AFDBM */ 675 676 if (kvm_lpa2_is_enabled()) 677 vtcr |= VTCR_EL2_DS; 678 679 /* Set the vmid bits */ 680 vtcr |= (get_vmid_bits(mmfr1) == 16) ? 681 VTCR_EL2_VS_16BIT : 682 VTCR_EL2_VS_8BIT; 683 684 return vtcr; 685 } 686 687 static bool stage2_has_fwb(struct kvm_pgtable *pgt) 688 { 689 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 690 return false; 691 692 return !(pgt->flags & KVM_PGTABLE_S2_NOFWB); 693 } 694 695 void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, 696 phys_addr_t addr, size_t size) 697 { 698 unsigned long pages, inval_pages; 699 700 if (!system_supports_tlb_range()) { 701 kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); 702 return; 703 } 704 705 pages = size >> PAGE_SHIFT; 706 while (pages > 0) { 707 inval_pages = min(pages, MAX_TLBI_RANGE_PAGES); 708 kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, inval_pages); 709 710 addr += inval_pages << PAGE_SHIFT; 711 pages -= inval_pages; 712 } 713 } 714 715 #define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt)) 716 717 static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot, 718 kvm_pte_t *ptep) 719 { 720 bool device = prot & KVM_PGTABLE_PROT_DEVICE; 721 kvm_pte_t attr = device ? KVM_S2_MEMATTR(pgt, DEVICE_nGnRE) : 722 KVM_S2_MEMATTR(pgt, NORMAL); 723 u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS; 724 725 if (!(prot & KVM_PGTABLE_PROT_X)) 726 attr |= KVM_PTE_LEAF_ATTR_HI_S2_XN; 727 else if (device) 728 return -EINVAL; 729 730 if (prot & KVM_PGTABLE_PROT_R) 731 attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R; 732 733 if (prot & KVM_PGTABLE_PROT_W) 734 attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; 735 736 if (!kvm_lpa2_is_enabled()) 737 attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh); 738 739 attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF; 740 attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW; 741 *ptep = attr; 742 743 return 0; 744 } 745 746 enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte) 747 { 748 enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW; 749 750 if (!kvm_pte_valid(pte)) 751 return prot; 752 753 if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R) 754 prot |= KVM_PGTABLE_PROT_R; 755 if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W) 756 prot |= KVM_PGTABLE_PROT_W; 757 if (!(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN)) 758 prot |= KVM_PGTABLE_PROT_X; 759 760 return prot; 761 } 762 763 static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new) 764 { 765 if (!kvm_pte_valid(old) || !kvm_pte_valid(new)) 766 return true; 767 768 return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)); 769 } 770 771 static bool stage2_pte_is_counted(kvm_pte_t pte) 772 { 773 /* 774 * The refcount tracks valid entries as well as invalid entries if they 775 * encode ownership of a page to another entity than the page-table 776 * owner, whose id is 0. 777 */ 778 return !!pte; 779 } 780 781 static bool stage2_pte_is_locked(kvm_pte_t pte) 782 { 783 return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED); 784 } 785 786 static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new) 787 { 788 if (!kvm_pgtable_walk_shared(ctx)) { 789 WRITE_ONCE(*ctx->ptep, new); 790 return true; 791 } 792 793 return cmpxchg(ctx->ptep, ctx->old, new) == ctx->old; 794 } 795 796 /** 797 * stage2_try_break_pte() - Invalidates a pte according to the 798 * 'break-before-make' requirements of the 799 * architecture. 800 * 801 * @ctx: context of the visited pte. 802 * @mmu: stage-2 mmu 803 * 804 * Returns: true if the pte was successfully broken. 805 * 806 * If the removed pte was valid, performs the necessary serialization and TLB 807 * invalidation for the old value. For counted ptes, drops the reference count 808 * on the containing table page. 809 */ 810 static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, 811 struct kvm_s2_mmu *mmu) 812 { 813 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 814 815 if (stage2_pte_is_locked(ctx->old)) { 816 /* 817 * Should never occur if this walker has exclusive access to the 818 * page tables. 819 */ 820 WARN_ON(!kvm_pgtable_walk_shared(ctx)); 821 return false; 822 } 823 824 if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED)) 825 return false; 826 827 if (!kvm_pgtable_walk_skip_bbm_tlbi(ctx)) { 828 /* 829 * Perform the appropriate TLB invalidation based on the 830 * evicted pte value (if any). 831 */ 832 if (kvm_pte_table(ctx->old, ctx->level)) 833 kvm_tlb_flush_vmid_range(mmu, ctx->addr, 834 kvm_granule_size(ctx->level)); 835 else if (kvm_pte_valid(ctx->old)) 836 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, 837 ctx->addr, ctx->level); 838 } 839 840 if (stage2_pte_is_counted(ctx->old)) 841 mm_ops->put_page(ctx->ptep); 842 843 return true; 844 } 845 846 static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new) 847 { 848 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 849 850 WARN_ON(!stage2_pte_is_locked(*ctx->ptep)); 851 852 if (stage2_pte_is_counted(new)) 853 mm_ops->get_page(ctx->ptep); 854 855 smp_store_release(ctx->ptep, new); 856 } 857 858 static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt) 859 { 860 /* 861 * If FEAT_TLBIRANGE is implemented, defer the individual 862 * TLB invalidations until the entire walk is finished, and 863 * then use the range-based TLBI instructions to do the 864 * invalidations. Condition deferred TLB invalidation on the 865 * system supporting FWB as the optimization is entirely 866 * pointless when the unmap walker needs to perform CMOs. 867 */ 868 return system_supports_tlb_range() && stage2_has_fwb(pgt); 869 } 870 871 static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx, 872 struct kvm_s2_mmu *mmu, 873 struct kvm_pgtable_mm_ops *mm_ops) 874 { 875 struct kvm_pgtable *pgt = ctx->arg; 876 877 /* 878 * Clear the existing PTE, and perform break-before-make if it was 879 * valid. Depending on the system support, defer the TLB maintenance 880 * for the same until the entire unmap walk is completed. 881 */ 882 if (kvm_pte_valid(ctx->old)) { 883 kvm_clear_pte(ctx->ptep); 884 885 if (!stage2_unmap_defer_tlb_flush(pgt)) 886 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, 887 ctx->addr, ctx->level); 888 } 889 890 mm_ops->put_page(ctx->ptep); 891 } 892 893 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte) 894 { 895 u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR; 896 return memattr == KVM_S2_MEMATTR(pgt, NORMAL); 897 } 898 899 static bool stage2_pte_executable(kvm_pte_t pte) 900 { 901 return !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN); 902 } 903 904 static u64 stage2_map_walker_phys_addr(const struct kvm_pgtable_visit_ctx *ctx, 905 const struct stage2_map_data *data) 906 { 907 u64 phys = data->phys; 908 909 /* 910 * Stage-2 walks to update ownership data are communicated to the map 911 * walker using an invalid PA. Avoid offsetting an already invalid PA, 912 * which could overflow and make the address valid again. 913 */ 914 if (!kvm_phys_is_valid(phys)) 915 return phys; 916 917 /* 918 * Otherwise, work out the correct PA based on how far the walk has 919 * gotten. 920 */ 921 return phys + (ctx->addr - ctx->start); 922 } 923 924 static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx, 925 struct stage2_map_data *data) 926 { 927 u64 phys = stage2_map_walker_phys_addr(ctx, data); 928 929 if (data->force_pte && ctx->level < KVM_PGTABLE_LAST_LEVEL) 930 return false; 931 932 return kvm_block_mapping_supported(ctx, phys); 933 } 934 935 static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, 936 struct stage2_map_data *data) 937 { 938 kvm_pte_t new; 939 u64 phys = stage2_map_walker_phys_addr(ctx, data); 940 u64 granule = kvm_granule_size(ctx->level); 941 struct kvm_pgtable *pgt = data->mmu->pgt; 942 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 943 944 if (!stage2_leaf_mapping_allowed(ctx, data)) 945 return -E2BIG; 946 947 if (kvm_phys_is_valid(phys)) 948 new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level); 949 else 950 new = kvm_init_invalid_leaf_owner(data->owner_id); 951 952 /* 953 * Skip updating the PTE if we are trying to recreate the exact 954 * same mapping or only change the access permissions. Instead, 955 * the vCPU will exit one more time from guest if still needed 956 * and then go through the path of relaxing permissions. 957 */ 958 if (!stage2_pte_needs_update(ctx->old, new)) 959 return -EAGAIN; 960 961 if (!stage2_try_break_pte(ctx, data->mmu)) 962 return -EAGAIN; 963 964 /* Perform CMOs before installation of the guest stage-2 PTE */ 965 if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc && 966 stage2_pte_cacheable(pgt, new)) 967 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), 968 granule); 969 970 if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou && 971 stage2_pte_executable(new)) 972 mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); 973 974 stage2_make_pte(ctx, new); 975 976 return 0; 977 } 978 979 static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx, 980 struct stage2_map_data *data) 981 { 982 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 983 kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops); 984 int ret; 985 986 if (!stage2_leaf_mapping_allowed(ctx, data)) 987 return 0; 988 989 ret = stage2_map_walker_try_leaf(ctx, data); 990 if (ret) 991 return ret; 992 993 mm_ops->free_unlinked_table(childp, ctx->level); 994 return 0; 995 } 996 997 static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx, 998 struct stage2_map_data *data) 999 { 1000 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 1001 kvm_pte_t *childp, new; 1002 int ret; 1003 1004 ret = stage2_map_walker_try_leaf(ctx, data); 1005 if (ret != -E2BIG) 1006 return ret; 1007 1008 if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL)) 1009 return -EINVAL; 1010 1011 if (!data->memcache) 1012 return -ENOMEM; 1013 1014 childp = mm_ops->zalloc_page(data->memcache); 1015 if (!childp) 1016 return -ENOMEM; 1017 1018 if (!stage2_try_break_pte(ctx, data->mmu)) { 1019 mm_ops->put_page(childp); 1020 return -EAGAIN; 1021 } 1022 1023 /* 1024 * If we've run into an existing block mapping then replace it with 1025 * a table. Accesses beyond 'end' that fall within the new table 1026 * will be mapped lazily. 1027 */ 1028 new = kvm_init_table_pte(childp, mm_ops); 1029 stage2_make_pte(ctx, new); 1030 1031 return 0; 1032 } 1033 1034 /* 1035 * The TABLE_PRE callback runs for table entries on the way down, looking 1036 * for table entries which we could conceivably replace with a block entry 1037 * for this mapping. If it finds one it replaces the entry and calls 1038 * kvm_pgtable_mm_ops::free_unlinked_table() to tear down the detached table. 1039 * 1040 * Otherwise, the LEAF callback performs the mapping at the existing leaves 1041 * instead. 1042 */ 1043 static int stage2_map_walker(const struct kvm_pgtable_visit_ctx *ctx, 1044 enum kvm_pgtable_walk_flags visit) 1045 { 1046 struct stage2_map_data *data = ctx->arg; 1047 1048 switch (visit) { 1049 case KVM_PGTABLE_WALK_TABLE_PRE: 1050 return stage2_map_walk_table_pre(ctx, data); 1051 case KVM_PGTABLE_WALK_LEAF: 1052 return stage2_map_walk_leaf(ctx, data); 1053 default: 1054 return -EINVAL; 1055 } 1056 } 1057 1058 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, 1059 u64 phys, enum kvm_pgtable_prot prot, 1060 void *mc, enum kvm_pgtable_walk_flags flags) 1061 { 1062 int ret; 1063 struct stage2_map_data map_data = { 1064 .phys = ALIGN_DOWN(phys, PAGE_SIZE), 1065 .mmu = pgt->mmu, 1066 .memcache = mc, 1067 .force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot), 1068 }; 1069 struct kvm_pgtable_walker walker = { 1070 .cb = stage2_map_walker, 1071 .flags = flags | 1072 KVM_PGTABLE_WALK_TABLE_PRE | 1073 KVM_PGTABLE_WALK_LEAF, 1074 .arg = &map_data, 1075 }; 1076 1077 if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys))) 1078 return -EINVAL; 1079 1080 ret = stage2_set_prot_attr(pgt, prot, &map_data.attr); 1081 if (ret) 1082 return ret; 1083 1084 ret = kvm_pgtable_walk(pgt, addr, size, &walker); 1085 dsb(ishst); 1086 return ret; 1087 } 1088 1089 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, 1090 void *mc, u8 owner_id) 1091 { 1092 int ret; 1093 struct stage2_map_data map_data = { 1094 .phys = KVM_PHYS_INVALID, 1095 .mmu = pgt->mmu, 1096 .memcache = mc, 1097 .owner_id = owner_id, 1098 .force_pte = true, 1099 }; 1100 struct kvm_pgtable_walker walker = { 1101 .cb = stage2_map_walker, 1102 .flags = KVM_PGTABLE_WALK_TABLE_PRE | 1103 KVM_PGTABLE_WALK_LEAF, 1104 .arg = &map_data, 1105 }; 1106 1107 if (owner_id > KVM_MAX_OWNER_ID) 1108 return -EINVAL; 1109 1110 ret = kvm_pgtable_walk(pgt, addr, size, &walker); 1111 return ret; 1112 } 1113 1114 static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx, 1115 enum kvm_pgtable_walk_flags visit) 1116 { 1117 struct kvm_pgtable *pgt = ctx->arg; 1118 struct kvm_s2_mmu *mmu = pgt->mmu; 1119 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 1120 kvm_pte_t *childp = NULL; 1121 bool need_flush = false; 1122 1123 if (!kvm_pte_valid(ctx->old)) { 1124 if (stage2_pte_is_counted(ctx->old)) { 1125 kvm_clear_pte(ctx->ptep); 1126 mm_ops->put_page(ctx->ptep); 1127 } 1128 return 0; 1129 } 1130 1131 if (kvm_pte_table(ctx->old, ctx->level)) { 1132 childp = kvm_pte_follow(ctx->old, mm_ops); 1133 1134 if (mm_ops->page_count(childp) != 1) 1135 return 0; 1136 } else if (stage2_pte_cacheable(pgt, ctx->old)) { 1137 need_flush = !stage2_has_fwb(pgt); 1138 } 1139 1140 /* 1141 * This is similar to the map() path in that we unmap the entire 1142 * block entry and rely on the remaining portions being faulted 1143 * back lazily. 1144 */ 1145 stage2_unmap_put_pte(ctx, mmu, mm_ops); 1146 1147 if (need_flush && mm_ops->dcache_clean_inval_poc) 1148 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops), 1149 kvm_granule_size(ctx->level)); 1150 1151 if (childp) 1152 mm_ops->put_page(childp); 1153 1154 return 0; 1155 } 1156 1157 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) 1158 { 1159 int ret; 1160 struct kvm_pgtable_walker walker = { 1161 .cb = stage2_unmap_walker, 1162 .arg = pgt, 1163 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, 1164 }; 1165 1166 ret = kvm_pgtable_walk(pgt, addr, size, &walker); 1167 if (stage2_unmap_defer_tlb_flush(pgt)) 1168 /* Perform the deferred TLB invalidations */ 1169 kvm_tlb_flush_vmid_range(pgt->mmu, addr, size); 1170 1171 return ret; 1172 } 1173 1174 struct stage2_attr_data { 1175 kvm_pte_t attr_set; 1176 kvm_pte_t attr_clr; 1177 kvm_pte_t pte; 1178 s8 level; 1179 }; 1180 1181 static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx, 1182 enum kvm_pgtable_walk_flags visit) 1183 { 1184 kvm_pte_t pte = ctx->old; 1185 struct stage2_attr_data *data = ctx->arg; 1186 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 1187 1188 if (!kvm_pte_valid(ctx->old)) 1189 return -EAGAIN; 1190 1191 data->level = ctx->level; 1192 data->pte = pte; 1193 pte &= ~data->attr_clr; 1194 pte |= data->attr_set; 1195 1196 /* 1197 * We may race with the CPU trying to set the access flag here, 1198 * but worst-case the access flag update gets lost and will be 1199 * set on the next access instead. 1200 */ 1201 if (data->pte != pte) { 1202 /* 1203 * Invalidate instruction cache before updating the guest 1204 * stage-2 PTE if we are going to add executable permission. 1205 */ 1206 if (mm_ops->icache_inval_pou && 1207 stage2_pte_executable(pte) && !stage2_pte_executable(ctx->old)) 1208 mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops), 1209 kvm_granule_size(ctx->level)); 1210 1211 if (!stage2_try_set_pte(ctx, pte)) 1212 return -EAGAIN; 1213 } 1214 1215 return 0; 1216 } 1217 1218 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, 1219 u64 size, kvm_pte_t attr_set, 1220 kvm_pte_t attr_clr, kvm_pte_t *orig_pte, 1221 s8 *level, enum kvm_pgtable_walk_flags flags) 1222 { 1223 int ret; 1224 kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI; 1225 struct stage2_attr_data data = { 1226 .attr_set = attr_set & attr_mask, 1227 .attr_clr = attr_clr & attr_mask, 1228 }; 1229 struct kvm_pgtable_walker walker = { 1230 .cb = stage2_attr_walker, 1231 .arg = &data, 1232 .flags = flags | KVM_PGTABLE_WALK_LEAF, 1233 }; 1234 1235 ret = kvm_pgtable_walk(pgt, addr, size, &walker); 1236 if (ret) 1237 return ret; 1238 1239 if (orig_pte) 1240 *orig_pte = data.pte; 1241 1242 if (level) 1243 *level = data.level; 1244 return 0; 1245 } 1246 1247 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) 1248 { 1249 return stage2_update_leaf_attrs(pgt, addr, size, 0, 1250 KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W, 1251 NULL, NULL, 0); 1252 } 1253 1254 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr) 1255 { 1256 kvm_pte_t pte = 0; 1257 int ret; 1258 1259 ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0, 1260 &pte, NULL, 1261 KVM_PGTABLE_WALK_HANDLE_FAULT | 1262 KVM_PGTABLE_WALK_SHARED); 1263 if (!ret) 1264 dsb(ishst); 1265 1266 return pte; 1267 } 1268 1269 struct stage2_age_data { 1270 bool mkold; 1271 bool young; 1272 }; 1273 1274 static int stage2_age_walker(const struct kvm_pgtable_visit_ctx *ctx, 1275 enum kvm_pgtable_walk_flags visit) 1276 { 1277 kvm_pte_t new = ctx->old & ~KVM_PTE_LEAF_ATTR_LO_S2_AF; 1278 struct stage2_age_data *data = ctx->arg; 1279 1280 if (!kvm_pte_valid(ctx->old) || new == ctx->old) 1281 return 0; 1282 1283 data->young = true; 1284 1285 /* 1286 * stage2_age_walker() is always called while holding the MMU lock for 1287 * write, so this will always succeed. Nonetheless, this deliberately 1288 * follows the race detection pattern of the other stage-2 walkers in 1289 * case the locking mechanics of the MMU notifiers is ever changed. 1290 */ 1291 if (data->mkold && !stage2_try_set_pte(ctx, new)) 1292 return -EAGAIN; 1293 1294 /* 1295 * "But where's the TLBI?!", you scream. 1296 * "Over in the core code", I sigh. 1297 * 1298 * See the '->clear_flush_young()' callback on the KVM mmu notifier. 1299 */ 1300 return 0; 1301 } 1302 1303 bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, 1304 u64 size, bool mkold) 1305 { 1306 struct stage2_age_data data = { 1307 .mkold = mkold, 1308 }; 1309 struct kvm_pgtable_walker walker = { 1310 .cb = stage2_age_walker, 1311 .arg = &data, 1312 .flags = KVM_PGTABLE_WALK_LEAF, 1313 }; 1314 1315 WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker)); 1316 return data.young; 1317 } 1318 1319 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, 1320 enum kvm_pgtable_prot prot) 1321 { 1322 int ret; 1323 s8 level; 1324 kvm_pte_t set = 0, clr = 0; 1325 1326 if (prot & KVM_PTE_LEAF_ATTR_HI_SW) 1327 return -EINVAL; 1328 1329 if (prot & KVM_PGTABLE_PROT_R) 1330 set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R; 1331 1332 if (prot & KVM_PGTABLE_PROT_W) 1333 set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; 1334 1335 if (prot & KVM_PGTABLE_PROT_X) 1336 clr |= KVM_PTE_LEAF_ATTR_HI_S2_XN; 1337 1338 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, 1339 KVM_PGTABLE_WALK_HANDLE_FAULT | 1340 KVM_PGTABLE_WALK_SHARED); 1341 if (!ret || ret == -EAGAIN) 1342 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); 1343 return ret; 1344 } 1345 1346 static int stage2_flush_walker(const struct kvm_pgtable_visit_ctx *ctx, 1347 enum kvm_pgtable_walk_flags visit) 1348 { 1349 struct kvm_pgtable *pgt = ctx->arg; 1350 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; 1351 1352 if (!kvm_pte_valid(ctx->old) || !stage2_pte_cacheable(pgt, ctx->old)) 1353 return 0; 1354 1355 if (mm_ops->dcache_clean_inval_poc) 1356 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops), 1357 kvm_granule_size(ctx->level)); 1358 return 0; 1359 } 1360 1361 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) 1362 { 1363 struct kvm_pgtable_walker walker = { 1364 .cb = stage2_flush_walker, 1365 .flags = KVM_PGTABLE_WALK_LEAF, 1366 .arg = pgt, 1367 }; 1368 1369 if (stage2_has_fwb(pgt)) 1370 return 0; 1371 1372 return kvm_pgtable_walk(pgt, addr, size, &walker); 1373 } 1374 1375 kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, 1376 u64 phys, s8 level, 1377 enum kvm_pgtable_prot prot, 1378 void *mc, bool force_pte) 1379 { 1380 struct stage2_map_data map_data = { 1381 .phys = phys, 1382 .mmu = pgt->mmu, 1383 .memcache = mc, 1384 .force_pte = force_pte, 1385 }; 1386 struct kvm_pgtable_walker walker = { 1387 .cb = stage2_map_walker, 1388 .flags = KVM_PGTABLE_WALK_LEAF | 1389 KVM_PGTABLE_WALK_SKIP_BBM_TLBI | 1390 KVM_PGTABLE_WALK_SKIP_CMO, 1391 .arg = &map_data, 1392 }; 1393 /* 1394 * The input address (.addr) is irrelevant for walking an 1395 * unlinked table. Construct an ambiguous IA range to map 1396 * kvm_granule_size(level) worth of memory. 1397 */ 1398 struct kvm_pgtable_walk_data data = { 1399 .walker = &walker, 1400 .addr = 0, 1401 .end = kvm_granule_size(level), 1402 }; 1403 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; 1404 kvm_pte_t *pgtable; 1405 int ret; 1406 1407 if (!IS_ALIGNED(phys, kvm_granule_size(level))) 1408 return ERR_PTR(-EINVAL); 1409 1410 ret = stage2_set_prot_attr(pgt, prot, &map_data.attr); 1411 if (ret) 1412 return ERR_PTR(ret); 1413 1414 pgtable = mm_ops->zalloc_page(mc); 1415 if (!pgtable) 1416 return ERR_PTR(-ENOMEM); 1417 1418 ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable, 1419 level + 1); 1420 if (ret) { 1421 kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level); 1422 mm_ops->put_page(pgtable); 1423 return ERR_PTR(ret); 1424 } 1425 1426 return pgtable; 1427 } 1428 1429 /* 1430 * Get the number of page-tables needed to replace a block with a 1431 * fully populated tree up to the PTE entries. Note that @level is 1432 * interpreted as in "level @level entry". 1433 */ 1434 static int stage2_block_get_nr_page_tables(s8 level) 1435 { 1436 switch (level) { 1437 case 1: 1438 return PTRS_PER_PTE + 1; 1439 case 2: 1440 return 1; 1441 case 3: 1442 return 0; 1443 default: 1444 WARN_ON_ONCE(level < KVM_PGTABLE_MIN_BLOCK_LEVEL || 1445 level > KVM_PGTABLE_LAST_LEVEL); 1446 return -EINVAL; 1447 }; 1448 } 1449 1450 static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, 1451 enum kvm_pgtable_walk_flags visit) 1452 { 1453 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 1454 struct kvm_mmu_memory_cache *mc = ctx->arg; 1455 struct kvm_s2_mmu *mmu; 1456 kvm_pte_t pte = ctx->old, new, *childp; 1457 enum kvm_pgtable_prot prot; 1458 s8 level = ctx->level; 1459 bool force_pte; 1460 int nr_pages; 1461 u64 phys; 1462 1463 /* No huge-pages exist at the last level */ 1464 if (level == KVM_PGTABLE_LAST_LEVEL) 1465 return 0; 1466 1467 /* We only split valid block mappings */ 1468 if (!kvm_pte_valid(pte)) 1469 return 0; 1470 1471 nr_pages = stage2_block_get_nr_page_tables(level); 1472 if (nr_pages < 0) 1473 return nr_pages; 1474 1475 if (mc->nobjs >= nr_pages) { 1476 /* Build a tree mapped down to the PTE granularity. */ 1477 force_pte = true; 1478 } else { 1479 /* 1480 * Don't force PTEs, so create_unlinked() below does 1481 * not populate the tree up to the PTE level. The 1482 * consequence is that the call will require a single 1483 * page of level 2 entries at level 1, or a single 1484 * page of PTEs at level 2. If we are at level 1, the 1485 * PTEs will be created recursively. 1486 */ 1487 force_pte = false; 1488 nr_pages = 1; 1489 } 1490 1491 if (mc->nobjs < nr_pages) 1492 return -ENOMEM; 1493 1494 mmu = container_of(mc, struct kvm_s2_mmu, split_page_cache); 1495 phys = kvm_pte_to_phys(pte); 1496 prot = kvm_pgtable_stage2_pte_prot(pte); 1497 1498 childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys, 1499 level, prot, mc, force_pte); 1500 if (IS_ERR(childp)) 1501 return PTR_ERR(childp); 1502 1503 if (!stage2_try_break_pte(ctx, mmu)) { 1504 kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level); 1505 mm_ops->put_page(childp); 1506 return -EAGAIN; 1507 } 1508 1509 /* 1510 * Note, the contents of the page table are guaranteed to be made 1511 * visible before the new PTE is assigned because stage2_make_pte() 1512 * writes the PTE using smp_store_release(). 1513 */ 1514 new = kvm_init_table_pte(childp, mm_ops); 1515 stage2_make_pte(ctx, new); 1516 dsb(ishst); 1517 return 0; 1518 } 1519 1520 int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, 1521 struct kvm_mmu_memory_cache *mc) 1522 { 1523 struct kvm_pgtable_walker walker = { 1524 .cb = stage2_split_walker, 1525 .flags = KVM_PGTABLE_WALK_LEAF, 1526 .arg = mc, 1527 }; 1528 1529 return kvm_pgtable_walk(pgt, addr, size, &walker); 1530 } 1531 1532 int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, 1533 struct kvm_pgtable_mm_ops *mm_ops, 1534 enum kvm_pgtable_stage2_flags flags, 1535 kvm_pgtable_force_pte_cb_t force_pte_cb) 1536 { 1537 size_t pgd_sz; 1538 u64 vtcr = mmu->vtcr; 1539 u32 ia_bits = VTCR_EL2_IPA(vtcr); 1540 u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 1541 s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; 1542 1543 pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE; 1544 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz); 1545 if (!pgt->pgd) 1546 return -ENOMEM; 1547 1548 pgt->ia_bits = ia_bits; 1549 pgt->start_level = start_level; 1550 pgt->mm_ops = mm_ops; 1551 pgt->mmu = mmu; 1552 pgt->flags = flags; 1553 pgt->force_pte_cb = force_pte_cb; 1554 1555 /* Ensure zeroed PGD pages are visible to the hardware walker */ 1556 dsb(ishst); 1557 return 0; 1558 } 1559 1560 size_t kvm_pgtable_stage2_pgd_size(u64 vtcr) 1561 { 1562 u32 ia_bits = VTCR_EL2_IPA(vtcr); 1563 u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 1564 s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; 1565 1566 return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE; 1567 } 1568 1569 static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx, 1570 enum kvm_pgtable_walk_flags visit) 1571 { 1572 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; 1573 1574 if (!stage2_pte_is_counted(ctx->old)) 1575 return 0; 1576 1577 mm_ops->put_page(ctx->ptep); 1578 1579 if (kvm_pte_table(ctx->old, ctx->level)) 1580 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops)); 1581 1582 return 0; 1583 } 1584 1585 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) 1586 { 1587 size_t pgd_sz; 1588 struct kvm_pgtable_walker walker = { 1589 .cb = stage2_free_walker, 1590 .flags = KVM_PGTABLE_WALK_LEAF | 1591 KVM_PGTABLE_WALK_TABLE_POST, 1592 }; 1593 1594 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); 1595 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE; 1596 pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz); 1597 pgt->pgd = NULL; 1598 } 1599 1600 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level) 1601 { 1602 kvm_pteref_t ptep = (kvm_pteref_t)pgtable; 1603 struct kvm_pgtable_walker walker = { 1604 .cb = stage2_free_walker, 1605 .flags = KVM_PGTABLE_WALK_LEAF | 1606 KVM_PGTABLE_WALK_TABLE_POST, 1607 }; 1608 struct kvm_pgtable_walk_data data = { 1609 .walker = &walker, 1610 1611 /* 1612 * At this point the IPA really doesn't matter, as the page 1613 * table being traversed has already been removed from the stage 1614 * 2. Set an appropriate range to cover the entire page table. 1615 */ 1616 .addr = 0, 1617 .end = kvm_granule_size(level), 1618 }; 1619 1620 WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1)); 1621 1622 WARN_ON(mm_ops->page_count(pgtable) != 1); 1623 mm_ops->put_page(pgtable); 1624 } 1625