1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include "mmu.h" 5 #include "mmu_internal.h" 6 #include "mmutrace.h" 7 #include "tdp_iter.h" 8 #include "tdp_mmu.h" 9 #include "spte.h" 10 11 #include <asm/cmpxchg.h> 12 #include <trace/events/kvm.h> 13 14 /* Initializes the TDP MMU for the VM, if enabled. */ 15 void kvm_mmu_init_tdp_mmu(struct kvm *kvm) 16 { 17 INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); 18 spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); 19 } 20 21 /* Arbitrarily returns true so that this may be used in if statements. */ 22 static __always_inline bool kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm, 23 bool shared) 24 { 25 if (shared) 26 lockdep_assert_held_read(&kvm->mmu_lock); 27 else 28 lockdep_assert_held_write(&kvm->mmu_lock); 29 30 return true; 31 } 32 33 void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) 34 { 35 /* 36 * Invalidate all roots, which besides the obvious, schedules all roots 37 * for zapping and thus puts the TDP MMU's reference to each root, i.e. 38 * ultimately frees all roots. 39 */ 40 kvm_tdp_mmu_invalidate_all_roots(kvm); 41 kvm_tdp_mmu_zap_invalidated_roots(kvm); 42 43 WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages)); 44 WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots)); 45 46 /* 47 * Ensure that all the outstanding RCU callbacks to free shadow pages 48 * can run before the VM is torn down. Putting the last reference to 49 * zapped roots will create new callbacks. 50 */ 51 rcu_barrier(); 52 } 53 54 static void tdp_mmu_free_sp(struct kvm_mmu_page *sp) 55 { 56 free_page((unsigned long)sp->spt); 57 kmem_cache_free(mmu_page_header_cache, sp); 58 } 59 60 /* 61 * This is called through call_rcu in order to free TDP page table memory 62 * safely with respect to other kernel threads that may be operating on 63 * the memory. 64 * By only accessing TDP MMU page table memory in an RCU read critical 65 * section, and freeing it after a grace period, lockless access to that 66 * memory won't use it after it is freed. 67 */ 68 static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) 69 { 70 struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page, 71 rcu_head); 72 73 tdp_mmu_free_sp(sp); 74 } 75 76 void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) 77 { 78 if (!refcount_dec_and_test(&root->tdp_mmu_root_count)) 79 return; 80 81 /* 82 * The TDP MMU itself holds a reference to each root until the root is 83 * explicitly invalidated, i.e. the final reference should be never be 84 * put for a valid root. 85 */ 86 KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm); 87 88 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 89 list_del_rcu(&root->link); 90 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 91 call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback); 92 } 93 94 /* 95 * Returns the next root after @prev_root (or the first root if @prev_root is 96 * NULL). A reference to the returned root is acquired, and the reference to 97 * @prev_root is released (the caller obviously must hold a reference to 98 * @prev_root if it's non-NULL). 99 * 100 * If @only_valid is true, invalid roots are skipped. 101 * 102 * Returns NULL if the end of tdp_mmu_roots was reached. 103 */ 104 static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, 105 struct kvm_mmu_page *prev_root, 106 bool only_valid) 107 { 108 struct kvm_mmu_page *next_root; 109 110 /* 111 * While the roots themselves are RCU-protected, fields such as 112 * role.invalid are protected by mmu_lock. 113 */ 114 lockdep_assert_held(&kvm->mmu_lock); 115 116 rcu_read_lock(); 117 118 if (prev_root) 119 next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 120 &prev_root->link, 121 typeof(*prev_root), link); 122 else 123 next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots, 124 typeof(*next_root), link); 125 126 while (next_root) { 127 if ((!only_valid || !next_root->role.invalid) && 128 kvm_tdp_mmu_get_root(next_root)) 129 break; 130 131 next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots, 132 &next_root->link, typeof(*next_root), link); 133 } 134 135 rcu_read_unlock(); 136 137 if (prev_root) 138 kvm_tdp_mmu_put_root(kvm, prev_root); 139 140 return next_root; 141 } 142 143 /* 144 * Note: this iterator gets and puts references to the roots it iterates over. 145 * This makes it safe to release the MMU lock and yield within the loop, but 146 * if exiting the loop early, the caller must drop the reference to the most 147 * recent root. (Unless keeping a live reference is desirable.) 148 * 149 * If shared is set, this function is operating under the MMU lock in read 150 * mode. 151 */ 152 #define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _only_valid) \ 153 for (_root = tdp_mmu_next_root(_kvm, NULL, _only_valid); \ 154 ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root; \ 155 _root = tdp_mmu_next_root(_kvm, _root, _only_valid)) \ 156 if (_as_id >= 0 && kvm_mmu_page_as_id(_root) != _as_id) { \ 157 } else 158 159 #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \ 160 __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, true) 161 162 #define for_each_tdp_mmu_root_yield_safe(_kvm, _root) \ 163 for (_root = tdp_mmu_next_root(_kvm, NULL, false); \ 164 ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root; \ 165 _root = tdp_mmu_next_root(_kvm, _root, false)) 166 167 /* 168 * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write, 169 * the implication being that any flow that holds mmu_lock for read is 170 * inherently yield-friendly and should use the yield-safe variant above. 171 * Holding mmu_lock for write obviates the need for RCU protection as the list 172 * is guaranteed to be stable. 173 */ 174 #define __for_each_tdp_mmu_root(_kvm, _root, _as_id, _only_valid) \ 175 list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) \ 176 if (kvm_lockdep_assert_mmu_lock_held(_kvm, false) && \ 177 ((_as_id >= 0 && kvm_mmu_page_as_id(_root) != _as_id) || \ 178 ((_only_valid) && (_root)->role.invalid))) { \ 179 } else 180 181 #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \ 182 __for_each_tdp_mmu_root(_kvm, _root, _as_id, false) 183 184 #define for_each_valid_tdp_mmu_root(_kvm, _root, _as_id) \ 185 __for_each_tdp_mmu_root(_kvm, _root, _as_id, true) 186 187 static struct kvm_mmu_page *tdp_mmu_alloc_sp(struct kvm_vcpu *vcpu) 188 { 189 struct kvm_mmu_page *sp; 190 191 sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache); 192 sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache); 193 194 return sp; 195 } 196 197 static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, tdp_ptep_t sptep, 198 gfn_t gfn, union kvm_mmu_page_role role) 199 { 200 INIT_LIST_HEAD(&sp->possible_nx_huge_page_link); 201 202 set_page_private(virt_to_page(sp->spt), (unsigned long)sp); 203 204 sp->role = role; 205 sp->gfn = gfn; 206 sp->ptep = sptep; 207 sp->tdp_mmu_page = true; 208 209 trace_kvm_mmu_get_page(sp, true); 210 } 211 212 static void tdp_mmu_init_child_sp(struct kvm_mmu_page *child_sp, 213 struct tdp_iter *iter) 214 { 215 struct kvm_mmu_page *parent_sp; 216 union kvm_mmu_page_role role; 217 218 parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); 219 220 role = parent_sp->role; 221 role.level--; 222 223 tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role); 224 } 225 226 int kvm_tdp_mmu_alloc_root(struct kvm_vcpu *vcpu) 227 { 228 struct kvm_mmu *mmu = vcpu->arch.mmu; 229 union kvm_mmu_page_role role = mmu->root_role; 230 int as_id = kvm_mmu_role_as_id(role); 231 struct kvm *kvm = vcpu->kvm; 232 struct kvm_mmu_page *root; 233 234 /* 235 * Check for an existing root before acquiring the pages lock to avoid 236 * unnecessary serialization if multiple vCPUs are loading a new root. 237 * E.g. when bringing up secondary vCPUs, KVM will already have created 238 * a valid root on behalf of the primary vCPU. 239 */ 240 read_lock(&kvm->mmu_lock); 241 242 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, as_id) { 243 if (root->role.word == role.word) 244 goto out_read_unlock; 245 } 246 247 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 248 249 /* 250 * Recheck for an existing root after acquiring the pages lock, another 251 * vCPU may have raced ahead and created a new usable root. Manually 252 * walk the list of roots as the standard macros assume that the pages 253 * lock is *not* held. WARN if grabbing a reference to a usable root 254 * fails, as the last reference to a root can only be put *after* the 255 * root has been invalidated, which requires holding mmu_lock for write. 256 */ 257 list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) { 258 if (root->role.word == role.word && 259 !WARN_ON_ONCE(!kvm_tdp_mmu_get_root(root))) 260 goto out_spin_unlock; 261 } 262 263 root = tdp_mmu_alloc_sp(vcpu); 264 tdp_mmu_init_sp(root, NULL, 0, role); 265 266 /* 267 * TDP MMU roots are kept until they are explicitly invalidated, either 268 * by a memslot update or by the destruction of the VM. Initialize the 269 * refcount to two; one reference for the vCPU, and one reference for 270 * the TDP MMU itself, which is held until the root is invalidated and 271 * is ultimately put by kvm_tdp_mmu_zap_invalidated_roots(). 272 */ 273 refcount_set(&root->tdp_mmu_root_count, 2); 274 list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots); 275 276 out_spin_unlock: 277 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 278 out_read_unlock: 279 read_unlock(&kvm->mmu_lock); 280 /* 281 * Note, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS will prevent entering the guest 282 * and actually consuming the root if it's invalidated after dropping 283 * mmu_lock, and the root can't be freed as this vCPU holds a reference. 284 */ 285 mmu->root.hpa = __pa(root->spt); 286 mmu->root.pgd = 0; 287 return 0; 288 } 289 290 static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 291 u64 old_spte, u64 new_spte, int level, 292 bool shared); 293 294 static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) 295 { 296 kvm_account_pgtable_pages((void *)sp->spt, +1); 297 atomic64_inc(&kvm->arch.tdp_mmu_pages); 298 } 299 300 static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) 301 { 302 kvm_account_pgtable_pages((void *)sp->spt, -1); 303 atomic64_dec(&kvm->arch.tdp_mmu_pages); 304 } 305 306 /** 307 * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages 308 * 309 * @kvm: kvm instance 310 * @sp: the page to be removed 311 */ 312 static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 313 { 314 tdp_unaccount_mmu_page(kvm, sp); 315 316 if (!sp->nx_huge_page_disallowed) 317 return; 318 319 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 320 sp->nx_huge_page_disallowed = false; 321 untrack_possible_nx_huge_page(kvm, sp); 322 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 323 } 324 325 /** 326 * handle_removed_pt() - handle a page table removed from the TDP structure 327 * 328 * @kvm: kvm instance 329 * @pt: the page removed from the paging structure 330 * @shared: This operation may not be running under the exclusive use 331 * of the MMU lock and the operation must synchronize with other 332 * threads that might be modifying SPTEs. 333 * 334 * Given a page table that has been removed from the TDP paging structure, 335 * iterates through the page table to clear SPTEs and free child page tables. 336 * 337 * Note that pt is passed in as a tdp_ptep_t, but it does not need RCU 338 * protection. Since this thread removed it from the paging structure, 339 * this thread will be responsible for ensuring the page is freed. Hence the 340 * early rcu_dereferences in the function. 341 */ 342 static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared) 343 { 344 struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(pt)); 345 int level = sp->role.level; 346 gfn_t base_gfn = sp->gfn; 347 int i; 348 349 trace_kvm_mmu_prepare_zap_page(sp); 350 351 tdp_mmu_unlink_sp(kvm, sp); 352 353 for (i = 0; i < SPTE_ENT_PER_PAGE; i++) { 354 tdp_ptep_t sptep = pt + i; 355 gfn_t gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level); 356 u64 old_spte; 357 358 if (shared) { 359 /* 360 * Set the SPTE to a nonpresent value that other 361 * threads will not overwrite. If the SPTE was 362 * already marked as frozen then another thread 363 * handling a page fault could overwrite it, so 364 * set the SPTE until it is set from some other 365 * value to the frozen SPTE value. 366 */ 367 for (;;) { 368 old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, FROZEN_SPTE); 369 if (!is_frozen_spte(old_spte)) 370 break; 371 cpu_relax(); 372 } 373 } else { 374 /* 375 * If the SPTE is not MMU-present, there is no backing 376 * page associated with the SPTE and so no side effects 377 * that need to be recorded, and exclusive ownership of 378 * mmu_lock ensures the SPTE can't be made present. 379 * Note, zapping MMIO SPTEs is also unnecessary as they 380 * are guarded by the memslots generation, not by being 381 * unreachable. 382 */ 383 old_spte = kvm_tdp_mmu_read_spte(sptep); 384 if (!is_shadow_present_pte(old_spte)) 385 continue; 386 387 /* 388 * Use the common helper instead of a raw WRITE_ONCE as 389 * the SPTE needs to be updated atomically if it can be 390 * modified by a different vCPU outside of mmu_lock. 391 * Even though the parent SPTE is !PRESENT, the TLB 392 * hasn't yet been flushed, and both Intel and AMD 393 * document that A/D assists can use upper-level PxE 394 * entries that are cached in the TLB, i.e. the CPU can 395 * still access the page and mark it dirty. 396 * 397 * No retry is needed in the atomic update path as the 398 * sole concern is dropping a Dirty bit, i.e. no other 399 * task can zap/remove the SPTE as mmu_lock is held for 400 * write. Marking the SPTE as a frozen SPTE is not 401 * strictly necessary for the same reason, but using 402 * the frozen SPTE value keeps the shared/exclusive 403 * paths consistent and allows the handle_changed_spte() 404 * call below to hardcode the new value to FROZEN_SPTE. 405 * 406 * Note, even though dropping a Dirty bit is the only 407 * scenario where a non-atomic update could result in a 408 * functional bug, simply checking the Dirty bit isn't 409 * sufficient as a fast page fault could read the upper 410 * level SPTE before it is zapped, and then make this 411 * target SPTE writable, resume the guest, and set the 412 * Dirty bit between reading the SPTE above and writing 413 * it here. 414 */ 415 old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, 416 FROZEN_SPTE, level); 417 } 418 handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, 419 old_spte, FROZEN_SPTE, level, shared); 420 } 421 422 call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); 423 } 424 425 /** 426 * handle_changed_spte - handle bookkeeping associated with an SPTE change 427 * @kvm: kvm instance 428 * @as_id: the address space of the paging structure the SPTE was a part of 429 * @gfn: the base GFN that was mapped by the SPTE 430 * @old_spte: The value of the SPTE before the change 431 * @new_spte: The value of the SPTE after the change 432 * @level: the level of the PT the SPTE is part of in the paging structure 433 * @shared: This operation may not be running under the exclusive use of 434 * the MMU lock and the operation must synchronize with other 435 * threads that might be modifying SPTEs. 436 * 437 * Handle bookkeeping that might result from the modification of a SPTE. Note, 438 * dirty logging updates are handled in common code, not here (see make_spte() 439 * and fast_pf_fix_direct_spte()). 440 */ 441 static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 442 u64 old_spte, u64 new_spte, int level, 443 bool shared) 444 { 445 bool was_present = is_shadow_present_pte(old_spte); 446 bool is_present = is_shadow_present_pte(new_spte); 447 bool was_leaf = was_present && is_last_spte(old_spte, level); 448 bool is_leaf = is_present && is_last_spte(new_spte, level); 449 bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 450 451 WARN_ON_ONCE(level > PT64_ROOT_MAX_LEVEL); 452 WARN_ON_ONCE(level < PG_LEVEL_4K); 453 WARN_ON_ONCE(gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); 454 455 /* 456 * If this warning were to trigger it would indicate that there was a 457 * missing MMU notifier or a race with some notifier handler. 458 * A present, leaf SPTE should never be directly replaced with another 459 * present leaf SPTE pointing to a different PFN. A notifier handler 460 * should be zapping the SPTE before the main MM's page table is 461 * changed, or the SPTE should be zeroed, and the TLBs flushed by the 462 * thread before replacement. 463 */ 464 if (was_leaf && is_leaf && pfn_changed) { 465 pr_err("Invalid SPTE change: cannot replace a present leaf\n" 466 "SPTE with another present leaf SPTE mapping a\n" 467 "different PFN!\n" 468 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 469 as_id, gfn, old_spte, new_spte, level); 470 471 /* 472 * Crash the host to prevent error propagation and guest data 473 * corruption. 474 */ 475 BUG(); 476 } 477 478 if (old_spte == new_spte) 479 return; 480 481 trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); 482 483 if (is_leaf) 484 check_spte_writable_invariants(new_spte); 485 486 /* 487 * The only times a SPTE should be changed from a non-present to 488 * non-present state is when an MMIO entry is installed/modified/ 489 * removed. In that case, there is nothing to do here. 490 */ 491 if (!was_present && !is_present) { 492 /* 493 * If this change does not involve a MMIO SPTE or frozen SPTE, 494 * it is unexpected. Log the change, though it should not 495 * impact the guest since both the former and current SPTEs 496 * are nonpresent. 497 */ 498 if (WARN_ON_ONCE(!is_mmio_spte(kvm, old_spte) && 499 !is_mmio_spte(kvm, new_spte) && 500 !is_frozen_spte(new_spte))) 501 pr_err("Unexpected SPTE change! Nonpresent SPTEs\n" 502 "should not be replaced with another,\n" 503 "different nonpresent SPTE, unless one or both\n" 504 "are MMIO SPTEs, or the new SPTE is\n" 505 "a temporary frozen SPTE.\n" 506 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 507 as_id, gfn, old_spte, new_spte, level); 508 return; 509 } 510 511 if (is_leaf != was_leaf) 512 kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1); 513 514 /* 515 * Recursively handle child PTs if the change removed a subtree from 516 * the paging structure. Note the WARN on the PFN changing without the 517 * SPTE being converted to a hugepage (leaf) or being zapped. Shadow 518 * pages are kernel allocations and should never be migrated. 519 */ 520 if (was_present && !was_leaf && 521 (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed))) 522 handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared); 523 } 524 525 static inline int __must_check __tdp_mmu_set_spte_atomic(struct tdp_iter *iter, 526 u64 new_spte) 527 { 528 u64 *sptep = rcu_dereference(iter->sptep); 529 530 /* 531 * The caller is responsible for ensuring the old SPTE is not a FROZEN 532 * SPTE. KVM should never attempt to zap or manipulate a FROZEN SPTE, 533 * and pre-checking before inserting a new SPTE is advantageous as it 534 * avoids unnecessary work. 535 */ 536 WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte)); 537 538 /* 539 * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and 540 * does not hold the mmu_lock. On failure, i.e. if a different logical 541 * CPU modified the SPTE, try_cmpxchg64() updates iter->old_spte with 542 * the current value, so the caller operates on fresh data, e.g. if it 543 * retries tdp_mmu_set_spte_atomic() 544 */ 545 if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte)) 546 return -EBUSY; 547 548 return 0; 549 } 550 551 /* 552 * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically 553 * and handle the associated bookkeeping. Do not mark the page dirty 554 * in KVM's dirty bitmaps. 555 * 556 * If setting the SPTE fails because it has changed, iter->old_spte will be 557 * refreshed to the current value of the spte. 558 * 559 * @kvm: kvm instance 560 * @iter: a tdp_iter instance currently on the SPTE that should be set 561 * @new_spte: The value the SPTE should be set to 562 * Return: 563 * * 0 - If the SPTE was set. 564 * * -EBUSY - If the SPTE cannot be set. In this case this function will have 565 * no side-effects other than setting iter->old_spte to the last 566 * known value of the spte. 567 */ 568 static inline int __must_check tdp_mmu_set_spte_atomic(struct kvm *kvm, 569 struct tdp_iter *iter, 570 u64 new_spte) 571 { 572 int ret; 573 574 lockdep_assert_held_read(&kvm->mmu_lock); 575 576 ret = __tdp_mmu_set_spte_atomic(iter, new_spte); 577 if (ret) 578 return ret; 579 580 handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte, 581 new_spte, iter->level, true); 582 583 return 0; 584 } 585 586 /* 587 * tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping 588 * @kvm: KVM instance 589 * @as_id: Address space ID, i.e. regular vs. SMM 590 * @sptep: Pointer to the SPTE 591 * @old_spte: The current value of the SPTE 592 * @new_spte: The new value that will be set for the SPTE 593 * @gfn: The base GFN that was (or will be) mapped by the SPTE 594 * @level: The level _containing_ the SPTE (its parent PT's level) 595 * 596 * Returns the old SPTE value, which _may_ be different than @old_spte if the 597 * SPTE had voldatile bits. 598 */ 599 static u64 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, 600 u64 old_spte, u64 new_spte, gfn_t gfn, int level) 601 { 602 lockdep_assert_held_write(&kvm->mmu_lock); 603 604 /* 605 * No thread should be using this function to set SPTEs to or from the 606 * temporary frozen SPTE value. 607 * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic 608 * should be used. If operating under the MMU lock in write mode, the 609 * use of the frozen SPTE should not be necessary. 610 */ 611 WARN_ON_ONCE(is_frozen_spte(old_spte) || is_frozen_spte(new_spte)); 612 613 old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level); 614 615 handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false); 616 return old_spte; 617 } 618 619 static inline void tdp_mmu_iter_set_spte(struct kvm *kvm, struct tdp_iter *iter, 620 u64 new_spte) 621 { 622 WARN_ON_ONCE(iter->yielded); 623 iter->old_spte = tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep, 624 iter->old_spte, new_spte, 625 iter->gfn, iter->level); 626 } 627 628 #define tdp_root_for_each_pte(_iter, _root, _start, _end) \ 629 for_each_tdp_pte(_iter, _root, _start, _end) 630 631 #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \ 632 tdp_root_for_each_pte(_iter, _root, _start, _end) \ 633 if (!is_shadow_present_pte(_iter.old_spte) || \ 634 !is_last_spte(_iter.old_spte, _iter.level)) \ 635 continue; \ 636 else 637 638 #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \ 639 for_each_tdp_pte(_iter, root_to_sp(_mmu->root.hpa), _start, _end) 640 641 static inline bool __must_check tdp_mmu_iter_need_resched(struct kvm *kvm, 642 struct tdp_iter *iter) 643 { 644 if (!need_resched() && !rwlock_needbreak(&kvm->mmu_lock)) 645 return false; 646 647 /* Ensure forward progress has been made before yielding. */ 648 return iter->next_last_level_gfn != iter->yielded_gfn; 649 } 650 651 /* 652 * Yield if the MMU lock is contended or this thread needs to return control 653 * to the scheduler. 654 * 655 * If this function should yield and flush is set, it will perform a remote 656 * TLB flush before yielding. 657 * 658 * If this function yields, iter->yielded is set and the caller must skip to 659 * the next iteration, where tdp_iter_next() will reset the tdp_iter's walk 660 * over the paging structures to allow the iterator to continue its traversal 661 * from the paging structure root. 662 * 663 * Returns true if this function yielded. 664 */ 665 static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm, 666 struct tdp_iter *iter, 667 bool flush, bool shared) 668 { 669 KVM_MMU_WARN_ON(iter->yielded); 670 671 if (!tdp_mmu_iter_need_resched(kvm, iter)) 672 return false; 673 674 if (flush) 675 kvm_flush_remote_tlbs(kvm); 676 677 rcu_read_unlock(); 678 679 if (shared) 680 cond_resched_rwlock_read(&kvm->mmu_lock); 681 else 682 cond_resched_rwlock_write(&kvm->mmu_lock); 683 684 rcu_read_lock(); 685 686 WARN_ON_ONCE(iter->gfn > iter->next_last_level_gfn); 687 688 iter->yielded = true; 689 return true; 690 } 691 692 static inline gfn_t tdp_mmu_max_gfn_exclusive(void) 693 { 694 /* 695 * Bound TDP MMU walks at host.MAXPHYADDR. KVM disallows memslots with 696 * a gpa range that would exceed the max gfn, and KVM does not create 697 * MMIO SPTEs for "impossible" gfns, instead sending such accesses down 698 * the slow emulation path every time. 699 */ 700 return kvm_mmu_max_gfn() + 1; 701 } 702 703 static void __tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root, 704 bool shared, int zap_level) 705 { 706 struct tdp_iter iter; 707 708 gfn_t end = tdp_mmu_max_gfn_exclusive(); 709 gfn_t start = 0; 710 711 for_each_tdp_pte_min_level(iter, root, zap_level, start, end) { 712 retry: 713 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) 714 continue; 715 716 if (!is_shadow_present_pte(iter.old_spte)) 717 continue; 718 719 if (iter.level > zap_level) 720 continue; 721 722 if (!shared) 723 tdp_mmu_iter_set_spte(kvm, &iter, SHADOW_NONPRESENT_VALUE); 724 else if (tdp_mmu_set_spte_atomic(kvm, &iter, SHADOW_NONPRESENT_VALUE)) 725 goto retry; 726 } 727 } 728 729 static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root, 730 bool shared) 731 { 732 733 /* 734 * The root must have an elevated refcount so that it's reachable via 735 * mmu_notifier callbacks, which allows this path to yield and drop 736 * mmu_lock. When handling an unmap/release mmu_notifier command, KVM 737 * must drop all references to relevant pages prior to completing the 738 * callback. Dropping mmu_lock with an unreachable root would result 739 * in zapping SPTEs after a relevant mmu_notifier callback completes 740 * and lead to use-after-free as zapping a SPTE triggers "writeback" of 741 * dirty accessed bits to the SPTE's associated struct page. 742 */ 743 WARN_ON_ONCE(!refcount_read(&root->tdp_mmu_root_count)); 744 745 kvm_lockdep_assert_mmu_lock_held(kvm, shared); 746 747 rcu_read_lock(); 748 749 /* 750 * Zap roots in multiple passes of decreasing granularity, i.e. zap at 751 * 4KiB=>2MiB=>1GiB=>root, in order to better honor need_resched() (all 752 * preempt models) or mmu_lock contention (full or real-time models). 753 * Zapping at finer granularity marginally increases the total time of 754 * the zap, but in most cases the zap itself isn't latency sensitive. 755 * 756 * If KVM is configured to prove the MMU, skip the 4KiB and 2MiB zaps 757 * in order to mimic the page fault path, which can replace a 1GiB page 758 * table with an equivalent 1GiB hugepage, i.e. can get saddled with 759 * zapping a 1GiB region that's fully populated with 4KiB SPTEs. This 760 * allows verifying that KVM can safely zap 1GiB regions, e.g. without 761 * inducing RCU stalls, without relying on a relatively rare event 762 * (zapping roots is orders of magnitude more common). Note, because 763 * zapping a SP recurses on its children, stepping down to PG_LEVEL_4K 764 * in the iterator itself is unnecessary. 765 */ 766 if (!IS_ENABLED(CONFIG_KVM_PROVE_MMU)) { 767 __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_4K); 768 __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_2M); 769 } 770 __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_1G); 771 __tdp_mmu_zap_root(kvm, root, shared, root->role.level); 772 773 rcu_read_unlock(); 774 } 775 776 bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 777 { 778 u64 old_spte; 779 780 /* 781 * This helper intentionally doesn't allow zapping a root shadow page, 782 * which doesn't have a parent page table and thus no associated entry. 783 */ 784 if (WARN_ON_ONCE(!sp->ptep)) 785 return false; 786 787 old_spte = kvm_tdp_mmu_read_spte(sp->ptep); 788 if (WARN_ON_ONCE(!is_shadow_present_pte(old_spte))) 789 return false; 790 791 tdp_mmu_set_spte(kvm, kvm_mmu_page_as_id(sp), sp->ptep, old_spte, 792 SHADOW_NONPRESENT_VALUE, sp->gfn, sp->role.level + 1); 793 794 return true; 795 } 796 797 /* 798 * If can_yield is true, will release the MMU lock and reschedule if the 799 * scheduler needs the CPU or there is contention on the MMU lock. If this 800 * function cannot yield, it will not release the MMU lock or reschedule and 801 * the caller must ensure it does not supply too large a GFN range, or the 802 * operation can cause a soft lockup. 803 */ 804 static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root, 805 gfn_t start, gfn_t end, bool can_yield, bool flush) 806 { 807 struct tdp_iter iter; 808 809 end = min(end, tdp_mmu_max_gfn_exclusive()); 810 811 lockdep_assert_held_write(&kvm->mmu_lock); 812 813 rcu_read_lock(); 814 815 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) { 816 if (can_yield && 817 tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) { 818 flush = false; 819 continue; 820 } 821 822 if (!is_shadow_present_pte(iter.old_spte) || 823 !is_last_spte(iter.old_spte, iter.level)) 824 continue; 825 826 tdp_mmu_iter_set_spte(kvm, &iter, SHADOW_NONPRESENT_VALUE); 827 828 /* 829 * Zappings SPTEs in invalid roots doesn't require a TLB flush, 830 * see kvm_tdp_mmu_zap_invalidated_roots() for details. 831 */ 832 if (!root->role.invalid) 833 flush = true; 834 } 835 836 rcu_read_unlock(); 837 838 /* 839 * Because this flow zaps _only_ leaf SPTEs, the caller doesn't need 840 * to provide RCU protection as no 'struct kvm_mmu_page' will be freed. 841 */ 842 return flush; 843 } 844 845 /* 846 * Zap leaf SPTEs for the range of gfns, [start, end), for all *VALID** roots. 847 * Returns true if a TLB flush is needed before releasing the MMU lock, i.e. if 848 * one or more SPTEs were zapped since the MMU lock was last acquired. 849 */ 850 bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush) 851 { 852 struct kvm_mmu_page *root; 853 854 lockdep_assert_held_write(&kvm->mmu_lock); 855 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, -1) 856 flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush); 857 858 return flush; 859 } 860 861 void kvm_tdp_mmu_zap_all(struct kvm *kvm) 862 { 863 struct kvm_mmu_page *root; 864 865 /* 866 * Zap all roots, including invalid roots, as all SPTEs must be dropped 867 * before returning to the caller. Zap directly even if the root is 868 * also being zapped by a worker. Walking zapped top-level SPTEs isn't 869 * all that expensive and mmu_lock is already held, which means the 870 * worker has yielded, i.e. flushing the work instead of zapping here 871 * isn't guaranteed to be any faster. 872 * 873 * A TLB flush is unnecessary, KVM zaps everything if and only the VM 874 * is being destroyed or the userspace VMM has exited. In both cases, 875 * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request. 876 */ 877 lockdep_assert_held_write(&kvm->mmu_lock); 878 for_each_tdp_mmu_root_yield_safe(kvm, root) 879 tdp_mmu_zap_root(kvm, root, false); 880 } 881 882 /* 883 * Zap all invalidated roots to ensure all SPTEs are dropped before the "fast 884 * zap" completes. 885 */ 886 void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm) 887 { 888 struct kvm_mmu_page *root; 889 890 read_lock(&kvm->mmu_lock); 891 892 for_each_tdp_mmu_root_yield_safe(kvm, root) { 893 if (!root->tdp_mmu_scheduled_root_to_zap) 894 continue; 895 896 root->tdp_mmu_scheduled_root_to_zap = false; 897 KVM_BUG_ON(!root->role.invalid, kvm); 898 899 /* 900 * A TLB flush is not necessary as KVM performs a local TLB 901 * flush when allocating a new root (see kvm_mmu_load()), and 902 * when migrating a vCPU to a different pCPU. Note, the local 903 * TLB flush on reuse also invalidates paging-structure-cache 904 * entries, i.e. TLB entries for intermediate paging structures, 905 * that may be zapped, as such entries are associated with the 906 * ASID on both VMX and SVM. 907 */ 908 tdp_mmu_zap_root(kvm, root, true); 909 910 /* 911 * The referenced needs to be put *after* zapping the root, as 912 * the root must be reachable by mmu_notifiers while it's being 913 * zapped 914 */ 915 kvm_tdp_mmu_put_root(kvm, root); 916 } 917 918 read_unlock(&kvm->mmu_lock); 919 } 920 921 /* 922 * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that 923 * is about to be zapped, e.g. in response to a memslots update. The actual 924 * zapping is done separately so that it happens with mmu_lock with read, 925 * whereas invalidating roots must be done with mmu_lock held for write (unless 926 * the VM is being destroyed). 927 * 928 * Note, kvm_tdp_mmu_zap_invalidated_roots() is gifted the TDP MMU's reference. 929 * See kvm_tdp_mmu_alloc_root(). 930 */ 931 void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) 932 { 933 struct kvm_mmu_page *root; 934 935 /* 936 * mmu_lock must be held for write to ensure that a root doesn't become 937 * invalid while there are active readers (invalidating a root while 938 * there are active readers may or may not be problematic in practice, 939 * but it's uncharted territory and not supported). 940 * 941 * Waive the assertion if there are no users of @kvm, i.e. the VM is 942 * being destroyed after all references have been put, or if no vCPUs 943 * have been created (which means there are no roots), i.e. the VM is 944 * being destroyed in an error path of KVM_CREATE_VM. 945 */ 946 if (IS_ENABLED(CONFIG_PROVE_LOCKING) && 947 refcount_read(&kvm->users_count) && kvm->created_vcpus) 948 lockdep_assert_held_write(&kvm->mmu_lock); 949 950 /* 951 * As above, mmu_lock isn't held when destroying the VM! There can't 952 * be other references to @kvm, i.e. nothing else can invalidate roots 953 * or get/put references to roots. 954 */ 955 list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) { 956 /* 957 * Note, invalid roots can outlive a memslot update! Invalid 958 * roots must be *zapped* before the memslot update completes, 959 * but a different task can acquire a reference and keep the 960 * root alive after its been zapped. 961 */ 962 if (!root->role.invalid) { 963 root->tdp_mmu_scheduled_root_to_zap = true; 964 root->role.invalid = true; 965 } 966 } 967 } 968 969 /* 970 * Installs a last-level SPTE to handle a TDP page fault. 971 * (NPT/EPT violation/misconfiguration) 972 */ 973 static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, 974 struct kvm_page_fault *fault, 975 struct tdp_iter *iter) 976 { 977 struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); 978 u64 new_spte; 979 int ret = RET_PF_FIXED; 980 bool wrprot = false; 981 982 if (WARN_ON_ONCE(sp->role.level != fault->goal_level)) 983 return RET_PF_RETRY; 984 985 if (fault->prefetch && is_shadow_present_pte(iter->old_spte)) 986 return RET_PF_SPURIOUS; 987 988 if (is_shadow_present_pte(iter->old_spte) && 989 is_access_allowed(fault, iter->old_spte) && 990 is_last_spte(iter->old_spte, iter->level)) 991 return RET_PF_SPURIOUS; 992 993 if (unlikely(!fault->slot)) 994 new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); 995 else 996 wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn, 997 fault->pfn, iter->old_spte, fault->prefetch, 998 false, fault->map_writable, &new_spte); 999 1000 if (new_spte == iter->old_spte) 1001 ret = RET_PF_SPURIOUS; 1002 else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) 1003 return RET_PF_RETRY; 1004 else if (is_shadow_present_pte(iter->old_spte) && 1005 (!is_last_spte(iter->old_spte, iter->level) || 1006 WARN_ON_ONCE(leaf_spte_change_needs_tlb_flush(iter->old_spte, new_spte)))) 1007 kvm_flush_remote_tlbs_gfn(vcpu->kvm, iter->gfn, iter->level); 1008 1009 /* 1010 * If the page fault was caused by a write but the page is write 1011 * protected, emulation is needed. If the emulation was skipped, 1012 * the vCPU would have the same fault again. 1013 */ 1014 if (wrprot && fault->write) 1015 ret = RET_PF_WRITE_PROTECTED; 1016 1017 /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ 1018 if (unlikely(is_mmio_spte(vcpu->kvm, new_spte))) { 1019 vcpu->stat.pf_mmio_spte_created++; 1020 trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, 1021 new_spte); 1022 ret = RET_PF_EMULATE; 1023 } else { 1024 trace_kvm_mmu_set_spte(iter->level, iter->gfn, 1025 rcu_dereference(iter->sptep)); 1026 } 1027 1028 return ret; 1029 } 1030 1031 /* 1032 * tdp_mmu_link_sp - Replace the given spte with an spte pointing to the 1033 * provided page table. 1034 * 1035 * @kvm: kvm instance 1036 * @iter: a tdp_iter instance currently on the SPTE that should be set 1037 * @sp: The new TDP page table to install. 1038 * @shared: This operation is running under the MMU lock in read mode. 1039 * 1040 * Returns: 0 if the new page table was installed. Non-0 if the page table 1041 * could not be installed (e.g. the atomic compare-exchange failed). 1042 */ 1043 static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, 1044 struct kvm_mmu_page *sp, bool shared) 1045 { 1046 u64 spte = make_nonleaf_spte(sp->spt, !kvm_ad_enabled); 1047 int ret = 0; 1048 1049 if (shared) { 1050 ret = tdp_mmu_set_spte_atomic(kvm, iter, spte); 1051 if (ret) 1052 return ret; 1053 } else { 1054 tdp_mmu_iter_set_spte(kvm, iter, spte); 1055 } 1056 1057 tdp_account_mmu_page(kvm, sp); 1058 1059 return 0; 1060 } 1061 1062 static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, 1063 struct kvm_mmu_page *sp, bool shared); 1064 1065 /* 1066 * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing 1067 * page tables and SPTEs to translate the faulting guest physical address. 1068 */ 1069 int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) 1070 { 1071 struct kvm_mmu *mmu = vcpu->arch.mmu; 1072 struct kvm *kvm = vcpu->kvm; 1073 struct tdp_iter iter; 1074 struct kvm_mmu_page *sp; 1075 int ret = RET_PF_RETRY; 1076 1077 kvm_mmu_hugepage_adjust(vcpu, fault); 1078 1079 trace_kvm_mmu_spte_requested(fault); 1080 1081 rcu_read_lock(); 1082 1083 tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) { 1084 int r; 1085 1086 if (fault->nx_huge_page_workaround_enabled) 1087 disallowed_hugepage_adjust(fault, iter.old_spte, iter.level); 1088 1089 /* 1090 * If SPTE has been frozen by another thread, just give up and 1091 * retry, avoiding unnecessary page table allocation and free. 1092 */ 1093 if (is_frozen_spte(iter.old_spte)) 1094 goto retry; 1095 1096 if (iter.level == fault->goal_level) 1097 goto map_target_level; 1098 1099 /* Step down into the lower level page table if it exists. */ 1100 if (is_shadow_present_pte(iter.old_spte) && 1101 !is_large_pte(iter.old_spte)) 1102 continue; 1103 1104 /* 1105 * The SPTE is either non-present or points to a huge page that 1106 * needs to be split. 1107 */ 1108 sp = tdp_mmu_alloc_sp(vcpu); 1109 tdp_mmu_init_child_sp(sp, &iter); 1110 1111 sp->nx_huge_page_disallowed = fault->huge_page_disallowed; 1112 1113 if (is_shadow_present_pte(iter.old_spte)) 1114 r = tdp_mmu_split_huge_page(kvm, &iter, sp, true); 1115 else 1116 r = tdp_mmu_link_sp(kvm, &iter, sp, true); 1117 1118 /* 1119 * Force the guest to retry if installing an upper level SPTE 1120 * failed, e.g. because a different task modified the SPTE. 1121 */ 1122 if (r) { 1123 tdp_mmu_free_sp(sp); 1124 goto retry; 1125 } 1126 1127 if (fault->huge_page_disallowed && 1128 fault->req_level >= iter.level) { 1129 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 1130 if (sp->nx_huge_page_disallowed) 1131 track_possible_nx_huge_page(kvm, sp); 1132 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); 1133 } 1134 } 1135 1136 /* 1137 * The walk aborted before reaching the target level, e.g. because the 1138 * iterator detected an upper level SPTE was frozen during traversal. 1139 */ 1140 WARN_ON_ONCE(iter.level == fault->goal_level); 1141 goto retry; 1142 1143 map_target_level: 1144 ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter); 1145 1146 retry: 1147 rcu_read_unlock(); 1148 return ret; 1149 } 1150 1151 bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, 1152 bool flush) 1153 { 1154 struct kvm_mmu_page *root; 1155 1156 __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false) 1157 flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end, 1158 range->may_block, flush); 1159 1160 return flush; 1161 } 1162 1163 /* 1164 * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero 1165 * if any of the GFNs in the range have been accessed. 1166 * 1167 * No need to mark the corresponding PFN as accessed as this call is coming 1168 * from the clear_young() or clear_flush_young() notifier, which uses the 1169 * return value to determine if the page has been accessed. 1170 */ 1171 static void kvm_tdp_mmu_age_spte(struct tdp_iter *iter) 1172 { 1173 u64 new_spte; 1174 1175 if (spte_ad_enabled(iter->old_spte)) { 1176 iter->old_spte = tdp_mmu_clear_spte_bits(iter->sptep, 1177 iter->old_spte, 1178 shadow_accessed_mask, 1179 iter->level); 1180 new_spte = iter->old_spte & ~shadow_accessed_mask; 1181 } else { 1182 new_spte = mark_spte_for_access_track(iter->old_spte); 1183 iter->old_spte = kvm_tdp_mmu_write_spte(iter->sptep, 1184 iter->old_spte, new_spte, 1185 iter->level); 1186 } 1187 1188 trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, 1189 iter->old_spte, new_spte); 1190 } 1191 1192 static bool __kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, 1193 struct kvm_gfn_range *range, 1194 bool test_only) 1195 { 1196 struct kvm_mmu_page *root; 1197 struct tdp_iter iter; 1198 bool ret = false; 1199 1200 /* 1201 * Don't support rescheduling, none of the MMU notifiers that funnel 1202 * into this helper allow blocking; it'd be dead, wasteful code. Note, 1203 * this helper must NOT be used to unmap GFNs, as it processes only 1204 * valid roots! 1205 */ 1206 for_each_valid_tdp_mmu_root(kvm, root, range->slot->as_id) { 1207 guard(rcu)(); 1208 1209 tdp_root_for_each_leaf_pte(iter, root, range->start, range->end) { 1210 if (!is_accessed_spte(iter.old_spte)) 1211 continue; 1212 1213 if (test_only) 1214 return true; 1215 1216 ret = true; 1217 kvm_tdp_mmu_age_spte(&iter); 1218 } 1219 } 1220 1221 return ret; 1222 } 1223 1224 bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) 1225 { 1226 return __kvm_tdp_mmu_age_gfn_range(kvm, range, false); 1227 } 1228 1229 bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 1230 { 1231 return __kvm_tdp_mmu_age_gfn_range(kvm, range, true); 1232 } 1233 1234 /* 1235 * Remove write access from all SPTEs at or above min_level that map GFNs 1236 * [start, end). Returns true if an SPTE has been changed and the TLBs need to 1237 * be flushed. 1238 */ 1239 static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1240 gfn_t start, gfn_t end, int min_level) 1241 { 1242 struct tdp_iter iter; 1243 u64 new_spte; 1244 bool spte_set = false; 1245 1246 rcu_read_lock(); 1247 1248 BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 1249 1250 for_each_tdp_pte_min_level(iter, root, min_level, start, end) { 1251 retry: 1252 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 1253 continue; 1254 1255 if (!is_shadow_present_pte(iter.old_spte) || 1256 !is_last_spte(iter.old_spte, iter.level) || 1257 !(iter.old_spte & PT_WRITABLE_MASK)) 1258 continue; 1259 1260 new_spte = iter.old_spte & ~PT_WRITABLE_MASK; 1261 1262 if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) 1263 goto retry; 1264 1265 spte_set = true; 1266 } 1267 1268 rcu_read_unlock(); 1269 return spte_set; 1270 } 1271 1272 /* 1273 * Remove write access from all the SPTEs mapping GFNs in the memslot. Will 1274 * only affect leaf SPTEs down to min_level. 1275 * Returns true if an SPTE has been changed and the TLBs need to be flushed. 1276 */ 1277 bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, 1278 const struct kvm_memory_slot *slot, int min_level) 1279 { 1280 struct kvm_mmu_page *root; 1281 bool spte_set = false; 1282 1283 lockdep_assert_held_read(&kvm->mmu_lock); 1284 1285 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) 1286 spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn, 1287 slot->base_gfn + slot->npages, min_level); 1288 1289 return spte_set; 1290 } 1291 1292 static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(void) 1293 { 1294 struct kvm_mmu_page *sp; 1295 1296 sp = kmem_cache_zalloc(mmu_page_header_cache, GFP_KERNEL_ACCOUNT); 1297 if (!sp) 1298 return NULL; 1299 1300 sp->spt = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 1301 if (!sp->spt) { 1302 kmem_cache_free(mmu_page_header_cache, sp); 1303 return NULL; 1304 } 1305 1306 return sp; 1307 } 1308 1309 /* Note, the caller is responsible for initializing @sp. */ 1310 static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter, 1311 struct kvm_mmu_page *sp, bool shared) 1312 { 1313 const u64 huge_spte = iter->old_spte; 1314 const int level = iter->level; 1315 int ret, i; 1316 1317 /* 1318 * No need for atomics when writing to sp->spt since the page table has 1319 * not been linked in yet and thus is not reachable from any other CPU. 1320 */ 1321 for (i = 0; i < SPTE_ENT_PER_PAGE; i++) 1322 sp->spt[i] = make_small_spte(kvm, huge_spte, sp->role, i); 1323 1324 /* 1325 * Replace the huge spte with a pointer to the populated lower level 1326 * page table. Since we are making this change without a TLB flush vCPUs 1327 * will see a mix of the split mappings and the original huge mapping, 1328 * depending on what's currently in their TLB. This is fine from a 1329 * correctness standpoint since the translation will be the same either 1330 * way. 1331 */ 1332 ret = tdp_mmu_link_sp(kvm, iter, sp, shared); 1333 if (ret) 1334 goto out; 1335 1336 /* 1337 * tdp_mmu_link_sp_atomic() will handle subtracting the huge page we 1338 * are overwriting from the page stats. But we have to manually update 1339 * the page stats with the new present child pages. 1340 */ 1341 kvm_update_page_stats(kvm, level - 1, SPTE_ENT_PER_PAGE); 1342 1343 out: 1344 trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret); 1345 return ret; 1346 } 1347 1348 static int tdp_mmu_split_huge_pages_root(struct kvm *kvm, 1349 struct kvm_mmu_page *root, 1350 gfn_t start, gfn_t end, 1351 int target_level, bool shared) 1352 { 1353 struct kvm_mmu_page *sp = NULL; 1354 struct tdp_iter iter; 1355 1356 rcu_read_lock(); 1357 1358 /* 1359 * Traverse the page table splitting all huge pages above the target 1360 * level into one lower level. For example, if we encounter a 1GB page 1361 * we split it into 512 2MB pages. 1362 * 1363 * Since the TDP iterator uses a pre-order traversal, we are guaranteed 1364 * to visit an SPTE before ever visiting its children, which means we 1365 * will correctly recursively split huge pages that are more than one 1366 * level above the target level (e.g. splitting a 1GB to 512 2MB pages, 1367 * and then splitting each of those to 512 4KB pages). 1368 */ 1369 for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) { 1370 retry: 1371 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared)) 1372 continue; 1373 1374 if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte)) 1375 continue; 1376 1377 if (!sp) { 1378 rcu_read_unlock(); 1379 1380 if (shared) 1381 read_unlock(&kvm->mmu_lock); 1382 else 1383 write_unlock(&kvm->mmu_lock); 1384 1385 sp = tdp_mmu_alloc_sp_for_split(); 1386 1387 if (shared) 1388 read_lock(&kvm->mmu_lock); 1389 else 1390 write_lock(&kvm->mmu_lock); 1391 1392 if (!sp) { 1393 trace_kvm_mmu_split_huge_page(iter.gfn, 1394 iter.old_spte, 1395 iter.level, -ENOMEM); 1396 return -ENOMEM; 1397 } 1398 1399 rcu_read_lock(); 1400 1401 iter.yielded = true; 1402 continue; 1403 } 1404 1405 tdp_mmu_init_child_sp(sp, &iter); 1406 1407 if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared)) 1408 goto retry; 1409 1410 sp = NULL; 1411 } 1412 1413 rcu_read_unlock(); 1414 1415 /* 1416 * It's possible to exit the loop having never used the last sp if, for 1417 * example, a vCPU doing HugePage NX splitting wins the race and 1418 * installs its own sp in place of the last sp we tried to split. 1419 */ 1420 if (sp) 1421 tdp_mmu_free_sp(sp); 1422 1423 return 0; 1424 } 1425 1426 1427 /* 1428 * Try to split all huge pages mapped by the TDP MMU down to the target level. 1429 */ 1430 void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, 1431 const struct kvm_memory_slot *slot, 1432 gfn_t start, gfn_t end, 1433 int target_level, bool shared) 1434 { 1435 struct kvm_mmu_page *root; 1436 int r = 0; 1437 1438 kvm_lockdep_assert_mmu_lock_held(kvm, shared); 1439 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) { 1440 r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); 1441 if (r) { 1442 kvm_tdp_mmu_put_root(kvm, root); 1443 break; 1444 } 1445 } 1446 } 1447 1448 static bool tdp_mmu_need_write_protect(struct kvm_mmu_page *sp) 1449 { 1450 /* 1451 * All TDP MMU shadow pages share the same role as their root, aside 1452 * from level, so it is valid to key off any shadow page to determine if 1453 * write protection is needed for an entire tree. 1454 */ 1455 return kvm_mmu_page_ad_need_write_protect(sp) || !kvm_ad_enabled; 1456 } 1457 1458 static void clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, 1459 gfn_t start, gfn_t end) 1460 { 1461 const u64 dbit = tdp_mmu_need_write_protect(root) ? PT_WRITABLE_MASK : 1462 shadow_dirty_mask; 1463 struct tdp_iter iter; 1464 1465 rcu_read_lock(); 1466 1467 tdp_root_for_each_pte(iter, root, start, end) { 1468 retry: 1469 if (!is_shadow_present_pte(iter.old_spte) || 1470 !is_last_spte(iter.old_spte, iter.level)) 1471 continue; 1472 1473 if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true)) 1474 continue; 1475 1476 KVM_MMU_WARN_ON(dbit == shadow_dirty_mask && 1477 spte_ad_need_write_protect(iter.old_spte)); 1478 1479 if (!(iter.old_spte & dbit)) 1480 continue; 1481 1482 if (tdp_mmu_set_spte_atomic(kvm, &iter, iter.old_spte & ~dbit)) 1483 goto retry; 1484 } 1485 1486 rcu_read_unlock(); 1487 } 1488 1489 /* 1490 * Clear the dirty status (D-bit or W-bit) of all the SPTEs mapping GFNs in the 1491 * memslot. 1492 */ 1493 void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, 1494 const struct kvm_memory_slot *slot) 1495 { 1496 struct kvm_mmu_page *root; 1497 1498 lockdep_assert_held_read(&kvm->mmu_lock); 1499 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) 1500 clear_dirty_gfn_range(kvm, root, slot->base_gfn, 1501 slot->base_gfn + slot->npages); 1502 } 1503 1504 static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root, 1505 gfn_t gfn, unsigned long mask, bool wrprot) 1506 { 1507 const u64 dbit = (wrprot || tdp_mmu_need_write_protect(root)) ? PT_WRITABLE_MASK : 1508 shadow_dirty_mask; 1509 struct tdp_iter iter; 1510 1511 lockdep_assert_held_write(&kvm->mmu_lock); 1512 1513 rcu_read_lock(); 1514 1515 tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask), 1516 gfn + BITS_PER_LONG) { 1517 if (!mask) 1518 break; 1519 1520 KVM_MMU_WARN_ON(dbit == shadow_dirty_mask && 1521 spte_ad_need_write_protect(iter.old_spte)); 1522 1523 if (iter.level > PG_LEVEL_4K || 1524 !(mask & (1UL << (iter.gfn - gfn)))) 1525 continue; 1526 1527 mask &= ~(1UL << (iter.gfn - gfn)); 1528 1529 if (!(iter.old_spte & dbit)) 1530 continue; 1531 1532 iter.old_spte = tdp_mmu_clear_spte_bits(iter.sptep, 1533 iter.old_spte, dbit, 1534 iter.level); 1535 1536 trace_kvm_tdp_mmu_spte_changed(iter.as_id, iter.gfn, iter.level, 1537 iter.old_spte, 1538 iter.old_spte & ~dbit); 1539 } 1540 1541 rcu_read_unlock(); 1542 } 1543 1544 /* 1545 * Clear the dirty status (D-bit or W-bit) of all the 4k SPTEs mapping GFNs for 1546 * which a bit is set in mask, starting at gfn. The given memslot is expected to 1547 * contain all the GFNs represented by set bits in the mask. 1548 */ 1549 void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, 1550 struct kvm_memory_slot *slot, 1551 gfn_t gfn, unsigned long mask, 1552 bool wrprot) 1553 { 1554 struct kvm_mmu_page *root; 1555 1556 for_each_valid_tdp_mmu_root(kvm, root, slot->as_id) 1557 clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot); 1558 } 1559 1560 static int tdp_mmu_make_huge_spte(struct kvm *kvm, 1561 struct tdp_iter *parent, 1562 u64 *huge_spte) 1563 { 1564 struct kvm_mmu_page *root = spte_to_child_sp(parent->old_spte); 1565 gfn_t start = parent->gfn; 1566 gfn_t end = start + KVM_PAGES_PER_HPAGE(parent->level); 1567 struct tdp_iter iter; 1568 1569 tdp_root_for_each_leaf_pte(iter, root, start, end) { 1570 /* 1571 * Use the parent iterator when checking for forward progress so 1572 * that KVM doesn't get stuck continuously trying to yield (i.e. 1573 * returning -EAGAIN here and then failing the forward progress 1574 * check in the caller ad nauseam). 1575 */ 1576 if (tdp_mmu_iter_need_resched(kvm, parent)) 1577 return -EAGAIN; 1578 1579 *huge_spte = make_huge_spte(kvm, iter.old_spte, parent->level); 1580 return 0; 1581 } 1582 1583 return -ENOENT; 1584 } 1585 1586 static void recover_huge_pages_range(struct kvm *kvm, 1587 struct kvm_mmu_page *root, 1588 const struct kvm_memory_slot *slot) 1589 { 1590 gfn_t start = slot->base_gfn; 1591 gfn_t end = start + slot->npages; 1592 struct tdp_iter iter; 1593 int max_mapping_level; 1594 bool flush = false; 1595 u64 huge_spte; 1596 int r; 1597 1598 if (WARN_ON_ONCE(kvm_slot_dirty_track_enabled(slot))) 1599 return; 1600 1601 rcu_read_lock(); 1602 1603 for_each_tdp_pte_min_level(iter, root, PG_LEVEL_2M, start, end) { 1604 retry: 1605 if (tdp_mmu_iter_cond_resched(kvm, &iter, flush, true)) { 1606 flush = false; 1607 continue; 1608 } 1609 1610 if (iter.level > KVM_MAX_HUGEPAGE_LEVEL || 1611 !is_shadow_present_pte(iter.old_spte)) 1612 continue; 1613 1614 /* 1615 * Don't zap leaf SPTEs, if a leaf SPTE could be replaced with 1616 * a large page size, then its parent would have been zapped 1617 * instead of stepping down. 1618 */ 1619 if (is_last_spte(iter.old_spte, iter.level)) 1620 continue; 1621 1622 /* 1623 * If iter.gfn resides outside of the slot, i.e. the page for 1624 * the current level overlaps but is not contained by the slot, 1625 * then the SPTE can't be made huge. More importantly, trying 1626 * to query that info from slot->arch.lpage_info will cause an 1627 * out-of-bounds access. 1628 */ 1629 if (iter.gfn < start || iter.gfn >= end) 1630 continue; 1631 1632 max_mapping_level = kvm_mmu_max_mapping_level(kvm, slot, iter.gfn); 1633 if (max_mapping_level < iter.level) 1634 continue; 1635 1636 r = tdp_mmu_make_huge_spte(kvm, &iter, &huge_spte); 1637 if (r == -EAGAIN) 1638 goto retry; 1639 else if (r) 1640 continue; 1641 1642 if (tdp_mmu_set_spte_atomic(kvm, &iter, huge_spte)) 1643 goto retry; 1644 1645 flush = true; 1646 } 1647 1648 if (flush) 1649 kvm_flush_remote_tlbs_memslot(kvm, slot); 1650 1651 rcu_read_unlock(); 1652 } 1653 1654 /* 1655 * Recover huge page mappings within the slot by replacing non-leaf SPTEs with 1656 * huge SPTEs where possible. 1657 */ 1658 void kvm_tdp_mmu_recover_huge_pages(struct kvm *kvm, 1659 const struct kvm_memory_slot *slot) 1660 { 1661 struct kvm_mmu_page *root; 1662 1663 lockdep_assert_held_read(&kvm->mmu_lock); 1664 for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) 1665 recover_huge_pages_range(kvm, root, slot); 1666 } 1667 1668 /* 1669 * Removes write access on the last level SPTE mapping this GFN and unsets the 1670 * MMU-writable bit to ensure future writes continue to be intercepted. 1671 * Returns true if an SPTE was set and a TLB flush is needed. 1672 */ 1673 static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, 1674 gfn_t gfn, int min_level) 1675 { 1676 struct tdp_iter iter; 1677 u64 new_spte; 1678 bool spte_set = false; 1679 1680 BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); 1681 1682 rcu_read_lock(); 1683 1684 for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) { 1685 if (!is_shadow_present_pte(iter.old_spte) || 1686 !is_last_spte(iter.old_spte, iter.level)) 1687 continue; 1688 1689 new_spte = iter.old_spte & 1690 ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); 1691 1692 if (new_spte == iter.old_spte) 1693 break; 1694 1695 tdp_mmu_iter_set_spte(kvm, &iter, new_spte); 1696 spte_set = true; 1697 } 1698 1699 rcu_read_unlock(); 1700 1701 return spte_set; 1702 } 1703 1704 /* 1705 * Removes write access on the last level SPTE mapping this GFN and unsets the 1706 * MMU-writable bit to ensure future writes continue to be intercepted. 1707 * Returns true if an SPTE was set and a TLB flush is needed. 1708 */ 1709 bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, 1710 struct kvm_memory_slot *slot, gfn_t gfn, 1711 int min_level) 1712 { 1713 struct kvm_mmu_page *root; 1714 bool spte_set = false; 1715 1716 lockdep_assert_held_write(&kvm->mmu_lock); 1717 for_each_valid_tdp_mmu_root(kvm, root, slot->as_id) 1718 spte_set |= write_protect_gfn(kvm, root, gfn, min_level); 1719 1720 return spte_set; 1721 } 1722 1723 /* 1724 * Return the level of the lowest level SPTE added to sptes. 1725 * That SPTE may be non-present. 1726 * 1727 * Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 1728 */ 1729 int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, 1730 int *root_level) 1731 { 1732 struct tdp_iter iter; 1733 struct kvm_mmu *mmu = vcpu->arch.mmu; 1734 gfn_t gfn = addr >> PAGE_SHIFT; 1735 int leaf = -1; 1736 1737 *root_level = vcpu->arch.mmu->root_role.level; 1738 1739 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 1740 leaf = iter.level; 1741 sptes[leaf] = iter.old_spte; 1742 } 1743 1744 return leaf; 1745 } 1746 1747 /* 1748 * Returns the last level spte pointer of the shadow page walk for the given 1749 * gpa, and sets *spte to the spte value. This spte may be non-preset. If no 1750 * walk could be performed, returns NULL and *spte does not contain valid data. 1751 * 1752 * Contract: 1753 * - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}. 1754 * - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end. 1755 * 1756 * WARNING: This function is only intended to be called during fast_page_fault. 1757 */ 1758 u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gfn_t gfn, 1759 u64 *spte) 1760 { 1761 struct tdp_iter iter; 1762 struct kvm_mmu *mmu = vcpu->arch.mmu; 1763 tdp_ptep_t sptep = NULL; 1764 1765 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { 1766 *spte = iter.old_spte; 1767 sptep = iter.sptep; 1768 } 1769 1770 /* 1771 * Perform the rcu_dereference to get the raw spte pointer value since 1772 * we are passing it up to fast_page_fault, which is shared with the 1773 * legacy MMU and thus does not retain the TDP MMU-specific __rcu 1774 * annotation. 1775 * 1776 * This is safe since fast_page_fault obeys the contracts of this 1777 * function as well as all TDP MMU contracts around modifying SPTEs 1778 * outside of mmu_lock. 1779 */ 1780 return rcu_dereference(sptep); 1781 } 1782