1 // SPDX-License-Identifier: GPL-2.0
2
3 #ifndef __KVM_X86_MMU_TDP_MMU_H
4 #define __KVM_X86_MMU_TDP_MMU_H
5
6 #include <linux/kvm_host.h>
7
8 #include "spte.h"
9
10 void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
11 void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
12
13 void kvm_tdp_mmu_alloc_root(struct kvm_vcpu *vcpu, bool private);
14
kvm_tdp_mmu_get_root(struct kvm_mmu_page * root)15 __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root)
16 {
17 return refcount_inc_not_zero(&root->tdp_mmu_root_count);
18 }
19
20 void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root);
21
22 enum kvm_tdp_mmu_root_types {
23 KVM_INVALID_ROOTS = BIT(0),
24 KVM_DIRECT_ROOTS = BIT(1),
25 KVM_MIRROR_ROOTS = BIT(2),
26 KVM_VALID_ROOTS = KVM_DIRECT_ROOTS | KVM_MIRROR_ROOTS,
27 KVM_ALL_ROOTS = KVM_VALID_ROOTS | KVM_INVALID_ROOTS,
28 };
29
kvm_gfn_range_filter_to_root_types(struct kvm * kvm,enum kvm_gfn_range_filter process)30 static inline enum kvm_tdp_mmu_root_types kvm_gfn_range_filter_to_root_types(struct kvm *kvm,
31 enum kvm_gfn_range_filter process)
32 {
33 enum kvm_tdp_mmu_root_types ret = 0;
34
35 if (!kvm_has_mirrored_tdp(kvm))
36 return KVM_DIRECT_ROOTS;
37
38 if (process & KVM_FILTER_PRIVATE)
39 ret |= KVM_MIRROR_ROOTS;
40 if (process & KVM_FILTER_SHARED)
41 ret |= KVM_DIRECT_ROOTS;
42
43 WARN_ON_ONCE(!ret);
44
45 return ret;
46 }
47
tdp_mmu_get_root_for_fault(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)48 static inline struct kvm_mmu_page *tdp_mmu_get_root_for_fault(struct kvm_vcpu *vcpu,
49 struct kvm_page_fault *fault)
50 {
51 if (unlikely(!kvm_is_addr_direct(vcpu->kvm, fault->addr)))
52 return root_to_sp(vcpu->arch.mmu->mirror_root_hpa);
53
54 return root_to_sp(vcpu->arch.mmu->root.hpa);
55 }
56
tdp_mmu_get_root(struct kvm_vcpu * vcpu,enum kvm_tdp_mmu_root_types type)57 static inline struct kvm_mmu_page *tdp_mmu_get_root(struct kvm_vcpu *vcpu,
58 enum kvm_tdp_mmu_root_types type)
59 {
60 if (unlikely(type == KVM_MIRROR_ROOTS))
61 return root_to_sp(vcpu->arch.mmu->mirror_root_hpa);
62
63 return root_to_sp(vcpu->arch.mmu->root.hpa);
64 }
65
66 bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
67 bool kvm_tdp_mmu_zap_possible_nx_huge_page(struct kvm *kvm,
68 struct kvm_mmu_page *sp);
69 void kvm_tdp_mmu_zap_all(struct kvm *kvm);
70 void kvm_tdp_mmu_invalidate_roots(struct kvm *kvm,
71 enum kvm_tdp_mmu_root_types root_types);
72 void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared);
73
74 int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
75
76 bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
77 bool flush);
78 bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
79 bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
80
81 bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
82 const struct kvm_memory_slot *slot, int min_level);
83 void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
84 const struct kvm_memory_slot *slot);
85 void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
86 struct kvm_memory_slot *slot,
87 gfn_t gfn, unsigned long mask,
88 bool wrprot);
89 void kvm_tdp_mmu_recover_huge_pages(struct kvm *kvm,
90 const struct kvm_memory_slot *slot);
91
92 bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
93 struct kvm_memory_slot *slot, gfn_t gfn,
94 int min_level);
95
96 void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm,
97 const struct kvm_memory_slot *slot,
98 gfn_t start, gfn_t end,
99 int target_level, bool shared);
100
kvm_tdp_mmu_walk_lockless_begin(void)101 static inline void kvm_tdp_mmu_walk_lockless_begin(void)
102 {
103 rcu_read_lock();
104 }
105
kvm_tdp_mmu_walk_lockless_end(void)106 static inline void kvm_tdp_mmu_walk_lockless_end(void)
107 {
108 rcu_read_unlock();
109 }
110
111 int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
112 int *root_level);
113 u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gfn_t gfn,
114 u64 *spte);
115
116 #ifdef CONFIG_X86_64
is_tdp_mmu_page(struct kvm_mmu_page * sp)117 static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; }
118 #else
is_tdp_mmu_page(struct kvm_mmu_page * sp)119 static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; }
120 #endif
121
122 #endif /* __KVM_X86_MMU_TDP_MMU_H */
123