1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2007, 2016, 2025 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 * Claudio Imbrenda <imbrenda@linux.ibm.com> 8 */ 9 10 #ifndef ARCH_KVM_GMAP_GMAP_H 11 #define ARCH_KVM_GMAP_GMAP_H 12 13 #include <linux/kvm_host.h> 14 15 #include "dat.h" 16 17 /** 18 * enum gmap_flags - Flags of a gmap. 19 * 20 * @GMAP_FLAG_SHADOW: The gmap is a vsie shadow gmap. 21 * @GMAP_FLAG_OWNS_PAGETABLES: The gmap owns all dat levels; normally 1, is 0 22 * only for ucontrol per-cpu gmaps, since they 23 * share the page tables with the main gmap. 24 * @GMAP_FLAG_IS_UCONTROL: The gmap is ucontrol (main gmap or per-cpu gmap). 25 * @GMAP_FLAG_ALLOW_HPAGE_1M: 1M hugepages are allowed for this gmap, 26 * independently of the page size used by userspace. 27 * @GMAP_FLAG_ALLOW_HPAGE_2G: 2G hugepages are allowed for this gmap, 28 * independently of the page size used by userspace. 29 * @GMAP_FLAG_PFAULT_ENABLED: Pfault is enabled for the gmap. 30 * @GMAP_FLAG_USES_SKEYS: If the guest uses storage keys. 31 * @GMAP_FLAG_USES_CMM: Whether the guest uses CMMA. 32 * @GMAP_FLAG_EXPORT_ON_UNMAP: Whether to export guest pages when unmapping. 33 */ 34 enum gmap_flags { 35 GMAP_FLAG_SHADOW = 0, 36 GMAP_FLAG_OWNS_PAGETABLES, 37 GMAP_FLAG_IS_UCONTROL, 38 GMAP_FLAG_ALLOW_HPAGE_1M, 39 GMAP_FLAG_ALLOW_HPAGE_2G, 40 GMAP_FLAG_PFAULT_ENABLED, 41 GMAP_FLAG_USES_SKEYS, 42 GMAP_FLAG_USES_CMM, 43 GMAP_FLAG_EXPORT_ON_UNMAP, 44 }; 45 46 /** 47 * struct gmap_struct - Guest address space. 48 * 49 * @flags: GMAP_FLAG_* flags. 50 * @edat_level: The edat level of this shadow gmap. 51 * @kvm: The vm. 52 * @asce: The ASCE used by this gmap. 53 * @list: List head used in children gmaps for the children gmap list. 54 * @children_lock: Protects children and scb_users. 55 * @children: List of child gmaps of this gmap. 56 * @scb_users: List of vsie_scb that use this shadow gmap. 57 * @parent: Parent gmap of a child gmap. 58 * @guest_asce: Original ASCE of this shadow gmap. 59 * @host_to_rmap_lock: Protects host_to_rmap. 60 * @host_to_rmap: Radix tree mapping host addresses to guest addresses. 61 */ 62 struct gmap { 63 unsigned long flags; 64 unsigned char edat_level; 65 bool invalidated; 66 struct kvm *kvm; 67 union asce asce; 68 struct list_head list; 69 spinlock_t children_lock; /* Protects: children, scb_users */ 70 struct list_head children; 71 struct list_head scb_users; 72 struct gmap *parent; 73 union asce guest_asce; 74 spinlock_t host_to_rmap_lock; /* Protects host_to_rmap */ 75 struct radix_tree_root host_to_rmap; 76 refcount_t refcount; 77 }; 78 79 struct gmap_cache { 80 struct list_head list; 81 struct gmap *gmap; 82 }; 83 84 #define gmap_for_each_rmap_safe(pos, n, head) \ 85 for (pos = (head); n = pos ? pos->next : NULL, pos; pos = n) 86 87 int s390_replace_asce(struct gmap *gmap); 88 bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end); 89 bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end); 90 int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault); 91 struct gmap *gmap_new(struct kvm *kvm, gfn_t limit); 92 struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit); 93 void gmap_remove_child(struct gmap *child); 94 void gmap_dispose(struct gmap *gmap); 95 int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *fault, 96 struct kvm_memory_slot *slot); 97 void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end); 98 int gmap_set_limit(struct gmap *gmap, gfn_t limit); 99 int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr); 100 int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count); 101 void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count); 102 103 #if KVM_S390_MANAGES_S390_GUEST 104 int gmap_enable_skeys(struct gmap *gmap); 105 #endif /* KVM_S390_MANAGES_S390_GUEST */ 106 107 int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible); 108 int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, 109 gfn_t r_gfn, int level); 110 int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, 111 kvm_pfn_t pfn, int level, bool wr); 112 void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn); 113 struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, 114 union asce asce, int edat_level); 115 void gmap_split_huge_pages(struct gmap *gmap); 116 117 static inline bool uses_skeys(struct gmap *gmap) 118 { 119 return test_bit(GMAP_FLAG_USES_SKEYS, &gmap->flags); 120 } 121 122 static inline bool uses_cmm(struct gmap *gmap) 123 { 124 return test_bit(GMAP_FLAG_USES_CMM, &gmap->flags); 125 } 126 127 static inline bool pfault_enabled(struct gmap *gmap) 128 { 129 return test_bit(GMAP_FLAG_PFAULT_ENABLED, &gmap->flags); 130 } 131 132 static inline bool is_ucontrol(struct gmap *gmap) 133 { 134 return test_bit(GMAP_FLAG_IS_UCONTROL, &gmap->flags); 135 } 136 137 static inline bool is_shadow(struct gmap *gmap) 138 { 139 return test_bit(GMAP_FLAG_SHADOW, &gmap->flags); 140 } 141 142 static inline bool owns_page_tables(struct gmap *gmap) 143 { 144 return test_bit(GMAP_FLAG_OWNS_PAGETABLES, &gmap->flags); 145 } 146 147 static inline struct gmap *gmap_put(struct gmap *gmap) 148 { 149 if (refcount_dec_and_test(&gmap->refcount)) 150 gmap_dispose(gmap); 151 return NULL; 152 } 153 154 static inline void gmap_get(struct gmap *gmap) 155 { 156 WARN_ON_ONCE(unlikely(!refcount_inc_not_zero(&gmap->refcount))); 157 } 158 159 static inline void gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn) 160 { 161 scoped_guard(spinlock, &parent->children_lock) 162 _gmap_handle_vsie_unshadow_event(parent, gfn); 163 } 164 165 #if KVM_S390_MANAGES_S390_GUEST 166 bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); 167 #else 168 static inline bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) 169 { 170 return true; 171 } 172 #endif /* KVM_S390_MANAGES_S390_GUEST */ 173 static inline bool gmap_mkold_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) 174 { 175 return _gmap_unmap_prefix(gmap, gfn, end, true); 176 } 177 178 static inline bool gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) 179 { 180 return _gmap_unmap_prefix(gmap, gfn, end, false); 181 } 182 183 /** 184 * pte_needs_unshadow() -- Check if the pte operations triggers unshadowing. 185 * @oldpte: the previous value for the guest pte. 186 * @newpte: the new pte being set. 187 * @pgste: the pgste for the pte entry. 188 * 189 * If the pgste.vsie_notif bit is not set, return false: the page is not 190 * involved in vsie and thus should not trigger an unshadow operation. 191 * 192 * If the pgste.vsie_gmem bit is set, this pte represents shadowed guest 193 * memory. The access rights on g3's memory should be synchronized with g1's 194 * and g2's. Therefore unshadowing is triggered if the new and old pte 195 * differ in protection, or if the new pte is invalid. 196 * 197 * If the pgste.vsie_gmem bit is not set, this pte maps the g2 dat tables 198 * for g3. If the entry becomes writable or absent, it becomes impossible to 199 * guarantee that the shadow mapping will match g2's mapping. In that case, 200 * trigger an unshadow event. 201 * 202 * Return: true if an unshadow event should be triggered, otherwise false. 203 */ 204 static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union pgste pgste) 205 { 206 if (!pgste.vsie_notif) 207 return false; 208 if (pgste.vsie_gmem) 209 return (oldpte.h.p != newpte.h.p) || newpte.h.i; 210 return !newpte.h.p || !newpte.s.pr; 211 } 212 213 #if KVM_S390_MANAGES_S390_GUEST 214 void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); 215 static inline void gmap_set_cmma_all_dirty(struct gmap *gmap) 216 { 217 _gmap_set_cmma_all(gmap, true); 218 } 219 220 static inline void gmap_set_cmma_all_clean(struct gmap *gmap) 221 { 222 _gmap_set_cmma_all(gmap, false); 223 } 224 #endif /* KVM_S390_MANAGES_S390_GUEST */ 225 226 static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, 227 union pgste pgste, gfn_t gfn, bool needs_lock) 228 { 229 lockdep_assert_held(&gmap->kvm->mmu_lock); 230 if (!needs_lock) 231 lockdep_assert_held(&gmap->children_lock); 232 else 233 lockdep_assert_not_held(&gmap->children_lock); 234 235 if (pgste.prefix_notif && (newpte.h.p || newpte.h.i)) { 236 pgste.prefix_notif = 0; 237 gmap_unmap_prefix(gmap, gfn, gfn + 1); 238 } 239 if (pte_needs_unshadow(*ptep, newpte, pgste)) { 240 pgste.vsie_notif = 0; 241 pgste.vsie_gmem = 0; 242 if (needs_lock) 243 gmap_handle_vsie_unshadow_event(gmap, gfn); 244 else 245 _gmap_handle_vsie_unshadow_event(gmap, gfn); 246 } 247 if (!ptep->s.d && newpte.s.d && !newpte.s.s) 248 SetPageDirty(pfn_to_page(newpte.h.pfra)); 249 pgste.zero = 0; 250 return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap)); 251 } 252 253 static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, 254 union pgste pgste, gfn_t gfn) 255 { 256 return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true); 257 } 258 259 /** 260 * crste_needs_unshadow() -- Check if the crste operations triggers unshadowing. 261 * @oldcrste: the previous value for the crste. 262 * @newcrste: the new value for the crste. 263 * 264 * If the old crste did not have the vsie_notif bit set, return false: the 265 * page is not involved in vsie and thus should not trigger an unshadow 266 * operation. Conversely, if the bit is set, it can only be g3 memory, since 267 * dat tables are never mapped using large pages. 268 * 269 * Similar to the pgste.vsie_gmem case of pte_needs_unshadow(), if the 270 * protection bit is changing or the new page is invalid, trigger an 271 * unshadow event. Also trigger an unshadow event if the new crste does not 272 * have the vsie_notif bit set. 273 * 274 * Return: true if an unshadow event should be triggered, otherwise false. 275 */ 276 static inline bool crste_needs_unshadow(union crste oldcrste, union crste newcrste) 277 { 278 if (!oldcrste.s.fc1.vsie_notif) 279 return false; 280 return (newcrste.h.p != oldcrste.h.p) || newcrste.h.i || !newcrste.s.fc1.vsie_notif; 281 } 282 283 static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, 284 union crste oldcrste, union crste newcrste, 285 gfn_t gfn, bool needs_lock) 286 { 287 unsigned long align = is_pmd(newcrste) ? _PAGE_ENTRIES : _PAGE_ENTRIES * _CRST_ENTRIES; 288 289 if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm)) 290 return true; 291 292 lockdep_assert_held(&gmap->kvm->mmu_lock); 293 if (!needs_lock) 294 lockdep_assert_held(&gmap->children_lock); 295 296 gfn = ALIGN_DOWN(gfn, align); 297 if (crste_prefix(oldcrste) && (newcrste.h.p || newcrste.h.i || !crste_prefix(newcrste))) { 298 newcrste.s.fc1.prefix_notif = 0; 299 gmap_unmap_prefix(gmap, gfn, gfn + align); 300 } 301 if (crste_leaf(oldcrste) && crste_needs_unshadow(oldcrste, newcrste)) { 302 newcrste = oldcrste; 303 newcrste.s.fc1.vsie_notif = 0; 304 if (needs_lock) 305 gmap_handle_vsie_unshadow_event(gmap, gfn); 306 else 307 _gmap_handle_vsie_unshadow_event(gmap, gfn); 308 if (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce)) 309 return false; 310 /* 311 * Return false even if the swap was successful, as it only 312 * indicates that the best effort clearing of the vsie_notif 313 * bit was successful. The caller will have to try again 314 * regardless, since the desired value has not been set. 315 * This pointless check is needed to silence a potential 316 * __must_check warning. 317 */ 318 return false; 319 } 320 if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s) 321 SetPageDirty(phys_to_page(crste_origin_large(newcrste))); 322 return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); 323 } 324 325 static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, 326 union crste oldcrste, union crste newcrste, 327 gfn_t gfn) 328 { 329 return _gmap_crstep_xchg_atomic(gmap, crstep, oldcrste, newcrste, gfn, true); 330 } 331 332 /** 333 * gmap_is_shadow_valid() - check if a shadow guest address space matches the 334 * given properties and is still valid. 335 * @sg: Pointer to the shadow guest address space structure. 336 * @asce: ASCE for which the shadow table is requested. 337 * @edat_level: Edat level to be used for the shadow translation. 338 * 339 * Return: true if the gmap shadow is still valid and matches the given 340 * properties and the caller can continue using it; false otherwise, the 341 * caller has to request a new shadow gmap in this case. 342 */ 343 static inline bool gmap_is_shadow_valid(struct gmap *sg, union asce asce, int edat_level) 344 { 345 return sg->guest_asce.val == asce.val && sg->edat_level == edat_level; 346 } 347 348 #endif /* ARCH_KVM_GMAP_GMAP_H */ 349