1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Helper functions for KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2007, 2025 6 */ 7 8 #include <linux/export.h> 9 #include <linux/mm_types.h> 10 #include <linux/mmap_lock.h> 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/swap.h> 14 #include <linux/leafops.h> 15 #include <linux/pagewalk.h> 16 #include <linux/ksm.h> 17 #include <asm/gmap_helpers.h> 18 19 /** 20 * try_get_locked_pte() - like get_locked_pte(), but atomic and with trylock 21 * @mm: the mm 22 * @vmaddr: the userspace virtual address whose pte is to be found 23 * @ptl: will be set to the pointer to the lock used to lock the pte in case 24 * of success. 25 * 26 * This function returns the pointer to the pte corresponding to @addr in @mm, 27 * similarly to get_locked_pte(). Unlike get_locked_pte(), no attempt is made 28 * to allocate missing page tables. If a missing or large entry is found, the 29 * function will return NULL. If the ptl lock is contended, %-EAGAIN is 30 * returned. 31 * 32 * In case of success, *@ptl will point to the locked pte lock for the returned 33 * pte, like get_locked_pte() does. 34 * 35 * Context: mmap_lock or vma lock for read or for write needs to be held. 36 * Return: 37 * * %NULL if the pte cannot be reached. 38 * * %-EAGAIN if the pte can be reached, but cannot be locked. 39 * * the pointer to the pte corresponding to @addr in @mm, if it can be reached 40 * and locked. 41 */ 42 pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t **ptl) 43 { 44 pmd_t *pmdp, pmd, pmdval; 45 pud_t *pudp, pud; 46 p4d_t *p4dp, p4d; 47 pgd_t *pgdp, pgd; 48 pte_t *ptep; 49 50 pgdp = pgd_offset(mm, vmaddr); 51 pgd = pgdp_get(pgdp); 52 if (pgd_none(pgd) || !pgd_present(pgd)) 53 return NULL; 54 p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr); 55 p4d = p4dp_get(p4dp); 56 if (p4d_none(p4d) || !p4d_present(p4d)) 57 return NULL; 58 pudp = pud_offset_lockless(p4dp, p4d, vmaddr); 59 pud = pudp_get(pudp); 60 if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud)) 61 return NULL; 62 pmdp = pmd_offset_lockless(pudp, pud, vmaddr); 63 pmd = pmdp_get_lockless(pmdp); 64 if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd)) 65 return NULL; 66 ptep = pte_offset_map_rw_nolock(mm, pmdp, vmaddr, &pmdval, ptl); 67 if (!ptep) 68 return NULL; 69 70 if (spin_trylock(*ptl)) { 71 if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmdp)))) { 72 pte_unmap_unlock(ptep, *ptl); 73 return ERR_PTR(-EAGAIN); 74 } 75 return ptep; 76 } 77 78 pte_unmap(ptep); 79 return ERR_PTR(-EAGAIN); 80 } 81 EXPORT_SYMBOL_GPL(try_get_locked_pte); 82 83 /** 84 * gmap_helper_zap_one_page() - discard a page if it was swapped. 85 * @mm: the mm 86 * @vmaddr: the userspace virtual address that needs to be discarded 87 * 88 * If the given address maps to a swap entry, discard it. 89 * 90 * Context: needs to be called while holding the mmap lock. 91 */ 92 void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) 93 { 94 struct vm_area_struct *vma; 95 spinlock_t *ptl; /* Lock for the host (userspace) page table */ 96 softleaf_t sl; 97 pte_t *ptep; 98 99 mmap_assert_locked(mm); 100 101 /* Find the vm address for the guest address */ 102 vma = vma_lookup(mm, vmaddr); 103 if (!vma || is_vm_hugetlb_page(vma)) 104 return; 105 106 /* Get pointer to the page table entry */ 107 ptep = try_get_locked_pte(mm, vmaddr, &ptl); 108 if (IS_ERR_OR_NULL(ptep)) 109 return; 110 sl = softleaf_from_pte(*ptep); 111 if (pte_swap(*ptep) && softleaf_is_swap(sl)) { 112 dec_mm_counter(mm, MM_SWAPENTS); 113 swap_put_entries_direct(sl, 1); 114 pte_clear(mm, vmaddr, ptep); 115 } 116 pte_unmap_unlock(ptep, ptl); 117 } 118 EXPORT_SYMBOL_GPL(gmap_helper_zap_one_page); 119 120 /** 121 * gmap_helper_discard() - discard user pages in the given range 122 * @mm: the mm 123 * @vmaddr: starting userspace address 124 * @end: end address (first address outside the range) 125 * 126 * All userpace pages in the range [@vamddr, @end) are discarded and unmapped. 127 * 128 * Context: needs to be called while holding the mmap lock. 129 */ 130 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end) 131 { 132 struct vm_area_struct *vma; 133 134 mmap_assert_locked(mm); 135 136 while (vmaddr < end) { 137 vma = find_vma_intersection(mm, vmaddr, end); 138 if (!vma) 139 return; 140 if (!is_vm_hugetlb_page(vma)) 141 zap_vma_range(vma, vmaddr, min(end, vma->vm_end) - vmaddr); 142 vmaddr = vma->vm_end; 143 } 144 } 145 EXPORT_SYMBOL_GPL(gmap_helper_discard); 146 147 /** 148 * gmap_helper_try_set_pte_unused() - mark a pte entry as unused 149 * @mm: the mm 150 * @vmaddr: the userspace address whose pte is to be marked 151 * 152 * Mark the pte corresponding the given address as unused. This will cause 153 * core mm code to just drop this page instead of swapping it. 154 * 155 * This function needs to be called with interrupts disabled (for example 156 * while holding a spinlock), or while holding the mmap lock. Normally this 157 * function is called as a result of an unmap operation, and thus KVM common 158 * code will already hold kvm->mmu_lock in write mode. 159 * 160 * Context: Needs to be called while holding the mmap lock or with interrupts 161 * disabled. 162 */ 163 void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr) 164 { 165 spinlock_t *ptl; /* Lock for the host (userspace) page table */ 166 pte_t *ptep; 167 168 /* 169 * Several paths exists that takes the ptl lock and then call the 170 * mmu_notifier, which takes the mmu_lock. The unmap path, instead, 171 * takes the mmu_lock in write mode first, and then potentially 172 * calls this function, which takes the ptl lock. This can lead to a 173 * deadlock. 174 * The unused page mechanism is only an optimization, if the 175 * _PAGE_UNUSED bit is not set, the unused page is swapped as normal 176 * instead of being discarded. 177 * If the lock is contended the bit is not set and the deadlock is 178 * avoided. 179 */ 180 ptep = try_get_locked_pte(mm, vmaddr, &ptl); 181 if (IS_ERR_OR_NULL(ptep)) 182 return; 183 184 if (pte_present(*ptep)) 185 __atomic64_or(_PAGE_UNUSED, (long *)ptep); 186 pte_unmap_unlock(ptep, ptl); 187 } 188 EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused); 189 190 static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, 191 unsigned long end, struct mm_walk *walk) 192 { 193 unsigned long *found_addr = walk->private; 194 195 /* Return 1 of the page is a zeropage. */ 196 if (is_zero_pfn(pte_pfn(*pte))) { 197 /* 198 * Shared zeropage in e.g., a FS DAX mapping? We cannot do the 199 * right thing and likely don't care: FAULT_FLAG_UNSHARE 200 * currently only works in COW mappings, which is also where 201 * mm_forbids_zeropage() is checked. 202 */ 203 if (!is_cow_mapping(walk->vma->vm_flags)) 204 return -EFAULT; 205 206 *found_addr = addr; 207 return 1; 208 } 209 return 0; 210 } 211 212 static const struct mm_walk_ops find_zeropage_ops = { 213 .pte_entry = find_zeropage_pte_entry, 214 .walk_lock = PGWALK_WRLOCK, 215 }; 216 217 /** __gmap_helper_unshare_zeropages() - unshare all shared zeropages 218 * @mm: the mm whose zero pages are to be unshared 219 * 220 * Unshare all shared zeropages, replacing them by anonymous pages. Note that 221 * we cannot simply zap all shared zeropages, because this could later 222 * trigger unexpected userfaultfd missing events. 223 * 224 * This must be called after mm->context.allow_cow_sharing was 225 * set to 0, to avoid future mappings of shared zeropages. 226 * 227 * mm contracts with s390, that even if mm were to remove a page table, 228 * and racing with walk_page_range_vma() calling pte_offset_map_lock() 229 * would fail, it will never insert a page table containing empty zero 230 * pages once mm_forbids_zeropage(mm) i.e. 231 * mm->context.allow_cow_sharing is set to 0. 232 */ 233 static int __gmap_helper_unshare_zeropages(struct mm_struct *mm) 234 { 235 struct vm_area_struct *vma; 236 VMA_ITERATOR(vmi, mm, 0); 237 unsigned long addr; 238 vm_fault_t fault; 239 int rc; 240 241 for_each_vma(vmi, vma) { 242 /* 243 * We could only look at COW mappings, but it's more future 244 * proof to catch unexpected zeropages in other mappings and 245 * fail. 246 */ 247 if ((vma->vm_flags & VM_PFNMAP) || is_vm_hugetlb_page(vma)) 248 continue; 249 addr = vma->vm_start; 250 251 retry: 252 rc = walk_page_range_vma(vma, addr, vma->vm_end, 253 &find_zeropage_ops, &addr); 254 if (rc < 0) 255 return rc; 256 else if (!rc) 257 continue; 258 259 /* addr was updated by find_zeropage_pte_entry() */ 260 fault = handle_mm_fault(vma, addr, 261 FAULT_FLAG_UNSHARE | FAULT_FLAG_REMOTE, 262 NULL); 263 if (fault & VM_FAULT_OOM) 264 return -ENOMEM; 265 /* 266 * See break_ksm(): even after handle_mm_fault() returned 0, we 267 * must start the lookup from the current address, because 268 * handle_mm_fault() may back out if there's any difficulty. 269 * 270 * VM_FAULT_SIGBUS and VM_FAULT_SIGSEGV are unexpected but 271 * maybe they could trigger in the future on concurrent 272 * truncation. In that case, the shared zeropage would be gone 273 * and we can simply retry and make progress. 274 */ 275 cond_resched(); 276 goto retry; 277 } 278 279 return 0; 280 } 281 282 /** 283 * gmap_helper_disable_cow_sharing() - disable all COW sharing 284 * 285 * Disable most COW-sharing of memory pages for the whole process: 286 * (1) Disable KSM and unmerge/unshare any KSM pages. 287 * (2) Disallow shared zeropages and unshare any zerpages that are mapped. 288 * 289 * Not that we currently don't bother with COW-shared pages that are shared 290 * with parent/child processes due to fork(). 291 */ 292 int gmap_helper_disable_cow_sharing(void) 293 { 294 struct mm_struct *mm = current->mm; 295 int rc; 296 297 mmap_assert_write_locked(mm); 298 299 if (!mm->context.allow_cow_sharing) 300 return 0; 301 302 mm->context.allow_cow_sharing = 0; 303 304 /* Replace all shared zeropages by anonymous pages. */ 305 rc = __gmap_helper_unshare_zeropages(mm); 306 /* 307 * Make sure to disable KSM (if enabled for the whole process or 308 * individual VMAs). Note that nothing currently hinders user space 309 * from re-enabling it. 310 */ 311 if (!rc) 312 rc = ksm_disable(mm); 313 if (rc) 314 mm->context.allow_cow_sharing = 1; 315 return rc; 316 } 317 EXPORT_SYMBOL_GPL(gmap_helper_disable_cow_sharing); 318