1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2007, 2011 4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 5 */ 6 7 #include <linux/cpufeature.h> 8 #include <linux/export.h> 9 #include <linux/sched.h> 10 #include <linux/kernel.h> 11 #include <linux/errno.h> 12 #include <linux/gfp.h> 13 #include <linux/mm.h> 14 #include <linux/swap.h> 15 #include <linux/smp.h> 16 #include <linux/spinlock.h> 17 #include <linux/rcupdate.h> 18 #include <linux/slab.h> 19 #include <linux/leafops.h> 20 #include <linux/sysctl.h> 21 #include <linux/ksm.h> 22 #include <linux/mman.h> 23 24 #include <asm/tlbflush.h> 25 #include <asm/mmu_context.h> 26 #include <asm/page-states.h> 27 #include <asm/machine.h> 28 29 pgprot_t pgprot_writecombine(pgprot_t prot) 30 { 31 /* 32 * mio_wb_bit_mask may be set on a different CPU, but it is only set 33 * once at init and only read afterwards. 34 */ 35 return __pgprot(pgprot_val(prot) | mio_wb_bit_mask); 36 } 37 EXPORT_SYMBOL_GPL(pgprot_writecombine); 38 39 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, 40 pte_t *ptep, int nodat) 41 { 42 unsigned long opt, asce; 43 44 if (machine_has_tlb_guest()) { 45 opt = 0; 46 asce = READ_ONCE(mm->context.gmap_asce); 47 if (asce == 0UL || nodat) 48 opt |= IPTE_NODAT; 49 if (asce != -1UL) { 50 asce = asce ? : mm->context.asce; 51 opt |= IPTE_GUEST_ASCE; 52 } 53 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); 54 } else { 55 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); 56 } 57 } 58 59 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, 60 pte_t *ptep, int nodat) 61 { 62 unsigned long opt, asce; 63 64 if (machine_has_tlb_guest()) { 65 opt = 0; 66 asce = READ_ONCE(mm->context.gmap_asce); 67 if (asce == 0UL || nodat) 68 opt |= IPTE_NODAT; 69 if (asce != -1UL) { 70 asce = asce ? : mm->context.asce; 71 opt |= IPTE_GUEST_ASCE; 72 } 73 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); 74 } else { 75 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); 76 } 77 } 78 79 static inline pte_t ptep_flush_direct(struct mm_struct *mm, 80 unsigned long addr, pte_t *ptep, 81 int nodat) 82 { 83 pte_t old; 84 85 old = *ptep; 86 if (unlikely(pte_val(old) & _PAGE_INVALID)) 87 return old; 88 atomic_inc(&mm->context.flush_count); 89 if (cpu_has_tlb_lc() && 90 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 91 ptep_ipte_local(mm, addr, ptep, nodat); 92 else 93 ptep_ipte_global(mm, addr, ptep, nodat); 94 atomic_dec(&mm->context.flush_count); 95 return old; 96 } 97 98 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, 99 unsigned long addr, pte_t *ptep, 100 int nodat) 101 { 102 pte_t old; 103 104 old = *ptep; 105 if (unlikely(pte_val(old) & _PAGE_INVALID)) 106 return old; 107 atomic_inc(&mm->context.flush_count); 108 if (cpumask_equal(&mm->context.cpu_attach_mask, 109 cpumask_of(smp_processor_id()))) { 110 set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID))); 111 mm->context.flush_mm = 1; 112 } else 113 ptep_ipte_global(mm, addr, ptep, nodat); 114 atomic_dec(&mm->context.flush_count); 115 return old; 116 } 117 118 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, 119 pte_t *ptep, pte_t new) 120 { 121 pte_t old; 122 123 preempt_disable(); 124 old = ptep_flush_direct(mm, addr, ptep, 1); 125 set_pte(ptep, new); 126 preempt_enable(); 127 return old; 128 } 129 EXPORT_SYMBOL(ptep_xchg_direct); 130 131 /* 132 * Caller must check that new PTE only differs in _PAGE_PROTECT HW bit, so that 133 * RDP can be used instead of IPTE. See also comments at pte_allow_rdp(). 134 */ 135 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep, 136 pte_t new) 137 { 138 preempt_disable(); 139 atomic_inc(&mm->context.flush_count); 140 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 141 __ptep_rdp(addr, ptep, 1); 142 else 143 __ptep_rdp(addr, ptep, 0); 144 /* 145 * PTE is not invalidated by RDP, only _PAGE_PROTECT is cleared. That 146 * means it is still valid and active, and must not be changed according 147 * to the architecture. But writing a new value that only differs in SW 148 * bits is allowed. 149 */ 150 set_pte(ptep, new); 151 atomic_dec(&mm->context.flush_count); 152 preempt_enable(); 153 } 154 EXPORT_SYMBOL(ptep_reset_dat_prot); 155 156 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, 157 pte_t *ptep, pte_t new) 158 { 159 pte_t old; 160 161 preempt_disable(); 162 old = ptep_flush_lazy(mm, addr, ptep, 1); 163 set_pte(ptep, new); 164 preempt_enable(); 165 return old; 166 } 167 EXPORT_SYMBOL(ptep_xchg_lazy); 168 169 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, 170 pte_t *ptep) 171 { 172 return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1); 173 } 174 175 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, 176 pte_t *ptep, pte_t old_pte, pte_t pte) 177 { 178 set_pte(ptep, pte); 179 } 180 181 static inline void pmdp_idte_local(struct mm_struct *mm, 182 unsigned long addr, pmd_t *pmdp) 183 { 184 if (machine_has_tlb_guest()) 185 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_LOCAL); 186 else 187 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); 188 } 189 190 static inline void pmdp_idte_global(struct mm_struct *mm, 191 unsigned long addr, pmd_t *pmdp) 192 { 193 if (machine_has_tlb_guest()) { 194 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, 195 mm->context.asce, IDTE_GLOBAL); 196 } else { 197 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); 198 } 199 } 200 201 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, 202 unsigned long addr, pmd_t *pmdp) 203 { 204 pmd_t old; 205 206 old = *pmdp; 207 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) 208 return old; 209 atomic_inc(&mm->context.flush_count); 210 if (cpu_has_tlb_lc() && 211 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 212 pmdp_idte_local(mm, addr, pmdp); 213 else 214 pmdp_idte_global(mm, addr, pmdp); 215 atomic_dec(&mm->context.flush_count); 216 return old; 217 } 218 219 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, 220 unsigned long addr, pmd_t *pmdp) 221 { 222 pmd_t old; 223 224 old = *pmdp; 225 if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) 226 return old; 227 atomic_inc(&mm->context.flush_count); 228 if (cpumask_equal(&mm->context.cpu_attach_mask, 229 cpumask_of(smp_processor_id()))) { 230 set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); 231 mm->context.flush_mm = 1; 232 } else { 233 pmdp_idte_global(mm, addr, pmdp); 234 } 235 atomic_dec(&mm->context.flush_count); 236 return old; 237 } 238 239 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, 240 pmd_t *pmdp, pmd_t new) 241 { 242 pmd_t old; 243 244 preempt_disable(); 245 old = pmdp_flush_direct(mm, addr, pmdp); 246 set_pmd(pmdp, new); 247 preempt_enable(); 248 return old; 249 } 250 EXPORT_SYMBOL(pmdp_xchg_direct); 251 252 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, 253 pmd_t *pmdp, pmd_t new) 254 { 255 pmd_t old; 256 257 preempt_disable(); 258 old = pmdp_flush_lazy(mm, addr, pmdp); 259 set_pmd(pmdp, new); 260 preempt_enable(); 261 return old; 262 } 263 EXPORT_SYMBOL(pmdp_xchg_lazy); 264 265 static inline void pudp_idte_local(struct mm_struct *mm, 266 unsigned long addr, pud_t *pudp) 267 { 268 if (machine_has_tlb_guest()) 269 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, 270 mm->context.asce, IDTE_LOCAL); 271 else 272 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); 273 } 274 275 static inline void pudp_idte_global(struct mm_struct *mm, 276 unsigned long addr, pud_t *pudp) 277 { 278 if (machine_has_tlb_guest()) 279 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, 280 mm->context.asce, IDTE_GLOBAL); 281 else 282 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); 283 } 284 285 static inline pud_t pudp_flush_direct(struct mm_struct *mm, 286 unsigned long addr, pud_t *pudp) 287 { 288 pud_t old; 289 290 old = *pudp; 291 if (pud_val(old) & _REGION_ENTRY_INVALID) 292 return old; 293 atomic_inc(&mm->context.flush_count); 294 if (cpu_has_tlb_lc() && 295 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) 296 pudp_idte_local(mm, addr, pudp); 297 else 298 pudp_idte_global(mm, addr, pudp); 299 atomic_dec(&mm->context.flush_count); 300 return old; 301 } 302 303 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, 304 pud_t *pudp, pud_t new) 305 { 306 pud_t old; 307 308 preempt_disable(); 309 old = pudp_flush_direct(mm, addr, pudp); 310 set_pud(pudp, new); 311 preempt_enable(); 312 return old; 313 } 314 EXPORT_SYMBOL(pudp_xchg_direct); 315 316 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 317 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 318 pgtable_t pgtable) 319 { 320 struct list_head *lh = (struct list_head *) pgtable; 321 322 assert_spin_locked(pmd_lockptr(mm, pmdp)); 323 324 /* FIFO */ 325 if (!pmd_huge_pte(mm, pmdp)) 326 INIT_LIST_HEAD(lh); 327 else 328 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); 329 pmd_huge_pte(mm, pmdp) = pgtable; 330 } 331 332 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 333 { 334 struct list_head *lh; 335 pgtable_t pgtable; 336 pte_t *ptep; 337 338 assert_spin_locked(pmd_lockptr(mm, pmdp)); 339 340 /* FIFO */ 341 pgtable = pmd_huge_pte(mm, pmdp); 342 lh = (struct list_head *) pgtable; 343 if (list_empty(lh)) 344 pmd_huge_pte(mm, pmdp) = NULL; 345 else { 346 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; 347 list_del(lh); 348 } 349 ptep = (pte_t *) pgtable; 350 set_pte(ptep, __pte(_PAGE_INVALID)); 351 ptep++; 352 set_pte(ptep, __pte(_PAGE_INVALID)); 353 return pgtable; 354 } 355 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 356