1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * SPARC64 Huge TLB page support. 4 * 5 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/mm.h> 10 #include <linux/sched/mm.h> 11 #include <linux/hugetlb.h> 12 #include <linux/pagemap.h> 13 #include <linux/sysctl.h> 14 15 #include <asm/mman.h> 16 #include <asm/pgalloc.h> 17 #include <asm/tlb.h> 18 #include <asm/tlbflush.h> 19 #include <asm/cacheflush.h> 20 #include <asm/mmu_context.h> 21 22 23 static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) 24 { 25 return entry; 26 } 27 28 static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) 29 { 30 unsigned long hugepage_size = _PAGE_SZ4MB_4V; 31 32 pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; 33 34 switch (shift) { 35 case HPAGE_16GB_SHIFT: 36 hugepage_size = _PAGE_SZ16GB_4V; 37 pte_val(entry) |= _PAGE_PUD_HUGE; 38 break; 39 case HPAGE_2GB_SHIFT: 40 hugepage_size = _PAGE_SZ2GB_4V; 41 pte_val(entry) |= _PAGE_PMD_HUGE; 42 break; 43 case HPAGE_256MB_SHIFT: 44 hugepage_size = _PAGE_SZ256MB_4V; 45 pte_val(entry) |= _PAGE_PMD_HUGE; 46 break; 47 case HPAGE_SHIFT: 48 pte_val(entry) |= _PAGE_PMD_HUGE; 49 break; 50 case HPAGE_64K_SHIFT: 51 hugepage_size = _PAGE_SZ64K_4V; 52 break; 53 default: 54 WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); 55 } 56 57 pte_val(entry) = pte_val(entry) | hugepage_size; 58 return entry; 59 } 60 61 static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift) 62 { 63 if (tlb_type == hypervisor) 64 return sun4v_hugepage_shift_to_tte(entry, shift); 65 else 66 return sun4u_hugepage_shift_to_tte(entry, shift); 67 } 68 69 pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) 70 { 71 pte_t pte; 72 73 entry = pte_mkhuge(entry); 74 pte = hugepage_shift_to_tte(entry, shift); 75 76 #ifdef CONFIG_SPARC64 77 /* If this vma has ADI enabled on it, turn on TTE.mcd 78 */ 79 if (flags & VM_SPARC_ADI) 80 return pte_mkmcd(pte); 81 else 82 return pte_mknotmcd(pte); 83 #else 84 return pte; 85 #endif 86 } 87 88 static unsigned int sun4v_huge_tte_to_shift(pte_t entry) 89 { 90 unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V; 91 unsigned int shift; 92 93 switch (tte_szbits) { 94 case _PAGE_SZ16GB_4V: 95 shift = HPAGE_16GB_SHIFT; 96 break; 97 case _PAGE_SZ2GB_4V: 98 shift = HPAGE_2GB_SHIFT; 99 break; 100 case _PAGE_SZ256MB_4V: 101 shift = HPAGE_256MB_SHIFT; 102 break; 103 case _PAGE_SZ4MB_4V: 104 shift = REAL_HPAGE_SHIFT; 105 break; 106 case _PAGE_SZ64K_4V: 107 shift = HPAGE_64K_SHIFT; 108 break; 109 default: 110 shift = PAGE_SHIFT; 111 break; 112 } 113 return shift; 114 } 115 116 static unsigned int sun4u_huge_tte_to_shift(pte_t entry) 117 { 118 unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U; 119 unsigned int shift; 120 121 switch (tte_szbits) { 122 case _PAGE_SZ256MB_4U: 123 shift = HPAGE_256MB_SHIFT; 124 break; 125 case _PAGE_SZ4MB_4U: 126 shift = REAL_HPAGE_SHIFT; 127 break; 128 case _PAGE_SZ64K_4U: 129 shift = HPAGE_64K_SHIFT; 130 break; 131 default: 132 shift = PAGE_SHIFT; 133 break; 134 } 135 return shift; 136 } 137 138 static unsigned long tte_to_shift(pte_t entry) 139 { 140 if (tlb_type == hypervisor) 141 return sun4v_huge_tte_to_shift(entry); 142 143 return sun4u_huge_tte_to_shift(entry); 144 } 145 146 static unsigned int huge_tte_to_shift(pte_t entry) 147 { 148 unsigned long shift = tte_to_shift(entry); 149 150 if (shift == PAGE_SHIFT) 151 WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n", 152 pte_val(entry)); 153 154 return shift; 155 } 156 157 static unsigned long huge_tte_to_size(pte_t pte) 158 { 159 unsigned long size = 1UL << huge_tte_to_shift(pte); 160 161 if (size == REAL_HPAGE_SIZE) 162 size = HPAGE_SIZE; 163 return size; 164 } 165 166 unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); } 167 unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); } 168 unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); } 169 170 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 171 unsigned long addr, unsigned long sz) 172 { 173 pgd_t *pgd; 174 p4d_t *p4d; 175 pud_t *pud; 176 pmd_t *pmd; 177 178 pgd = pgd_offset(mm, addr); 179 p4d = p4d_offset(pgd, addr); 180 pud = pud_alloc(mm, p4d, addr); 181 if (!pud) 182 return NULL; 183 if (sz >= PUD_SIZE) 184 return (pte_t *)pud; 185 pmd = pmd_alloc(mm, pud, addr); 186 if (!pmd) 187 return NULL; 188 if (sz >= PMD_SIZE) 189 return (pte_t *)pmd; 190 return pte_alloc_huge(mm, pmd, addr); 191 } 192 193 pte_t *huge_pte_offset(struct mm_struct *mm, 194 unsigned long addr, unsigned long sz) 195 { 196 pgd_t *pgd; 197 p4d_t *p4d; 198 pud_t *pud; 199 pmd_t *pmd; 200 201 pgd = pgd_offset(mm, addr); 202 if (pgd_none(*pgd)) 203 return NULL; 204 p4d = p4d_offset(pgd, addr); 205 if (p4d_none(*p4d)) 206 return NULL; 207 pud = pud_offset(p4d, addr); 208 if (pud_none(*pud)) 209 return NULL; 210 if (is_hugetlb_pud(*pud)) 211 return (pte_t *)pud; 212 pmd = pmd_offset(pud, addr); 213 if (pmd_none(*pmd)) 214 return NULL; 215 if (is_hugetlb_pmd(*pmd)) 216 return (pte_t *)pmd; 217 return pte_offset_huge(pmd, addr); 218 } 219 220 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 221 pte_t *ptep, pte_t entry) 222 { 223 unsigned int nptes, orig_shift, shift; 224 unsigned long i, size; 225 pte_t orig; 226 227 size = huge_tte_to_size(entry); 228 229 shift = PAGE_SHIFT; 230 if (size >= PUD_SIZE) 231 shift = PUD_SHIFT; 232 else if (size >= PMD_SIZE) 233 shift = PMD_SHIFT; 234 else 235 shift = PAGE_SHIFT; 236 237 nptes = size >> shift; 238 239 if (!pte_present(*ptep) && pte_present(entry)) 240 mm->context.hugetlb_pte_count += nptes; 241 242 addr &= ~(size - 1); 243 orig = *ptep; 244 orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig); 245 246 for (i = 0; i < nptes; i++) 247 ptep[i] = __pte(pte_val(entry) + (i << shift)); 248 249 maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift); 250 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ 251 if (size == HPAGE_SIZE) 252 maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0, 253 orig_shift); 254 } 255 256 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 257 pte_t *ptep, pte_t entry, unsigned long sz) 258 { 259 __set_huge_pte_at(mm, addr, ptep, entry); 260 } 261 262 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, 263 pte_t *ptep) 264 { 265 unsigned int i, nptes, orig_shift, shift; 266 unsigned long size; 267 pte_t entry; 268 269 entry = *ptep; 270 size = huge_tte_to_size(entry); 271 272 shift = PAGE_SHIFT; 273 if (size >= PUD_SIZE) 274 shift = PUD_SHIFT; 275 else if (size >= PMD_SIZE) 276 shift = PMD_SHIFT; 277 else 278 shift = PAGE_SHIFT; 279 280 nptes = size >> shift; 281 orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry); 282 283 if (pte_present(entry)) 284 mm->context.hugetlb_pte_count -= nptes; 285 286 addr &= ~(size - 1); 287 for (i = 0; i < nptes; i++) 288 ptep[i] = __pte(0UL); 289 290 maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift); 291 /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ 292 if (size == HPAGE_SIZE) 293 maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, 294 orig_shift); 295 296 return entry; 297 } 298 299 static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 300 unsigned long addr) 301 { 302 pgtable_t token = pmd_pgtable(*pmd); 303 304 pmd_clear(pmd); 305 pte_free_tlb(tlb, token, addr); 306 mm_dec_nr_ptes(tlb->mm); 307 } 308 309 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 310 unsigned long addr, unsigned long end, 311 unsigned long floor, unsigned long ceiling) 312 { 313 pmd_t *pmd; 314 unsigned long next; 315 unsigned long start; 316 317 start = addr; 318 pmd = pmd_offset(pud, addr); 319 do { 320 next = pmd_addr_end(addr, end); 321 if (pmd_none(*pmd)) 322 continue; 323 if (is_hugetlb_pmd(*pmd)) 324 pmd_clear(pmd); 325 else 326 hugetlb_free_pte_range(tlb, pmd, addr); 327 } while (pmd++, addr = next, addr != end); 328 329 start &= PUD_MASK; 330 if (start < floor) 331 return; 332 if (ceiling) { 333 ceiling &= PUD_MASK; 334 if (!ceiling) 335 return; 336 } 337 if (end - 1 > ceiling - 1) 338 return; 339 340 pmd = pmd_offset(pud, start); 341 pud_clear(pud); 342 pmd_free_tlb(tlb, pmd, start); 343 mm_dec_nr_pmds(tlb->mm); 344 } 345 346 static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, 347 unsigned long addr, unsigned long end, 348 unsigned long floor, unsigned long ceiling) 349 { 350 pud_t *pud; 351 unsigned long next; 352 unsigned long start; 353 354 start = addr; 355 pud = pud_offset(p4d, addr); 356 do { 357 next = pud_addr_end(addr, end); 358 if (pud_none_or_clear_bad(pud)) 359 continue; 360 if (is_hugetlb_pud(*pud)) 361 pud_clear(pud); 362 else 363 hugetlb_free_pmd_range(tlb, pud, addr, next, floor, 364 ceiling); 365 } while (pud++, addr = next, addr != end); 366 367 start &= PGDIR_MASK; 368 if (start < floor) 369 return; 370 if (ceiling) { 371 ceiling &= PGDIR_MASK; 372 if (!ceiling) 373 return; 374 } 375 if (end - 1 > ceiling - 1) 376 return; 377 378 pud = pud_offset(p4d, start); 379 p4d_clear(p4d); 380 pud_free_tlb(tlb, pud, start); 381 mm_dec_nr_puds(tlb->mm); 382 } 383 384 void hugetlb_free_pgd_range(struct mmu_gather *tlb, 385 unsigned long addr, unsigned long end, 386 unsigned long floor, unsigned long ceiling) 387 { 388 pgd_t *pgd; 389 p4d_t *p4d; 390 unsigned long next; 391 392 addr &= PMD_MASK; 393 if (addr < floor) { 394 addr += PMD_SIZE; 395 if (!addr) 396 return; 397 } 398 if (ceiling) { 399 ceiling &= PMD_MASK; 400 if (!ceiling) 401 return; 402 } 403 if (end - 1 > ceiling - 1) 404 end -= PMD_SIZE; 405 if (addr > end - 1) 406 return; 407 408 pgd = pgd_offset(tlb->mm, addr); 409 p4d = p4d_offset(pgd, addr); 410 do { 411 next = p4d_addr_end(addr, end); 412 if (p4d_none_or_clear_bad(p4d)) 413 continue; 414 hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); 415 } while (p4d++, addr = next, addr != end); 416 } 417