1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/mm.h> 3 #include <linux/gfp.h> 4 #include <linux/hugetlb.h> 5 #include <asm/pgalloc.h> 6 #include <asm/tlb.h> 7 #include <asm/fixmap.h> 8 #include <asm/mtrr.h> 9 10 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK 11 phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; 12 EXPORT_SYMBOL(physical_mask); 13 #endif 14 15 #ifdef CONFIG_HIGHPTE 16 #define PGTABLE_HIGHMEM __GFP_HIGHMEM 17 #else 18 #define PGTABLE_HIGHMEM 0 19 #endif 20 21 #ifndef CONFIG_PARAVIRT 22 static inline 23 void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) 24 { 25 tlb_remove_page(tlb, table); 26 } 27 #endif 28 29 gfp_t __userpte_alloc_gfp = GFP_PGTABLE_USER | PGTABLE_HIGHMEM; 30 31 pgtable_t pte_alloc_one(struct mm_struct *mm) 32 { 33 return __pte_alloc_one(mm, __userpte_alloc_gfp); 34 } 35 36 static int __init setup_userpte(char *arg) 37 { 38 if (!arg) 39 return -EINVAL; 40 41 /* 42 * "userpte=nohigh" disables allocation of user pagetables in 43 * high memory. 44 */ 45 if (strcmp(arg, "nohigh") == 0) 46 __userpte_alloc_gfp &= ~__GFP_HIGHMEM; 47 else 48 return -EINVAL; 49 return 0; 50 } 51 early_param("userpte", setup_userpte); 52 53 void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) 54 { 55 pagetable_pte_dtor(page_ptdesc(pte)); 56 paravirt_release_pte(page_to_pfn(pte)); 57 paravirt_tlb_remove_table(tlb, pte); 58 } 59 60 #if CONFIG_PGTABLE_LEVELS > 2 61 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) 62 { 63 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); 64 paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT); 65 /* 66 * NOTE! For PAE, any changes to the top page-directory-pointer-table 67 * entries need a full cr3 reload to flush. 68 */ 69 #ifdef CONFIG_X86_PAE 70 tlb->need_flush_all = 1; 71 #endif 72 pagetable_pmd_dtor(ptdesc); 73 paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc)); 74 } 75 76 #if CONFIG_PGTABLE_LEVELS > 3 77 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) 78 { 79 paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); 80 paravirt_tlb_remove_table(tlb, virt_to_page(pud)); 81 } 82 83 #if CONFIG_PGTABLE_LEVELS > 4 84 void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) 85 { 86 paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); 87 paravirt_tlb_remove_table(tlb, virt_to_page(p4d)); 88 } 89 #endif /* CONFIG_PGTABLE_LEVELS > 4 */ 90 #endif /* CONFIG_PGTABLE_LEVELS > 3 */ 91 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 92 93 static inline void pgd_list_add(pgd_t *pgd) 94 { 95 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); 96 97 list_add(&ptdesc->pt_list, &pgd_list); 98 } 99 100 static inline void pgd_list_del(pgd_t *pgd) 101 { 102 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); 103 104 list_del(&ptdesc->pt_list); 105 } 106 107 #define UNSHARED_PTRS_PER_PGD \ 108 (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) 109 #define MAX_UNSHARED_PTRS_PER_PGD \ 110 max_t(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) 111 112 113 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) 114 { 115 virt_to_ptdesc(pgd)->pt_mm = mm; 116 } 117 118 struct mm_struct *pgd_page_get_mm(struct page *page) 119 { 120 return page_ptdesc(page)->pt_mm; 121 } 122 123 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) 124 { 125 /* If the pgd points to a shared pagetable level (either the 126 ptes in non-PAE, or shared PMD in PAE), then just copy the 127 references from swapper_pg_dir. */ 128 if (CONFIG_PGTABLE_LEVELS == 2 || 129 (CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) || 130 CONFIG_PGTABLE_LEVELS >= 4) { 131 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, 132 swapper_pg_dir + KERNEL_PGD_BOUNDARY, 133 KERNEL_PGD_PTRS); 134 } 135 136 /* list required to sync kernel mapping updates */ 137 if (!SHARED_KERNEL_PMD) { 138 pgd_set_mm(pgd, mm); 139 pgd_list_add(pgd); 140 } 141 } 142 143 static void pgd_dtor(pgd_t *pgd) 144 { 145 if (SHARED_KERNEL_PMD) 146 return; 147 148 spin_lock(&pgd_lock); 149 pgd_list_del(pgd); 150 spin_unlock(&pgd_lock); 151 } 152 153 /* 154 * List of all pgd's needed for non-PAE so it can invalidate entries 155 * in both cached and uncached pgd's; not needed for PAE since the 156 * kernel pmd is shared. If PAE were not to share the pmd a similar 157 * tactic would be needed. This is essentially codepath-based locking 158 * against pageattr.c; it is the unique case in which a valid change 159 * of kernel pagetables can't be lazily synchronized by vmalloc faults. 160 * vmalloc faults work because attached pagetables are never freed. 161 * -- nyc 162 */ 163 164 #ifdef CONFIG_X86_PAE 165 /* 166 * In PAE mode, we need to do a cr3 reload (=tlb flush) when 167 * updating the top-level pagetable entries to guarantee the 168 * processor notices the update. Since this is expensive, and 169 * all 4 top-level entries are used almost immediately in a 170 * new process's life, we just pre-populate them here. 171 * 172 * Also, if we're in a paravirt environment where the kernel pmd is 173 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate 174 * and initialize the kernel pmds here. 175 */ 176 #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD 177 #define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD 178 179 /* 180 * We allocate separate PMDs for the kernel part of the user page-table 181 * when PTI is enabled. We need them to map the per-process LDT into the 182 * user-space page-table. 183 */ 184 #define PREALLOCATED_USER_PMDS (boot_cpu_has(X86_FEATURE_PTI) ? \ 185 KERNEL_PGD_PTRS : 0) 186 #define MAX_PREALLOCATED_USER_PMDS KERNEL_PGD_PTRS 187 188 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) 189 { 190 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 191 192 /* Note: almost everything apart from _PAGE_PRESENT is 193 reserved at the pmd (PDPT) level. */ 194 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); 195 196 /* 197 * According to Intel App note "TLBs, Paging-Structure Caches, 198 * and Their Invalidation", April 2007, document 317080-001, 199 * section 8.1: in PAE mode we explicitly have to flush the 200 * TLB via cr3 if the top-level pgd is changed... 201 */ 202 flush_tlb_mm(mm); 203 } 204 #else /* !CONFIG_X86_PAE */ 205 206 /* No need to prepopulate any pagetable entries in non-PAE modes. */ 207 #define PREALLOCATED_PMDS 0 208 #define MAX_PREALLOCATED_PMDS 0 209 #define PREALLOCATED_USER_PMDS 0 210 #define MAX_PREALLOCATED_USER_PMDS 0 211 #endif /* CONFIG_X86_PAE */ 212 213 static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) 214 { 215 int i; 216 struct ptdesc *ptdesc; 217 218 for (i = 0; i < count; i++) 219 if (pmds[i]) { 220 ptdesc = virt_to_ptdesc(pmds[i]); 221 222 pagetable_pmd_dtor(ptdesc); 223 pagetable_free(ptdesc); 224 mm_dec_nr_pmds(mm); 225 } 226 } 227 228 static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) 229 { 230 int i; 231 bool failed = false; 232 gfp_t gfp = GFP_PGTABLE_USER; 233 234 if (mm == &init_mm) 235 gfp &= ~__GFP_ACCOUNT; 236 gfp &= ~__GFP_HIGHMEM; 237 238 for (i = 0; i < count; i++) { 239 pmd_t *pmd = NULL; 240 struct ptdesc *ptdesc = pagetable_alloc(gfp, 0); 241 242 if (!ptdesc) 243 failed = true; 244 if (ptdesc && !pagetable_pmd_ctor(ptdesc)) { 245 pagetable_free(ptdesc); 246 ptdesc = NULL; 247 failed = true; 248 } 249 if (ptdesc) { 250 mm_inc_nr_pmds(mm); 251 pmd = ptdesc_address(ptdesc); 252 } 253 254 pmds[i] = pmd; 255 } 256 257 if (failed) { 258 free_pmds(mm, pmds, count); 259 return -ENOMEM; 260 } 261 262 return 0; 263 } 264 265 /* 266 * Mop up any pmd pages which may still be attached to the pgd. 267 * Normally they will be freed by munmap/exit_mmap, but any pmd we 268 * preallocate which never got a corresponding vma will need to be 269 * freed manually. 270 */ 271 static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) 272 { 273 pgd_t pgd = *pgdp; 274 275 if (pgd_val(pgd) != 0) { 276 pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); 277 278 pgd_clear(pgdp); 279 280 paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); 281 pmd_free(mm, pmd); 282 mm_dec_nr_pmds(mm); 283 } 284 } 285 286 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) 287 { 288 int i; 289 290 for (i = 0; i < PREALLOCATED_PMDS; i++) 291 mop_up_one_pmd(mm, &pgdp[i]); 292 293 #ifdef CONFIG_PAGE_TABLE_ISOLATION 294 295 if (!boot_cpu_has(X86_FEATURE_PTI)) 296 return; 297 298 pgdp = kernel_to_user_pgdp(pgdp); 299 300 for (i = 0; i < PREALLOCATED_USER_PMDS; i++) 301 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); 302 #endif 303 } 304 305 static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[]) 306 { 307 p4d_t *p4d; 308 pud_t *pud; 309 int i; 310 311 p4d = p4d_offset(pgd, 0); 312 pud = pud_offset(p4d, 0); 313 314 for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) { 315 pmd_t *pmd = pmds[i]; 316 317 if (i >= KERNEL_PGD_BOUNDARY) 318 memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]), 319 sizeof(pmd_t) * PTRS_PER_PMD); 320 321 pud_populate(mm, pud, pmd); 322 } 323 } 324 325 #ifdef CONFIG_PAGE_TABLE_ISOLATION 326 static void pgd_prepopulate_user_pmd(struct mm_struct *mm, 327 pgd_t *k_pgd, pmd_t *pmds[]) 328 { 329 pgd_t *s_pgd = kernel_to_user_pgdp(swapper_pg_dir); 330 pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); 331 p4d_t *u_p4d; 332 pud_t *u_pud; 333 int i; 334 335 u_p4d = p4d_offset(u_pgd, 0); 336 u_pud = pud_offset(u_p4d, 0); 337 338 s_pgd += KERNEL_PGD_BOUNDARY; 339 u_pud += KERNEL_PGD_BOUNDARY; 340 341 for (i = 0; i < PREALLOCATED_USER_PMDS; i++, u_pud++, s_pgd++) { 342 pmd_t *pmd = pmds[i]; 343 344 memcpy(pmd, (pmd_t *)pgd_page_vaddr(*s_pgd), 345 sizeof(pmd_t) * PTRS_PER_PMD); 346 347 pud_populate(mm, u_pud, pmd); 348 } 349 350 } 351 #else 352 static void pgd_prepopulate_user_pmd(struct mm_struct *mm, 353 pgd_t *k_pgd, pmd_t *pmds[]) 354 { 355 } 356 #endif 357 /* 358 * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also 359 * assumes that pgd should be in one page. 360 * 361 * But kernel with PAE paging that is not running as a Xen domain 362 * only needs to allocate 32 bytes for pgd instead of one page. 363 */ 364 #ifdef CONFIG_X86_PAE 365 366 #include <linux/slab.h> 367 368 #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 369 #define PGD_ALIGN 32 370 371 static struct kmem_cache *pgd_cache; 372 373 void __init pgtable_cache_init(void) 374 { 375 /* 376 * When PAE kernel is running as a Xen domain, it does not use 377 * shared kernel pmd. And this requires a whole page for pgd. 378 */ 379 if (!SHARED_KERNEL_PMD) 380 return; 381 382 /* 383 * when PAE kernel is not running as a Xen domain, it uses 384 * shared kernel pmd. Shared kernel pmd does not require a whole 385 * page for pgd. We are able to just allocate a 32-byte for pgd. 386 * During boot time, we create a 32-byte slab for pgd table allocation. 387 */ 388 pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN, 389 SLAB_PANIC, NULL); 390 } 391 392 static inline pgd_t *_pgd_alloc(void) 393 { 394 /* 395 * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain. 396 * We allocate one page for pgd. 397 */ 398 if (!SHARED_KERNEL_PMD) 399 return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, 400 PGD_ALLOCATION_ORDER); 401 402 /* 403 * Now PAE kernel is not running as a Xen domain. We can allocate 404 * a 32-byte slab for pgd to save memory space. 405 */ 406 return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER); 407 } 408 409 static inline void _pgd_free(pgd_t *pgd) 410 { 411 if (!SHARED_KERNEL_PMD) 412 free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); 413 else 414 kmem_cache_free(pgd_cache, pgd); 415 } 416 #else 417 418 static inline pgd_t *_pgd_alloc(void) 419 { 420 return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, 421 PGD_ALLOCATION_ORDER); 422 } 423 424 static inline void _pgd_free(pgd_t *pgd) 425 { 426 free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); 427 } 428 #endif /* CONFIG_X86_PAE */ 429 430 pgd_t *pgd_alloc(struct mm_struct *mm) 431 { 432 pgd_t *pgd; 433 pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS]; 434 pmd_t *pmds[MAX_PREALLOCATED_PMDS]; 435 436 pgd = _pgd_alloc(); 437 438 if (pgd == NULL) 439 goto out; 440 441 mm->pgd = pgd; 442 443 if (sizeof(pmds) != 0 && 444 preallocate_pmds(mm, pmds, PREALLOCATED_PMDS) != 0) 445 goto out_free_pgd; 446 447 if (sizeof(u_pmds) != 0 && 448 preallocate_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS) != 0) 449 goto out_free_pmds; 450 451 if (paravirt_pgd_alloc(mm) != 0) 452 goto out_free_user_pmds; 453 454 /* 455 * Make sure that pre-populating the pmds is atomic with 456 * respect to anything walking the pgd_list, so that they 457 * never see a partially populated pgd. 458 */ 459 spin_lock(&pgd_lock); 460 461 pgd_ctor(mm, pgd); 462 if (sizeof(pmds) != 0) 463 pgd_prepopulate_pmd(mm, pgd, pmds); 464 465 if (sizeof(u_pmds) != 0) 466 pgd_prepopulate_user_pmd(mm, pgd, u_pmds); 467 468 spin_unlock(&pgd_lock); 469 470 return pgd; 471 472 out_free_user_pmds: 473 if (sizeof(u_pmds) != 0) 474 free_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS); 475 out_free_pmds: 476 if (sizeof(pmds) != 0) 477 free_pmds(mm, pmds, PREALLOCATED_PMDS); 478 out_free_pgd: 479 _pgd_free(pgd); 480 out: 481 return NULL; 482 } 483 484 void pgd_free(struct mm_struct *mm, pgd_t *pgd) 485 { 486 pgd_mop_up_pmds(mm, pgd); 487 pgd_dtor(pgd); 488 paravirt_pgd_free(mm, pgd); 489 _pgd_free(pgd); 490 } 491 492 /* 493 * Used to set accessed or dirty bits in the page table entries 494 * on other architectures. On x86, the accessed and dirty bits 495 * are tracked by hardware. However, do_wp_page calls this function 496 * to also make the pte writeable at the same time the dirty bit is 497 * set. In that case we do actually need to write the PTE. 498 */ 499 int ptep_set_access_flags(struct vm_area_struct *vma, 500 unsigned long address, pte_t *ptep, 501 pte_t entry, int dirty) 502 { 503 int changed = !pte_same(*ptep, entry); 504 505 if (changed && dirty) 506 set_pte(ptep, entry); 507 508 return changed; 509 } 510 511 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 512 int pmdp_set_access_flags(struct vm_area_struct *vma, 513 unsigned long address, pmd_t *pmdp, 514 pmd_t entry, int dirty) 515 { 516 int changed = !pmd_same(*pmdp, entry); 517 518 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 519 520 if (changed && dirty) { 521 set_pmd(pmdp, entry); 522 /* 523 * We had a write-protection fault here and changed the pmd 524 * to to more permissive. No need to flush the TLB for that, 525 * #PF is architecturally guaranteed to do that and in the 526 * worst-case we'll generate a spurious fault. 527 */ 528 } 529 530 return changed; 531 } 532 533 int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, 534 pud_t *pudp, pud_t entry, int dirty) 535 { 536 int changed = !pud_same(*pudp, entry); 537 538 VM_BUG_ON(address & ~HPAGE_PUD_MASK); 539 540 if (changed && dirty) { 541 set_pud(pudp, entry); 542 /* 543 * We had a write-protection fault here and changed the pud 544 * to to more permissive. No need to flush the TLB for that, 545 * #PF is architecturally guaranteed to do that and in the 546 * worst-case we'll generate a spurious fault. 547 */ 548 } 549 550 return changed; 551 } 552 #endif 553 554 int ptep_test_and_clear_young(struct vm_area_struct *vma, 555 unsigned long addr, pte_t *ptep) 556 { 557 int ret = 0; 558 559 if (pte_young(*ptep)) 560 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 561 (unsigned long *) &ptep->pte); 562 563 return ret; 564 } 565 566 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 567 int pmdp_test_and_clear_young(struct vm_area_struct *vma, 568 unsigned long addr, pmd_t *pmdp) 569 { 570 int ret = 0; 571 572 if (pmd_young(*pmdp)) 573 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 574 (unsigned long *)pmdp); 575 576 return ret; 577 } 578 #endif 579 580 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 581 int pudp_test_and_clear_young(struct vm_area_struct *vma, 582 unsigned long addr, pud_t *pudp) 583 { 584 int ret = 0; 585 586 if (pud_young(*pudp)) 587 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 588 (unsigned long *)pudp); 589 590 return ret; 591 } 592 #endif 593 594 int ptep_clear_flush_young(struct vm_area_struct *vma, 595 unsigned long address, pte_t *ptep) 596 { 597 /* 598 * On x86 CPUs, clearing the accessed bit without a TLB flush 599 * doesn't cause data corruption. [ It could cause incorrect 600 * page aging and the (mistaken) reclaim of hot pages, but the 601 * chance of that should be relatively low. ] 602 * 603 * So as a performance optimization don't flush the TLB when 604 * clearing the accessed bit, it will eventually be flushed by 605 * a context switch or a VM operation anyway. [ In the rare 606 * event of it not getting flushed for a long time the delay 607 * shouldn't really matter because there's no real memory 608 * pressure for swapout to react to. ] 609 */ 610 return ptep_test_and_clear_young(vma, address, ptep); 611 } 612 613 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 614 int pmdp_clear_flush_young(struct vm_area_struct *vma, 615 unsigned long address, pmd_t *pmdp) 616 { 617 int young; 618 619 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 620 621 young = pmdp_test_and_clear_young(vma, address, pmdp); 622 if (young) 623 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 624 625 return young; 626 } 627 628 pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, 629 pmd_t *pmdp) 630 { 631 /* 632 * No flush is necessary. Once an invalid PTE is established, the PTE's 633 * access and dirty bits cannot be updated. 634 */ 635 return pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); 636 } 637 #endif 638 639 /** 640 * reserve_top_address - reserves a hole in the top of kernel address space 641 * @reserve - size of hole to reserve 642 * 643 * Can be used to relocate the fixmap area and poke a hole in the top 644 * of kernel address space to make room for a hypervisor. 645 */ 646 void __init reserve_top_address(unsigned long reserve) 647 { 648 #ifdef CONFIG_X86_32 649 BUG_ON(fixmaps_set > 0); 650 __FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE; 651 printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n", 652 -reserve, __FIXADDR_TOP + PAGE_SIZE); 653 #endif 654 } 655 656 int fixmaps_set; 657 658 void __native_set_fixmap(enum fixed_addresses idx, pte_t pte) 659 { 660 unsigned long address = __fix_to_virt(idx); 661 662 #ifdef CONFIG_X86_64 663 /* 664 * Ensure that the static initial page tables are covering the 665 * fixmap completely. 666 */ 667 BUILD_BUG_ON(__end_of_permanent_fixed_addresses > 668 (FIXMAP_PMD_NUM * PTRS_PER_PTE)); 669 #endif 670 671 if (idx >= __end_of_fixed_addresses) { 672 BUG(); 673 return; 674 } 675 set_pte_vaddr(address, pte); 676 fixmaps_set++; 677 } 678 679 void native_set_fixmap(unsigned /* enum fixed_addresses */ idx, 680 phys_addr_t phys, pgprot_t flags) 681 { 682 /* Sanitize 'prot' against any unsupported bits: */ 683 pgprot_val(flags) &= __default_kernel_pte_mask; 684 685 __native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags)); 686 } 687 688 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 689 #ifdef CONFIG_X86_5LEVEL 690 /** 691 * p4d_set_huge - setup kernel P4D mapping 692 * 693 * No 512GB pages yet -- always return 0 694 */ 695 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 696 { 697 return 0; 698 } 699 700 /** 701 * p4d_clear_huge - clear kernel P4D mapping when it is set 702 * 703 * No 512GB pages yet -- always return 0 704 */ 705 void p4d_clear_huge(p4d_t *p4d) 706 { 707 } 708 #endif 709 710 /** 711 * pud_set_huge - setup kernel PUD mapping 712 * 713 * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this 714 * function sets up a huge page only if the complete range has the same MTRR 715 * caching mode. 716 * 717 * Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger 718 * page mapping attempt fails. 719 * 720 * Returns 1 on success and 0 on failure. 721 */ 722 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 723 { 724 u8 uniform; 725 726 mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform); 727 if (!uniform) 728 return 0; 729 730 /* Bail out if we are we on a populated non-leaf entry: */ 731 if (pud_present(*pud) && !pud_huge(*pud)) 732 return 0; 733 734 set_pte((pte_t *)pud, pfn_pte( 735 (u64)addr >> PAGE_SHIFT, 736 __pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE))); 737 738 return 1; 739 } 740 741 /** 742 * pmd_set_huge - setup kernel PMD mapping 743 * 744 * See text over pud_set_huge() above. 745 * 746 * Returns 1 on success and 0 on failure. 747 */ 748 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 749 { 750 u8 uniform; 751 752 mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform); 753 if (!uniform) { 754 pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n", 755 __func__, addr, addr + PMD_SIZE); 756 return 0; 757 } 758 759 /* Bail out if we are we on a populated non-leaf entry: */ 760 if (pmd_present(*pmd) && !pmd_huge(*pmd)) 761 return 0; 762 763 set_pte((pte_t *)pmd, pfn_pte( 764 (u64)addr >> PAGE_SHIFT, 765 __pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE))); 766 767 return 1; 768 } 769 770 /** 771 * pud_clear_huge - clear kernel PUD mapping when it is set 772 * 773 * Returns 1 on success and 0 on failure (no PUD map is found). 774 */ 775 int pud_clear_huge(pud_t *pud) 776 { 777 if (pud_large(*pud)) { 778 pud_clear(pud); 779 return 1; 780 } 781 782 return 0; 783 } 784 785 /** 786 * pmd_clear_huge - clear kernel PMD mapping when it is set 787 * 788 * Returns 1 on success and 0 on failure (no PMD map is found). 789 */ 790 int pmd_clear_huge(pmd_t *pmd) 791 { 792 if (pmd_large(*pmd)) { 793 pmd_clear(pmd); 794 return 1; 795 } 796 797 return 0; 798 } 799 800 #ifdef CONFIG_X86_64 801 /** 802 * pud_free_pmd_page - Clear pud entry and free pmd page. 803 * @pud: Pointer to a PUD. 804 * @addr: Virtual address associated with pud. 805 * 806 * Context: The pud range has been unmapped and TLB purged. 807 * Return: 1 if clearing the entry succeeded. 0 otherwise. 808 * 809 * NOTE: Callers must allow a single page allocation. 810 */ 811 int pud_free_pmd_page(pud_t *pud, unsigned long addr) 812 { 813 pmd_t *pmd, *pmd_sv; 814 pte_t *pte; 815 int i; 816 817 pmd = pud_pgtable(*pud); 818 pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL); 819 if (!pmd_sv) 820 return 0; 821 822 for (i = 0; i < PTRS_PER_PMD; i++) { 823 pmd_sv[i] = pmd[i]; 824 if (!pmd_none(pmd[i])) 825 pmd_clear(&pmd[i]); 826 } 827 828 pud_clear(pud); 829 830 /* INVLPG to clear all paging-structure caches */ 831 flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); 832 833 for (i = 0; i < PTRS_PER_PMD; i++) { 834 if (!pmd_none(pmd_sv[i])) { 835 pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); 836 free_page((unsigned long)pte); 837 } 838 } 839 840 free_page((unsigned long)pmd_sv); 841 842 pagetable_pmd_dtor(virt_to_ptdesc(pmd)); 843 free_page((unsigned long)pmd); 844 845 return 1; 846 } 847 848 /** 849 * pmd_free_pte_page - Clear pmd entry and free pte page. 850 * @pmd: Pointer to a PMD. 851 * @addr: Virtual address associated with pmd. 852 * 853 * Context: The pmd range has been unmapped and TLB purged. 854 * Return: 1 if clearing the entry succeeded. 0 otherwise. 855 */ 856 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 857 { 858 pte_t *pte; 859 860 pte = (pte_t *)pmd_page_vaddr(*pmd); 861 pmd_clear(pmd); 862 863 /* INVLPG to clear all paging-structure caches */ 864 flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); 865 866 free_page((unsigned long)pte); 867 868 return 1; 869 } 870 871 #else /* !CONFIG_X86_64 */ 872 873 /* 874 * Disable free page handling on x86-PAE. This assures that ioremap() 875 * does not update sync'd pmd entries. See vmalloc_sync_one(). 876 */ 877 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 878 { 879 return pmd_none(*pmd); 880 } 881 882 #endif /* CONFIG_X86_64 */ 883 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 884 885 pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma) 886 { 887 if (vma->vm_flags & VM_SHADOW_STACK) 888 return pte_mkwrite_shstk(pte); 889 890 pte = pte_mkwrite_novma(pte); 891 892 return pte_clear_saveddirty(pte); 893 } 894 895 pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 896 { 897 if (vma->vm_flags & VM_SHADOW_STACK) 898 return pmd_mkwrite_shstk(pmd); 899 900 pmd = pmd_mkwrite_novma(pmd); 901 902 return pmd_clear_saveddirty(pmd); 903 } 904 905 void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte) 906 { 907 /* 908 * Hardware before shadow stack can (rarely) set Dirty=1 909 * on a Write=0 PTE. So the below condition 910 * only indicates a software bug when shadow stack is 911 * supported by the HW. This checking is covered in 912 * pte_shstk(). 913 */ 914 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) && 915 pte_shstk(pte)); 916 } 917 918 void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd) 919 { 920 /* See note in arch_check_zapped_pte() */ 921 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) && 922 pmd_shstk(pmd)); 923 } 924