1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PGTABLE_H 3 #define _LINUX_PGTABLE_H 4 5 #include <linux/pfn.h> 6 #include <asm/pgtable.h> 7 8 #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) 9 #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) 10 11 #ifndef __ASSEMBLY__ 12 #ifdef CONFIG_MMU 13 14 #include <linux/mm_types.h> 15 #include <linux/bug.h> 16 #include <linux/errno.h> 17 #include <asm-generic/pgtable_uffd.h> 18 #include <linux/page_table_check.h> 19 20 #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ 21 defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS 22 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED 23 #endif 24 25 /* 26 * This defines the generic helper for accessing PMD page 27 * table page. Although platforms can still override this 28 * via their respective <asm/pgtable.h>. 29 */ 30 #ifndef pmd_pgtable 31 #define pmd_pgtable(pmd) pmd_page(pmd) 32 #endif 33 34 #define pmd_folio(pmd) page_folio(pmd_page(pmd)) 35 36 /* 37 * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] 38 * 39 * The pXx_index() functions return the index of the entry in the page 40 * table page which would control the given virtual address 41 * 42 * As these functions may be used by the same code for different levels of 43 * the page table folding, they are always available, regardless of 44 * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 45 * because in such cases PTRS_PER_PxD equals 1. 46 */ 47 48 static inline unsigned long pte_index(unsigned long address) 49 { 50 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 51 } 52 53 #ifndef pmd_index 54 static inline unsigned long pmd_index(unsigned long address) 55 { 56 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 57 } 58 #define pmd_index pmd_index 59 #endif 60 61 #ifndef pud_index 62 static inline unsigned long pud_index(unsigned long address) 63 { 64 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 65 } 66 #define pud_index pud_index 67 #endif 68 69 #ifndef pgd_index 70 /* Must be a compile-time constant, so implement it as a macro */ 71 #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 72 #endif 73 74 #ifndef kernel_pte_init 75 static inline void kernel_pte_init(void *addr) 76 { 77 } 78 #define kernel_pte_init kernel_pte_init 79 #endif 80 81 #ifndef pmd_init 82 static inline void pmd_init(void *addr) 83 { 84 } 85 #define pmd_init pmd_init 86 #endif 87 88 #ifndef pud_init 89 static inline void pud_init(void *addr) 90 { 91 } 92 #define pud_init pud_init 93 #endif 94 95 #ifndef pte_offset_kernel 96 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 97 { 98 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 99 } 100 #define pte_offset_kernel pte_offset_kernel 101 #endif 102 103 #ifdef CONFIG_HIGHPTE 104 #define __pte_map(pmd, address) \ 105 ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address))) 106 #define pte_unmap(pte) do { \ 107 kunmap_local((pte)); \ 108 rcu_read_unlock(); \ 109 } while (0) 110 #else 111 static inline pte_t *__pte_map(pmd_t *pmd, unsigned long address) 112 { 113 return pte_offset_kernel(pmd, address); 114 } 115 static inline void pte_unmap(pte_t *pte) 116 { 117 rcu_read_unlock(); 118 } 119 #endif 120 121 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); 122 123 /* Find an entry in the second-level page table.. */ 124 #ifndef pmd_offset 125 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 126 { 127 return pud_pgtable(*pud) + pmd_index(address); 128 } 129 #define pmd_offset pmd_offset 130 #endif 131 132 #ifndef pud_offset 133 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 134 { 135 return p4d_pgtable(*p4d) + pud_index(address); 136 } 137 #define pud_offset pud_offset 138 #endif 139 140 static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) 141 { 142 return (pgd + pgd_index(address)); 143 }; 144 145 /* 146 * a shortcut to get a pgd_t in a given mm 147 */ 148 #ifndef pgd_offset 149 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) 150 #endif 151 152 /* 153 * a shortcut which implies the use of the kernel's pgd, instead 154 * of a process's 155 */ 156 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 157 158 /* 159 * In many cases it is known that a virtual address is mapped at PMD or PTE 160 * level, so instead of traversing all the page table levels, we can get a 161 * pointer to the PMD entry in user or kernel page table or translate a virtual 162 * address to the pointer in the PTE in the kernel page tables with simple 163 * helpers. 164 */ 165 static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) 166 { 167 return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); 168 } 169 170 static inline pmd_t *pmd_off_k(unsigned long va) 171 { 172 return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); 173 } 174 175 static inline pte_t *virt_to_kpte(unsigned long vaddr) 176 { 177 pmd_t *pmd = pmd_off_k(vaddr); 178 179 return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); 180 } 181 182 #ifndef pmd_young 183 static inline int pmd_young(pmd_t pmd) 184 { 185 return 0; 186 } 187 #endif 188 189 #ifndef pmd_dirty 190 static inline int pmd_dirty(pmd_t pmd) 191 { 192 return 0; 193 } 194 #endif 195 196 /* 197 * A facility to provide lazy MMU batching. This allows PTE updates and 198 * page invalidations to be delayed until a call to leave lazy MMU mode 199 * is issued. Some architectures may benefit from doing this, and it is 200 * beneficial for both shadow and direct mode hypervisors, which may batch 201 * the PTE updates which happen during this window. Note that using this 202 * interface requires that read hazards be removed from the code. A read 203 * hazard could result in the direct mode hypervisor case, since the actual 204 * write to the page tables may not yet have taken place, so reads though 205 * a raw PTE pointer after it has been modified are not guaranteed to be 206 * up to date. 207 * 208 * In the general case, no lock is guaranteed to be held between entry and exit 209 * of the lazy mode. (In practice, for user PTE updates, the appropriate page 210 * table lock(s) are held, but for kernel PTE updates, no lock is held). 211 * The implementation must therefore assume preemption may be enabled upon 212 * entry to the mode and cpu migration is possible; it must take steps to be 213 * robust against this. An implementation may handle this by disabling 214 * preemption, as a consequence generic code may not sleep while the lazy MMU 215 * mode is active. 216 * 217 * The mode is disabled in interrupt context and calls to the lazy_mmu API have 218 * no effect. 219 * 220 * The lazy MMU mode is enabled for a given block of code using: 221 * 222 * lazy_mmu_mode_enable(); 223 * <code> 224 * lazy_mmu_mode_disable(); 225 * 226 * Nesting is permitted: <code> may itself use an enable()/disable() pair. 227 * A nested call to enable() has no functional effect; however disable() causes 228 * any batched architectural state to be flushed regardless of nesting. After a 229 * call to disable(), the caller can therefore rely on all previous page table 230 * modifications to have taken effect, but the lazy MMU mode may still be 231 * enabled. 232 * 233 * In certain cases, it may be desirable to temporarily pause the lazy MMU mode. 234 * This can be done using: 235 * 236 * lazy_mmu_mode_pause(); 237 * <code> 238 * lazy_mmu_mode_resume(); 239 * 240 * pause() ensures that the mode is exited regardless of the nesting level; 241 * resume() re-enters the mode at the same nesting level. Any call to the 242 * lazy_mmu_mode_* API between those two calls has no effect. In particular, 243 * this means that pause()/resume() pairs may nest. 244 * 245 * is_lazy_mmu_mode_active() can be used to check whether the lazy MMU mode is 246 * currently enabled. 247 */ 248 #ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE 249 /** 250 * lazy_mmu_mode_enable() - Enable the lazy MMU mode. 251 * 252 * Enters a new lazy MMU mode section; if the mode was not already enabled, 253 * enables it and calls arch_enter_lazy_mmu_mode(). 254 * 255 * Must be paired with a call to lazy_mmu_mode_disable(). 256 * 257 * Has no effect if called: 258 * - While paused - see lazy_mmu_mode_pause() 259 * - In interrupt context 260 */ 261 static inline void lazy_mmu_mode_enable(void) 262 { 263 struct lazy_mmu_state *state = ¤t->lazy_mmu_state; 264 265 if (in_interrupt() || state->pause_count > 0) 266 return; 267 268 VM_WARN_ON_ONCE(state->enable_count == U8_MAX); 269 270 if (state->enable_count++ == 0) 271 arch_enter_lazy_mmu_mode(); 272 } 273 274 /** 275 * lazy_mmu_mode_disable() - Disable the lazy MMU mode. 276 * 277 * Exits the current lazy MMU mode section. If it is the outermost section, 278 * disables the mode and calls arch_leave_lazy_mmu_mode(). Otherwise (nested 279 * section), calls arch_flush_lazy_mmu_mode(). 280 * 281 * Must match a call to lazy_mmu_mode_enable(). 282 * 283 * Has no effect if called: 284 * - While paused - see lazy_mmu_mode_pause() 285 * - In interrupt context 286 */ 287 static inline void lazy_mmu_mode_disable(void) 288 { 289 struct lazy_mmu_state *state = ¤t->lazy_mmu_state; 290 291 if (in_interrupt() || state->pause_count > 0) 292 return; 293 294 VM_WARN_ON_ONCE(state->enable_count == 0); 295 296 if (--state->enable_count == 0) 297 arch_leave_lazy_mmu_mode(); 298 else /* Exiting a nested section */ 299 arch_flush_lazy_mmu_mode(); 300 301 } 302 303 /** 304 * lazy_mmu_mode_pause() - Pause the lazy MMU mode. 305 * 306 * Pauses the lazy MMU mode; if it is currently active, disables it and calls 307 * arch_leave_lazy_mmu_mode(). 308 * 309 * Must be paired with a call to lazy_mmu_mode_resume(). Calls to the 310 * lazy_mmu_mode_* API have no effect until the matching resume() call. 311 * 312 * Has no effect if called: 313 * - While paused (inside another pause()/resume() pair) 314 * - In interrupt context 315 */ 316 static inline void lazy_mmu_mode_pause(void) 317 { 318 struct lazy_mmu_state *state = ¤t->lazy_mmu_state; 319 320 if (in_interrupt()) 321 return; 322 323 VM_WARN_ON_ONCE(state->pause_count == U8_MAX); 324 325 if (state->pause_count++ == 0 && state->enable_count > 0) 326 arch_leave_lazy_mmu_mode(); 327 } 328 329 /** 330 * lazy_mmu_mode_resume() - Resume the lazy MMU mode. 331 * 332 * Resumes the lazy MMU mode; if it was active at the point where the matching 333 * call to lazy_mmu_mode_pause() was made, re-enables it and calls 334 * arch_enter_lazy_mmu_mode(). 335 * 336 * Must match a call to lazy_mmu_mode_pause(). 337 * 338 * Has no effect if called: 339 * - While paused (inside another pause()/resume() pair) 340 * - In interrupt context 341 */ 342 static inline void lazy_mmu_mode_resume(void) 343 { 344 struct lazy_mmu_state *state = ¤t->lazy_mmu_state; 345 346 if (in_interrupt()) 347 return; 348 349 VM_WARN_ON_ONCE(state->pause_count == 0); 350 351 if (--state->pause_count == 0 && state->enable_count > 0) 352 arch_enter_lazy_mmu_mode(); 353 } 354 #else 355 static inline void lazy_mmu_mode_enable(void) {} 356 static inline void lazy_mmu_mode_disable(void) {} 357 static inline void lazy_mmu_mode_pause(void) {} 358 static inline void lazy_mmu_mode_resume(void) {} 359 #endif 360 361 #ifndef pte_batch_hint 362 /** 363 * pte_batch_hint - Number of pages that can be added to batch without scanning. 364 * @ptep: Page table pointer for the entry. 365 * @pte: Page table entry. 366 * 367 * Some architectures know that a set of contiguous ptes all map the same 368 * contiguous memory with the same permissions. In this case, it can provide a 369 * hint to aid pte batching without the core code needing to scan every pte. 370 * 371 * An architecture implementation may ignore the PTE accessed state. Further, 372 * the dirty state must apply atomically to all the PTEs described by the hint. 373 * 374 * May be overridden by the architecture, else pte_batch_hint is always 1. 375 */ 376 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) 377 { 378 return 1; 379 } 380 #endif 381 382 #ifndef pte_advance_pfn 383 static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) 384 { 385 return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT)); 386 } 387 #endif 388 389 #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) 390 391 #ifndef set_ptes 392 /** 393 * set_ptes - Map consecutive pages to a contiguous range of addresses. 394 * @mm: Address space to map the pages into. 395 * @addr: Address to map the first page at. 396 * @ptep: Page table pointer for the first entry. 397 * @pte: Page table entry for the first page. 398 * @nr: Number of pages to map. 399 * 400 * When nr==1, initial state of pte may be present or not present, and new state 401 * may be present or not present. When nr>1, initial state of all ptes must be 402 * not present, and new state must be present. 403 * 404 * May be overridden by the architecture, or the architecture can define 405 * set_pte() and PFN_PTE_SHIFT. 406 * 407 * Context: The caller holds the page table lock. The pages all belong 408 * to the same folio. The PTEs are all in the same PMD. 409 */ 410 static inline void set_ptes(struct mm_struct *mm, unsigned long addr, 411 pte_t *ptep, pte_t pte, unsigned int nr) 412 { 413 page_table_check_ptes_set(mm, addr, ptep, pte, nr); 414 415 for (;;) { 416 set_pte(ptep, pte); 417 if (--nr == 0) 418 break; 419 ptep++; 420 pte = pte_next_pfn(pte); 421 } 422 } 423 #endif 424 #define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) 425 426 #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 427 extern int ptep_set_access_flags(struct vm_area_struct *vma, 428 unsigned long address, pte_t *ptep, 429 pte_t entry, int dirty); 430 #endif 431 432 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 433 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 434 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 435 unsigned long address, pmd_t *pmdp, 436 pmd_t entry, int dirty); 437 extern int pudp_set_access_flags(struct vm_area_struct *vma, 438 unsigned long address, pud_t *pudp, 439 pud_t entry, int dirty); 440 #else 441 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 442 unsigned long address, pmd_t *pmdp, 443 pmd_t entry, int dirty) 444 { 445 BUILD_BUG(); 446 return 0; 447 } 448 static inline int pudp_set_access_flags(struct vm_area_struct *vma, 449 unsigned long address, pud_t *pudp, 450 pud_t entry, int dirty) 451 { 452 BUILD_BUG(); 453 return 0; 454 } 455 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 456 #endif 457 458 #ifndef ptep_get 459 static inline pte_t ptep_get(pte_t *ptep) 460 { 461 return READ_ONCE(*ptep); 462 } 463 #endif 464 465 #ifndef pmdp_get 466 static inline pmd_t pmdp_get(pmd_t *pmdp) 467 { 468 return READ_ONCE(*pmdp); 469 } 470 #endif 471 472 #ifndef pudp_get 473 static inline pud_t pudp_get(pud_t *pudp) 474 { 475 return READ_ONCE(*pudp); 476 } 477 #endif 478 479 #ifndef p4dp_get 480 static inline p4d_t p4dp_get(p4d_t *p4dp) 481 { 482 return READ_ONCE(*p4dp); 483 } 484 #endif 485 486 #ifndef pgdp_get 487 static inline pgd_t pgdp_get(pgd_t *pgdp) 488 { 489 return READ_ONCE(*pgdp); 490 } 491 #endif 492 493 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 494 static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, 495 unsigned long address, pte_t *ptep) 496 { 497 pte_t pte = ptep_get(ptep); 498 bool young = true; 499 500 if (!pte_young(pte)) 501 young = false; 502 else 503 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); 504 return young; 505 } 506 #endif 507 508 #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 509 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 510 static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, 511 unsigned long address, pmd_t *pmdp) 512 { 513 pmd_t pmd = *pmdp; 514 bool young = true; 515 516 if (!pmd_young(pmd)) 517 young = false; 518 else 519 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); 520 return young; 521 } 522 #else 523 static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, 524 unsigned long address, pmd_t *pmdp) 525 { 526 BUILD_BUG(); 527 return false; 528 } 529 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ 530 #endif 531 532 #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 533 bool ptep_clear_flush_young(struct vm_area_struct *vma, 534 unsigned long address, pte_t *ptep); 535 #endif 536 537 #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 538 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 539 bool pmdp_clear_flush_young(struct vm_area_struct *vma, 540 unsigned long address, pmd_t *pmdp); 541 #else 542 /* 543 * Despite relevant to THP only, this API is called from generic rmap code 544 * under PageTransHuge(), hence needs a dummy implementation for !THP 545 */ 546 static inline bool pmdp_clear_flush_young(struct vm_area_struct *vma, 547 unsigned long address, pmd_t *pmdp) 548 { 549 BUILD_BUG(); 550 return false; 551 } 552 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 553 #endif 554 555 #ifndef arch_has_hw_nonleaf_pmd_young 556 /* 557 * Return whether the accessed bit in non-leaf PMD entries is supported on the 558 * local CPU. 559 */ 560 static inline bool arch_has_hw_nonleaf_pmd_young(void) 561 { 562 return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG); 563 } 564 #endif 565 566 #ifndef arch_has_hw_pte_young 567 /* 568 * Return whether the accessed bit is supported on the local CPU. 569 * 570 * This stub assumes accessing through an old PTE triggers a page fault. 571 * Architectures that automatically set the access bit should overwrite it. 572 */ 573 static inline bool arch_has_hw_pte_young(void) 574 { 575 return IS_ENABLED(CONFIG_ARCH_HAS_HW_PTE_YOUNG); 576 } 577 #endif 578 579 #ifndef exec_folio_order 580 /* 581 * Returns preferred minimum folio order for executable file-backed memory. Must 582 * be in range [0, PMD_ORDER). Default to order-0. 583 */ 584 static inline unsigned int exec_folio_order(void) 585 { 586 return 0; 587 } 588 #endif 589 590 #ifndef arch_check_zapped_pte 591 static inline void arch_check_zapped_pte(struct vm_area_struct *vma, 592 pte_t pte) 593 { 594 } 595 #endif 596 597 #ifndef arch_check_zapped_pmd 598 static inline void arch_check_zapped_pmd(struct vm_area_struct *vma, 599 pmd_t pmd) 600 { 601 } 602 #endif 603 604 #ifndef arch_check_zapped_pud 605 static inline void arch_check_zapped_pud(struct vm_area_struct *vma, pud_t pud) 606 { 607 } 608 #endif 609 610 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR 611 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 612 unsigned long address, 613 pte_t *ptep) 614 { 615 pte_t pte = ptep_get(ptep); 616 pte_clear(mm, address, ptep); 617 page_table_check_pte_clear(mm, address, pte); 618 return pte; 619 } 620 #endif 621 622 #ifndef clear_young_dirty_ptes 623 /** 624 * clear_young_dirty_ptes - Mark PTEs that map consecutive pages of the 625 * same folio as old/clean. 626 * @mm: Address space the pages are mapped into. 627 * @addr: Address the first page is mapped at. 628 * @ptep: Page table pointer for the first entry. 629 * @nr: Number of entries to mark old/clean. 630 * @flags: Flags to modify the PTE batch semantics. 631 * 632 * May be overridden by the architecture; otherwise, implemented by 633 * get_and_clear/modify/set for each pte in the range. 634 * 635 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 636 * some PTEs might be write-protected. 637 * 638 * Context: The caller holds the page table lock. The PTEs map consecutive 639 * pages that belong to the same folio. The PTEs are all in the same PMD. 640 */ 641 static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, 642 unsigned long addr, pte_t *ptep, 643 unsigned int nr, cydp_t flags) 644 { 645 pte_t pte; 646 647 for (;;) { 648 if (flags == CYDP_CLEAR_YOUNG) 649 ptep_test_and_clear_young(vma, addr, ptep); 650 else { 651 pte = ptep_get_and_clear(vma->vm_mm, addr, ptep); 652 if (flags & CYDP_CLEAR_YOUNG) 653 pte = pte_mkold(pte); 654 if (flags & CYDP_CLEAR_DIRTY) 655 pte = pte_mkclean(pte); 656 set_pte_at(vma->vm_mm, addr, ptep, pte); 657 } 658 if (--nr == 0) 659 break; 660 ptep++; 661 addr += PAGE_SIZE; 662 } 663 } 664 #endif 665 666 static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, 667 pte_t *ptep) 668 { 669 pte_t pte = ptep_get(ptep); 670 671 pte_clear(mm, addr, ptep); 672 /* 673 * No need for ptep_get_and_clear(): page table check doesn't care about 674 * any bits that could have been set by HW concurrently. 675 */ 676 page_table_check_pte_clear(mm, addr, pte); 677 } 678 679 #ifdef CONFIG_GUP_GET_PXX_LOW_HIGH 680 /* 681 * For walking the pagetables without holding any locks. Some architectures 682 * (eg x86-32 PAE) cannot load the entries atomically without using expensive 683 * instructions. We are guaranteed that a PTE will only either go from not 684 * present to present, or present to not present -- it will not switch to a 685 * completely different present page without a TLB flush inbetween; which we 686 * are blocking by holding interrupts off. 687 * 688 * Setting ptes from not present to present goes: 689 * 690 * ptep->pte_high = h; 691 * smp_wmb(); 692 * ptep->pte_low = l; 693 * 694 * And present to not present goes: 695 * 696 * ptep->pte_low = 0; 697 * smp_wmb(); 698 * ptep->pte_high = 0; 699 * 700 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'. 701 * We load pte_high *after* loading pte_low, which ensures we don't see an older 702 * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't 703 * picked up a changed pte high. We might have gotten rubbish values from 704 * pte_low and pte_high, but we are guaranteed that pte_low will not have the 705 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only 706 * operates on present ptes we're safe. 707 */ 708 static inline pte_t ptep_get_lockless(pte_t *ptep) 709 { 710 pte_t pte; 711 712 do { 713 pte.pte_low = ptep->pte_low; 714 smp_rmb(); 715 pte.pte_high = ptep->pte_high; 716 smp_rmb(); 717 } while (unlikely(pte.pte_low != ptep->pte_low)); 718 719 return pte; 720 } 721 #define ptep_get_lockless ptep_get_lockless 722 723 #if CONFIG_PGTABLE_LEVELS > 2 724 static inline pmd_t pmdp_get_lockless(pmd_t *pmdp) 725 { 726 pmd_t pmd; 727 728 do { 729 pmd.pmd_low = pmdp->pmd_low; 730 smp_rmb(); 731 pmd.pmd_high = pmdp->pmd_high; 732 smp_rmb(); 733 } while (unlikely(pmd.pmd_low != pmdp->pmd_low)); 734 735 return pmd; 736 } 737 #define pmdp_get_lockless pmdp_get_lockless 738 #define pmdp_get_lockless_sync() tlb_remove_table_sync_one() 739 #endif /* CONFIG_PGTABLE_LEVELS > 2 */ 740 #endif /* CONFIG_GUP_GET_PXX_LOW_HIGH */ 741 742 /* 743 * We require that the PTE can be read atomically. 744 */ 745 #ifndef ptep_get_lockless 746 static inline pte_t ptep_get_lockless(pte_t *ptep) 747 { 748 return ptep_get(ptep); 749 } 750 #endif 751 752 #ifndef pmdp_get_lockless 753 static inline pmd_t pmdp_get_lockless(pmd_t *pmdp) 754 { 755 return pmdp_get(pmdp); 756 } 757 static inline void pmdp_get_lockless_sync(void) 758 { 759 } 760 #endif 761 762 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 763 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 764 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 765 unsigned long address, 766 pmd_t *pmdp) 767 { 768 pmd_t pmd = *pmdp; 769 770 pmd_clear(pmdp); 771 page_table_check_pmd_clear(mm, address, pmd); 772 773 return pmd; 774 } 775 #endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ 776 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR 777 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, 778 unsigned long address, 779 pud_t *pudp) 780 { 781 pud_t pud = *pudp; 782 783 pud_clear(pudp); 784 page_table_check_pud_clear(mm, address, pud); 785 786 return pud; 787 } 788 #endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ 789 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 790 791 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 792 #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 793 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, 794 unsigned long address, pmd_t *pmdp, 795 int full) 796 { 797 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 798 } 799 #endif 800 801 #ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL 802 static inline pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma, 803 unsigned long address, pud_t *pudp, 804 int full) 805 { 806 return pudp_huge_get_and_clear(vma->vm_mm, address, pudp); 807 } 808 #endif 809 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 810 811 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 812 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 813 unsigned long address, pte_t *ptep, 814 int full) 815 { 816 return ptep_get_and_clear(mm, address, ptep); 817 } 818 #endif 819 820 #ifndef get_and_clear_full_ptes 821 /** 822 * get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of 823 * the same folio, collecting dirty/accessed bits. 824 * @mm: Address space the pages are mapped into. 825 * @addr: Address the first page is mapped at. 826 * @ptep: Page table pointer for the first entry. 827 * @nr: Number of entries to clear. 828 * @full: Whether we are clearing a full mm. 829 * 830 * May be overridden by the architecture; otherwise, implemented as a simple 831 * loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the 832 * returned PTE. 833 * 834 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 835 * some PTEs might be write-protected. 836 * 837 * Context: The caller holds the page table lock. The PTEs map consecutive 838 * pages that belong to the same folio. The PTEs are all in the same PMD. 839 */ 840 static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, 841 unsigned long addr, pte_t *ptep, unsigned int nr, int full) 842 { 843 pte_t pte, tmp_pte; 844 845 pte = ptep_get_and_clear_full(mm, addr, ptep, full); 846 while (--nr) { 847 ptep++; 848 addr += PAGE_SIZE; 849 tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full); 850 if (pte_dirty(tmp_pte)) 851 pte = pte_mkdirty(pte); 852 if (pte_young(tmp_pte)) 853 pte = pte_mkyoung(pte); 854 } 855 return pte; 856 } 857 #endif 858 859 /** 860 * get_and_clear_ptes - Clear present PTEs that map consecutive pages of 861 * the same folio, collecting dirty/accessed bits. 862 * @mm: Address space the pages are mapped into. 863 * @addr: Address the first page is mapped at. 864 * @ptep: Page table pointer for the first entry. 865 * @nr: Number of entries to clear. 866 * 867 * Use this instead of get_and_clear_full_ptes() if it is known that we don't 868 * need to clear the full mm, which is mostly the case. 869 * 870 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 871 * some PTEs might be write-protected. 872 * 873 * Context: The caller holds the page table lock. The PTEs map consecutive 874 * pages that belong to the same folio. The PTEs are all in the same PMD. 875 */ 876 static inline pte_t get_and_clear_ptes(struct mm_struct *mm, unsigned long addr, 877 pte_t *ptep, unsigned int nr) 878 { 879 return get_and_clear_full_ptes(mm, addr, ptep, nr, 0); 880 } 881 882 #ifndef clear_full_ptes 883 /** 884 * clear_full_ptes - Clear present PTEs that map consecutive pages of the same 885 * folio. 886 * @mm: Address space the pages are mapped into. 887 * @addr: Address the first page is mapped at. 888 * @ptep: Page table pointer for the first entry. 889 * @nr: Number of entries to clear. 890 * @full: Whether we are clearing a full mm. 891 * 892 * May be overridden by the architecture; otherwise, implemented as a simple 893 * loop over ptep_get_and_clear_full(). 894 * 895 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 896 * some PTEs might be write-protected. 897 * 898 * Context: The caller holds the page table lock. The PTEs map consecutive 899 * pages that belong to the same folio. The PTEs are all in the same PMD. 900 */ 901 static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, 902 pte_t *ptep, unsigned int nr, int full) 903 { 904 for (;;) { 905 ptep_get_and_clear_full(mm, addr, ptep, full); 906 if (--nr == 0) 907 break; 908 ptep++; 909 addr += PAGE_SIZE; 910 } 911 } 912 #endif 913 914 /** 915 * clear_ptes - Clear present PTEs that map consecutive pages of the same folio. 916 * @mm: Address space the pages are mapped into. 917 * @addr: Address the first page is mapped at. 918 * @ptep: Page table pointer for the first entry. 919 * @nr: Number of entries to clear. 920 * 921 * Use this instead of clear_full_ptes() if it is known that we don't need to 922 * clear the full mm, which is mostly the case. 923 * 924 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 925 * some PTEs might be write-protected. 926 * 927 * Context: The caller holds the page table lock. The PTEs map consecutive 928 * pages that belong to the same folio. The PTEs are all in the same PMD. 929 */ 930 static inline void clear_ptes(struct mm_struct *mm, unsigned long addr, 931 pte_t *ptep, unsigned int nr) 932 { 933 clear_full_ptes(mm, addr, ptep, nr, 0); 934 } 935 936 /* 937 * If two threads concurrently fault at the same page, the thread that 938 * won the race updates the PTE and its local TLB/Cache. The other thread 939 * gives up, simply does nothing, and continues; on architectures where 940 * software can update TLB, local TLB can be updated here to avoid next page 941 * fault. This function updates TLB only, do nothing with cache or others. 942 * It is the difference with function update_mmu_cache. 943 */ 944 #ifndef update_mmu_tlb_range 945 static inline void update_mmu_tlb_range(struct vm_area_struct *vma, 946 unsigned long address, pte_t *ptep, unsigned int nr) 947 { 948 } 949 #endif 950 951 static inline void update_mmu_tlb(struct vm_area_struct *vma, 952 unsigned long address, pte_t *ptep) 953 { 954 update_mmu_tlb_range(vma, address, ptep, 1); 955 } 956 957 /* 958 * Some architectures may be able to avoid expensive synchronization 959 * primitives when modifications are made to PTE's which are already 960 * not present, or in the process of an address space destruction. 961 */ 962 #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL 963 static inline void pte_clear_not_present_full(struct mm_struct *mm, 964 unsigned long address, 965 pte_t *ptep, 966 int full) 967 { 968 pte_clear(mm, address, ptep); 969 } 970 #endif 971 972 #ifndef clear_not_present_full_ptes 973 /** 974 * clear_not_present_full_ptes - Clear multiple not present PTEs which are 975 * consecutive in the pgtable. 976 * @mm: Address space the ptes represent. 977 * @addr: Address of the first pte. 978 * @ptep: Page table pointer for the first entry. 979 * @nr: Number of entries to clear. 980 * @full: Whether we are clearing a full mm. 981 * 982 * May be overridden by the architecture; otherwise, implemented as a simple 983 * loop over pte_clear_not_present_full(). 984 * 985 * Context: The caller holds the page table lock. The PTEs are all not present. 986 * The PTEs are all in the same PMD. 987 */ 988 static inline void clear_not_present_full_ptes(struct mm_struct *mm, 989 unsigned long addr, pte_t *ptep, unsigned int nr, int full) 990 { 991 for (;;) { 992 pte_clear_not_present_full(mm, addr, ptep, full); 993 if (--nr == 0) 994 break; 995 ptep++; 996 addr += PAGE_SIZE; 997 } 998 } 999 #endif 1000 1001 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH 1002 extern pte_t ptep_clear_flush(struct vm_area_struct *vma, 1003 unsigned long address, 1004 pte_t *ptep); 1005 #endif 1006 1007 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 1008 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 1009 unsigned long address, 1010 pmd_t *pmdp); 1011 extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, 1012 unsigned long address, 1013 pud_t *pudp); 1014 #endif 1015 1016 #ifndef pte_mkwrite 1017 static inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma) 1018 { 1019 return pte_mkwrite_novma(pte); 1020 } 1021 #endif 1022 1023 #if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite) 1024 static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1025 { 1026 return pmd_mkwrite_novma(pmd); 1027 } 1028 #endif 1029 1030 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT 1031 struct mm_struct; 1032 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) 1033 { 1034 pte_t old_pte = ptep_get(ptep); 1035 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); 1036 } 1037 #endif 1038 1039 #ifndef wrprotect_ptes 1040 /** 1041 * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same 1042 * folio. 1043 * @mm: Address space the pages are mapped into. 1044 * @addr: Address the first page is mapped at. 1045 * @ptep: Page table pointer for the first entry. 1046 * @nr: Number of entries to write-protect. 1047 * 1048 * May be overridden by the architecture; otherwise, implemented as a simple 1049 * loop over ptep_set_wrprotect(). 1050 * 1051 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 1052 * some PTEs might be write-protected. 1053 * 1054 * Context: The caller holds the page table lock. The PTEs map consecutive 1055 * pages that belong to the same folio. The PTEs are all in the same PMD. 1056 */ 1057 static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, 1058 pte_t *ptep, unsigned int nr) 1059 { 1060 for (;;) { 1061 ptep_set_wrprotect(mm, addr, ptep); 1062 if (--nr == 0) 1063 break; 1064 ptep++; 1065 addr += PAGE_SIZE; 1066 } 1067 } 1068 #endif 1069 1070 #ifndef clear_flush_young_ptes 1071 /** 1072 * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same 1073 * folio as old and flush the TLB. 1074 * @vma: The virtual memory area the pages are mapped into. 1075 * @addr: Address the first page is mapped at. 1076 * @ptep: Page table pointer for the first entry. 1077 * @nr: Number of entries to clear access bit. 1078 * 1079 * May be overridden by the architecture; otherwise, implemented as a simple 1080 * loop over ptep_clear_flush_young(). 1081 * 1082 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 1083 * some PTEs might be write-protected. 1084 * 1085 * Context: The caller holds the page table lock. The PTEs map consecutive 1086 * pages that belong to the same folio. The PTEs are all in the same PMD. 1087 */ 1088 static inline bool clear_flush_young_ptes(struct vm_area_struct *vma, 1089 unsigned long addr, pte_t *ptep, unsigned int nr) 1090 { 1091 bool young = false; 1092 1093 for (;;) { 1094 young |= ptep_clear_flush_young(vma, addr, ptep); 1095 if (--nr == 0) 1096 break; 1097 ptep++; 1098 addr += PAGE_SIZE; 1099 } 1100 1101 return young; 1102 } 1103 #endif 1104 1105 #ifndef test_and_clear_young_ptes 1106 /** 1107 * test_and_clear_young_ptes - Mark PTEs that map consecutive pages of the same 1108 * folio as old 1109 * @vma: The virtual memory area the pages are mapped into. 1110 * @addr: Address the first page is mapped at. 1111 * @ptep: Page table pointer for the first entry. 1112 * @nr: Number of entries to clear access bit. 1113 * 1114 * May be overridden by the architecture; otherwise, implemented as a simple 1115 * loop over ptep_test_and_clear_young(). 1116 * 1117 * Note that PTE bits in the PTE range besides the PFN can differ. For example, 1118 * some PTEs might be write-protected. 1119 * 1120 * Context: The caller holds the page table lock. The PTEs map consecutive 1121 * pages that belong to the same folio. The PTEs are all in the same PMD. 1122 * 1123 * Returns: whether any PTE was young. 1124 */ 1125 static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma, 1126 unsigned long addr, pte_t *ptep, unsigned int nr) 1127 { 1128 bool young = false; 1129 1130 for (;;) { 1131 young |= ptep_test_and_clear_young(vma, addr, ptep); 1132 if (--nr == 0) 1133 break; 1134 ptep++; 1135 addr += PAGE_SIZE; 1136 } 1137 1138 return young; 1139 } 1140 #endif 1141 1142 /* 1143 * On some architectures hardware does not set page access bit when accessing 1144 * memory page, it is responsibility of software setting this bit. It brings 1145 * out extra page fault penalty to track page access bit. For optimization page 1146 * access bit can be set during all page fault flow on these arches. 1147 * To be differentiate with macro pte_mkyoung, this macro is used on platforms 1148 * where software maintains page access bit. 1149 */ 1150 #ifndef pte_sw_mkyoung 1151 static inline pte_t pte_sw_mkyoung(pte_t pte) 1152 { 1153 return pte; 1154 } 1155 #define pte_sw_mkyoung pte_sw_mkyoung 1156 #endif 1157 1158 #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT 1159 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1160 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 1161 unsigned long address, pmd_t *pmdp) 1162 { 1163 pmd_t old_pmd = *pmdp; 1164 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); 1165 } 1166 #else 1167 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 1168 unsigned long address, pmd_t *pmdp) 1169 { 1170 BUILD_BUG(); 1171 } 1172 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1173 #endif 1174 #ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT 1175 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1176 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1177 static inline void pudp_set_wrprotect(struct mm_struct *mm, 1178 unsigned long address, pud_t *pudp) 1179 { 1180 pud_t old_pud = *pudp; 1181 1182 set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); 1183 } 1184 #else 1185 static inline void pudp_set_wrprotect(struct mm_struct *mm, 1186 unsigned long address, pud_t *pudp) 1187 { 1188 BUILD_BUG(); 1189 } 1190 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1191 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1192 #endif 1193 1194 #ifndef pmdp_collapse_flush 1195 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1196 extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 1197 unsigned long address, pmd_t *pmdp); 1198 #else 1199 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 1200 unsigned long address, 1201 pmd_t *pmdp) 1202 { 1203 BUILD_BUG(); 1204 return *pmdp; 1205 } 1206 #define pmdp_collapse_flush pmdp_collapse_flush 1207 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1208 #endif 1209 1210 #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 1211 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 1212 pgtable_t pgtable); 1213 #endif 1214 1215 #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 1216 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 1217 #endif 1218 1219 #ifndef arch_needs_pgtable_deposit 1220 #define arch_needs_pgtable_deposit() (false) 1221 #endif 1222 1223 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1224 /* 1225 * This is an implementation of pmdp_establish() that is only suitable for an 1226 * architecture that doesn't have hardware dirty/accessed bits. In this case we 1227 * can't race with CPU which sets these bits and non-atomic approach is fine. 1228 */ 1229 static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, 1230 unsigned long address, pmd_t *pmdp, pmd_t pmd) 1231 { 1232 pmd_t old_pmd = *pmdp; 1233 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 1234 return old_pmd; 1235 } 1236 #endif 1237 1238 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 1239 extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 1240 pmd_t *pmdp); 1241 #endif 1242 1243 #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD 1244 1245 /* 1246 * pmdp_invalidate_ad() invalidates the PMD while changing a transparent 1247 * hugepage mapping in the page tables. This function is similar to 1248 * pmdp_invalidate(), but should only be used if the access and dirty bits would 1249 * not be cleared by the software in the new PMD value. The function ensures 1250 * that hardware changes of the access and dirty bits updates would not be lost. 1251 * 1252 * Doing so can allow in certain architectures to avoid a TLB flush in most 1253 * cases. Yet, another TLB flush might be necessary later if the PMD update 1254 * itself requires such flush (e.g., if protection was set to be stricter). Yet, 1255 * even when a TLB flush is needed because of the update, the caller may be able 1256 * to batch these TLB flushing operations, so fewer TLB flush operations are 1257 * needed. 1258 */ 1259 extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, 1260 unsigned long address, pmd_t *pmdp); 1261 #endif 1262 1263 #ifndef __HAVE_ARCH_PTE_SAME 1264 static inline int pte_same(pte_t pte_a, pte_t pte_b) 1265 { 1266 return pte_val(pte_a) == pte_val(pte_b); 1267 } 1268 #endif 1269 1270 #ifndef __HAVE_ARCH_PTE_UNUSED 1271 /* 1272 * Some architectures provide facilities to virtualization guests 1273 * so that they can flag allocated pages as unused. This allows the 1274 * host to transparently reclaim unused pages. This function returns 1275 * whether the pte's page is unused. 1276 */ 1277 static inline int pte_unused(pte_t pte) 1278 { 1279 return 0; 1280 } 1281 #endif 1282 1283 #ifndef pte_access_permitted 1284 #define pte_access_permitted(pte, write) \ 1285 (pte_present(pte) && (!(write) || pte_write(pte))) 1286 #endif 1287 1288 #ifndef pmd_access_permitted 1289 #define pmd_access_permitted(pmd, write) \ 1290 (pmd_present(pmd) && (!(write) || pmd_write(pmd))) 1291 #endif 1292 1293 #ifndef pud_access_permitted 1294 #define pud_access_permitted(pud, write) \ 1295 (pud_present(pud) && (!(write) || pud_write(pud))) 1296 #endif 1297 1298 #ifndef p4d_access_permitted 1299 #define p4d_access_permitted(p4d, write) \ 1300 (p4d_present(p4d) && (!(write) || p4d_write(p4d))) 1301 #endif 1302 1303 #ifndef pgd_access_permitted 1304 #define pgd_access_permitted(pgd, write) \ 1305 (pgd_present(pgd) && (!(write) || pgd_write(pgd))) 1306 #endif 1307 1308 #ifndef __HAVE_ARCH_PMD_SAME 1309 static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) 1310 { 1311 return pmd_val(pmd_a) == pmd_val(pmd_b); 1312 } 1313 #endif 1314 1315 #ifndef pud_same 1316 static inline int pud_same(pud_t pud_a, pud_t pud_b) 1317 { 1318 return pud_val(pud_a) == pud_val(pud_b); 1319 } 1320 #define pud_same pud_same 1321 #endif 1322 1323 #ifndef __HAVE_ARCH_P4D_SAME 1324 static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) 1325 { 1326 return p4d_val(p4d_a) == p4d_val(p4d_b); 1327 } 1328 #endif 1329 1330 #ifndef __HAVE_ARCH_PGD_SAME 1331 static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) 1332 { 1333 return pgd_val(pgd_a) == pgd_val(pgd_b); 1334 } 1335 #endif 1336 1337 #ifndef __HAVE_ARCH_DO_SWAP_PAGE 1338 static inline void arch_do_swap_page_nr(struct mm_struct *mm, 1339 struct vm_area_struct *vma, 1340 unsigned long addr, 1341 pte_t pte, pte_t oldpte, 1342 int nr) 1343 { 1344 1345 } 1346 #else 1347 /* 1348 * Some architectures support metadata associated with a page. When a 1349 * page is being swapped out, this metadata must be saved so it can be 1350 * restored when the page is swapped back in. SPARC M7 and newer 1351 * processors support an ADI (Application Data Integrity) tag for the 1352 * page as metadata for the page. arch_do_swap_page() can restore this 1353 * metadata when a page is swapped back in. 1354 */ 1355 static inline void arch_do_swap_page_nr(struct mm_struct *mm, 1356 struct vm_area_struct *vma, 1357 unsigned long addr, 1358 pte_t pte, pte_t oldpte, 1359 int nr) 1360 { 1361 for (int i = 0; i < nr; i++) { 1362 arch_do_swap_page(vma->vm_mm, vma, addr + i * PAGE_SIZE, 1363 pte_advance_pfn(pte, i), 1364 pte_advance_pfn(oldpte, i)); 1365 } 1366 } 1367 #endif 1368 1369 #ifndef __HAVE_ARCH_UNMAP_ONE 1370 /* 1371 * Some architectures support metadata associated with a page. When a 1372 * page is being swapped out, this metadata must be saved so it can be 1373 * restored when the page is swapped back in. SPARC M7 and newer 1374 * processors support an ADI (Application Data Integrity) tag for the 1375 * page as metadata for the page. arch_unmap_one() can save this 1376 * metadata on a swap-out of a page. 1377 */ 1378 static inline int arch_unmap_one(struct mm_struct *mm, 1379 struct vm_area_struct *vma, 1380 unsigned long addr, 1381 pte_t orig_pte) 1382 { 1383 return 0; 1384 } 1385 #endif 1386 1387 /* 1388 * Allow architectures to preserve additional metadata associated with 1389 * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function 1390 * prototypes must be defined in the arch-specific asm/pgtable.h file. 1391 */ 1392 #ifndef __HAVE_ARCH_PREPARE_TO_SWAP 1393 static inline int arch_prepare_to_swap(struct folio *folio) 1394 { 1395 return 0; 1396 } 1397 #endif 1398 1399 #ifndef __HAVE_ARCH_SWAP_INVALIDATE 1400 static inline void arch_swap_invalidate_page(int type, pgoff_t offset) 1401 { 1402 } 1403 1404 static inline void arch_swap_invalidate_area(int type) 1405 { 1406 } 1407 #endif 1408 1409 #ifndef __HAVE_ARCH_SWAP_RESTORE 1410 static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) 1411 { 1412 } 1413 #endif 1414 1415 #ifndef __HAVE_ARCH_MOVE_PTE 1416 #define move_pte(pte, old_addr, new_addr) (pte) 1417 #endif 1418 1419 #ifndef pte_accessible 1420 # define pte_accessible(mm, pte) ((void)(pte), 1) 1421 #endif 1422 1423 #ifndef flush_tlb_fix_spurious_fault 1424 #define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address) 1425 #endif 1426 1427 #ifndef flush_tlb_fix_spurious_fault_pmd 1428 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) do { } while (0) 1429 #endif 1430 1431 /* 1432 * When walking page tables, get the address of the next boundary, 1433 * or the end address of the range if that comes earlier. Although no 1434 * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. 1435 */ 1436 1437 #define pgd_addr_end(addr, end) \ 1438 ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ 1439 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 1440 }) 1441 1442 #ifndef p4d_addr_end 1443 #define p4d_addr_end(addr, end) \ 1444 ({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ 1445 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 1446 }) 1447 #endif 1448 1449 #ifndef pud_addr_end 1450 #define pud_addr_end(addr, end) \ 1451 ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ 1452 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 1453 }) 1454 #endif 1455 1456 #ifndef pmd_addr_end 1457 #define pmd_addr_end(addr, end) \ 1458 ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ 1459 (__boundary - 1 < (end) - 1)? __boundary: (end); \ 1460 }) 1461 #endif 1462 1463 /* 1464 * When walking page tables, we usually want to skip any p?d_none entries; 1465 * and any p?d_bad entries - reporting the error before resetting to none. 1466 * Do the tests inline, but report and clear the bad entry in mm/memory.c. 1467 */ 1468 void pgd_clear_bad(pgd_t *); 1469 1470 #ifndef __PAGETABLE_P4D_FOLDED 1471 void p4d_clear_bad(p4d_t *); 1472 #else 1473 #define p4d_clear_bad(p4d) do { } while (0) 1474 #endif 1475 1476 #ifndef __PAGETABLE_PUD_FOLDED 1477 void pud_clear_bad(pud_t *); 1478 #else 1479 #define pud_clear_bad(p4d) do { } while (0) 1480 #endif 1481 1482 void pmd_clear_bad(pmd_t *); 1483 1484 static inline int pgd_none_or_clear_bad(pgd_t *pgd) 1485 { 1486 if (pgd_none(*pgd)) 1487 return 1; 1488 if (unlikely(pgd_bad(*pgd))) { 1489 pgd_clear_bad(pgd); 1490 return 1; 1491 } 1492 return 0; 1493 } 1494 1495 static inline int p4d_none_or_clear_bad(p4d_t *p4d) 1496 { 1497 if (p4d_none(*p4d)) 1498 return 1; 1499 if (unlikely(p4d_bad(*p4d))) { 1500 p4d_clear_bad(p4d); 1501 return 1; 1502 } 1503 return 0; 1504 } 1505 1506 static inline int pud_none_or_clear_bad(pud_t *pud) 1507 { 1508 if (pud_none(*pud)) 1509 return 1; 1510 if (unlikely(pud_bad(*pud))) { 1511 pud_clear_bad(pud); 1512 return 1; 1513 } 1514 return 0; 1515 } 1516 1517 static inline int pmd_none_or_clear_bad(pmd_t *pmd) 1518 { 1519 if (pmd_none(*pmd)) 1520 return 1; 1521 if (unlikely(pmd_bad(*pmd))) { 1522 pmd_clear_bad(pmd); 1523 return 1; 1524 } 1525 return 0; 1526 } 1527 1528 static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, 1529 unsigned long addr, 1530 pte_t *ptep) 1531 { 1532 /* 1533 * Get the current pte state, but zero it out to make it 1534 * non-present, preventing the hardware from asynchronously 1535 * updating it. 1536 */ 1537 return ptep_get_and_clear(vma->vm_mm, addr, ptep); 1538 } 1539 1540 static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, 1541 unsigned long addr, 1542 pte_t *ptep, pte_t pte) 1543 { 1544 /* 1545 * The pte is non-present, so there's no hardware state to 1546 * preserve. 1547 */ 1548 set_pte_at(vma->vm_mm, addr, ptep, pte); 1549 } 1550 1551 #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 1552 /* 1553 * Start a pte protection read-modify-write transaction, which 1554 * protects against asynchronous hardware modifications to the pte. 1555 * The intention is not to prevent the hardware from making pte 1556 * updates, but to prevent any updates it may make from being lost. 1557 * 1558 * This does not protect against other software modifications of the 1559 * pte; the appropriate pte lock must be held over the transaction. 1560 * 1561 * Note that this interface is intended to be batchable, meaning that 1562 * ptep_modify_prot_commit may not actually update the pte, but merely 1563 * queue the update to be done at some later time. The update must be 1564 * actually committed before the pte lock is released, however. 1565 */ 1566 static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, 1567 unsigned long addr, 1568 pte_t *ptep) 1569 { 1570 return __ptep_modify_prot_start(vma, addr, ptep); 1571 } 1572 1573 /* 1574 * Commit an update to a pte, leaving any hardware-controlled bits in 1575 * the PTE unmodified. The pte returned from ptep_modify_prot_start() may 1576 * additionally have young and/or dirty bits set where previously they were not, 1577 * so the updated pte may have these additional changes. 1578 */ 1579 static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, 1580 unsigned long addr, 1581 pte_t *ptep, pte_t old_pte, pte_t pte) 1582 { 1583 __ptep_modify_prot_commit(vma, addr, ptep, pte); 1584 } 1585 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ 1586 1587 /** 1588 * modify_prot_start_ptes - Start a pte protection read-modify-write transaction 1589 * over a batch of ptes, which protects against asynchronous hardware 1590 * modifications to the ptes. The intention is not to prevent the hardware from 1591 * making pte updates, but to prevent any updates it may make from being lost. 1592 * Please see the comment above ptep_modify_prot_start() for full description. 1593 * 1594 * @vma: The virtual memory area the pages are mapped into. 1595 * @addr: Address the first page is mapped at. 1596 * @ptep: Page table pointer for the first entry. 1597 * @nr: Number of entries. 1598 * 1599 * May be overridden by the architecture; otherwise, implemented as a simple 1600 * loop over ptep_modify_prot_start(), collecting the a/d bits from each pte 1601 * in the batch. 1602 * 1603 * Note that PTE bits in the PTE batch besides the PFN can differ. 1604 * 1605 * Context: The caller holds the page table lock. The PTEs map consecutive 1606 * pages that belong to the same folio. All other PTE bits must be identical for 1607 * all PTEs in the batch except for young and dirty bits. The PTEs are all in 1608 * the same PMD. 1609 */ 1610 #ifndef modify_prot_start_ptes 1611 static inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma, 1612 unsigned long addr, pte_t *ptep, unsigned int nr) 1613 { 1614 pte_t pte, tmp_pte; 1615 1616 pte = ptep_modify_prot_start(vma, addr, ptep); 1617 while (--nr) { 1618 ptep++; 1619 addr += PAGE_SIZE; 1620 tmp_pte = ptep_modify_prot_start(vma, addr, ptep); 1621 if (pte_dirty(tmp_pte)) 1622 pte = pte_mkdirty(pte); 1623 if (pte_young(tmp_pte)) 1624 pte = pte_mkyoung(pte); 1625 } 1626 return pte; 1627 } 1628 #endif 1629 1630 /** 1631 * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any 1632 * hardware-controlled bits in the PTE unmodified. 1633 * 1634 * @vma: The virtual memory area the pages are mapped into. 1635 * @addr: Address the first page is mapped at. 1636 * @ptep: Page table pointer for the first entry. 1637 * @old_pte: Old page table entry (for the first entry) which is now cleared. 1638 * @pte: New page table entry to be set. 1639 * @nr: Number of entries. 1640 * 1641 * May be overridden by the architecture; otherwise, implemented as a simple 1642 * loop over ptep_modify_prot_commit(). 1643 * 1644 * Context: The caller holds the page table lock. The PTEs are all in the same 1645 * PMD. On exit, the set ptes in the batch map the same folio. The ptes set by 1646 * ptep_modify_prot_start() may additionally have young and/or dirty bits set 1647 * where previously they were not, so the updated ptes may have these 1648 * additional changes. 1649 */ 1650 #ifndef modify_prot_commit_ptes 1651 static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr, 1652 pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr) 1653 { 1654 int i; 1655 1656 for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE) { 1657 ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte); 1658 1659 /* Advance PFN only, set same prot */ 1660 old_pte = pte_next_pfn(old_pte); 1661 pte = pte_next_pfn(pte); 1662 } 1663 } 1664 #endif 1665 1666 /* 1667 * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values 1668 * and let generic vmalloc, ioremap and page table update code know when 1669 * arch_sync_kernel_mappings() needs to be called. 1670 */ 1671 #ifndef ARCH_PAGE_TABLE_SYNC_MASK 1672 #define ARCH_PAGE_TABLE_SYNC_MASK 0 1673 #endif 1674 1675 /* 1676 * There is no default implementation for arch_sync_kernel_mappings(). It is 1677 * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK 1678 * is 0. 1679 */ 1680 void arch_sync_kernel_mappings(unsigned long start, unsigned long end); 1681 1682 #endif /* CONFIG_MMU */ 1683 1684 /* 1685 * On almost all architectures and configurations, 0 can be used as the 1686 * upper ceiling to free_pgtables(): on many architectures it has the same 1687 * effect as using TASK_SIZE. However, there is one configuration which 1688 * must impose a more careful limit, to avoid freeing kernel pgtables. 1689 */ 1690 #ifndef USER_PGTABLES_CEILING 1691 #define USER_PGTABLES_CEILING 0UL 1692 #endif 1693 1694 /* 1695 * This defines the first usable user address. Platforms 1696 * can override its value with custom FIRST_USER_ADDRESS 1697 * defined in their respective <asm/pgtable.h>. 1698 */ 1699 #ifndef FIRST_USER_ADDRESS 1700 #define FIRST_USER_ADDRESS 0UL 1701 #endif 1702 1703 /* 1704 * No-op macros that just return the current protection value. Defined here 1705 * because these macros can be used even if CONFIG_MMU is not defined. 1706 */ 1707 1708 #ifndef pgprot_nx 1709 #define pgprot_nx(prot) (prot) 1710 #endif 1711 1712 #ifndef pgprot_noncached 1713 #define pgprot_noncached(prot) (prot) 1714 #endif 1715 1716 #ifndef pgprot_writecombine 1717 #define pgprot_writecombine pgprot_noncached 1718 #endif 1719 1720 #ifndef pgprot_writethrough 1721 #define pgprot_writethrough pgprot_noncached 1722 #endif 1723 1724 #ifndef pgprot_device 1725 #define pgprot_device pgprot_noncached 1726 #endif 1727 1728 #ifndef pgprot_mhp 1729 #define pgprot_mhp(prot) (prot) 1730 #endif 1731 1732 #ifdef CONFIG_MMU 1733 #ifndef pgprot_modify 1734 #define pgprot_modify pgprot_modify 1735 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 1736 { 1737 if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) 1738 newprot = pgprot_noncached(newprot); 1739 if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) 1740 newprot = pgprot_writecombine(newprot); 1741 if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) 1742 newprot = pgprot_device(newprot); 1743 return newprot; 1744 } 1745 #endif 1746 #endif /* CONFIG_MMU */ 1747 1748 #ifndef pgprot_encrypted 1749 #define pgprot_encrypted(prot) (prot) 1750 #endif 1751 1752 #ifndef pgprot_decrypted 1753 #define pgprot_decrypted(prot) (prot) 1754 #endif 1755 1756 /* 1757 * A facility to provide batching of the reload of page tables and 1758 * other process state with the actual context switch code for 1759 * paravirtualized guests. By convention, only one of the batched 1760 * update (lazy) modes (CPU, MMU) should be active at any given time, 1761 * entry should never be nested, and entry and exits should always be 1762 * paired. This is for sanity of maintaining and reasoning about the 1763 * kernel code. In this case, the exit (end of the context switch) is 1764 * in architecture-specific code, and so doesn't need a generic 1765 * definition. 1766 */ 1767 #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH 1768 #define arch_start_context_switch(prev) do {} while (0) 1769 #endif 1770 1771 /* 1772 * Some platforms can customize the PTE soft-dirty bit making it unavailable 1773 * even if the architecture provides the resource. 1774 * Adding this API allows architectures to add their own checks for the 1775 * devices on which the kernel is running. 1776 * Note: When overriding it, please make sure the CONFIG_MEM_SOFT_DIRTY 1777 * is part of this macro. 1778 */ 1779 #ifndef pgtable_supports_soft_dirty 1780 #define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) 1781 #endif 1782 1783 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY 1784 #ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION 1785 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 1786 { 1787 return pmd; 1788 } 1789 1790 static inline int pmd_swp_soft_dirty(pmd_t pmd) 1791 { 1792 return 0; 1793 } 1794 1795 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 1796 { 1797 return pmd; 1798 } 1799 #endif 1800 #else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ 1801 static inline int pte_soft_dirty(pte_t pte) 1802 { 1803 return 0; 1804 } 1805 1806 static inline int pmd_soft_dirty(pmd_t pmd) 1807 { 1808 return 0; 1809 } 1810 1811 static inline pte_t pte_mksoft_dirty(pte_t pte) 1812 { 1813 return pte; 1814 } 1815 1816 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 1817 { 1818 return pmd; 1819 } 1820 1821 static inline pte_t pte_clear_soft_dirty(pte_t pte) 1822 { 1823 return pte; 1824 } 1825 1826 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 1827 { 1828 return pmd; 1829 } 1830 1831 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 1832 { 1833 return pte; 1834 } 1835 1836 static inline int pte_swp_soft_dirty(pte_t pte) 1837 { 1838 return 0; 1839 } 1840 1841 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 1842 { 1843 return pte; 1844 } 1845 1846 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) 1847 { 1848 return pmd; 1849 } 1850 1851 static inline int pmd_swp_soft_dirty(pmd_t pmd) 1852 { 1853 return 0; 1854 } 1855 1856 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) 1857 { 1858 return pmd; 1859 } 1860 #endif 1861 1862 #ifndef __HAVE_PFNMAP_TRACKING 1863 /* 1864 * Interfaces that can be used by architecture code to keep track of 1865 * memory type of pfn mappings specified by the remap_pfn_range, 1866 * vmf_insert_pfn. 1867 */ 1868 1869 static inline int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, 1870 pgprot_t *prot) 1871 { 1872 return 0; 1873 } 1874 1875 static inline int pfnmap_track(unsigned long pfn, unsigned long size, 1876 pgprot_t *prot) 1877 { 1878 return 0; 1879 } 1880 1881 static inline void pfnmap_untrack(unsigned long pfn, unsigned long size) 1882 { 1883 } 1884 #else 1885 /** 1886 * pfnmap_setup_cachemode - setup the cachemode in the pgprot for a pfn range 1887 * @pfn: the start of the pfn range 1888 * @size: the size of the pfn range in bytes 1889 * @prot: the pgprot to modify 1890 * 1891 * Lookup the cachemode for the pfn range starting at @pfn with the size 1892 * @size and store it in @prot, leaving other data in @prot unchanged. 1893 * 1894 * This allows for a hardware implementation to have fine-grained control of 1895 * memory cache behavior at page level granularity. Without a hardware 1896 * implementation, this function does nothing. 1897 * 1898 * Currently there is only one implementation for this - x86 Page Attribute 1899 * Table (PAT). See Documentation/arch/x86/pat.rst for more details. 1900 * 1901 * This function can fail if the pfn range spans pfns that require differing 1902 * cachemodes. If the pfn range was previously verified to have a single 1903 * cachemode, it is sufficient to query only a single pfn. The assumption is 1904 * that this is the case for drivers using the vmf_insert_pfn*() interface. 1905 * 1906 * Returns 0 on success and -EINVAL on error. 1907 */ 1908 int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, 1909 pgprot_t *prot); 1910 1911 /** 1912 * pfnmap_track - track a pfn range 1913 * @pfn: the start of the pfn range 1914 * @size: the size of the pfn range in bytes 1915 * @prot: the pgprot to track 1916 * 1917 * Requested the pfn range to be 'tracked' by a hardware implementation and 1918 * setup the cachemode in @prot similar to pfnmap_setup_cachemode(). 1919 * 1920 * This allows for fine-grained control of memory cache behaviour at page 1921 * level granularity. Tracking memory this way is persisted across VMA splits 1922 * (VMA merging does not apply for VM_PFNMAP). 1923 * 1924 * Currently, there is only one implementation for this - x86 Page Attribute 1925 * Table (PAT). See Documentation/arch/x86/pat.rst for more details. 1926 * 1927 * Returns 0 on success and -EINVAL on error. 1928 */ 1929 int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot); 1930 1931 /** 1932 * pfnmap_untrack - untrack a pfn range 1933 * @pfn: the start of the pfn range 1934 * @size: the size of the pfn range in bytes 1935 * 1936 * Untrack a pfn range previously tracked through pfnmap_track(). 1937 */ 1938 void pfnmap_untrack(unsigned long pfn, unsigned long size); 1939 #endif 1940 1941 /** 1942 * pfnmap_setup_cachemode_pfn - setup the cachemode in the pgprot for a pfn 1943 * @pfn: the pfn 1944 * @prot: the pgprot to modify 1945 * 1946 * Lookup the cachemode for @pfn and store it in @prot, leaving other 1947 * data in @prot unchanged. 1948 * 1949 * See pfnmap_setup_cachemode() for details. 1950 */ 1951 static inline void pfnmap_setup_cachemode_pfn(unsigned long pfn, pgprot_t *prot) 1952 { 1953 pfnmap_setup_cachemode(pfn, PAGE_SIZE, prot); 1954 } 1955 1956 /* 1957 * ZERO_PAGE() is global shared page(s) that is always zero. It is used for 1958 * zero-mapped memory areas, CoW etc. 1959 * 1960 * On architectures that __HAVE_COLOR_ZERO_PAGE there are several such pages 1961 * for different ranges in the virtual address space. 1962 * 1963 * zero_page_pfn identifies the first (or the only) pfn for these pages. 1964 * 1965 * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in 1966 * empty_zero_page in BSS. 1967 */ 1968 void arch_setup_zero_pages(void); 1969 1970 #ifdef __HAVE_COLOR_ZERO_PAGE 1971 static inline int is_zero_pfn(unsigned long pfn) 1972 { 1973 extern unsigned long zero_page_pfn; 1974 unsigned long offset_from_zero_pfn = pfn - zero_page_pfn; 1975 1976 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); 1977 } 1978 1979 #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) 1980 1981 #else 1982 static inline int is_zero_pfn(unsigned long pfn) 1983 { 1984 extern unsigned long zero_page_pfn; 1985 1986 return pfn == zero_page_pfn; 1987 } 1988 1989 static inline unsigned long zero_pfn(unsigned long addr) 1990 { 1991 extern unsigned long zero_page_pfn; 1992 1993 return zero_page_pfn; 1994 } 1995 1996 extern uint8_t empty_zero_page[PAGE_SIZE]; 1997 extern struct page *__zero_page; 1998 1999 static inline struct page *_zero_page(unsigned long addr) 2000 { 2001 return __zero_page; 2002 } 2003 #define ZERO_PAGE(vaddr) _zero_page(vaddr) 2004 2005 #endif /* __HAVE_COLOR_ZERO_PAGE */ 2006 2007 #ifdef CONFIG_MMU 2008 2009 #ifndef CONFIG_TRANSPARENT_HUGEPAGE 2010 static inline int pmd_trans_huge(pmd_t pmd) 2011 { 2012 return 0; 2013 } 2014 #ifndef pmd_write 2015 static inline int pmd_write(pmd_t pmd) 2016 { 2017 BUG(); 2018 return 0; 2019 } 2020 #endif /* pmd_write */ 2021 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2022 2023 #ifndef pud_write 2024 static inline int pud_write(pud_t pud) 2025 { 2026 BUG(); 2027 return 0; 2028 } 2029 #endif /* pud_write */ 2030 2031 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ 2032 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 2033 static inline int pud_trans_huge(pud_t pud) 2034 { 2035 return 0; 2036 } 2037 #endif 2038 2039 static inline int pud_trans_unstable(pud_t *pud) 2040 { 2041 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 2042 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 2043 pud_t pudval = pudp_get(pud); 2044 2045 if (pud_none(pudval) || pud_trans_huge(pudval)) 2046 return 1; 2047 if (unlikely(pud_bad(pudval))) { 2048 pud_clear_bad(pud); 2049 return 1; 2050 } 2051 #endif 2052 return 0; 2053 } 2054 2055 #ifndef CONFIG_NUMA_BALANCING 2056 /* 2057 * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is 2058 * perfectly valid to indicate "no" in that case, which is why our default 2059 * implementation defaults to "always no". 2060 * 2061 * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE 2062 * page protection due to NUMA hinting. NUMA hinting faults only apply in 2063 * accessible VMAs. 2064 * 2065 * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, 2066 * looking at the VMA accessibility is sufficient. 2067 */ 2068 static inline int pte_protnone(pte_t pte) 2069 { 2070 return 0; 2071 } 2072 2073 static inline int pmd_protnone(pmd_t pmd) 2074 { 2075 return 0; 2076 } 2077 #endif /* CONFIG_NUMA_BALANCING */ 2078 2079 #endif /* CONFIG_MMU */ 2080 2081 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 2082 2083 #ifndef __PAGETABLE_P4D_FOLDED 2084 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); 2085 void p4d_clear_huge(p4d_t *p4d); 2086 #else 2087 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 2088 { 2089 return 0; 2090 } 2091 static inline void p4d_clear_huge(p4d_t *p4d) { } 2092 #endif /* !__PAGETABLE_P4D_FOLDED */ 2093 2094 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); 2095 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); 2096 int pud_clear_huge(pud_t *pud); 2097 int pmd_clear_huge(pmd_t *pmd); 2098 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); 2099 int pud_free_pmd_page(pud_t *pud, unsigned long addr); 2100 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); 2101 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 2102 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 2103 { 2104 return 0; 2105 } 2106 static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 2107 { 2108 return 0; 2109 } 2110 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 2111 { 2112 return 0; 2113 } 2114 static inline void p4d_clear_huge(p4d_t *p4d) { } 2115 static inline int pud_clear_huge(pud_t *pud) 2116 { 2117 return 0; 2118 } 2119 static inline int pmd_clear_huge(pmd_t *pmd) 2120 { 2121 return 0; 2122 } 2123 static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) 2124 { 2125 return 0; 2126 } 2127 static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) 2128 { 2129 return 0; 2130 } 2131 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 2132 { 2133 return 0; 2134 } 2135 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 2136 2137 #ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE 2138 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2139 /* 2140 * ARCHes with special requirements for evicting THP backing TLB entries can 2141 * implement this. Otherwise also, it can help optimize normal TLB flush in 2142 * THP regime. Stock flush_tlb_range() typically has optimization to nuke the 2143 * entire TLB if flush span is greater than a threshold, which will 2144 * likely be true for a single huge page. Thus a single THP flush will 2145 * invalidate the entire TLB which is not desirable. 2146 * e.g. see arch/arc: flush_pmd_tlb_range 2147 */ 2148 #define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 2149 #define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 2150 #else 2151 #define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() 2152 #define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() 2153 #endif 2154 #endif 2155 2156 struct file; 2157 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, 2158 unsigned long size, pgprot_t *vma_prot); 2159 2160 #ifndef CONFIG_X86_ESPFIX64 2161 static inline void init_espfix_bsp(void) { } 2162 #endif 2163 2164 extern void __init pgtable_cache_init(void); 2165 2166 #ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED 2167 static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) 2168 { 2169 return true; 2170 } 2171 2172 static inline bool arch_has_pfn_modify_check(void) 2173 { 2174 return false; 2175 } 2176 #endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ 2177 2178 /* 2179 * Architecture PAGE_KERNEL_* fallbacks 2180 * 2181 * Some architectures don't define certain PAGE_KERNEL_* flags. This is either 2182 * because they really don't support them, or the port needs to be updated to 2183 * reflect the required functionality. Below are a set of relatively safe 2184 * fallbacks, as best effort, which we can count on in lieu of the architectures 2185 * not defining them on their own yet. 2186 */ 2187 2188 #ifndef PAGE_KERNEL_RO 2189 # define PAGE_KERNEL_RO PAGE_KERNEL 2190 #endif 2191 2192 #ifndef PAGE_KERNEL_EXEC 2193 # define PAGE_KERNEL_EXEC PAGE_KERNEL 2194 #endif 2195 2196 /* 2197 * Page Table Modification bits for pgtbl_mod_mask. 2198 * 2199 * These are used by the p?d_alloc_track*() and p*d_populate_kernel() 2200 * functions in the generic vmalloc, ioremap and page table update code 2201 * to track at which page-table levels entries have been modified. 2202 * Based on that the code can better decide when page table changes need 2203 * to be synchronized to other page-tables in the system. 2204 */ 2205 #define __PGTBL_PGD_MODIFIED 0 2206 #define __PGTBL_P4D_MODIFIED 1 2207 #define __PGTBL_PUD_MODIFIED 2 2208 #define __PGTBL_PMD_MODIFIED 3 2209 #define __PGTBL_PTE_MODIFIED 4 2210 2211 #define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) 2212 #define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) 2213 #define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) 2214 #define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) 2215 #define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) 2216 2217 /* Page-Table Modification Mask */ 2218 typedef unsigned int pgtbl_mod_mask; 2219 2220 enum pgtable_level { 2221 PGTABLE_LEVEL_PTE = 0, 2222 PGTABLE_LEVEL_PMD, 2223 PGTABLE_LEVEL_PUD, 2224 PGTABLE_LEVEL_P4D, 2225 PGTABLE_LEVEL_PGD, 2226 }; 2227 2228 static inline const char *pgtable_level_to_str(enum pgtable_level level) 2229 { 2230 switch (level) { 2231 case PGTABLE_LEVEL_PTE: 2232 return "pte"; 2233 case PGTABLE_LEVEL_PMD: 2234 return "pmd"; 2235 case PGTABLE_LEVEL_PUD: 2236 return "pud"; 2237 case PGTABLE_LEVEL_P4D: 2238 return "p4d"; 2239 case PGTABLE_LEVEL_PGD: 2240 return "pgd"; 2241 default: 2242 return "unknown"; 2243 } 2244 } 2245 2246 #endif /* !__ASSEMBLY__ */ 2247 2248 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) 2249 #ifdef CONFIG_PHYS_ADDR_T_64BIT 2250 /* 2251 * ZSMALLOC needs to know the highest PFN on 32-bit architectures 2252 * with physical address space extension, but falls back to 2253 * BITS_PER_LONG otherwise. 2254 */ 2255 #error Missing MAX_POSSIBLE_PHYSMEM_BITS definition 2256 #else 2257 #define MAX_POSSIBLE_PHYSMEM_BITS 32 2258 #endif 2259 #endif 2260 2261 #ifndef has_transparent_hugepage 2262 #define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE) 2263 #endif 2264 2265 #ifndef has_transparent_pud_hugepage 2266 #define has_transparent_pud_hugepage() IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 2267 #endif 2268 /* 2269 * On some architectures it depends on the mm if the p4d/pud or pmd 2270 * layer of the page table hierarchy is folded or not. 2271 */ 2272 #ifndef mm_p4d_folded 2273 #define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) 2274 #endif 2275 2276 #ifndef mm_pud_folded 2277 #define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) 2278 #endif 2279 2280 #ifndef mm_pmd_folded 2281 #define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) 2282 #endif 2283 2284 #ifndef p4d_offset_lockless 2285 #define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address) 2286 #endif 2287 #ifndef pud_offset_lockless 2288 #define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address) 2289 #endif 2290 #ifndef pmd_offset_lockless 2291 #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) 2292 #endif 2293 2294 /* 2295 * pXd_leaf() is the API to check whether a pgtable entry is a huge page 2296 * mapping. It should work globally across all archs, without any 2297 * dependency on CONFIG_* options. For architectures that do not support 2298 * huge mappings on specific levels, below fallbacks will be used. 2299 * 2300 * A leaf pgtable entry should always imply the following: 2301 * 2302 * - It is a "present" entry. IOW, before using this API, please check it 2303 * with pXd_present() first. NOTE: it may not always mean the "present 2304 * bit" is set. For example, PROT_NONE entries are always "present". 2305 * 2306 * - It should _never_ be a swap entry of any type. Above "present" check 2307 * should have guarded this, but let's be crystal clear on this. 2308 * 2309 * - It should contain a huge PFN, which points to a huge page larger than 2310 * PAGE_SIZE of the platform. The PFN format isn't important here. 2311 * 2312 * - It should cover all kinds of huge mappings (i.e. pXd_trans_huge() 2313 * or hugetlb mappings). 2314 */ 2315 #ifndef pgd_leaf 2316 #define pgd_leaf(x) false 2317 #endif 2318 #ifndef p4d_leaf 2319 #define p4d_leaf(x) false 2320 #endif 2321 #ifndef pud_leaf 2322 #define pud_leaf(x) false 2323 #endif 2324 #ifndef pmd_leaf 2325 #define pmd_leaf(x) false 2326 #endif 2327 2328 #ifndef pgd_leaf_size 2329 #define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT) 2330 #endif 2331 #ifndef p4d_leaf_size 2332 #define p4d_leaf_size(x) P4D_SIZE 2333 #endif 2334 #ifndef pud_leaf_size 2335 #define pud_leaf_size(x) PUD_SIZE 2336 #endif 2337 #ifndef pmd_leaf_size 2338 #define pmd_leaf_size(x) PMD_SIZE 2339 #endif 2340 #ifndef __pte_leaf_size 2341 #ifndef pte_leaf_size 2342 #define pte_leaf_size(x) PAGE_SIZE 2343 #endif 2344 #define __pte_leaf_size(x,y) pte_leaf_size(y) 2345 #endif 2346 2347 /* 2348 * We always define pmd_pfn for all archs as it's used in lots of generic 2349 * code. Now it happens too for pud_pfn (and can happen for larger 2350 * mappings too in the future; we're not there yet). Instead of defining 2351 * it for all archs (like pmd_pfn), provide a fallback. 2352 * 2353 * Note that returning 0 here means any arch that didn't define this can 2354 * get severely wrong when it hits a real pud leaf. It's arch's 2355 * responsibility to properly define it when a huge pud is possible. 2356 */ 2357 #ifndef pud_pfn 2358 #define pud_pfn(x) 0 2359 #endif 2360 2361 /* 2362 * Some architectures have MMUs that are configurable or selectable at boot 2363 * time. These lead to variable PTRS_PER_x. For statically allocated arrays it 2364 * helps to have a static maximum value. 2365 */ 2366 2367 #ifndef MAX_PTRS_PER_PTE 2368 #define MAX_PTRS_PER_PTE PTRS_PER_PTE 2369 #endif 2370 2371 #ifndef MAX_PTRS_PER_PMD 2372 #define MAX_PTRS_PER_PMD PTRS_PER_PMD 2373 #endif 2374 2375 #ifndef MAX_PTRS_PER_PUD 2376 #define MAX_PTRS_PER_PUD PTRS_PER_PUD 2377 #endif 2378 2379 #ifndef MAX_PTRS_PER_P4D 2380 #define MAX_PTRS_PER_P4D PTRS_PER_P4D 2381 #endif 2382 2383 #ifndef pte_pgprot 2384 #define pte_pgprot(x) ((pgprot_t) {0}) 2385 #endif 2386 2387 #ifndef pmd_pgprot 2388 #define pmd_pgprot(x) ((pgprot_t) {0}) 2389 #endif 2390 2391 #ifndef pud_pgprot 2392 #define pud_pgprot(x) ((pgprot_t) {0}) 2393 #endif 2394 2395 /* description of effects of mapping type and prot in current implementation. 2396 * this is due to the limited x86 page protection hardware. The expected 2397 * behavior is in parens: 2398 * 2399 * map_type prot 2400 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC 2401 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes 2402 * w: (no) no w: (no) no w: (yes) yes w: (no) no 2403 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes 2404 * 2405 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes 2406 * w: (no) no w: (no) no w: (copy) copy w: (no) no 2407 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes 2408 * 2409 * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and 2410 * MAP_PRIVATE (with Enhanced PAN supported): 2411 * r: (no) no 2412 * w: (no) no 2413 * x: (yes) yes 2414 */ 2415 #define DECLARE_VM_GET_PAGE_PROT \ 2416 pgprot_t vm_get_page_prot(vm_flags_t vm_flags) \ 2417 { \ 2418 return protection_map[vm_flags & \ 2419 (VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \ 2420 } \ 2421 EXPORT_SYMBOL(vm_get_page_prot); 2422 2423 #endif /* _LINUX_PGTABLE_H */ 2424