1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <asm/pgalloc.h> 4 #include <linux/gfp.h> 5 #include <linux/kernel.h> 6 #include <linux/pgtable.h> 7 8 int ptep_set_access_flags(struct vm_area_struct *vma, 9 unsigned long address, pte_t *ptep, 10 pte_t entry, int dirty) 11 { 12 asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) 13 : : : : svvptc); 14 15 if (!pte_same(ptep_get(ptep), entry)) 16 __set_pte_at(vma->vm_mm, ptep, entry); 17 /* 18 * update_mmu_cache will unconditionally execute, handling both 19 * the case that the PTE changed and the spurious fault case. 20 */ 21 return true; 22 23 svvptc: 24 if (!pte_same(ptep_get(ptep), entry)) { 25 __set_pte_at(vma->vm_mm, ptep, entry); 26 /* Here only not svadu is impacted */ 27 flush_tlb_page(vma, address); 28 return true; 29 } 30 31 return false; 32 } 33 34 int ptep_test_and_clear_young(struct vm_area_struct *vma, 35 unsigned long address, 36 pte_t *ptep) 37 { 38 if (!pte_young(ptep_get(ptep))) 39 return 0; 40 return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); 41 } 42 EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); 43 44 #ifdef CONFIG_64BIT 45 pud_t *pud_offset(p4d_t *p4d, unsigned long address) 46 { 47 if (pgtable_l4_enabled) 48 return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); 49 50 return (pud_t *)p4d; 51 } 52 53 p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 54 { 55 if (pgtable_l5_enabled) 56 return pgd_pgtable(pgdp_get(pgd)) + p4d_index(address); 57 58 return (p4d_t *)pgd; 59 } 60 #endif 61 62 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 63 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 64 { 65 return 0; 66 } 67 68 void p4d_clear_huge(p4d_t *p4d) 69 { 70 } 71 72 int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) 73 { 74 pud_t new_pud = pfn_pud(__phys_to_pfn(phys), prot); 75 76 set_pud(pud, new_pud); 77 return 1; 78 } 79 80 int pud_clear_huge(pud_t *pud) 81 { 82 if (!pud_leaf(pudp_get(pud))) 83 return 0; 84 pud_clear(pud); 85 return 1; 86 } 87 88 int pud_free_pmd_page(pud_t *pud, unsigned long addr) 89 { 90 pmd_t *pmd = pud_pgtable(pudp_get(pud)); 91 int i; 92 93 pud_clear(pud); 94 95 flush_tlb_kernel_range(addr, addr + PUD_SIZE); 96 97 for (i = 0; i < PTRS_PER_PMD; i++) { 98 if (!pmd_none(pmd[i])) { 99 pte_t *pte = (pte_t *)pmd_page_vaddr(pmd[i]); 100 101 pte_free_kernel(NULL, pte); 102 } 103 } 104 105 pmd_free(NULL, pmd); 106 107 return 1; 108 } 109 110 int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) 111 { 112 pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), prot); 113 114 set_pmd(pmd, new_pmd); 115 return 1; 116 } 117 118 int pmd_clear_huge(pmd_t *pmd) 119 { 120 if (!pmd_leaf(pmdp_get(pmd))) 121 return 0; 122 pmd_clear(pmd); 123 return 1; 124 } 125 126 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 127 { 128 pte_t *pte = (pte_t *)pmd_page_vaddr(pmdp_get(pmd)); 129 130 pmd_clear(pmd); 131 132 flush_tlb_kernel_range(addr, addr + PMD_SIZE); 133 pte_free_kernel(NULL, pte); 134 return 1; 135 } 136 137 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 138 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 139 pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 140 unsigned long address, pmd_t *pmdp) 141 { 142 pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 143 144 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 145 VM_BUG_ON(pmd_trans_huge(pmdp_get(pmdp))); 146 /* 147 * When leaf PTE entries (regular pages) are collapsed into a leaf 148 * PMD entry (huge page), a valid non-leaf PTE is converted into a 149 * valid leaf PTE at the level 1 page table. Since the sfence.vma 150 * forms that specify an address only apply to leaf PTEs, we need a 151 * global flush here. collapse_huge_page() assumes these flushes are 152 * eager, so just do the fence here. 153 */ 154 flush_tlb_mm(vma->vm_mm); 155 return pmd; 156 } 157 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 158