1 /* 2 * This file contains common routines for dealing with free of page tables 3 * Along with common page table handling code 4 * 5 * Derived from arch/powerpc/mm/tlb_64.c: 6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 7 * 8 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 9 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 10 * Copyright (C) 1996 Paul Mackerras 11 * 12 * Derived from "arch/i386/mm/init.c" 13 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 14 * 15 * Dave Engebretsen <engebret@us.ibm.com> 16 * Rework for PPC64 port. 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License 20 * as published by the Free Software Foundation; either version 21 * 2 of the License, or (at your option) any later version. 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/gfp.h> 26 #include <linux/mm.h> 27 #include <linux/percpu.h> 28 #include <linux/hardirq.h> 29 #include <linux/hugetlb.h> 30 #include <asm/pgalloc.h> 31 #include <asm/tlbflush.h> 32 #include <asm/tlb.h> 33 34 static inline int is_exec_fault(void) 35 { 36 return current->thread.regs && TRAP(current->thread.regs) == 0x400; 37 } 38 39 /* We only try to do i/d cache coherency on stuff that looks like 40 * reasonably "normal" PTEs. We currently require a PTE to be present 41 * and we avoid _PAGE_SPECIAL and cache inhibited pte. We also only do that 42 * on userspace PTEs 43 */ 44 static inline int pte_looks_normal(pte_t pte) 45 { 46 47 #if defined(CONFIG_PPC_BOOK3S_64) 48 if ((pte_val(pte) & (_PAGE_PRESENT | _PAGE_SPECIAL)) == _PAGE_PRESENT) { 49 if (pte_ci(pte)) 50 return 0; 51 if (pte_user(pte)) 52 return 1; 53 } 54 return 0; 55 #else 56 return (pte_val(pte) & 57 (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER)) == 58 (_PAGE_PRESENT | _PAGE_USER); 59 #endif 60 } 61 62 static struct page *maybe_pte_to_page(pte_t pte) 63 { 64 unsigned long pfn = pte_pfn(pte); 65 struct page *page; 66 67 if (unlikely(!pfn_valid(pfn))) 68 return NULL; 69 page = pfn_to_page(pfn); 70 if (PageReserved(page)) 71 return NULL; 72 return page; 73 } 74 75 #if defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0 76 77 /* Server-style MMU handles coherency when hashing if HW exec permission 78 * is supposed per page (currently 64-bit only). If not, then, we always 79 * flush the cache for valid PTEs in set_pte. Embedded CPU without HW exec 80 * support falls into the same category. 81 */ 82 83 static pte_t set_pte_filter(pte_t pte) 84 { 85 if (radix_enabled()) 86 return pte; 87 88 pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); 89 if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || 90 cpu_has_feature(CPU_FTR_NOEXECUTE))) { 91 struct page *pg = maybe_pte_to_page(pte); 92 if (!pg) 93 return pte; 94 if (!test_bit(PG_arch_1, &pg->flags)) { 95 flush_dcache_icache_page(pg); 96 set_bit(PG_arch_1, &pg->flags); 97 } 98 } 99 return pte; 100 } 101 102 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, 103 int dirty) 104 { 105 return pte; 106 } 107 108 #else /* defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0 */ 109 110 /* Embedded type MMU with HW exec support. This is a bit more complicated 111 * as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so 112 * instead we "filter out" the exec permission for non clean pages. 113 */ 114 static pte_t set_pte_filter(pte_t pte) 115 { 116 struct page *pg; 117 118 /* No exec permission in the first place, move on */ 119 if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte)) 120 return pte; 121 122 /* If you set _PAGE_EXEC on weird pages you're on your own */ 123 pg = maybe_pte_to_page(pte); 124 if (unlikely(!pg)) 125 return pte; 126 127 /* If the page clean, we move on */ 128 if (test_bit(PG_arch_1, &pg->flags)) 129 return pte; 130 131 /* If it's an exec fault, we flush the cache and make it clean */ 132 if (is_exec_fault()) { 133 flush_dcache_icache_page(pg); 134 set_bit(PG_arch_1, &pg->flags); 135 return pte; 136 } 137 138 /* Else, we filter out _PAGE_EXEC */ 139 return __pte(pte_val(pte) & ~_PAGE_EXEC); 140 } 141 142 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, 143 int dirty) 144 { 145 struct page *pg; 146 147 /* So here, we only care about exec faults, as we use them 148 * to recover lost _PAGE_EXEC and perform I$/D$ coherency 149 * if necessary. Also if _PAGE_EXEC is already set, same deal, 150 * we just bail out 151 */ 152 if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault()) 153 return pte; 154 155 #ifdef CONFIG_DEBUG_VM 156 /* So this is an exec fault, _PAGE_EXEC is not set. If it was 157 * an error we would have bailed out earlier in do_page_fault() 158 * but let's make sure of it 159 */ 160 if (WARN_ON(!(vma->vm_flags & VM_EXEC))) 161 return pte; 162 #endif /* CONFIG_DEBUG_VM */ 163 164 /* If you set _PAGE_EXEC on weird pages you're on your own */ 165 pg = maybe_pte_to_page(pte); 166 if (unlikely(!pg)) 167 goto bail; 168 169 /* If the page is already clean, we move on */ 170 if (test_bit(PG_arch_1, &pg->flags)) 171 goto bail; 172 173 /* Clean the page and set PG_arch_1 */ 174 flush_dcache_icache_page(pg); 175 set_bit(PG_arch_1, &pg->flags); 176 177 bail: 178 return __pte(pte_val(pte) | _PAGE_EXEC); 179 } 180 181 #endif /* !(defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0) */ 182 183 /* 184 * set_pte stores a linux PTE into the linux page table. 185 */ 186 void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, 187 pte_t pte) 188 { 189 /* 190 * When handling numa faults, we already have the pte marked 191 * _PAGE_PRESENT, but we can be sure that it is not in hpte. 192 * Hence we can use set_pte_at for them. 193 */ 194 VM_WARN_ON(pte_present(*ptep) && !pte_protnone(*ptep)); 195 196 /* 197 * Add the pte bit when tryint set a pte 198 */ 199 pte = __pte(pte_val(pte) | _PAGE_PTE); 200 201 /* Note: mm->context.id might not yet have been assigned as 202 * this context might not have been activated yet when this 203 * is called. 204 */ 205 pte = set_pte_filter(pte); 206 207 /* Perform the setting of the PTE */ 208 __set_pte_at(mm, addr, ptep, pte, 0); 209 } 210 211 /* 212 * This is called when relaxing access to a PTE. It's also called in the page 213 * fault path when we don't hit any of the major fault cases, ie, a minor 214 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have 215 * handled those two for us, we additionally deal with missing execute 216 * permission here on some processors 217 */ 218 int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, 219 pte_t *ptep, pte_t entry, int dirty) 220 { 221 int changed; 222 entry = set_access_flags_filter(entry, vma, dirty); 223 changed = !pte_same(*(ptep), entry); 224 if (changed) { 225 if (!is_vm_hugetlb_page(vma)) 226 assert_pte_locked(vma->vm_mm, address); 227 __ptep_set_access_flags(ptep, entry); 228 flush_tlb_page(vma, address); 229 } 230 return changed; 231 } 232 233 #ifdef CONFIG_DEBUG_VM 234 void assert_pte_locked(struct mm_struct *mm, unsigned long addr) 235 { 236 pgd_t *pgd; 237 pud_t *pud; 238 pmd_t *pmd; 239 240 if (mm == &init_mm) 241 return; 242 pgd = mm->pgd + pgd_index(addr); 243 BUG_ON(pgd_none(*pgd)); 244 pud = pud_offset(pgd, addr); 245 BUG_ON(pud_none(*pud)); 246 pmd = pmd_offset(pud, addr); 247 /* 248 * khugepaged to collapse normal pages to hugepage, first set 249 * pmd to none to force page fault/gup to take mmap_sem. After 250 * pmd is set to none, we do a pte_clear which does this assertion 251 * so if we find pmd none, return. 252 */ 253 if (pmd_none(*pmd)) 254 return; 255 BUG_ON(!pmd_present(*pmd)); 256 assert_spin_locked(pte_lockptr(mm, pmd)); 257 } 258 #endif /* CONFIG_DEBUG_VM */ 259 260 unsigned long vmalloc_to_phys(void *va) 261 { 262 unsigned long pfn = vmalloc_to_pfn(va); 263 264 BUG_ON(!pfn); 265 return __pa(pfn_to_kaddr(pfn)) + offset_in_page(va); 266 } 267 EXPORT_SYMBOL_GPL(vmalloc_to_phys); 268