1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <linux/ksm.h> 27 #include <asm/uaccess.h> 28 #include <asm/pgtable.h> 29 #include <asm/cacheflush.h> 30 #include <asm/tlbflush.h> 31 32 /* 33 * For a prot_numa update we only hold mmap_sem for read so there is a 34 * potential race with faulting where a pmd was temporarily none. This 35 * function checks for a transhuge pmd under the appropriate lock. It 36 * returns a pte if it was successfully locked or NULL if it raced with 37 * a transhuge insertion. 38 */ 39 static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd, 40 unsigned long addr, int prot_numa, spinlock_t **ptl) 41 { 42 pte_t *pte; 43 spinlock_t *pmdl; 44 45 /* !prot_numa is protected by mmap_sem held for write */ 46 if (!prot_numa) 47 return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 48 49 pmdl = pmd_lock(vma->vm_mm, pmd); 50 if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) { 51 spin_unlock(pmdl); 52 return NULL; 53 } 54 55 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 56 spin_unlock(pmdl); 57 return pte; 58 } 59 60 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 61 unsigned long addr, unsigned long end, pgprot_t newprot, 62 int dirty_accountable, int prot_numa) 63 { 64 struct mm_struct *mm = vma->vm_mm; 65 pte_t *pte, oldpte; 66 spinlock_t *ptl; 67 unsigned long pages = 0; 68 69 pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl); 70 if (!pte) 71 return 0; 72 73 arch_enter_lazy_mmu_mode(); 74 do { 75 oldpte = *pte; 76 if (pte_present(oldpte)) { 77 pte_t ptent; 78 bool preserve_write = prot_numa && pte_write(oldpte); 79 80 /* 81 * Avoid trapping faults against the zero or KSM 82 * pages. See similar comment in change_huge_pmd. 83 */ 84 if (prot_numa) { 85 struct page *page; 86 87 page = vm_normal_page(vma, addr, oldpte); 88 if (!page || PageKsm(page)) 89 continue; 90 91 /* Avoid TLB flush if possible */ 92 if (pte_protnone(oldpte)) 93 continue; 94 } 95 96 ptent = ptep_modify_prot_start(mm, addr, pte); 97 ptent = pte_modify(ptent, newprot); 98 if (preserve_write) 99 ptent = pte_mkwrite(ptent); 100 101 /* Avoid taking write faults for known dirty pages */ 102 if (dirty_accountable && pte_dirty(ptent) && 103 (pte_soft_dirty(ptent) || 104 !(vma->vm_flags & VM_SOFTDIRTY))) { 105 ptent = pte_mkwrite(ptent); 106 } 107 ptep_modify_prot_commit(mm, addr, pte, ptent); 108 pages++; 109 } else if (IS_ENABLED(CONFIG_MIGRATION)) { 110 swp_entry_t entry = pte_to_swp_entry(oldpte); 111 112 if (is_write_migration_entry(entry)) { 113 pte_t newpte; 114 /* 115 * A protection check is difficult so 116 * just be safe and disable write 117 */ 118 make_migration_entry_read(&entry); 119 newpte = swp_entry_to_pte(entry); 120 if (pte_swp_soft_dirty(oldpte)) 121 newpte = pte_swp_mksoft_dirty(newpte); 122 set_pte_at(mm, addr, pte, newpte); 123 124 pages++; 125 } 126 } 127 } while (pte++, addr += PAGE_SIZE, addr != end); 128 arch_leave_lazy_mmu_mode(); 129 pte_unmap_unlock(pte - 1, ptl); 130 131 return pages; 132 } 133 134 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 135 pud_t *pud, unsigned long addr, unsigned long end, 136 pgprot_t newprot, int dirty_accountable, int prot_numa) 137 { 138 pmd_t *pmd; 139 struct mm_struct *mm = vma->vm_mm; 140 unsigned long next; 141 unsigned long pages = 0; 142 unsigned long nr_huge_updates = 0; 143 unsigned long mni_start = 0; 144 145 pmd = pmd_offset(pud, addr); 146 do { 147 unsigned long this_pages; 148 149 next = pmd_addr_end(addr, end); 150 if (!pmd_trans_huge(*pmd) && pmd_none_or_clear_bad(pmd)) 151 continue; 152 153 /* invoke the mmu notifier if the pmd is populated */ 154 if (!mni_start) { 155 mni_start = addr; 156 mmu_notifier_invalidate_range_start(mm, mni_start, end); 157 } 158 159 if (pmd_trans_huge(*pmd)) { 160 if (next - addr != HPAGE_PMD_SIZE) 161 split_huge_page_pmd(vma, addr, pmd); 162 else { 163 int nr_ptes = change_huge_pmd(vma, pmd, addr, 164 newprot, prot_numa); 165 166 if (nr_ptes) { 167 if (nr_ptes == HPAGE_PMD_NR) { 168 pages += HPAGE_PMD_NR; 169 nr_huge_updates++; 170 } 171 172 /* huge pmd was handled */ 173 continue; 174 } 175 } 176 /* fall through, the trans huge pmd just split */ 177 } 178 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 179 dirty_accountable, prot_numa); 180 pages += this_pages; 181 } while (pmd++, addr = next, addr != end); 182 183 if (mni_start) 184 mmu_notifier_invalidate_range_end(mm, mni_start, end); 185 186 if (nr_huge_updates) 187 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 188 return pages; 189 } 190 191 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 192 pgd_t *pgd, unsigned long addr, unsigned long end, 193 pgprot_t newprot, int dirty_accountable, int prot_numa) 194 { 195 pud_t *pud; 196 unsigned long next; 197 unsigned long pages = 0; 198 199 pud = pud_offset(pgd, addr); 200 do { 201 next = pud_addr_end(addr, end); 202 if (pud_none_or_clear_bad(pud)) 203 continue; 204 pages += change_pmd_range(vma, pud, addr, next, newprot, 205 dirty_accountable, prot_numa); 206 } while (pud++, addr = next, addr != end); 207 208 return pages; 209 } 210 211 static unsigned long change_protection_range(struct vm_area_struct *vma, 212 unsigned long addr, unsigned long end, pgprot_t newprot, 213 int dirty_accountable, int prot_numa) 214 { 215 struct mm_struct *mm = vma->vm_mm; 216 pgd_t *pgd; 217 unsigned long next; 218 unsigned long start = addr; 219 unsigned long pages = 0; 220 221 BUG_ON(addr >= end); 222 pgd = pgd_offset(mm, addr); 223 flush_cache_range(vma, addr, end); 224 set_tlb_flush_pending(mm); 225 do { 226 next = pgd_addr_end(addr, end); 227 if (pgd_none_or_clear_bad(pgd)) 228 continue; 229 pages += change_pud_range(vma, pgd, addr, next, newprot, 230 dirty_accountable, prot_numa); 231 } while (pgd++, addr = next, addr != end); 232 233 /* Only flush the TLB if we actually modified any entries: */ 234 if (pages) 235 flush_tlb_range(vma, start, end); 236 clear_tlb_flush_pending(mm); 237 238 return pages; 239 } 240 241 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 242 unsigned long end, pgprot_t newprot, 243 int dirty_accountable, int prot_numa) 244 { 245 unsigned long pages; 246 247 if (is_vm_hugetlb_page(vma)) 248 pages = hugetlb_change_protection(vma, start, end, newprot); 249 else 250 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 251 252 return pages; 253 } 254 255 int 256 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 257 unsigned long start, unsigned long end, unsigned long newflags) 258 { 259 struct mm_struct *mm = vma->vm_mm; 260 unsigned long oldflags = vma->vm_flags; 261 long nrpages = (end - start) >> PAGE_SHIFT; 262 unsigned long charged = 0; 263 pgoff_t pgoff; 264 int error; 265 int dirty_accountable = 0; 266 267 if (newflags == oldflags) { 268 *pprev = vma; 269 return 0; 270 } 271 272 /* 273 * If we make a private mapping writable we increase our commit; 274 * but (without finer accounting) cannot reduce our commit if we 275 * make it unwritable again. hugetlb mapping were accounted for 276 * even if read-only so there is no need to account for them here 277 */ 278 if (newflags & VM_WRITE) { 279 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 280 VM_SHARED|VM_NORESERVE))) { 281 charged = nrpages; 282 if (security_vm_enough_memory_mm(mm, charged)) 283 return -ENOMEM; 284 newflags |= VM_ACCOUNT; 285 } 286 } 287 288 /* 289 * First try to merge with previous and/or next vma. 290 */ 291 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 292 *pprev = vma_merge(mm, *pprev, start, end, newflags, 293 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); 294 if (*pprev) { 295 vma = *pprev; 296 goto success; 297 } 298 299 *pprev = vma; 300 301 if (start != vma->vm_start) { 302 error = split_vma(mm, vma, start, 1); 303 if (error) 304 goto fail; 305 } 306 307 if (end != vma->vm_end) { 308 error = split_vma(mm, vma, end, 0); 309 if (error) 310 goto fail; 311 } 312 313 success: 314 /* 315 * vm_flags and vm_page_prot are protected by the mmap_sem 316 * held in write mode. 317 */ 318 vma->vm_flags = newflags; 319 dirty_accountable = vma_wants_writenotify(vma); 320 vma_set_page_prot(vma); 321 322 change_protection(vma, start, end, vma->vm_page_prot, 323 dirty_accountable, 0); 324 325 vm_stat_account(mm, oldflags, vma->vm_file, -nrpages); 326 vm_stat_account(mm, newflags, vma->vm_file, nrpages); 327 perf_event_mmap(vma); 328 return 0; 329 330 fail: 331 vm_unacct_memory(charged); 332 return error; 333 } 334 335 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 336 unsigned long, prot) 337 { 338 unsigned long vm_flags, nstart, end, tmp, reqprot; 339 struct vm_area_struct *vma, *prev; 340 int error = -EINVAL; 341 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 342 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 343 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 344 return -EINVAL; 345 346 if (start & ~PAGE_MASK) 347 return -EINVAL; 348 if (!len) 349 return 0; 350 len = PAGE_ALIGN(len); 351 end = start + len; 352 if (end <= start) 353 return -ENOMEM; 354 if (!arch_validate_prot(prot)) 355 return -EINVAL; 356 357 reqprot = prot; 358 /* 359 * Does the application expect PROT_READ to imply PROT_EXEC: 360 */ 361 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) 362 prot |= PROT_EXEC; 363 364 vm_flags = calc_vm_prot_bits(prot); 365 366 down_write(¤t->mm->mmap_sem); 367 368 vma = find_vma(current->mm, start); 369 error = -ENOMEM; 370 if (!vma) 371 goto out; 372 prev = vma->vm_prev; 373 if (unlikely(grows & PROT_GROWSDOWN)) { 374 if (vma->vm_start >= end) 375 goto out; 376 start = vma->vm_start; 377 error = -EINVAL; 378 if (!(vma->vm_flags & VM_GROWSDOWN)) 379 goto out; 380 } else { 381 if (vma->vm_start > start) 382 goto out; 383 if (unlikely(grows & PROT_GROWSUP)) { 384 end = vma->vm_end; 385 error = -EINVAL; 386 if (!(vma->vm_flags & VM_GROWSUP)) 387 goto out; 388 } 389 } 390 if (start > vma->vm_start) 391 prev = vma; 392 393 for (nstart = start ; ; ) { 394 unsigned long newflags; 395 396 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 397 398 newflags = vm_flags; 399 newflags |= (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); 400 401 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 402 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 403 error = -EACCES; 404 goto out; 405 } 406 407 error = security_file_mprotect(vma, reqprot, prot); 408 if (error) 409 goto out; 410 411 tmp = vma->vm_end; 412 if (tmp > end) 413 tmp = end; 414 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 415 if (error) 416 goto out; 417 nstart = tmp; 418 419 if (nstart < prev->vm_end) 420 nstart = prev->vm_end; 421 if (nstart >= end) 422 goto out; 423 424 vma = prev->vm_next; 425 if (!vma || vma->vm_start != nstart) { 426 error = -ENOMEM; 427 goto out; 428 } 429 } 430 out: 431 up_write(¤t->mm->mmap_sem); 432 return error; 433 } 434